diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f25bf01892..0f327c9152 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,6 +31,7 @@ build: artifacts: paths: - ./streampipes-backend/target/*.war + - ./streampipes-connect-container/target/*.jar - ./target/site/apidocs/** - ./target/mvn_version expire_in: 1 week @@ -69,6 +70,21 @@ docker-backend: only: - dev +docker-connect-container: + image: docker:17.06.0-ce + stage: docker + dependencies: + - build + script: + - export MVN_VERSION=$(cat ./target/mvn_version) + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $REGISTRY_HOST + - docker login -u riemer -p $HARBOR_PASSWORD laus.fzi.de:8201 + - docker build --pull -t $IMAGE_NAME/streampipes-connect-container:latest -t $IMAGE_NAME/streampipes-connect-container:$MVN_VERSION -t $HARBOR_IMAGE_NAME/streampipes-connect-container:latest -t $HARBOR_IMAGE_NAME/streampipes-connect-container:$MVN_VERSION ./streampipes-connect-container/ + - docker push $IMAGE_NAME/streampipes-connect-container:$MVN_VERSION + - docker push $IMAGE_NAME/streampipes-connect-container:latest + - docker push $HARBOR_IMAGE_NAME/streampipes-connect-container:$MVN_VERSION + - docker push $HARBOR_IMAGE_NAME/streampipes-connect-container:latest + deploy: image: maven:3-jdk-8 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000..99c51136ff --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,38 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + + +## [0.60.0] - 2018-11-14 +### Added +- Beta release of StreamPipes Connect Library +- Tutorials for better user guidance +- New Wrapper for the Siddhi CEP engine +- New Project streampipes-pipeline-elements contains more than 40 new pipeline elements + +### Changed +- Various bug fixes and stability improvements +- Many UX improvements (e.g., harmonized styles) +- Dashboard does not reload after new visualization type has been created +- Improved test coverage + +### Removed + +## [0.55.2] - 2018-05-08 +### Added +- The [installer](https://www.github.com/streampipes/streampipes-installer) makes it easy to install StreamPipes on Linux, MacOS and Windows +- Live data preview for data streams in the pipeline editor +- Initial support for data sets +- Default for configurations can now be provided as environment variable, with the same name + +### Changed +- Pipeline elements can be directly installed at installation time +- Extended the SDK to create new pipeline elements +- Several UI improvements to make the definition of pipelines more intuitive +- Several bug fixes and code improvements + +### Removed \ No newline at end of file diff --git a/README.md b/README.md index d21572ccec..deec42fb6c 100644 --- a/README.md +++ b/README.md @@ -20,15 +20,20 @@ Read the full documentation at [https://docs.streampipes.org](https://docs.strea ### Getting started It's easy to get started: -* Download the `docker-compose.yml` file from [https://www.github.com/streampipes/preview-docker](https://www.github.com/streampipes/preview-docker) -* Follow the installation guide at [https://docs.streampipes.org/quick_start/installation](https://docs.streampipes.org/quick_start/installation) +* Clone the installer script from [https://www.github.com/streampipes/streampipes-installer](https://www.github.com/streampipes/streampipes-installer) +* Follow the installation guide at [https://docs.streampipes.org/docs/user-guide-installation](https://docs.streampipes.org/docs/user-guide-installation) * Check the [tour](https://docs.streampipes.org/user_guide/features) and build your first pipeline! ### Extending StreamPipes You can easily add your own data streams, processors or sinks. -Check our developer guide at [https://docs.streampipes.org/developer_guide/introduction](https://docs.streampipes.org/developer_guide/introduction) +Check our developer guide at [https://docs.streampipes.org/docs/dev-guide-introduction](https://docs.streampipes.org/docs/dev-guide-introduction) + +### Community + +- [Twitter](https://twitter.com/streampipes) +- [Email](mailto:feedback@streampipes.org) ### Contributing diff --git a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/META-INF/archetype.xml b/archetypes/streampipes-archetype-esper-standalone/src/main/resources/META-INF/archetype.xml deleted file mode 100644 index 1881e447f2..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/META-INF/archetype.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - streampipes-archetype-storm - - src/main/java/__elementName__/__classNamePrefix__.java - src/main/java/__elementName__/__classNamePrefix__Controller.java - src/main/java/__elementName__/__classNamePrefix__Parameters.java - src/main/java/main/Init.java - - - - - \ No newline at end of file diff --git a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/META-INF/maven/archetype-metadata.xml b/archetypes/streampipes-archetype-esper-standalone/src/main/resources/META-INF/maven/archetype-metadata.xml deleted file mode 100644 index 24f6bac4b3..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/META-INF/maven/archetype-metadata.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - src/main/java - - **/*.java - - - - - - - - \ No newline at end of file diff --git a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/pom.xml b/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/pom.xml deleted file mode 100644 index 6f954c049c..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/pom.xml +++ /dev/null @@ -1,100 +0,0 @@ - - 4.0.0 - - ${groupId} - ${artifactId} - ${version} - jar - - - de.fzi.cep.sepa - semantic-epa-commons - 0.0.1-SNAPSHOT - - - org.apache.kafka - kafka_2.10 - - - - - de.fzi.cep.sepa - semantic-epa - 0.0.1-SNAPSHOT - - - com.espertech - esper - 5.2.0 - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.1 - - 1.8 - 1.8 - UTF-8 - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 2.3 - - - package - - shade - - - - - ${package}.main.Init - - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - META-INF/maven/com.github.jsonld-java/jsonld-java/pom.xml - META-INF/maven/com.github.jsonld-java/jsonld-java-sesame/pom.xml - - - - - - - - - - - - laus - nexus repository - http://laus.fzi.de/nexus/content/repositories/public/ - - true - daily - - - true - - - - \ No newline at end of file diff --git a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/__elementName__/__classNamePrefix__.java b/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/__elementName__/__classNamePrefix__.java deleted file mode 100644 index 1113303cae..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/__elementName__/__classNamePrefix__.java +++ /dev/null @@ -1,25 +0,0 @@ -package ${package}.${elementName}; - -import java.util.ArrayList; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -import ${package}.esper.EsperEventEngine; - - -public class ${classNamePrefix} extends EsperEventEngine<${classNamePrefix}Parameters>{ - - private static final Logger logger = LoggerFactory.getLogger(${classNamePrefix}.class.getSimpleName()); - - protected List statements(final ${classNamePrefix}Parameters params) { - - List statements = new ArrayList(); - - return statements; - - } -} - diff --git a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/__elementName__/__classNamePrefix__Controller.java b/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/__elementName__/__classNamePrefix__Controller.java deleted file mode 100644 index fc8f9f58fd..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/__elementName__/__classNamePrefix__Controller.java +++ /dev/null @@ -1,44 +0,0 @@ -package ${package}.${elementName}; - -import com.google.common.io.Resources; - -import de.fzi.cep.sepa.commons.exceptions.SepaParseException; - -import de.fzi.cep.sepa.desc.EpDeclarer; - -import de.fzi.cep.sepa.model.impl.Response; -import de.fzi.cep.sepa.model.impl.graph.SepaDescription; -import de.fzi.cep.sepa.model.impl.graph.SepaInvocation; - -import de.fzi.cep.sepa.model.util.SepaUtils; - -import de.fzi.cep.sepa.client.util.DeclarerUtils; - -public class ${classNamePrefix}Controller extends EpDeclarer<${classNamePrefix}Parameters> { - - @Override - public SepaDescription declareModel() { - - try { - return DeclarerUtils.descriptionFromResources(Resources.getResource("${elementName}.jsonLd"), SepaDescription.class); - } catch (SepaParseException e) { - e.printStackTrace(); - return null; - } - - } - - @Override - public Response invokeRuntime(SepaInvocation sepa) { - - ${classNamePrefix}Parameters staticParams = new ${classNamePrefix}Parameters(sepa); - - try { - invokeEPRuntime(staticParams, ${classNamePrefix}::new, sepa); - return new Response(sepa.getElementId(), true); - } catch (Exception e) { - e.printStackTrace(); - return new Response(sepa.getElementId(), false, e.getMessage()); - } - } -} diff --git a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/__elementName__/__classNamePrefix__Parameters.java b/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/__elementName__/__classNamePrefix__Parameters.java deleted file mode 100644 index cca8beaee2..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/__elementName__/__classNamePrefix__Parameters.java +++ /dev/null @@ -1,12 +0,0 @@ -package ${package}.${elementName}; - -import de.fzi.cep.sepa.model.impl.graph.SepaInvocation; -import de.fzi.cep.sepa.runtime.param.BindingParameters; - -public class ${classNamePrefix}Parameters extends BindingParameters { - - public ${classNamePrefix}Parameters(SepaInvocation graph) { - super(graph); - } - -} diff --git a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/esper/AbstractQueueRunnable.java b/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/esper/AbstractQueueRunnable.java deleted file mode 100644 index 4396561dac..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/esper/AbstractQueueRunnable.java +++ /dev/null @@ -1,66 +0,0 @@ -package ${package}.esper; - -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; - -public abstract class AbstractQueueRunnable extends Thread -{ - protected BlockingQueue queue; - protected long closeAfter = 0; - protected long currentTimestamp; - protected boolean autoClose; - private boolean running; - - public AbstractQueueRunnable(int maxQueueSize, int closeAfter) - { - queue = new ArrayBlockingQueue(maxQueueSize); - this.autoClose = true; - this.closeAfter = closeAfter * 1000; - this.currentTimestamp = System.currentTimeMillis(); - } - - public AbstractQueueRunnable(int maxQueueSize) - { - queue = new ArrayBlockingQueue(maxQueueSize); - this.autoClose = false; - this.currentTimestamp = System.currentTimeMillis(); - } - - @Override - public void run() - { - running = true; - while (running) - { - if (autoClose) - if (System.currentTimeMillis()-currentTimestamp > closeAfter) break; - try - { - T data = queue.take(); - currentTimestamp = System.currentTimeMillis(); - doNext(data); - } - catch (Exception e) - { - e.printStackTrace(); - if (e instanceof InterruptedException) - Thread.currentThread().interrupt(); - else - e.printStackTrace(); - } - } - System.out.println("Interrupted"); - } - - public void interrupt() - { - running = false; - } - - public void add(T data) throws InterruptedException - { - queue.put(data); - } - - protected abstract void doNext(T data) throws Exception; -} \ No newline at end of file diff --git a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/esper/EsperEventEngine.java b/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/esper/EsperEventEngine.java deleted file mode 100644 index a3dcc85e4b..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/esper/EsperEventEngine.java +++ /dev/null @@ -1,181 +0,0 @@ -package ${package}.esper; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.espertech.esper.client.ConfigurationException; -import com.espertech.esper.client.EPServiceProvider; -import com.espertech.esper.client.EPServiceProviderManager; -import com.espertech.esper.client.EPStatement; -import com.espertech.esper.client.EventBean; -import com.espertech.esper.client.UpdateListener; - -import de.fzi.cep.sepa.commons.Utils; -import de.fzi.cep.sepa.model.impl.graph.SepaInvocation; -import de.fzi.cep.sepa.runtime.EPEngine; -import de.fzi.cep.sepa.runtime.OutputCollector; -import de.fzi.cep.sepa.runtime.param.BindingParameters; -import de.fzi.cep.sepa.runtime.param.EngineParameters; - -public abstract class EsperEventEngine implements EPEngine{ - - protected EPServiceProvider epService; - protected List epStatements; - - private AbstractQueueRunnable queue; - private List eventTypeNames = new ArrayList<>(); - - private static final Logger logger = LoggerFactory.getLogger(EsperEventEngine.class.getSimpleName()); - - @Override - public void bind(EngineParameters parameters, OutputCollector collector, SepaInvocation graph) { - if (parameters.getInEventTypes().size() != graph.getInputStreams().size()) - throw new IllegalArgumentException("Input parameters do not match!"); - - epService = EPServiceProviderManager.getDefaultProvider(); - - logger.info("Configuring event types for graph " +graph.getName()); - parameters.getInEventTypes().entrySet().forEach(e -> { - Map inTypeMap = e.getValue(); - checkAndRegisterEventType(e.getKey(), inTypeMap); - }); - - checkAndRegisterEventType("topic://" +graph.getOutputStream().getEventGrounding().getTransportProtocol().getTopicName(), parameters.getOutEventType()); - - List statements = statements(parameters.getStaticProperty()); - registerStatements(statements, collector, parameters.getStaticProperty()); - - } - - private void checkAndRegisterEventType(String key, Map typeMap) - { - Map newTypeMap = new HashMap(); - Iterator it = typeMap.keySet().iterator(); - while(it.hasNext()) - { - String objKey = it.next(); - Object obj = typeMap.get(objKey); - if (obj instanceof java.util.List) - { - String eventName = StringUtils.capitalize(objKey); - registerEventTypeIfNotExists(eventName, (Map) ((java.util.List) obj).get(0)); - newTypeMap.put(objKey, eventName +"[]"); - } - else { - newTypeMap.put(objKey, obj); - } - } - //MapUtils.debugPrint(System.out, key, newTypeMap); - registerEventTypeIfNotExists(key, newTypeMap); - - } - - private void registerEventTypeIfNotExists(String eventTypeName, Map typeMap) - { - try { - logger.info("Registering event type, " +eventTypeName); - epService.getEPAdministrator().getConfiguration().addEventType(eventTypeName, typeMap); - eventTypeNames.add(eventTypeName); - } catch (ConfigurationException e) - { - e.printStackTrace(); - logger.info("Event type does already exist, " +eventTypeName); - } - } - - private void registerStatements(List statements, OutputCollector collector, T params) - { - toEpStatement(statements); - queue = new StatementAwareQueue(getWriter(collector, params), 50000); - queue.start(); - for(EPStatement epStatement : epStatements) - { - logger.info("Registering statement " +epStatement.getText()); - - if (epStatement.getText().startsWith("select")) - { - epStatement.addListener(listenerSendingTo(queue)); - } - epStatement.start(); - - } - - } - - private void toEpStatement(List statements) - { - if (epStatements == null) epStatements = new ArrayList<>(); - for(String statement : statements) - { - epStatements.add(epService.getEPAdministrator().createEPL(statement)); - } - epStatements.add(epService.getEPAdministrator().createEPL("select * from StatusEvent")); - } - - @Override - public void onEvent(Map event, String sourceInfo) { - epService.getEPRuntime().sendEvent(event, sourceInfo); - } - - @Override - public void discard() { - logger.info("Removing existing statements"); - for(EPStatement epStatement : epStatements) - { - epService.getEPAdministrator().getStatement(epStatement.getName()).removeAllListeners(); - epService.getEPAdministrator().getStatement(epStatement.getName()).stop(); - epService.getEPAdministrator().getStatement(epStatement.getName()).destroy(); - } - epStatements.clear(); - for(String eventName : eventTypeNames) - { - try { - epService.getEPAdministrator().getConfiguration().removeEventType(eventName, false); - } catch (ConfigurationException ce) - { - logger.info("Event type used in another statement which is still running, skipping..."); - } - } - - queue.interrupt(); - } - - private static UpdateListener listenerSendingTo(AbstractQueueRunnable queue) { - return new UpdateListener() { - - @Override - public void update(EventBean[] newEvents, EventBean[] oldEvents) { - try { - if (newEvents != null) queue.add(newEvents); - else queue.add(oldEvents); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - }; - } - - protected abstract List statements(final T bindingParameters); - - protected String fixEventName(String eventName) - { - return "`" +eventName +"`"; - } - - protected List makeStatementList(String statement) - { - return Utils.createList(statement); - } - - protected Writer getWriter(OutputCollector collector, T params) - { - return new SEPAWriter(collector); - } -} diff --git a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/esper/SEPAWriter.java b/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/esper/SEPAWriter.java deleted file mode 100644 index 79c50bc2b7..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/esper/SEPAWriter.java +++ /dev/null @@ -1,20 +0,0 @@ -package ${package}.esper; - -import com.espertech.esper.client.EventBean; - -import de.fzi.cep.sepa.runtime.OutputCollector; - -public class SEPAWriter implements Writer { - - private OutputCollector collector; - - public SEPAWriter(OutputCollector collector) { - this.collector = collector; - } - - @Override - public void onEvent(EventBean bean) { - collector.send(bean.getUnderlying()); - } - -} diff --git a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/esper/StatementAwareQueue.java b/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/esper/StatementAwareQueue.java deleted file mode 100644 index c2d5378620..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/esper/StatementAwareQueue.java +++ /dev/null @@ -1,28 +0,0 @@ -package ${package}.esper; - -import com.espertech.esper.client.EventBean; - -public class StatementAwareQueue extends AbstractQueueRunnable{ - - private int counter = 0; - private Writer writer; - - public StatementAwareQueue(Writer writer, int maxQueueSize, int closeAfter) { - super(maxQueueSize, closeAfter); - this.writer = writer; - } - - public StatementAwareQueue(Writer writer, int maxQueueSize) { - super(maxQueueSize); - this.writer = writer; - } - - @Override - protected void doNext(EventBean[] newEvents) throws Exception { - currentTimestamp = System.currentTimeMillis(); - counter++; - if (counter % 100000 == 0) System.out.println(counter + " Events received."); - writer.onEvent(newEvents[0]); - } - -} diff --git a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/esper/Writer.java b/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/esper/Writer.java deleted file mode 100644 index 681ed064ab..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/esper/Writer.java +++ /dev/null @@ -1,8 +0,0 @@ -package ${package}.esper; - -import com.espertech.esper.client.EventBean; - -public interface Writer { - - public void onEvent(EventBean bean); -} diff --git a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/main/Init.java b/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/main/Init.java deleted file mode 100644 index bf0048a7c2..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/src/main/resources/archetype-resources/src/main/java/main/Init.java +++ /dev/null @@ -1,19 +0,0 @@ -package ${package}.main; - -import java.util.Arrays; - -import de.fzi.cep.sepa.desc.ModelSubmitter; - -import ${package}.${elementName}.${classNamePrefix}Controller; - -public class Init { - - public static void main(String[] args) - { - try { - ModelSubmitter.submitAgent(Arrays.asList(new ${classNamePrefix}Controller())); - } catch (Exception e) { - e.printStackTrace(); - } - } -} diff --git a/archetypes/streampipes-archetype-esper-standalone/target/classes/META-INF/MANIFEST.MF b/archetypes/streampipes-archetype-esper-standalone/target/classes/META-INF/MANIFEST.MF deleted file mode 100644 index eaca22bf14..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/target/classes/META-INF/MANIFEST.MF +++ /dev/null @@ -1,5 +0,0 @@ -Manifest-Version: 1.0 -Build-Jdk: 1.7.0_101 -Built-By: philipp -Created-By: Maven Integration for Eclipse - diff --git a/archetypes/streampipes-archetype-esper-standalone/target/classes/META-INF/archetype.xml b/archetypes/streampipes-archetype-esper-standalone/target/classes/META-INF/archetype.xml deleted file mode 100644 index 1881e447f2..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/target/classes/META-INF/archetype.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - streampipes-archetype-storm - - src/main/java/__elementName__/__classNamePrefix__.java - src/main/java/__elementName__/__classNamePrefix__Controller.java - src/main/java/__elementName__/__classNamePrefix__Parameters.java - src/main/java/main/Init.java - - - - - \ No newline at end of file diff --git a/archetypes/streampipes-archetype-esper-standalone/target/classes/META-INF/maven/archetype-metadata.xml b/archetypes/streampipes-archetype-esper-standalone/target/classes/META-INF/maven/archetype-metadata.xml deleted file mode 100644 index 24f6bac4b3..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/target/classes/META-INF/maven/archetype-metadata.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - src/main/java - - **/*.java - - - - - - - - \ No newline at end of file diff --git a/archetypes/streampipes-archetype-esper-standalone/target/classes/META-INF/maven/de.fzi.cep.sepa/streampipes-archetype-esper-standalone/pom.properties b/archetypes/streampipes-archetype-esper-standalone/target/classes/META-INF/maven/de.fzi.cep.sepa/streampipes-archetype-esper-standalone/pom.properties deleted file mode 100644 index 77bdefec8a..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/target/classes/META-INF/maven/de.fzi.cep.sepa/streampipes-archetype-esper-standalone/pom.properties +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright 2018 FZI Forschungszentrum Informatik -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -#Generated by Maven Integration for Eclipse -#Thu May 05 14:09:02 CEST 2016 -version=0.0.1-SNAPSHOT -groupId=de.fzi.cep.sepa -m2e.projectName=streampipes-archetype-esper-standalone -m2e.projectLocation=/home/philipp/Coding/fzi/icep/semantic-epa-parent/archetypes/streampipes-archetype-esper-standalone -artifactId=streampipes-archetype-esper-standalone diff --git a/archetypes/streampipes-archetype-esper-standalone/target/classes/META-INF/maven/de.fzi.cep.sepa/streampipes-archetype-esper-standalone/pom.xml b/archetypes/streampipes-archetype-esper-standalone/target/classes/META-INF/maven/de.fzi.cep.sepa/streampipes-archetype-esper-standalone/pom.xml deleted file mode 100644 index 8c3ffb21a4..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/target/classes/META-INF/maven/de.fzi.cep.sepa/streampipes-archetype-esper-standalone/pom.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - 4.0.0 - de.fzi.cep.sepa - streampipes-archetype-esper-standalone - 0.0.1-SNAPSHOT - \ No newline at end of file diff --git a/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/pom.xml b/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/pom.xml deleted file mode 100644 index f26630054f..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/pom.xml +++ /dev/null @@ -1,117 +0,0 @@ - - - - 4.0.0 - - ${groupId} - ${artifactId} - ${version} - jar - - - de.fzi.cep.sepa - semantic-epa-commons - 0.0.1-SNAPSHOT - - - org.apache.kafka - kafka_2.10 - - - - - de.fzi.cep.sepa - semantic-epa - 0.0.1-SNAPSHOT - - - com.espertech - esper - 5.2.0 - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.1 - - 1.8 - 1.8 - UTF-8 - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 2.3 - - - package - - shade - - - - - ${package}.main.Init - - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - META-INF/maven/com.github.jsonld-java/jsonld-java/pom.xml - META-INF/maven/com.github.jsonld-java/jsonld-java-sesame/pom.xml - - - - - - - - - - - - laus - nexus repository - http://laus.fzi.de/nexus/content/repositories/public/ - - true - daily - - - true - - - - \ No newline at end of file diff --git a/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/__elementName__/__classNamePrefix__.java b/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/__elementName__/__classNamePrefix__.java deleted file mode 100644 index 1113303cae..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/__elementName__/__classNamePrefix__.java +++ /dev/null @@ -1,25 +0,0 @@ -package ${package}.${elementName}; - -import java.util.ArrayList; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -import ${package}.esper.EsperEventEngine; - - -public class ${classNamePrefix} extends EsperEventEngine<${classNamePrefix}Parameters>{ - - private static final Logger logger = LoggerFactory.getLogger(${classNamePrefix}.class.getSimpleName()); - - protected List statements(final ${classNamePrefix}Parameters params) { - - List statements = new ArrayList(); - - return statements; - - } -} - diff --git a/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/__elementName__/__classNamePrefix__Controller.java b/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/__elementName__/__classNamePrefix__Controller.java deleted file mode 100644 index fc8f9f58fd..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/__elementName__/__classNamePrefix__Controller.java +++ /dev/null @@ -1,44 +0,0 @@ -package ${package}.${elementName}; - -import com.google.common.io.Resources; - -import de.fzi.cep.sepa.commons.exceptions.SepaParseException; - -import de.fzi.cep.sepa.desc.EpDeclarer; - -import de.fzi.cep.sepa.model.impl.Response; -import de.fzi.cep.sepa.model.impl.graph.SepaDescription; -import de.fzi.cep.sepa.model.impl.graph.SepaInvocation; - -import de.fzi.cep.sepa.model.util.SepaUtils; - -import de.fzi.cep.sepa.client.util.DeclarerUtils; - -public class ${classNamePrefix}Controller extends EpDeclarer<${classNamePrefix}Parameters> { - - @Override - public SepaDescription declareModel() { - - try { - return DeclarerUtils.descriptionFromResources(Resources.getResource("${elementName}.jsonLd"), SepaDescription.class); - } catch (SepaParseException e) { - e.printStackTrace(); - return null; - } - - } - - @Override - public Response invokeRuntime(SepaInvocation sepa) { - - ${classNamePrefix}Parameters staticParams = new ${classNamePrefix}Parameters(sepa); - - try { - invokeEPRuntime(staticParams, ${classNamePrefix}::new, sepa); - return new Response(sepa.getElementId(), true); - } catch (Exception e) { - e.printStackTrace(); - return new Response(sepa.getElementId(), false, e.getMessage()); - } - } -} diff --git a/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/__elementName__/__classNamePrefix__Parameters.java b/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/__elementName__/__classNamePrefix__Parameters.java deleted file mode 100644 index cca8beaee2..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/__elementName__/__classNamePrefix__Parameters.java +++ /dev/null @@ -1,12 +0,0 @@ -package ${package}.${elementName}; - -import de.fzi.cep.sepa.model.impl.graph.SepaInvocation; -import de.fzi.cep.sepa.runtime.param.BindingParameters; - -public class ${classNamePrefix}Parameters extends BindingParameters { - - public ${classNamePrefix}Parameters(SepaInvocation graph) { - super(graph); - } - -} diff --git a/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/esper/AbstractQueueRunnable.java b/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/esper/AbstractQueueRunnable.java deleted file mode 100644 index 4396561dac..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/esper/AbstractQueueRunnable.java +++ /dev/null @@ -1,66 +0,0 @@ -package ${package}.esper; - -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; - -public abstract class AbstractQueueRunnable extends Thread -{ - protected BlockingQueue queue; - protected long closeAfter = 0; - protected long currentTimestamp; - protected boolean autoClose; - private boolean running; - - public AbstractQueueRunnable(int maxQueueSize, int closeAfter) - { - queue = new ArrayBlockingQueue(maxQueueSize); - this.autoClose = true; - this.closeAfter = closeAfter * 1000; - this.currentTimestamp = System.currentTimeMillis(); - } - - public AbstractQueueRunnable(int maxQueueSize) - { - queue = new ArrayBlockingQueue(maxQueueSize); - this.autoClose = false; - this.currentTimestamp = System.currentTimeMillis(); - } - - @Override - public void run() - { - running = true; - while (running) - { - if (autoClose) - if (System.currentTimeMillis()-currentTimestamp > closeAfter) break; - try - { - T data = queue.take(); - currentTimestamp = System.currentTimeMillis(); - doNext(data); - } - catch (Exception e) - { - e.printStackTrace(); - if (e instanceof InterruptedException) - Thread.currentThread().interrupt(); - else - e.printStackTrace(); - } - } - System.out.println("Interrupted"); - } - - public void interrupt() - { - running = false; - } - - public void add(T data) throws InterruptedException - { - queue.put(data); - } - - protected abstract void doNext(T data) throws Exception; -} \ No newline at end of file diff --git a/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/esper/EsperEventEngine.java b/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/esper/EsperEventEngine.java deleted file mode 100644 index a3dcc85e4b..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/esper/EsperEventEngine.java +++ /dev/null @@ -1,181 +0,0 @@ -package ${package}.esper; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.espertech.esper.client.ConfigurationException; -import com.espertech.esper.client.EPServiceProvider; -import com.espertech.esper.client.EPServiceProviderManager; -import com.espertech.esper.client.EPStatement; -import com.espertech.esper.client.EventBean; -import com.espertech.esper.client.UpdateListener; - -import de.fzi.cep.sepa.commons.Utils; -import de.fzi.cep.sepa.model.impl.graph.SepaInvocation; -import de.fzi.cep.sepa.runtime.EPEngine; -import de.fzi.cep.sepa.runtime.OutputCollector; -import de.fzi.cep.sepa.runtime.param.BindingParameters; -import de.fzi.cep.sepa.runtime.param.EngineParameters; - -public abstract class EsperEventEngine implements EPEngine{ - - protected EPServiceProvider epService; - protected List epStatements; - - private AbstractQueueRunnable queue; - private List eventTypeNames = new ArrayList<>(); - - private static final Logger logger = LoggerFactory.getLogger(EsperEventEngine.class.getSimpleName()); - - @Override - public void bind(EngineParameters parameters, OutputCollector collector, SepaInvocation graph) { - if (parameters.getInEventTypes().size() != graph.getInputStreams().size()) - throw new IllegalArgumentException("Input parameters do not match!"); - - epService = EPServiceProviderManager.getDefaultProvider(); - - logger.info("Configuring event types for graph " +graph.getName()); - parameters.getInEventTypes().entrySet().forEach(e -> { - Map inTypeMap = e.getValue(); - checkAndRegisterEventType(e.getKey(), inTypeMap); - }); - - checkAndRegisterEventType("topic://" +graph.getOutputStream().getEventGrounding().getTransportProtocol().getTopicName(), parameters.getOutEventType()); - - List statements = statements(parameters.getStaticProperty()); - registerStatements(statements, collector, parameters.getStaticProperty()); - - } - - private void checkAndRegisterEventType(String key, Map typeMap) - { - Map newTypeMap = new HashMap(); - Iterator it = typeMap.keySet().iterator(); - while(it.hasNext()) - { - String objKey = it.next(); - Object obj = typeMap.get(objKey); - if (obj instanceof java.util.List) - { - String eventName = StringUtils.capitalize(objKey); - registerEventTypeIfNotExists(eventName, (Map) ((java.util.List) obj).get(0)); - newTypeMap.put(objKey, eventName +"[]"); - } - else { - newTypeMap.put(objKey, obj); - } - } - //MapUtils.debugPrint(System.out, key, newTypeMap); - registerEventTypeIfNotExists(key, newTypeMap); - - } - - private void registerEventTypeIfNotExists(String eventTypeName, Map typeMap) - { - try { - logger.info("Registering event type, " +eventTypeName); - epService.getEPAdministrator().getConfiguration().addEventType(eventTypeName, typeMap); - eventTypeNames.add(eventTypeName); - } catch (ConfigurationException e) - { - e.printStackTrace(); - logger.info("Event type does already exist, " +eventTypeName); - } - } - - private void registerStatements(List statements, OutputCollector collector, T params) - { - toEpStatement(statements); - queue = new StatementAwareQueue(getWriter(collector, params), 50000); - queue.start(); - for(EPStatement epStatement : epStatements) - { - logger.info("Registering statement " +epStatement.getText()); - - if (epStatement.getText().startsWith("select")) - { - epStatement.addListener(listenerSendingTo(queue)); - } - epStatement.start(); - - } - - } - - private void toEpStatement(List statements) - { - if (epStatements == null) epStatements = new ArrayList<>(); - for(String statement : statements) - { - epStatements.add(epService.getEPAdministrator().createEPL(statement)); - } - epStatements.add(epService.getEPAdministrator().createEPL("select * from StatusEvent")); - } - - @Override - public void onEvent(Map event, String sourceInfo) { - epService.getEPRuntime().sendEvent(event, sourceInfo); - } - - @Override - public void discard() { - logger.info("Removing existing statements"); - for(EPStatement epStatement : epStatements) - { - epService.getEPAdministrator().getStatement(epStatement.getName()).removeAllListeners(); - epService.getEPAdministrator().getStatement(epStatement.getName()).stop(); - epService.getEPAdministrator().getStatement(epStatement.getName()).destroy(); - } - epStatements.clear(); - for(String eventName : eventTypeNames) - { - try { - epService.getEPAdministrator().getConfiguration().removeEventType(eventName, false); - } catch (ConfigurationException ce) - { - logger.info("Event type used in another statement which is still running, skipping..."); - } - } - - queue.interrupt(); - } - - private static UpdateListener listenerSendingTo(AbstractQueueRunnable queue) { - return new UpdateListener() { - - @Override - public void update(EventBean[] newEvents, EventBean[] oldEvents) { - try { - if (newEvents != null) queue.add(newEvents); - else queue.add(oldEvents); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - }; - } - - protected abstract List statements(final T bindingParameters); - - protected String fixEventName(String eventName) - { - return "`" +eventName +"`"; - } - - protected List makeStatementList(String statement) - { - return Utils.createList(statement); - } - - protected Writer getWriter(OutputCollector collector, T params) - { - return new SEPAWriter(collector); - } -} diff --git a/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/esper/SEPAWriter.java b/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/esper/SEPAWriter.java deleted file mode 100644 index 79c50bc2b7..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/esper/SEPAWriter.java +++ /dev/null @@ -1,20 +0,0 @@ -package ${package}.esper; - -import com.espertech.esper.client.EventBean; - -import de.fzi.cep.sepa.runtime.OutputCollector; - -public class SEPAWriter implements Writer { - - private OutputCollector collector; - - public SEPAWriter(OutputCollector collector) { - this.collector = collector; - } - - @Override - public void onEvent(EventBean bean) { - collector.send(bean.getUnderlying()); - } - -} diff --git a/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/esper/StatementAwareQueue.java b/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/esper/StatementAwareQueue.java deleted file mode 100644 index c2d5378620..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/esper/StatementAwareQueue.java +++ /dev/null @@ -1,28 +0,0 @@ -package ${package}.esper; - -import com.espertech.esper.client.EventBean; - -public class StatementAwareQueue extends AbstractQueueRunnable{ - - private int counter = 0; - private Writer writer; - - public StatementAwareQueue(Writer writer, int maxQueueSize, int closeAfter) { - super(maxQueueSize, closeAfter); - this.writer = writer; - } - - public StatementAwareQueue(Writer writer, int maxQueueSize) { - super(maxQueueSize); - this.writer = writer; - } - - @Override - protected void doNext(EventBean[] newEvents) throws Exception { - currentTimestamp = System.currentTimeMillis(); - counter++; - if (counter % 100000 == 0) System.out.println(counter + " Events received."); - writer.onEvent(newEvents[0]); - } - -} diff --git a/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/esper/Writer.java b/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/esper/Writer.java deleted file mode 100644 index 681ed064ab..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/esper/Writer.java +++ /dev/null @@ -1,8 +0,0 @@ -package ${package}.esper; - -import com.espertech.esper.client.EventBean; - -public interface Writer { - - public void onEvent(EventBean bean); -} diff --git a/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/main/Init.java b/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/main/Init.java deleted file mode 100644 index bf0048a7c2..0000000000 --- a/archetypes/streampipes-archetype-esper-standalone/target/classes/archetype-resources/src/main/java/main/Init.java +++ /dev/null @@ -1,19 +0,0 @@ -package ${package}.main; - -import java.util.Arrays; - -import de.fzi.cep.sepa.desc.ModelSubmitter; - -import ${package}.${elementName}.${classNamePrefix}Controller; - -public class Init { - - public static void main(String[] args) - { - try { - ModelSubmitter.submitAgent(Arrays.asList(new ${classNamePrefix}Controller())); - } catch (Exception e) { - e.printStackTrace(); - } - } -} diff --git a/archetypes/streampipes-archetype-pe-jvm/src/main/resources/archetype-resources/Dockerfile b/archetypes/streampipes-archetype-pe-jvm/src/main/resources/archetype-resources/Dockerfile deleted file mode 100644 index f024d3ba43..0000000000 --- a/archetypes/streampipes-archetype-pe-jvm/src/main/resources/archetype-resources/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM anapsix/alpine-java - -ENV CONSUL_LOCATION consul - -EXPOSE 8090 - -RUN echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf - -ADD target/${artifactId}.jar /${artifactId}.jar - -ENTRYPOINT ["java", "-jar", "/${artifactId}.jar"] diff --git a/archetypes/streampipes-archetype-pe-processors-flink/README.md b/archetypes/streampipes-archetype-pe-processors-flink/README.md new file mode 100644 index 0000000000..0f2f5b3689 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-processors-flink/README.md @@ -0,0 +1,18 @@ +## StreamPipes Maven Archetype for Flink-based Data Processors + +### Usage + +mvn archetype:generate \ + -DarchetypeGroupId=org.streampipes \ + -DarchetypeArtifactId=streampipes-archetype-pe-processors-flink \ + -DarchetypeVersion=0.55.3-SNAPSHOT \ + -DgroupId=my.test.groupId \ + -DartifactId=my-test-artifact-id + -DclassNamePrefix=MyProcessor + -DpackageName=mypackagename + +### Variables + +* classNamePrefix: Will be used as a prefix to name your controller & parameter classes +* packageName: Will be used as the package name + diff --git a/archetypes/streampipes-archetype-pe-processors-flink/pom.xml b/archetypes/streampipes-archetype-pe-processors-flink/pom.xml new file mode 100644 index 0000000000..8791c5a179 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-processors-flink/pom.xml @@ -0,0 +1,59 @@ + + + 4.0.0 + + org.streampipes + streampipes-parent + 0.60.0 + ../../pom.xml + + streampipes-archetype-pe-processors-flink + maven-archetype + + streampipes-archetype-pe-processors-flink + + + + + org.apache.maven.archetype + archetype-packaging + 3.0.1 + + + + + + + maven-archetype-plugin + 3.0.1 + + + + + + + + scm:git:ssh://git@ipe-wim-gitlab.fzi.de:2222/streampipes/ce-backend.git/archetypes/streampipes-archetype-pe-sinks-jvm + + + + + + deployment + Internal Releases + https://laus.fzi.de/nexus/content/repositories/public/ + + + + + deployment + Internal Releases + https://laus.fzi.de/nexus/content/repositories/releases/ + + + deployment + Internal Releases + https://laus.fzi.de/nexus/content/repositories/snapshots/ + + + diff --git a/archetypes/streampipes-archetype-pe-jvm/src/main/resources/META-INF/maven/archetype-metadata.xml b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/META-INF/maven/archetype-metadata.xml similarity index 96% rename from archetypes/streampipes-archetype-pe-jvm/src/main/resources/META-INF/maven/archetype-metadata.xml rename to archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/META-INF/maven/archetype-metadata.xml index f562ab08d5..c309cbf6b0 100644 --- a/archetypes/streampipes-archetype-pe-jvm/src/main/resources/META-INF/maven/archetype-metadata.xml +++ b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/META-INF/maven/archetype-metadata.xml @@ -28,7 +28,7 @@ Example - + example diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/Dockerfile b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/Dockerfile new file mode 100644 index 0000000000..06f89bbaae --- /dev/null +++ b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/Dockerfile @@ -0,0 +1,8 @@ +FROM anapsix/alpine-java:8 + +EXPOSE 8090 +ENV CONSUL_LOCATION consul + +COPY ./target/${artifactId}.jar /streampipes-processing-element-container.jar + +ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"] diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/pom.xml b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/pom.xml new file mode 100644 index 0000000000..5a80f20dd8 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/pom.xml @@ -0,0 +1,154 @@ + + + 4.0.0 + ${groupId} + ${artifactId} + ${version} + + + 0.55.3-SNAPSHOT + + + + + org.streampipes + streampipes-container-standalone + ${sp.version} + + + org.apache.kafka + kafka_2.10 + + + org.apache.kafka + kafka-clients + + + + + org.streampipes + streampipes-commons + ${sp.version} + + + org.apache.kafka + kafka_2.10 + + + org.apache.kafka + kafka-clients + + + + + org.streampipes + streampipes-wrapper-flink + ${sp.version} + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + + + commons-beanutils + commons-beanutils + 1.9.2 + + + org.streampipes + streampipes-sdk + ${sp.version} + + + org.apache.logging.log4j + log4j-to-slf4j + 2.8.2 + + + org.slf4j + slf4j-api + 1.7.24 + + + org.streampipes + streampipes-config + ${sp.version} + + + + + + laus + nexus repository + http://laus.fzi.de/nexus/content/repositories/public/ + + true + daily + + + true + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.3 + + + package + + shade + + + false + + + ${package}.main.Init + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + META-INF/maven/com.github.jsonld-java/jsonld-java/pom.xml + META-INF/maven/com.github.jsonld-java/jsonld-java-sesame/pom.xml + + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + 1.8 + 1.8 + UTF-8 + + + + ${artifactId} + + diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/config/Config.java b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/config/Config.java new file mode 100644 index 0000000000..e742629bca --- /dev/null +++ b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/config/Config.java @@ -0,0 +1,84 @@ +#set( $symbol_pound = '#' ) +#set( $symbol_dollar = '$' ) +#set( $symbol_escape = '\' ) +package ${package}.config; + +import org.streampipes.config.SpConfig; +import org.streampipes.container.model.PeConfig; + +public enum Config implements PeConfig { + + INSTANCE; + + private SpConfig config; + public static final String JAR_FILE = "./streampipes-processing-element-container.jar"; + + private final static String service_id = "pe/${package}"; + private final static String service_name = "${packageName}"; + private final static String service_container_name = "${artifactId}"; + + + Config() { + config = SpConfig.getSpConfig(service_id); + + config.register(ConfigKeys.HOST, service_container_name, "Hostname for the pe mixed flink component"); + config.register(ConfigKeys.PORT, 8090, "Port for the pe mixed flink component"); + config.register(ConfigKeys.FLINK_HOST, "jobmanager", "Host for the flink cluster"); + config.register(ConfigKeys.FLINK_PORT, 6123, "Port for the flink cluster"); + config.register(ConfigKeys.ICON_HOST, "backend", "Hostname for the icon host"); + config.register(ConfigKeys.ICON_PORT, 80, "Port for the icons in nginx"); + + config.register(ConfigKeys.DEBUG, false, "When set to true programs are not deployed to cluster, but executed locally"); + + config.register(ConfigKeys.SERVICE_NAME, service_name, "The name of the service"); + + } + + @Override + public String getHost() { + return config.getString(ConfigKeys.HOST); + } + + @Override + public int getPort() { + return config.getInteger(ConfigKeys.PORT); + } + + public String getFlinkHost() { + return config.getString(ConfigKeys.FLINK_HOST); + } + + public int getFlinkPort() { + return config.getInteger(ConfigKeys.FLINK_PORT); + } + + public static final String iconBaseUrl = "http://" + Config.INSTANCE.getIconHost() + ":" + + Config.INSTANCE.getIconPort() + "/assets/img/pe_icons"; + + public static final String getIconUrl(String pictureName) { + return iconBaseUrl + "/" + pictureName + ".png"; + } + + public String getIconHost() { + return config.getString(ConfigKeys.ICON_HOST); + } + + public int getIconPort() { + return config.getInteger(ConfigKeys.ICON_PORT); + } + + public boolean getDebug() { + return config.getBoolean(ConfigKeys.DEBUG); + } + + @Override + public String getId() { + return service_id; + } + + @Override + public String getName() { + return config.getString(ConfigKeys.SERVICE_NAME); + } + +} diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/config/ConfigKeys.java b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/config/ConfigKeys.java new file mode 100644 index 0000000000..4110b02b99 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/config/ConfigKeys.java @@ -0,0 +1,15 @@ +#set( $symbol_pound = '#' ) +#set( $symbol_dollar = '$' ) +#set( $symbol_escape = '\' ) +package ${package}.config; + +public class ConfigKeys { + final static String HOST = "SP_HOST"; + final static String PORT = "SP_PORT"; + final static String FLINK_HOST = "SP_FLINK_HOST"; + final static String FLINK_PORT = "SP_FLINK_PORT"; + final static String ICON_HOST = "SP_ICON_HOST"; + final static String ICON_PORT = "SP_ICON_PORT"; + final static String SERVICE_NAME = "SP_SERVICE_NAME"; + final static String DEBUG = "SP_FLINK_DEBUG"; +} \ No newline at end of file diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/main/Init.java b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/main/Init.java new file mode 100644 index 0000000000..628bce1d70 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/main/Init.java @@ -0,0 +1,23 @@ +#set( $symbol_pound = '#' ) +#set( $symbol_dollar = '$' ) +#set( $symbol_escape = '\' ) +package ${package}.main; + +import org.streampipes.container.init.DeclarersSingleton; +import org.streampipes.container.standalone.init.StandaloneModelSubmitter; + +import ${package}.config.Config; +import ${package}.pe.processor.${packageName}.${classNamePrefix}Controller; + +public class Init extends StandaloneModelSubmitter { + + public static void main(String[] args) throws Exception { + DeclarersSingleton.getInstance() + .add(new ${classNamePrefix}Controller()); + + new Init().init(Config.INSTANCE); + + } + + +} diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__.java b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__.java new file mode 100644 index 0000000000..013c3d49b2 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__.java @@ -0,0 +1,21 @@ +#set( $symbol_pound = '#' ) +#set( $symbol_dollar = '$' ) +#set( $symbol_escape = '\' ) + +package ${package}.pe.processor.${packageName}; + +import org.apache.flink.api.common.functions.FlatMapFunction; +import org.apache.flink.util.Collector; + +import java.util.Map; + +public class ${classNamePrefix} implements FlatMapFunction, Map> { + + @Override + public void flatMap(Map in, + Collector> out) throws Exception { + + out.collect(in); + } +} diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Controller.java b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Controller.java new file mode 100644 index 0000000000..c5f510a077 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Controller.java @@ -0,0 +1,55 @@ +#set( $symbol_pound = '#' ) +#set( $symbol_dollar = '$' ) +#set( $symbol_escape = '\' ) +package ${package}.pe.processor.${packageName}; + +import ${package}.config.Config; + +import org.streampipes.model.DataProcessorType; +import org.streampipes.model.graph.DataProcessorDescription; +import org.streampipes.model.graph.DataProcessorInvocation; +import org.streampipes.sdk.builder.ProcessingElementBuilder; +import org.streampipes.sdk.builder.StreamRequirementsBuilder; +import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor; +import org.streampipes.sdk.helpers.EpRequirements; +import org.streampipes.sdk.helpers.Labels; +import org.streampipes.sdk.helpers.OutputStrategies; +import org.streampipes.sdk.helpers.SupportedFormats; +import org.streampipes.sdk.helpers.SupportedProtocols; +import org.streampipes.wrapper.flink.FlinkDataProcessorDeclarer; +import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime; + +public class ${classNamePrefix}Controller extends + FlinkDataProcessorDeclarer<${classNamePrefix}Parameters> { + + private static final String EXAMPLE_KEY = "example-key"; + + @Override + public DataProcessorDescription declareModel() { + return ProcessingElementBuilder.create("${package}-${packageName}", "${classNamePrefix}", + "Description") + .category(DataProcessorType.ENRICH) + .requiredStream(StreamRequirementsBuilder + .create() + .requiredProperty(EpRequirements.anyProperty()) + .build()) + .supportedFormats(SupportedFormats.jsonFormat()) + .supportedProtocols(SupportedProtocols.kafka()) + .outputStrategy(OutputStrategies.keep()) + .requiredTextParameter(Labels.from(EXAMPLE_KEY, "Example Text Parameter", "Example " + + "Text Parameter Description")) + .build(); + } + + @Override + public FlinkDataProcessorRuntime<${classNamePrefix}Parameters> getRuntime(DataProcessorInvocation + graph, ProcessingElementParameterExtractor extractor) { + + String exampleString = extractor.singleValueParameter(EXAMPLE_KEY, String.class); + + ${classNamePrefix}Parameters params = new ${classNamePrefix}Parameters(graph, exampleString); + + return new ${classNamePrefix}Program(params, Config.INSTANCE.getDebug()); + } + +} diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Parameters.java b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Parameters.java new file mode 100644 index 0000000000..4e9eaaec03 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Parameters.java @@ -0,0 +1,22 @@ +#set( $symbol_pound = '#' ) +#set( $symbol_dollar = '$' ) +#set( $symbol_escape = '\' ) +package ${package}.pe.processor.${packageName}; + +import org.streampipes.model.graph.DataProcessorInvocation; +import org.streampipes.wrapper.params.binding.EventProcessorBindingParams; + +public class ${classNamePrefix}Parameters extends EventProcessorBindingParams { + + private String exampleText; + + public ${classNamePrefix}Parameters(DataProcessorInvocation graph, String exampleText) { + super(graph); + this.exampleText = exampleText; + } + + public String getExampleText() { + return exampleText; + } + +} diff --git a/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Program.java b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Program.java new file mode 100644 index 0000000000..1e0e995b8d --- /dev/null +++ b/archetypes/streampipes-archetype-pe-processors-flink/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Program.java @@ -0,0 +1,39 @@ +#set( $symbol_pound = '#' ) +#set( $symbol_dollar = '$' ) +#set( $symbol_escape = '\' ) +package ${package}.pe.processor.${packageName}; + +import ${package}.config.Config; + +import org.apache.flink.streaming.api.datastream.DataStream; +import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime; +import org.streampipes.wrapper.flink.FlinkDeploymentConfig; + +import java.io.Serializable; +import java.util.Map; + +public class ${classNamePrefix}Program extends + FlinkDataProcessorRuntime<${classNamePrefix}Parameters> +implements Serializable { + + private static final long serialVersionUID = 1L; + private static final String INDEX_NAME_PREFIX = "sp_"; + + public ${classNamePrefix}Program(${classNamePrefix}Parameters params, boolean debug) { + super(params, debug); + } + + @Override + protected FlinkDeploymentConfig getDeploymentConfig() { + return new FlinkDeploymentConfig(Config.JAR_FILE, + Config.INSTANCE.getFlinkHost(), Config.INSTANCE.getFlinkPort()); + } + + @Override + protected DataStream> getApplicationLogic( + DataStream>... messageStream) { + + return messageStream[0] + .flatMap(new ${classNamePrefix}()); + } +} diff --git a/archetypes/streampipes-archetype-pe-jvm/src/test/resources/projects/basic/archetype.properties b/archetypes/streampipes-archetype-pe-processors-flink/src/test/resources/projects/basic/archetype.properties similarity index 87% rename from archetypes/streampipes-archetype-pe-jvm/src/test/resources/projects/basic/archetype.properties rename to archetypes/streampipes-archetype-pe-processors-flink/src/test/resources/projects/basic/archetype.properties index b81587bae5..48bcb20ea5 100644 --- a/archetypes/streampipes-archetype-pe-jvm/src/test/resources/projects/basic/archetype.properties +++ b/archetypes/streampipes-archetype-pe-processors-flink/src/test/resources/projects/basic/archetype.properties @@ -4,4 +4,4 @@ version=0.1-SNAPSHOT groupId=test-pe artifactId=basic classNamePrefix=Example -elementName=example +packageName=example diff --git a/archetypes/streampipes-archetype-pe-processors-jvm/README.md b/archetypes/streampipes-archetype-pe-processors-jvm/README.md new file mode 100644 index 0000000000..a8900d563c --- /dev/null +++ b/archetypes/streampipes-archetype-pe-processors-jvm/README.md @@ -0,0 +1,18 @@ +## StreamPipes Maven Archetype for Standalone Processors + +### Usage + +mvn archetype:generate \ + -DarchetypeGroupId=org.streampipes \ + -DarchetypeArtifactId=streampipes-archetype-pe-processors-jvm \ + -DarchetypeVersion=0.55.3-SNAPSHOT \ + -DgroupId=my.test.groupId \ + -DartifactId=my-test-artifact-id + -DclassNamePrefix=MyProcessor + -DpackageName=mypackagename + +### Variables + +* classNamePrefix: Will be used as a prefix to name your controller & parameter classes +* packageName: Will be used as the package name + diff --git a/archetypes/streampipes-archetype-pe-processors-jvm/pom.xml b/archetypes/streampipes-archetype-pe-processors-jvm/pom.xml new file mode 100644 index 0000000000..df0c5c921b --- /dev/null +++ b/archetypes/streampipes-archetype-pe-processors-jvm/pom.xml @@ -0,0 +1,59 @@ + + + 4.0.0 + + org.streampipes + streampipes-parent + 0.60.0 + ../../pom.xml + + streampipes-archetype-pe-processors-jvm + maven-archetype + + streampipes-archetype-pe-processors-jvm + + + + + org.apache.maven.archetype + archetype-packaging + 3.0.1 + + + + + + + maven-archetype-plugin + 3.0.1 + + + + + + + + scm:git:ssh://git@ipe-wim-gitlab.fzi.de:2222/streampipes/ce-backend.git/archetypes/streampipes-archetype-pe-processors-jvm + + + + + + deployment + Internal Releases + https://laus.fzi.de/nexus/content/repositories/public/ + + + + + deployment + Internal Releases + https://laus.fzi.de/nexus/content/repositories/releases/ + + + deployment + Internal Releases + https://laus.fzi.de/nexus/content/repositories/snapshots/ + + + diff --git a/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/META-INF/maven/archetype-metadata.xml b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/META-INF/maven/archetype-metadata.xml new file mode 100644 index 0000000000..c309cbf6b0 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/META-INF/maven/archetype-metadata.xml @@ -0,0 +1,35 @@ + + + + + src/main/java + + **/*.java + + + + src/test/java + + **/*.java + + + + + + Dockerfile + + + + + + Example + + + example + + + diff --git a/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/Dockerfile b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/Dockerfile new file mode 100644 index 0000000000..06f89bbaae --- /dev/null +++ b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/Dockerfile @@ -0,0 +1,8 @@ +FROM anapsix/alpine-java:8 + +EXPOSE 8090 +ENV CONSUL_LOCATION consul + +COPY ./target/${artifactId}.jar /streampipes-processing-element-container.jar + +ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"] diff --git a/archetypes/streampipes-archetype-pe-jvm/src/main/resources/archetype-resources/pom.xml b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/pom.xml similarity index 95% rename from archetypes/streampipes-archetype-pe-jvm/src/main/resources/archetype-resources/pom.xml rename to archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/pom.xml index c05e8b461a..3ce1679429 100644 --- a/archetypes/streampipes-archetype-pe-jvm/src/main/resources/archetype-resources/pom.xml +++ b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/pom.xml @@ -7,48 +7,43 @@ ${version} - 0.51.2-SNAPSHOT + 0.55.3-SNAPSHOT org.streampipes - streampipes-commons - ${sp.version} - - - org.streampipes - streampipes-messaging-kafka + streampipes-wrapper-standalone ${sp.version} org.streampipes - streampipes-messaging-jms + streampipes-sdk ${sp.version} org.streampipes - streampipes-dataformat-json + streampipes-container-standalone ${sp.version} org.streampipes - streampipes-wrapper-standalone + streampipes-config ${sp.version} org.streampipes - streampipes-container-standalone + streampipes-dataformat-json ${sp.version} org.streampipes - streampipes-sdk + streampipes-messaging-kafka ${sp.version} org.streampipes - streampipes-config + streampipes-messaging-jms ${sp.version} diff --git a/archetypes/streampipes-archetype-pe-jvm/src/main/resources/archetype-resources/src/main/java/config/Config.java b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/src/main/java/config/Config.java similarity index 78% rename from archetypes/streampipes-archetype-pe-jvm/src/main/resources/archetype-resources/src/main/java/config/Config.java rename to archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/src/main/java/config/Config.java index 328e3418f6..51cf66e4ae 100644 --- a/archetypes/streampipes-archetype-pe-jvm/src/main/resources/archetype-resources/src/main/java/config/Config.java +++ b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/src/main/java/config/Config.java @@ -3,34 +3,21 @@ #set( $symbol_escape = '\' ) package ${package}.config; - import org.streampipes.config.SpConfig; import org.streampipes.container.model.PeConfig; +import static ${package}.config.ConfigKeys.*; + public enum Config implements PeConfig { INSTANCE; private SpConfig config; - private final static String HOST = "host"; - private final static String PORT = "port"; - private final static String KAFKA_HOST = "kafka_host"; - private final static String KAFKA_PORT = "kafka_port"; - private final static String ZOOKEEPER_HOST = "zookeeper_host"; - private final static String ZOOKEEPER_PORT = "zookeeper_port"; - private final static String JMS_HOST = "jms_host"; - private final static String JMS_PORT = "jms_port"; - private final static String NGINX_HOST = "nginx_host"; - private final static String NGINX_PORT = "nginx_port"; - private final static String ICON_HOST = "icon_host"; - private final static String ICON_PORT = "icon_port"; - public final static String serverUrl; public final static String iconBaseUrl; private final static String SERVICE_ID= "pe/${package}"; - private final static String SERVICE_NAME = "service_name"; Config() { config = SpConfig.getSpConfig("pe/${package}"); @@ -52,7 +39,7 @@ public enum Config implements PeConfig { static { serverUrl = Config.INSTANCE.getHost() + ":" + Config.INSTANCE.getPort(); - iconBaseUrl = Config.INSTANCE.getIconHost() + ":" + Config.INSTANCE.getIconPort() + "/img/pe_icons"; + iconBaseUrl = "http://" + Config.INSTANCE.getIconHost() + ":" + Config.INSTANCE.getIconPort() + "/assets/img/pe_icons"; } public static final String getIconUrl(String pictureName) { diff --git a/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/src/main/java/config/ConfigKeys.java b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/src/main/java/config/ConfigKeys.java new file mode 100644 index 0000000000..43922c7a21 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/src/main/java/config/ConfigKeys.java @@ -0,0 +1,22 @@ +#set( $symbol_pound = '#' ) +#set( $symbol_dollar = '$' ) +#set( $symbol_escape = '\' ) +package ${package}.config; + +public class ConfigKeys { + final static String HOST = "SP_HOST"; + final static String PORT = "SP_PORT"; + final static String ICON_HOST = "SP_ICON_HOST"; + final static String ICON_PORT = "SP_ICON_PORT"; + final static String KAFKA_HOST = "SP_KAFKA_HOST"; + final static String KAFKA_PORT = "SP_KAFKA_PORT"; + final static String ZOOKEEPER_HOST = "SP_ZOOKEEPER_HOST"; + final static String ZOOKEEPER_PORT = "SP_ZOOKEEPER_PORT"; + final static String COUCHDB_HOST = "SP_COUCHDB_HOST"; + final static String COUCHDB_PORT = "SP_COCHDB_PORT"; + final static String JMS_HOST = "SP_JMS_HOST"; + final static String JMS_PORT = "SP_JMS_PORT"; + final static String NGINX_HOST = "SP_NGINX_HOST"; + final static String NGINX_PORT = "SP_NGINX_PORT"; + final static String SERVICE_NAME = "SP_SERVICE_NAME"; +} \ No newline at end of file diff --git a/archetypes/streampipes-archetype-pe-jvm/src/main/resources/archetype-resources/src/main/java/main/Init.java b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/src/main/java/main/Init.java similarity index 92% rename from archetypes/streampipes-archetype-pe-jvm/src/main/resources/archetype-resources/src/main/java/main/Init.java rename to archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/src/main/java/main/Init.java index e464d68637..c36e4dec52 100644 --- a/archetypes/streampipes-archetype-pe-jvm/src/main/resources/archetype-resources/src/main/java/main/Init.java +++ b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/src/main/java/main/Init.java @@ -9,7 +9,7 @@ import org.streampipes.messaging.kafka.SpKafkaProtocolFactory; import ${package}.config.Config; -import ${package}.pe.${elementName}.${classNamePrefix}Controller; +import ${package}.pe.processor.${packageName}.${classNamePrefix}Controller; public class Init extends StandaloneModelSubmitter { diff --git a/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__.java b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__.java new file mode 100644 index 0000000000..260a65278b --- /dev/null +++ b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__.java @@ -0,0 +1,39 @@ +#set( $symbol_pound = '#' ) +#set( $symbol_dollar = '$' ) +#set( $symbol_escape = '\' ) + +package ${package}.pe.processor.${packageName}; + +import org.streampipes.commons.exceptions.SpRuntimeException; +import org.streampipes.model.graph.DataProcessorInvocation; +import org.streampipes.wrapper.routing.SpOutputCollector; +import org.streampipes.wrapper.standalone.engine.StandaloneEventProcessorEngine; + +import org.slf4j.Logger; + +import java.util.Map; + +public class ${classNamePrefix} extends StandaloneEventProcessorEngine<${classNamePrefix}Parameters> { + + private static Logger LOG; + + public ${classNamePrefix}(${classNamePrefix}Parameters params) { + super(params); + } + + @Override + public void onInvocation(${classNamePrefix}Parameters parameters, + DataProcessorInvocation graph) { + + } + + @Override + public void onEvent(Map in, String sourceInfo, SpOutputCollector out) { + + } + + @Override + public void onDetach() { + + } +} diff --git a/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Controller.java b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Controller.java new file mode 100644 index 0000000000..4f1c059264 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Controller.java @@ -0,0 +1,53 @@ +#set( $symbol_pound = '#' ) +#set( $symbol_dollar = '$' ) +#set( $symbol_escape = '\' ) +package ${package}.pe.processor.${packageName}; + +import org.streampipes.model.DataProcessorType; +import org.streampipes.model.graph.DataProcessorDescription; +import org.streampipes.model.graph.DataProcessorInvocation; +import org.streampipes.sdk.builder.ProcessingElementBuilder; +import org.streampipes.sdk.builder.StreamRequirementsBuilder; +import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor; +import org.streampipes.sdk.helpers.EpRequirements; +import org.streampipes.sdk.helpers.Labels; +import org.streampipes.sdk.helpers.OutputStrategies; +import org.streampipes.sdk.helpers.SupportedFormats; +import org.streampipes.sdk.helpers.SupportedProtocols; +import org.streampipes.wrapper.standalone.ConfiguredEventProcessor; +import org.streampipes.wrapper.standalone.declarer.StandaloneEventProcessingDeclarer; + +public class ${classNamePrefix}Controller extends StandaloneEventProcessingDeclarer<${classNamePrefix}Parameters> { + + private static final String EXAMPLE_KEY = "example-key"; + + @Override + public DataProcessorDescription declareModel() { + return ProcessingElementBuilder.create("${package}-${packageName}", "${classNamePrefix}", + "Description") + .category(DataProcessorType.AGGREGATE) + .requiredStream(StreamRequirementsBuilder + .create() + .requiredProperty(EpRequirements.anyProperty()) + .build()) + .supportedFormats(SupportedFormats.jsonFormat()) + .supportedProtocols(SupportedProtocols.kafka()) + .requiredTextParameter(Labels.from(EXAMPLE_KEY, "Example Text Parameter", "Example " + + "Text Parameter Description")) + .outputStrategy(OutputStrategies.keep()) + .build(); + } + + @Override + public ConfiguredEventProcessor<${classNamePrefix}Parameters> onInvocation + (DataProcessorInvocation graph) { + ProcessingElementParameterExtractor extractor = getExtractor(graph); + + String exampleString = extractor.singleValueParameter(EXAMPLE_KEY, String.class); + + ${classNamePrefix}Parameters params = new ${classNamePrefix}Parameters(graph, exampleString); + + return new ConfiguredEventProcessor<>(params, () -> new ${classNamePrefix}(params)); + } + +} diff --git a/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Parameters.java b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Parameters.java new file mode 100644 index 0000000000..4e9eaaec03 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-processors-jvm/src/main/resources/archetype-resources/src/main/java/pe/processor/__packageName__/__classNamePrefix__Parameters.java @@ -0,0 +1,22 @@ +#set( $symbol_pound = '#' ) +#set( $symbol_dollar = '$' ) +#set( $symbol_escape = '\' ) +package ${package}.pe.processor.${packageName}; + +import org.streampipes.model.graph.DataProcessorInvocation; +import org.streampipes.wrapper.params.binding.EventProcessorBindingParams; + +public class ${classNamePrefix}Parameters extends EventProcessorBindingParams { + + private String exampleText; + + public ${classNamePrefix}Parameters(DataProcessorInvocation graph, String exampleText) { + super(graph); + this.exampleText = exampleText; + } + + public String getExampleText() { + return exampleText; + } + +} diff --git a/archetypes/streampipes-archetype-pe-processors-jvm/src/test/resources/projects/basic/archetype.properties b/archetypes/streampipes-archetype-pe-processors-jvm/src/test/resources/projects/basic/archetype.properties new file mode 100644 index 0000000000..48bcb20ea5 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-processors-jvm/src/test/resources/projects/basic/archetype.properties @@ -0,0 +1,7 @@ +#Tue Nov 21 10:33:48 CET 2017 +package=org.streampipes.test +version=0.1-SNAPSHOT +groupId=test-pe +artifactId=basic +classNamePrefix=Example +packageName=example diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/README.md b/archetypes/streampipes-archetype-pe-sinks-flink/README.md new file mode 100644 index 0000000000..8be1f4adac --- /dev/null +++ b/archetypes/streampipes-archetype-pe-sinks-flink/README.md @@ -0,0 +1,18 @@ +## StreamPipes Maven Archetype for Flink-based Sinks + +### Usage + +mvn archetype:generate \ + -DarchetypeGroupId=org.streampipes \ + -DarchetypeArtifactId=streampipes-archetype-pe-sinks-flink \ + -DarchetypeVersion=0.55.3-SNAPSHOT \ + -DgroupId=my.test.groupId \ + -DartifactId=my-test-artifact-id + -DclassNamePrefix=MySink + -DpackageName=mypackagename + +### Variables + +* classNamePrefix: Will be used as a prefix to name your controller & parameter classes +* packageName: Will be used as the package name + diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/pom.xml b/archetypes/streampipes-archetype-pe-sinks-flink/pom.xml new file mode 100644 index 0000000000..a08e64ab7b --- /dev/null +++ b/archetypes/streampipes-archetype-pe-sinks-flink/pom.xml @@ -0,0 +1,59 @@ + + + 4.0.0 + + org.streampipes + streampipes-parent + 0.60.0 + ../../pom.xml + + streampipes-archetype-pe-sinks-flink + maven-archetype + + streampipes-archetype-pe-sinks-flink + + + + + org.apache.maven.archetype + archetype-packaging + 3.0.1 + + + + + + + maven-archetype-plugin + 3.0.1 + + + + + + + + scm:git:ssh://git@ipe-wim-gitlab.fzi.de:2222/streampipes/ce-backend.git/archetypes/streampipes-archetype-pe-sinks-jvm + + + + + + deployment + Internal Releases + https://laus.fzi.de/nexus/content/repositories/public/ + + + + + deployment + Internal Releases + https://laus.fzi.de/nexus/content/repositories/releases/ + + + deployment + Internal Releases + https://laus.fzi.de/nexus/content/repositories/snapshots/ + + + diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/META-INF/maven/archetype-metadata.xml b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/META-INF/maven/archetype-metadata.xml new file mode 100644 index 0000000000..c309cbf6b0 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/META-INF/maven/archetype-metadata.xml @@ -0,0 +1,35 @@ + + + + + src/main/java + + **/*.java + + + + src/test/java + + **/*.java + + + + + + Dockerfile + + + + + + Example + + + example + + + diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/Dockerfile b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/Dockerfile new file mode 100644 index 0000000000..06f89bbaae --- /dev/null +++ b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/Dockerfile @@ -0,0 +1,8 @@ +FROM anapsix/alpine-java:8 + +EXPOSE 8090 +ENV CONSUL_LOCATION consul + +COPY ./target/${artifactId}.jar /streampipes-processing-element-container.jar + +ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"] diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/pom.xml b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/pom.xml new file mode 100644 index 0000000000..5a80f20dd8 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/pom.xml @@ -0,0 +1,154 @@ + + + 4.0.0 + ${groupId} + ${artifactId} + ${version} + + + 0.55.3-SNAPSHOT + + + + + org.streampipes + streampipes-container-standalone + ${sp.version} + + + org.apache.kafka + kafka_2.10 + + + org.apache.kafka + kafka-clients + + + + + org.streampipes + streampipes-commons + ${sp.version} + + + org.apache.kafka + kafka_2.10 + + + org.apache.kafka + kafka-clients + + + + + org.streampipes + streampipes-wrapper-flink + ${sp.version} + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + + + commons-beanutils + commons-beanutils + 1.9.2 + + + org.streampipes + streampipes-sdk + ${sp.version} + + + org.apache.logging.log4j + log4j-to-slf4j + 2.8.2 + + + org.slf4j + slf4j-api + 1.7.24 + + + org.streampipes + streampipes-config + ${sp.version} + + + + + + laus + nexus repository + http://laus.fzi.de/nexus/content/repositories/public/ + + true + daily + + + true + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.3 + + + package + + shade + + + false + + + ${package}.main.Init + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + META-INF/maven/com.github.jsonld-java/jsonld-java/pom.xml + META-INF/maven/com.github.jsonld-java/jsonld-java-sesame/pom.xml + + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + 1.8 + 1.8 + UTF-8 + + + + ${artifactId} + + diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/config/Config.java b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/config/Config.java new file mode 100644 index 0000000000..e742629bca --- /dev/null +++ b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/config/Config.java @@ -0,0 +1,84 @@ +#set( $symbol_pound = '#' ) +#set( $symbol_dollar = '$' ) +#set( $symbol_escape = '\' ) +package ${package}.config; + +import org.streampipes.config.SpConfig; +import org.streampipes.container.model.PeConfig; + +public enum Config implements PeConfig { + + INSTANCE; + + private SpConfig config; + public static final String JAR_FILE = "./streampipes-processing-element-container.jar"; + + private final static String service_id = "pe/${package}"; + private final static String service_name = "${packageName}"; + private final static String service_container_name = "${artifactId}"; + + + Config() { + config = SpConfig.getSpConfig(service_id); + + config.register(ConfigKeys.HOST, service_container_name, "Hostname for the pe mixed flink component"); + config.register(ConfigKeys.PORT, 8090, "Port for the pe mixed flink component"); + config.register(ConfigKeys.FLINK_HOST, "jobmanager", "Host for the flink cluster"); + config.register(ConfigKeys.FLINK_PORT, 6123, "Port for the flink cluster"); + config.register(ConfigKeys.ICON_HOST, "backend", "Hostname for the icon host"); + config.register(ConfigKeys.ICON_PORT, 80, "Port for the icons in nginx"); + + config.register(ConfigKeys.DEBUG, false, "When set to true programs are not deployed to cluster, but executed locally"); + + config.register(ConfigKeys.SERVICE_NAME, service_name, "The name of the service"); + + } + + @Override + public String getHost() { + return config.getString(ConfigKeys.HOST); + } + + @Override + public int getPort() { + return config.getInteger(ConfigKeys.PORT); + } + + public String getFlinkHost() { + return config.getString(ConfigKeys.FLINK_HOST); + } + + public int getFlinkPort() { + return config.getInteger(ConfigKeys.FLINK_PORT); + } + + public static final String iconBaseUrl = "http://" + Config.INSTANCE.getIconHost() + ":" + + Config.INSTANCE.getIconPort() + "/assets/img/pe_icons"; + + public static final String getIconUrl(String pictureName) { + return iconBaseUrl + "/" + pictureName + ".png"; + } + + public String getIconHost() { + return config.getString(ConfigKeys.ICON_HOST); + } + + public int getIconPort() { + return config.getInteger(ConfigKeys.ICON_PORT); + } + + public boolean getDebug() { + return config.getBoolean(ConfigKeys.DEBUG); + } + + @Override + public String getId() { + return service_id; + } + + @Override + public String getName() { + return config.getString(ConfigKeys.SERVICE_NAME); + } + +} diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/config/ConfigKeys.java b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/config/ConfigKeys.java new file mode 100644 index 0000000000..4110b02b99 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/config/ConfigKeys.java @@ -0,0 +1,15 @@ +#set( $symbol_pound = '#' ) +#set( $symbol_dollar = '$' ) +#set( $symbol_escape = '\' ) +package ${package}.config; + +public class ConfigKeys { + final static String HOST = "SP_HOST"; + final static String PORT = "SP_PORT"; + final static String FLINK_HOST = "SP_FLINK_HOST"; + final static String FLINK_PORT = "SP_FLINK_PORT"; + final static String ICON_HOST = "SP_ICON_HOST"; + final static String ICON_PORT = "SP_ICON_PORT"; + final static String SERVICE_NAME = "SP_SERVICE_NAME"; + final static String DEBUG = "SP_FLINK_DEBUG"; +} \ No newline at end of file diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/main/Init.java b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/main/Init.java new file mode 100644 index 0000000000..9886522bb5 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/main/Init.java @@ -0,0 +1,23 @@ +#set( $symbol_pound = '#' ) +#set( $symbol_dollar = '$' ) +#set( $symbol_escape = '\' ) +package ${package}.main; + +import org.streampipes.container.init.DeclarersSingleton; +import org.streampipes.container.standalone.init.StandaloneModelSubmitter; + +import ${package}.config.Config; +import ${package}.pe.sink.${packageName}.${classNamePrefix}Controller; + +public class Init extends StandaloneModelSubmitter { + + public static void main(String[] args) throws Exception { + DeclarersSingleton.getInstance() + .add(new ${classNamePrefix}Controller()); + + new Init().init(Config.INSTANCE); + + } + + +} diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Controller.java b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Controller.java new file mode 100644 index 0000000000..08c9f57554 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Controller.java @@ -0,0 +1,50 @@ +#set( $symbol_pound = '#' ) +#set( $symbol_dollar = '$' ) +#set( $symbol_escape = '\' ) +package ${package}.pe.sink.${packageName}; + +import ${package}.config.Config; + +import org.streampipes.model.DataSinkType; +import org.streampipes.model.graph.DataSinkDescription; +import org.streampipes.model.graph.DataSinkInvocation; +import org.streampipes.sdk.builder.DataSinkBuilder; +import org.streampipes.sdk.builder.StreamRequirementsBuilder; +import org.streampipes.sdk.extractor.DataSinkParameterExtractor; +import org.streampipes.sdk.helpers.EpRequirements; +import org.streampipes.sdk.helpers.Labels; +import org.streampipes.sdk.helpers.SupportedFormats; +import org.streampipes.sdk.helpers.SupportedProtocols; +import org.streampipes.wrapper.flink.FlinkDataSinkDeclarer; +import org.streampipes.wrapper.flink.FlinkDataSinkRuntime; + +public class ${classNamePrefix}Controller extends FlinkDataSinkDeclarer<${classNamePrefix}Parameters> { + + private static final String EXAMPLE_KEY = "example-key"; + + @Override + public DataSinkDescription declareModel() { + return DataSinkBuilder.create("${package}-${packageName}", "${classNamePrefix}", "Description") + .category(DataSinkType.NOTIFICATION) + .requiredStream(StreamRequirementsBuilder + .create() + .requiredProperty(EpRequirements.anyProperty()) + .build()) + .supportedFormats(SupportedFormats.jsonFormat()) + .supportedProtocols(SupportedProtocols.kafka()) + .requiredTextParameter(Labels.from(EXAMPLE_KEY, "Example Text Parameter", "Example " + + "Text Parameter Description")) + .build(); + } + + @Override + public FlinkDataSinkRuntime<${classNamePrefix}Parameters> getRuntime(DataSinkInvocation graph, DataSinkParameterExtractor extractor) { + + String exampleString = extractor.singleValueParameter(EXAMPLE_KEY, String.class); + + ${classNamePrefix}Parameters params = new ${classNamePrefix}Parameters(graph, exampleString); + + return new ${classNamePrefix}Program(params, Config.INSTANCE.getDebug()); + } + +} diff --git a/archetypes/streampipes-archetype-pe-jvm/src/main/resources/archetype-resources/src/main/java/pe/__elementName__/__classNamePrefix__Parameters.java b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Parameters.java similarity index 92% rename from archetypes/streampipes-archetype-pe-jvm/src/main/resources/archetype-resources/src/main/java/pe/__elementName__/__classNamePrefix__Parameters.java rename to archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Parameters.java index 0a31777ee6..631a156668 100644 --- a/archetypes/streampipes-archetype-pe-jvm/src/main/resources/archetype-resources/src/main/java/pe/__elementName__/__classNamePrefix__Parameters.java +++ b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Parameters.java @@ -1,7 +1,7 @@ #set( $symbol_pound = '#' ) #set( $symbol_dollar = '$' ) #set( $symbol_escape = '\' ) -package ${package}.pe.${elementName}; +package ${package}.pe.sink.${packageName}; import org.streampipes.model.graph.DataSinkInvocation; import org.streampipes.wrapper.params.binding.EventSinkBindingParams; diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Program.java b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Program.java new file mode 100644 index 0000000000..7f53162e69 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-sinks-flink/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Program.java @@ -0,0 +1,37 @@ +#set( $symbol_pound = '#' ) +#set( $symbol_dollar = '$' ) +#set( $symbol_escape = '\' ) +package ${package}.pe.sink.${packageName}; + +import ${package}.config.Config; + +import org.apache.flink.streaming.api.datastream.DataStream; +import org.streampipes.wrapper.flink.FlinkDataSinkRuntime; +import org.streampipes.wrapper.flink.FlinkDeploymentConfig; + +import java.io.Serializable; +import java.util.Map; + +public class ${classNamePrefix}Program extends FlinkDataSinkRuntime<${classNamePrefix}Parameters> +implements Serializable { + + private static final long serialVersionUID = 1L; + private static final String INDEX_NAME_PREFIX = "sp_"; + + public ${classNamePrefix}Program(${classNamePrefix}Parameters params, boolean debug) { + super(params, debug); + } + + @Override + protected FlinkDeploymentConfig getDeploymentConfig() { + return new FlinkDeploymentConfig(Config.JAR_FILE, + Config.INSTANCE.getFlinkHost(), Config.INSTANCE.getFlinkPort()); + } + + @Override + public void getSink( + DataStream>... convertedStream) { + + // TODO add logic here + } +} diff --git a/archetypes/streampipes-archetype-pe-sinks-flink/src/test/resources/projects/basic/archetype.properties b/archetypes/streampipes-archetype-pe-sinks-flink/src/test/resources/projects/basic/archetype.properties new file mode 100644 index 0000000000..48bcb20ea5 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-sinks-flink/src/test/resources/projects/basic/archetype.properties @@ -0,0 +1,7 @@ +#Tue Nov 21 10:33:48 CET 2017 +package=org.streampipes.test +version=0.1-SNAPSHOT +groupId=test-pe +artifactId=basic +classNamePrefix=Example +packageName=example diff --git a/archetypes/streampipes-archetype-pe-sinks-jvm/README.md b/archetypes/streampipes-archetype-pe-sinks-jvm/README.md new file mode 100644 index 0000000000..4d3fa415ba --- /dev/null +++ b/archetypes/streampipes-archetype-pe-sinks-jvm/README.md @@ -0,0 +1,18 @@ +## StreamPipes Maven Archetype for Standalone Sinks + +### Usage + +mvn archetype:generate \ + -DarchetypeGroupId=org.streampipes \ + -DarchetypeArtifactId=streampipes-archetype-pe-sinks-jvm \ + -DarchetypeVersion=0.55.3-SNAPSHOT \ + -DgroupId=my.test.groupId \ + -DartifactId=my-test-artifact-id + -DclassNamePrefix=MySink + -DpackageName=mypackagename + +### Variables + +* classNamePrefix: Will be used as a prefix to name your controller & parameter classes +* packageName: Will be used as the package name + diff --git a/archetypes/streampipes-archetype-pe-jvm/pom.xml b/archetypes/streampipes-archetype-pe-sinks-jvm/pom.xml similarity index 89% rename from archetypes/streampipes-archetype-pe-jvm/pom.xml rename to archetypes/streampipes-archetype-pe-sinks-jvm/pom.xml index a269103818..c120d8a585 100644 --- a/archetypes/streampipes-archetype-pe-jvm/pom.xml +++ b/archetypes/streampipes-archetype-pe-sinks-jvm/pom.xml @@ -4,13 +4,13 @@ org.streampipes streampipes-parent - 0.55.2 + 0.60.0 ../../pom.xml - streampipes-archetype-pe-jvm + streampipes-archetype-pe-sinks-jvm maven-archetype - streampipes-archetype-pe-jvm + streampipes-archetype-pe-sinks-jvm @@ -33,7 +33,7 @@ - scm:git:ssh://git@ipe-wim-gitlab.fzi.de:2222/streampipes/ce-backend.git/archetypes/streampipes-archetype-pe-sinks-standalone + scm:git:ssh://git@ipe-wim-gitlab.fzi.de:2222/streampipes/ce-backend.git/archetypes/streampipes-archetype-pe-sinks-jvm diff --git a/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/META-INF/maven/archetype-metadata.xml b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/META-INF/maven/archetype-metadata.xml new file mode 100644 index 0000000000..c309cbf6b0 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/META-INF/maven/archetype-metadata.xml @@ -0,0 +1,35 @@ + + + + + src/main/java + + **/*.java + + + + src/test/java + + **/*.java + + + + + + Dockerfile + + + + + + Example + + + example + + + diff --git a/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/Dockerfile b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/Dockerfile new file mode 100644 index 0000000000..06f89bbaae --- /dev/null +++ b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/Dockerfile @@ -0,0 +1,8 @@ +FROM anapsix/alpine-java:8 + +EXPOSE 8090 +ENV CONSUL_LOCATION consul + +COPY ./target/${artifactId}.jar /streampipes-processing-element-container.jar + +ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"] diff --git a/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/pom.xml b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/pom.xml new file mode 100644 index 0000000000..3ce1679429 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/pom.xml @@ -0,0 +1,119 @@ + + + 4.0.0 + ${groupId} + ${artifactId} + ${version} + + + 0.55.3-SNAPSHOT + + + + + org.streampipes + streampipes-wrapper-standalone + ${sp.version} + + + org.streampipes + streampipes-sdk + ${sp.version} + + + org.streampipes + streampipes-container-standalone + ${sp.version} + + + org.streampipes + streampipes-config + ${sp.version} + + + org.streampipes + streampipes-dataformat-json + ${sp.version} + + + org.streampipes + streampipes-messaging-kafka + ${sp.version} + + + org.streampipes + streampipes-messaging-jms + ${sp.version} + + + + + + laus + nexus repository + http://laus.fzi.de/nexus/content/repositories/public/ + + true + daily + + + true + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.3 + + + package + + shade + + + false + + + ${package}.main.Init + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + META-INF/maven/com.github.jsonld-java/jsonld-java/pom.xml + META-INF/maven/com.github.jsonld-java/jsonld-java-sesame/pom.xml + + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + 1.8 + 1.8 + UTF-8 + + + + ${artifactId} + + diff --git a/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/src/main/java/config/Config.java b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/src/main/java/config/Config.java new file mode 100644 index 0000000000..51cf66e4ae --- /dev/null +++ b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/src/main/java/config/Config.java @@ -0,0 +1,115 @@ +#set( $symbol_pound = '#' ) +#set( $symbol_dollar = '$' ) +#set( $symbol_escape = '\' ) +package ${package}.config; + +import org.streampipes.config.SpConfig; +import org.streampipes.container.model.PeConfig; + +import static ${package}.config.ConfigKeys.*; + +public enum Config implements PeConfig { + + INSTANCE; + + private SpConfig config; + + public final static String serverUrl; + public final static String iconBaseUrl; + + private final static String SERVICE_ID= "pe/${package}"; + + Config() { + config = SpConfig.getSpConfig("pe/${package}"); + + config.register(HOST, "${projectName}", "Hostname for the pe sinks"); + config.register(PORT, 8090, "Port for the pe sinks"); + config.register(NGINX_HOST, System.getenv("STREAMPIPES_HOST"), "External hostname of " + + "StreamPipes Nginx"); + config.register(NGINX_PORT, 80, "External port of StreamPipes Nginx"); + config.register(KAFKA_HOST, "kafka", "Host for kafka of the pe sinks project"); + config.register(KAFKA_PORT, 9092, "Port for kafka of the pe sinks project"); + config.register(ZOOKEEPER_HOST, "zookeeper", "Host for zookeeper of the pe sinks project"); + config.register(ZOOKEEPER_PORT, 2181, "Port for zookeeper of the pe sinks project"); + config.register(JMS_HOST, "tcp://activemq", "Hostname for pe actions service for active mq"); + config.register(JMS_PORT, 61616, "Port for pe actions service for active mq"); + config.register(ICON_HOST, "backend", "Hostname for the icon host"); + config.register(ICON_PORT, 80, "Port for the icons in nginx"); + } + + static { + serverUrl = Config.INSTANCE.getHost() + ":" + Config.INSTANCE.getPort(); + iconBaseUrl = "http://" + Config.INSTANCE.getIconHost() + ":" + Config.INSTANCE.getIconPort() + "/assets/img/pe_icons"; + } + + public static final String getIconUrl(String pictureName) { + return iconBaseUrl + "/" + pictureName + ".png"; + } + + public String getHost() { + return config.getString(HOST); + } + + public int getPort() { + return config.getInteger(PORT); + } + + public String getKafkaHost() { + return config.getString(KAFKA_HOST); + } + + public int getKafkaPort() { + return config.getInteger(KAFKA_PORT); + } + + public String getKafkaUrl() { + return getKafkaHost() + ":" + getKafkaPort(); + } + + public String getZookeeperHost() { + return config.getString(ZOOKEEPER_HOST); + } + + public int getZookeeperPort() { + return config.getInteger(ZOOKEEPER_PORT); + } + + public String getJmsHost() { + return config.getString(JMS_HOST); + } + + public int getJmsPort() { + return config.getInteger(JMS_PORT); + } + + public String getJmsUrl() { + return getJmsHost() + ":" + getJmsPort(); + } + + public String getNginxHost() { + return config.getString(NGINX_HOST); + } + + public Integer getNginxPort() { + return config.getInteger(NGINX_PORT); + } + + public String getIconHost() { + return config.getString(ICON_HOST); + } + + public int getIconPort() { + return config.getInteger(ICON_PORT); + } + + @Override + public String getId() { + return SERVICE_ID; + } + + @Override + public String getName() { + return config.getString(SERVICE_NAME); + } + +} diff --git a/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/src/main/java/config/ConfigKeys.java b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/src/main/java/config/ConfigKeys.java new file mode 100644 index 0000000000..43922c7a21 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/src/main/java/config/ConfigKeys.java @@ -0,0 +1,22 @@ +#set( $symbol_pound = '#' ) +#set( $symbol_dollar = '$' ) +#set( $symbol_escape = '\' ) +package ${package}.config; + +public class ConfigKeys { + final static String HOST = "SP_HOST"; + final static String PORT = "SP_PORT"; + final static String ICON_HOST = "SP_ICON_HOST"; + final static String ICON_PORT = "SP_ICON_PORT"; + final static String KAFKA_HOST = "SP_KAFKA_HOST"; + final static String KAFKA_PORT = "SP_KAFKA_PORT"; + final static String ZOOKEEPER_HOST = "SP_ZOOKEEPER_HOST"; + final static String ZOOKEEPER_PORT = "SP_ZOOKEEPER_PORT"; + final static String COUCHDB_HOST = "SP_COUCHDB_HOST"; + final static String COUCHDB_PORT = "SP_COCHDB_PORT"; + final static String JMS_HOST = "SP_JMS_HOST"; + final static String JMS_PORT = "SP_JMS_PORT"; + final static String NGINX_HOST = "SP_NGINX_HOST"; + final static String NGINX_PORT = "SP_NGINX_PORT"; + final static String SERVICE_NAME = "SP_SERVICE_NAME"; +} \ No newline at end of file diff --git a/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/src/main/java/main/Init.java b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/src/main/java/main/Init.java new file mode 100644 index 0000000000..c36e4dec52 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/src/main/java/main/Init.java @@ -0,0 +1,31 @@ +#set( $symbol_pound = '#' ) +#set( $symbol_dollar = '$' ) +#set( $symbol_escape = '\' ) +package ${package}.main; + +import org.streampipes.container.init.DeclarersSingleton; +import org.streampipes.container.standalone.init.StandaloneModelSubmitter; +import org.streampipes.dataformat.json.JsonDataFormatFactory; +import org.streampipes.messaging.kafka.SpKafkaProtocolFactory; + +import ${package}.config.Config; +import ${package}.pe.processor.${packageName}.${classNamePrefix}Controller; + +public class Init extends StandaloneModelSubmitter { + + public static void main(String[] args) throws Exception { + DeclarersSingleton.getInstance() + .add(new ${classNamePrefix}Controller()); + + DeclarersSingleton.getInstance().setPort(Config.INSTANCE.getPort()); + DeclarersSingleton.getInstance().setHostName(Config.INSTANCE.getHost()); + + DeclarersSingleton.getInstance().registerDataFormat(new JsonDataFormatFactory()); + DeclarersSingleton.getInstance().registerProtocol(new SpKafkaProtocolFactory()); + + new Init().init(Config.INSTANCE); + + } + + +} diff --git a/archetypes/streampipes-archetype-pe-jvm/src/main/resources/archetype-resources/src/main/java/pe/__elementName__/__classNamePrefix__.java b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__.java similarity index 64% rename from archetypes/streampipes-archetype-pe-jvm/src/main/resources/archetype-resources/src/main/java/pe/__elementName__/__classNamePrefix__.java rename to archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__.java index d035876975..80c7ee8314 100644 --- a/archetypes/streampipes-archetype-pe-jvm/src/main/resources/archetype-resources/src/main/java/pe/__elementName__/__classNamePrefix__.java +++ b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__.java @@ -2,15 +2,22 @@ #set( $symbol_dollar = '$' ) #set( $symbol_escape = '\' ) -package ${package}.pe.${elementName}; +package ${package}.pe.sink.${packageName}; import org.streampipes.commons.exceptions.SpRuntimeException; import org.streampipes.wrapper.runtime.EventSink; -import java.util.Date; +import org.slf4j.Logger; + import java.util.Map; -public class ${classNamePrefix} implements EventSink<${classNamePrefix}Parameters> { +public class ${classNamePrefix} extends EventSink<${classNamePrefix}Parameters> { + +private static Logger LOG; + + public ${classNamePrefix}(${classNamePrefix}Parameters params) { + super(params); + } @Override public void bind(${classNamePrefix}Parameters parameters) throws SpRuntimeException { diff --git a/archetypes/streampipes-archetype-pe-jvm/src/main/resources/archetype-resources/src/main/java/pe/__elementName__/__classNamePrefix__Controller.java b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Controller.java similarity index 71% rename from archetypes/streampipes-archetype-pe-jvm/src/main/resources/archetype-resources/src/main/java/pe/__elementName__/__classNamePrefix__Controller.java rename to archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Controller.java index 2377de36fa..1046dce9b7 100644 --- a/archetypes/streampipes-archetype-pe-jvm/src/main/resources/archetype-resources/src/main/java/pe/__elementName__/__classNamePrefix__Controller.java +++ b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Controller.java @@ -1,19 +1,19 @@ #set( $symbol_pound = '#' ) #set( $symbol_dollar = '$' ) #set( $symbol_escape = '\' ) -package ${package}.pe.${elementName}; +package ${package}.pe.sink.${packageName}; import org.streampipes.model.DataSinkType; import org.streampipes.model.graph.DataSinkDescription; import org.streampipes.model.graph.DataSinkInvocation; import org.streampipes.sdk.builder.DataSinkBuilder; +import org.streampipes.sdk.builder.StreamRequirementsBuilder; import org.streampipes.sdk.extractor.DataSinkParameterExtractor; import org.streampipes.sdk.helpers.EpRequirements; import org.streampipes.sdk.helpers.Labels; import org.streampipes.sdk.helpers.SupportedFormats; import org.streampipes.sdk.helpers.SupportedProtocols; import org.streampipes.wrapper.standalone.ConfiguredEventSink; -import org.streampipes.wrapper.runtime.EventSink; import org.streampipes.wrapper.standalone.declarer.StandaloneEventSinkDeclarer; public class ${classNamePrefix}Controller extends StandaloneEventSinkDeclarer<${classNamePrefix}Parameters> { @@ -22,24 +22,27 @@ public class ${classNamePrefix}Controller extends StandaloneEventSinkDeclarer<${ @Override public DataSinkDescription declareModel() { - return DataSinkBuilder.create("${package}-${elementName}", "${classNamePrefix}", "Description") + return DataSinkBuilder.create("${package}-${packageName}", "${classNamePrefix}", "Description") .category(DataSinkType.NOTIFICATION) - .requiredPropertyStream1(EpRequirements.anyProperty()) + .requiredStream(StreamRequirementsBuilder + .create() + .requiredProperty(EpRequirements.anyProperty()) + .build()) .supportedFormats(SupportedFormats.jsonFormat()) .supportedProtocols(SupportedProtocols.kafka()) - .requiredTextParameter(EXAMPLE_KEY, "Example Text Parameter", "Example Text Parameter Description") + .requiredTextParameter(Labels.from(EXAMPLE_KEY, "Example Text Parameter", "Example " + + "Text Parameter Description")) .build(); } @Override - public ConfiguredEventSink<${classNamePrefix}Parameters> onInvocation(DataSinkInvocation graph, DataSinkParameterExtractor - extractor) { + public ConfiguredEventSink<${classNamePrefix}Parameters> onInvocation(DataSinkInvocation graph) { String exampleString = extractor.singleValueParameter(EXAMPLE_KEY, String.class); ${classNamePrefix}Parameters params = new ${classNamePrefix}Parameters(graph, exampleString); - return new ConfiguredEventSink<>(params, ${classNamePrefix}::new); + return new ConfiguredEventSink<>(params, () -> new ${classNamePrefix}(params)); } } diff --git a/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Parameters.java b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Parameters.java new file mode 100644 index 0000000000..631a156668 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-sinks-jvm/src/main/resources/archetype-resources/src/main/java/pe/sink/__packageName__/__classNamePrefix__Parameters.java @@ -0,0 +1,22 @@ +#set( $symbol_pound = '#' ) +#set( $symbol_dollar = '$' ) +#set( $symbol_escape = '\' ) +package ${package}.pe.sink.${packageName}; + +import org.streampipes.model.graph.DataSinkInvocation; +import org.streampipes.wrapper.params.binding.EventSinkBindingParams; + +public class ${classNamePrefix}Parameters extends EventSinkBindingParams { + + private String exampleText; + + public ${classNamePrefix}Parameters(DataSinkInvocation graph, String exampleText) { + super(graph); + this.exampleText = exampleText; + } + + public String getExampleText() { + return exampleText; + } + +} diff --git a/archetypes/streampipes-archetype-pe-sinks-jvm/src/test/resources/projects/basic/archetype.properties b/archetypes/streampipes-archetype-pe-sinks-jvm/src/test/resources/projects/basic/archetype.properties new file mode 100644 index 0000000000..48bcb20ea5 --- /dev/null +++ b/archetypes/streampipes-archetype-pe-sinks-jvm/src/test/resources/projects/basic/archetype.properties @@ -0,0 +1,7 @@ +#Tue Nov 21 10:33:48 CET 2017 +package=org.streampipes.test +version=0.1-SNAPSHOT +groupId=test-pe +artifactId=basic +classNamePrefix=Example +packageName=example diff --git a/archetypes/streampipes-archetype-pe-jvm/src/test/resources/projects/basic/goal.txt b/archetypes/streampipes-archetype-pe-sinks-jvm/src/test/resources/projects/basic/goal.txt similarity index 100% rename from archetypes/streampipes-archetype-pe-jvm/src/test/resources/projects/basic/goal.txt rename to archetypes/streampipes-archetype-pe-sinks-jvm/src/test/resources/projects/basic/goal.txt diff --git a/archetypes/streampipes-archetype-pe-sources/pom.xml b/archetypes/streampipes-archetype-pe-sources/pom.xml index e0240bc972..399236e230 100644 --- a/archetypes/streampipes-archetype-pe-sources/pom.xml +++ b/archetypes/streampipes-archetype-pe-sources/pom.xml @@ -4,7 +4,7 @@ org.streampipes streampipes-parent - 0.55.2 + 0.60.0 ../../pom.xml streampipes-archetype-pe-sources diff --git a/archetypes/streampipes-archetype-pe-sources/src/main/resources/archetype-resources/Dockerfile b/archetypes/streampipes-archetype-pe-sources/src/main/resources/archetype-resources/Dockerfile index f024d3ba43..9716ca8075 100644 --- a/archetypes/streampipes-archetype-pe-sources/src/main/resources/archetype-resources/Dockerfile +++ b/archetypes/streampipes-archetype-pe-sources/src/main/resources/archetype-resources/Dockerfile @@ -1,4 +1,4 @@ -FROM anapsix/alpine-java +FROM anapsix/alpine-java:8 ENV CONSUL_LOCATION consul @@ -6,6 +6,6 @@ EXPOSE 8090 RUN echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf -ADD target/${artifactId}.jar /${artifactId}.jar +COPY target/${artifactId}.jar /${artifactId}.jar ENTRYPOINT ["java", "-jar", "/${artifactId}.jar"] diff --git a/archetypes/streampipes-archetype-pe-sources/src/main/resources/archetype-resources/src/main/java/config/Config.java b/archetypes/streampipes-archetype-pe-sources/src/main/resources/archetype-resources/src/main/java/config/Config.java index 0841fdea4c..e39ba888b9 100644 --- a/archetypes/streampipes-archetype-pe-sources/src/main/resources/archetype-resources/src/main/java/config/Config.java +++ b/archetypes/streampipes-archetype-pe-sources/src/main/resources/archetype-resources/src/main/java/config/Config.java @@ -48,7 +48,7 @@ public enum ActionConfig { static { serverUrl = ActionConfig.INSTANCE.getHost() + ":" + ActionConfig.INSTANCE.getPort(); - iconBaseUrl = ActionConfig.INSTANCE.getIconHost() + ":" + ActionConfig.INSTANCE.getIconPort() + "/img/pe_icons"; + iconBaseUrl = "http://" + ActionConfig.INSTANCE.getIconHost() + ":" + ActionConfig.INSTANCE.getIconPort() + "/assets/img/pe_icons"; } public static final String getIconUrl(String pictureName) { diff --git a/archetypes/streampipes-archetype-sources-embedded/.settings/org.eclipse.jdt.core.prefs b/archetypes/streampipes-archetype-sources-embedded/.settings/org.eclipse.jdt.core.prefs deleted file mode 100644 index 8626026241..0000000000 --- a/archetypes/streampipes-archetype-sources-embedded/.settings/org.eclipse.jdt.core.prefs +++ /dev/null @@ -1,5 +0,0 @@ -eclipse.preferences.version=1 -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5 -org.eclipse.jdt.core.compiler.compliance=1.5 -org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning -org.eclipse.jdt.core.compiler.source=1.5 diff --git a/archetypes/streampipes-archetype-sources-embedded/.settings/org.eclipse.m2e.core.prefs b/archetypes/streampipes-archetype-sources-embedded/.settings/org.eclipse.m2e.core.prefs deleted file mode 100644 index 14b697b7bb..0000000000 --- a/archetypes/streampipes-archetype-sources-embedded/.settings/org.eclipse.m2e.core.prefs +++ /dev/null @@ -1,4 +0,0 @@ -activeProfiles= -eclipse.preferences.version=1 -resolveWorkspaceProjects=true -version=1 diff --git a/archetypes/streampipes-archetype-sources-embedded/pom.xml b/archetypes/streampipes-archetype-sources-embedded/pom.xml deleted file mode 100644 index 2c32b3ced5..0000000000 --- a/archetypes/streampipes-archetype-sources-embedded/pom.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - 4.0.0 - de.fzi.cep.sepa - streampipes-archetype-sources-embedded - 0.0.1-SNAPSHOT - \ No newline at end of file diff --git a/archetypes/streampipes-archetype-sources-embedded/src/main/resources/META-INF/archetype.xml b/archetypes/streampipes-archetype-sources-embedded/src/main/resources/META-INF/archetype.xml deleted file mode 100644 index c1a6dd79f3..0000000000 --- a/archetypes/streampipes-archetype-sources-embedded/src/main/resources/META-INF/archetype.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - streampipes-archetype-storm - - src/main/java/streams/__streamName__Stream.java - src/main/java/sources/__sourceName__Producer.java - src/main/java/main/Init.java - - - src/main/resources/WEB-INF/web.xml - - - - - \ No newline at end of file diff --git a/archetypes/streampipes-archetype-sources-embedded/src/main/resources/META-INF/maven/archetype-metadata.xml b/archetypes/streampipes-archetype-sources-embedded/src/main/resources/META-INF/maven/archetype-metadata.xml deleted file mode 100644 index bed07c2f1e..0000000000 --- a/archetypes/streampipes-archetype-sources-embedded/src/main/resources/META-INF/maven/archetype-metadata.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - src/main/java - - **/*.java - - - - - - - - - - - \ No newline at end of file diff --git a/archetypes/streampipes-archetype-sources-embedded/src/main/resources/archetype-resources/pom.xml b/archetypes/streampipes-archetype-sources-embedded/src/main/resources/archetype-resources/pom.xml deleted file mode 100644 index cd5b0c9bea..0000000000 --- a/archetypes/streampipes-archetype-sources-embedded/src/main/resources/archetype-resources/pom.xml +++ /dev/null @@ -1,47 +0,0 @@ - - 4.0.0 - - ${groupId} - ${artifactId} - ${version} - war - - - de.fzi.cep.sepa - semantic-epa - 0.0.1-SNAPSHOT - - - - - laus - nexus repository - http://laus.fzi.de/nexus/content/repositories/public/ - - true - daily - - - true - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.1 - - 1.8 - 1.8 - UTF-8 - - - - - ${contextPath} - - \ No newline at end of file diff --git a/archetypes/streampipes-archetype-sources-embedded/src/main/resources/archetype-resources/src/main/java/main/Init.java b/archetypes/streampipes-archetype-sources-embedded/src/main/resources/archetype-resources/src/main/java/main/Init.java deleted file mode 100644 index 74ef7196c4..0000000000 --- a/archetypes/streampipes-archetype-sources-embedded/src/main/resources/archetype-resources/src/main/java/main/Init.java +++ /dev/null @@ -1,42 +0,0 @@ -package ${package}.main; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import de.fzi.cep.sepa.desc.EmbeddedModelSubmitter; -import de.fzi.cep.sepa.desc.declarer.SemanticEventConsumerDeclarer; -import de.fzi.cep.sepa.desc.declarer.SemanticEventProcessingAgentDeclarer; -import de.fzi.cep.sepa.desc.declarer.SemanticEventProducerDeclarer; - -import ${package}.sources.${sourceName}Producer; - -public class Init extends EmbeddedModelSubmitter { - - @Override - protected List epaDeclarers() { - return new ArrayList<>(); - } - - @Override - protected List sourceDeclarers() { - return Arrays.asList(new ${sourceName}Producer()); - } - - @Override - protected List consumerDeclarers() { - return new ArrayList<>(); - } - - @Override - protected int port() { - return ${port}; - } - - @Override - protected String contextPath() { - return "/${contextPath}"; - } - - -} diff --git a/archetypes/streampipes-archetype-sources-embedded/src/main/resources/archetype-resources/src/main/java/sources/__sourceName__Producer.java b/archetypes/streampipes-archetype-sources-embedded/src/main/resources/archetype-resources/src/main/java/sources/__sourceName__Producer.java deleted file mode 100644 index ba09b7985f..0000000000 --- a/archetypes/streampipes-archetype-sources-embedded/src/main/resources/archetype-resources/src/main/java/sources/__sourceName__Producer.java +++ /dev/null @@ -1,32 +0,0 @@ -package ${package}.sources; - -import java.util.ArrayList; -import java.util.List; - -import de.fzi.cep.sepa.desc.declarer.EventStreamDeclarer; -import de.fzi.cep.sepa.desc.declarer.SemanticEventProducerDeclarer; -import de.fzi.cep.sepa.model.impl.graph.SepDescription; - -import ${package}.streams.${streamName}Stream; - -public class ${sourceName}Producer implements SemanticEventProducerDeclarer { - - @Override - public SepDescription declareModel() { - - SepDescription sep = new SepDescription("source/{sourceName.toLowerCase()}", "", ""); - - return sep; - } - - - @Override - public List getEventStreams() { - - List streams = new ArrayList(); - - streams.add(new ${streamName}Stream()); - - return streams; - } -} diff --git a/archetypes/streampipes-archetype-sources-embedded/src/main/resources/archetype-resources/src/main/java/streams/__streamName__Stream.java b/archetypes/streampipes-archetype-sources-embedded/src/main/resources/archetype-resources/src/main/java/streams/__streamName__Stream.java deleted file mode 100644 index 14c961b557..0000000000 --- a/archetypes/streampipes-archetype-sources-embedded/src/main/resources/archetype-resources/src/main/java/streams/__streamName__Stream.java +++ /dev/null @@ -1,28 +0,0 @@ -package ${package}.streams; - -import de.fzi.cep.sepa.desc.declarer.EventStreamDeclarer; -import de.fzi.cep.sepa.model.impl.EventStream; -import de.fzi.cep.sepa.model.impl.graph.SepDescription; - -public class ${streamName}Stream implements EventStreamDeclarer { - - @Override - public EventStream declareModel(SepDescription sep) { - - EventStream stream = new EventStream(); - - return stream; - } - - @Override - public void executeStream() { - - } - - - @Override - public boolean isExecutable() { - return false; - } - -} diff --git a/archetypes/streampipes-archetype-sources-embedded/src/main/resources/archetype-resources/src/main/webapp/WEB-INF/web.xml b/archetypes/streampipes-archetype-sources-embedded/src/main/resources/archetype-resources/src/main/webapp/WEB-INF/web.xml deleted file mode 100644 index a589fbcf2e..0000000000 --- a/archetypes/streampipes-archetype-sources-embedded/src/main/resources/archetype-resources/src/main/webapp/WEB-INF/web.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - - Archetype Created Web Application - - - org.restlet.application - - main.Init - - - - RestletServlet - - org.restlet.ext.servlet.ServerServlet - - - - RestletServlet - / - - diff --git a/archetypes/streampipes-archetype-sources-embedded/target/classes/META-INF/MANIFEST.MF b/archetypes/streampipes-archetype-sources-embedded/target/classes/META-INF/MANIFEST.MF deleted file mode 100644 index eaca22bf14..0000000000 --- a/archetypes/streampipes-archetype-sources-embedded/target/classes/META-INF/MANIFEST.MF +++ /dev/null @@ -1,5 +0,0 @@ -Manifest-Version: 1.0 -Build-Jdk: 1.7.0_101 -Built-By: philipp -Created-By: Maven Integration for Eclipse - diff --git a/archetypes/streampipes-archetype-sources-embedded/target/classes/META-INF/archetype.xml b/archetypes/streampipes-archetype-sources-embedded/target/classes/META-INF/archetype.xml deleted file mode 100644 index c1a6dd79f3..0000000000 --- a/archetypes/streampipes-archetype-sources-embedded/target/classes/META-INF/archetype.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - streampipes-archetype-storm - - src/main/java/streams/__streamName__Stream.java - src/main/java/sources/__sourceName__Producer.java - src/main/java/main/Init.java - - - src/main/resources/WEB-INF/web.xml - - - - - \ No newline at end of file diff --git a/archetypes/streampipes-archetype-sources-embedded/target/classes/META-INF/maven/archetype-metadata.xml b/archetypes/streampipes-archetype-sources-embedded/target/classes/META-INF/maven/archetype-metadata.xml deleted file mode 100644 index bed07c2f1e..0000000000 --- a/archetypes/streampipes-archetype-sources-embedded/target/classes/META-INF/maven/archetype-metadata.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - src/main/java - - **/*.java - - - - - - - - - - - \ No newline at end of file diff --git a/archetypes/streampipes-archetype-sources-embedded/target/classes/META-INF/maven/de.fzi.cep.sepa/streampipes-archetype-sources-embedded/pom.properties b/archetypes/streampipes-archetype-sources-embedded/target/classes/META-INF/maven/de.fzi.cep.sepa/streampipes-archetype-sources-embedded/pom.properties deleted file mode 100644 index 36133baa52..0000000000 --- a/archetypes/streampipes-archetype-sources-embedded/target/classes/META-INF/maven/de.fzi.cep.sepa/streampipes-archetype-sources-embedded/pom.properties +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright 2018 FZI Forschungszentrum Informatik -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -#Generated by Maven Integration for Eclipse -#Thu May 05 14:09:02 CEST 2016 -version=0.0.1-SNAPSHOT -groupId=de.fzi.cep.sepa -m2e.projectName=streampipes-archetype-sources-embedded -m2e.projectLocation=/home/philipp/Coding/fzi/icep/semantic-epa-parent/archetypes/streampipes-archetype-sources-embedded -artifactId=streampipes-archetype-sources-embedded diff --git a/archetypes/streampipes-archetype-sources-embedded/target/classes/META-INF/maven/de.fzi.cep.sepa/streampipes-archetype-sources-embedded/pom.xml b/archetypes/streampipes-archetype-sources-embedded/target/classes/META-INF/maven/de.fzi.cep.sepa/streampipes-archetype-sources-embedded/pom.xml deleted file mode 100644 index 2c32b3ced5..0000000000 --- a/archetypes/streampipes-archetype-sources-embedded/target/classes/META-INF/maven/de.fzi.cep.sepa/streampipes-archetype-sources-embedded/pom.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - 4.0.0 - de.fzi.cep.sepa - streampipes-archetype-sources-embedded - 0.0.1-SNAPSHOT - \ No newline at end of file diff --git a/archetypes/streampipes-archetype-sources-embedded/target/classes/archetype-resources/pom.xml b/archetypes/streampipes-archetype-sources-embedded/target/classes/archetype-resources/pom.xml deleted file mode 100644 index 43b25f5a16..0000000000 --- a/archetypes/streampipes-archetype-sources-embedded/target/classes/archetype-resources/pom.xml +++ /dev/null @@ -1,64 +0,0 @@ - - - - 4.0.0 - - ${groupId} - ${artifactId} - ${version} - war - - - de.fzi.cep.sepa - semantic-epa - 0.0.1-SNAPSHOT - - - - - laus - nexus repository - http://laus.fzi.de/nexus/content/repositories/public/ - - true - daily - - - true - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.1 - - 1.8 - 1.8 - UTF-8 - - - - - ${contextPath} - - \ No newline at end of file diff --git a/archetypes/streampipes-archetype-sources-embedded/target/classes/archetype-resources/src/main/java/main/Init.java b/archetypes/streampipes-archetype-sources-embedded/target/classes/archetype-resources/src/main/java/main/Init.java deleted file mode 100644 index 74ef7196c4..0000000000 --- a/archetypes/streampipes-archetype-sources-embedded/target/classes/archetype-resources/src/main/java/main/Init.java +++ /dev/null @@ -1,42 +0,0 @@ -package ${package}.main; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import de.fzi.cep.sepa.desc.EmbeddedModelSubmitter; -import de.fzi.cep.sepa.desc.declarer.SemanticEventConsumerDeclarer; -import de.fzi.cep.sepa.desc.declarer.SemanticEventProcessingAgentDeclarer; -import de.fzi.cep.sepa.desc.declarer.SemanticEventProducerDeclarer; - -import ${package}.sources.${sourceName}Producer; - -public class Init extends EmbeddedModelSubmitter { - - @Override - protected List epaDeclarers() { - return new ArrayList<>(); - } - - @Override - protected List sourceDeclarers() { - return Arrays.asList(new ${sourceName}Producer()); - } - - @Override - protected List consumerDeclarers() { - return new ArrayList<>(); - } - - @Override - protected int port() { - return ${port}; - } - - @Override - protected String contextPath() { - return "/${contextPath}"; - } - - -} diff --git a/archetypes/streampipes-archetype-sources-embedded/target/classes/archetype-resources/src/main/java/sources/__sourceName__Producer.java b/archetypes/streampipes-archetype-sources-embedded/target/classes/archetype-resources/src/main/java/sources/__sourceName__Producer.java deleted file mode 100644 index ba09b7985f..0000000000 --- a/archetypes/streampipes-archetype-sources-embedded/target/classes/archetype-resources/src/main/java/sources/__sourceName__Producer.java +++ /dev/null @@ -1,32 +0,0 @@ -package ${package}.sources; - -import java.util.ArrayList; -import java.util.List; - -import de.fzi.cep.sepa.desc.declarer.EventStreamDeclarer; -import de.fzi.cep.sepa.desc.declarer.SemanticEventProducerDeclarer; -import de.fzi.cep.sepa.model.impl.graph.SepDescription; - -import ${package}.streams.${streamName}Stream; - -public class ${sourceName}Producer implements SemanticEventProducerDeclarer { - - @Override - public SepDescription declareModel() { - - SepDescription sep = new SepDescription("source/{sourceName.toLowerCase()}", "", ""); - - return sep; - } - - - @Override - public List getEventStreams() { - - List streams = new ArrayList(); - - streams.add(new ${streamName}Stream()); - - return streams; - } -} diff --git a/archetypes/streampipes-archetype-sources-embedded/target/classes/archetype-resources/src/main/java/streams/__streamName__Stream.java b/archetypes/streampipes-archetype-sources-embedded/target/classes/archetype-resources/src/main/java/streams/__streamName__Stream.java deleted file mode 100644 index 14c961b557..0000000000 --- a/archetypes/streampipes-archetype-sources-embedded/target/classes/archetype-resources/src/main/java/streams/__streamName__Stream.java +++ /dev/null @@ -1,28 +0,0 @@ -package ${package}.streams; - -import de.fzi.cep.sepa.desc.declarer.EventStreamDeclarer; -import de.fzi.cep.sepa.model.impl.EventStream; -import de.fzi.cep.sepa.model.impl.graph.SepDescription; - -public class ${streamName}Stream implements EventStreamDeclarer { - - @Override - public EventStream declareModel(SepDescription sep) { - - EventStream stream = new EventStream(); - - return stream; - } - - @Override - public void executeStream() { - - } - - - @Override - public boolean isExecutable() { - return false; - } - -} diff --git a/archetypes/streampipes-archetype-sources-embedded/target/classes/archetype-resources/src/main/webapp/WEB-INF/web.xml b/archetypes/streampipes-archetype-sources-embedded/target/classes/archetype-resources/src/main/webapp/WEB-INF/web.xml deleted file mode 100644 index 634bc6ac41..0000000000 --- a/archetypes/streampipes-archetype-sources-embedded/target/classes/archetype-resources/src/main/webapp/WEB-INF/web.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - Archetype Created Web Application - - - org.restlet.application - - main.Init - - - - RestletServlet - - org.restlet.ext.servlet.ServerServlet - - - - RestletServlet - / - - diff --git a/pom.xml b/pom.xml index 45a93301de..778fdd3822 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ org.streampipes streampipes-parent - 0.55.2 + 0.60.0 pom UTF-8 @@ -38,6 +38,8 @@ 1.2.3 1.9.8 false + 2.18.3 + 2.0.0-beta.5 StreamPipes Parent Pom @@ -66,6 +68,11 @@ jackson-annotations ${jackson.version} + + com.google.code.gson + gson + 2.8.5 + com.orbitz.consul consul-client @@ -339,7 +346,7 @@ org.streampipes - streampipes-vocabulary + streampipes-rest-shared ${project.version} @@ -367,6 +374,11 @@ streampipes-storage-rdf4j ${project.version} + + org.streampipes + streampipes-vocabulary + ${project.version} + org.streampipes streampipes-empire-rdf4j @@ -382,50 +394,141 @@ streampipes-logging ${project.version} + + com.github.tomakehurst + wiremock + 2.18.0 + test + + + org.mockito + mockito-core + ${mockito.version} + test + + + org.powermock + powermock-module-junit4 + ${powermock.version} + test + + + org.powermock + powermock-api-mockito2 + ${powermock.version} + test + + + com.jayway.restassured + rest-assured + 2.3.2 + test + + + org.eclipse.jetty + jetty-server + 9.2.24.v20180105 + + + org.eclipse.jetty + jetty-util + 9.2.24.v20180105 + + + org.eclipse.jetty + jetty-continuation + 9.2.24.v20180105 + + + + junit + junit + ${junit.version} + test + + + com.github.tomakehurst + wiremock + 2.18.0 + test + + + org.mockito + mockito-core + ${mockito.version} + test + + + com.jayway.restassured + rest-assured + 2.3.2 + test + + + org.powermock + powermock-module-junit4 + ${powermock.version} + test + + + org.powermock + powermock-api-mockito2 + ${powermock.version} + test + + + + archetypes/streampipes-archetype-pe-sources + archetypes/streampipes-archetype-pe-sinks-jvm + archetypes/streampipes-archetype-pe-processors-jvm + archetypes/streampipes-archetype-pe-sinks-flink + archetypes/streampipes-archetype-pe-processors-flink + streampipes-app-file-export streampipes-backend streampipes-code-generation streampipes-commons streampipes-config streampipes-connect + streampipes-connect-container streampipes-container streampipes-container-embedded streampipes-container-standalone streampipes-dataformat streampipes-dataformat-json - streampipes-pipeline-management + streampipes-logging streampipes-measurement-units streampipes-messaging-kafka streampipes-messaging streampipes-messaging-jms streampipes-model streampipes-model-client + streampipes-performance-tests + streampipes-pipeline-management streampipes-sources streampipes-rest + streampipes-rest-shared streampipes-sdk + streampipes-serializers streampipes-storage-api + streampipes-storage-management + streampipes-storage-couchdb + streampipes-storage-rdf4j + streampipes-test-utils streampipes-user-management + streampipes-vocabulary streampipes-wrapper + streampipes-wrapper-distributed streampipes-wrapper-esper streampipes-wrapper-flink + streampipes-wrapper-kafka-streams + streampipes-wrapper-siddhi streampipes-wrapper-spark streampipes-wrapper-standalone - streampipes-performance-tests - streampipes-storage-rdf4j - streampipes-storage-couchdb - streampipes-app-file-export - streampipes-vocabulary - streampipes-serializers - streampipes-wrapper-kafka-streams - streampipes-wrapper-distributed - streampipes-storage-management - streampipes-logging - archetypes/streampipes-archetype-pe-sources - archetypes/streampipes-archetype-pe-jvm @@ -483,7 +586,6 @@ feature- release- hotfix- - version- false true @@ -566,6 +668,19 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.1.0 + + + + true + true + + + + diff --git a/streampipes-app-file-export/pom.xml b/streampipes-app-file-export/pom.xml index 0a8bb1e4bf..7903d68e98 100644 --- a/streampipes-app-file-export/pom.xml +++ b/streampipes-app-file-export/pom.xml @@ -21,7 +21,7 @@ streampipes-parent org.streampipes - 0.55.2 + 0.60.0 StreamPipes App File Export streampipes-app-file-export diff --git a/streampipes-app-file-export/src/main/java/org/streampipes/app/file/export/ElasticsearchAppData.java b/streampipes-app-file-export/src/main/java/org/streampipes/app/file/export/ElasticsearchAppData.java index 4fcb9fcc92..78dab37e46 100644 --- a/streampipes-app-file-export/src/main/java/org/streampipes/app/file/export/ElasticsearchAppData.java +++ b/streampipes-app-file-export/src/main/java/org/streampipes/app/file/export/ElasticsearchAppData.java @@ -33,7 +33,7 @@ public ElasticsearchAppData() { public ElasticsearchAppData(String index, long timestampFrom, long timeStampTo, String output) { this.index = index; this.timestampFrom = timestampFrom; - this.timestampTo = timestampTo; + this.timestampTo = timeStampTo; this.output = output; } diff --git a/streampipes-app-file-export/src/main/java/org/streampipes/app/file/export/impl/Elasticsearch.java b/streampipes-app-file-export/src/main/java/org/streampipes/app/file/export/impl/Elasticsearch.java index e0b7c657bf..bf1b9c613c 100644 --- a/streampipes-app-file-export/src/main/java/org/streampipes/app/file/export/impl/Elasticsearch.java +++ b/streampipes-app-file-export/src/main/java/org/streampipes/app/file/export/impl/Elasticsearch.java @@ -35,18 +35,29 @@ import org.streampipes.app.file.export.model.IndexInfo; import org.streampipes.storage.couchdb.utils.Utils; -import javax.ws.rs.*; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; import java.io.File; import java.io.FileWriter; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; @Path("/v1/elasticsearch") public class Elasticsearch implements IElasticsearch { - static String mainFilePath = ElasticsearchConfig.INSTANCE.getDataLocation(); + private static String mainFilePath = ElasticsearchConfig.INSTANCE.getDataLocation(); private static final List excludedIndices = Collections.singletonList(".kibana"); Logger LOG = LoggerFactory.getLogger(Elasticsearch.class); @@ -71,7 +82,7 @@ public Response createFiles(ElasticsearchAppData data) { .asJson(); String response = jsonResponse.getBody().getObject().toString(); - if (output.equals("csv")) { + if (("csv").equals(output)) { response = new JsonConverter(response).convertToCsv(); } else { response = new JsonConverter(response).convertToJson(); @@ -98,11 +109,7 @@ public Response createFiles(ElasticsearchAppData data) { return Response.ok().build(); - } catch (IOException e) { - e.printStackTrace(); - LOG.error(e.getMessage()); - return Response.status(500).entity(e).build(); - } catch (UnirestException e) { + } catch (IOException | UnirestException e) { e.printStackTrace(); LOG.error(e.getMessage()); return Response.status(500).entity(e).build(); diff --git a/streampipes-backend/Dockerfile b/streampipes-backend/Dockerfile index 4ebc0b07d5..3cbdcba081 100644 --- a/streampipes-backend/Dockerfile +++ b/streampipes-backend/Dockerfile @@ -1,5 +1,5 @@ #FROM tomcat:8-jre8 -FROM anapsix/alpine-java +FROM anapsix/alpine-java:8 @@ -47,12 +47,12 @@ RUN set -x \ EXPOSE 8080 -ADD target/streampipes-backend.war /usr/local/tomcat/webapps/streampipes-backend.war -ADD ./deployment-config/rdf4j-server.war /usr/local/tomcat/webapps/rdf4j-server.war -ADD ./deployment-config/rdf4j-workbench.war /usr/local/tomcat/webapps/rdf4j-workbench.war +COPY target/streampipes-backend.war /usr/local/tomcat/webapps/streampipes-backend.war +COPY ./deployment-config/rdf4j-server.war /usr/local/tomcat/webapps/rdf4j-server.war +COPY ./deployment-config/rdf4j-workbench.war /usr/local/tomcat/webapps/rdf4j-workbench.war #ADD ./org.streampipes/streampipes-pe-slack/streampipes-pe-slack.war /usr/local/tomcat/webapps/slack.war -ADD ./deployment-config/catalina.properties /usr/local/tomcat/conf/catalina.properties -ADD ./deployment-config/server.xml /usr/local/tomcat/conf/server.xml +COPY ./deployment-config/catalina.properties /usr/local/tomcat/conf/catalina.properties +COPY ./deployment-config/server.xml /usr/local/tomcat/conf/server.xml CMD ["catalina.sh", "run"] diff --git a/streampipes-backend/pom.xml b/streampipes-backend/pom.xml index 15066dd94d..f1cd415a32 100644 --- a/streampipes-backend/pom.xml +++ b/streampipes-backend/pom.xml @@ -3,7 +3,7 @@ org.streampipes streampipes-parent - 0.55.2 + 0.60.0 streampipes-backend war @@ -55,6 +55,12 @@ 2.4 src\main\webapp\WEB-INF\web.xml + + + true + true + + diff --git a/streampipes-backend/src/main/webapp/WEB-INF/shiro.ini b/streampipes-backend/src/main/webapp/WEB-INF/shiro.ini index 8472faad30..8607a13805 100644 --- a/streampipes-backend/src/main/webapp/WEB-INF/shiro.ini +++ b/streampipes-backend/src/main/webapp/WEB-INF/shiro.ini @@ -32,6 +32,10 @@ securityManager.rememberMeManager.cookie.maxAge = 1000000000 /api/v2/setup/install/* = anon /api/v2/admin/configuration = anon /api/v2/admin/authc = anon +/api/v2/pipelines/*/* = anon +/api/v2/noauth/users/*/* = anon +/api/v2/noauth/users/*/element/* = anon +/api/v2/noauth/element/* = anon /api/v2/kpis = anon /api/v2/kpis/* = anon /api/v2/admin/login/* = anon diff --git a/streampipes-code-generation/pom.xml b/streampipes-code-generation/pom.xml index d319ba3066..32f522ffcb 100644 --- a/streampipes-code-generation/pom.xml +++ b/streampipes-code-generation/pom.xml @@ -20,7 +20,7 @@ org.streampipes streampipes-parent - 0.55.2 + 0.60.0 streampipes-code-generation diff --git a/streampipes-code-generation/src/main/java/org/streampipes/codegeneration/ZipFileGenerator.java b/streampipes-code-generation/src/main/java/org/streampipes/codegeneration/ZipFileGenerator.java index a635afd8c0..322fec3854 100644 --- a/streampipes-code-generation/src/main/java/org/streampipes/codegeneration/ZipFileGenerator.java +++ b/streampipes-code-generation/src/main/java/org/streampipes/codegeneration/ZipFileGenerator.java @@ -34,7 +34,7 @@ public class ZipFileGenerator { private File outputFile; public ZipFileGenerator(File inputDirectory, File outputFile) { - this.fileList = new ArrayList(); + this.fileList = new ArrayList<>(); this.inputDirectory = inputDirectory; this.outputFile = outputFile; generateFileList(inputDirectory); diff --git a/streampipes-code-generation/src/main/java/org/streampipes/codegeneration/flink/InitGenerator.java b/streampipes-code-generation/src/main/java/org/streampipes/codegeneration/flink/InitGenerator.java index 52f961379f..0ab84ed35d 100644 --- a/streampipes-code-generation/src/main/java/org/streampipes/codegeneration/flink/InitGenerator.java +++ b/streampipes-code-generation/src/main/java/org/streampipes/codegeneration/flink/InitGenerator.java @@ -40,7 +40,7 @@ public InitGenerator(ConsumableStreamPipesEntity sepa, String name, String packa @Override public JavaFile build() { - List controllers = new ArrayList(); + List controllers = new ArrayList<>(); controllers.add(ClassName.get("", name + "Controller")); TypeSpec controllerClass = TypeSpec.classBuilder(name + "Init").addModifiers(Modifier.PUBLIC) diff --git a/streampipes-code-generation/src/main/java/org/streampipes/codegeneration/flink/sepa/FlinkSepaControllerGenerator.java b/streampipes-code-generation/src/main/java/org/streampipes/codegeneration/flink/sepa/FlinkSepaControllerGenerator.java index 27b3cc7b54..74970d1d2d 100644 --- a/streampipes-code-generation/src/main/java/org/streampipes/codegeneration/flink/sepa/FlinkSepaControllerGenerator.java +++ b/streampipes-code-generation/src/main/java/org/streampipes/codegeneration/flink/sepa/FlinkSepaControllerGenerator.java @@ -17,22 +17,21 @@ package org.streampipes.codegeneration.flink.sepa; -import javax.lang.model.element.Modifier; - import com.squareup.javapoet.ClassName; import com.squareup.javapoet.JavaFile; import com.squareup.javapoet.MethodSpec; import com.squareup.javapoet.MethodSpec.Builder; import com.squareup.javapoet.ParameterizedTypeName; import com.squareup.javapoet.TypeSpec; - +import org.streampipes.codegeneration.ControllerGenerator; +import org.streampipes.codegeneration.utils.JFC; import org.streampipes.model.base.ConsumableStreamPipesEntity; import org.streampipes.model.graph.DataProcessorInvocation; import org.streampipes.model.staticproperty.FreeTextStaticProperty; import org.streampipes.model.staticproperty.MappingProperty; import org.streampipes.model.staticproperty.StaticProperty; -import org.streampipes.codegeneration.ControllerGenerator; -import org.streampipes.codegeneration.utils.JFC; + +import javax.lang.model.element.Modifier; public class FlinkSepaControllerGenerator extends ControllerGenerator { @@ -73,11 +72,13 @@ public Builder getRuntime() { private Builder getStaticProperty(Builder b, StaticProperty sp) { String name = sp.getInternalName().replaceAll("-", "_").replaceAll("/", "_"); if (sp instanceof MappingProperty) { - b.addStatement("String $L = $T.getMappingPropertyName(graph, $S)", name, JFC.SEPA_UTILS, - sp.getInternalName()); + // TODO use SDK +// b.addStatement("String $L = $T.getMappingPropertyName(graph, $S)", name, JFC.SEPA_UTILS, +// sp.getInternalName()); } else if (sp instanceof FreeTextStaticProperty) { - b.addStatement("String $L = $T.getFreeTextStaticPropertyValue(graph, $S)", name, JFC.SEPA_UTILS, - sp.getInternalName()); + // TODO use SDK +// b.addStatement("String $L = $T.getFreeTextStaticPropertyValue(graph, $S)", name, JFC.SEPA_UTILS, +// sp.getInternalName()); } else { // TODO add implementation for the other strategies try { diff --git a/streampipes-code-generation/src/main/java/org/streampipes/codegeneration/utils/JFC.java b/streampipes-code-generation/src/main/java/org/streampipes/codegeneration/utils/JFC.java index cba0369763..327526b9c0 100644 --- a/streampipes-code-generation/src/main/java/org/streampipes/codegeneration/utils/JFC.java +++ b/streampipes-code-generation/src/main/java/org/streampipes/codegeneration/utils/JFC.java @@ -28,7 +28,6 @@ import org.streampipes.model.output.AppendOutputStrategy; import org.streampipes.model.output.OutputStrategy; import org.streampipes.model.schema.EventProperty; -import org.streampipes.model.util.SepaUtils; import org.streampipes.sdk.PrimitivePropertyBuilder; import org.streampipes.sdk.helpers.EpProperties; import org.streampipes.sdk.stream.SchemaBuilder; @@ -69,7 +68,6 @@ public abstract class JFC { public static ClassName SEC_DESCRIPTION = ClassName.get(DataSinkDescription.class); public static ClassName SEPA_INVOCATION = ClassName.get(DataProcessorInvocation.class); public static ClassName SEC_INVOCATION = ClassName.get(DataSinkInvocation.class); - public static ClassName SEPA_UTILS = ClassName.get(SepaUtils.class); public static ClassName EVENT_STREAM = ClassName.get(SpDataStream.class); public static ClassName STREAM_BUILDER = ClassName.get(StreamBuilder.class); public static ClassName SCHEMA_BUILDER = ClassName.get(SchemaBuilder.class); diff --git a/streampipes-commons/pom.xml b/streampipes-commons/pom.xml index 43ba2afdd4..f4e3579c82 100644 --- a/streampipes-commons/pom.xml +++ b/streampipes-commons/pom.xml @@ -20,7 +20,7 @@ org.streampipes streampipes-parent - 0.55.2 + 0.60.0 streampipes-commons StreamPipes Commons @@ -38,7 +38,6 @@ com.google.code.gson gson - 2.3.1 org.streampipes diff --git a/streampipes-commons/src/main/java/org/streampipes/commons/Utils.java b/streampipes-commons/src/main/java/org/streampipes/commons/Utils.java index d3601c1b65..4673641051 100644 --- a/streampipes-commons/src/main/java/org/streampipes/commons/Utils.java +++ b/streampipes-commons/src/main/java/org/streampipes/commons/Utils.java @@ -65,7 +65,7 @@ public static String asString(Graph graph) throws RDFHandlerException { } - public static RDFWriter getRioWriter(OutputStream stream) throws RDFHandlerException { + private static RDFWriter getRioWriter(OutputStream stream) throws RDFHandlerException { RDFWriter writer = Rio.createWriter(RDFFormat.JSONLD, stream); writer.handleNamespace("sp", "https://streampipes.org/vocabulary/v1/"); diff --git a/streampipes-config/pom.xml b/streampipes-config/pom.xml index 98b1f1ced3..6d91f808d6 100644 --- a/streampipes-config/pom.xml +++ b/streampipes-config/pom.xml @@ -20,7 +20,7 @@ streampipes-parent org.streampipes - 0.55.2 + 0.60.0 4.0.0 @@ -31,6 +31,10 @@ com.orbitz.consul consul-client + + com.google.code.gson + gson + \ No newline at end of file diff --git a/streampipes-config/src/main/java/org/streampipes/config/SpConfig.java b/streampipes-config/src/main/java/org/streampipes/config/SpConfig.java index 8d201b3899..a5f5687d5d 100644 --- a/streampipes-config/src/main/java/org/streampipes/config/SpConfig.java +++ b/streampipes-config/src/main/java/org/streampipes/config/SpConfig.java @@ -18,6 +18,8 @@ package org.streampipes.config; import org.streampipes.config.consul.ConsulSpConfig; +import org.streampipes.config.model.ConfigItem; +import org.streampipes.config.model.ConfigurationScope; public abstract class SpConfig { @@ -39,6 +41,8 @@ public static SpConfig getSpConfig(String serviceName, SpConfigChangeCallback ca return new ConsulSpConfig(serviceName, callback); } + public abstract void register(String key, T defaultValue, String description, ConfigurationScope configurationScope); + public abstract void register(String key, boolean defaultValue, String description); public abstract void register(String key, int defaultValue, String description); @@ -57,6 +61,8 @@ public static SpConfig getSpConfig(String serviceName, SpConfigChangeCallback ca public abstract String getString(String key); + public abstract ConfigItem getConfigItem(String key); + public abstract void setBoolean(String key, Boolean value); public abstract void setInteger(String key, int value); diff --git a/streampipes-config/src/main/java/org/streampipes/config/SpConfigChangeCallback.java b/streampipes-config/src/main/java/org/streampipes/config/SpConfigChangeCallback.java index b9dfb07891..3d2dbfaa3a 100644 --- a/streampipes-config/src/main/java/org/streampipes/config/SpConfigChangeCallback.java +++ b/streampipes-config/src/main/java/org/streampipes/config/SpConfigChangeCallback.java @@ -19,5 +19,5 @@ public interface SpConfigChangeCallback { - public void onChange(); + void onChange(); } diff --git a/streampipes-config/src/main/java/org/streampipes/config/backend/BackendConfig.java b/streampipes-config/src/main/java/org/streampipes/config/backend/BackendConfig.java index 58cc4e47f2..7825700ef8 100644 --- a/streampipes-config/src/main/java/org/streampipes/config/backend/BackendConfig.java +++ b/streampipes-config/src/main/java/org/streampipes/config/backend/BackendConfig.java @@ -24,65 +24,55 @@ public enum BackendConfig { INSTANCE; private SpConfig config; - private final static String BACKEND_HOST = "backend_host"; - private final static String BACKEND_PORT = "backend_port"; - private final static String JMS_HOST = "jms_host"; - private final static String JMS_PORT = "jms_port"; - private final static String KAFKA_HOST = "kafka_host"; - private final static String KAFKA_PORT = "kafka_port"; - private final static String ZOOKEEPER_HOST = "zookeeper_host"; - private final static String ZOOKEEPER_PORT = "zookeeper_port"; - private final static String ELASTICSEARCH_HOST ="elasticsearch_host"; - private final static String ELASTICSEARCH_PORT ="elasticsearch_port"; - private final static String ELASTICSEARCH_PROTOCOL = "elasticsearch_protocol"; - private final static String IS_CONFIGURED = "is_configured"; - private final static String KAFKA_REST_HOST = "kafka_rest_host"; - private final static String KAFKA_REST_PORT = "kafka_rest_port"; BackendConfig() { config = SpConfig.getSpConfig("backend"); - config.register(BACKEND_HOST, "backend", "Hostname for backend"); - config.register(BACKEND_PORT, 8082, "Port for backend"); - - config.register(JMS_HOST, "activemq", "Hostname for backend service for active mq"); - config.register(JMS_PORT, 61616, "Port for backend service for active mq"); - config.register(KAFKA_HOST, "kafka", "Hostname for backend service for kafka"); - config.register(KAFKA_PORT, 9092, "Port for backend service for kafka"); - config.register(ZOOKEEPER_HOST, "zookeeper", "Hostname for backend service for zookeeper"); - config.register(ZOOKEEPER_PORT, 2181, "Port for backend service for zookeeper"); - config.register(ELASTICSEARCH_HOST, "elasticsearch", "Hostname for elasticsearch service"); - config.register(ELASTICSEARCH_PORT, 9200, "Port for elasticsearch service"); - config.register(ELASTICSEARCH_PROTOCOL, "http", "Protocol the elasticsearch service"); - config.register(IS_CONFIGURED, false, "Boolean that indicates whether streampipes is " + + config.register(BackendConfigKeys.SERVICE_NAME, "Backend", "Backend Configuration"); + + config.register(BackendConfigKeys.BACKEND_HOST, "backend", "Hostname for backend"); + config.register(BackendConfigKeys.BACKEND_PORT, 8082, "Port for backend"); + + config.register(BackendConfigKeys.JMS_HOST, "activemq", "Hostname for backend service for active mq"); + config.register(BackendConfigKeys.JMS_PORT, 61616, "Port for backend service for active mq"); + config.register(BackendConfigKeys.KAFKA_HOST, "kafka", "Hostname for backend service for kafka"); + config.register(BackendConfigKeys.KAFKA_PORT, 9092, "Port for backend service for kafka"); + config.register(BackendConfigKeys.ZOOKEEPER_HOST, "zookeeper", "Hostname for backend service for zookeeper"); + config.register(BackendConfigKeys.ZOOKEEPER_PORT, 2181, "Port for backend service for zookeeper"); + config.register(BackendConfigKeys.ELASTICSEARCH_HOST, "elasticsearch", "Hostname for elasticsearch service"); + config.register(BackendConfigKeys.ELASTICSEARCH_PORT, 9200, "Port for elasticsearch service"); + config.register(BackendConfigKeys.ELASTICSEARCH_PROTOCOL, "http", "Protocol the elasticsearch service"); + config.register(BackendConfigKeys.IS_CONFIGURED, false, "Boolean that indicates whether streampipes is " + "already configured or not"); - config.register(KAFKA_REST_HOST, "kafka-rest", "The hostname of the kafka-rest module"); - config.register(KAFKA_REST_PORT, 8073, "The port of the kafka-rest module"); + config.register(BackendConfigKeys.KAFKA_REST_HOST, "kafka-rest", "The hostname of the kafka-rest module"); + config.register(BackendConfigKeys.KAFKA_REST_PORT, 8082, "The port of the kafka-rest module"); + config.register(BackendConfigKeys.KAFKA_REST_HOST, "kafka-rest", "The hostname of the kafka-rest module"); + } public String getBackendHost() { - return config.getString(BACKEND_HOST); + return config.getString(BackendConfigKeys.BACKEND_HOST); } public int getBackendPort() { - return config.getInteger(BACKEND_PORT); + return config.getInteger(BackendConfigKeys.BACKEND_PORT); } public String getJmsHost() { - return config.getString(JMS_HOST); + return config.getString(BackendConfigKeys.JMS_HOST); } public int getJmsPort() { - return config.getInteger(JMS_PORT); + return config.getInteger(BackendConfigKeys.JMS_PORT); } public String getKafkaHost() { - return config.getString(KAFKA_HOST); + return config.getString(BackendConfigKeys.KAFKA_HOST); } public int getKafkaPort() { - return config.getInteger(KAFKA_PORT); + return config.getInteger(BackendConfigKeys.KAFKA_PORT); } public String getKafkaUrl() { @@ -90,43 +80,43 @@ public String getKafkaUrl() { } public String getZookeeperHost() { - return config.getString(ZOOKEEPER_HOST); + return config.getString(BackendConfigKeys.ZOOKEEPER_HOST); } public int getZookeeperPort() { - return config.getInteger(ZOOKEEPER_PORT); + return config.getInteger(BackendConfigKeys.ZOOKEEPER_PORT); } public boolean isConfigured() { - return config.getBoolean(IS_CONFIGURED); + return config.getBoolean(BackendConfigKeys.IS_CONFIGURED); } public void setKafkaHost(String s) { - config.setString(KAFKA_HOST, s); + config.setString(BackendConfigKeys.KAFKA_HOST, s); } public void setZookeeperHost(String s) { - config.setString(ZOOKEEPER_HOST, s); + config.setString(BackendConfigKeys.ZOOKEEPER_HOST, s); } public void setJmsHost(String s) { - config.setString(JMS_HOST, s); + config.setString(BackendConfigKeys.JMS_HOST, s); } public void setIsConfigured(boolean b) { - config.setBoolean(IS_CONFIGURED, b); + config.setBoolean(BackendConfigKeys.IS_CONFIGURED, b); } public String getElasticsearchHost() { - return config.getString(ELASTICSEARCH_HOST); + return config.getString(BackendConfigKeys.ELASTICSEARCH_HOST); } public int getElasticsearchPort() { - return config.getInteger(ELASTICSEARCH_PORT); + return config.getInteger(BackendConfigKeys.ELASTICSEARCH_PORT); } public String getElasticsearchProtocol() { - return config.getString(ELASTICSEARCH_PROTOCOL); + return config.getString(BackendConfigKeys.ELASTICSEARCH_PROTOCOL); } public String getElasticsearchURL() { @@ -134,11 +124,11 @@ public String getElasticsearchURL() { } public String getKafkaRestHost() { - return config.getString(KAFKA_REST_HOST); + return config.getString(BackendConfigKeys.KAFKA_REST_HOST); } public Integer getKafkaRestPort() { - return config.getInteger(KAFKA_REST_PORT); + return config.getInteger(BackendConfigKeys.KAFKA_REST_PORT); } public String getKafkaRestUrl() { diff --git a/streampipes-config/src/main/java/org/streampipes/config/backend/BackendConfigKeys.java b/streampipes-config/src/main/java/org/streampipes/config/backend/BackendConfigKeys.java new file mode 100644 index 0000000000..0dab4f7a46 --- /dev/null +++ b/streampipes-config/src/main/java/org/streampipes/config/backend/BackendConfigKeys.java @@ -0,0 +1,37 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.config.backend; + +public class BackendConfigKeys { + final static String BACKEND_HOST = "SP_BACKEND_HOST"; + final static String BACKEND_PORT = "SP_BACKEND_PORT"; + final static String JMS_HOST = "SP_JMS_HOST"; + final static String JMS_PORT = "SP_JMS_PORT"; + final static String KAFKA_HOST = "SP_KAFKA_HOST"; + final static String KAFKA_PORT = "SP_KAFKA_PORT"; + final static String ZOOKEEPER_HOST = "SP_ZOOKEEPER_HOST"; + final static String ZOOKEEPER_PORT = "SP_ZOOKEEPER_PORT"; + final static String ELASTICSEARCH_HOST = "SP_ELASTICSEARCH_HOST"; + final static String ELASTICSEARCH_PORT = "SP_ELASTICSEARCH_PORT"; + final static String ELASTICSEARCH_PROTOCOL = "SP_ELASTICSEARCH_PROTOCOL"; + final static String IS_CONFIGURED = "SP_IS_CONFIGURED"; + final static String KAFKA_REST_HOST = "SP_KAFKA_REST_HOST"; + final static String KAFKA_REST_PORT = "SP_KAFKA_REST_PORT"; + + final static String SERVICE_NAME = "SP_SERVICE_NAME"; +} diff --git a/streampipes-config/src/main/java/org/streampipes/config/consul/ConsulSpConfig.java b/streampipes-config/src/main/java/org/streampipes/config/consul/ConsulSpConfig.java index 5dff0e1632..02ad1c9f37 100644 --- a/streampipes-config/src/main/java/org/streampipes/config/consul/ConsulSpConfig.java +++ b/streampipes-config/src/main/java/org/streampipes/config/consul/ConsulSpConfig.java @@ -17,11 +17,14 @@ package org.streampipes.config.consul; +import com.google.gson.Gson; import com.orbitz.consul.Consul; import com.orbitz.consul.KeyValueClient; import com.orbitz.consul.model.kv.Value; import org.streampipes.config.SpConfig; import org.streampipes.config.SpConfigChangeCallback; +import org.streampipes.config.model.ConfigItem; +import org.streampipes.config.model.ConfigurationScope; import java.net.MalformedURLException; import java.net.URL; @@ -30,16 +33,14 @@ import java.util.Optional; public class ConsulSpConfig extends SpConfig implements Runnable { -// final static Logger logger = LogUtil.getInstance(ConsulSpConfig.class); -//public class ConsulSpConfig extends SpConfig { private static final String CONSUL_ENV_LOCATION = "CONSUL_LOCATION"; public static final String SERVICE_ROUTE_PREFIX = "sp/v1/"; private String serviceName; private KeyValueClient kvClient; - // TODO Implement mechanism to update the client when some configutation parameters change in Consul + // TODO Implement mechanism to update the client when some configuration parameters change in Consul private SpConfigChangeCallback callback; private Map configProps; @@ -79,7 +80,7 @@ public void run() { KeyValueClient kvClientThread = consulThread.keyValueClient(); while (true) { - configProps.keySet().stream().forEach((s) -> { + configProps.keySet().forEach((s) -> { Optional te = kvClientThread.getValue(addSn(s)); if (!te.get().getValueAsString().get().equals(configProps.get(s))) { callback.onChange(); @@ -95,49 +96,64 @@ public void run() { } } + @Override + public void register(String key, T defaultValue, String description, ConfigurationScope configurationScope) { + register(key, String.valueOf(defaultValue), getValueType(defaultValue), description, configurationScope, false); + } + + private String getValueType(T defaultValue) { + if (defaultValue instanceof Boolean) { + return "xs:boolean"; + } else if (defaultValue instanceof Integer) { + return "xs:integer"; + } else if (defaultValue instanceof Double) { + return "xs:double"; + } else { + return "xs:string"; + } + } + @Override public void register(String key, boolean defaultValue, String description) { - register(key, Boolean.toString(defaultValue), "xs:boolean", description, false); + register(key, Boolean.toString(defaultValue), "xs:boolean", description, ConfigurationScope.CONTAINER_STARTUP_CONFIG, false); } @Override public void register(String key, int defaultValue, String description) { - register(key, Integer.toString(defaultValue), "xs:integer", description, false); + register(key, Integer.toString(defaultValue), "xs:integer", description, ConfigurationScope.CONTAINER_STARTUP_CONFIG, false); } @Override public void register(String key, double defaultValue, String description) { - register(key, Double.toString(defaultValue), "xs:double", description, false); + register(key, Double.toString(defaultValue), "xs:double", description, ConfigurationScope.CONTAINER_STARTUP_CONFIG, false); } @Override public void register(String key, String defaultValue, String description) { - register(key, defaultValue, "xs:string", description, false); + register(key, defaultValue, "xs:string", description, ConfigurationScope.CONTAINER_STARTUP_CONFIG, false); } @Override public void registerPassword(String key, String defaultValue, String description) { - register(key, defaultValue, "xs:string", description, true); + register(key, defaultValue, "xs:string", description, ConfigurationScope.CONTAINER_STARTUP_CONFIG, true); } - private void register(String key, String defaultValue, String valueType, String description, boolean isPassword) { + private void register(String key, String defaultValue, String valueType, String description, ConfigurationScope configurationScope, boolean isPassword) { Optional i = kvClient.getValueAsString(addSn(key)); + ConfigItem configItem = prepareConfigItem(valueType, description, configurationScope, isPassword); // TODO this check does not work if (!i.isPresent()) { // Set the value of environment variable as default String envVariable = System.getenv(key); if (envVariable != null) { - kvClient.putValue(addSn(key), envVariable); + configItem.setValue(envVariable); + kvClient.putValue(addSn(key), toJson(configItem)); } else { - kvClient.putValue(addSn(key), defaultValue); + configItem.setValue(defaultValue); + kvClient.putValue(addSn(key), toJson(configItem)); } - - kvClient.putValue(addSn(key) + "_description", description); - kvClient.putValue(addSn(key) + "_type", valueType); - if(isPassword) - kvClient.putValue(addSn(key) + "_isPassword", "true"); } if (configProps != null) { @@ -162,12 +178,14 @@ public double getDouble(String key) { @Override public String getString(String key) { - Optional os = kvClient.getValueAsString(addSn(key)); + return getConfigItem(key).getValue(); + } - String s = os.get(); - return s; + @Override + public ConfigItem getConfigItem(String key) { + Optional os = kvClient.getValueAsString(addSn(key)); -// return kvClient.getValueAsString(addSn(key)).get(); + return fromJson(os.get()); } @Override @@ -194,5 +212,28 @@ private String addSn(String key) { return SERVICE_ROUTE_PREFIX + serviceName + "/" + key; } + private ConfigItem fromJson(String content) { + try { + return new Gson().fromJson(content, ConfigItem.class); + } catch (Exception e) { + // if old config is used, this is a fallback + ConfigItem configItem = new ConfigItem(); + configItem.setValue(content); + return configItem; + } + } + + private ConfigItem prepareConfigItem(String valueType, String description, ConfigurationScope configurationScope, boolean password) { + ConfigItem configItem = new ConfigItem(); + configItem.setValueType(valueType); + configItem.setDescription(description); + configItem.setPassword(password); + configItem.setConfigurationScope(configurationScope); + + return configItem; + } + private String toJson(ConfigItem configItem) { + return new Gson().toJson(configItem); + } } diff --git a/streampipes-config/src/main/java/org/streampipes/config/model/ConfigItem.java b/streampipes-config/src/main/java/org/streampipes/config/model/ConfigItem.java index 875ccf0a23..7faf1f4d70 100644 --- a/streampipes-config/src/main/java/org/streampipes/config/model/ConfigItem.java +++ b/streampipes-config/src/main/java/org/streampipes/config/model/ConfigItem.java @@ -23,6 +23,7 @@ public class ConfigItem { private String description; private String value; private String valueType; + private ConfigurationScope configurationScope; private boolean isPassword; public ConfigItem() { @@ -68,4 +69,12 @@ public boolean isPassword() { public void setPassword(boolean state) { isPassword = state; } + + public ConfigurationScope getConfigurationScope() { + return configurationScope; + } + + public void setConfigurationScope(ConfigurationScope configurationScope) { + this.configurationScope = configurationScope; + } } diff --git a/streampipes-config/src/main/java/org/streampipes/config/model/ConfigurationScope.java b/streampipes-config/src/main/java/org/streampipes/config/model/ConfigurationScope.java new file mode 100644 index 0000000000..1f54543aa8 --- /dev/null +++ b/streampipes-config/src/main/java/org/streampipes/config/model/ConfigurationScope.java @@ -0,0 +1,23 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.streampipes.config.model; + +public enum ConfigurationScope { + CONTAINER_STARTUP_CONFIG, + CONTAINER_GLOBAL_CONFIG, + PIPELINE_ELEMENT_CONFIG +} diff --git a/streampipes-connect-container/Dockerfile b/streampipes-connect-container/Dockerfile new file mode 100644 index 0000000000..30209b27ef --- /dev/null +++ b/streampipes-connect-container/Dockerfile @@ -0,0 +1,9 @@ +FROM anapsix/alpine-java:8 + +ENV CONSUL_LOCATION consul + +EXPOSE 8099 + +COPY target/streampipes-connect-container.jar /streampipes-connect-container.jar + +ENTRYPOINT ["java", "-jar", "/streampipes-connect-container.jar"] diff --git a/streampipes-connect-container/development/.env b/streampipes-connect-container/development/.env new file mode 100644 index 0000000000..6a1c555ee8 --- /dev/null +++ b/streampipes-connect-container/development/.env @@ -0,0 +1,5 @@ +# Those parameters are used by IntelliJ to set the default consul parameters for development +SP_BACKEND_HOST=localhost +SP_KAFKA_HOST=localhost +SP_CONNECT_CONTAINER_HOST=localhost +SP_CONNECT_CONTAINER_WORKER_HOST=localhost diff --git a/streampipes-connect-container/pom.xml b/streampipes-connect-container/pom.xml new file mode 100644 index 0000000000..36bb440dd4 --- /dev/null +++ b/streampipes-connect-container/pom.xml @@ -0,0 +1,112 @@ + + + + streampipes-parent + org.streampipes + 0.60.0 + + 4.0.0 + + streampipes-connect-container + + + 2.22.2 + + + + + org.streampipes + streampipes-serializers + + + org.streampipes + streampipes-container + + + org.streampipes + streampipes-config + + + org.streampipes + streampipes-connect + + + org.streampipes + streampipes-commons + + + org.streampipes + streampipes-storage-couchdb + + + org.streampipes + streampipes-rest-shared + + + org.glassfish.jersey.containers + jersey-container-jetty-http + ${jersey.version} + + + + org.rendersnake + rendersnake + 1.8 + + + io.rest-assured + json-path + 3.1.0 + + + io.rest-assured + json-path + 3.1.0 + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.3 + + + package + + shade + + + false + + + org.streampipes.connect.init.Main + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + META-INF/maven/com.github.jsonld-java/jsonld-java/pom.xml + META-INF/maven/com.github.jsonld-java/jsonld-java-sesame/pom.xml + + + + + + + + + + streampipes-connect-container + + + \ No newline at end of file diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/config/ConfigKeys.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/config/ConfigKeys.java new file mode 100644 index 0000000000..2badeab7e7 --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/config/ConfigKeys.java @@ -0,0 +1,32 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.config; + +public class ConfigKeys { + final static String BACKEND_HOST = "SP_BACKEND_HOST"; + final static String BACKEND_PORT = "SP_BACKEND_PORT"; + + final static String KAFKA_HOST = "SP_KAFKA_HOST"; + final static String KAFKA_PORT = "SP_KAFKA_PORT"; + + final static String CONNECT_CONTAINER_MASTER_HOST = "SP_CONNECT_CONTAINER_HOST"; + final static String CONNECT_CONTAINER_MASTER_PORT = "SP_CONNECT_CONTAINER_PORT"; + + final static String CONNECT_CONTAINER_WORKER_HOST = "SP_CONNECT_CONTAINER_WORKER_HOST"; + final static String CONNECT_CONTAINER_WORKER_PORT = "SP_CONNECT_CONTAINER_WORKER_PORT"; +} diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/config/ConnectContainerConfig.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/config/ConnectContainerConfig.java new file mode 100644 index 0000000000..cc5f91b6a5 --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/config/ConnectContainerConfig.java @@ -0,0 +1,99 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.config; + + +import org.streampipes.config.SpConfig; +import org.streampipes.connect.init.Config; + +public enum ConnectContainerConfig { + INSTANCE; + + private SpConfig config; + + ConnectContainerConfig() { + config = SpConfig.getSpConfig("connect-container"); + + config.register(ConfigKeys.BACKEND_HOST, "backend", "Hostname for backend"); + config.register(ConfigKeys.BACKEND_PORT, 8030, "Port for backend"); + + config.register(ConfigKeys.KAFKA_HOST, "kafka", "Hostname for backend service for kafka"); + config.register(ConfigKeys.KAFKA_PORT, 9092, "Port for backend service for kafka"); + + config.register(ConfigKeys.CONNECT_CONTAINER_MASTER_PORT, Config.MASTER_PORT, "The port of the connect container"); + config.register(ConfigKeys.CONNECT_CONTAINER_MASTER_HOST, "connect-master", "The hostname of the connect container"); + + config.register(ConfigKeys.CONNECT_CONTAINER_WORKER_PORT, Config.WORKER_PORT, "The port of the connect container"); + config.register(ConfigKeys.CONNECT_CONTAINER_WORKER_HOST, "connect-worker", "The hostname of the connect container"); + } + + public String getBackendApiUrl() { + return config.getString(ConfigKeys.BACKEND_HOST) + ":" + config.getInteger(ConfigKeys.BACKEND_PORT) + "/streampipes-backend/"; + } + + public String getConnectContainerWorkerUrl() { + return "http://" + config.getString(ConfigKeys.CONNECT_CONTAINER_WORKER_HOST) + ":" + config.getInteger(ConfigKeys.CONNECT_CONTAINER_WORKER_PORT) + "/"; + } + + public String getConnectContainerMasterUrl() { + return "http://" + getConnectContainerMasterHost() + ":" + getConnectContainerMasterPort() + "/"; + } + + public String getBackendHost() { + return config.getString(ConfigKeys.BACKEND_HOST); + } + + public int getBackendPort() { + return config.getInteger(ConfigKeys.BACKEND_PORT); + } + + public String getKafkaHost() { + return config.getString(ConfigKeys.KAFKA_HOST); + } + + public int getKafkaPort() { + return config.getInteger(ConfigKeys.KAFKA_PORT); + } + + public String getKafkaUrl() { + return getKafkaHost() + ":" + getKafkaPort(); + } + + public void setKafkaHost(String s) { + config.setString(ConfigKeys.KAFKA_HOST, s); + } + + public String getConnectContainerMasterHost() { + return config.getString(ConfigKeys.CONNECT_CONTAINER_MASTER_HOST); + } + + public Integer getConnectContainerMasterPort() { + return config.getInteger(ConfigKeys.CONNECT_CONTAINER_MASTER_PORT); + } + + + + public String getConnectContainerWorkerHost() { + return config.getString(ConfigKeys.CONNECT_CONTAINER_WORKER_HOST); + } + + public Integer getConnectContainerWorkerPort() { + return config.getInteger(ConfigKeys.CONNECT_CONTAINER_WORKER_PORT); + } + +} diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/init/Config.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/init/Config.java new file mode 100644 index 0000000000..97e578aeab --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/init/Config.java @@ -0,0 +1,58 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.init; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Config { + + private static final Logger LOG = LoggerFactory.getLogger(Config.class); + + public static final String EXECUTION_MODE = "EXECUTION_MODE"; + public static final String MASTER = "MASTER"; + public static final String WORKER = "WORKER"; + + + public static String CONNECTOR_CONTAINER_ID = "MAIN_CONTAINER"; + + + + public static int MASTER_PORT = 8099; + public static int WORKER_PORT = 8098; + public static String HOST = "localhost"; + + public static String getMasterBaseUrl() { + return "http://" + HOST + ":" + MASTER_PORT + "/"; + } + + public static String getWorkerBaseUrl() { + return "http://" + HOST + ":" + WORKER_PORT + "/"; + } + + public static String getEnv(String envName) { + + String envVarianble = System.getenv(envName); + if (envVarianble == null) { + LOG.error("Environment variable " + envName + " is not set"); + return ""; + } else { + return envVarianble; + } + } +} diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/init/Main.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/init/Main.java new file mode 100644 index 0000000000..3ecf3dfe00 --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/init/Main.java @@ -0,0 +1,138 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.init; + +import org.apache.http.client.fluent.Request; +import org.eclipse.jetty.server.Server; +import org.glassfish.jersey.jetty.JettyHttpContainerFactory; +import org.glassfish.jersey.server.ResourceConfig; +import org.lightcouch.CouchDbClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.connect.management.master.AdapterMasterManagement; +import org.streampipes.connect.rest.master.*; +import org.streampipes.connect.rest.worker.WelcomePageWorker; +import org.streampipes.connect.rest.worker.WorkerResource; +import org.streampipes.rest.shared.serializer.GsonClientModelProvider; +import org.streampipes.rest.shared.serializer.GsonWithIdProvider; +import org.streampipes.rest.shared.serializer.GsonWithoutIdProvider; +import org.streampipes.rest.shared.serializer.JsonLdProvider; +import org.streampipes.storage.couchdb.impl.AdapterStorageImpl; +import org.streampipes.storage.couchdb.utils.Utils; + +import javax.ws.rs.core.UriBuilder; +import java.net.URI; +import java.util.HashSet; +import java.util.Set; + +public class Main { + + private static final Logger LOG = LoggerFactory.getLogger(Main.class); + + public static void main(String... args) throws InterruptedException { + + // TODO Two different execution modes + + String executionMode = Config.getEnv(Config.EXECUTION_MODE); + + ResourceConfig config = null; + URI baseUri = null; + + switch (executionMode) { + case Config.MASTER: + LOG.info("Started StreamPipes Connect Resource in MASTER mode"); + config = new ResourceConfig(getMasterApiClasses()); + baseUri = UriBuilder + .fromUri(Config.getMasterBaseUrl()) + .build(); + + boolean couchDbAvailable = true; + + do { + + // Start all installed adapters on restart of master + try { + AdapterMasterManagement.startAllStreamAdapters(); + couchDbAvailable = true; + } catch (AdapterException e) { + LOG.error("Could not start all installed stream adapters", e); + couchDbAvailable = true; + } catch (Exception e) { + LOG.error("Could not connect to couch db. Try again in 2 seconds"); + couchDbAvailable = false; + Thread.sleep(2000); + } + } while (!couchDbAvailable); + + break; + case Config.WORKER: + LOG.info("Started StreamPipes Connect Resource in WORKER mode"); + config = new ResourceConfig(getWorkerApiClasses()); + baseUri = UriBuilder + .fromUri(Config.getWorkerBaseUrl()) + .build(); + + break; + default: + LOG.error("Environment Variable EXECUTION_MODE is not set correctly. Must be " + Config.MASTER + " or " + Config.WORKER); + System.exit(0); + } + + + Server server = JettyHttpContainerFactory.createServer(baseUri, config); + + } + + private static Set> getMasterApiClasses() { + Set> allClasses = new HashSet<>(); + + allClasses.add(WelcomePageMaster.class); + allClasses.add(AdapterResource.class); + allClasses.add(DescriptionResource.class); + allClasses.add(SourcesResource.class); + allClasses.add(GuessResource.class); + + allClasses.addAll(getApiClasses()); + + return allClasses; + } + + private static Set> getWorkerApiClasses() { + Set> allClasses = new HashSet<>(); + + allClasses.add(WelcomePageWorker.class); + allClasses.add(WorkerResource.class); + + allClasses.addAll(getApiClasses()); + + return allClasses; + } + + private static Set> getApiClasses() { + Set> allClasses = new HashSet<>(); + + // Serializers + allClasses.add(GsonWithIdProvider.class); + allClasses.add(GsonWithoutIdProvider.class); + allClasses.add(GsonClientModelProvider.class); + allClasses.add(JsonLdProvider.class); + + return allClasses; + } +} diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/management/AdapterDeserializer.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/management/AdapterDeserializer.java new file mode 100644 index 0000000000..11a8a5d345 --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/management/AdapterDeserializer.java @@ -0,0 +1,55 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.management; + +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.connect.adapter.AdapterRegistry; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.rest.shared.util.JsonLdUtils; + +import java.util.Set; + +public class AdapterDeserializer { + + public static AdapterDescription getAdapterDescription(String jsonld) throws AdapterException { + Set adapterIds = AdapterRegistry.getAllAdapters().keySet(); + + AdapterDescription result = null; + + for (String key : adapterIds) { + if (jsonld.contains(key)) { + Adapter adapter = AdapterRegistry.getAllAdapters().get(key); + AdapterDescription resultDescription = adapter.declareModel(); + + result = JsonLdUtils.fromJsonLd(jsonld, resultDescription.getClass()); + + if (result.getAdapterId() == null) { + result.setAdapterId(result.getUri()); + } + + } + } + + if (result == null) { + throw new AdapterException("Json-Ld of adapter description could not be deserialized: " + jsonld); + } + + return result; + } +} diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/management/AdapterUtils.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/management/AdapterUtils.java new file mode 100644 index 0000000000..b49cf9348f --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/management/AdapterUtils.java @@ -0,0 +1,52 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.management; + +import org.apache.http.client.fluent.Request; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.connect.config.ConnectContainerConfig; + +import java.io.IOException; + +public class AdapterUtils { + private static final Logger logger = LoggerFactory.getLogger(AdapterUtils .class); + + public static String stopPipeline(String url) { + logger.info("Send stopAdapter pipeline request on URL: " + url); + + String result = ""; + try { + result = Request.Get(url) + .connectTimeout(1000) + .socketTimeout(100000) + .execute().returnContent().asString(); + } catch (IOException e) { + e.printStackTrace(); + result = e.getMessage(); + } + + logger.info("Successfully stopped pipeline"); + + return result; + } + + public static String getUrl(String baseUrl, String pipelineId) { + return "http://" +baseUrl + "api/v2/pipelines/" + pipelineId + "/stopAdapter"; + } +} diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/management/master/AdapterMasterManagement.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/management/master/AdapterMasterManagement.java new file mode 100644 index 0000000000..de53ae5edb --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/management/master/AdapterMasterManagement.java @@ -0,0 +1,218 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.management.master; + +import org.apache.http.client.fluent.Form; +import org.apache.http.client.fluent.Request; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.connect.adapter.GroundingService; +import org.streampipes.connect.config.ConnectContainerConfig; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.model.connect.adapter.AdapterSetDescription; +import org.streampipes.model.connect.adapter.AdapterStreamDescription; +import org.streampipes.model.grounding.EventGrounding; +import org.streampipes.rest.shared.util.JsonLdUtils; +import org.streampipes.storage.couchdb.impl.AdapterStorageImpl; + +import java.io.IOException; +import java.util.List; +import java.util.UUID; + + +public class AdapterMasterManagement { + + private static final Logger logger = LoggerFactory.getLogger(AdapterMasterManagement.class); + + public static void startAllStreamAdapters() throws AdapterException { + AdapterStorageImpl adapterStorage = new AdapterStorageImpl(); + List allAdapters = adapterStorage.getAllAdapters(); + + for (AdapterDescription ad : allAdapters) { + if (ad instanceof AdapterStreamDescription) { + String url = Utils.addUserNameToApi(ConnectContainerConfig.INSTANCE.getConnectContainerWorkerUrl(), ad.getUserName()); + + WorkerRestClient.invokeStreamAdapter(url, (AdapterStreamDescription) ad); + } + } + } + + public String addAdapter(AdapterDescription ad, String baseUrl, AdapterStorageImpl + adapterStorage, String username) + throws AdapterException { + + // Add EventGrounding to AdapterDescription + EventGrounding eventGrounding = GroundingService.createEventGrounding( + ConnectContainerConfig.INSTANCE.getKafkaHost(), ConnectContainerConfig.INSTANCE.getKafkaPort(), null); + ad.setEventGrounding(eventGrounding); + + String newId = ad.getElementId() + UUID.randomUUID().toString(); + ad.changeElementId(newId); + ad.setElementId(newId); + + // store in db + adapterStorage.storeAdapter(ad); + + // start when stream adapter + if (ad instanceof AdapterStreamDescription) { + // TODO + WorkerRestClient.invokeStreamAdapter(baseUrl, (AdapterStreamDescription) ad); + System.out.println("Start adapter"); +// SpConnect.startStreamAdapter((AdapterStreamDescription) ad, baseUrl); + } + + List allAdapters = adapterStorage.getAllAdapters(); + String adapterCouchdbId = ""; + for (AdapterDescription a : allAdapters) { + if (a.getElementId().equals(ad.getElementId())) { + adapterCouchdbId = a.getId(); + } + } + + String backendBaseUrl = "http://" + ConnectContainerConfig.INSTANCE.getBackendApiUrl() +"api/v2/"; +// String userName = ad.getUserName(); + String requestUrl = backendBaseUrl + "noauth/users/" + username + "/element"; + String elementUrl = ConnectContainerConfig.INSTANCE.getConnectContainerMasterUrl() + + "api/v1/" + username + "/master/sources/" + adapterCouchdbId; + + logger.info("Install source (source URL: " + elementUrl +" in backend over URL: " + requestUrl); + + installDataSource(requestUrl, elementUrl); + + return new SourcesManagement().getAdapterDataSource(adapterCouchdbId).getElementId(); + } + + public boolean installDataSource(String requestUrl, String elementIdUrl) throws AdapterException { + + try { + String responseString = Request.Post(requestUrl) + .bodyForm( + Form.form() + .add("uri", elementIdUrl) + .add("publicElement", "true").build()) + .connectTimeout(1000) + .socketTimeout(100000) + .execute().returnContent().asString(); + + logger.info(responseString); + } catch (IOException e) { + logger.error("Error while installing data source: " + requestUrl, e); + throw new AdapterException(); + } + + return true; + } + + public AdapterDescription getAdapter(String id, AdapterStorageImpl adapterStorage) throws AdapterException { + + List allAdapters = adapterStorage.getAllAdapters(); + + if (allAdapters != null && id != null) { + for (AdapterDescription ad : allAdapters) { + if (id.equals(ad.getAdapterId())) { + return ad; + } + } + } + + throw new AdapterException("Could not find adapter with id: " + id); + } + + public void deleteAdapter(String id, String baseUrl) throws AdapterException { + // // IF Stream adapter delete it + AdapterStorageImpl adapterStorage = new AdapterStorageImpl(); + boolean isStreamAdapter = isStreamAdapter(id, adapterStorage); + + if (isStreamAdapter) { + stopStreamAdapter(id, baseUrl, adapterStorage); + } + AdapterDescription ad = adapterStorage.getAdapter(id); + String username = ad.getUserName(); + + adapterStorage.deleteAdapter(id); + + String backendBaseUrl = "http://" + ConnectContainerConfig.INSTANCE.getBackendApiUrl() + "api/v2/noauth/users/"+ username + "/element/delete"; + +// String elementUrl = ConnectContainerConfig.INSTANCE.getConnectContainerMasterUrl() + "api/v1/" + username + "/master/sources/" + id; + String elementUrl = ad.getUri() + "/" + ad.getId(); + +// deleteDataSource(backendBaseUrl, elementUrl); + + + boolean response = true; + + String responseString = null; + logger.info("Delete data source in backend with request URL: " + backendBaseUrl); + try { + responseString = Request.Post(backendBaseUrl) + .connectTimeout(1000) + .socketTimeout(100000) + .bodyForm(Form.form() + .add("uri", elementUrl).build()) + .execute().returnContent().asString(); + } catch (IOException e) { + e.printStackTrace(); + responseString = e.toString(); + response = false; + } + + logger.info("Response of the deletion request" + responseString); + } + + public List getAllAdapters(AdapterStorageImpl adapterStorage) throws AdapterException { + + List allAdapters = adapterStorage.getAllAdapters(); + + if (allAdapters == null) { + throw new AdapterException("Could not get all adapters"); + } + + return allAdapters; + } + + public static void stopSetAdapter(String adapterId, String baseUrl, AdapterStorageImpl adapterStorage) throws AdapterException { + + AdapterSetDescription ad = (AdapterSetDescription) adapterStorage.getAdapter(adapterId); + + WorkerRestClient.stopSetAdapter(baseUrl, ad); + } + + public static void stopStreamAdapter(String adapterId, String baseUrl, AdapterStorageImpl adapterStorage) throws AdapterException { + AdapterStreamDescription ad = (AdapterStreamDescription) adapterStorage.getAdapter(adapterId); + + WorkerRestClient.stopStreamAdapter(baseUrl, ad); + } + + public static boolean isStreamAdapter(String id, AdapterStorageImpl adapterStorage) { + AdapterDescription ad = adapterStorage.getAdapter(id); + + return ad instanceof AdapterStreamDescription; + } + + private static String toJsonLd(T object) { + JsonLdUtils.toJsonLD(object); + String s = JsonLdUtils.toJsonLD(object); + + if (s == null) { + logger.error("Could not serialize Object " + object + " into json ld"); + } + + return s; + } +} diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/management/master/DescriptionManagement.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/management/master/DescriptionManagement.java new file mode 100644 index 0000000000..85aa97a26f --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/management/master/DescriptionManagement.java @@ -0,0 +1,67 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.management.master; + +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.connect.adapter.AdapterRegistry; +import org.streampipes.connect.adapter.generic.format.Format; +import org.streampipes.connect.adapter.generic.protocol.Protocol; +import org.streampipes.model.connect.adapter.AdapterDescriptionList; +import org.streampipes.model.connect.grounding.FormatDescriptionList; +import org.streampipes.model.connect.grounding.ProtocolDescriptionList; + +import java.util.Map; + +public class DescriptionManagement { + + public ProtocolDescriptionList getProtocols() { + Map allProtocols = AdapterRegistry.getAllProtocols(); + + ProtocolDescriptionList result = new ProtocolDescriptionList(); + + for (Protocol p : allProtocols.values()) { + result.getList().add(p.declareModel()); + } + + return result; + } + + public FormatDescriptionList getFormats() { + Map allFormats = AdapterRegistry.getAllFormats(); + + FormatDescriptionList result = new FormatDescriptionList(); + + for (Format f : allFormats.values()) { + result.getList().add(f.declareModel()); + } + + return result; + } + + public AdapterDescriptionList getAdapters() { + Map allAdapters = AdapterRegistry.getAllAdapters(); + + AdapterDescriptionList result = new AdapterDescriptionList(); + + for (Adapter a : allAdapters.values()) { + result.getList().add(a.declareModel()); + } + + return result; + } +} diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/management/master/GuessManagement.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/management/master/GuessManagement.java new file mode 100644 index 0000000000..8057e43ab4 --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/management/master/GuessManagement.java @@ -0,0 +1,52 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.management.master; + +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.connect.adapter.AdapterRegistry; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.model.connect.guess.GuessSchema; +import org.streampipes.model.schema.EventPropertyPrimitive; +import org.streampipes.model.schema.EventSchema; + +import java.util.Arrays; + +public class GuessManagement { + + public GuessSchema guessSchema(AdapterDescription adapterDescription) throws AdapterException { + + Adapter adapter = AdapterRegistry.getAdapter(adapterDescription); + + GuessSchema guessSchema = adapter.getSchema(adapterDescription); + + return guessSchema; + + } + + public void guessFormat() { + // TODO implement + } + + + public void guessFormatDescription() { + // TODO implement + } + + +} diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/management/master/SourcesManagement.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/management/master/SourcesManagement.java new file mode 100644 index 0000000000..25c354405b --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/management/master/SourcesManagement.java @@ -0,0 +1,164 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.management.master; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.config.backend.BackendConfig; +import org.streampipes.connect.config.ConnectContainerConfig; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.container.html.JSONGenerator; +import org.streampipes.container.html.model.DataSourceDescriptionHtml; +import org.streampipes.container.html.model.Description; +import org.streampipes.model.SpDataSet; +import org.streampipes.model.SpDataStream; +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.model.connect.adapter.AdapterSetDescription; +import org.streampipes.model.connect.adapter.AdapterStreamDescription; +import org.streampipes.model.graph.DataSourceDescription; +import org.streampipes.model.grounding.EventGrounding; +import org.streampipes.model.grounding.TransportProtocol; +import org.streampipes.sdk.helpers.Formats; +import org.streampipes.sdk.helpers.Protocols; +import org.streampipes.sdk.helpers.SupportedFormats; +import org.streampipes.sdk.helpers.SupportedProtocols; +import org.streampipes.storage.couchdb.impl.AdapterStorageImpl; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class SourcesManagement { + + private Logger logger = LoggerFactory.getLogger(SourcesManagement.class); + + private AdapterStorageImpl adapterStorage; + private String connectHost = null; + + public SourcesManagement(AdapterStorageImpl adapterStorage) { + this.adapterStorage = adapterStorage; + } + + public SourcesManagement() { + this.adapterStorage = new AdapterStorageImpl(); + } + + public void addAdapter(String baseUrl, String streamId, SpDataSet dataSet) throws AdapterException { + AdapterSetDescription adapterDescription = (AdapterSetDescription) this.adapterStorage.getAdapter(streamId); + adapterDescription.setDataSet(dataSet); + + WorkerRestClient.invokeSetAdapter(baseUrl, adapterDescription); + } + + public void detachAdapter(String baseUrl, String streamId, String runningInstanceId) throws AdapterException { + AdapterSetDescription adapterDescription = (AdapterSetDescription) this.adapterStorage.getAdapter(streamId); + + WorkerRestClient.stopSetAdapter(baseUrl, adapterDescription); + } + + + public String getAllAdaptersInstallDescription(String user) throws AdapterException { + String host = getConnectHost(); + + List allAdapters = adapterStorage.getAllAdapters(); + List allAdapterDescriptions = new ArrayList<>(); + + for (AdapterDescription ad : allAdapters) { + URI uri = null; + String uriString = null; + try { +// uriString = "http://" + host + "/streampipes-connect/api/v1/" + user + "/master/adapters/" + ad.getId(); + uriString = "http://" + host + "/streampipes-connect/api/v1/" + user + "/master/sources/" + ad.getId(); + uri = new URI(uriString); + } catch (URISyntaxException e) { + logger.error("URI for the sources endpoint is not correct: " + uriString, e); + throw new AdapterException("Username " + user + " not allowed"); + } + + + List streams = new ArrayList<>(); + Description d = new Description(ad.getName(), "", uri); + d.setType("set"); + streams.add(d); + DataSourceDescriptionHtml dsd = new DataSourceDescriptionHtml("Adapter Stream", + "This stream is generated by an StreamPipes Connect adapter. ID of adapter: " + ad.getId(), uri, streams); + dsd.setType("source"); + allAdapterDescriptions.add(dsd); + } + + JSONGenerator json = new JSONGenerator(allAdapterDescriptions); + + return json.buildJson(); + } + + public DataSourceDescription getAdapterDataSource(String id) throws AdapterException { + + AdapterDescription adapterDescription = new AdapterStorageImpl().getAdapter(id); + + SpDataStream ds; + if (adapterDescription instanceof AdapterSetDescription) { + ds = ((AdapterSetDescription) adapterDescription).getDataSet(); + EventGrounding eg = new EventGrounding(); + eg.setTransportProtocol(SupportedProtocols.kafka()); + eg.setTransportFormats(Arrays.asList(SupportedFormats.jsonFormat())); + ((SpDataSet) ds).setSupportedGrounding(eg); + } else { + ds = ((AdapterStreamDescription) adapterDescription).getDataStream(); + + + String topic = adapterDescription.getEventGrounding().getTransportProtocol().getTopicDefinition().getActualTopicName(); + + TransportProtocol tp = Protocols.kafka(BackendConfig.INSTANCE.getKafkaHost(), BackendConfig.INSTANCE.getKafkaPort(), topic); + EventGrounding eg = new EventGrounding(); + eg.setTransportFormats(Arrays.asList(Formats.jsonFormat())); + eg.setTransportProtocol(tp); + + ds.setEventGrounding(eg); + } + + + String url = adapterDescription.getUri().toString() + "/" + adapterDescription.getId(); + + ds.setName(adapterDescription.getName()); + ds.setDescription("Description"); + + ds.setUri(url + "/streams"); + + DataSourceDescription dataSourceDescription = new DataSourceDescription( + url, "Adaper Data Source", + "This data source contains one data stream from the adapters"); + + dataSourceDescription.addEventStream(ds); + + return dataSourceDescription; + } + + public String getConnectHost() { + if (connectHost == null) { + return ConnectContainerConfig.INSTANCE.getBackendHost() + ":" + ConnectContainerConfig.INSTANCE.getBackendPort(); + } else { + return connectHost; + } + } + + public void setConnectHost(String connectHost) { + this.connectHost = connectHost; + } +} diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/management/master/Utils.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/management/master/Utils.java new file mode 100644 index 0000000000..3cf55fb083 --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/management/master/Utils.java @@ -0,0 +1,24 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.management.master; + +public class Utils { + public static String addUserNameToApi(String url, String userName) { + return url + "api/v1/" + userName + "/"; + } +} diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/management/master/WorkerRestClient.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/management/master/WorkerRestClient.java new file mode 100644 index 0000000000..8b9cb2fed2 --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/management/master/WorkerRestClient.java @@ -0,0 +1,127 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.management.master; + +import org.apache.http.client.fluent.Request; +import org.apache.http.entity.ContentType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.model.connect.adapter.AdapterSetDescription; +import org.streampipes.model.connect.adapter.AdapterStreamDescription; +import org.streampipes.rest.shared.util.JsonLdUtils; +import org.streampipes.storage.couchdb.impl.AdapterStorageImpl; + +import java.io.IOException; +import java.util.UUID; + +public class WorkerRestClient { + + private static final Logger logger = LoggerFactory.getLogger(WorkerRestClient.class); + + public static void invokeStreamAdapter(String baseUrl, AdapterStreamDescription adapterStreamDescription) throws AdapterException { + + String url = baseUrl + "worker/stream/invoke"; + + startAdapter(url, adapterStreamDescription); + } + + public static void stopStreamAdapter(String baseUrl, AdapterStreamDescription adapterStreamDescription) throws AdapterException { + String url = baseUrl + "worker/stream/stop"; + + stopAdapter(adapterStreamDescription.getId(), new AdapterStorageImpl(), url); + } + + public static void invokeSetAdapter(String baseUrl, AdapterSetDescription adapterSetDescription) throws AdapterException { + String url = baseUrl + "worker/set/invoke"; + + startAdapter(url, adapterSetDescription); + } + + public static void stopSetAdapter(String baseUrl, AdapterSetDescription adapterSetDescription) throws AdapterException { + String url = baseUrl + "worker/set/stop"; + + stopAdapter(adapterSetDescription.getAdapterId(), new AdapterStorageImpl(), url); + } + + public static void startAdapter(String url, AdapterDescription ad) throws AdapterException { + try { + logger.info("Trying to start adpater on endpoint: " + url); + + // this ensures that all adapters have a valid uri otherwise the json-ld serializer fails + if (ad.getUri() == null) { + ad.setUri("https://streampipes.org/adapter/" + UUID.randomUUID()); + } + + String adapterDescription = JsonLdUtils.toJsonLD(ad); + + String responseString = Request.Post(url) + .bodyString(adapterDescription, ContentType.APPLICATION_JSON) + .connectTimeout(1000) + .socketTimeout(100000) + .execute().returnContent().asString(); + + logger.info("Adapter started on endpoint: " + url + " with Response: " + responseString); + + } catch (IOException e) { + logger.error("Adapter did not start", e); + throw new AdapterException("Adapter with URL: " + url + " did not start"); + } + } + + + public static void stopAdapter(String adapterId, AdapterStorageImpl adapterStorage, String url) throws AdapterException { + + //Delete from database + AdapterDescription ad = adapterStorage.getAdapter(adapterId); + +// System.out.println("blll: " + adapterId); + +// adapterStorage.deleteAdapter(adapterId); + + // Stop execution of adatper + try { + logger.info("Trying to stopAdapter adpater on endpoint: " + url); + + // TODO quick fix because otherwise it is not serialized to json-ld + if (ad.getUri() == null) { + logger.error("Adapter uri is null this should not happen " + ad); + } + + String adapterDescription = JsonLdUtils.toJsonLD(ad); + + // TODO change this to a delete request + String responseString = Request.Post(url) + .bodyString(adapterDescription, ContentType.APPLICATION_JSON) + .connectTimeout(1000) + .socketTimeout(100000) + .execute().returnContent().asString(); + + logger.info("Adapter stopped on endpoint: " + url + " with Response: " + responseString); + + } catch (IOException e) { + logger.error("Adapter was not stopped successfully", e); + throw new AdapterException("Adapter was not stopped successfully with url: " + url); + } + + } + + +} + diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/management/worker/AdapterWorkerManagement.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/management/worker/AdapterWorkerManagement.java new file mode 100644 index 0000000000..90cd185766 --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/management/worker/AdapterWorkerManagement.java @@ -0,0 +1,97 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.management.worker; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.connect.RunningAdapterInstances; +import org.streampipes.connect.adapter.AdapterRegistry; +import org.streampipes.connect.config.ConnectContainerConfig; +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.connect.management.AdapterUtils; +import org.streampipes.model.SpDataSet; +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.model.connect.adapter.AdapterSetDescription; +import org.streampipes.model.connect.adapter.AdapterStreamDescription; + +public class AdapterWorkerManagement { + + private static final Logger logger = LoggerFactory.getLogger(AdapterWorkerManagement.class); + + public void invokeStreamAdapter(AdapterStreamDescription adapterStreamDescription) throws AdapterException { + + Adapter adapter = AdapterRegistry.getAdapter(adapterStreamDescription); + + RunningAdapterInstances.INSTANCE.addAdapter(adapterStreamDescription.getUri(), adapter); + adapter.startAdapter(); + + } + + public void stopStreamAdapter(AdapterStreamDescription adapterStreamDescription) throws AdapterException { + stopAdapter(adapterStreamDescription); + + } + + public void invokeSetAdapter (AdapterSetDescription adapterSetDescription) throws AdapterException { + SpDataSet dataSet = adapterSetDescription.getDataSet(); + + Adapter adapter = AdapterRegistry.getAdapter(adapterSetDescription); + RunningAdapterInstances.INSTANCE.addAdapter(dataSet.getDatasetInvocationId(), adapter); + + + // Set adapters run the whole set in one thread, once all data is processed the corresponding pipeline is stopped + Runnable r = () -> { + try { + adapter.startAdapter(); + } catch (AdapterException e) { + e.printStackTrace(); + } + + // TODO wait till all components are done with their calculations + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + String url = AdapterUtils.getUrl(ConnectContainerConfig.INSTANCE.getBackendApiUrl(), dataSet.getCorrespondingPipeline()); + String result = AdapterUtils.stopPipeline(url); + logger.info(result); + }; + + new Thread(r).start(); + } + + public void stopSetAdapter (AdapterSetDescription adapterSetDescription) throws AdapterException { + stopAdapter(adapterSetDescription); + } + + private void stopAdapter(AdapterDescription adapterDescription) throws AdapterException { + + String adapterUri = adapterDescription.getUri(); + + Adapter adapter = RunningAdapterInstances.INSTANCE.removeAdapter(adapterUri); + + if (adapter == null) { + throw new AdapterException("Adapter with id " + adapterUri + " was not found in this container and cannot be stopped."); + } + + adapter.stopAdapter(); + } + +} diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/AbstractContainerResource.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/AbstractContainerResource.java new file mode 100644 index 0000000000..72f39acf3e --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/AbstractContainerResource.java @@ -0,0 +1,40 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.rest; + +import org.streampipes.model.client.messages.Message; + +import javax.ws.rs.core.Response; + +public abstract class AbstractContainerResource { + + protected Response ok(T entity) { + return Response + .ok() + .entity(entity) + .build(); + } + + protected Response statusMessage(Message message) { + return ok(message); + } + + protected Response fail() { + return Response.serverError().build(); + } +} diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/SpConnect.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/SpConnect.java new file mode 100644 index 0000000000..529b4969a7 --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/SpConnect.java @@ -0,0 +1,239 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.rest; + +@Deprecated +public class SpConnect { +// +// private static final Logger logger = LoggerFactory.getLogger(SpConnectResource.class); +// +//// public String addAdapter(AdapterDescription ad, String baseUrl, AdapterStorageImpl adapterStorage) { +//// +//// // store in db +//// adapterStorage.storeAdapter(ad); +//// +//// +//// // start when stream adapter +//// if (ad instanceof AdapterStreamDescription) { +//// SpConnect.startStreamAdapter((AdapterStreamDescription) ad, baseUrl); +//// } +//// +//// List allAdapters = adapterStorage.getAllAdapters(); +//// String adapterCouchdbId = ""; +//// for (AdapterDescription a : allAdapters) { +//// if (a.getAdapterId().equals(ad.getAdapterId())) { +//// adapterCouchdbId = a.getId(); +//// } +//// } +//// +//// String backendBaseUrl = "http://" + BackendConfig.INSTANCE.getBackendHost() + ":" + "8030" + "/streampipes-backend/api/v2/"; +//// String userName = ad.getUserName(); +//// String requestUrl = backendBaseUrl + "noauth/users/" + userName + "/element"; +//// logger.info("Request URL: " + requestUrl); +//// +//// String elementUrl = backendBaseUrl + "adapter/all/" + adapterCouchdbId; +//// logger.info("Element URL: " + elementUrl); +//// +//// installDataSource(requestUrl, elementUrl); +//// +//// +//// return SpConnectUtils.SUCCESS; +//// } +// +//// public boolean installDataSource(String requestUrl, String elementIdUrl) { +//// +//// try { +//// String responseString = Request.Post(requestUrl) +//// .bodyForm( +//// Form.form() +//// .add("uri", elementIdUrl) +//// .add("publicElement", "true").build()) +//// .connectTimeout(1000) +//// .socketTimeout(100000) +//// .execute().returnContent().asString(); +//// +//// logger.info(responseString); +//// } catch (IOException e) { +//// e.printStackTrace(); +//// } +//// +//// return true; +//// } +// +// +// public static boolean deleteDataSource(String requestUrl, String elementUri) { +// boolean response = true; +// +// String responseString = null; +// logger.info("Delete data source in backend with request URL: " + requestUrl); +// try { +// responseString = Request.Post(requestUrl) +// .connectTimeout(1000) +// .socketTimeout(100000) +// .bodyForm(Form.form() +// .add("uri", elementUri).build()) +// .execute().returnContent().asString(); +// } catch (IOException e) { +// e.printStackTrace(); +// responseString = e.toString(); +// response = false; +// } +// +// logger.info("Response of the deletion request" + responseString); +// return response; +// } +// +// public static boolean isStreamAdapter(String id, AdapterStorageImpl adapterStorage) { +// AdapterDescription ad = adapterStorage.getAdapter(id); +// +// return ad instanceof AdapterStreamDescription; +// } +// +// public static AdapterDescription getAdapterDescription(String ads) { +// +// +// AdapterDescription a = null; +// +// if (ads.contains("AdapterSetDescription")){ +// JsonLdTransformer jsonLdTransformer = new JsonLdTransformer(StreamPipes.ADAPTER_SET_DESCRIPTION); +// a = getDescription(jsonLdTransformer, ads, AdapterSetDescription.class); +// } else { +// JsonLdTransformer jsonLdTransformer = new JsonLdTransformer(StreamPipes.ADAPTER_STREAM_DESCRIPTION); +// a = getDescription(jsonLdTransformer, ads, AdapterStreamDescription.class); +// } +// +// logger.info("Add Adapter Description " + a.getUri()); +// +// return a; +// } +// +// public static T getDescription(JsonLdTransformer jsonLdTransformer, String s, Class theClass) { +// +// T a = null; +// +// try { +// a = jsonLdTransformer.fromJsonLd(s, theClass); +// } catch (IOException e) { +// e.printStackTrace(); +// } +// +// return a; +// } +// +// public static String stopSetAdapter(String adapterId, String baseUrl, AdapterStorageImpl adapterStorage) { +// String url = baseUrl + "api/v1/stopAdapter/set"; +// +// return stopAdapter(adapterId, adapterStorage, url); +// } +// +// public static String stopStreamAdapter(String adapterId, String baseUrl, AdapterStorageImpl adapterStorage) { +// String url = baseUrl + "api/v1/stopAdapter/stream"; +// +// return stopAdapter(adapterId, adapterStorage, url); +// } +// +// private static String stopAdapter(String adapterId, AdapterStorageImpl adapterStorage, String url) { +// +// //Delete from database +// AdapterDescription ad = adapterStorage.getAdapter(adapterId); +// +// // Stop execution of adatper +// try { +// logger.info("Trying to stopAdapter adpater on endpoint: " + url); +// +// // TODO quick fix because otherwise it is not serialized to json-ld +// if (ad.getUri() == null) { +// logger.error("Adapter uri is null this should not happen " + ad); +// } +// +// String adapterDescription = toJsonLd(ad); +// +// // TODO change this to a delete request +// String responseString = Request.Post(url) +// .bodyString(adapterDescription, ContentType.APPLICATION_JSON) +// .connectTimeout(1000) +// .socketTimeout(100000) +// .execute().returnContent().asString(); +// +// logger.info("Adapter stopped on endpoint: " + url + " with Response: " + responseString); +// +// } catch (IOException e) { +// e.printStackTrace(); +// +// return "Adapter was not stopped successfully"; +// } +// +// return SpConnectUtils.SUCCESS; +// } +// +// +// public static String startStreamAdapter(AdapterStreamDescription asd, String baseUrl) { +// String url = baseUrl + "api/v1/invoke/stream"; +// +// return postStartAdapter(url, asd); +// } +// +// public String invokeAdapter(String streamId, SpDataSet dataSet, String baseUrl, AdapterStorageImpl adapterStorage) { +// String url = baseUrl + "api/v1/invoke/set"; +// +// AdapterSetDescription adapterDescription = (AdapterSetDescription) adapterStorage.getAdapter(streamId); +// adapterDescription.setDataSet(dataSet); +// +// return postStartAdapter(url, adapterDescription); +// } +// +// private static String postStartAdapter(String url, AdapterDescription ad) { +// try { +// logger.info("Trying to start adpater on endpoint: " + url); +// +// // this ensures that all adapters have a valid uri otherwise the json-ld serializer fails +// if (ad.getUri() == null) { +// ad.setUri("https://streampipes.org/adapter/" + UUID.randomUUID()); +// } +// String adapterDescription = toJsonLd(ad); +// +// String responseString = Request.Post(url) +// .bodyString(adapterDescription, ContentType.APPLICATION_JSON) +// .connectTimeout(1000) +// .socketTimeout(100000) +// .execute().returnContent().asString(); +// +// logger.info("Adapter started on endpoint: " + url + " with Response: " + responseString); +// +// } catch (IOException e) { +// e.printStackTrace(); +// +// return "Adapter was not started successfully"; +// } +// +// return SpConnectUtils.SUCCESS; +// } +// +// private static String toJsonLd(T object) { +// JsonLdUtils.toJsonLD(object); +// String s = JsonLdUtils.toJsonLD(object); +// +// if (s == null) { +// logger.error("Could not serialize Object " + object + " into json ld"); +// } +// +// return s; +// } + + +} diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/SpConnectResource.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/SpConnectResource.java new file mode 100644 index 0000000000..61a59462ba --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/SpConnectResource.java @@ -0,0 +1,314 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.rest; + + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.config.backend.BackendConfig; +import org.streampipes.connect.adapter.generic.format.csv.CsvFormat; +import org.streampipes.connect.adapter.generic.format.geojson.GeoJsonFormat; +import org.streampipes.connect.adapter.generic.format.json.arraykey.JsonFormat; +import org.streampipes.connect.adapter.generic.format.json.object.JsonObjectFormat; +import org.streampipes.connect.adapter.generic.format.xml.XmlFormat; +import org.streampipes.connect.adapter.generic.protocol.set.FileProtocol; +import org.streampipes.connect.adapter.generic.protocol.set.HttpProtocol; +import org.streampipes.connect.adapter.generic.protocol.stream.HttpStreamProtocol; +import org.streampipes.connect.adapter.generic.protocol.stream.KafkaProtocol; +import org.streampipes.connect.adapter.generic.protocol.stream.MqttProtocol; +import org.streampipes.container.html.JSONGenerator; +import org.streampipes.container.html.model.DataSourceDescriptionHtml; +import org.streampipes.container.html.model.Description; +import org.streampipes.container.util.Util; +import org.streampipes.model.SpDataSet; +import org.streampipes.model.SpDataStream; +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.model.connect.adapter.AdapterDescriptionList; +import org.streampipes.model.connect.adapter.AdapterSetDescription; +import org.streampipes.model.connect.adapter.AdapterStreamDescription; +import org.streampipes.model.connect.grounding.FormatDescriptionList; +import org.streampipes.model.connect.grounding.ProtocolDescriptionList; +import org.streampipes.model.graph.DataSourceDescription; +import org.streampipes.model.grounding.EventGrounding; +import org.streampipes.model.grounding.TransportProtocol; +import org.streampipes.rest.shared.annotation.GsonWithIds; +import org.streampipes.rest.shared.annotation.JsonLdSerialized; +import org.streampipes.rest.shared.util.JsonLdUtils; +import org.streampipes.rest.shared.util.SpMediaType; +import org.streampipes.sdk.helpers.Formats; +import org.streampipes.sdk.helpers.Protocols; +import org.streampipes.sdk.helpers.SupportedFormats; +import org.streampipes.sdk.helpers.SupportedProtocols; +import org.streampipes.serializers.jsonld.JsonLdTransformer; +import org.streampipes.storage.couchdb.impl.AdapterStorageImpl; +import org.streampipes.vocabulary.StreamPipes; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +//@Path("/v2/adapter") +//@Deprecated +public class SpConnectResource { +// +// private static final Logger logger = LoggerFactory.getLogger(SpConnectResource.class); +// private SpConnect spConnect; +// private String connectContainerEndpoint; +// +//// @Context +//// UriInfo uri; +// +// public SpConnectResource() { +// spConnect = new SpConnect(); +//// connectContainerEndpoint = BackendConfig.INSTANCE.getConnectContainerMasterUrl(); +// } +// +// public SpConnectResource(SpConnect spConnect, String connectContainerEndpoint) { +// this.spConnect = spConnect; +// this.connectContainerEndpoint = connectContainerEndpoint; +// } +// +//// @GET +//// @JsonLdSerialized +//// @Produces(SpMediaType.JSONLD) +// @Path("/allProtocols") +// public Response getAllProtocols() { +// ProtocolDescriptionList pdl = new ProtocolDescriptionList(); +// pdl.addDesctiption(new HttpProtocol().declareModel()); +// pdl.addDesctiption(new FileProtocol().declareModel()); +// pdl.addDesctiption(new KafkaProtocol().declareModel()); +// pdl.addDesctiption(new MqttProtocol().declareModel()); +// pdl.addDesctiption(new HttpStreamProtocol().declareModel()); +// +// return ok(pdl); +// } +//// +//// @GET +//// @Produces(MediaType.APPLICATION_JSON) +//// @Path("/allFormats") +//// public Response getAllFormats() { +//// FormatDescriptionList fdl = new FormatDescriptionList(); +//// fdl.addDesctiption(new JsonFormat().declareModel()); +//// fdl.addDesctiption(new JsonObjectFormat().declareModel()); +//// fdl.addDesctiption(new CsvFormat().declareModel()); +//// fdl.addDesctiption(new GeoJsonFormat().declareModel()); +//// fdl.addDesctiption(new XmlFormat().declareModel()); +//// +//// return ok(JsonLdUtils.toJsonLD(fdl)); +//// } +// +//// @GET +//// @Produces(MediaType.APPLICATION_JSON) +//// @GsonWithIds +//// @Path("/all") +//// public Response getAllAdapters() { +//// String host = BackendConfig.INSTANCE.getBackendHost() + ":" + BackendConfig.INSTANCE.getBackendPort(); +//// +//// List allAdapters = new AdapterStorageImpl().getAllAdapters(); +//// List allAdapterDescriptions = new ArrayList<>(); +//// +//// for (AdapterDescription ad : allAdapters) { +//// URI uri = null; +//// try { +//// uri = new URI("http://" + host + "/streampipes-backend/api/v2/adapter/all/" + ad.getId()); +//// } catch (URISyntaxException e) { +//// e.printStackTrace(); +//// } +//// List streams = new ArrayList<>(); +//// Description d = new Description(ad.getName(), "", uri); +//// d.setType("set"); +//// streams.add(d); +//// DataSourceDescriptionHtml dsd = new DataSourceDescriptionHtml("Adapter Stream", +//// "This stream is generated by an StreamPipes Connect adapter. ID of adapter: " + ad.getId(), uri, streams); +//// dsd.setType("source"); +//// allAdapterDescriptions.add(dsd); +//// } +//// +//// JSONGenerator json = new JSONGenerator(allAdapterDescriptions); +//// +//// return ok(json.buildJson()); +//// } +//// +//// @GET +//// @Produces(MediaType.APPLICATION_JSON) +//// @Path("/all/{id}") +//// public Response getAdapter(@Context HttpServletRequest request, @PathParam("id") String id) { +//// +//// AdapterDescription adapterDescription = new AdapterStorageImpl().getAdapter(id); +//// +//// SpDataStream ds; +//// if (adapterDescription instanceof AdapterSetDescription) { +//// ds = ((AdapterSetDescription) adapterDescription).getDataSet(); +//// EventGrounding eg = new EventGrounding(); +//// eg.setTransportProtocol(SupportedProtocols.kafka()); +//// eg.setTransportFormats(Arrays.asList(SupportedFormats.jsonFormat())); +//// ((SpDataSet) ds).setSupportedGrounding(eg); +//// } else { +//// ds = ((AdapterStreamDescription) adapterDescription).getDataStream(); +//// +//// String topic = getTopicPrefix() + adapterDescription.getName(); +//// +//// TransportProtocol tp = Protocols.kafka(BackendConfig.INSTANCE.getKafkaHost(), BackendConfig.INSTANCE.getKafkaPort(), topic); +//// EventGrounding eg = new EventGrounding(); +//// eg.setTransportFormats(Arrays.asList(Formats.jsonFormat())); +//// eg.setTransportProtocol(tp); +//// +//// ds.setEventGrounding(eg); +//// } +//// +//// +//// String url = request.getRequestURL().toString(); +//// +//// ds.setName(adapterDescription.getName()); +//// ds.setDescription("Description"); +//// +//// ds.setUri(url + "/streams"); +//// +//// DataSourceDescription dataSourceDescription = new DataSourceDescription( +//// url, "Adaper Data Source", +//// "This data source contains one data stream from the adapters"); +//// +//// dataSourceDescription.addEventStream(ds); +//// +//// return ok(JsonLdUtils.toJsonLD(dataSourceDescription)); +//// } +// +// @POST +// @Path("/all/{streamId}/streams") +// @Consumes(MediaType.APPLICATION_JSON) +// @Produces(MediaType.APPLICATION_JSON) +// public String invokeAdapter(@PathParam("streamId") String streamId, String +// payload) { +// +// JsonLdTransformer jsonLdTransformer = new JsonLdTransformer(StreamPipes.DATA_SET); +// +// SpDataSet dataSet = SpConnect.getDescription(jsonLdTransformer, payload, SpDataSet.class); +// +// String result = spConnect.invokeAdapter(streamId, dataSet, connectContainerEndpoint, new AdapterStorageImpl()); +// +// return getResponse(result, streamId); +// } +// +// +// @DELETE +// @Path("/all/{streamId}/streams/{runningInstanceId}") +// @Produces(MediaType.APPLICATION_JSON) +// public String detach(@PathParam("streamId") String elementId, @PathParam("runningInstanceId") String runningInstanceId) { +// String result = SpConnect.stopSetAdapter(elementId, connectContainerEndpoint, new AdapterStorageImpl()); +// +// org.streampipes.model.Response resp = new org.streampipes.model.Response(runningInstanceId, true); +// return Util.toResponseString(resp); +// +//// return getResponse(result, runningInstanceId); +// } +// +// +//// @POST +//// @JsonLdSerialized +//// @Produces(MediaType.APPLICATION_JSON) +//// @Consumes(SpMediaType.JSONLD) +//// public String addAdapter(AdapterDescription a) { +//// //logger.info("Received request add adapter with json-ld: " + ar); +//// +//// //AdapterDescription a = SpConnect.getAdapterDescription(ar); +//// UUID id = UUID.randomUUID(); +//// if (a.getUri() == null) { +//// a.setUri("https://streampipes.org/adapter/" + id); +//// } +//// +//// a.setAdapterId(id.toString()); +//// +//// String success = spConnect.addAdapter(a, connectContainerEndpoint, new AdapterStorageImpl()); +//// +//// return getResponse(success, a.getUri()); +//// } +// +// +//// @DELETE +//// @Produces(MediaType.APPLICATION_JSON) +//// @Path("{adapterId}") +//// public String deleteAdapter(@PathParam("adapterId") String couchDbadapterId) { +//// String result = ""; +//// +//// // IF Stream adapter delete it +//// AdapterStorageImpl adapterStorage = new AdapterStorageImpl(); +//// boolean isStreamAdapter = SpConnect.isStreamAdapter(couchDbadapterId, adapterStorage); +//// +//// if (isStreamAdapter) { +//// result = SpConnect.stopStreamAdapter(couchDbadapterId, connectContainerEndpoint, adapterStorage); +//// } +//// AdapterDescription ad = adapterStorage.getAdapter(couchDbadapterId); +//// String username = ad.getUserName(); +//// +//// adapterStorage.deleteAdapter(couchDbadapterId); +//// +//// String backendBaseUrl = "http://" + BackendConfig.INSTANCE.getBackendHost() + ":" + "8030" + +//// "/streampipes-backend/api/v2/noauth/users/"+ username + "/element/"; +//// backendBaseUrl = backendBaseUrl + couchDbadapterId; +//// SpConnect.deleteDataSource(backendBaseUrl); +//// +//// +//// return getResponse(result, couchDbadapterId); +//// } +// +//// @GET +//// @Produces(MediaType.APPLICATION_JSON) +//// @GsonWithIds +//// @Path("/allrunning") +//// public Response getAllRunningAdapters() { +//// +//// AdapterDescriptionList adapterDescriptionList = new AdapterDescriptionList(); +//// +//// List allAdapters = new AdapterStorageImpl().getAllAdapters(); +//// adapterDescriptionList.setList(allAdapters); +//// +//// for(AdapterDescription ad : adapterDescriptionList.getList()) { +//// ad.setUri("https://www.streampipes.org/adapter/" + UUID.randomUUID()); +//// } +//// +//// return ok(JsonLdUtils.toJsonLD(adapterDescriptionList)); +//// } +// +// private String getTopicPrefix() { +// return "org.streampipes.autogenerated.adapters."; +// } +// +// public void setSpConnect(SpConnect spConnect) { +// this.spConnect = spConnect; +// } +// +// public void setConnectContainerEndpoint(String connectContainerEndpoint) { +// this.connectContainerEndpoint = connectContainerEndpoint; +// } +// +// private String getResponse(String result, String id) { +// if (result == SpConnectUtils.SUCCESS) { +// return new org.streampipes.model.Response(id, true).toString(); +// } else { +// return new org.streampipes.model.Response(id, false, result).toString(); +// } +// } +} diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/SpConnectUtils.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/SpConnectUtils.java new file mode 100644 index 0000000000..b2d04fd2ed --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/SpConnectUtils.java @@ -0,0 +1,22 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.rest; + +public class SpConnectUtils { + public static final String SUCCESS = "success"; +} diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/master/AdapterResource.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/master/AdapterResource.java new file mode 100644 index 0000000000..9546acd29c --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/master/AdapterResource.java @@ -0,0 +1,155 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.rest.master; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.connect.config.ConnectContainerConfig; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.connect.management.AdapterDeserializer; +import org.streampipes.connect.management.master.AdapterMasterManagement; +import org.streampipes.connect.management.master.Utils; +import org.streampipes.connect.rest.AbstractContainerResource; +import org.streampipes.model.client.messages.Notifications; +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.model.connect.adapter.AdapterDescriptionList; +import org.streampipes.rest.shared.annotation.GsonWithIds; +import org.streampipes.rest.shared.annotation.JsonLdSerialized; +import org.streampipes.rest.shared.util.SpMediaType; +import org.streampipes.storage.couchdb.impl.AdapterStorageImpl; + +import java.util.List; + +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +@Path("/api/v1/{username}/master/adapters") +public class AdapterResource extends AbstractContainerResource { + + private Logger logger = LoggerFactory.getLogger(AdapterResource.class); + + private AdapterMasterManagement adapterMasterManagement; + + private String connectContainerEndpoint; + + public AdapterResource() { + this.adapterMasterManagement = new AdapterMasterManagement(); + this.connectContainerEndpoint = ConnectContainerConfig.INSTANCE.getConnectContainerWorkerUrl(); + } + + public AdapterResource(String connectContainerEndpoint) { + this.adapterMasterManagement = new AdapterMasterManagement(); + this.connectContainerEndpoint = connectContainerEndpoint; + } + + @POST +// @JsonLdSerialized + @Path("/") + @GsonWithIds + @Produces(MediaType.APPLICATION_JSON) + public Response addAdapter(String s, @PathParam("username") String userName) { + + AdapterDescription adapterDescription = null; + String adapterId; + + try { + adapterDescription = AdapterDeserializer.getAdapterDescription(s); + } catch (AdapterException e) { + logger.error("Could not deserialize AdapterDescription: " + s, e); + e.printStackTrace(); + } + + logger.info("User: " + userName + " starts adapter " + adapterDescription.getAdapterId()); + + String newUrl = Utils.addUserNameToApi(connectContainerEndpoint, userName); + + try { + adapterId = adapterMasterManagement.addAdapter(adapterDescription, newUrl, new + AdapterStorageImpl(), userName); + } catch (AdapterException e) { + logger.error("Error while starting adapter with id " + adapterDescription.getUri(), e); + return ok(Notifications.error(e.getMessage())); + } + + logger.info("Stream adapter with id " + adapterId + " successfully added"); + return ok(Notifications.success(adapterId)); + } + + @GET + @JsonLdSerialized + @Path("/{id}") + @Produces(SpMediaType.JSONLD) + public Response getAdapter(@PathParam("id") String id, @PathParam("username") String userName) { + + try { + AdapterDescription adapterDescription = adapterMasterManagement.getAdapter(id, new AdapterStorageImpl()); + + return ok(adapterDescription); + } catch (AdapterException e) { + logger.error("Error while getting adapter with id " + id, e); + return fail(); + } + + } + + @DELETE + @JsonLdSerialized + @Path("/{id}") + @Produces(MediaType.APPLICATION_JSON) + public Response deleteAdapter(@PathParam("id") String id, @PathParam("username") String userName) { + + try { + + String newUrl = Utils.addUserNameToApi(connectContainerEndpoint, userName); + adapterMasterManagement.deleteAdapter(id, newUrl); + return ok(true); + } catch (AdapterException e) { + logger.error("Error while deleting adapter with id " + id, e); + return fail(); + } + } + + @GET + @JsonLdSerialized + @Path("/") + @Produces(SpMediaType.JSONLD) + public Response getAllAdapters(String id, @PathParam("username") String userName) { + try { + List allAdapterDescription = adapterMasterManagement.getAllAdapters(new AdapterStorageImpl()); + AdapterDescriptionList result = new AdapterDescriptionList(); + result.setList(allAdapterDescription); + + return ok(result); + } catch (AdapterException e) { + logger.error("Error while getting all adapters", e); + return fail(); + } + + } + + public void setAdapterMasterManagement(AdapterMasterManagement adapterMasterManagement) { + this.adapterMasterManagement = adapterMasterManagement; + } + +} diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/master/DescriptionResource.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/master/DescriptionResource.java new file mode 100644 index 0000000000..d12d3edfc2 --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/master/DescriptionResource.java @@ -0,0 +1,83 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.rest.master; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.connect.management.master.DescriptionManagement; +import org.streampipes.connect.rest.AbstractContainerResource; +import org.streampipes.model.connect.adapter.AdapterDescriptionList; +import org.streampipes.model.connect.grounding.FormatDescriptionList; +import org.streampipes.model.connect.grounding.ProtocolDescriptionList; +import org.streampipes.rest.shared.annotation.JsonLdSerialized; +import org.streampipes.rest.shared.util.SpMediaType; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Response; + +@Path("/api/v1/{username}/master/description") +public class DescriptionResource extends AbstractContainerResource { + + private Logger logger = LoggerFactory.getLogger(DescriptionResource.class); + + private DescriptionManagement descriptionManagement; + + public DescriptionResource() { + descriptionManagement = new DescriptionManagement(); + } + + public DescriptionResource(DescriptionManagement descriptionManagement) { + this.descriptionManagement = descriptionManagement; + } + + @GET + @JsonLdSerialized + @Path("/formats") + @Produces(SpMediaType.JSONLD) + public Response getFormats() { + FormatDescriptionList result = descriptionManagement.getFormats(); + + return ok(result); + } + + @GET + @JsonLdSerialized + @Path("/protocols") + @Produces(SpMediaType.JSONLD) + public Response getProtocols() { + ProtocolDescriptionList result = descriptionManagement.getProtocols(); + + return ok(result); + } + + @GET + @JsonLdSerialized + @Path("/adapters") + @Produces(SpMediaType.JSONLD) + public Response getAdapters() { + AdapterDescriptionList result = descriptionManagement.getAdapters(); + + return ok(result); + } + + public void setDescriptionManagement(DescriptionManagement descriptionManagement) { + this.descriptionManagement = descriptionManagement; + } +} diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/master/GuessResource.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/master/GuessResource.java new file mode 100644 index 0000000000..499007085c --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/master/GuessResource.java @@ -0,0 +1,102 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.rest.master; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.connect.management.AdapterDeserializer; +import org.streampipes.connect.management.master.AdapterMasterManagement; +import org.streampipes.connect.management.master.GuessManagement; +import org.streampipes.connect.rest.AbstractContainerResource; +import org.streampipes.model.client.messages.Notifications; +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.model.connect.adapter.AdapterSetDescription; +import org.streampipes.model.connect.adapter.AdapterStreamDescription; +import org.streampipes.model.connect.guess.GuessSchema; +import org.streampipes.model.schema.EventPropertyPrimitive; +import org.streampipes.model.schema.EventSchema; +import org.streampipes.rest.shared.annotation.JsonLdSerialized; +import org.streampipes.rest.shared.util.JsonLdUtils; +import org.streampipes.rest.shared.util.SpMediaType; +import org.streampipes.serializers.jsonld.JsonLdTransformer; + + +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.util.Arrays; + + +@Path("/api/v1/{username}/master/guess") +public class GuessResource extends AbstractContainerResource { + + private static final Logger logger = LoggerFactory.getLogger(GuessResource.class); + + private GuessManagement guessManagement; + + public GuessResource() { + this.guessManagement = new GuessManagement(); + } + + public GuessResource(GuessManagement guessManagement) { + this.guessManagement = guessManagement; + } + + @POST + @JsonLdSerialized + @Path("/schema") + @Produces(SpMediaType.JSONLD) + public Response guessSchema(String s, @PathParam("username") String userName) { + + try { + AdapterDescription adapterDescription = AdapterDeserializer.getAdapterDescription(s); + GuessSchema result = guessManagement.guessSchema(adapterDescription); + + return ok(result); + } catch (AdapterException e) { + logger.error("Could not deserialize AdapterDescription: " + s, e); + return fail(); + } + + } + + @GET + @Produces(SpMediaType.JSONLD) + @Path("/format") + public Response guessFormat() { + //TODO + return ok(true); + } + + + @GET + @Produces(SpMediaType.JSONLD) + @Path("/formatdescription") + public Response guessFormatDescription() { + //TODO + return ok(true); + } + + public void setGuessManagement(GuessManagement guessManagement) { + this.guessManagement = guessManagement; + } + +} + diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/master/SourcesResource.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/master/SourcesResource.java new file mode 100644 index 0000000000..3c81b849ff --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/master/SourcesResource.java @@ -0,0 +1,135 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.rest.master; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.connect.config.ConnectContainerConfig; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.connect.management.master.SourcesManagement; +import org.streampipes.connect.rest.AbstractContainerResource; +import org.streampipes.model.SpDataSet; +import org.streampipes.model.client.messages.Notifications; +import org.streampipes.model.graph.DataSourceDescription; +import org.streampipes.rest.shared.annotation.GsonWithIds; +import org.streampipes.rest.shared.annotation.JsonLdSerialized; +import org.streampipes.rest.shared.util.JsonLdUtils; +import org.streampipes.rest.shared.util.SpMediaType; +import org.streampipes.vocabulary.StreamPipes; + +import javax.json.Json; +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +@Path("/api/v1/{username}/master/sources") +public class SourcesResource extends AbstractContainerResource { + + private Logger logger = LoggerFactory.getLogger(SourcesResource.class); + + private String connectContainerBaseUrl; + + private SourcesManagement sourcesManagement; + + public SourcesResource() { + this.connectContainerBaseUrl = ConnectContainerConfig.INSTANCE.getConnectContainerMasterUrl(); + this.sourcesManagement = new SourcesManagement(); + } + + public SourcesResource(String connectContainerBaseUrl) { + this.connectContainerBaseUrl = connectContainerBaseUrl; + } + + + @GET + @Produces(MediaType.APPLICATION_JSON) + @GsonWithIds + @Path("/") + public Response getAllAdaptersInstallDescription(@PathParam("username") String username) { + + try { + String resultingJson = this.sourcesManagement.getAllAdaptersInstallDescription(username); + return ok(resultingJson); + } catch (AdapterException e) { + logger.error("Error while getting all adapter descriptions", e); + return fail(); + } + } + + + + @GET + @Path("/{id}") + @JsonLdSerialized + @Produces(SpMediaType.JSONLD) + public Response getAdapterDataSource(@PathParam("id") String id) { + + try { + DataSourceDescription result = this.sourcesManagement.getAdapterDataSource(id); + return ok(result); + } catch (AdapterException e) { + logger.error("Error while retrieving DataSourceDescription with id: " + id); + return fail(); + } + } + + + + @POST +// @JsonLdSerialized + @Consumes(SpMediaType.JSONLD) + @Path("/{streamId}/streams") + @Produces(MediaType.APPLICATION_JSON) + public Response addAdapter(@PathParam("streamId") String elementId, String dataSetSet) { + + SpDataSet dataSet = JsonLdUtils.fromJsonLd(dataSetSet, SpDataSet.class, StreamPipes.DATA_SET); + + String responseMessage = "Instance of data set " + dataSet.getUri() + " successfully started"; + + try { + this.sourcesManagement.addAdapter(elementId, this.connectContainerBaseUrl, dataSet); + } catch (AdapterException e) { + logger.error("Could not set data set instance: " + dataSet.getUri(), e); + return ok(Notifications.error("Could not set data set instance: " + dataSet.getUri())); + } + + + return ok(Notifications.success(responseMessage)); + } + + @DELETE + @Path("/{streamId}/streams/{runningInstanceId}") + @Produces(MediaType.APPLICATION_JSON) + public Response detach(@PathParam("streamId") String elementId, @PathParam("runningInstanceId") String runningInstanceId) { + String responseMessage = "Instance of set id: " + elementId + " with instance id: "+ runningInstanceId + " successfully started"; + + try { + this.sourcesManagement.detachAdapter(this.connectContainerBaseUrl, elementId, runningInstanceId); + } catch (AdapterException e) { + logger.error("Could not set set id "+ elementId + " with instance id: "+ runningInstanceId, e); + return fail(); + } + + + return ok(Notifications.success(responseMessage)); + } + + public void setSourcesManagement(SourcesManagement sourcesManagement) { + this.sourcesManagement = sourcesManagement; + } +} diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/master/WelcomePageMaster.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/master/WelcomePageMaster.java new file mode 100644 index 0000000000..6d139734f7 --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/master/WelcomePageMaster.java @@ -0,0 +1,106 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.rest.master; + + +import org.rendersnake.HtmlCanvas; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.connect.adapter.GroundingService; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.connect.management.master.AdapterMasterManagement; +import org.streampipes.connect.rest.AbstractContainerResource; +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.storage.couchdb.impl.AdapterStorageImpl; +import org.streampipes.storage.couchdb.utils.CouchDbConfig; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import java.io.IOException; +import java.util.List; + + +@Path("/") +public class WelcomePageMaster extends AbstractContainerResource { + + private Logger logger = LoggerFactory.getLogger(WelcomePageMaster.class); + + private AdapterMasterManagement adapterMasterManagement; + + public WelcomePageMaster() { + this.adapterMasterManagement = new AdapterMasterManagement(); + } + + public WelcomePageMaster(AdapterMasterManagement adapterMasterManagement) { + this.adapterMasterManagement = adapterMasterManagement; + } + @GET + @Produces(MediaType.TEXT_HTML) + public String getWelcomePageHtml() { + return buildHtml(); + } + + private String buildHtml() { + HtmlCanvas html = new HtmlCanvas(); + try { + HtmlCanvas tmp = html + .head() + .title() + .content("StreamPipes Connector Master Container") + ._head() + .body() + .h1().write("Connector Master Container")._h1() + .h2().write("All Running Adapters")._h2() + .ol(); + + tmp = getAllRunningAdapters(tmp); + + html = tmp._ol() + ._body(); + } catch (IOException e) { + logger.error("Error in SP Connect Master Container: ", e); + } + + return html.toHtml(); + } + + private HtmlCanvas getAllRunningAdapters(HtmlCanvas canvas) throws IOException { + + List allAdapterDescriptions = null; + try { + allAdapterDescriptions = this.adapterMasterManagement.getAllAdapters(new AdapterStorageImpl()); + } catch (AdapterException e) { + logger.error("Could not connect to couchdb on URL: " + CouchDbConfig.INSTANCE.getHost(), e); + canvas.li().write("Error while connecting to CouchDB on Host: " + CouchDbConfig.INSTANCE.getHost())._li(); + return canvas; + } + + for (AdapterDescription ad : allAdapterDescriptions) { + canvas.li().write(ad.getAdapterId())._li(); + canvas.ul().li().write("Kafka Topic: " + GroundingService.extractTopic(ad))._li()._ul(); + } + + return canvas; + } + + public void setAdapterMasterManagement(AdapterMasterManagement adapterMasterManagement) { + this.adapterMasterManagement = adapterMasterManagement; + } +} diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/worker/WelcomePageWorker.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/worker/WelcomePageWorker.java new file mode 100644 index 0000000000..e1c172d5c0 --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/worker/WelcomePageWorker.java @@ -0,0 +1,65 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.rest.worker; + + +import org.rendersnake.HtmlCanvas; +import org.streampipes.connect.rest.AbstractContainerResource; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import java.io.IOException; + +@Path("/") +public class WelcomePageWorker extends AbstractContainerResource { + + private String id; + + public WelcomePageWorker() { + this.id = "Worker01"; + } + + public WelcomePageWorker(String id) { + this.id = id; + } + + @GET + @Produces(MediaType.TEXT_HTML) + public String getWelcomePageHtml() { + return buildHtml(); + } + + private String buildHtml() { + HtmlCanvas html = new HtmlCanvas(); + try { + html + .head() + .title() + .content("StreamPipes Connector Worker Container") + ._head() + .body() + .h1().write("Worker Connector Container with ID: " + this.id)._h1() + ._body(); + } catch (IOException e) { + e.printStackTrace(); + } + return html.toHtml(); + } +} diff --git a/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/worker/WorkerResource.java b/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/worker/WorkerResource.java new file mode 100644 index 0000000000..15897c6a25 --- /dev/null +++ b/streampipes-connect-container/src/main/java/org/streampipes/connect/rest/worker/WorkerResource.java @@ -0,0 +1,153 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.rest.worker; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.connect.management.AdapterDeserializer; +import org.streampipes.connect.management.worker.AdapterWorkerManagement; +import org.streampipes.connect.rest.AbstractContainerResource; +import org.streampipes.model.client.messages.Notifications; +import org.streampipes.model.connect.adapter.AdapterSetDescription; +import org.streampipes.model.connect.adapter.AdapterStreamDescription; +import org.streampipes.rest.shared.annotation.JsonLdSerialized; + +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +@Path("/api/v1/{username}/worker") +public class WorkerResource extends AbstractContainerResource { + + private static final Logger logger = LoggerFactory.getLogger(WorkerResource.class); + + private AdapterWorkerManagement adapterManagement; + + public WorkerResource() { + adapterManagement = new AdapterWorkerManagement(); + } + + public WorkerResource(AdapterWorkerManagement adapterManagement) { + this.adapterManagement = adapterManagement; + } + + @POST + @JsonLdSerialized + @Path("/stream/invoke") +// @Consumes(SpMediaType.JSONLD) + @Produces(MediaType.APPLICATION_JSON) + public Response invokeStreamAdapter(String adapterDescription) { + + AdapterStreamDescription adapterStreamDescription = null; + + try { + adapterStreamDescription = (AdapterStreamDescription) + AdapterDeserializer.getAdapterDescription(adapterDescription); + + adapterManagement.invokeStreamAdapter(adapterStreamDescription); + } catch (AdapterException e) { + logger.error("Error while starting adapter with id " + adapterStreamDescription.getUri(), e); + return ok(Notifications.error(e.getMessage())); + } + String responseMessage = "Stream adapter with id " + adapterStreamDescription.getUri() + " successfully started"; + + logger.info(responseMessage); + return ok(Notifications.success(responseMessage)); + } + + @POST + @JsonLdSerialized + @Path("/stream/stop") +// @Consumes(SpMediaType.JSONLD) + @Produces(MediaType.APPLICATION_JSON) + public Response stopStreamAdapter(String adapterDescription) { + + AdapterStreamDescription adapterStreamDescription = null; + + try { + adapterStreamDescription = (AdapterStreamDescription) + AdapterDeserializer.getAdapterDescription(adapterDescription); + + adapterManagement.stopStreamAdapter(adapterStreamDescription); + } catch (AdapterException e) { + logger.error("Error while stopping adapter with id " + adapterStreamDescription.getUri(), e); + return ok(Notifications.error(e.getMessage())); + } + + String responseMessage = "Stream adapter with id " + adapterStreamDescription.getUri() + " successfully stopped"; + + logger.info(responseMessage); + return ok(Notifications.success(responseMessage)); + } + + @POST + @JsonLdSerialized + @Path("/set/invoke") +// @Consumes(SpMediaType.JSONLD) + @Produces(MediaType.APPLICATION_JSON) + public Response invokeSetAdapter(String adapterDescription) { + + AdapterSetDescription adapterSetDescription = null; + + try { + adapterSetDescription = (AdapterSetDescription) + AdapterDeserializer.getAdapterDescription(adapterDescription); + + adapterManagement.invokeSetAdapter(adapterSetDescription); + } catch (AdapterException e) { + logger.error("Error while starting adapter with id " + adapterSetDescription.getUri(), e); + return ok(Notifications.error(e.getMessage())); + } + + String responseMessage = "Set adapter with id " + adapterSetDescription.getUri() + " successfully started"; + + logger.info(responseMessage); + return ok(Notifications.success(responseMessage)); + } + + @POST + @JsonLdSerialized + @Path("/set/stop") +// @Consumes(SpMediaType.JSONLD) + @Produces(MediaType.APPLICATION_JSON) + public Response stopSetAdapter(String adapterDescription){ + + AdapterSetDescription adapterSetDescription = null; + + try { + adapterSetDescription = (AdapterSetDescription) + AdapterDeserializer.getAdapterDescription(adapterDescription); + + adapterManagement.stopSetAdapter(adapterSetDescription); + } catch (AdapterException e) { + logger.error("Error while stopping adapter with id " + adapterSetDescription.getUri(), e); + return ok(Notifications.error(e.getMessage())); + } + + String responseMessage = "Set adapter with id " + adapterSetDescription.getUri() + " successfully stopped"; + + logger.info(responseMessage); + return ok(Notifications.success(responseMessage)); + } + + public void setAdapterManagement(AdapterWorkerManagement adapterManagement) { + this.adapterManagement = adapterManagement; + } + +} diff --git a/streampipes-connect-container/src/test/SpConnectResourceTest.java b/streampipes-connect-container/src/test/SpConnectResourceTest.java new file mode 100644 index 0000000000..ae1891ca84 --- /dev/null +++ b/streampipes-connect-container/src/test/SpConnectResourceTest.java @@ -0,0 +1,128 @@ +package org.streampipes.rest.impl.connect; + +import com.jayway.restassured.RestAssured; +import org.eclipse.jetty.server.Server; +import org.glassfish.jersey.jetty.JettyHttpContainerFactory; +import org.glassfish.jersey.server.ResourceConfig; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.streampipes.model.SpDataSet; +import org.streampipes.model.connect.adapter.AdapterStreamDescription; +import org.streampipes.rest.Mock; +import org.streampipes.rest.TestUtil; +import org.streampipes.storage.couchdb.impl.AdapterStorageImpl; + +import javax.ws.rs.core.UriBuilder; +import java.net.URI; + +import static com.jayway.restassured.RestAssured.given; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; + +public class SpConnectResourceTest { + + SpConnectResource spConnectResource; + Server server; + + @Before + public void before() { + + spConnectResource = new SpConnectResource(new SpConnect(), ""); + RestAssured.port = Mock.PORT; + + ResourceConfig config = new ResourceConfig().register(spConnectResource); + + URI baseUri = UriBuilder + .fromUri(Mock.HOST) + .build(); + + server = JettyHttpContainerFactory.createServer(baseUri, config); + } + + @After + public void after() { + try { + server.stop(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Test + public void addAdapterSuccess() { + + // Mock adatper management + SpConnect spConnect = mock(SpConnect.class); + org.mockito.Mockito.when(spConnect.addAdapter(any(AdapterStreamDescription.class), any(String.class), any())) + .thenReturn(SpConnectUtils.SUCCESS); + spConnectResource.setSpConnect(spConnect); + + // perform test + String data = TestUtil.getMinimalStreamAdapterJsonLD(); + given().contentType("application/json").body(data).when() + .post("v2/adapter").then().assertThat() + .body("success", equalTo(true)) + .body("optionalMessage", equalTo("")) + .body("elementId", equalTo("http://test.de/1")); + } + + @Test + public void addAdapterFail() { + + String errorMessage = "ERROR"; + + SpConnect spConnect = mock(SpConnect.class); + org.mockito.Mockito.when(spConnect.addAdapter(any(AdapterStreamDescription.class), any(String.class), any())) + .thenReturn(errorMessage); + spConnectResource.setSpConnect(spConnect); + spConnectResource.setConnectContainerEndpoint(Mock.HOST); + + SpConnect backendConfig = mock(SpConnect.class); + + String data = TestUtil.getMinimalStreamAdapterJsonLD(); + given().contentType("application/json").body(data).when() + .post("v2/adapter").then().assertThat() + .body("success", equalTo(false)) + .body("optionalMessage", equalTo(errorMessage)) + .body("elementId", equalTo("http://test.de/1")); + } + + @Test + public void invokeAdapterSuccess() { + + SpConnect spConnect = mock(SpConnect.class); + org.mockito.Mockito.when(spConnect.invokeAdapter(any(String.class), any(SpDataSet.class), any(String.class), + any(AdapterStorageImpl.class))) + .thenReturn(SpConnectUtils.SUCCESS); + spConnectResource.setSpConnect(spConnect); + + String data = TestUtil.getMinimalDataSet(); + given().contentType("application/json").body(data).when() + .post("v2/adapter/all/1234/streams").then().assertThat() + .body("success", equalTo(true)) + .body("optionalMessage", equalTo("")) + .body("elementId", equalTo("1234")); + } + + @Test + public void invokeAdapterFail() { + + String errorMessage = "ERROR"; + + SpConnect spConnect = mock(SpConnect.class); + org.mockito.Mockito.when(spConnect.invokeAdapter(any(String.class), any(SpDataSet.class), any(String.class), + any(AdapterStorageImpl.class))) + .thenReturn(errorMessage); + spConnectResource.setSpConnect(spConnect); + + String data = TestUtil.getMinimalDataSet(); + given().contentType("application/json").body(data).when() + .post("v2/adapter/all/1234/streams").then().assertThat() + .body("success", equalTo(false)) + .body("optionalMessage", equalTo(errorMessage)) + .body("elementId", equalTo("1234")); + } + +} \ No newline at end of file diff --git a/streampipes-connect-container/src/test/SpConnectTest.java b/streampipes-connect-container/src/test/SpConnectTest.java new file mode 100644 index 0000000000..589c5a5d89 --- /dev/null +++ b/streampipes-connect-container/src/test/SpConnectTest.java @@ -0,0 +1,187 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.rest.impl.connect; + +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.streampipes.model.Response; +import org.streampipes.model.SpDataSet; +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.model.connect.adapter.AdapterSetDescription; +import org.streampipes.model.connect.adapter.AdapterStreamDescription; +import org.streampipes.model.schema.EventSchema; +import org.streampipes.rest.Mock; +import org.streampipes.rest.TestUtil; +import org.streampipes.storage.couchdb.impl.AdapterStorageImpl; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; + +public class SpConnectTest { + + @Rule + public WireMockRule wireMockRule = new WireMockRule(Mock.PORT); + + @Before + public void before() { + } + + @Test + public void isStreamAdapterSuccess() { + AdapterStorageImpl adapterStorage = mock(AdapterStorageImpl.class); + org.mockito.Mockito.when(adapterStorage.getAdapter(org.mockito.ArgumentMatchers.any(String.class))) + .thenReturn(new AdapterStreamDescription()); + + + assertTrue(SpConnect.isStreamAdapter("", adapterStorage)); + } + + @Test + public void isStreamAdapterFail() { + AdapterStorageImpl adapterStorage = mock(AdapterStorageImpl.class); + org.mockito.Mockito.when(adapterStorage.getAdapter(org.mockito.ArgumentMatchers.any(String.class))) + .thenReturn(new AdapterSetDescription()); + + assertFalse(SpConnect.isStreamAdapter("", adapterStorage)); + } + + + @Test + public void getAdapterStreamDescriptionWithoutType() { + AdapterDescription asd = SpConnect + .getAdapterDescription(TestUtil.getMinimalStreamAdapterJsonLD()); + + assertTrue(asd instanceof AdapterStreamDescription); + } + + @Test + public void getAdapterSetDescriptionWithoutType() { + AdapterDescription asd = SpConnect. + getAdapterDescription(TestUtil.getMinimalSetAdapterJsonLD()); + + assertTrue(asd instanceof AdapterSetDescription); + } + + @Test + public void startStreamAdapterTest() { + // expected http request to connect-container /invoke/stream + Response expected = new Response("id",true); + stubFor(post(urlEqualTo("/api/v1/invoke/stream")) + .willReturn(aResponse() + .withStatus(200) + .withBody(expected.toString()))); + + AdapterStreamDescription adapter = new AdapterStreamDescription(); + adapter.setUri("http://test.de/1"); + + String result = SpConnect.startStreamAdapter(adapter, Mock.HOST + "/"); + + assertEquals(SpConnectUtils.SUCCESS, result); + verify(postRequestedFor(urlEqualTo("/api/v1/invoke/stream")) + .withHeader("Content-Type", equalTo("application/json; charset=UTF-8"))); + } + + @Test + public void invokeAdapterTest() { + Response expected = new Response("id",true); + stubFor(post(urlEqualTo("/api/v1/invoke/set")) + .willReturn(aResponse() + .withStatus(200) + .withBody(expected.toString()))); + + AdapterSetDescription adapterSetDescription = new AdapterSetDescription(); + adapterSetDescription.setUri("http://test.adapter"); + AdapterStorageImpl adapterStorage = mock(AdapterStorageImpl.class); + org.mockito.Mockito.when(adapterStorage.getAdapter(org.mockito.ArgumentMatchers.any(String.class))) + .thenReturn(adapterSetDescription); + + + SpDataSet dataSet = new SpDataSet("http://one.de", "name", "desc", new EventSchema()); + + SpConnect spConnect = new SpConnect(); + String result = spConnect.invokeAdapter("1234", dataSet, Mock.HOST + "/", adapterStorage); + + assertEquals(SpConnectUtils.SUCCESS, result); + verify(postRequestedFor(urlEqualTo("/api/v1/invoke/set")) + .withHeader("Content-Type", equalTo("application/json; charset=UTF-8"))); + + } + + @Test + public void stopStreamAdapterTest() { + Response expected = new Response("id",true); + stubFor(post(urlEqualTo("/api/v1/stopAdapter/stream")) + .willReturn(aResponse() + .withStatus(200) + .withBody(expected.toString()))); + + AdapterDescription adapterDescription = new AdapterSetDescription(); + adapterDescription.setUri("http://test.adapter"); + AdapterStorageImpl adapterStorage = mock(AdapterStorageImpl.class); + org.mockito.Mockito.when(adapterStorage.getAdapter(org.mockito.ArgumentMatchers.any(String.class))) + .thenReturn(adapterDescription); + + + SpConnect spConnect = new SpConnect(); + String result = spConnect.stopStreamAdapter("1234",Mock.HOST + "/", adapterStorage); + + assertEquals(SpConnectUtils.SUCCESS, result); + verify(postRequestedFor(urlEqualTo("/api/v1/stopAdapter/stream")) + .withHeader("Content-Type", equalTo("application/json; charset=UTF-8"))); + } + + @Test + public void stopSetAdapterTest() { + Response expected = new Response("id",true); + stubFor(post(urlEqualTo("/api/v1/stopAdapter/set")) + .willReturn(aResponse() + .withStatus(200) + .withBody(expected.toString()))); + + AdapterDescription adapterDescription = new AdapterSetDescription(); + adapterDescription.setUri("http://test.adapter"); + AdapterStorageImpl adapterStorage = mock(AdapterStorageImpl.class); + org.mockito.Mockito.when(adapterStorage.getAdapter(org.mockito.ArgumentMatchers.any(String.class))) + .thenReturn(adapterDescription); + + + SpConnect spConnect = new SpConnect(); + String result = spConnect.stopSetAdapter("1234",Mock.HOST + "/", adapterStorage); + + assertEquals(SpConnectUtils.SUCCESS, result); + verify(postRequestedFor(urlEqualTo("/api/v1/stopAdapter/set")) + .withHeader("Content-Type", equalTo("application/json; charset=UTF-8"))); + } + +// @Test +// public void installDatasource() { +//// stubFor(post(urlEqualTo("/api/v1/stopAdapter/set")) +//// .willReturn(aResponse() +//// .withStatus(200) +//// .wit +//// .withBody(expected.toString()))); +// +// SpConnect spConnect = new SpConnect(); +// boolean result = spConnect.installDatasource(); +// +// assertEquals(true, result); +// } +} \ No newline at end of file diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/Mock.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/Mock.java new file mode 100644 index 0000000000..e5b7d5ab57 --- /dev/null +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/Mock.java @@ -0,0 +1,25 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect; + +public class Mock { + public static int PORT = 8042; + + public static String HOST = "http://localhost:" + PORT; + +} diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/management/AdapterDeserializerTest.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/management/AdapterDeserializerTest.java new file mode 100644 index 0000000000..e8003ba9d2 --- /dev/null +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/management/AdapterDeserializerTest.java @@ -0,0 +1,200 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.management; + +import org.junit.Test; +import org.streampipes.connect.adapter.generic.format.json.object.JsonObjectFormat; +import org.streampipes.connect.adapter.generic.format.json.object.JsonObjectParser; +import org.streampipes.connect.adapter.generic.format.xml.XmlFormat; +import org.streampipes.connect.adapter.generic.format.xml.XmlParser; +import org.streampipes.connect.adapter.generic.protocol.set.HttpProtocol; +import org.streampipes.connect.adapter.generic.protocol.stream.KafkaProtocol; +import org.streampipes.connect.adapter.specific.twitter.TwitterAdapter; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.model.SpDataSet; +import org.streampipes.model.connect.adapter.*; +import org.streampipes.model.connect.grounding.FormatDescription; +import org.streampipes.model.connect.grounding.ProtocolDescription; +import org.streampipes.rest.shared.util.JsonLdUtils; +import org.streampipes.sdk.builder.DataSetBuilder; + +import static org.junit.Assert.*; +import static org.reflections.util.ConfigurationBuilder.build; + +public class AdapterDeserializerTest { + + @Test + public void getGenericAdapterStreamDescription() throws AdapterException { + AdapterDescription genericAdapterStreamDescription = new GenericAdapterStreamDescription(); + String jsonLd = JsonLdUtils.toJsonLD(genericAdapterStreamDescription); + + AdapterDescription a = AdapterDeserializer.getAdapterDescription(jsonLd); + + assertTrue(a instanceof GenericAdapterStreamDescription); + assertEquals(GenericAdapterStreamDescription.ID, a.getUri()); + } + + @Test + public void getGenericAdapterSetDescription() throws AdapterException { + AdapterDescription genericAdapterSetDescription = new GenericAdapterSetDescription(); + String jsonLd = JsonLdUtils.toJsonLD(genericAdapterSetDescription); + + AdapterDescription a = AdapterDeserializer.getAdapterDescription(jsonLd); + + assertTrue(a instanceof GenericAdapterSetDescription); + assertEquals(GenericAdapterSetDescription.ID, a.getUri()); + } + + @Test + public void getSpecificAdapterStreamDescription() throws AdapterException { + AdapterDescription specificAdapterStreamDescription = new TwitterAdapter().declareModel(); + String jsonLd = JsonLdUtils.toJsonLD(specificAdapterStreamDescription); + + AdapterDescription a = AdapterDeserializer.getAdapterDescription(jsonLd); + + assertTrue(a instanceof SpecificAdapterStreamDescription); + assertEquals(TwitterAdapter.ID, a.getUri()); + } + + @Test + public void getFormatDescriptionHttpProtocolXmlFormat() throws AdapterException { + AdapterDescription genericAdapterStreamDescription = new GenericAdapterStreamDescription(); + + XmlFormat xmlFormat = new XmlFormat("row"); + HttpProtocol httpProtocol = new HttpProtocol(new XmlParser("row"), xmlFormat, "URL"); + + FormatDescription formatDescription = xmlFormat.declareModel(); + ProtocolDescription protocolDescription = httpProtocol.declareModel(); + + + ((GenericAdapterStreamDescription) genericAdapterStreamDescription).setProtocolDescription(protocolDescription); + ((GenericAdapterStreamDescription) genericAdapterStreamDescription).setFormatDescription(formatDescription); + + String jsonLd = JsonLdUtils.toJsonLD(genericAdapterStreamDescription); + + AdapterDescription a = AdapterDeserializer.getAdapterDescription(jsonLd); + + assertNotNull(((GenericAdapterStreamDescription) a).getFormatDescription()); + } + + @Test + public void getFormatDescriptionKafkaProtocolJsobObjectFormat() throws AdapterException { + AdapterDescription genericAdapterStreamDescription = new GenericAdapterStreamDescription(); + + JsonObjectFormat jsonObjectFormat = new JsonObjectFormat(); + KafkaProtocol kafkaProtocol = new KafkaProtocol(new JsonObjectParser(), jsonObjectFormat, "URL", "broker"); + + FormatDescription formatDescription = jsonObjectFormat.declareModel(); + ProtocolDescription protocolDescription = kafkaProtocol.declareModel(); + + ((GenericAdapterStreamDescription) genericAdapterStreamDescription).setProtocolDescription(protocolDescription); + ((GenericAdapterStreamDescription) genericAdapterStreamDescription).setFormatDescription(formatDescription); + + String jsonLd = JsonLdUtils.toJsonLD(genericAdapterStreamDescription); + + System.out.println(jsonLd); + AdapterDescription a = AdapterDeserializer.getAdapterDescription(jsonLd); + + assertNotNull(((GenericAdapterStreamDescription) a).getFormatDescription()); + } + + @Test + public void testAdapterSetDeserialization() throws AdapterException { + String jsonld = getGenericAdapterSetDescripionGeneratedByFrontend(); + AdapterDescription a = AdapterDeserializer.getAdapterDescription(jsonld); + + + assertNotNull(((GenericAdapterSetDescription) a).getFormatDescription()); + assertNotNull(((GenericAdapterSetDescription) a).getProtocolDescription()); + + } + + private String getGenericAdapterSetDescripionGeneratedByFrontend() { + return "{ \n" + + " \"@context\":{ \n" + + " \"sp\":\"https://streampipes.org/vocabulary/v1/\",\n" + + " \"spi\":\"urn:streampipes.org:spi:\",\n" + + " \"foaf\":\"http://xmlns.com/foaf/0.1/\"\n" + + " },\n" + + " \"@graph\":[ \n" + + " { \n" + + " \"@id\":\"http://streampipes.org/dataset/69165e7e-e837-2b48-455b-909be67abb7e\",\n" + + " \"@type\":\"sp:DataSet\"\n" + + " },\n" + + " { \n" + + " \"@id\":\"http://streampipes.org/genericadaptersetdescription\",\n" + + " \"@type\":\"sp:GenericAdapterSetDescription\",\n" + + " \"http://www.w3.org/2000/01/rdf-schema#description\":\"This is the description for the http protocol\",\n" + + " \"http://www.w3.org/2000/01/rdf-schema#label\":\"HTTP (Set)\",\n" + + " \"sp:config\":[ \n" + + "\n" + + " ],\n" + + " \"sp:hasDataSet\":{ \n" + + " \"@id\":\"http://streampipes.org/dataset/69165e7e-e837-2b48-455b-909be67abb7e\"\n" + + " },\n" + + " \"sp:hasFormat\":{ \n" + + " \"@id\":\"sp:format/xml\"\n" + + " },\n" + + " \"sp:hasProtocol\":{ \n" + + " \"@id\":\"sp:protocol/set/http\"\n" + + " },\n" + + " \"sp:hasUri\":\"http://streampipes.org/genericadaptersetdescription\"\n" + + " },\n" + + " { \n" + + " \"@id\":\"sp:format/xml\",\n" + + " \"@type\":\"sp:FormatDescription\",\n" + + " \"http://www.w3.org/2000/01/rdf-schema#description\":\"This is the description for the XML format\",\n" + + " \"http://www.w3.org/2000/01/rdf-schema#label\":\"XML\",\n" + + " \"sp:config\":{ \n" + + " \"@id\":\"spi:freetextstaticproperty:CEVrUb\"\n" + + " },\n" + + " \"sp:hasUri\":\"https://streampipes.org/vocabulary/v1/format/xml\"\n" + + " },\n" + + " { \n" + + " \"@id\":\"sp:protocol/set/http\",\n" + + " \"@type\":\"sp:ProtocolDescription\",\n" + + " \"http://www.w3.org/2000/01/rdf-schema#description\":\"This is the description for the http protocol\",\n" + + " \"http://www.w3.org/2000/01/rdf-schema#label\":\"HTTP (Set)\",\n" + + " \"sp:config\":{ \n" + + " \"@id\":\"spi:freetextstaticproperty:dbiWfF\"\n" + + " },\n" + + " \"sp:hasUri\":\"https://streampipes.org/vocabulary/v1/protocol/set/http\",\n" + + " \"sp:sourceType\":\"SET\"\n" + + " },\n" + + " { \n" + + " \"@id\":\"spi:freetextstaticproperty:CEVrUb\",\n" + + " \"@type\":\"sp:FreeTextStaticProperty\",\n" + + " \"http://www.w3.org/2000/01/rdf-schema#description\":\"The Tag name of the events\",\n" + + " \"http://www.w3.org/2000/01/rdf-schema#label\":\"Tag\",\n" + + " \"sp:hasValue\":\"asd\",\n" + + " \"sp:internalName\":\"tag\",\n" + + " \"sp:requiredDomainProperty\":\"\"\n" + + " },\n" + + " { \n" + + " \"@id\":\"spi:freetextstaticproperty:dbiWfF\",\n" + + " \"@type\":\"sp:FreeTextStaticProperty\",\n" + + " \"http://www.w3.org/2000/01/rdf-schema#description\":\"This property defines the URL for the http request.\",\n" + + " \"http://www.w3.org/2000/01/rdf-schema#label\":\"url\",\n" + + " \"sp:hasValue\":\"sdf\",\n" + + " \"sp:internalName\":\"url\",\n" + + " \"sp:requiredDomainProperty\":\"\"\n" + + " }\n" + + " ]\n" + + "}"; + } +} \ No newline at end of file diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/management/AdapterMasterManagementTest.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/management/AdapterMasterManagementTest.java new file mode 100644 index 0000000000..2df59ec13c --- /dev/null +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/management/AdapterMasterManagementTest.java @@ -0,0 +1,96 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.management; + +import org.junit.Test; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.connect.management.master.AdapterMasterManagement; +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.model.connect.adapter.AdapterStreamDescription; +import org.streampipes.model.connect.adapter.GenericAdapterStreamDescription; +import org.streampipes.storage.couchdb.impl.AdapterStorageImpl; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class AdapterMasterManagementTest { + @Test + public void getAdapterSuccess() throws AdapterException { + String id = GenericAdapterStreamDescription.ID; + List adapterDescriptions = Arrays.asList(new GenericAdapterStreamDescription()); + AdapterStorageImpl adapterStorage = mock(AdapterStorageImpl.class); + when(adapterStorage.getAllAdapters()).thenReturn(adapterDescriptions); + + AdapterMasterManagement adapterMasterManagement = new AdapterMasterManagement(); + + AdapterDescription result = adapterMasterManagement.getAdapter(id, adapterStorage); + + assertEquals(id, result.getUri()); + } + + @Test(expected = AdapterException.class) + public void getAdapterFailNull() throws AdapterException { + AdapterStorageImpl adapterStorage = mock(AdapterStorageImpl.class); + when(adapterStorage.getAllAdapters()).thenReturn(null); + + AdapterMasterManagement adapterMasterManagement = new AdapterMasterManagement(); + + adapterMasterManagement.getAdapter("id2", adapterStorage); + } + + @Test(expected = AdapterException.class) + public void getAdapterFail() throws AdapterException { + List adapterDescriptions = Arrays.asList(new GenericAdapterStreamDescription()); + AdapterStorageImpl adapterStorage = mock(AdapterStorageImpl.class); + when(adapterStorage.getAllAdapters()).thenReturn(adapterDescriptions); + + String id = "http://t.id"; + AdapterMasterManagement adapterMasterManagement = new AdapterMasterManagement(); + + adapterMasterManagement.getAdapter("id2", adapterStorage); + } + + @Test + public void getAllAdaptersSuccess() throws AdapterException { + List adapterDescriptions = Arrays.asList(new GenericAdapterStreamDescription()); + AdapterStorageImpl adapterStorage = mock(AdapterStorageImpl.class); + when(adapterStorage.getAllAdapters()).thenReturn(adapterDescriptions); + + AdapterMasterManagement adapterMasterManagement = new AdapterMasterManagement(); + + List result = adapterMasterManagement.getAllAdapters(adapterStorage); + + assertEquals(1, result.size()); + assertEquals(GenericAdapterStreamDescription.ID, result.get(0).getUri()); + } + + @Test(expected = AdapterException.class) + public void getAllAdaptersFail() throws AdapterException { + AdapterStorageImpl adapterStorage = mock(AdapterStorageImpl.class); + when(adapterStorage.getAllAdapters()).thenReturn(null); + + AdapterMasterManagement adapterMasterManagement = new AdapterMasterManagement(); + + adapterMasterManagement.getAllAdapters(adapterStorage); + + } +} \ No newline at end of file diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/management/AdapterUtilsTest.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/management/AdapterUtilsTest.java new file mode 100644 index 0000000000..d90484f578 --- /dev/null +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/management/AdapterUtilsTest.java @@ -0,0 +1,60 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.management; + + +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import org.junit.Rule; +import org.junit.Test; +import org.streampipes.connect.Mock; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; + +import static org.junit.Assert.*; + +public class AdapterUtilsTest { + @Rule + public WireMockRule wireMockRule = new WireMockRule(Mock.PORT); + + + @Test + public void stopPipeline() { + String expected = ""; + + stubFor(get(urlEqualTo("/")) + .willReturn(aResponse() + .withStatus(200) + .withBody(expected))); + + + String result = AdapterUtils.stopPipeline(Mock.HOST + ":" + Mock.PORT + "/"); + + assertEquals(expected, result); + + verify(getRequestedFor(urlMatching("/"))); + } + + @Test + public void getUrlTest() { + String expected = "http://host:80/api/v2/pipelines/1/stopAdapter"; + + String result = AdapterUtils.getUrl("host:80/", "1"); + + assertEquals(expected, result); + } +} \ No newline at end of file diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/management/AdapterWorkerManagementTest.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/management/AdapterWorkerManagementTest.java new file mode 100644 index 0000000000..7532354add --- /dev/null +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/management/AdapterWorkerManagementTest.java @@ -0,0 +1,157 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.management; + +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.any; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.streampipes.connect.RunningAdapterInstances; +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.connect.adapter.AdapterRegistry; +import org.streampipes.connect.adapter.specific.SpecificDataSetAdapter; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.connect.management.worker.AdapterWorkerManagement; +import org.streampipes.connect.utils.Utils; +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.model.connect.adapter.AdapterSetDescription; +import org.streampipes.model.connect.adapter.AdapterStreamDescription; +import org.streampipes.model.connect.adapter.GenericAdapterSetDescription; +import org.streampipes.model.connect.adapter.GenericAdapterStreamDescription; +import org.streampipes.model.connect.adapter.SpecificAdapterSetDescription; +import org.streampipes.model.connect.guess.GuessSchema; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ AdapterRegistry.class }) +public class AdapterWorkerManagementTest { + + @Test + public void startStreamAdapterSucess() throws AdapterException { + TestAdapter testAdapter = new TestAdapter(); + + PowerMockito.mockStatic(AdapterRegistry.class); + Mockito.when(AdapterRegistry.getAdapter(any(AdapterDescription.class))) + .thenReturn(testAdapter); + AdapterWorkerManagement adapterWorkerManagement = new AdapterWorkerManagement(); + + adapterWorkerManagement.invokeStreamAdapter(Utils.getMinimalStreamAdapter()); + + assertTrue(testAdapter.calledStart); + assertNotNull(RunningAdapterInstances.INSTANCE.removeAdapter("http://t.de/")); + } + + @Test + public void stopStreamAdapterFail() { + String expected = "Adapter with id http://test.de was not found in this container and cannot be stopped."; + AdapterStreamDescription asd = new GenericAdapterStreamDescription(); + asd.setUri("http://test.de"); + + AdapterWorkerManagement adapterManagement = new AdapterWorkerManagement(); + + try { + adapterManagement.stopStreamAdapter(asd); + fail(); + } catch (AdapterException e) { + assertEquals(expected, e.getMessage()); + } + } + + @Test + public void stopStreamAdapterSuccess() throws AdapterException { + TestAdapter testAdapter = new TestAdapter(); + RunningAdapterInstances.INSTANCE.addAdapter("http://t.de/", testAdapter); + AdapterWorkerManagement adapterWorkerManagement = new AdapterWorkerManagement(); + adapterWorkerManagement.stopStreamAdapter(Utils.getMinimalStreamAdapter()); + + assertTrue(testAdapter.calledStop); + + } + + @Test + public void stopSetAdapterFail() { + String expected = "Adapter with id http://test.de was not found in this container and cannot be stopped."; + AdapterSetDescription asd = new GenericAdapterSetDescription(); + asd.setUri("http://test.de"); + + AdapterWorkerManagement adapterManagement = new AdapterWorkerManagement(); + + try { + adapterManagement.stopSetAdapter(asd); + fail(); + } catch (AdapterException e) { + assertEquals(expected, e.getMessage()); + } + } + + @Test + public void stopSetAdapterSuccess() throws AdapterException { + TestAdapter testAdapter = new TestAdapter(); + RunningAdapterInstances.INSTANCE.addAdapter("http://t.de/", testAdapter); + AdapterWorkerManagement adapterWorkerManagement = new AdapterWorkerManagement(); + adapterWorkerManagement.stopSetAdapter(Utils.getMinimalSetAdapter()); + + assertTrue(testAdapter.calledStop); + } + + private class TestAdapter extends SpecificDataSetAdapter { + + public TestAdapter() { + super(null); + } + + public boolean calledStart = false; + public boolean calledStop = false; + + + + @Override + public SpecificAdapterSetDescription declareModel() { + return null; + } + + @Override + public void startAdapter() throws AdapterException { + calledStart = true; + } + + @Override + public void stopAdapter() throws AdapterException { + calledStop = true; + } + + @Override + public Adapter getInstance(SpecificAdapterSetDescription adapterDescription) { + return null; + } + + @Override + public GuessSchema getSchema(SpecificAdapterSetDescription adapterDescription) { + return null; + } + + @Override + public String getId() { + return null; + } + } +} \ No newline at end of file diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/management/DescriptionManagementTest.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/management/DescriptionManagementTest.java new file mode 100644 index 0000000000..1bea4a713b --- /dev/null +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/management/DescriptionManagementTest.java @@ -0,0 +1,103 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.management; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.connect.adapter.AdapterRegistry; +import org.streampipes.connect.adapter.generic.GenericDataSetAdapter; +import org.streampipes.connect.adapter.generic.format.Format; +import org.streampipes.connect.adapter.generic.format.json.arraykey.JsonFormat; +import org.streampipes.connect.adapter.generic.protocol.Protocol; +import org.streampipes.connect.adapter.generic.protocol.set.HttpProtocol; +import org.streampipes.connect.management.master.DescriptionManagement; +import org.streampipes.model.connect.adapter.AdapterDescriptionList; +import org.streampipes.model.connect.grounding.FormatDescriptionList; +import org.streampipes.model.connect.grounding.ProtocolDescriptionList; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.*; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ AdapterRegistry.class }) +public class DescriptionManagementTest { + + @Test + public void getProtocols() { + Map allProtocols = new HashMap<>(); + allProtocols.put(HttpProtocol.ID, new HttpProtocol()); + + PowerMockito.mockStatic(AdapterRegistry.class); + Mockito.when(AdapterRegistry.getAllProtocols()) + .thenReturn(allProtocols); + + DescriptionManagement descriptionManagement = new DescriptionManagement(); + + ProtocolDescriptionList result = descriptionManagement.getProtocols(); + + assertNotNull(result); + assertNotNull(result.getList()); + assertEquals(1, result.getList().size()); + assertEquals(HttpProtocol.ID, result.getList().get(0).getUri()); + } + + @Test + public void getFormats() { + Map allFormats = new HashMap<>(); + allFormats.put(JsonFormat.ID, new JsonFormat()); + + PowerMockito.mockStatic(AdapterRegistry.class); + Mockito.when(AdapterRegistry.getAllFormats()) + .thenReturn(allFormats); + + DescriptionManagement descriptionManagement = new DescriptionManagement(); + + FormatDescriptionList result = descriptionManagement.getFormats(); + + assertNotNull(result); + assertNotNull(result.getList()); + assertEquals(1, result.getList().size()); + assertEquals(JsonFormat.ID, result.getList().get(0).getUri()); + } + + @Test + public void getAdapters() { + Map allAdapters = new HashMap<>(); + allAdapters.put(GenericDataSetAdapter.ID, new GenericDataSetAdapter()); + + PowerMockito.mockStatic(AdapterRegistry.class); + Mockito.when(AdapterRegistry.getAllAdapters()) + .thenReturn(allAdapters); + + DescriptionManagement descriptionManagement = new DescriptionManagement(); + + AdapterDescriptionList result = descriptionManagement.getAdapters(); + + assertNotNull(result); + assertNotNull(result.getList()); + assertEquals(1, result.getList().size()); + assertEquals(GenericDataSetAdapter.ID, result.getList().get(0).getAdapterId()); + } +} \ No newline at end of file diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/management/master/GuessManagementTest.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/management/master/GuessManagementTest.java new file mode 100644 index 0000000000..71ec3d0421 --- /dev/null +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/management/master/GuessManagementTest.java @@ -0,0 +1,106 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.management.master; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.connect.adapter.AdapterRegistry; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.model.connect.guess.GuessSchema; +import org.streampipes.model.schema.EventPropertyPrimitive; +import org.streampipes.model.schema.EventSchema; + +import java.util.Arrays; + +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.any; +import static org.powermock.api.mockito.PowerMockito.when; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ AdapterRegistry.class }) +public class GuessManagementTest { + + @Before + public void before() { + PowerMockito.mockStatic(AdapterRegistry.class); + } + + + @Test + public void guessSchema() throws Exception { + when(AdapterRegistry.class, "getAdapter", any()).thenReturn(new TestAdapter()); + + GuessManagement guessManagement = new GuessManagement(); + + GuessSchema result = guessManagement.guessSchema(null); + + assertNotNull(result); + assertNotNull(result.eventSchema); + assertNotNull(result.eventSchema.getEventProperties()); + assertEquals(1, result.eventSchema.getEventProperties().size()); + assertEquals("id", result.eventSchema.getEventProperties().get(0).getRuntimeName()); + } + + private class TestAdapter extends Adapter { + + @Override + public GuessSchema getSchema(AdapterDescription adapterDescription) { + + EventSchema eventSchema = new EventSchema(); + EventPropertyPrimitive eventPropertyPrimitive = new EventPropertyPrimitive(); + eventPropertyPrimitive.setRuntimeType("http://schema.org/Number"); + eventPropertyPrimitive.setRuntimeName("id"); + + eventSchema.setEventProperties(Arrays.asList(eventPropertyPrimitive)); + GuessSchema guessSchema = new GuessSchema(); + guessSchema.setEventSchema(eventSchema); + return guessSchema; + } + + @Override + public AdapterDescription declareModel() { + return null; + } + + @Override + public void startAdapter() throws AdapterException { + + } + + @Override + public void stopAdapter() throws AdapterException { + + } + + @Override + public Adapter getInstance(AdapterDescription adapterDescription) { + return null; + } + + @Override + public String getId() { + return null; + } + } +} \ No newline at end of file diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/management/master/SourcesManagementTest.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/management/master/SourcesManagementTest.java new file mode 100644 index 0000000000..f9c820aab0 --- /dev/null +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/management/master/SourcesManagementTest.java @@ -0,0 +1,146 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.management.master; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.model.SpDataSet; +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.model.connect.adapter.AdapterSetDescription; +import org.streampipes.model.connect.adapter.GenericAdapterSetDescription; +import org.streampipes.storage.couchdb.impl.AdapterStorageImpl; + +import java.util.Arrays; + +import static groovy.xml.Entity.iexcl; +import static groovy.xml.Entity.times; +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; +import static org.powermock.api.mockito.PowerMockito.doNothing; +import static org.powermock.api.mockito.PowerMockito.verifyStatic; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ WorkerRestClient.class }) +public class SourcesManagementTest { + @Before + public void before() { + PowerMockito.mockStatic(WorkerRestClient.class); + } + + @Test + public void addAdapterSuccess() throws Exception { + AdapterStorageImpl adapterStorage = mock(AdapterStorageImpl.class); + when(adapterStorage.getAdapter(anyString())).thenReturn(new GenericAdapterSetDescription()); + SourcesManagement sourcesManagement = new SourcesManagement(adapterStorage); + doNothing().when(WorkerRestClient.class, "invokeSetAdapter", anyString(), any()); + + sourcesManagement.addAdapter("/", "id", new SpDataSet()); + + verify(adapterStorage, times(1)).getAdapter(anyString()); + verifyStatic(WorkerRestClient.class, times(1)); + WorkerRestClient.invokeSetAdapter(eq("/"), any()); + + } + + @Test(expected = AdapterException.class) + public void addAdapterFail() throws Exception { + AdapterStorageImpl adapterStorage = mock(AdapterStorageImpl.class); + when(adapterStorage.getAdapter(anyString())).thenReturn(new GenericAdapterSetDescription()); + SourcesManagement sourcesManagement = new SourcesManagement(adapterStorage); + + org.powermock.api.mockito.PowerMockito.doThrow(new AdapterException()).when(WorkerRestClient.class, "stopSetAdapter", anyString(), any()); + + sourcesManagement.detachAdapter("/", "id0", "id1"); + } + + @Test + public void detachAdapterSuccess() throws Exception { + AdapterStorageImpl adapterStorage = mock(AdapterStorageImpl.class); + when(adapterStorage.getAdapter(anyString())).thenReturn(new GenericAdapterSetDescription()); + SourcesManagement sourcesManagement = new SourcesManagement(adapterStorage); + doNothing().when(WorkerRestClient.class, "stopSetAdapter", anyString(), any()); + + sourcesManagement.detachAdapter("/", "id0", "id1"); + + verify(adapterStorage, times(1)).getAdapter(anyString()); + verifyStatic(WorkerRestClient.class, times(1)); + WorkerRestClient.stopSetAdapter(eq("/"), any()); + } + + @Test(expected = AdapterException.class) + public void detachAdapterFail() throws Exception { + AdapterStorageImpl adapterStorage = mock(AdapterStorageImpl.class); + when(adapterStorage.getAdapter(anyString())).thenReturn(new GenericAdapterSetDescription()); + SourcesManagement sourcesManagement = new SourcesManagement(adapterStorage); + org.powermock.api.mockito.PowerMockito.doThrow(new AdapterException()).when(WorkerRestClient.class, "stopSetAdapter", anyString(), any()); + + sourcesManagement.detachAdapter("/", "id0", "id1"); + } + + @Test + public void getAllAdaptersInstallDescriptionSuccess() throws Exception { + AdapterDescription adapterDescription = new GenericAdapterSetDescription(); + adapterDescription.setId("1234"); + AdapterStorageImpl adapterStorage = mock(AdapterStorageImpl.class); + when(adapterStorage.getAllAdapters()).thenReturn(Arrays.asList(adapterDescription)); + SourcesManagement sourcesManagement = new SourcesManagement(adapterStorage); + sourcesManagement.setConnectHost("host"); + + String result = sourcesManagement.getAllAdaptersInstallDescription("user@fzi.de"); + assertEquals(getJsonString(), result); + + } + + @Test(expected = AdapterException.class) + public void getAllAdaptersInstallDescriptionFail() throws Exception { + AdapterStorageImpl adapterStorage = mock(AdapterStorageImpl.class); + when(adapterStorage.getAllAdapters()).thenReturn(Arrays.asList(new GenericAdapterSetDescription())); + SourcesManagement sourcesManagement = new SourcesManagement(adapterStorage); + sourcesManagement.setConnectHost("host"); + + sourcesManagement.getAllAdaptersInstallDescription(" "); + + } + + + private String getJsonString() { + return "[" + + "{" + + "\"uri\":\"http://host/streampipes-connect/api/v1/user@fzi.de/master/sources/1234\"," + + "\"name\":\"Adapter Stream\"," + + "\"description\":\"This stream is generated by an StreamPipes Connect adapter. ID of adapter: 1234\"," + + "\"type\":\"source\"," + + "\"streams\":[" + + "{" + + "\"uri\":\"http://host/streampipes-connect/api/v1/user@fzi.de/master/sources/1234\"," + + "\"name\":\"GenericAdapterSetDescription\"," + + "\"description\":\"\"," + + "\"type\":\"set\"" + + "}" + + "]" + + "}" + + "]"; + } +} \ No newline at end of file diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/management/master/WorkerRestClientTest.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/management/master/WorkerRestClientTest.java new file mode 100644 index 0000000000..9633a64aa0 --- /dev/null +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/management/master/WorkerRestClientTest.java @@ -0,0 +1,145 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.management.master; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.model.connect.adapter.GenericAdapterSetDescription; +import org.streampipes.model.connect.adapter.GenericAdapterStreamDescription; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.times; +import static org.powermock.api.mockito.PowerMockito.*; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ WorkerRestClient.class }) +public class WorkerRestClientTest { + + /** + * Notes: In this class I tested how powermock could be used to mok static methods + * One problem is to mock static methods that return void + */ + + @Before + public void before() { + PowerMockito.mockStatic(WorkerRestClient.class); + } + + @Test + public void invokeStreamAdapterSuccess() throws Exception { + doNothing().when(WorkerRestClient.class, "startAdapter", anyString(), any()); + when(WorkerRestClient.class, "invokeStreamAdapter", anyString(), any()).thenCallRealMethod(); + + WorkerRestClient.invokeStreamAdapter("", null); + + verifyStatic(WorkerRestClient.class, times(1)); + WorkerRestClient.startAdapter(eq("worker/stream/invoke"), any()); + } + + @Test(expected = AdapterException.class) + public void invokeStreamAdapterFail() throws Exception { + doThrow(new AdapterException()).when(WorkerRestClient.class, "startAdapter", anyString(), any()); + when(WorkerRestClient.class, "invokeStreamAdapter", anyString(), any()).thenCallRealMethod(); + + WorkerRestClient.invokeStreamAdapter("", null); + } + + @Test + public void stopStreamAdapterSuccess() throws Exception { + + doNothing().when(WorkerRestClient.class, "stopAdapter", anyString(), any(), anyString()); + when(WorkerRestClient.class, "stopStreamAdapter", anyString(), any()).thenCallRealMethod(); + GenericAdapterStreamDescription description = new GenericAdapterStreamDescription(); + description.setId("id1"); + + WorkerRestClient.stopStreamAdapter("", description); + + verifyStatic(WorkerRestClient.class, times(1)); + WorkerRestClient.stopAdapter(anyString(), any(), eq("worker/stream/stop")); + + } + + @Test(expected = AdapterException.class) + public void stopStreamAdapterFail() throws Exception { + doThrow(new AdapterException()).when(WorkerRestClient.class, "stopAdapter", anyString(), any(), anyString()); + when(WorkerRestClient.class, "stopStreamAdapter", anyString(), any()).thenCallRealMethod(); + + GenericAdapterStreamDescription description = new GenericAdapterStreamDescription(); + description.setId("id1"); + + WorkerRestClient.stopStreamAdapter("", description); + + } + + @Test + public void invokeSetAdapterSuccess() throws Exception { + + doNothing().when(WorkerRestClient.class, "startAdapter", anyString(), any()); + when(WorkerRestClient.class, "invokeSetAdapter", anyString(), any()).thenCallRealMethod(); + + GenericAdapterSetDescription description = new GenericAdapterSetDescription(); + description.setId("id1"); + WorkerRestClient.invokeSetAdapter("", description); + + verifyStatic(WorkerRestClient.class, times(1)); + WorkerRestClient.startAdapter(eq("worker/set/invoke"), any()); + + } + + @Test(expected = AdapterException.class) + public void invokeSetAdapterFail() throws Exception { + doThrow(new AdapterException()).when(WorkerRestClient.class, "startAdapter", anyString(), any()); + when(WorkerRestClient.class, "invokeSetAdapter", anyString(), any()).thenCallRealMethod(); + + WorkerRestClient.invokeSetAdapter("", null); + } + + @Test + public void stopSetAdapterSuccess() throws Exception { + + doNothing().when(WorkerRestClient.class, "stopAdapter", anyString(), any(), anyString()); + when(WorkerRestClient.class, "stopSetAdapter", anyString(), any()).thenCallRealMethod(); + + GenericAdapterSetDescription description = new GenericAdapterSetDescription(); + description.setId("id1"); + WorkerRestClient.stopSetAdapter("", description); + + verifyStatic(WorkerRestClient.class, times(1)); + WorkerRestClient.stopAdapter(anyString(), any(), eq("worker/set/stop")); + + } + + @Test(expected = AdapterException.class) + public void stopSetAdapterFail() throws Exception { + doThrow(new AdapterException()).when(WorkerRestClient.class, "stopAdapter", anyString(), any(), anyString()); + when(WorkerRestClient.class, "stopSetAdapter", anyString(), any()).thenCallRealMethod(); + + GenericAdapterSetDescription description = new GenericAdapterSetDescription(); + description.setId("id1"); + WorkerRestClient.stopSetAdapter("", description); + + } + +} \ No newline at end of file diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/master/AdapterResourceTest.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/master/AdapterResourceTest.java new file mode 100644 index 0000000000..271636daf2 --- /dev/null +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/master/AdapterResourceTest.java @@ -0,0 +1,153 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.rest.master; + +import com.jayway.restassured.RestAssured; +import org.eclipse.jetty.server.Server; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.connect.init.Config; +import org.streampipes.connect.management.master.AdapterMasterManagement; +import org.streampipes.connect.utils.ConnectContainerResourceTest; +import org.streampipes.connect.utils.Utils; +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.model.connect.adapter.GenericAdapterStreamDescription; + +import static com.jayway.restassured.RestAssured.given; +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +public class AdapterResourceTest extends ConnectContainerResourceTest { + + @Override + protected String getApi() { + return "/api/v1/admin@fzi.de/master/adapters"; + } + + private AdapterResource adapterResource; + + private Server server; + + private AdapterMasterManagement adapterMasterManagement; + + @Before + public void before() { + Config.MASTER_PORT = 8019; + RestAssured.port = 8019; + + adapterResource = new AdapterResource(""); + server = getMasterServer(adapterResource); + + adapterMasterManagement = mock(AdapterMasterManagement.class); + adapterResource.setAdapterMasterManagement(adapterMasterManagement); + } + + @After + public void after() { + try { + server.stop(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Test + public void addAdapterSuccess() throws AdapterException { + when(adapterMasterManagement.addAdapter(any(), anyString(), any(), anyString())).thenReturn("http://t.de/"); + String data = Utils.getMinimalStreamAdapterJsonLD(); + postJsonLdSuccessRequest(data, "/", "http://t.de/"); + } + + @Test + public void addAdapterFail() throws AdapterException { + doThrow(new AdapterException(ERROR_MESSAGE)).when(adapterMasterManagement).addAdapter(any(), any(), any(), anyString()); + adapterResource.setAdapterMasterManagement(adapterMasterManagement); + + String data = Utils.getMinimalStreamAdapterJsonLD(); + postJsonLdFailRequest(data,"/"); + } + + @Test + public void getAdapterSuccess() throws AdapterException { + AdapterMasterManagement adapterMasterManagement = mock(AdapterMasterManagement.class); + when(adapterMasterManagement.getAdapter(any(), any())).thenReturn(new GenericAdapterStreamDescription()); + adapterResource.setAdapterMasterManagement(adapterMasterManagement); + + AdapterDescription result = getJsonLdSucessRequest("/testid", GenericAdapterStreamDescription.class); + + assertEquals(GenericAdapterStreamDescription.ID, result.getUri()); + } + + @Test + public void getAdapterFail() throws AdapterException { + doThrow(new AdapterException(ERROR_MESSAGE)).when(adapterMasterManagement).getAdapter(anyString(), any()); + adapterResource.setAdapterMasterManagement(adapterMasterManagement); + + getJsonLdFailRequest("/testid"); + } + + @Test + public void deleteAdapterSuccess() throws AdapterException { + AdapterMasterManagement adapterMasterManagement = mock(AdapterMasterManagement.class); + doNothing().when(adapterMasterManagement).deleteAdapter(anyString(), anyString()); + adapterResource.setAdapterMasterManagement(adapterMasterManagement); + + deleteJsonLdSucessRequest("/testid"); + } + + @Test + public void deleteAdapterFail() throws AdapterException { + doThrow(new AdapterException(ERROR_MESSAGE)).when(adapterMasterManagement).deleteAdapter(anyString(), anyString()); + adapterResource.setAdapterMasterManagement(adapterMasterManagement); + + deleteJsonLdFailRequest("/testid"); + } + + // TODO + // This test currently is not active. The problem is that we currently cannot deserialize the list with adapter + // descriptions because AdpaterDesription is an abstract class and the concrete subclasses are not known. + // Have a look at class org.streampipes.connect.management.AdapterDeserializer, which is a workaround for + // AdapterDescriptions Objects + // +// @Test +// public void getAllAdaptersSuccess() throws AdapterException { +// List adapterDescriptions = Arrays.asList(new GenericAdapterStreamDescription()); +// IAdapterMasterManagement adapterMasterManagement = mock(AdapterMasterManagement.class); +// when(adapterMasterManagement.getAllAdapters(any())).thenReturn(adapterDescriptions); +// adapterResource.setAdapterMasterManagement(adapterMasterManagement); +// +// AdapterDescriptionList result = getJsonLdSucessRequest("/", AdapterDescriptionList.class, StreamPipes.ADAPTER_DESCRIPTION_LIST); +//// AdapterDescriptionList result = getJsonLdSucessRequest("/", AdapterDescriptionList.class); +// +// assertEquals("http://streampipes.org/adapterlist", result.getUri()); +// assertEquals(1, result.getList().size()); +// assertEquals("http://t.id", result.getList().get(0).getUri()); +// +// } + + @Test + public void getAllAdaptersFail() throws AdapterException { + doThrow(new AdapterException(ERROR_MESSAGE)).when(adapterMasterManagement).getAllAdapters(any()); + adapterResource.setAdapterMasterManagement(adapterMasterManagement); + + getJsonLdFailRequest("/"); + } +} \ No newline at end of file diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/master/DecriptionResourceTest.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/master/DecriptionResourceTest.java new file mode 100644 index 0000000000..d4788cd862 --- /dev/null +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/master/DecriptionResourceTest.java @@ -0,0 +1,190 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.rest.master; + +import com.jayway.restassured.RestAssured; +import org.eclipse.jetty.server.Server; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.streampipes.connect.init.Config; +import org.streampipes.connect.management.master.DescriptionManagement; +import org.streampipes.connect.utils.ConnectContainerResourceTest; +import org.streampipes.model.connect.adapter.*; +import org.streampipes.model.connect.grounding.FormatDescription; +import org.streampipes.model.connect.grounding.FormatDescriptionList; +import org.streampipes.model.connect.grounding.ProtocolDescription; +import org.streampipes.model.connect.grounding.ProtocolDescriptionList; + +import java.util.Arrays; +import java.util.List; + +import static com.jayway.restassured.RestAssured.given; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class DecriptionResourceTest extends ConnectContainerResourceTest { + + @Override + protected String getApi() { + return "/api/v1/riemer@fzi.de/master/description"; + } + + private Server server; + + private DescriptionResource descriptionResource; + + private DescriptionManagement descriptionManagement; + + + @Before + public void before() { + Config.MASTER_PORT = 8019; + RestAssured.port = 8019; + + descriptionResource = new DescriptionResource(); + server = getMasterServer(descriptionResource); + } + + @After + public void after() { + try { + server.stop(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + + @Test + public void getFormatsEmptySuccess() { + mockDescriptionManagerFormats(new FormatDescriptionList()); + + FormatDescriptionList resultObject = getJsonLdSucessRequest("/formats", FormatDescriptionList.class); + + assertEquals(resultObject.getUri(), "http://bla.de#2"); + assertNotNull(resultObject.getList()); + assertEquals(resultObject.getList().size(), 0); + } + + @Test + public void getFormatsSuccess() { + List list = Arrays.asList( + new FormatDescription("http://id/1", "name1", ""), + new FormatDescription("http://id/2", "name2", "")); + mockDescriptionManagerFormats(new FormatDescriptionList(list)); + + FormatDescriptionList resultObject = getJsonLdSucessRequest("/formats", FormatDescriptionList.class); + + assertEquals(resultObject.getUri(), "http://bla.de#2"); + assertNotNull(resultObject.getList()); + assertEquals(2, resultObject.getList().size()); + assertEquals("http://id/1", resultObject.getList().get(0).getUri()); + assertEquals("name1", resultObject.getList().get(0).getName()); + assertEquals("http://id/2", resultObject.getList().get(1).getUri()); + assertEquals("name2", resultObject.getList().get(1).getName()); + } + + @Test + public void getProtocolsEmptySuccess() { + mockDescriptionManagerProtocols(new ProtocolDescriptionList()); + + ProtocolDescriptionList resultObject = getJsonLdSucessRequest("/protocols", ProtocolDescriptionList.class); + + assertEquals(resultObject.getUri(), "http://bla.de#1"); + assertNotNull(resultObject.getList()); + assertEquals(resultObject.getList().size(), 0); + } + + @Test + public void getProtocolsSuccess() { + List list = Arrays.asList( + new ProtocolDescription("http://id/1", "name1", ""), + new ProtocolDescription("http://id/2", "name2", "")); + mockDescriptionManagerProtocols(new ProtocolDescriptionList(list)); + + ProtocolDescriptionList resultObject = getJsonLdSucessRequest("/protocols", ProtocolDescriptionList.class); + + assertEquals(resultObject.getUri(), "http://bla.de#1"); + assertNotNull(resultObject.getList()); + assertEquals(2, resultObject.getList().size()); + assertEquals("http://id/1", resultObject.getList().get(0).getUri()); + assertEquals("name1", resultObject.getList().get(0).getName()); + assertEquals("http://id/2", resultObject.getList().get(1).getUri()); + assertEquals("name2", resultObject.getList().get(1).getName()); + } + + @Test + public void getAdaptersEmptySucess() { + mockDescriptionManagerAdapters(new AdapterDescriptionList()); + + AdapterDescriptionList resultObject = getJsonLdSucessRequest("/adapters", AdapterDescriptionList.class); + + assertNotNull(resultObject.getList()); + assertEquals(resultObject.getList().size(), 0); + } + + // TODO + // This test currently is not active. The problem is that we currently cannot deserialize the list with adapter + // descriptions because AdpaterDesription is an abstract class and the concrete subclasses are not known. + // Have a look at class org.streampipes.connect.management.AdapterDeserializer, which is a workaround for + // AdapterDescriptions Objects + // +// @Test +// public void getAdaptersSucess() { +// List list = Arrays.asList( +// new GenericAdapterStreamDescription(), +// new GenericAdapterSetDescription()); +// mockDescriptionManagerAdapters(new AdapterDescriptionList(list)); +// +// // TODO not sure how to fix +// AdapterDescriptionList resultObject = getJsonLdSucessRequest("/adapters", AdapterDescriptionList.class, StreamPipes.ADAPTER_DESCRIPTION_LIST); +// +//// assertEquals(resultObject.getUri(), "http://bla.de#2"); +// assertNotNull(resultObject.getList()); +// assertEquals(2, resultObject.getList().size()); +// assertEquals("http://id/1", resultObject.getList().get(0).getUri()); +// assertEquals("name1", resultObject.getList().get(0).getName()); +// assertEquals("http://id/2", resultObject.getList().get(1).getUri()); +// assertEquals("name2", resultObject.getList().get(1).getName()); +// } + + private void mockDescriptionManagerFormats(FormatDescriptionList formatDescriptionList){ + DescriptionManagement descriptionManagement = mock(DescriptionManagement.class); + when(descriptionManagement.getFormats()).thenReturn(formatDescriptionList); + + descriptionResource.setDescriptionManagement(descriptionManagement); + } + + private void mockDescriptionManagerProtocols(ProtocolDescriptionList protocolDescriptionList){ + DescriptionManagement descriptionManagement = mock(DescriptionManagement.class); + when(descriptionManagement.getProtocols()).thenReturn(protocolDescriptionList); + + descriptionResource.setDescriptionManagement(descriptionManagement); + } + + private void mockDescriptionManagerAdapters(AdapterDescriptionList adapterDescriptionList){ + DescriptionManagement descriptionManagement = mock(DescriptionManagement.class); + when(descriptionManagement.getAdapters()).thenReturn(adapterDescriptionList); + + descriptionResource.setDescriptionManagement(descriptionManagement); + } +} \ No newline at end of file diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/master/GuessResourceTest.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/master/GuessResourceTest.java new file mode 100644 index 0000000000..bf15c9997c --- /dev/null +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/master/GuessResourceTest.java @@ -0,0 +1,136 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.rest.master; + +import com.jayway.restassured.RestAssured; +import com.jayway.restassured.response.Response; +import org.eclipse.jetty.server.Server; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.connect.init.Config; +import org.streampipes.connect.management.master.GuessManagement; +import org.streampipes.connect.utils.ConnectContainerResourceTest; +import org.streampipes.connect.utils.Utils; +import org.streampipes.model.connect.guess.DomainPropertyProbabilityList; +import org.streampipes.model.connect.guess.GuessSchema; +import org.streampipes.model.schema.EventPropertyPrimitive; +import org.streampipes.model.schema.EventSchema; +import org.streampipes.rest.shared.util.JsonLdUtils; +import org.streampipes.vocabulary.StreamPipes; + +import java.util.Arrays; + +import static com.jayway.restassured.RestAssured.given; +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class GuessResourceTest extends ConnectContainerResourceTest { + + @Override + protected String getApi() { + return "/api/v1/riemer@fzi.de/master/guess"; + } + + private GuessResource guessResource; + private GuessManagement guessManagement; + private Server server; + + @Before + public void before() { + Config.MASTER_PORT = 8019; + RestAssured.port = 8019; + + guessResource = new GuessResource(); + server = getMasterServer(guessResource); + + guessManagement = mock(GuessManagement.class); + guessResource.setGuessManagement(guessManagement); + } + + @After + public void after() { + try { + server.stop(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + + @Test + public void guessSchemaSuccess() throws AdapterException { + GuessSchema guessSchema = getGuessSchema(); + when(guessManagement.guessSchema(any())).thenReturn(guessSchema); + String data = Utils.getMinimalStreamAdapterJsonLD(); + + Response res = given() + .body(data) + .when() + .post(getApi() + "/schema"); + + res.then() + .assertThat() + .statusCode(200); + + String resultString = res.body().print(); + + GuessSchema resultObject = JsonLdUtils.fromJsonLd(resultString, GuessSchema.class, StreamPipes.GUESS_SCHEMA); + + assertNotNull(resultObject); + assertNotNull(resultObject.propertyProbabilityList); + assertEquals(0, resultObject.propertyProbabilityList.size()); + assertNotNull(resultObject.eventSchema); + assertNotNull(resultObject.eventSchema.getEventProperties()); + assertEquals(1, resultObject.eventSchema.getEventProperties().size()); + assertEquals("id", resultObject.eventSchema.getEventProperties().get(0).getRuntimeName()); + } + + @Test + public void guessSchemaFail() throws AdapterException { + doThrow(new AdapterException()).when(guessManagement).guessSchema(any()); + String data = Utils.getMinimalStreamAdapterJsonLD(); + + Response res = given() + .body(data) + .when() + .post(getApi() + "/schema"); + + res.then() + .statusCode(500); + + } + + private GuessSchema getGuessSchema() { + EventSchema eventSchema = new EventSchema(); + EventPropertyPrimitive eventPropertyPrimitive = new EventPropertyPrimitive(); + eventPropertyPrimitive.setRuntimeType("http://schema.org/Number"); + eventPropertyPrimitive.setRuntimeName("id"); + + eventSchema.setEventProperties(Arrays.asList(eventPropertyPrimitive)); + GuessSchema guessSchema = new GuessSchema(); + guessSchema.setEventSchema(eventSchema); + + return guessSchema; + + } +} \ No newline at end of file diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/master/SourcesResourceTest.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/master/SourcesResourceTest.java new file mode 100644 index 0000000000..528ae6c646 --- /dev/null +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/master/SourcesResourceTest.java @@ -0,0 +1,194 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.rest.master; + + +import com.jayway.restassured.RestAssured; +import com.jayway.restassured.response.Response; +import org.eclipse.jetty.server.Server; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.connect.init.Config; +import org.streampipes.connect.management.master.SourcesManagement; +import org.streampipes.connect.utils.ConnectContainerResourceTest; +import org.streampipes.model.SpDataSet; +import org.streampipes.model.connect.grounding.FormatDescriptionList; +import org.streampipes.model.graph.DataSourceDescription; +import org.streampipes.rest.shared.util.JsonLdUtils; + +import static com.jayway.restassured.RestAssured.given; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; +import static org.powermock.api.mockito.PowerMockito.doNothing; + +public class SourcesResourceTest extends ConnectContainerResourceTest { + + @Override + protected String getApi() { + return "/api/v1/riemer@fzi.de/master/sources"; + } + + private Server server; + + private SourcesResource sourcesResource; + + private SourcesManagement sourcesManagement; + + + @Before + public void before() { + Config.MASTER_PORT = 8019; + RestAssured.port = 8019; + + sourcesResource = new SourcesResource(""); + server = getMasterServer(sourcesResource); + } + + @After + public void after() { + try { + server.stop(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + + @Test + public void getAllAdaptersInstallDescriptionSuccess() throws Exception { + SourcesManagement sourcesManagement = mock(SourcesManagement.class); + when(sourcesManagement.getAllAdaptersInstallDescription(anyString())).thenReturn("test"); + sourcesResource.setSourcesManagement(sourcesManagement); + + Response response = given().contentType("application/json") + .when() + .get(getApi() + "/"); + + response.then() + .statusCode(200); + + String resultString = response.body().print(); + + + assertEquals("test", resultString); + + } + + @Test + public void getAllAdaptersInstallDescriptionFail() throws Exception { + SourcesManagement sourcesManagement = mock(SourcesManagement.class); + doThrow(new AdapterException()).when(sourcesManagement).getAllAdaptersInstallDescription(anyString()); + sourcesResource.setSourcesManagement(sourcesManagement); + + given().contentType("application/json") + .when() + .get(getApi() + "/") + .then() + .statusCode(500); + } + + @Test + public void getAdapterDataSourceSuccess() throws AdapterException { + DataSourceDescription dataSourceDescription = new DataSourceDescription("http://a.d", "a", ""); + SourcesManagement sourcesManagement = mock(SourcesManagement.class); + when(sourcesManagement.getAdapterDataSource(anyString())).thenReturn(dataSourceDescription); + sourcesResource.setSourcesManagement(sourcesManagement); + + Response response = given().contentType("application/json") + .when() + .get(getApi() + "/1234"); + + response.then() + .statusCode(200); + + String resultString = response.body().print(); + + DataSourceDescription result = JsonLdUtils.fromJsonLd(resultString, DataSourceDescription.class); + + assertEquals(dataSourceDescription.getUri(), result.getUri()); + assertEquals(dataSourceDescription.getName(), result.getName()); + } + + @Test + public void getAdapterDataSourceFail() throws AdapterException { + SourcesManagement sourcesManagement = mock(SourcesManagement.class); + doThrow(new AdapterException()).when(sourcesManagement).getAdapterDataSource(anyString()); + sourcesResource.setSourcesManagement(sourcesManagement); + + given().contentType("application/json") + .when() + .get(getApi() + "/1234") + .then() + .statusCode(500); + } + + + @Test + public void addAdapterSuccess() throws Exception { + SourcesManagement sourcesManagement = mock(SourcesManagement.class); + doNothing().when(sourcesManagement).addAdapter(anyString(), anyString(), any()); + sourcesResource.setSourcesManagement(sourcesManagement); + + String data = getMinimalDataSetJsonLd(); + postJsonLdSuccessRequest(data, "/id/streams", "Instance of data set http://dataset.de/1 successfully started"); + + verify(sourcesManagement, times(1)).addAdapter(anyString(), anyString(), any()); + } + + @Test + public void addAdapterFail() throws AdapterException { + SourcesManagement sourcesManagement = mock(SourcesManagement.class); + doThrow(AdapterException.class).when(sourcesManagement).addAdapter(anyString(), anyString(), any()); + sourcesResource.setSourcesManagement(sourcesManagement); + + String data = getMinimalDataSetJsonLd(); + postJsonLdFailRequest(data, "/id/streams", "Could not set data set instance: http://dataset.de/1"); + + } + + @Test + public void detachSuccess() throws AdapterException { + SourcesManagement sourcesManagement = mock(SourcesManagement.class); + doNothing().when(sourcesManagement).detachAdapter(anyString(), anyString(), anyString()); + sourcesResource.setSourcesManagement(sourcesManagement); + + deleteJsonLdSucessRequest("/id0/streams/id1"); + + verify(sourcesManagement, times(1)).detachAdapter(anyString(), anyString(), anyString()); + } + + @Test + public void detachFail() throws AdapterException { + SourcesManagement sourcesManagement = mock(SourcesManagement.class); + doThrow(AdapterException.class).when(sourcesManagement).detachAdapter(anyString(), anyString(), anyString()); + sourcesResource.setSourcesManagement(sourcesManagement); + + deleteJsonLdFailRequest("/id0/streams/id1"); + } + + private String getMinimalDataSetJsonLd() { + SpDataSet dataSet = new SpDataSet(); + dataSet.setUri("http://dataset.de/1"); + + return JsonLdUtils.toJsonLD(dataSet); + } +} \ No newline at end of file diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/master/WelcomePageMasterTest.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/master/WelcomePageMasterTest.java new file mode 100644 index 0000000000..db3329a15b --- /dev/null +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/master/WelcomePageMasterTest.java @@ -0,0 +1,96 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.rest.master; + + +import com.jayway.restassured.RestAssured; +import org.eclipse.jetty.server.Server; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.connect.init.Config; +import org.streampipes.connect.management.master.AdapterMasterManagement; +import org.streampipes.connect.utils.ConnectContainerResourceTest; +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.model.connect.adapter.GenericAdapterStreamDescription; +import org.streampipes.model.grounding.*; + +import java.util.Arrays; + +import static com.jayway.restassured.RestAssured.get; +import static org.hamcrest.collection.IsCollectionWithSize.hasSize; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class WelcomePageMasterTest extends ConnectContainerResourceTest { + private Server server; + + private WelcomePageMaster welcomePage; + + @Before + public void before() { + Config.MASTER_PORT = 8019; + RestAssured.port = 8019; + + welcomePage = new WelcomePageMaster(); + server = getMasterServer(welcomePage); + } + + @After + public void after() { + try { + server.stop(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Test + public void getWelcomePageHtmlWithAdaptersSucessTest() throws AdapterException { + testMainPart(); + get("/").then().body("html.body.ol.size()", equalTo(1)); + get("/").then().body("html.body.ol[0].toString()", equalTo("testId")); + } + + private void testMainPart() throws AdapterException { + String id = "testId"; + AdapterDescription adapterDescription = new GenericAdapterStreamDescription(); + adapterDescription.setAdapterId(id); + EventGrounding eventGrounding = new EventGrounding(); + //TODO add eeventGrounding + TransportProtocol transportProtocol = new KafkaTransportProtocol(); + TopicDefinition topicDefinition = new SimpleTopicDefinition("test"); + transportProtocol.setTopicDefinition(topicDefinition); + eventGrounding.setTransportProtocol(transportProtocol); + adapterDescription.setEventGrounding(eventGrounding); + AdapterMasterManagement adapterManagement = mock(AdapterMasterManagement.class); + welcomePage.setAdapterMasterManagement(adapterManagement); + when(adapterManagement.getAllAdapters(any())).thenReturn(Arrays.asList(adapterDescription)); + get("/").then().body("html.head.title", equalTo("StreamPipes Connector Master Container")); + get("/").then().body("html.body.h1", equalTo("Connector Master Container")); + get("/").then().body("html.body.h2", equalTo("All Running Adapters")); + } + + @Override + protected String getApi() { + return ""; + } +} \ No newline at end of file diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/worker/WelcomePageWorkerTest.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/worker/WelcomePageWorkerTest.java new file mode 100644 index 0000000000..864ac4b911 --- /dev/null +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/worker/WelcomePageWorkerTest.java @@ -0,0 +1,64 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.rest.worker; + +import com.jayway.restassured.RestAssured; +import org.eclipse.jetty.server.Server; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.streampipes.connect.init.Config; +import org.streampipes.connect.utils.ConnectContainerResourceTest; + +import static com.jayway.restassured.RestAssured.get; +import static org.hamcrest.core.IsEqual.equalTo; + +public class WelcomePageWorkerTest extends ConnectContainerResourceTest { + + + private Server server; + + @Before + public void before() { + Config.WORKER_PORT = 8019; + RestAssured.port = 8019; + + WelcomePageWorker welcomePage = new WelcomePageWorker("WORKER_01"); + server = getWorkerServer(welcomePage); + } + + @After + public void after() { + try { + server.stop(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Test + public void getWelcomePageHtmlTest() { + get("/").then().body("html.head.title", equalTo("StreamPipes Connector Worker Container")); + get("/").then().body("html.body.h1", equalTo("Worker Connector Container with ID: WORKER_01")); + } + + @Override + protected String getApi() { + return ""; + } +} diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/worker/WorkerResourceTest.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/worker/WorkerResourceTest.java new file mode 100644 index 0000000000..f7d011cfd7 --- /dev/null +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/rest/worker/WorkerResourceTest.java @@ -0,0 +1,145 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.rest.worker; + +import com.jayway.restassured.RestAssured; +import org.eclipse.jetty.server.Server; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.connect.init.Config; +import org.streampipes.connect.management.worker.AdapterWorkerManagement; +import org.streampipes.connect.utils.ConnectContainerResourceTest; +import org.streampipes.connect.utils.Utils; +import org.streampipes.model.connect.adapter.AdapterSetDescription; +import org.streampipes.model.connect.adapter.AdapterStreamDescription; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; + + +public class WorkerResourceTest extends ConnectContainerResourceTest { + + @Override + protected String getApi() { + return "/api/v1/riemer@fzi.de/worker"; + } + + private WorkerResource workerResource; + + private Server server; + + private AdapterWorkerManagement adapterManagement; + + @Before + public void before() { + Config.WORKER_PORT = 8019; + RestAssured.port = 8019; + + workerResource = new WorkerResource(); + server = getWorkerServer(workerResource); + + adapterManagement = mock(AdapterWorkerManagement.class); + workerResource.setAdapterManagement(adapterManagement); + } + + @After + public void after() { + try { + server.stop(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Test + public void invokeStreamAdapterSuccess() { + + String data = Utils.getMinimalStreamAdapterJsonLD(); + postJsonLdSuccessRequest(data, "/stream/invoke", "Stream adapter with id http://t.de/ successfully started"); + + } + + @Test + public void invokeStreamAdapterFail() throws AdapterException { + + doThrow(new AdapterException(ERROR_MESSAGE)).when(adapterManagement).invokeStreamAdapter(any(AdapterStreamDescription.class)); + workerResource.setAdapterManagement(adapterManagement); + + String data = Utils.getMinimalStreamAdapterJsonLD(); + postJsonLdFailRequest(data,"/stream/invoke"); + + } + + @Test + public void stopStreamAdapterSuccess() { + + String data = Utils.getMinimalStreamAdapterJsonLD(); + postJsonLdSuccessRequest(data,"/stream/stop", "Stream adapter with id http://t.de/ successfully stopped"); + + } + + @Test + public void stopStreamAdapterFail() throws AdapterException { + + doThrow(new AdapterException(ERROR_MESSAGE)).when(adapterManagement).stopStreamAdapter(any(AdapterStreamDescription.class)); + workerResource.setAdapterManagement(adapterManagement); + + String data = Utils.getMinimalStreamAdapterJsonLD(); + postJsonLdFailRequest(data,"/stream/stop"); + + } + + + @Test + public void invokeSetAdapterSuccess() { + String data = Utils.getMinimalSetAdapterJsonLD(); + postJsonLdSuccessRequest(data,"/set/invoke", "Set adapter with id http://t.de/ successfully started"); + } + + @Test + public void invokeSetAdapterFail() throws AdapterException { + + doThrow(new AdapterException(ERROR_MESSAGE)).when(adapterManagement).invokeSetAdapter(any(AdapterSetDescription.class)); + workerResource.setAdapterManagement(adapterManagement); + + String data = Utils.getMinimalSetAdapterJsonLD(); + postJsonLdFailRequest(data, "/set/invoke"); + } + + @Test + public void stopSetAdapterSuccess() { + + String data = Utils.getMinimalSetAdapterJsonLD(); + postJsonLdSuccessRequest(data, "/set/stop", "Set adapter with id http://t.de/ successfully stopped"); + } + + @Test + public void stopSetAdapterFail() throws AdapterException { + + doThrow(new AdapterException(ERROR_MESSAGE)).when(adapterManagement).stopSetAdapter(any(AdapterSetDescription.class)); + workerResource.setAdapterManagement(adapterManagement); + + String data = Utils.getMinimalSetAdapterJsonLD(); + postJsonLdFailRequest(data, "/set/stop"); + } + + +} \ No newline at end of file diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/utils/ConnectContainerResourceTest.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/utils/ConnectContainerResourceTest.java new file mode 100644 index 0000000000..2f6173c7ce --- /dev/null +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/utils/ConnectContainerResourceTest.java @@ -0,0 +1,149 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.utils; + +import java.net.URI; + +import com.jayway.restassured.response.Response; +import com.jayway.restassured.response.ValidatableResponseOptions; +import org.eclipse.jetty.server.Server; +import org.glassfish.jersey.jetty.JettyHttpContainerFactory; +import org.glassfish.jersey.server.ResourceConfig; +import org.streampipes.connect.init.Config; +import org.streampipes.connect.rest.AbstractContainerResource; +import org.streampipes.rest.shared.serializer.GsonClientModelProvider; +import org.streampipes.rest.shared.serializer.GsonWithIdProvider; +import org.streampipes.rest.shared.serializer.GsonWithoutIdProvider; +import org.streampipes.rest.shared.serializer.JsonLdProvider; +import org.streampipes.rest.shared.util.JsonLdUtils; + +import static org.hamcrest.core.IsEqual.equalTo; +import static com.jayway.restassured.RestAssured.given; + +import javax.ws.rs.core.UriBuilder; + +public abstract class ConnectContainerResourceTest { + + protected static final String ERROR_MESSAGE = "error"; + + protected Server getMasterServer(AbstractContainerResource resource) { + return getServer(resource, Config.getMasterBaseUrl()); + } + + protected Server getWorkerServer(AbstractContainerResource resource) { + return getServer(resource, Config.getWorkerBaseUrl()); + + } + + private Server getServer(AbstractContainerResource resource, String url) { + + ResourceConfig config = new ResourceConfig() + .register(GsonWithIdProvider.class) + .register(GsonWithoutIdProvider.class) + .register(GsonClientModelProvider.class) + .register(JsonLdProvider.class) + .register(resource); + + URI baseUri = UriBuilder + .fromUri(url) + .build(); + + return JettyHttpContainerFactory.createServer(baseUri, config); + } + + + protected abstract String getApi(); + + protected T getJsonLdSucessRequest(String route, Class clazz) { + return getJsonLdSucessRequest(route, clazz, ""); + + } + + protected T getJsonLdSucessRequest(String route, Class clazz, String rootElement) { + Response response = given().contentType("application/ld+json") + .when() + .get(getApi() + route); + + response.then() + .statusCode(200); + + String resultString = response.body().print(); + + T resultObject; + if (rootElement.equals("")) { + resultObject = JsonLdUtils.fromJsonLd(resultString, clazz); + } else { + resultObject = JsonLdUtils.fromJsonLd(resultString, clazz, rootElement); + } + + return resultObject; + + } + + protected void getJsonLdFailRequest(String route) { + given().contentType("application/ld+json") + .when() + .get(getApi() + route) + .then() + .statusCode(500); + + } + + protected void deleteJsonLdSucessRequest(String route) { + Response response = given().contentType("application/ld+json") + .when() + .delete(getApi() + route); + + response.then() + .statusCode(200); + } + + protected void deleteJsonLdFailRequest(String route) { + Response response = given().contentType("application/ld+json") + .when() + .delete(getApi() + route); + + response.then() + .statusCode(500); + } + + protected ValidatableResponseOptions postJsonLdSuccessRequest(String data, String route, String responseMessage) { + return postJsonLdRequest(data, route) + .body("success", equalTo(true)) + .body("notifications[0].title", equalTo(responseMessage)); + } + + protected ValidatableResponseOptions postJsonLdFailRequest(String data, String route) { + return postJsonLdFailRequest(data, route, ERROR_MESSAGE); + } + + protected ValidatableResponseOptions postJsonLdFailRequest(String data, String route, String errorMessage) { + return postJsonLdRequest(data, route) + .body("success", equalTo(false)) + .body("notifications[0].title", equalTo(errorMessage)); + } + protected ValidatableResponseOptions postJsonLdRequest(String data, String route) { + return given().contentType("application/ld+json") + .body(data) + .when() + .post(getApi() + route) + .then() + .assertThat() + .statusCode(200); + } +} diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/utils/TestAdapterSerializer.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/utils/TestAdapterSerializer.java new file mode 100644 index 0000000000..7883e72493 --- /dev/null +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/utils/TestAdapterSerializer.java @@ -0,0 +1,73 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.streampipes.connect.utils; + +import org.junit.Test; +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.model.connect.adapter.GenericAdapterStreamDescription; +import org.streampipes.model.connect.grounding.FormatDescription; +import org.streampipes.model.schema.EventProperty; +import org.streampipes.model.schema.EventPropertyPrimitive; +import org.streampipes.model.schema.EventSchema; +import org.streampipes.model.staticproperty.FreeTextStaticProperty; +import org.streampipes.serializers.json.GsonSerializer; +import org.streampipes.storage.couchdb.impl.AdapterStorageImpl; + +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; + +public class TestAdapterSerializer { + + @Test + public void testGenericAdapterStreamSerialization() { + GenericAdapterStreamDescription desc = new GenericAdapterStreamDescription(); + FormatDescription formatDescription = new FormatDescription(); + formatDescription.addConfig(new FreeTextStaticProperty("internal-id", "test", "test")); + desc.setFormatDescription(formatDescription); + + // Add _id field to json, probably there is a cleaner solution + String serialized = (GsonSerializer.getAdapterGson().toJson(desc)); + serialized = serialized.substring(1); + serialized = "{\"_id\": \"1\", \"_rev\": \"2\"," + serialized; + + AdapterDescription deserialized = GsonSerializer.getAdapterGson().fromJson(serialized, AdapterDescription.class); + assertEquals(deserialized.getAdapterId(), "http://streampipes.org/genericadapterstreamdescription"); + } + + @Test + public void testEventPropertySerialization() { + EventSchema schema = new EventSchema(); + EventProperty primitive = new EventPropertyPrimitive(); + primitive.setRuntimeName("test"); + ((EventPropertyPrimitive) primitive).setRuntimeType("test"); + schema.setEventProperties(Arrays.asList(primitive)); + String serialized = GsonSerializer.getGson().toJson(primitive); + System.out.println(serialized); + } + +// @Test +// public void testCouchDbSerialization() { +// GenericAdapterStreamDescription desc = new GenericAdapterStreamDescription(); +// +// new AdapterStorageImpl().storeAdapter(desc); +// +// AdapterDescription deserialized = new AdapterStorageImpl().getAllAdapters().get(0); +// System.out.println(deserialized.getAdapterId()); +// } + +} diff --git a/streampipes-connect-container/src/test/java/org/streampipes/connect/utils/Utils.java b/streampipes-connect-container/src/test/java/org/streampipes/connect/utils/Utils.java new file mode 100644 index 0000000000..07f93b5466 --- /dev/null +++ b/streampipes-connect-container/src/test/java/org/streampipes/connect/utils/Utils.java @@ -0,0 +1,75 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.utils; + +import org.streampipes.empire.core.empire.annotation.InvalidRdfException; +import org.streampipes.model.connect.adapter.*; +import org.streampipes.serializers.jsonld.JsonLdTransformer; +import org.streampipes.vocabulary.StreamPipes; + +import java.lang.reflect.InvocationTargetException; + +public class Utils { + + public static String getMinimalStreamAdapterJsonLD() { + return getMinimalAdapterJsonLD(new GenericAdapterStreamDescription(), StreamPipes.GENERIC_ADAPTER_SET_DESCRIPTION); + } + + public static String getMinimalSetAdapterJsonLD() { + return getMinimalAdapterJsonLD(new GenericAdapterSetDescription(), StreamPipes.GENERIC_ADAPTER_SET_DESCRIPTION); + } + + private static String getMinimalAdapterJsonLD(AdapterDescription asd, String rootElement) { + String id = "http://t.de/"; + asd.setUri(id); + asd.setId(id); + + JsonLdTransformer jsonLdTransformer = new JsonLdTransformer(rootElement); + + try { + return org.streampipes.commons.Utils.asString(jsonLdTransformer.toJsonLd(asd)); + } catch (IllegalAccessException | InvocationTargetException | ClassNotFoundException e) { + e.printStackTrace(); + } + catch (InvalidRdfException e) { + e.printStackTrace(); + } + + return ""; + } + + + public static AdapterStreamDescription getMinimalStreamAdapter() { + AdapterStreamDescription result = new GenericAdapterStreamDescription(); + String id = "http://t.de/"; + result.setUri(id); + result.setId(id); + + return result; + } + + public static AdapterSetDescription getMinimalSetAdapter() { + AdapterSetDescription result = new GenericAdapterSetDescription(); + String id = "http://t.de/"; + result.setUri(id); + result.setId(id); + + return result; + } + +} diff --git a/streampipes-connect/pom.xml b/streampipes-connect/pom.xml index 3126630759..84d0b2e2de 100755 --- a/streampipes-connect/pom.xml +++ b/streampipes-connect/pom.xml @@ -1,10 +1,10 @@ - org.streampipes - streampipes-parent - 0.55.2 - + org.streampipes + streampipes-parent + 0.60.0 + 4.0.0 streampipes-connect @@ -26,10 +26,15 @@ org.streampipes streampipes-serializers + - org.streampipes - streampipes-commons - + org.streampipes + streampipes-rest-shared + + + org.streampipes + streampipes-measurement-units + org.glassfish javax.json @@ -59,6 +64,42 @@ org.streampipes streampipes-messaging-kafka - + + org.json + json + 20180130 + test + + + org.fusesource.mqtt-client + mqtt-client + 1.12 + + + de.grundid.opendatalab + geojson-jackson + 1.8 + + + org.json + json + 20171018 + + + org.twitter4j + twitter4j-core + [4.0,) + + + org.twitter4j + twitter4j-stream + 4.0.1 + compile + + + com.opencsv + opencsv + 4.0 + \ No newline at end of file diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/EmitBinaryEvent.java b/streampipes-connect/src/main/java/org/streampipes/connect/EmitBinaryEvent.java index a2a0736251..3587ff4bde 100644 --- a/streampipes-connect/src/main/java/org/streampipes/connect/EmitBinaryEvent.java +++ b/streampipes-connect/src/main/java/org/streampipes/connect/EmitBinaryEvent.java @@ -1,3 +1,20 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package org.streampipes.connect; public interface EmitBinaryEvent { diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/GetNEvents.java b/streampipes-connect/src/main/java/org/streampipes/connect/GetNEvents.java index f4fe9e37ce..1ea2703c0c 100644 --- a/streampipes-connect/src/main/java/org/streampipes/connect/GetNEvents.java +++ b/streampipes-connect/src/main/java/org/streampipes/connect/GetNEvents.java @@ -1,3 +1,20 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package org.streampipes.connect; diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/GetTrainingData.java b/streampipes-connect/src/main/java/org/streampipes/connect/GetTrainingData.java deleted file mode 100644 index 4594946f34..0000000000 --- a/streampipes-connect/src/main/java/org/streampipes/connect/GetTrainingData.java +++ /dev/null @@ -1,153 +0,0 @@ -package org.streampipes.connect; - - -import org.apache.http.client.fluent.Request; -import org.streampipes.connect.firstconnector.format.json.JsonFormat; -import org.streampipes.connect.firstconnector.format.json.JsonParser; -import org.streampipes.connect.firstconnector.protocol.Protocol; -import org.streampipes.connect.firstconnector.protocol.set.HttpProtocol; -import org.streampipes.dataformat.json.JsonDataFormatDefinition; -import org.streampipes.model.modelconnect.DomainPropertyProbability; -import org.streampipes.model.modelconnect.DomainPropertyProbabilityList; -import org.streampipes.model.modelconnect.ProtocolDescription; - -import java.io.IOException; -import java.io.PrintStream; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.util.List; -import java.util.Map; - -/** - * This is just a wrapper to find the best solution to extract training data from multiple different sources to train our model repository - */ -public class GetTrainingData { - public static void main(String... args) { - new GetTrainingData().getTrainingData(); - -// Double[] x = {34292.0, 34292.0, 34292.0, 84155.0, 34466.0, 83352.0, 84503.0, 63916.0, 9456.0, 9456.0, 9456.0, 8359.0, 84371.0, 63743.0, 8280.0, 8280.0, 34454.0, 84364.0, 94081.0, 57334.0}; -// String[] x = {"a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a", "a"}; - -// new GetTrainingData().getDomainPropertyProbability(x); - - } - - public static DomainPropertyProbabilityList getDomainPropertyProbability(Object[] sampleData) { - - String url = "http://localhost/predict"; - - String numbers = ""; - for (Object d : sampleData) { - if (d instanceof String || d == null) { -// if (d == null) { -// d = ""; -// } - numbers = numbers + "\"" + d + "\","; - } else { - numbers = numbers + d + ","; - } - - } - numbers = numbers.substring(0, numbers.length() - 1) + ""; - - numbers = numbers.replaceAll("\"null\"", "0.0"); - - try { - url = url + "?X=[" + URLEncoder.encode(numbers, "UTF-8") + "]"; - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - } - - DomainPropertyProbabilityList result = new DomainPropertyProbabilityList(); - try { - url = url.replaceAll("%2C", ","); - System.out.println(url); - String s = Request.Get(url) - .connectTimeout(10000) - .socketTimeout(100000) - .execute().returnContent().asString(); - - - JsonDataFormatDefinition jsonDataFormatDefinition = new JsonDataFormatDefinition(); - - s = s.substring(2, s.length()); - - while(s.indexOf("[") > -1) { - String tmpString = s.substring(s.indexOf("["), s.indexOf("]") + 1); - s = s.substring(s.indexOf("]") + 1, s.length()); - result.addDomainPropertyProbability(parseDomainPropertyProbability(tmpString)); - } - - } catch (IOException e) { - e.printStackTrace(); - } - return result; - } - - private static DomainPropertyProbability parseDomainPropertyProbability(String s) { - DomainPropertyProbability result = new DomainPropertyProbability(); - String property = s.substring(s.indexOf("\"") + 1, s.lastIndexOf("\"")); - String numberString = s.substring(s.indexOf(",") + 1, s.indexOf("]")); -// double number = Double.parseDouble(numberString); - - result.setDomainProperty(property); - result.setProbability(numberString); - return result; - } - - public void getTrainingData() { - ProtocolDescription httpDescription = new HttpProtocol().declareModel(); - - - JsonParser parser = new JsonParser(true, "records"); - JsonFormat format = new JsonFormat(); - - - Protocol httpProtocol = new HttpProtocol().getInstance(httpDescription, parser, format); - - List> result = httpProtocol.getNElements(100); - -// String[] tagsLabels = {"addr:city", "addr:country", "addr:housenumber", "addr:postcode", "addr:street", "amenity", "dispensing", -// "email", "fax", "name", "opening_hours", "operator", "phone", "website", "wheelchair"}; - - String[] tagsLabels = {"name", "strae", "hausnummer", "plz", "stadteil", "bezirk", "nvr", "trger", "kurzbeschreibung", "erweit__informationen", - "ffnungszeiten", "urls", "gf1", "gf2", "gf3", "feeds_flatstore_entry_id", "timestamp", "feeds_entity_id"}; - - - String resultString = ""; - - - for (String label : tagsLabels) { - for (int j = 0; j < result.size(); j = j + 20) { - String subString = "["; - for (int i = 0; i < 20; i++) { -// Map tags = (Map) result.get(i + j).get("tags"); -// String s = (String) tags.get(label); -// subString = subString + "\"" + s + "\", "; -// subString = subString + "" + s + ", "; - - subString = subString + "" + "\"" + result.get(i + j).get(label) + "\", "; -// subString = subString + "" + result.get(i + j).get(label) + ", "; - } - - subString = subString.substring(0, subString.length() - 2); - subString = subString + "]"; - - resultString = resultString + subString + ",\n"; - } - - - resultString = resultString.substring(0, resultString.length() - 2); - System.out.println(label); - try { - new PrintStream(System.out, true, "UTF-8").println(resultString); - System.out.println(); - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - } - resultString = ""; - } - - - } -} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/RunningAdapterInstances.java b/streampipes-connect/src/main/java/org/streampipes/connect/RunningAdapterInstances.java index c675de214f..829dbce82a 100644 --- a/streampipes-connect/src/main/java/org/streampipes/connect/RunningAdapterInstances.java +++ b/streampipes-connect/src/main/java/org/streampipes/connect/RunningAdapterInstances.java @@ -1,6 +1,23 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + package org.streampipes.connect; -import org.streampipes.connect.firstconnector.Adapter; +import org.streampipes.connect.adapter.Adapter; import java.util.HashMap; import java.util.Map; diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/SendToKafka.java b/streampipes-connect/src/main/java/org/streampipes/connect/SendToKafka.java deleted file mode 100644 index 083f90bcaa..0000000000 --- a/streampipes-connect/src/main/java/org/streampipes/connect/SendToKafka.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.streampipes.connect; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.gson.JsonObject; -import org.streampipes.connect.firstconnector.format.Format; -import org.streampipes.messaging.kafka.SpKafkaProducer; - -import java.util.Map; - -public class SendToKafka implements EmitBinaryEvent { - - private Format format; - - private SpKafkaProducer producer; - private ObjectMapper objectMapper; - - public SendToKafka(Format format, String brokerUrl, String topic) { - this.format = format; - - producer = new SpKafkaProducer(brokerUrl, topic); - objectMapper = new ObjectMapper(); - - } - - @Override - public Boolean emit(byte[] event) { - - Map result = format.parse(event); - System.out.println("send to kafka: " + result); - - try { - producer.publish(objectMapper.writeValueAsBytes(result)); - } catch (JsonProcessingException e) { - e.printStackTrace(); - } - return true; - } -} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/SendToPipeline.java b/streampipes-connect/src/main/java/org/streampipes/connect/SendToPipeline.java new file mode 100644 index 0000000000..9de6b64d9b --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/SendToPipeline.java @@ -0,0 +1,76 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.streampipes.connect.adapter.generic.format.Format; +import org.streampipes.connect.adapter.generic.pipeline.AdapterPipeline; +import org.streampipes.messaging.kafka.SpKafkaProducer; + +import java.util.Map; + +public class SendToPipeline implements EmitBinaryEvent { + + private Format format; + + private SpKafkaProducer producer; + private ObjectMapper objectMapper; + + private AdapterPipeline adapterPipeline; + + @Deprecated + // TODO remove + public SendToPipeline(Format format, String brokerUrl, String topic) { + this.format = format; + + producer = new SpKafkaProducer(brokerUrl, topic); + objectMapper = new ObjectMapper(); + } + + public SendToPipeline(Format format, AdapterPipeline adapterPipeline) { + this.format = format; + this.adapterPipeline = adapterPipeline; + } + + @Override + public Boolean emit(byte[] event) { + + Map result = format.parse(event); + + if (result != null) { + adapterPipeline.process(result); + } + + // TODO Get the rules + + + // TODO Apply on events + + + +// try { +// if (result != null) { +// producer.publish(objectMapper.writeValueAsBytes(result)); +// System.out.println("send to kafka: " + result); +// } +// } catch (JsonProcessingException e) { +// e.printStackTrace(); +// } + return true; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/TestMain.java b/streampipes-connect/src/main/java/org/streampipes/connect/TestMain.java new file mode 100644 index 0000000000..cc0b68c512 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/TestMain.java @@ -0,0 +1,165 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect; + +import org.apache.http.Header; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicHeader; +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.connect.adapter.generic.format.json.object.JsonObjectFormat; +import org.streampipes.connect.adapter.generic.protocol.stream.KafkaProtocol; +import org.streampipes.connect.adapter.specific.sensemap.OpenSenseMapAdapter; +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.model.connect.adapter.GenericAdapterStreamDescription; +import org.streampipes.model.connect.grounding.FormatDescription; +import org.streampipes.model.connect.grounding.ProtocolDescription; +import org.streampipes.model.staticproperty.FreeTextStaticProperty; +import org.streampipes.model.staticproperty.StaticProperty; +import org.streampipes.rest.shared.util.JsonLdUtils; + +import javax.swing.text.AbstractDocument; +import java.io.IOException; + +public class TestMain { + + public static void main(String... args) throws IOException { + +// addAdapter(); + startSenseMapAdapter(); +// deleteAdapter("77932edb92be4a06bd8386d23e3f9f9d"); + + + + } + + + // http://localhost:8099/api/v1/riemer@fzi.de/master/adapters/ + + // http://localhost:8099/api/v1/master/adpaters/8d8dc565cf694dd78913df920d3023f4 + + private static void deleteAdapter(String id) throws IOException { + HttpDelete delete = new HttpDelete("http://localhost:8099/api/v1/riemer@fzi.de/master/adapters/" + id); + Header headers[] = { + new BasicHeader("Content-type", "application/ld+json"), + }; + delete.setHeaders(headers); +// post.setEntity(new StringEntity(jsonld)); + + HttpClient client = HttpClients.custom().build(); + HttpResponse response = client.execute(delete); + + System.out.println(response); + } + + + private static void startSenseMapAdapter() throws IOException { + + AdapterDescription adapterDescription = new OpenSenseMapAdapter().declareModel(); + + // Set broker URL + for (StaticProperty property: adapterDescription.getConfig()) { + if (property.getInternalName().equals("sensortype")) { + ((FreeTextStaticProperty)property).setValue("bla"); + } + } + + + + + String jsonld = JsonLdUtils.toJsonLD(adapterDescription); + + +// String s = Request.Post("http://localhost:8099/api/v1/riemer@fzi.de/master/adapters/") +//// .setHeader("content-type", "applicaito") +// .bodyString(jsonld, ContentType("application/ld+json", Consts.UTF_8)) +// .connectTimeout(1000) +// .socketTimeout(100000) +// .execute().returnContent().asString(); + +// String tmp = "{\"@context\":{\"sp\":\"https://streampipes.org/vocabulary/v1/\",\"spi\":\"urn:streampipes.org:spi:\",\"foaf\":\"http://xmlns.com/foaf/0.1/\"},\"@graph\":[{\"@id\":\"http://streampipes.org/genericadapterstreamdescription\",\"@type\":\"sp:GenericAdapterStreamDescription\",\"http://www.w3.org/2000/01/rdf-schema#description\":\"This is the description for the Apache Kafka protocol\",\"http://www.w3.org/2000/01/rdf-schema#label\":\"Apache Kafka (Stream)\",\"sp:config\":[],\"sp:hasFormat\":{\"@id\":\"sp:format/json/object\"},\"sp:hasProtocol\":{\"@id\":\"sp:protocol/stream/kafka\"},\"sp:hasUri\":\"http://streampipes.org/genericadapterstreamdescription\",\"sp:userName\":\"riemer@fzi.de\"},{\"@id\":\"sp:format/json/object\",\"@type\":\"sp:FormatDescription\",\"http://www.w3.org/2000/01/rdf-schema#description\":\"This is the descriptionfor json format\",\"http://www.w3.org/2000/01/rdf-schema#label\":\"Json Object\",\"sp:config\":[],\"sp:hasUri\":\"https://streampipes.org/vocabulary/v1/format/json/object\"},{\"@id\":\"sp:protocol/stream/kafka\",\"@type\":\"sp:ProtocolDescription\",\"http://www.w3.org/2000/01/rdf-schema#description\":\"This is the description for the Apache Kafka protocol\",\"http://www.w3.org/2000/01/rdf-schema#label\":\"Apache Kafka (Stream)\",\"sp:config\":[{\"@id\":\"spi:freetextstaticproperty:EMjdqQ\"},{\"@id\":\"spi:freetextstaticproperty:nHqyCU\"}],\"sp:hasUri\":\"https://streampipes.org/vocabulary/v1/protocol/stream/kafka\",\"sp:sourceType\":\"STREAM\"},{\"@id\":\"spi:freetextstaticproperty:EMjdqQ\",\"@type\":\"sp:FreeTextStaticProperty\",\"http://www.w3.org/2000/01/rdf-schema#description\":\"Topic in the broker\",\"http://www.w3.org/2000/01/rdf-schema#label\":\"Topic\",\"sp:hasValue\":\"org.streampipes.examples.waterlevel\",\"sp:internalName\":\"topic\",\"sp:requiredDomainProperty\":\"\"},{\"@id\":\"spi:freetextstaticproperty:nHqyCU\",\"@type\":\"sp:FreeTextStaticProperty\",\"http://www.w3.org/2000/01/rdf-schema#description\":\"This property defines the URL of the Kafka broker.\",\"http://www.w3.org/2000/01/rdf-schema#label\":\"Broker URL\",\"sp:hasValue\":\"ipe-koi04.fzi.de:9092\",\"sp:internalName\":\"broker_url\",\"sp:requiredDomainProperty\":\"\"}]}"; + + HttpPost post = new HttpPost("http://localhost:8099/api/v1/riemer@fzi.de/master/adapters/"); + Header headers[] = { + new BasicHeader("Content-type", "application/ld+json"), + }; + post.setHeaders(headers); + post.setEntity(new StringEntity(jsonld)); + + HttpClient client = HttpClients.custom().build(); + HttpResponse response = client.execute(post); + + System.out.println(response); + } + + private static void addAdapter() throws IOException { + + ProtocolDescription protocolDescription = new KafkaProtocol().declareModel(); + + + // Set broker URL + for (StaticProperty property: protocolDescription.getConfig()) { + if (property.getInternalName().equals("broker_url")) { + ((FreeTextStaticProperty)property).setValue("ipe-koi04.fzi.de:9092"); + } + + if (property.getInternalName().equals("topic")) { + ((FreeTextStaticProperty)property).setValue("org.streampipes.examples.waterlevel"); + } + } + + // Set topic + + FormatDescription formatDescription = new JsonObjectFormat().declareModel(); + + GenericAdapterStreamDescription genericSetDescription = new GenericAdapterStreamDescription(); + + genericSetDescription.setProtocolDescription(protocolDescription); + genericSetDescription.setFormatDescription(formatDescription); + + String jsonld = JsonLdUtils.toJsonLD(genericSetDescription); + + +// String s = Request.Post("http://localhost:8099/api/v1/riemer@fzi.de/master/adapters/") +//// .setHeader("content-type", "applicaito") +// .bodyString(jsonld, ContentType("application/ld+json", Consts.UTF_8)) +// .connectTimeout(1000) +// .socketTimeout(100000) +// .execute().returnContent().asString(); + +// String tmp = "{\"@context\":{\"sp\":\"https://streampipes.org/vocabulary/v1/\",\"spi\":\"urn:streampipes.org:spi:\",\"foaf\":\"http://xmlns.com/foaf/0.1/\"},\"@graph\":[{\"@id\":\"http://streampipes.org/genericadapterstreamdescription\",\"@type\":\"sp:GenericAdapterStreamDescription\",\"http://www.w3.org/2000/01/rdf-schema#description\":\"This is the description for the Apache Kafka protocol\",\"http://www.w3.org/2000/01/rdf-schema#label\":\"Apache Kafka (Stream)\",\"sp:config\":[],\"sp:hasFormat\":{\"@id\":\"sp:format/json/object\"},\"sp:hasProtocol\":{\"@id\":\"sp:protocol/stream/kafka\"},\"sp:hasUri\":\"http://streampipes.org/genericadapterstreamdescription\",\"sp:userName\":\"riemer@fzi.de\"},{\"@id\":\"sp:format/json/object\",\"@type\":\"sp:FormatDescription\",\"http://www.w3.org/2000/01/rdf-schema#description\":\"This is the descriptionfor json format\",\"http://www.w3.org/2000/01/rdf-schema#label\":\"Json Object\",\"sp:config\":[],\"sp:hasUri\":\"https://streampipes.org/vocabulary/v1/format/json/object\"},{\"@id\":\"sp:protocol/stream/kafka\",\"@type\":\"sp:ProtocolDescription\",\"http://www.w3.org/2000/01/rdf-schema#description\":\"This is the description for the Apache Kafka protocol\",\"http://www.w3.org/2000/01/rdf-schema#label\":\"Apache Kafka (Stream)\",\"sp:config\":[{\"@id\":\"spi:freetextstaticproperty:EMjdqQ\"},{\"@id\":\"spi:freetextstaticproperty:nHqyCU\"}],\"sp:hasUri\":\"https://streampipes.org/vocabulary/v1/protocol/stream/kafka\",\"sp:sourceType\":\"STREAM\"},{\"@id\":\"spi:freetextstaticproperty:EMjdqQ\",\"@type\":\"sp:FreeTextStaticProperty\",\"http://www.w3.org/2000/01/rdf-schema#description\":\"Topic in the broker\",\"http://www.w3.org/2000/01/rdf-schema#label\":\"Topic\",\"sp:hasValue\":\"org.streampipes.examples.waterlevel\",\"sp:internalName\":\"topic\",\"sp:requiredDomainProperty\":\"\"},{\"@id\":\"spi:freetextstaticproperty:nHqyCU\",\"@type\":\"sp:FreeTextStaticProperty\",\"http://www.w3.org/2000/01/rdf-schema#description\":\"This property defines the URL of the Kafka broker.\",\"http://www.w3.org/2000/01/rdf-schema#label\":\"Broker URL\",\"sp:hasValue\":\"ipe-koi04.fzi.de:9092\",\"sp:internalName\":\"broker_url\",\"sp:requiredDomainProperty\":\"\"}]}"; + + HttpPost post = new HttpPost("http://localhost:8099/api/v1/riemer@fzi.de/master/adapters/"); + Header headers[] = { + new BasicHeader("Content-type", "application/ld+json"), + }; + post.setHeaders(headers); + post.setEntity(new StringEntity(jsonld)); + + HttpClient client = HttpClients.custom().build(); + HttpResponse response = client.execute(post); + + System.out.println(response); + } + + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/Adapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/Adapter.java new file mode 100644 index 0000000000..ee7f887fc3 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/Adapter.java @@ -0,0 +1,85 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.model.connect.guess.GuessSchema; + +public abstract class Adapter { + Logger logger = LoggerFactory.getLogger(Adapter.class); + + @Deprecated + protected String kafkaUrl; + + @Deprecated + protected String topic; + + private boolean debug; + + protected T adapterDescription; + + public Adapter(T adapterDescription, boolean debug) { + this.adapterDescription = adapterDescription; + this.debug = debug; + } + + public Adapter(T adapterDescription) { + this(adapterDescription, false); + } + + public Adapter(boolean debug) { + this.debug = debug; + } + + public Adapter() { + this(false); + } + + @Deprecated + public Adapter(String kafkaUrl, String topic, boolean debug) { + this.kafkaUrl = kafkaUrl; + this.topic = topic; + this.debug = debug; + } + + @Deprecated + public Adapter(String kafkaUrl, String topic) { + this(kafkaUrl, topic, false); + } + + public abstract T declareModel(); + + // Decide which adapter to call + public abstract void startAdapter() throws AdapterException; + + public abstract void stopAdapter() throws AdapterException; + + public abstract Adapter getInstance(T adapterDescription); + + public abstract GuessSchema getSchema(T adapterDescription) throws AdapterException; + + public abstract String getId(); + + public boolean isDebug() { + return debug; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/AdapterRegistry.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/AdapterRegistry.java new file mode 100644 index 0000000000..6a8761e3ef --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/AdapterRegistry.java @@ -0,0 +1,118 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter; + +import org.streampipes.connect.adapter.generic.GenericDataSetAdapter; +import org.streampipes.connect.adapter.generic.GenericDataStreamAdapter; +import org.streampipes.connect.adapter.generic.format.Format; +import org.streampipes.connect.adapter.generic.format.Parser; +import org.streampipes.connect.adapter.generic.format.csv.CsvFormat; +import org.streampipes.connect.adapter.generic.format.csv.CsvParser; +import org.streampipes.connect.adapter.generic.format.geojson.GeoJsonFormat; +import org.streampipes.connect.adapter.generic.format.geojson.GeoJsonParser; +import org.streampipes.connect.adapter.generic.format.json.arraykey.JsonFormat; +import org.streampipes.connect.adapter.generic.format.json.arraykey.JsonParser; +import org.streampipes.connect.adapter.generic.format.json.arraynokey.JsonArrayFormat; +import org.streampipes.connect.adapter.generic.format.json.arraynokey.JsonArrayParser; +import org.streampipes.connect.adapter.generic.format.json.object.JsonObjectFormat; +import org.streampipes.connect.adapter.generic.format.json.object.JsonObjectParser; +import org.streampipes.connect.adapter.generic.format.xml.XmlFormat; +import org.streampipes.connect.adapter.generic.format.xml.XmlParser; +import org.streampipes.connect.adapter.generic.protocol.Protocol; +import org.streampipes.connect.adapter.generic.protocol.set.FileProtocol; +import org.streampipes.connect.adapter.generic.protocol.set.HttpProtocol; +import org.streampipes.connect.adapter.generic.protocol.stream.HttpStreamProtocol; +import org.streampipes.connect.adapter.generic.protocol.stream.KafkaProtocol; +import org.streampipes.connect.adapter.generic.protocol.stream.MqttProtocol; +import org.streampipes.connect.adapter.specific.gdelt.GdeltAdapter; +import org.streampipes.connect.adapter.specific.nswaustralia.trafficcamera.NswTrafficCameraAdapter; +import org.streampipes.connect.adapter.specific.sensemap.OpenSenseMapAdapter; +import org.streampipes.connect.adapter.specific.twitter.TwitterAdapter; +import org.streampipes.model.connect.adapter.AdapterDescription; + +import java.util.HashMap; +import java.util.Map; + +/** + * Contains all implemented adapters + */ +public class AdapterRegistry { + + public static Map getAllAdapters() { + Map allAdapters = new HashMap<>(); + +// allAdapters.put(GenericDataSetAdapter.ID, new GenericDataSetAdapter()); + allAdapters.put(GenericDataStreamAdapter.ID, new GenericDataStreamAdapter()); +// allAdapters.put(TwitterAdapter.ID, new TwitterAdapter()); + allAdapters.put(OpenSenseMapAdapter.ID, new OpenSenseMapAdapter()); + allAdapters.put(GdeltAdapter.ID, new GdeltAdapter()); +// allAdapters.put(NswTrafficCameraAdapter.ID, new NswTrafficCameraAdapter()); + + return allAdapters; + } + + public static Map getAllFormats() { + Map allFormats = new HashMap<>(); + + allFormats.put(JsonFormat.ID, new JsonFormat()); + allFormats.put(JsonObjectFormat.ID, new JsonObjectFormat()); + allFormats.put(JsonArrayFormat.ID, new JsonArrayFormat()); +// allFormats.put(CsvFormat.ID, new CsvFormat()); + allFormats.put(GeoJsonFormat.ID, new GeoJsonFormat()); +// allFormats.put(XmlFormat.ID, new XmlFormat()); + + + return allFormats; + } + + public static Map getAllParsers() { + Map allParsers = new HashMap<>(); + + allParsers.put(JsonFormat.ID, new JsonParser()); + allParsers.put(JsonObjectFormat.ID, new JsonObjectParser()); + allParsers.put(JsonArrayFormat.ID, new JsonArrayParser()); +// allParsers.put(CsvFormat.ID, new CsvParser()); + allParsers.put(GeoJsonFormat.ID, new GeoJsonParser()); +// allParsers.put(XmlFormat.ID, new XmlParser()); + + return allParsers; + } + + public static Map getAllProtocols() { + Map allProtocols = new HashMap<>(); + +// allProtocols.put(HttpProtocol.ID, new HttpProtocol()); +// allProtocols.put(FileProtocol.ID, new FileProtocol()); + allProtocols.put(KafkaProtocol.ID, new KafkaProtocol()); + allProtocols.put(MqttProtocol.ID, new MqttProtocol()); + allProtocols.put(HttpStreamProtocol.ID, new HttpStreamProtocol()); + + return allProtocols; + } + + public static Adapter getAdapter(AdapterDescription adapterDescription) { + if (adapterDescription != null) { + Map adapterMap = AdapterRegistry.getAllAdapters(); + + return adapterMap.get(adapterDescription.getAdapterId()).getInstance(adapterDescription); + } else { + return null; + } + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/GroundingService.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/GroundingService.java new file mode 100644 index 0000000000..f4db17fe76 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/GroundingService.java @@ -0,0 +1,57 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter; + +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.model.grounding.EventGrounding; +import org.streampipes.model.grounding.KafkaTransportProtocol; +import org.streampipes.model.grounding.SimpleTopicDefinition; +import org.streampipes.model.grounding.TopicDefinition; +import org.streampipes.model.schema.EventSchema; + +import java.util.UUID; + +public class GroundingService { + + public static String extractBroker(AdapterDescription adapterDescription) { + String host = adapterDescription.getEventGrounding().getTransportProtocol().getBrokerHostname(); + int port = ((KafkaTransportProtocol) adapterDescription.getEventGrounding().getTransportProtocol()).getKafkaPort(); + return host + ":" + port; + } + + public static String extractTopic(AdapterDescription adapterDescription) { + return adapterDescription.getEventGrounding().getTransportProtocol().getTopicDefinition().getActualTopicName(); + } + + public static EventGrounding createEventGrounding(String kafkaHost, int kafkaPort, EventSchema eventSchema) { + EventGrounding eventGrounding = new EventGrounding(); + KafkaTransportProtocol transportProtocol = new KafkaTransportProtocol(); + transportProtocol.setBrokerHostname(kafkaHost); + transportProtocol.setKafkaPort(kafkaPort); + + String topic = "org.streampipes.connect." + UUID.randomUUID(); + System.out.println("Topic: " + topic); + TopicDefinition topicDefinition = new SimpleTopicDefinition(topic); + transportProtocol.setTopicDefinition(topicDefinition); + + eventGrounding.setTransportProtocol(transportProtocol); + + + return eventGrounding; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/GenericAdapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/GenericAdapter.java new file mode 100644 index 0000000000..21abcb0549 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/GenericAdapter.java @@ -0,0 +1,108 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.connect.adapter.AdapterRegistry; +import org.streampipes.connect.adapter.generic.format.Format; +import org.streampipes.connect.adapter.generic.format.Parser; +import org.streampipes.connect.adapter.generic.pipeline.AdapterPipeline; +import org.streampipes.connect.adapter.generic.pipeline.AdapterPipelineElement; +import org.streampipes.connect.adapter.generic.pipeline.elements.SendToKafkaAdapterSink; +import org.streampipes.connect.adapter.generic.pipeline.elements.TransformSchemaAdapterPipelineElement; +import org.streampipes.connect.adapter.generic.protocol.Protocol; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.model.connect.adapter.GenericAdapterDescription; +import org.streampipes.model.connect.guess.GuessSchema; + +import java.util.ArrayList; +import java.util.List; + +public abstract class GenericAdapter extends Adapter { + + private static final Logger logger = LoggerFactory.getLogger(Adapter.class); + + public GenericAdapter(T adapterDescription) { + super(adapterDescription); + } + + public GenericAdapter(T adapterDescription, boolean debug) { + super(adapterDescription, debug); + } + + public GenericAdapter() { + super(); + } + + public abstract GenericAdapterDescription getAdapterDescription(); + + public abstract void setProtocol(Protocol protocol); + + @Override + public void startAdapter() throws AdapterException { + + GenericAdapterDescription adapterDescription = getAdapterDescription(); + + + Parser parser = getParser(adapterDescription); + Format format = getFormat(adapterDescription); + + Protocol protocol = getProtocol(adapterDescription, format, parser); + setProtocol(protocol); + + logger.debug("Start adatper with format: " + format.getId() + " and " + protocol.getId()); + + + List pipelineElements = new ArrayList<>(); + pipelineElements.add(new TransformSchemaAdapterPipelineElement(adapterDescription.getRules())); + pipelineElements.add(new SendToKafkaAdapterSink((AdapterDescription) adapterDescription)); + + AdapterPipeline adapterPipeline = new AdapterPipeline(pipelineElements); + + protocol.run(adapterPipeline); + } + + + @Override + public GuessSchema getSchema(T adapterDescription) throws AdapterException { + Parser parser = getParser((GenericAdapterDescription) adapterDescription); + Format format = getFormat((GenericAdapterDescription) adapterDescription); + + Protocol protocol = getProtocol((GenericAdapterDescription) adapterDescription, format, parser); + + logger.debug("Extract schema with format: " + format.getId() + " and " + protocol.getId()); + + return protocol.getGuessSchema(); + } + + private Parser getParser(GenericAdapterDescription adapterDescription) throws AdapterException { + if (adapterDescription.getFormatDescription() == null) throw new AdapterException("Format description of Adapter ist empty"); + return AdapterRegistry.getAllParsers().get(adapterDescription.getFormatDescription().getUri()).getInstance(adapterDescription.getFormatDescription()); + } + + private Format getFormat(GenericAdapterDescription adapterDescription) { + return AdapterRegistry.getAllFormats().get(adapterDescription.getFormatDescription().getUri()).getInstance(adapterDescription.getFormatDescription()); + } + + private Protocol getProtocol(GenericAdapterDescription adapterDescription, Format format, Parser parser) { + return AdapterRegistry.getAllProtocols().get(adapterDescription.getProtocolDescription().getUri()).getInstance(adapterDescription.getProtocolDescription(), parser, format); + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/GenericDataSetAdapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/GenericDataSetAdapter.java new file mode 100644 index 0000000000..f6752ff33b --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/GenericDataSetAdapter.java @@ -0,0 +1,79 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.connect.adapter.generic.protocol.Protocol; +import org.streampipes.model.connect.adapter.GenericAdapterDescription; +import org.streampipes.model.connect.adapter.GenericAdapterSetDescription; + +public class GenericDataSetAdapter extends GenericAdapter { + + public static final String ID = GenericAdapterSetDescription.ID; + + private Protocol protocol; + + Logger logger = LoggerFactory.getLogger(Adapter.class); + + public GenericDataSetAdapter() { + super(); + } + + + public GenericDataSetAdapter(GenericAdapterSetDescription adapterDescription, boolean debug) { + super(adapterDescription, debug); + } + + public GenericDataSetAdapter(GenericAdapterSetDescription adapterDescription) { + super(adapterDescription); + } + + @Override + public GenericAdapterSetDescription declareModel() { + GenericAdapterSetDescription adapterDescription = new GenericAdapterSetDescription(); + adapterDescription.setAdapterId(GenericAdapterSetDescription.ID); + adapterDescription.setUri(GenericAdapterSetDescription.ID); + return adapterDescription; + } + + @Override + public Adapter getInstance(GenericAdapterSetDescription adapterDescription) { + return new GenericDataSetAdapter(adapterDescription); + } + + @Override + public String getId() { + return ID; + } + + public void stopAdapter() { + protocol.stop(); + } + + @Override + public GenericAdapterDescription getAdapterDescription() { + return adapterDescription; + } + + @Override + public void setProtocol(Protocol protocol) { + this.protocol = protocol; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/GenericDataStreamAdapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/GenericDataStreamAdapter.java new file mode 100644 index 0000000000..b6656ee81f --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/GenericDataStreamAdapter.java @@ -0,0 +1,78 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.connect.adapter.generic.protocol.Protocol; +import org.streampipes.model.connect.adapter.GenericAdapterDescription; +import org.streampipes.model.connect.adapter.GenericAdapterStreamDescription; + +public class GenericDataStreamAdapter extends GenericAdapter { + + public static final String ID = GenericAdapterStreamDescription.ID; + + private Protocol protocol; + + Logger logger = LoggerFactory.getLogger(Adapter.class); + + public GenericDataStreamAdapter() { + super(); + } + + public GenericDataStreamAdapter(GenericAdapterStreamDescription adapterDescription, boolean debug) { + super(adapterDescription, debug); + } + + public GenericDataStreamAdapter(GenericAdapterStreamDescription adapterDescription) { + super(adapterDescription); + } + + @Override + public GenericAdapterStreamDescription declareModel() { + GenericAdapterStreamDescription adapterDescription = new GenericAdapterStreamDescription(); + adapterDescription.setAdapterId(GenericAdapterStreamDescription.ID); + adapterDescription.setUri(GenericAdapterStreamDescription.ID); + return adapterDescription; + } + + @Override + public Adapter getInstance(GenericAdapterStreamDescription adapterDescription) { + return new GenericDataStreamAdapter(adapterDescription); + } + + @Override + public String getId() { + return ID; + } + + public void stopAdapter() { + protocol.stop(); + } + + @Override + public GenericAdapterDescription getAdapterDescription() { + return adapterDescription; + } + + @Override + public void setProtocol(Protocol protocol) { + this.protocol = protocol; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/Format.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/Format.java new file mode 100644 index 0000000000..ef65b5e802 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/Format.java @@ -0,0 +1,40 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.format; + + +import org.streampipes.model.connect.grounding.FormatDescription; + +import java.util.Map; + +public abstract class Format { + + public abstract Format getInstance(FormatDescription formatDescription); + + public abstract FormatDescription declareModel(); + + public abstract String getId(); + + /** + * This method parses a byte[] and transforms the event object into a serialized version of the internal + * representation + */ + public abstract Map parse(byte[] object); + + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/Parser.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/Parser.java new file mode 100644 index 0000000000..9f5cebd788 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/Parser.java @@ -0,0 +1,49 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.format; + + +import org.streampipes.connect.GetNEvents; +import org.streampipes.model.connect.grounding.FormatDescription; +import org.streampipes.model.schema.EventSchema; +import org.streampipes.connect.EmitBinaryEvent; + +import java.io.InputStream; +import java.util.List; + +public abstract class Parser { + + public abstract Parser getInstance(FormatDescription formatDescription); + + public abstract void parse(InputStream data, EmitBinaryEvent emitBinaryEvent); + + public List parseNEvents(InputStream data, int n) { + GetNEvents gne = new GetNEvents(n); + + parse(data, gne); + + return gne.getEvents(); + } + + /** + * Pass one event to Parser to get the event schema + * @param oneEvent + * @return + */ + public abstract EventSchema getEventSchema(List oneEvent); +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/csv/CsvFormat.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/csv/CsvFormat.java new file mode 100644 index 0000000000..5ad53cb8b8 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/csv/CsvFormat.java @@ -0,0 +1,118 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.format.csv; + + +import org.streampipes.connect.adapter.generic.format.Format; +import org.streampipes.connect.adapter.generic.sdk.ParameterExtractor; +import org.streampipes.model.connect.grounding.FormatDescription; +import org.streampipes.model.staticproperty.AnyStaticProperty; +import org.streampipes.model.staticproperty.FreeTextStaticProperty; +import org.streampipes.model.staticproperty.Option; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +public class CsvFormat extends Format { + + public static String HEADER_NAME = "header"; + public static String DELIMITER_NAME = "delimiter"; + + private String[] keyValues = null; + private String delimiter; + private Boolean header; + + public static final String ID = "https://streampipes.org/vocabulary/v1/format/csv"; + + public CsvFormat() { + + } + + public CsvFormat(String delimiter, Boolean header) { + this.delimiter = delimiter; + this.header = header; + } + + @Override + public Format getInstance(FormatDescription formatDescription) { + ParameterExtractor extractor = new ParameterExtractor(formatDescription.getConfig()); + boolean header = extractor.singleValue(HEADER_NAME) == null ? false : true; + String delimiter = extractor.singleValue(DELIMITER_NAME); + + return new CsvFormat(delimiter, header); + } + + @Override + public Map parse(byte[] object) { + String[] arr = new String(object).split(delimiter); + Map map = new HashMap<>(); + + if (keyValues == null && !header) { + keyValues = new String[arr.length]; + for (int i = 0; i < arr.length; i++) { + keyValues[i] = "key_" + i; + } + } + + if (keyValues == null) { + keyValues = new String[arr.length]; + for (int i = 0; i < arr.length; i++) { + keyValues[i] = arr[i]; + } + + } else { + for (int i = 0; i <= arr.length - 1; i++) { + map.put(keyValues[i], arr[i]); + } + + } + + if (map.keySet().size() == 0) { + return null; + } else { + return map; + } + } + + @Override + public FormatDescription declareModel() { + FormatDescription fd = new FormatDescription(ID, "Csv", "This is the description" + + "for csv format"); + FreeTextStaticProperty delimiterProperty = new FreeTextStaticProperty("delimiter", + "Delimiter", "The delimiter for json. Mostly either , or ;"); + + + AnyStaticProperty offset = new AnyStaticProperty("header", "Header", "Does the CSV file include a header or not"); + offset.setOptions(Arrays.asList(new Option("Header","Header"))); + +// FreeTextStaticProperty offset = new FreeTextStaticProperty("header", +// "Includes Header", "Description"); + + fd.addConfig(delimiterProperty); + fd.addConfig(offset); + + return fd; + } + + + @Override + public String getId() { + return ID; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/csv/CsvParser.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/csv/CsvParser.java new file mode 100644 index 0000000000..543aca615c --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/csv/CsvParser.java @@ -0,0 +1,128 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.format.csv; + + +import org.streampipes.connect.EmitBinaryEvent; +import org.streampipes.connect.adapter.generic.format.Parser; +import org.streampipes.connect.adapter.generic.sdk.ParameterExtractor; +import org.streampipes.model.connect.grounding.FormatDescription; +import org.streampipes.model.schema.EventPropertyPrimitive; +import org.streampipes.model.schema.EventSchema; +import org.streampipes.vocabulary.SO; +import org.streampipes.vocabulary.XSD; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.List; + + +public class CsvParser extends Parser { + + private String delimiter; + private Boolean header; + + public CsvParser() { + } + + public CsvParser(String delimiter, Boolean header) { + this.delimiter = delimiter; + this.header = header; + } + + @Override + public Parser getInstance(FormatDescription formatDescription) { + ParameterExtractor extractor = new ParameterExtractor(formatDescription.getConfig()); + + boolean header = extractor.singleValue(CsvFormat.HEADER_NAME) == null ? false : true; + String delimiter = extractor.singleValue(CsvFormat.DELIMITER_NAME); + + return new CsvParser(delimiter, header); + } + + @Override + public void parse(InputStream data, EmitBinaryEvent emitBinaryEvent) { + BufferedReader reader = new BufferedReader(new InputStreamReader(data)); + + boolean result = true; + + try { + while (reader.ready() && result) { + String s = reader.readLine(); + byte[] parseResult = s.getBytes(); + if (parseResult != null) { + result = emitBinaryEvent.emit(parseResult); + } + } + } catch (IOException e) { + e.printStackTrace(); + } + + } + + @Override + public EventSchema getEventSchema(List oneEvent) { + String[] keys; + String[] data; + + if (!this.header) { + keys = new String (oneEvent.get(0)).split(delimiter); + data = new String (oneEvent.get(1)).split(delimiter); + } else { + data = new String (oneEvent.get(0)).split(delimiter); + keys = new String[data.length]; + for (int i = 0; i < data.length; i++) { + keys[i] = "key_" + i; + } + } + + EventSchema resultSchema = new EventSchema(); + for (int i = 0; i < keys.length; i++) { + EventPropertyPrimitive p = new EventPropertyPrimitive(); + p.setRuntimeName(keys[i]); + p.setRuntimeType(getTypeString(data[i])); + resultSchema.addEventProperty(p); + } + + return resultSchema; + } + + public static void main(String... args) { + System.out.println(Boolean.parseBoolean("2")); + System.out.println(Integer.parseInt("dd")); + } + + private String getTypeString(String o) { + + try { + Double.parseDouble(o); + return SO.Number.toString(); + } catch (NumberFormatException e) { + + } + + if (o.equalsIgnoreCase("true") || o.equalsIgnoreCase("false")) { + return XSD._boolean.toString(); + } + + return XSD._string.toString(); + } + +} \ No newline at end of file diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/geojson/GeoJsonFormat.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/geojson/GeoJsonFormat.java new file mode 100644 index 0000000000..7f991292ca --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/geojson/GeoJsonFormat.java @@ -0,0 +1,131 @@ +/* +Copyright 2018 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package org.streampipes.connect.adapter.generic.format.geojson; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.commons.exceptions.SpRuntimeException; +import org.streampipes.connect.adapter.generic.format.Format; +import org.streampipes.dataformat.json.JsonDataFormatDefinition; +import org.streampipes.model.connect.grounding.FormatDescription; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class GeoJsonFormat extends Format { + + public static final String ID = "https://streampipes.org/vocabulary/v1/format/geojson"; + private static final Logger logger = LoggerFactory.getLogger(GeoJsonFormat.class); + + @Override + public FormatDescription declareModel() { + FormatDescription description = new FormatDescription(ID, "GeoJSON", "This is the description " + + "for the geoJSON format"); + return description; + } + + @Override + public Format getInstance(FormatDescription formatDescription) { + return new GeoJsonFormat(); + } + + @Override + public String getId() { + return ID; + } + + @Override + public Map parse(byte[] object) { + JsonDataFormatDefinition jsonDefinition = new JsonDataFormatDefinition(); + Map result = null; + + try { + result = jsonDefinition.toMap(object); + } catch (SpRuntimeException e) { + e.printStackTrace(); + } + + return geoJsonFormatter(result); + } + + private Map geoJsonFormatter(Map map) { + Map geoJson = new HashMap(); + Boolean foundGeometry = false; + Boolean foundProperties = false; + + for (Map.Entry entry : map.entrySet()) + { + if (entry.getKey().equalsIgnoreCase("GEOMETRY")) { + foundGeometry = true; + geoJson.putAll(formatGeometryField( (Map) entry.getValue())); + } + if(entry.getKey().equalsIgnoreCase("PROPERTIES")) { + foundProperties = true; + for (Map.Entry innerEntry : ((Map) entry.getValue()).entrySet()) { + geoJson.put(innerEntry.getKey(), innerEntry.getValue()); + } + } + } + + if (!foundGeometry) logger.warn("Geometry field not found"); + if (!foundProperties) logger.warn("Property field not found"); + + return geoJson; + } + + private Map formatGeometryField(Map map) { + Map geometryFields = new HashMap(); + + String type = (String) map.get("type"); + + if(type.equalsIgnoreCase("POINT")) { + List coordinates = (List) map.get("coordinates"); + + try { + geometryFields.put("longitude", coordinates.get(0)); + geometryFields.put("latitude", coordinates.get(1)); + if (coordinates.size() == 3) { + geometryFields.put("altitude", coordinates.get(2)); + } + } catch (IndexOutOfBoundsException e) { + logger.error(e.getMessage()); + } + + } else if (type.equalsIgnoreCase("LINESTRING")) { + geometryFields.put("coordinatesLineString", map.get("coordinates").toString()); + + } else if (type.equalsIgnoreCase("POLYGON")) { + geometryFields.put("coordinatesPolygon", map.get("coordinates").toString()); + + } else if (type.equalsIgnoreCase("MULTIPOINT")) { + geometryFields.put("coordinatesMultiPoint", map.get("coordinates").toString()); + + } else if (type.equalsIgnoreCase("MULTILINESTRING")) { + geometryFields.put("coordinatesMultiString", map.get("coordinates").toString()); + + } else if (type.equalsIgnoreCase("MULTIPOLYGON")) { + geometryFields.put("coordinatesMultiPolygon", map.get("coordinates").toString()); + + } else { + logger.error(type + "is not a suppported field type"); + } + + return geometryFields; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/geojson/GeoJsonParser.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/geojson/GeoJsonParser.java new file mode 100644 index 0000000000..0cda7504d2 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/geojson/GeoJsonParser.java @@ -0,0 +1,143 @@ +/* +Copyright 2018 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package org.streampipes.connect.adapter.generic.format.geojson; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Charsets; +import com.google.common.io.CharStreams; +import com.google.gson.Gson; +import org.geojson.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.connect.EmitBinaryEvent; +import org.streampipes.connect.adapter.generic.format.Parser; +import org.streampipes.connect.adapter.generic.format.util.JsonEventProperty; +import org.streampipes.model.connect.grounding.FormatDescription; +import org.streampipes.model.schema.*; +import org.streampipes.vocabulary.SO; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.*; + +public class GeoJsonParser extends Parser { + + Logger logger = LoggerFactory.getLogger(GeoJsonParser.class); + + @Override + public Parser getInstance(FormatDescription formatDescription) { + return new GeoJsonParser(); + } + + @Override + public void parse(InputStream data, EmitBinaryEvent emitBinaryEvent) { + FeatureCollection geoFeature; + Gson gson = new Gson(); + + try { + String dataString = CharStreams.toString(new InputStreamReader(data, Charsets.UTF_8)); + List features = (List) gson.fromJson(dataString, HashMap.class).get("features"); + + for(Map feature : features) { + byte[] bytes = gson.toJson(feature).getBytes(); + emitBinaryEvent.emit(bytes); + } + + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Override + public EventSchema getEventSchema(List oneEvent) { + EventSchema resultSchema = new EventSchema(); + + Feature geoFeature = null; + try { + geoFeature = new ObjectMapper().readValue(oneEvent.get(0), Feature.class); + + } catch (IOException e) { + logger.error(e.toString()); + } + + for (Map.Entry entry : geoFeature.getProperties().entrySet()) { + EventProperty p = JsonEventProperty.getEventProperty(entry.getKey(), entry.getValue()); + resultSchema.addEventProperty(p); + } + + List eventProperties = parseGeometryField(geoFeature); + eventProperties.forEach(eventProperty -> resultSchema.addEventProperty(eventProperty)); + + return resultSchema; + } + + private List parseGeometryField(Feature geoFeature) { + List eventProperties = new LinkedList<>(); + + if(geoFeature.getGeometry() instanceof Point) { + Point point = (Point) geoFeature.getGeometry(); + eventProperties.add(getEventPropertyGeoJson("longitude", point.getCoordinates().getLongitude(), SO.Longitude)); + eventProperties.add(getEventPropertyGeoJson("latitude", point.getCoordinates().getLatitude(), SO.Latitude)); + if (point.getCoordinates().hasAltitude()) { + eventProperties.add(getEventPropertyGeoJson("altitude", point.getCoordinates().getAltitude(), SO.Altitude)); + } + + } else if (geoFeature.getGeometry() instanceof LineString) { + LineString lineString = (LineString) geoFeature.getGeometry(); + eventProperties.add(JsonEventProperty.getEventProperty("coorindatesLineString", lineString.getCoordinates())); + + } else if (geoFeature.getGeometry() instanceof Polygon) { + Polygon polygon = (Polygon) geoFeature.getGeometry(); + eventProperties.add(JsonEventProperty.getEventProperty("coorindatesPolygon", polygon.getCoordinates())); + + } else if (geoFeature.getGeometry() instanceof MultiPoint) { + MultiPoint multiPoint = (MultiPoint) geoFeature.getGeometry(); + eventProperties.add(JsonEventProperty.getEventProperty("coorindatesMultiPoint", multiPoint.getCoordinates())); + + } else if (geoFeature.getGeometry() instanceof MultiLineString) { + MultiLineString multiLineString = (MultiLineString) geoFeature.getGeometry(); + eventProperties.add(JsonEventProperty.getEventProperty("coorindatesMultiLineString", multiLineString.getCoordinates())); + + } else if (geoFeature.getGeometry() instanceof MultiPolygon) { + MultiPolygon multiPolygon = (MultiPolygon) geoFeature.getGeometry(); + eventProperties.add(JsonEventProperty.getEventProperty("coorindatesMultiPolygon", multiPolygon.getCoordinates())); + } else { + logger.error("No geometry field found in geofeature: " + geoFeature.toString()); + } + + return eventProperties; + + } + + private EventProperty getEventPropertyGeoJson(String name, Object value, String domain) { + EventProperty eventProperty = JsonEventProperty.getEventProperty(name, value); + try { + ((EventPropertyPrimitive) eventProperty).setDomainProperties(Arrays.asList(new URI(domain))); + + } catch (URISyntaxException e) { + logger.error(e.getMessage()); + } + return eventProperty; + } + + + + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/json/AbstractJsonFormat.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/json/AbstractJsonFormat.java new file mode 100644 index 0000000000..d94bd7b645 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/json/AbstractJsonFormat.java @@ -0,0 +1,45 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.streampipes.connect.adapter.generic.format.json; + +import org.streampipes.commons.exceptions.SpRuntimeException; +import org.streampipes.connect.adapter.generic.format.Format; +import org.streampipes.dataformat.json.JsonDataFormatDefinition; +import org.streampipes.model.schema.EventSchema; + +import java.util.Map; + +public abstract class AbstractJsonFormat extends Format { + + @Override + public Map parse(byte[] object) { + EventSchema resultSchema = new EventSchema(); + + JsonDataFormatDefinition jsonDefinition = new JsonDataFormatDefinition(); + + Map result = null; + + try { + result = jsonDefinition.toMap(object); + } catch (SpRuntimeException e) { + e.printStackTrace(); + } + + return result; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/json/arraykey/JsonFormat.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/json/arraykey/JsonFormat.java new file mode 100644 index 0000000000..d8a32afee9 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/json/arraykey/JsonFormat.java @@ -0,0 +1,54 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.format.json.arraykey; + + +import org.streampipes.connect.adapter.generic.format.Format; +import org.streampipes.connect.adapter.generic.format.json.AbstractJsonFormat; +import org.streampipes.model.connect.grounding.FormatDescription; +import org.streampipes.model.staticproperty.FreeTextStaticProperty; + +public class JsonFormat extends AbstractJsonFormat { + + public static final String ID = "https://streampipes.org/vocabulary/v1/format/json/arraykey"; + + @Override + public Format getInstance(FormatDescription formatDescription) { + return new JsonFormat(); + } + + + @Override + public FormatDescription declareModel() { + FormatDescription fd = new FormatDescription(ID, "Json Array Key", "This is the description" + + "for json format"); + FreeTextStaticProperty keyProperty = new FreeTextStaticProperty("key", + "key", "Key of the json part that should be parsed"); + + fd.addConfig(keyProperty); + + return fd; + } + + @Override + public String getId() { + return ID; + } + + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/json/arraykey/JsonParser.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/json/arraykey/JsonParser.java new file mode 100644 index 0000000000..8a2d37050e --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/json/arraykey/JsonParser.java @@ -0,0 +1,232 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.format.json.arraykey; + + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.commons.exceptions.SpRuntimeException; +import org.streampipes.connect.EmitBinaryEvent; +import org.streampipes.connect.adapter.generic.format.Parser; +import org.streampipes.connect.adapter.generic.format.util.JsonEventProperty; +import org.streampipes.connect.adapter.generic.sdk.ParameterExtractor; +import org.streampipes.dataformat.json.JsonDataFormatDefinition; +import org.streampipes.model.connect.grounding.FormatDescription; +import org.streampipes.model.schema.EventProperty; +import org.streampipes.model.schema.EventSchema; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.json.Json; +import javax.json.stream.JsonParserFactory; + +public class JsonParser extends Parser { + + Logger logger = LoggerFactory.getLogger(JsonParser.class); + private boolean isArray; + private String key = ""; + + + @Override + public Parser getInstance(FormatDescription formatDescription) { + ParameterExtractor extractor = new ParameterExtractor(formatDescription.getConfig()); + String key = extractor.singleValue("key"); + + return new JsonParser(true, key); + } + + /** + * Use this constructor when just a specific key of the object should be parsed + * + * @param isArray + * @param key + */ + public JsonParser(boolean isArray, String key) { + this.isArray = isArray; + this.key = key; + } + + public JsonParser() { + } + + @Override + public void parse(InputStream data, EmitBinaryEvent emitBinaryEvent) { + JsonParserFactory factory = Json.createParserFactory(null); + String s = data.toString(); + javax.json.stream.JsonParser jsonParser = factory.createParser(data); + + //Find the array in the document by key + if (!key.equals("")) { + boolean found = false; + while (jsonParser.hasNext() && !found) { + javax.json.stream.JsonParser.Event event = jsonParser.next(); + if (event.equals(javax.json.stream.JsonParser.Event.KEY_NAME) && jsonParser.getString().equals(key)) { + found = true; + } + } + } + + // Find first event in array + boolean foundBeginning = false; + while (jsonParser.hasNext() && !foundBeginning) { + javax.json.stream.JsonParser.Event event = jsonParser.next(); + if (isArray && event.equals(javax.json.stream.JsonParser.Event.START_ARRAY)) { + foundBeginning = true; + } + } + + // Parse all events + JsonDataFormatDefinition jsonDefinition = new JsonDataFormatDefinition(); + boolean isEvent = true; + boolean result = true; + int objectCount = 0; + while (jsonParser.hasNext() && isEvent && result) { + Map objectMap = parseObject(jsonParser, true, 1); + if (objectMap != null) { + byte[] tmp = new byte[0]; + try { + tmp = jsonDefinition.fromMap(objectMap); + } catch (SpRuntimeException e) { + e.printStackTrace(); + } +// handleEvent(new EventObjectEndEvent(parseObject(tmp))); + // TODO decide what happens id emit returns false + result = emitBinaryEvent.emit(tmp); + } else { + isEvent = false; + } + + } + } + + @Override + public EventSchema getEventSchema(List oneEvent) { + EventSchema resultSchema = new EventSchema(); + + JsonDataFormatDefinition jsonDefinition = new JsonDataFormatDefinition(); + + Map exampleEvent = null; + + try { + exampleEvent = jsonDefinition.toMap(oneEvent.get(0)); + } catch (SpRuntimeException e) { + e.printStackTrace(); + } + + for (Map.Entry entry : exampleEvent.entrySet()) { +// System.out.println(entry.getKey() + "/" + entry.getValue()); + EventProperty p = JsonEventProperty.getEventProperty(entry.getKey(), entry.getValue()); + + resultSchema.addEventProperty(p); + + } + + return resultSchema; + } + + public Map parseObject(javax.json.stream.JsonParser jsonParser, boolean root, int start) { + // this variable is needed to skip the first object start + String mapKey = ""; + Map result = new HashMap<>(); + List arr = null; + + while (jsonParser.hasNext()) { + javax.json.stream.JsonParser.Event event = jsonParser.next(); + switch (event) { + case KEY_NAME: + mapKey = jsonParser.getString(); + logger.debug("key: " + mapKey); + break; + case START_OBJECT: + if (start == 0) { + Map ob = parseObject(jsonParser, false, 0); + if (arr == null) { + result.put(mapKey, ob); + } else { + arr.add(ob); + } + } else { + start--; + } + logger.debug("start object"); + break; + case END_OBJECT: + + logger.debug("end object"); + return result; + case START_ARRAY: + arr = new ArrayList<>(); + logger.debug("start array"); + break; + case END_ARRAY: + // Check if just the end of array is entered + if (result.keySet().size() == 0 && mapKey.equals("")) { + return null; + } + result.put(mapKey, arr); + arr = null; + logger.debug("end array"); + break; + case VALUE_TRUE: + if (arr == null) { + result.put(mapKey, true); + } else { + arr.add(true); + } + logger.debug("value: true"); + break; + case VALUE_FALSE: + if (arr == null) { + result.put(mapKey, false); + } else { + arr.add(false); + } + logger.debug("value: false"); + break; + case VALUE_STRING: + if (arr == null) { + result.put(mapKey, jsonParser.getString()); + } else { + arr.add(jsonParser.getString()); + } + logger.debug("value string: " + jsonParser.getString()); + break; + case VALUE_NUMBER: + if (arr == null) { + result.put(mapKey, jsonParser.getBigDecimal()); + } else { + arr.add(jsonParser.getBigDecimal()); + } + logger.debug("value number: " + jsonParser.getBigDecimal()); + break; + case VALUE_NULL: + logger.debug("value null"); + break; + default: + logger.error("Error: " + event + " event is not handled in the JSON parser"); + break; + } + } + + return result; + } +} \ No newline at end of file diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/json/arraynokey/JsonArrayFormat.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/json/arraynokey/JsonArrayFormat.java new file mode 100644 index 0000000000..37c7790a99 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/json/arraynokey/JsonArrayFormat.java @@ -0,0 +1,48 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.format.json.arraynokey; + + +import org.streampipes.connect.adapter.generic.format.Format; +import org.streampipes.connect.adapter.generic.format.json.AbstractJsonFormat; +import org.streampipes.model.connect.grounding.FormatDescription; + +public class JsonArrayFormat extends AbstractJsonFormat { + + public static final String ID = "https://streampipes.org/vocabulary/v1/format/json/arraynokey"; + + @Override + public Format getInstance(FormatDescription formatDescription) { + return new JsonArrayFormat(); + } + + @Override + public FormatDescription declareModel() { + FormatDescription fd = new FormatDescription(ID, "Json Array No Key", "This is the description" + + "for json format"); + + return fd; + } + + @Override + public String getId() { + return ID; + } + + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/firstconnector/format/json/JsonParser.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/json/arraynokey/JsonArrayParser.java similarity index 60% rename from streampipes-connect/src/main/java/org/streampipes/connect/firstconnector/format/json/JsonParser.java rename to streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/json/arraynokey/JsonArrayParser.java index c30c4927cc..0f394d7888 100644 --- a/streampipes-connect/src/main/java/org/streampipes/connect/firstconnector/format/json/JsonParser.java +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/json/arraynokey/JsonArrayParser.java @@ -1,50 +1,52 @@ -package org.streampipes.connect.firstconnector.format.json; +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.format.json.arraynokey; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.streampipes.commons.exceptions.SpRuntimeException; -import org.streampipes.connect.firstconnector.sdk.ParameterExtractor; -import org.streampipes.model.modelconnect.FormatDescription; +import org.streampipes.connect.EmitBinaryEvent; +import org.streampipes.connect.adapter.generic.format.Parser; +import org.streampipes.connect.adapter.generic.format.util.JsonEventProperty; +import org.streampipes.connect.adapter.generic.sdk.ParameterExtractor; import org.streampipes.dataformat.json.JsonDataFormatDefinition; +import org.streampipes.model.connect.grounding.FormatDescription; import org.streampipes.model.schema.*; -import org.streampipes.connect.EmitBinaryEvent; -import org.streampipes.connect.firstconnector.format.Parser; -import org.streampipes.vocabulary.SO; -import org.streampipes.vocabulary.XSD; - -import javax.json.*; +import javax.json.Json; import javax.json.stream.JsonParserFactory; import java.io.InputStream; import java.util.*; -public class JsonParser extends Parser { +public class JsonArrayParser extends Parser { - Logger logger = LoggerFactory.getLogger(JsonParser.class); - private boolean isArray; - private String key = ""; + Logger logger = LoggerFactory.getLogger(JsonArrayParser.class); @Override public Parser getInstance(FormatDescription formatDescription) { ParameterExtractor extractor = new ParameterExtractor(formatDescription.getConfig()); - String key = extractor.singleValue("key"); - return new JsonParser(true, key); + return new JsonArrayParser(); } - /** - * Use this constructor when just a specific key of the object should be parsed - * @param isArray - * @param key - */ - public JsonParser(boolean isArray, String key) { - this.isArray = isArray; - this.key = key; - } - - public JsonParser() { + public JsonArrayParser() { } @Override @@ -53,22 +55,11 @@ public void parse(InputStream data, EmitBinaryEvent emitBinaryEvent) { String s = data.toString(); javax.json.stream.JsonParser jsonParser = factory.createParser(data); - //Find the array in the document by key - if (!key.equals("")) { - boolean found = false; - while (jsonParser.hasNext() && !found) { - javax.json.stream.JsonParser.Event event = jsonParser.next(); - if (event.equals(javax.json.stream.JsonParser.Event.KEY_NAME) && jsonParser.getString().equals(key)) { - found = true; - } - } - } - // Find first event in array boolean foundBeginning = false; while (jsonParser.hasNext() && !foundBeginning ) { javax.json.stream.JsonParser.Event event = jsonParser.next(); - if (isArray && event.equals(javax.json.stream.JsonParser.Event.START_ARRAY)) { + if (event.equals(javax.json.stream.JsonParser.Event.START_ARRAY)) { foundBeginning = true; } } @@ -98,17 +89,15 @@ public void parse(InputStream data, EmitBinaryEvent emitBinaryEvent) { } @Override - public EventSchema getSchema(InputStream data) { + public EventSchema getEventSchema(List oneEvent) { EventSchema resultSchema = new EventSchema(); - List nEvents = parseNEvents(data, 1); - JsonDataFormatDefinition jsonDefinition = new JsonDataFormatDefinition(); Map exampleEvent = null; try { - exampleEvent = jsonDefinition.toMap(nEvents.get(0)); + exampleEvent = jsonDefinition.toMap(oneEvent.get(0)); } catch (SpRuntimeException e) { e.printStackTrace(); } @@ -116,7 +105,7 @@ public EventSchema getSchema(InputStream data) { for (Map.Entry entry : exampleEvent.entrySet()) { // System.out.println(entry.getKey() + "/" + entry.getValue()); - EventProperty p = getEventProperty(entry.getKey(), entry.getValue()); + EventProperty p = JsonEventProperty.getEventProperty(entry.getKey(), entry.getValue()); resultSchema.addEventProperty(p); @@ -125,54 +114,6 @@ public EventSchema getSchema(InputStream data) { return resultSchema; } - private EventProperty getEventProperty(String key, Object o) { - EventProperty resultProperty = null; - - System.out.println("Key: " + key); - System.out.println("Class: " + o.getClass()); - System.out.println("Primitive: " + o.getClass().isPrimitive()); - System.out.println("Array: " + o.getClass().isArray()); - System.out.println("TypeName: " + o.getClass().getTypeName()); - - - System.out.println("======================="); - - if (o.getClass().equals(Boolean.class)) { - resultProperty = new EventPropertyPrimitive(); - resultProperty.setRuntimeName(key); - ((EventPropertyPrimitive) resultProperty).setRuntimeType(XSD._boolean.toString()); - } - else if (o.getClass().equals(String.class)) { - resultProperty = new EventPropertyPrimitive(); - resultProperty.setRuntimeName(key); - ((EventPropertyPrimitive) resultProperty).setRuntimeType(XSD._string.toString()); - } - else if (o.getClass().equals(Integer.class) || o.getClass().equals(Double.class)) { - resultProperty = new EventPropertyPrimitive(); - resultProperty.setRuntimeName(key); - ((EventPropertyPrimitive) resultProperty).setRuntimeType(SO.Number.toString()); - } - else if (o.getClass().equals(LinkedHashMap.class)) { - resultProperty = new EventPropertyNested(); - resultProperty.setRuntimeName(key); - List all = new ArrayList<>(); - for (Map.Entry entry : ((Map) o).entrySet()) { - all.add(getEventProperty(entry.getKey(), entry.getValue())); - } - - ((EventPropertyNested) resultProperty).setEventProperties(all); - - } else if (o.getClass().equals(ArrayList.class)) { - resultProperty = new EventPropertyList(); - resultProperty.setRuntimeName(key); - } - - if (resultProperty == null) { - logger.error("Property Type was not detected in JsonParser for the schema detection. This should never happen!"); - } - - return resultProperty; - } public Map parseObject(javax.json.stream.JsonParser jsonParser, boolean root, int start) { // this variable is needed to skip the first object start diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/json/object/JsonObjectFormat.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/json/object/JsonObjectFormat.java new file mode 100644 index 0000000000..50d2e88afb --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/json/object/JsonObjectFormat.java @@ -0,0 +1,47 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.format.json.object; + +import org.streampipes.connect.adapter.generic.format.Format; +import org.streampipes.connect.adapter.generic.format.json.AbstractJsonFormat; +import org.streampipes.model.connect.grounding.FormatDescription; + +public class JsonObjectFormat extends AbstractJsonFormat { + + public static final String ID = "https://streampipes.org/vocabulary/v1/format/json/object"; + + @Override + public Format getInstance(FormatDescription formatDescription) { + return new JsonObjectFormat(); + } + + @Override + public FormatDescription declareModel() { + FormatDescription fd = new FormatDescription(ID, "Json Object", "This is the description" + + "for json format"); + + return fd; + } + + @Override + public String getId() { + return ID; + } + + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/json/object/JsonObjectParser.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/json/object/JsonObjectParser.java new file mode 100644 index 0000000000..4b6a67a9d7 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/json/object/JsonObjectParser.java @@ -0,0 +1,209 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.format.json.object; + + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.commons.exceptions.SpRuntimeException; +import org.streampipes.connect.EmitBinaryEvent; +import org.streampipes.connect.adapter.generic.format.Parser; +import org.streampipes.connect.adapter.generic.format.util.JsonEventProperty; +import org.streampipes.dataformat.json.JsonDataFormatDefinition; +import org.streampipes.model.connect.grounding.FormatDescription; +import org.streampipes.model.schema.EventProperty; +import org.streampipes.model.schema.EventSchema; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.json.Json; +import javax.json.stream.JsonParserFactory; +import javax.json.stream.JsonParsingException; + +public class JsonObjectParser extends Parser { + + private static final Logger logger = LoggerFactory.getLogger(JsonObjectParser.class); + + @Override + public Parser getInstance(FormatDescription formatDescription) { + return new JsonObjectParser(); + } + + /** + * Use this constructor when just a specific key of the object should be parsed + */ + public JsonObjectParser() { + } + + @Override + public void parse(InputStream data, EmitBinaryEvent emitBinaryEvent) { + JsonParserFactory factory = Json.createParserFactory(null); + javax.json.stream.JsonParser jsonParser = factory.createParser(data); + + // Parse all events + JsonDataFormatDefinition jsonDefinition = new JsonDataFormatDefinition(); + boolean isEvent = true; + boolean result = true; + int objectCount = 0; + + try { + while (jsonParser.hasNext() && isEvent && result) { + Map objectMap = parseObject(jsonParser, true, 1); + if (objectMap != null) { + byte[] tmp = new byte[0]; + try { + tmp = jsonDefinition.fromMap(objectMap); + } catch (SpRuntimeException e) { + e.printStackTrace(); + } +// handleEvent(new EventObjectEndEvent(parseObject(tmp))); + // TODO decide what happens id emit returns false + result = emitBinaryEvent.emit(tmp); + } else { + isEvent = false; + } + } + + } catch(JsonParsingException e) { + logger.error("Error. Currently just one Object is supported in JSONObjectParser"); + + } + + + } + + @Override + public EventSchema getEventSchema(List oneEvent) { + EventSchema resultSchema = new EventSchema(); + +// resultSchema.setEventProperties(Arrays.asList(EpProperties.timestampProperty("timestamp"))); + + JsonDataFormatDefinition jsonDefinition = new JsonDataFormatDefinition(); + + + Map exampleEvent = null; + + try { + exampleEvent = jsonDefinition.toMap(oneEvent.get(0)); + } catch (SpRuntimeException e) { + e.printStackTrace(); + } + + for (Map.Entry entry : exampleEvent.entrySet()) + { +// System.out.println(entry.getKey() + "/" + entry.getValue()); + EventProperty p = JsonEventProperty.getEventProperty(entry.getKey(), entry.getValue()); + + resultSchema.addEventProperty(p); + + } + + return resultSchema; + } + + public Map parseObject(javax.json.stream.JsonParser jsonParser, boolean root, int start) { + // this variable is needed to skip the first object start + String mapKey = ""; + Map result = new HashMap<>(); + List arr = null; + + while (jsonParser.hasNext()) { + javax.json.stream.JsonParser.Event event = jsonParser.next(); + switch (event) { + case KEY_NAME: + mapKey = jsonParser.getString(); + logger.debug("key: " + mapKey ); + break; + case START_OBJECT: + if (start == 0) { + Map ob = parseObject(jsonParser, false, 0); + if (arr == null) { + result.put(mapKey, ob); + } else { + arr.add(ob); + } + } else { + start--; + } + logger.debug("start object"); + break; + case END_OBJECT: + + logger.debug("end object"); + return result; + case START_ARRAY: + arr = new ArrayList<>(); + logger.debug("start array"); + break; + case END_ARRAY: + // Check if just the end of array is entered + if (result.keySet().size() == 0 && mapKey.equals("")) { + return null; + } + result.put(mapKey, arr); + arr = null; + logger.debug("end array"); + break; + case VALUE_TRUE: + if (arr == null) { + result.put(mapKey, true); + } else { + arr.add(true); + } + logger.debug("value: true"); + break; + case VALUE_FALSE: + if (arr == null) { + result.put(mapKey, false); + } else { + arr.add(false); + } + logger.debug("value: false"); + break; + case VALUE_STRING: + if (arr == null) { + result.put(mapKey, jsonParser.getString()); + } else { + arr.add(jsonParser.getString()); + } + logger.debug("value string: " + jsonParser.getString()); + break; + case VALUE_NUMBER: + if (arr == null) { + result.put(mapKey, jsonParser.getBigDecimal()); + } else { + arr.add(jsonParser.getBigDecimal()); + } + logger.debug("value number: " + jsonParser.getBigDecimal()); + break; + case VALUE_NULL: + logger.debug("value null"); + break; + default: + logger.error("Error: " + event + " event is not handled in the JSON parser"); + break; + } + } + + return result; + } +} \ No newline at end of file diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/util/JsonEventProperty.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/util/JsonEventProperty.java new file mode 100644 index 0000000000..e98598675d --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/util/JsonEventProperty.java @@ -0,0 +1,86 @@ +/* +Copyright 2018 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package org.streampipes.connect.adapter.generic.format.util; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.model.schema.EventProperty; +import org.streampipes.model.schema.EventPropertyList; +import org.streampipes.model.schema.EventPropertyNested; +import org.streampipes.model.schema.EventPropertyPrimitive; +import org.streampipes.vocabulary.SO; +import org.streampipes.vocabulary.XSD; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class JsonEventProperty { + + static Logger logger = LoggerFactory.getLogger(JsonEventProperty.class); + + + public static EventProperty getEventProperty(String key, Object o) { + EventProperty resultProperty = null; + + System.out.println("Key: " + key); + System.out.println("Class: " + o.getClass()); + System.out.println("Primitive: " + o.getClass().isPrimitive()); + System.out.println("Array: " + o.getClass().isArray()); + System.out.println("TypeName: " + o.getClass().getTypeName()); + + + System.out.println("======================="); + + if (o.getClass().equals(Boolean.class)) { + resultProperty = new EventPropertyPrimitive(); + resultProperty.setRuntimeName(key); + ((EventPropertyPrimitive) resultProperty).setRuntimeType(XSD._boolean.toString()); + } + else if (o.getClass().equals(String.class)) { + resultProperty = new EventPropertyPrimitive(); + resultProperty.setRuntimeName(key); + ((EventPropertyPrimitive) resultProperty).setRuntimeType(XSD._string.toString()); + } + else if (o.getClass().equals(Integer.class) || o.getClass().equals(Double.class)|| o.getClass().equals(Long.class)) { + resultProperty = new EventPropertyPrimitive(); + resultProperty.setRuntimeName(key); + ((EventPropertyPrimitive) resultProperty).setRuntimeType(XSD._float.toString()); + } + else if (o.getClass().equals(LinkedHashMap.class)) { + resultProperty = new EventPropertyNested(); + resultProperty.setRuntimeName(key); + List all = new ArrayList<>(); + for (Map.Entry entry : ((Map) o).entrySet()) { + all.add(getEventProperty(entry.getKey(), entry.getValue())); + } + + ((EventPropertyNested) resultProperty).setEventProperties(all); + + } else if (o.getClass().equals(ArrayList.class)) { + resultProperty = new EventPropertyList(); + resultProperty.setRuntimeName(key); + } + + if (resultProperty == null) { + logger.error("Property Type was not detected in JsonParser for the schema detection. This should never happen!"); + } + + return resultProperty; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/xml/XmlFormat.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/xml/XmlFormat.java new file mode 100644 index 0000000000..147f48a638 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/xml/XmlFormat.java @@ -0,0 +1,86 @@ +/* +Copyright 2018 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package org.streampipes.connect.adapter.generic.format.xml; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.commons.exceptions.SpRuntimeException; +import org.streampipes.connect.adapter.generic.format.Format; +import org.streampipes.connect.adapter.generic.sdk.ParameterExtractor; +import org.streampipes.dataformat.json.JsonDataFormatDefinition; +import org.streampipes.model.connect.grounding.FormatDescription; +import org.streampipes.model.schema.EventSchema; +import org.streampipes.model.staticproperty.FreeTextStaticProperty; + +import java.util.Map; + +public class XmlFormat extends Format { + + public static final String ID = "https://streampipes.org/vocabulary/v1/format/xml"; + private String tag; + + Logger logger = LoggerFactory.getLogger(XmlFormat.class); + + public XmlFormat() { + } + + public XmlFormat(String tag) { + this.tag = tag; + } + + @Override + public Format getInstance(FormatDescription formatDescription) { + ParameterExtractor extractor = new ParameterExtractor(formatDescription.getConfig()); + String tag = extractor.singleValue("tag"); + + return new XmlFormat(tag); + } + + @Override + public FormatDescription declareModel() { + FormatDescription description = new FormatDescription(ID, "XML", "This is the description " + + "for the XML format"); + FreeTextStaticProperty tagProperty = new FreeTextStaticProperty("tag" , + "Tag", "The Tag name of the events"); + + description.addConfig(tagProperty); + + return description; + } + + @Override + public String getId() { + return ID; + } + + @Override + public Map parse(byte[] object) { + EventSchema resultSchema = new EventSchema(); + + JsonDataFormatDefinition jsonDefinition = new JsonDataFormatDefinition(); + + Map result = null; + + try { + result = jsonDefinition.toMap(object); + } catch (SpRuntimeException e) { + e.printStackTrace(); + } + + return result; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/xml/XmlParser.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/xml/XmlParser.java new file mode 100644 index 0000000000..b1fefbc46c --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/format/xml/XmlParser.java @@ -0,0 +1,131 @@ +/* +Copyright 2018 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package org.streampipes.connect.adapter.generic.format.xml; + +import com.google.common.base.Charsets; +import com.google.common.io.CharStreams; +import com.google.gson.Gson; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.XML; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.commons.exceptions.SpRuntimeException; +import org.streampipes.connect.EmitBinaryEvent; +import org.streampipes.connect.adapter.generic.format.Parser; +import org.streampipes.connect.adapter.generic.format.util.JsonEventProperty; +import org.streampipes.connect.adapter.generic.sdk.ParameterExtractor; +import org.streampipes.dataformat.json.JsonDataFormatDefinition; +import org.streampipes.model.connect.grounding.FormatDescription; +import org.streampipes.model.schema.EventProperty; +import org.streampipes.model.schema.EventSchema; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.*; + +public class XmlParser extends Parser { + + Logger logger = LoggerFactory.getLogger(XmlParser.class); + + private String tag; + + public XmlParser() { + + } + + public XmlParser(String tag) { + this.tag = tag; + } + + @Override + public Parser getInstance(FormatDescription formatDescription) { + ParameterExtractor extractor = new ParameterExtractor(formatDescription.getConfig()); + String tag = extractor.singleValue("tag"); + + return new XmlParser(tag); + } + + @Override + public void parse(InputStream data, EmitBinaryEvent emitBinaryEvent) { + + try { + String dataString = CharStreams.toString(new InputStreamReader(data, Charsets.UTF_8)); + + JSONObject xmlJSONObj = XML.toJSONObject(dataString); + searchAndEmitEvents(xmlJSONObj.toMap(), tag, emitBinaryEvent); + + } catch (JSONException e) { + logger.error(e.toString()); + } catch (IOException e) { + logger.error(e.toString()); + } + + } + + @Override + public EventSchema getEventSchema(List oneEvent) { + EventSchema resultSchema = new EventSchema(); + + JsonDataFormatDefinition jsonDefinition = new JsonDataFormatDefinition(); + + Map exampleEvent = null; + + try { + exampleEvent = jsonDefinition.toMap(oneEvent.get(0)); + } catch (SpRuntimeException e) { + logger.error(e.toString()); + } + + for (Map.Entry entry : exampleEvent.entrySet()) + { EventProperty p = JsonEventProperty.getEventProperty(entry.getKey(), entry.getValue()); + resultSchema.addEventProperty(p); + } + + return resultSchema; + } + + private void searchAndEmitEvents(Map map, String key, EmitBinaryEvent emitBinaryEvent) { + Gson gson = new Gson(); + + for (Map.Entry entry : map.entrySet()) { + if(entry.getKey().equals(key)) { + + if(entry.getValue() instanceof List) { + List list = (List) entry.getValue(); + + list.forEach(listEntry -> emitBinaryEvent.emit(gson.toJson(listEntry).getBytes())); + + } else if(entry.getValue() instanceof Map) { + byte[] bytes = gson.toJson(entry.getValue()).getBytes(); + emitBinaryEvent.emit(bytes); + + } else { + logger.error("Events are found, but could not disjunct: " + entry.toString()); + } + + } else if (entry.getValue() instanceof Map) { + searchAndEmitEvents((Map)entry.getValue(), key, emitBinaryEvent); + } else if (entry.getValue() instanceof List) { + List list = (List)entry.getValue(); + list.forEach(listEntry -> searchAndEmitEvents((Map)listEntry, key, emitBinaryEvent)); + } + } + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/guess/PropertyGuessResults.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/guess/PropertyGuessResults.java new file mode 100644 index 0000000000..f286c37d10 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/guess/PropertyGuessResults.java @@ -0,0 +1,39 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.guess; + +//TODO Replace this class with DomainPropertyProbabilityList +public class PropertyGuessResults { + private PropertyGuesses[] result; + + public PropertyGuessResults() { + } + + + public PropertyGuessResults(PropertyGuesses[] result) { + this.result = result; + } + + public PropertyGuesses[] getResult() { + return result; + } + + public void setResult(PropertyGuesses[] result) { + this.result = result; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/guess/PropertyGuesses.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/guess/PropertyGuesses.java new file mode 100644 index 0000000000..9ee83d8dec --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/guess/PropertyGuesses.java @@ -0,0 +1,53 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.guess; + +import com.google.gson.annotations.SerializedName; + +public class PropertyGuesses { + + @SerializedName("class") + private String clazz; + + private double probability; + + + public PropertyGuesses() { + } + + public PropertyGuesses(String clazz, double probability) { + this.clazz = clazz; + this.probability = probability; + } + + public String getClazz() { + return clazz; + } + + public void setClazz(String clazz) { + this.clazz = clazz; + } + + public double getProbability() { + return probability; + } + + public void setProbability(double probability) { + this.probability = probability; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/guess/SchemaGuesser.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/guess/SchemaGuesser.java new file mode 100644 index 0000000000..6aa15433df --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/guess/SchemaGuesser.java @@ -0,0 +1,158 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.guess; + +import org.apache.http.client.fluent.Form; +import org.apache.http.client.fluent.Request; +import org.streampipes.model.connect.guess.DomainPropertyProbability; +import org.streampipes.model.connect.guess.DomainPropertyProbabilityList; +import org.streampipes.model.connect.guess.GuessSchema; +import org.streampipes.model.schema.EventProperty; +import org.streampipes.model.schema.EventPropertyNested; +import org.streampipes.model.schema.EventSchema; +import org.streampipes.serializers.json.GsonSerializer; + + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class SchemaGuesser { + + public static int port = 80; + + public static GuessSchema guessSchma(EventSchema eventSchema, List> nElements) { + GuessSchema result = new GuessSchema(); + + List allDomainPropertyProbabilities = getDomainPropertyProbabitlyList(eventSchema.getEventProperties(), nElements, new ArrayList<>()); + + result.setEventSchema(eventSchema); + result.setPropertyProbabilityList(allDomainPropertyProbabilities); + + + return result; + } + + private static List getDomainPropertyProbabitlyList(List eventProperties, + List> nEventsParsed, + List keys) { + + List result = new ArrayList<>(); + for (EventProperty ep : eventProperties) { + if (ep instanceof EventPropertyNested) { + List li = ((EventPropertyNested) ep).getEventProperties(); + keys.add(ep.getRuntimeName()); + result.addAll(getDomainPropertyProbabitlyList(li, nEventsParsed, keys)); + } else { + List tmp = new ArrayList<>(); + for (Map event : nEventsParsed) { + Map subEvent = event; + for (String k : keys) { + subEvent = (Map) subEvent.get(k); + } + + if (subEvent != null) { + tmp.add(subEvent.get(ep.getRuntimeName())); + } + } + + DomainPropertyProbabilityList resultList = getDomainPropertyProbability(tmp.toArray()); + resultList.setRuntimeName(ep.getRuntimeName()); + result.add(resultList); + } + + } + + return result; + } + + /** + * TODO replace this method, change python API to variables of DomainPropertyProbabilityList + * @param objects + * @return + */ + public static PropertyGuessResults requestProbabilitiesObject(Object[] objects) { + + String probabilitiesJsonString = requestProbabilitiesString(objects); + PropertyGuessResults res = GsonSerializer.getGsonWithIds().fromJson(probabilitiesJsonString, + PropertyGuessResults.class); + return res; + } + + public static String requestProbabilitiesString(Object[] objects) { + String httpRequestBody = GsonSerializer.getGsonWithIds() + .toJson(objects); + + String httpResp = "{\"result\": []}"; + + try { + httpResp = Request.Post("http://localhost:" + port +"/predict") + .addHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8") + .bodyForm(Form.form().add("X", httpRequestBody).build()).execute().returnContent().asString(); + } catch (IOException e) { + e.printStackTrace(); + } + + + return httpResp; + } + + public static void main(String[] args) { + + int[] n = {1, 2}; + String[] st = {"f", "dd"}; + String s = "[34292.0, 34292.0, 34292.0, 84155.0, 34466.0, 83352.0, 84503.0, 63916.0, 9456.0, 9456.0, 9456.0, 8359.0, 84371.0, 63743.0, 8280.0, 8280.0, 34454.0, 84364.0, 94081.0, 57334.0]"; + + SchemaGuesser schemaGuesser = new SchemaGuesser(); + PropertyGuessResults prg = schemaGuesser.requestProbabilitiesObject(st); + + for (PropertyGuesses pg : prg.getResult()) { + System.out.println("Class " + pg.getClazz()); + System.out.println("Property " + pg.getProbability()); + System.out.println("========="); + + } + + DomainPropertyProbabilityList dppl = SchemaGuesser.getDomainPropertyProbability(st); + + for (DomainPropertyProbability pg: dppl.getList()) { + System.out.println("Class " + pg.getDomainProperty()); + System.out.println("Property " + pg.getProbability()); + System.out.println("========="); + + } + } + + + public static DomainPropertyProbabilityList getDomainPropertyProbability(Object[] sampleData) { + PropertyGuessResults pgr = requestProbabilitiesObject(sampleData); + + DomainPropertyProbabilityList result = new DomainPropertyProbabilityList(); + + + for (PropertyGuesses pg : pgr.getResult()) { + Double d = pg.getProbability(); + result.addDomainPropertyProbability(new DomainPropertyProbability(pg.getClazz(), d.toString())); + } + + + return result; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/pipeline/AdapterPipeline.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/pipeline/AdapterPipeline.java new file mode 100644 index 0000000000..1d1c7df31b --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/pipeline/AdapterPipeline.java @@ -0,0 +1,47 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.pipeline; + +import java.util.List; +import java.util.Map; + +public class AdapterPipeline { + + private List pipelineElements; + + + public AdapterPipeline(List pipelineElements) { + this.pipelineElements = pipelineElements; + } + + public void process(Map event) { + + for (AdapterPipelineElement pipelineElement : pipelineElements) { + event = pipelineElement.process(event); + } + + } + + public List getPipelineElements() { + return pipelineElements; + } + + public void setPipelineElements(List pipelineElements) { + this.pipelineElements = pipelineElements; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/pipeline/AdapterPipelineElement.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/pipeline/AdapterPipelineElement.java new file mode 100644 index 0000000000..cb9e753707 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/pipeline/AdapterPipelineElement.java @@ -0,0 +1,26 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.pipeline; + +import java.util.Map; + +public interface AdapterPipelineElement { + + public Map process(Map event); + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/pipeline/elements/AddTimestamp.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/pipeline/elements/AddTimestamp.java new file mode 100644 index 0000000000..3f54b8f08d --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/pipeline/elements/AddTimestamp.java @@ -0,0 +1,33 @@ +/* +Copyright 2018 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package org.streampipes.connect.adapter.generic.pipeline.elements; + +import org.streampipes.connect.adapter.generic.pipeline.AdapterPipelineElement; + +import java.util.Map; + +public class AddTimestamp implements AdapterPipelineElement { + + private static final String TIMESTAMP_PROPERTY_NAME = "timestamp"; + + @Override + public Map process(Map event) { + event.put(TIMESTAMP_PROPERTY_NAME, System.currentTimeMillis()); + return event; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/pipeline/elements/DuplicateFilter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/pipeline/elements/DuplicateFilter.java new file mode 100644 index 0000000000..26021f7fb3 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/pipeline/elements/DuplicateFilter.java @@ -0,0 +1,61 @@ +/* +Copyright 2018 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package org.streampipes.connect.adapter.generic.pipeline.elements; + +import org.streampipes.connect.adapter.generic.pipeline.AdapterPipelineElement; + +import java.util.HashMap; +import java.util.Map; + +public class DuplicateFilter implements AdapterPipelineElement { + + private static final long CLEAN_UP_INTERVAL_MILLI_SEC = 1000 * 60 * 60; //1 Hour + private static final long LIFE_TIME_EVENT_DUPLICATE = 1000 * 60 * 60 * 24; //24 Hour + + private Map eventState = new HashMap<>(); + private long lastCleanUpTimestamp = System.currentTimeMillis(); + + @Override + public Map process(Map event) { + cleanUpEvenState(); + + int hash = event.hashCode(); + + if (isDuplicatedEvent(hash)) { + saveEvent(hash); + return null; + } + saveEvent(hash); + return event; + } + + private boolean isDuplicatedEvent(int key) { + return eventState.containsKey(key); + } + + private void saveEvent(int key) { + eventState.put(key, System.currentTimeMillis()); + } + + private void cleanUpEvenState() { + if (System.currentTimeMillis() > lastCleanUpTimestamp + CLEAN_UP_INTERVAL_MILLI_SEC) { + + eventState.entrySet().removeIf(entry -> entry.getValue() + LIFE_TIME_EVENT_DUPLICATE < System.currentTimeMillis()); + + } + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/pipeline/elements/SendToKafkaAdapterSink.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/pipeline/elements/SendToKafkaAdapterSink.java new file mode 100644 index 0000000000..9863e10b8f --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/pipeline/elements/SendToKafkaAdapterSink.java @@ -0,0 +1,55 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.pipeline.elements; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.eclipse.rdf4j.query.algebra.Str; +import org.streampipes.connect.adapter.GroundingService; +import org.streampipes.connect.adapter.generic.pipeline.AdapterPipelineElement; +import org.streampipes.messaging.kafka.SpKafkaProducer; +import org.streampipes.model.connect.adapter.AdapterDescription; + +import java.util.Map; + +public class SendToKafkaAdapterSink implements AdapterPipelineElement { + private SpKafkaProducer producer; + private ObjectMapper objectMapper; + + public SendToKafkaAdapterSink(AdapterDescription adapterDescription) { + String brokerUrl = GroundingService.extractBroker(adapterDescription); + String topic = GroundingService.extractTopic(adapterDescription); + + producer = new SpKafkaProducer(brokerUrl, topic); + objectMapper = new ObjectMapper(); + } + + @Override + public Map process(Map event) { + try { + if (event != null) { + producer.publish(objectMapper.writeValueAsBytes(event)); + System.out.println("send to kafka: " + event); + } + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + + return null; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/pipeline/elements/TransformSchemaAdapterPipelineElement.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/pipeline/elements/TransformSchemaAdapterPipelineElement.java new file mode 100644 index 0000000000..c0b89e3fb5 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/pipeline/elements/TransformSchemaAdapterPipelineElement.java @@ -0,0 +1,85 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.pipeline.elements; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.connect.adapter.generic.pipeline.AdapterPipelineElement; +import org.streampipes.connect.adapter.generic.transform.*; +import org.streampipes.model.connect.rules.*; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +public class TransformSchemaAdapterPipelineElement implements AdapterPipelineElement { + + private EventTransformer eventTransformer; + Logger logger = LoggerFactory.getLogger(TransformSchemaAdapterPipelineElement.class); + + public TransformSchemaAdapterPipelineElement(List transformationRuleDescriptions) { + List rules = new ArrayList<>(); + + // transforms description to actual rules + for (TransformationRuleDescription ruleDescription : transformationRuleDescriptions) { + if (ruleDescription instanceof RenameRuleDescription) { + RenameRuleDescription tmp = (RenameRuleDescription) ruleDescription; + rules.add(new RenameTransformationRule(toKeyArray(tmp.getOldRuntimeKey()), getLastKey(tmp.getNewRuntimeKey()))); + } else if (ruleDescription instanceof MoveRuleDescription) { + MoveRuleDescription tmp = (MoveRuleDescription) ruleDescription; + rules.add(new MoveTransformationRule(toKeyArray(tmp.getOldRuntimeKey()), toKeyArray(tmp.getNewRuntimeKey()))); + } else if (ruleDescription instanceof CreateNestedRuleDescription) { + CreateNestedRuleDescription tmp = (CreateNestedRuleDescription) ruleDescription; + rules.add(new CreateNestedTransformationRule(toKeyArray(tmp.getRuntimeKey()))); + } else if (ruleDescription instanceof DeleteRuleDescription) { + DeleteRuleDescription tmp = (DeleteRuleDescription) ruleDescription; + rules.add(new DeleteTransformationRule(toKeyArray(tmp.getRuntimeKey()))); + } else { + logger.error("Could not find the class for the rule description. This should never happen. Talk to admins to extend the rule implementations to get rid of this error!"); + } + } + + eventTransformer = new EventTransformer(rules); + } + + private String getLastKey(String s) { + String[] list = s.split("\\."); + if (list.length == 0) { + return s; + } else { + return list[list.length - 1]; + } + } + + + + private List toKeyArray(String s) { + String[] split = s.split("\\."); + if (split.length == 0) { + return Arrays.asList(s); + } else { + return Arrays.asList(split); + } + } + + @Override + public Map process(Map event) { + return eventTransformer.transform(event); + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/Protocol.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/Protocol.java new file mode 100644 index 0000000000..edc306f3d7 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/Protocol.java @@ -0,0 +1,60 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.protocol; + +import org.streampipes.connect.adapter.generic.format.Format; +import org.streampipes.connect.adapter.generic.format.Parser; +import org.streampipes.connect.adapter.generic.pipeline.AdapterPipeline; +import org.streampipes.model.connect.guess.GuessSchema; +import org.streampipes.model.connect.grounding.ProtocolDescription; + +import java.util.List; +import java.util.Map; + + +public abstract class Protocol { + + protected Parser parser; + protected Format format; + + public Protocol() { + + } + + public Protocol(Parser parser, Format format) { + this.parser = parser; + this.format = format; + } + + public abstract Protocol getInstance(ProtocolDescription protocolDescription, Parser parser, Format format); + + public abstract ProtocolDescription declareModel(); + + public abstract GuessSchema getGuessSchema(); + + public abstract List> getNElements(int n); + + public abstract void run(AdapterPipeline adapterPipeline); + + /* + Stops the running protocol. Mainly relevant for streaming protocols + */ + public abstract void stop(); + + public abstract String getId(); +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/set/FileProtocol.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/set/FileProtocol.java new file mode 100644 index 0000000000..56bf836423 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/set/FileProtocol.java @@ -0,0 +1,192 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.protocol.set; + + + +import org.slf4j.LoggerFactory; +import org.slf4j.Logger; +import org.streampipes.connect.SendToPipeline; +import org.streampipes.connect.adapter.generic.format.Format; +import org.streampipes.connect.adapter.generic.guess.SchemaGuesser; +import org.streampipes.connect.adapter.generic.pipeline.AdapterPipeline; +import org.streampipes.connect.adapter.generic.protocol.Protocol; +import org.streampipes.connect.adapter.generic.sdk.ParameterExtractor; +import org.streampipes.model.connect.guess.GuessSchema; +import org.streampipes.model.connect.grounding.ProtocolDescription; +import org.streampipes.model.schema.EventSchema; +import org.streampipes.model.staticproperty.FreeTextStaticProperty; +import org.streampipes.connect.adapter.generic.format.Parser; + +import java.io.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class FileProtocol extends Protocol { + + private static Logger logger = LoggerFactory.getLogger(FileProtocol.class); + + public static final String ID = "https://streampipes.org/vocabulary/v1/protocol/set/file"; + + private String fileUri; + + public FileProtocol() { + } + + public FileProtocol(Parser parser, Format format, String fileUri) { + super(parser, format); + this.fileUri = fileUri; + } + + @Override + public ProtocolDescription declareModel() { + ProtocolDescription pd = new ProtocolDescription(ID,"File","This is the " + + "description for the File protocol"); + FreeTextStaticProperty urlProperty = new FreeTextStaticProperty("fileUri", "fileUri", + "This property defines the URL for the http request."); + pd.setSourceType("SET"); + pd.setIconUrl("file.png"); + pd.addConfig(urlProperty); + return pd; + } + + @Override + public Protocol getInstance(ProtocolDescription protocolDescription, Parser parser, Format format) { + ParameterExtractor extractor = new ParameterExtractor(protocolDescription.getConfig()); + + String fileUri = extractor.singleValue("fileUri"); + + return new FileProtocol(parser, format, fileUri); + } + + @Override + public void run(AdapterPipeline adapterPipeline) { + FileReader fr = null; + + // TODO fix this. Currently needed because it must be wait till the whole pipeline is up and running + try { + Thread.sleep(7000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + SendToPipeline stk = new SendToPipeline(format, adapterPipeline); + try { + fr = new FileReader(fileUri); + BufferedReader br = new BufferedReader(fr); + + InputStream inn = new FileInputStream(fileUri); + parser.parse(inn, stk); + + fr.close(); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Override + public void stop() { + + } + + + @Override + public GuessSchema getGuessSchema() { + + + InputStream dataInputStream = getDataFromEndpoint(); + + List dataByte = parser.parseNEvents(dataInputStream, 20); + + EventSchema eventSchema= parser.getEventSchema(dataByte); + + GuessSchema result = SchemaGuesser.guessSchma(eventSchema, getNElements(20)); + + return result; + +// EventSchema result = null; +// +// FileReader fr = null; +// +// try { +// fr = new FileReader(fileUri); +// BufferedReader br = new BufferedReader(fr); +// +// InputStream inn = new FileInputStream(fileUri); +// result = parser.getEventSchema(parser.parseNEvents(inn, 1).get(0)); +// +// fr.close(); +// } catch (FileNotFoundException e) { +// e.printStackTrace(); +// } catch (IOException e) { +// e.printStackTrace(); +// } + +// return result; + } + + + public InputStream getDataFromEndpoint() { + FileReader fr = null; + InputStream inn = null; + + try { + fr = new FileReader(fileUri); + BufferedReader br = new BufferedReader(fr); + + inn = new FileInputStream(fileUri); + + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + + return inn; + } + @Override + public List> getNElements(int n) { + List> result = new ArrayList<>(); + + InputStream dataInputStream = getDataFromEndpoint(); + + List dataByteArray = parser.parseNEvents(dataInputStream, n); + + // Check that result size is n. Currently just an error is logged. Maybe change to an exception + if (dataByteArray.size() < n) { + logger.error("Error in File Protocol! User required: " + n + " elements but the resource just had: " + + dataByteArray.size()); + } + + for (byte[] b : dataByteArray) { + result.add(format.parse(b)); + } + + return result; + } + + + @Override + public String getId() { + return ID; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/set/HttpProtocol.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/set/HttpProtocol.java new file mode 100644 index 0000000000..4c586c8e86 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/set/HttpProtocol.java @@ -0,0 +1,168 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.protocol.set; + +import org.apache.commons.io.IOUtils; +import org.apache.http.client.fluent.Request; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.connect.SendToPipeline; +import org.streampipes.connect.adapter.generic.format.Format; +import org.streampipes.connect.adapter.generic.format.Parser; +import org.streampipes.connect.adapter.generic.guess.SchemaGuesser; +import org.streampipes.connect.adapter.generic.pipeline.AdapterPipeline; +import org.streampipes.connect.adapter.generic.protocol.Protocol; +import org.streampipes.connect.adapter.generic.sdk.ParameterExtractor; +import org.streampipes.model.connect.grounding.ProtocolDescription; +import org.streampipes.model.connect.guess.GuessSchema; +import org.streampipes.model.schema.EventSchema; +import org.streampipes.model.staticproperty.FreeTextStaticProperty; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class HttpProtocol extends Protocol { + + Logger logger = LoggerFactory.getLogger(Protocol.class); + + public static final String ID = "https://streampipes.org/vocabulary/v1/protocol/set/http"; + + private String url; + + public HttpProtocol() { + } + + public HttpProtocol(Parser parser, Format format, String url) { + super(parser, format); + this.url = url; + } + + @Override + public ProtocolDescription declareModel() { + ProtocolDescription pd = new ProtocolDescription(ID,"HTTP Set","This is the " + + "description for the http protocol"); + FreeTextStaticProperty urlProperty = new FreeTextStaticProperty("url", "url", + "This property defines the URL for the http request."); + + pd.setSourceType("SET"); + pd.setIconUrl("rest.png"); + //TODO remove just for testing +// urlProperty.setValue("https://opendata.bonn.de/api/action/datastore/search.json?resource_id=0a41c514-f760-4a17-b0a8-e1b755204fee&limit=100"); + + pd.addConfig(urlProperty); + return pd; + } + + @Override + public Protocol getInstance(ProtocolDescription protocolDescription, Parser parser, Format format) { + ParameterExtractor extractor = new ParameterExtractor(protocolDescription.getConfig()); + String url = extractor.singleValue("url"); + + return new HttpProtocol(parser, format, url); + } + + @Override + public void run(AdapterPipeline adapterPipeline) { + + // TODO fix this. Currently needed because it must be wait till the whole pipeline is up and running + try { + Thread.sleep(7000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + SendToPipeline stk = new SendToPipeline(format, adapterPipeline); + + InputStream data = getDataFromEndpoint(); + + parser.parse(data, stk); + } + + @Override + public void stop() { + + } + + + @Override + public GuessSchema getGuessSchema() { + + + InputStream dataInputStream = getDataFromEndpoint(); + + List dataByte = parser.parseNEvents(dataInputStream, 20); + + EventSchema eventSchema= parser.getEventSchema(dataByte); + + GuessSchema result = SchemaGuesser.guessSchma(eventSchema, getNElements(20)); + + return result; + } + + @Override + public List> getNElements(int n) { + + List> result = new ArrayList<>(); + + InputStream dataInputStream = getDataFromEndpoint(); + + List dataByteArray = parser.parseNEvents(dataInputStream, n); + + // Check that result size is n. Currently just an error is logged. Maybe change to an exception + if (dataByteArray.size() < n) { + logger.error("Error in HttpProtocol! User required: " + n + " elements but the resource just had: " + + dataByteArray.size()); + } + + for (byte[] b : dataByteArray) { + result.add(format.parse(b)); + } + + return result; + } + + public InputStream getDataFromEndpoint() { + InputStream result = null; + + try { + String s = Request.Get(url) + .connectTimeout(1000) + .socketTimeout(100000) + .execute().returnContent().asString(); + + if (s.startsWith("ï")) { + s = s.substring(3); + } + + result = IOUtils.toInputStream(s, "UTF-8"); + + } catch (IOException e) { + e.printStackTrace(); + } + + return result; + } + + @Override + public String getId() { + return ID; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/BrokerProtocol.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/BrokerProtocol.java new file mode 100644 index 0000000000..167125d259 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/BrokerProtocol.java @@ -0,0 +1,67 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.streampipes.connect.adapter.generic.protocol.stream; + +import org.streampipes.connect.adapter.generic.format.Format; +import org.streampipes.connect.adapter.generic.format.Parser; +import org.streampipes.connect.adapter.generic.guess.SchemaGuesser; +import org.streampipes.connect.adapter.generic.protocol.Protocol; +import org.streampipes.model.connect.guess.GuessSchema; +import org.streampipes.model.schema.EventSchema; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public abstract class BrokerProtocol extends Protocol { + + protected String brokerUrl; + protected String topic; + + public BrokerProtocol() { + + } + + public BrokerProtocol(Parser parser, Format format, String brokerUrl, String topic) { + super(parser, format); + this.brokerUrl = brokerUrl; + this.topic = topic; + } + + @Override + public GuessSchema getGuessSchema() { + + List eventByte = getNByteElements(1); + EventSchema eventSchema = parser.getEventSchema(eventByte); + + return SchemaGuesser.guessSchma(eventSchema, getNElements(1)); + } + + @Override + public List> getNElements(int n) { + List resultEventsByte = getNByteElements(n); + List> result = new ArrayList<>(); + for (byte[] event : resultEventsByte) { + result.add(format.parse(event)); + } + + return result; + } + + protected abstract List getNByteElements(int n); + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/HttpStreamProtocol.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/HttpStreamProtocol.java new file mode 100644 index 0000000000..b7d88fc0b0 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/HttpStreamProtocol.java @@ -0,0 +1,178 @@ +/* +Copyright 2018 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package org.streampipes.connect.adapter.generic.protocol.stream; + +import org.apache.commons.io.IOUtils; +import org.apache.http.client.fluent.Request; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.connect.adapter.generic.format.Format; +import org.streampipes.connect.adapter.generic.format.Parser; +import org.streampipes.connect.adapter.generic.guess.SchemaGuesser; +import org.streampipes.connect.adapter.generic.protocol.Protocol; +import org.streampipes.connect.adapter.generic.sdk.ParameterExtractor; +import org.streampipes.model.connect.grounding.ProtocolDescription; +import org.streampipes.model.connect.guess.GuessSchema; +import org.streampipes.model.schema.EventSchema; +import org.streampipes.model.staticproperty.FreeTextStaticProperty; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class HttpStreamProtocol extends PullProtocol { + + Logger logger = LoggerFactory.getLogger(HttpStreamProtocol.class); + + public static final String ID = "https://streampipes.org/vocabulary/v1/protocol/stream/http"; + private static String URL_PROPERTY ="url"; + private static String INTERVAL_PROPERTY ="interval"; + private static String ACCESS_TOKEN_PROPERTY ="access_token"; + + private String url; + private String accessToken; + + public HttpStreamProtocol() { + } + + + + public HttpStreamProtocol(Parser parser, Format format, String url, long interval, String accessToken) { + super(parser, format, interval); + this.url = url; + this.accessToken = accessToken; + } + + @Override + public Protocol getInstance(ProtocolDescription protocolDescription, Parser parser, Format format) { + ParameterExtractor extractor = new ParameterExtractor(protocolDescription.getConfig()); + + String urlProperty = extractor.singleValue(URL_PROPERTY); + try { + long intervalProperty = Long.parseLong(extractor.singleValue(INTERVAL_PROPERTY)); + // TODO change access token to an optional parameter +// String accessToken = extractor.singleValue(ACCESS_TOKEN_PROPERTY); + String accessToken = ""; + return new HttpStreamProtocol(parser, format, urlProperty, intervalProperty, accessToken); + } catch (NumberFormatException e) { + logger.error("Could not parse" + extractor.singleValue(INTERVAL_PROPERTY) + "to int"); + return null; + } + + } + + @Override + public ProtocolDescription declareModel() { + ProtocolDescription description = new ProtocolDescription(ID, "HTTP Stream", "This is the " + + "description for the http stream protocol"); + + FreeTextStaticProperty urlProperty = new FreeTextStaticProperty(URL_PROPERTY, "URL", "This property " + + "defines the URL for the http request."); + + + FreeTextStaticProperty intervalProperty = new FreeTextStaticProperty(INTERVAL_PROPERTY, "Interval", "This property " + + "defines the pull interval in seconds."); + +// FreeTextStaticProperty accessToken = new FreeTextStaticProperty(ACCESS_TOKEN_PROPERTY, "Access Token", "Http Access Token"); + + description.setSourceType("STREAM"); + description.addConfig(urlProperty); + description.setIconUrl("rest.png"); + description.addConfig(intervalProperty); +// description.addConfig(accessToken); + + return description; + } + + @Override + public GuessSchema getGuessSchema() { + int n = 1; + + InputStream dataInputStream = getDataFromEndpoint(); + + List dataByte = parser.parseNEvents(dataInputStream, n); + if (dataByte.size() < n) { + logger.error("Error in HttpStreamProtocol! Required: " + n + " elements but the resource just had: " + + dataByte.size()); + + dataByte.addAll(dataByte); + } + EventSchema eventSchema= parser.getEventSchema(dataByte); + GuessSchema result = SchemaGuesser.guessSchma(eventSchema, getNElements(n)); + + return result; + } + + @Override + public List> getNElements(int n) { + List> result = new ArrayList<>(); + + InputStream dataInputStream = getDataFromEndpoint(); + + List dataByte = parser.parseNEvents(dataInputStream, n); + + // Check that result size is n. Currently just an error is logged. Maybe change to an exception + if (dataByte.size() < n) { + logger.error("Error in HttpStreamProtocol! User required: " + n + " elements but the resource just had: " + + dataByte.size()); + } + + for (byte[] b : dataByte) { + result.add(format.parse(b)); + } + + return result; + } + + + @Override + public String getId() { + return ID; + } + + @Override + InputStream getDataFromEndpoint() { + InputStream result = null; + + try { + Request request = Request.Get(url) + .connectTimeout(1000) + .socketTimeout(100000); + + if (this.accessToken != null && !this.accessToken.equals("")) { + request.setHeader("Authorization", "Bearer " + this.accessToken); + } + + String s = request + .execute().returnContent().asString(); + + if (s.startsWith("ï")) { + s = s.substring(3); + } + + result = IOUtils.toInputStream(s, "UTF-8"); + + } catch (Exception e) { + logger.error("Error while fetching data from URL: " + url, e); + +// throw new AdapterException(); + } + + return result; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/KafkaProtocol.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/KafkaProtocol.java new file mode 100644 index 0000000000..2a83a0cdd2 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/KafkaProtocol.java @@ -0,0 +1,227 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.protocol.stream; + +import org.apache.commons.io.IOUtils; +import org.apache.kafka.clients.consumer.*; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.serialization.LongDeserializer; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.commons.exceptions.SpRuntimeException; +import org.streampipes.connect.SendToPipeline; +import org.streampipes.connect.adapter.generic.format.Format; +import org.streampipes.connect.adapter.generic.format.Parser; +import org.streampipes.connect.adapter.generic.format.json.object.JsonObjectFormat; +import org.streampipes.connect.adapter.generic.format.json.object.JsonObjectParser; +import org.streampipes.connect.adapter.generic.pipeline.AdapterPipeline; +import org.streampipes.connect.adapter.generic.protocol.Protocol; +import org.streampipes.connect.adapter.generic.sdk.ParameterExtractor; +import org.streampipes.messaging.InternalEventProcessor; +import org.streampipes.messaging.kafka.SpKafkaConsumer; +import org.streampipes.model.connect.guess.GuessSchema; +import org.streampipes.model.connect.grounding.ProtocolDescription; +import org.streampipes.model.staticproperty.FreeTextStaticProperty; + +import java.io.IOException; +import java.io.InputStream; +import java.util.*; + +public class KafkaProtocol extends BrokerProtocol { + + Logger logger = LoggerFactory.getLogger(KafkaProtocol.class); + + public static final String ID = "https://streampipes.org/vocabulary/v1/protocol/stream/kafka"; + + private Thread thread; + private SpKafkaConsumer kafkaConsumer; + + public KafkaProtocol() { + } + + public KafkaProtocol(Parser parser, Format format, String brokerUrl, String topic) { + super(parser, format, brokerUrl, topic); + } + + @Override + public Protocol getInstance(ProtocolDescription protocolDescription, Parser parser, Format format) { + ParameterExtractor extractor = new ParameterExtractor(protocolDescription.getConfig()); + String brokerUrl = extractor.singleValue("broker_url"); + String topic = extractor.singleValue("topic"); + + return new KafkaProtocol(parser, format, brokerUrl, topic); + } + + @Override + public ProtocolDescription declareModel() { + ProtocolDescription pd = new ProtocolDescription(ID,"Apache Kafka","This is the " + + "description for the Apache Kafka protocol"); + FreeTextStaticProperty broker = new FreeTextStaticProperty("broker_url", "Broker URL", + "This property defines the URL of the Kafka broker."); + + + pd.setSourceType("STREAM"); + pd.setIconUrl("kafka.jpg"); + + FreeTextStaticProperty topic = new FreeTextStaticProperty("topic", "Topic", + "Topic in the broker"); + + //TODO remove, just for debugging +// broker.setValue("141.21.42.75:9092"); +// topic.setValue("SEPA.SEP.Random.Number.Json"); + + pd.addConfig(broker); + pd.addConfig(topic); + return pd; + } + + @Override + protected List getNByteElements(int n) { + final Consumer consumer = createConsumer(this.brokerUrl, this.topic); + + consumer.subscribe(Arrays.asList(this.topic), new ConsumerRebalanceListener() { + @Override + public void onPartitionsRevoked(Collection collection) { + + } + + @Override + public void onPartitionsAssigned(Collection collection) { + consumer.seekToBeginning(collection); + } + }); + + List nEventsByte = new ArrayList<>(); + List resultEventsByte = new ArrayList<>(); + + + while (true) { + final ConsumerRecords consumerRecords = + consumer.poll(1000); + + consumerRecords.forEach(record -> { + try { + InputStream inputStream = IOUtils.toInputStream(record.value(), "UTF-8"); + + nEventsByte.addAll(parser.parseNEvents(inputStream, n)); + } catch (IOException e) { + e.printStackTrace(); + } + }); + + if (nEventsByte.size() > n) { + resultEventsByte = nEventsByte.subList(0, n); + break; + } else if (nEventsByte.size() == n) { + resultEventsByte = nEventsByte; + break; + } + + consumer.commitAsync(); + } + + consumer.close(); + + return resultEventsByte; + } + + public static void main(String... args) { + Parser parser = new JsonObjectParser(); + Format format = new JsonObjectFormat(); + KafkaProtocol kp = new KafkaProtocol(parser, format, "localhost:9092", "org.streampipes.examples.flowrate-1"); + GuessSchema gs = kp.getGuessSchema(); + + System.out.println(gs); + + } + + + private static Consumer createConsumer(String broker, String topic) { + final Properties props = new Properties(); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, + broker); + + props.put(ConsumerConfig.GROUP_ID_CONFIG, + "KafkaExampleConsumer" + System.currentTimeMillis()); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + LongDeserializer.class.getName()); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + StringDeserializer.class.getName()); + + // Create the consumer using props. + final Consumer consumer = + new KafkaConsumer<>(props); + + // Subscribe to the topic. + consumer.subscribe(Collections.singletonList(topic)); + + return consumer; + } + + + @Override + public void run(AdapterPipeline adapterPipeline) { + SendToPipeline stk = new SendToPipeline(format, adapterPipeline); + this.kafkaConsumer = new SpKafkaConsumer(this.brokerUrl, this.topic, new EventProcessor(stk)); + + thread = new Thread(this.kafkaConsumer); + thread.start(); + } + + @Override + public void stop() { + try { + kafkaConsumer.disconnect(); + } catch (SpRuntimeException e) { + e.printStackTrace(); + } + + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + logger.info("Kafka Adapter was sucessfully stopped"); + thread.interrupt(); + } + + + private class EventProcessor implements InternalEventProcessor { + private SendToPipeline stk; + public EventProcessor(SendToPipeline stk) { + this.stk = stk; + } + + @Override + public void onEvent(byte[] payload) { + try { + parser.parse(IOUtils.toInputStream(new String(payload), "UTF-8"), stk); + } catch (IOException e) { + logger.error("Adapter " + ID + " could not read value!",e); + } + } + } + + @Override + public String getId() { + return ID; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/MqttConsumer.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/MqttConsumer.java new file mode 100644 index 0000000000..01bcd0ec6c --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/MqttConsumer.java @@ -0,0 +1,74 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.streampipes.connect.adapter.generic.protocol.stream; + +import org.fusesource.mqtt.client.*; +import org.streampipes.messaging.InternalEventProcessor; + +public class MqttConsumer implements Runnable { + + private String broker; + private String topic; + private InternalEventProcessor consumer; + private boolean running; + private int maxElementsToReceive = -1; + private int messageCount = 0; + + public MqttConsumer(String broker, String topic, InternalEventProcessor consumer) { + this.broker = broker; + this.topic = topic; + this.consumer = consumer; + } + + public MqttConsumer(String broker, String topic, InternalEventProcessor consumer, int maxElementsToReceive) { + this(broker, topic, consumer); + this.maxElementsToReceive = maxElementsToReceive; + } + + + @Override + public void run() { + this.running = true; + MQTT mqtt = new MQTT(); + try { + mqtt.setHost(broker); + BlockingConnection connection = mqtt.blockingConnection(); + connection.connect(); + Topic[] topics = {new Topic(topic, QoS.AT_LEAST_ONCE)}; + byte[] qoses = connection.subscribe(topics); + + while(running && ((maxElementsToReceive == -1) || (this.messageCount <= maxElementsToReceive))) { + Message message = connection.receive(); + byte[] payload = message.getPayload(); + consumer.onEvent(payload); + message.ack(); + this.messageCount++; + } + connection.disconnect(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public void close() { + this.running = false; + } + + public Integer getMessageCount() { + return messageCount; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/MqttProtocol.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/MqttProtocol.java new file mode 100644 index 0000000000..e90185b5b1 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/MqttProtocol.java @@ -0,0 +1,136 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.streampipes.connect.adapter.generic.protocol.stream; + +import org.apache.commons.io.IOUtils; +import org.streampipes.connect.SendToPipeline; +import org.streampipes.connect.adapter.generic.format.Format; +import org.streampipes.connect.adapter.generic.format.Parser; +import org.streampipes.connect.adapter.generic.pipeline.AdapterPipeline; +import org.streampipes.connect.adapter.generic.protocol.Protocol; +import org.streampipes.connect.adapter.generic.sdk.ParameterExtractor; +import org.streampipes.messaging.InternalEventProcessor; +import org.streampipes.model.connect.grounding.ProtocolDescription; +import org.streampipes.model.staticproperty.FreeTextStaticProperty; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class MqttProtocol extends BrokerProtocol { + + public static final String ID = "https://streampipes.org/vocabulary/v1/protocol/stream/mqtt"; + + private Thread thread; + private MqttConsumer mqttConsumer; + + public MqttProtocol() { + } + + public MqttProtocol(Parser parser, Format format, String brokerUrl, String topic) { + super(parser, format, brokerUrl, topic); + } + + @Override + public Protocol getInstance(ProtocolDescription protocolDescription, Parser parser, Format format) { + ParameterExtractor extractor = new ParameterExtractor(protocolDescription.getConfig()); + String brokerUrl = extractor.singleValue("broker_url"); + String topic = extractor.singleValue("topic"); + + return new MqttProtocol(parser, format, brokerUrl, topic); + } + + @Override + public ProtocolDescription declareModel() { + ProtocolDescription pd = new ProtocolDescription(ID, "MQTT", "This is the " + + "description for the MQTT protocol"); + FreeTextStaticProperty broker = new FreeTextStaticProperty("broker_url", "Broker URL", + "This property defines the URL of the MQTT broker."); + + pd.setSourceType("STREAM"); + pd.setIconUrl("mqtt.png"); + + FreeTextStaticProperty topic = new FreeTextStaticProperty("topic", "Topic", + "Topic in the broker"); + + pd.addConfig(broker); + pd.addConfig(topic); + return pd; + } + + + + @Override + protected List getNByteElements(int n) { + List elements = new ArrayList<>(); + int i = 0; + + InternalEventProcessor eventProcessor = elements::add; + + MqttConsumer consumer = new MqttConsumer(this.brokerUrl, this.topic, eventProcessor); + + Thread thread = new Thread(consumer); + thread.start(); + + while (consumer.getMessageCount() < n) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + return elements; + } + + @Override + public void run(AdapterPipeline adapterPipeline) { + SendToPipeline stk = new SendToPipeline(format, adapterPipeline); + this.mqttConsumer = new MqttConsumer(this.brokerUrl, this.topic, new MqttProtocol.EventProcessor(stk)); + + thread = new Thread(this.mqttConsumer); + thread.start(); + } + + @Override + public void stop() { + this.mqttConsumer.close(); + } + + @Override + public String getId() { + return ID; + } + + private class EventProcessor implements InternalEventProcessor { + private SendToPipeline stk; + + public EventProcessor(SendToPipeline stk) { + this.stk = stk; + } + + @Override + public void onEvent(byte[] payload) { + try { + parser.parse(IOUtils.toInputStream(new String(payload), "UTF-8"), stk); + } catch (IOException e) { + e.printStackTrace(); + //logger.error("Adapter " + ID + " could not read value!",e); + } + } + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/PullProtocol.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/PullProtocol.java new file mode 100644 index 0000000000..039e56a82e --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/protocol/stream/PullProtocol.java @@ -0,0 +1,85 @@ +/* +Copyright 2018 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package org.streampipes.connect.adapter.generic.protocol.stream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.connect.SendToPipeline; +import org.streampipes.connect.adapter.generic.format.Format; +import org.streampipes.connect.adapter.generic.format.Parser; +import org.streampipes.connect.adapter.generic.pipeline.AdapterPipeline; +import org.streampipes.connect.adapter.generic.protocol.Protocol; + +import java.io.InputStream; +import java.util.concurrent.*; + +public abstract class PullProtocol extends Protocol { + + private ScheduledExecutorService scheduler; + + private Logger logger = LoggerFactory.getLogger(PullProtocol.class); + + private long interval; + + + public PullProtocol() { + } + + public PullProtocol(Parser parser, Format format, long interval) { + super(parser, format); + this.interval = interval; + } + + @Override + public void run(AdapterPipeline adapterPipeline) { + final Runnable errorThread = () -> { + executeProtocolLogic(adapterPipeline); + }; + + + scheduler = Executors.newScheduledThreadPool(1); + scheduler.schedule(errorThread, 0, TimeUnit.MILLISECONDS); + + } + + + private void executeProtocolLogic(AdapterPipeline adapterPipeline) { + final Runnable task = () -> { + SendToPipeline stk = new SendToPipeline(format, adapterPipeline); + InputStream data = getDataFromEndpoint(); + + parser.parse(data, stk); + }; + + scheduler = Executors.newScheduledThreadPool(1); + ScheduledFuture handle = scheduler.scheduleAtFixedRate(task, 1, interval, TimeUnit.SECONDS); + try { + handle.get(); + } catch (ExecutionException e ) { + logger.error("Error", e); + } catch (InterruptedException e) { + logger.error("Error", e); + } + } + + @Override + public void stop() { + scheduler.shutdownNow(); + } + + abstract InputStream getDataFromEndpoint(); +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/firstconnector/sdk/ParameterExtractor.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/sdk/ParameterExtractor.java similarity index 51% rename from streampipes-connect/src/main/java/org/streampipes/connect/firstconnector/sdk/ParameterExtractor.java rename to streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/sdk/ParameterExtractor.java index bedb9ce682..ff756f8760 100644 --- a/streampipes-connect/src/main/java/org/streampipes/connect/firstconnector/sdk/ParameterExtractor.java +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/sdk/ParameterExtractor.java @@ -1,4 +1,21 @@ -package org.streampipes.connect.firstconnector.sdk; +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.sdk; import org.streampipes.model.staticproperty.FreeTextStaticProperty; import org.streampipes.model.staticproperty.StaticProperty; diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/transform/CreateNestedTransformationRule.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/transform/CreateNestedTransformationRule.java new file mode 100644 index 0000000000..ae0bac8ad1 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/transform/CreateNestedTransformationRule.java @@ -0,0 +1,54 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.transform; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class CreateNestedTransformationRule implements TransformationRule { + private List key; + + public CreateNestedTransformationRule(List key) { + this.key = key; + } + + @Override + public Map transform(Map event) { + return transform(event, key); + } + + private Map transform(Map event, List keys) { + + if (keys.size() == 1) { + event.put(keys.get(0), new HashMap<>()); + return event; + } else { + String key = keys.get(0); + List newKeysTmpList = keys.subList(1, keys.size()); + + Map newSubEvent = + transform((Map) event.get(keys.get(0)), newKeysTmpList); + + event.remove(key); + event.put(key, newSubEvent); + return event; + } + + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/transform/DeleteTransformationRule.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/transform/DeleteTransformationRule.java new file mode 100644 index 0000000000..1fd44713b0 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/transform/DeleteTransformationRule.java @@ -0,0 +1,53 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.transform; + +import java.util.List; +import java.util.Map; + +public class DeleteTransformationRule implements TransformationRule { + + private List key; + + public DeleteTransformationRule(List key) { + this.key = key; + } + + @Override + public Map transform(Map event) { + return transform(event, key); + } + + private Map transform(Map event, List keys) { + if (keys.size() == 1) { + event.remove(keys.get(0)); + return event; + } else { + String key = keys.get(0); + List newKeysTmpList = keys.subList(1, keys.size()); + + Map newSubEvent = + transform((Map) event.get(keys.get(0)), newKeysTmpList); + + event.remove(key); + event.put(key, newSubEvent); + return event; + } + + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/transform/EventTransformer.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/transform/EventTransformer.java new file mode 100644 index 0000000000..326787a7a3 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/transform/EventTransformer.java @@ -0,0 +1,116 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.transform; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class EventTransformer implements TransformationRule { + + private List renameTransformationRules; + private List createNestedTransformationRules; + private List moveTransformationRules; + private List deleteTransformationRules; + + public EventTransformer(List rules) { + this.renameTransformationRules = new ArrayList<>(); + this.createNestedTransformationRules = new ArrayList<>(); + this.moveTransformationRules = new ArrayList<>(); + this.deleteTransformationRules = new ArrayList<>(); + + for (TransformationRule rule : rules) { + if (rule instanceof RenameTransformationRule) { + this.renameTransformationRules.add((RenameTransformationRule) rule); + } else if (rule instanceof CreateNestedTransformationRule) { + this.createNestedTransformationRules.add((CreateNestedTransformationRule) rule); + } else if (rule instanceof MoveTransformationRule) { + this.moveTransformationRules.add((MoveTransformationRule) rule); + } else if (rule instanceof DeleteTransformationRule) { + this.deleteTransformationRules.add((DeleteTransformationRule) rule); + } + } + } + + + public EventTransformer(List renameTransformationRules, List createNestedTransformationRules, List moveTransformationRules, List deleteTransformationRules) { + this.renameTransformationRules = renameTransformationRules; + this.createNestedTransformationRules = createNestedTransformationRules; + this.moveTransformationRules = moveTransformationRules; + this.deleteTransformationRules = deleteTransformationRules; + } + + + @Override + public Map transform(Map event) { + + for (RenameTransformationRule renameRule : renameTransformationRules) { + event = renameRule.transform(event); + } + + for (CreateNestedTransformationRule createRule : createNestedTransformationRules) { + event = createRule.transform(event); + } + + for (MoveTransformationRule moveRule : moveTransformationRules) { + event = moveRule.transform(event); + } + + for (DeleteTransformationRule deleteRule : deleteTransformationRules) { + event = deleteRule.transform(event); + } + + return event; + } + + + + public List getRenameTransformationRules() { + return renameTransformationRules; + } + + public void setRenameTransformationRules(List renameTransformationRules) { + this.renameTransformationRules = renameTransformationRules; + } + + public List getCreateNestedTransformationRules() { + return createNestedTransformationRules; + } + + public void setCreateNestedTransformationRules(List createNestedTransformationRules) { + this.createNestedTransformationRules = createNestedTransformationRules; + } + + public List getMoveTransformationRules() { + return moveTransformationRules; + } + + public void setMoveTransformationRules(List moveTransformationRules) { + this.moveTransformationRules = moveTransformationRules; + } + + public List getDeleteTransformationRules() { + return deleteTransformationRules; + } + + public void setDeleteTransformationRules(List deleteTransformationRules) { + this.deleteTransformationRules = deleteTransformationRules; + } + + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/transform/MoveTransformationRule.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/transform/MoveTransformationRule.java new file mode 100644 index 0000000000..d45351223b --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/transform/MoveTransformationRule.java @@ -0,0 +1,101 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.transform; + +import java.util.*; + +public class MoveTransformationRule implements TransformationRule { + + private List oldKey; + private List newKey; + + public MoveTransformationRule(List oldKey, List newKey) { + this.oldKey = oldKey; + +// List tmp = new ArrayList<>(); +// for (int i = 0; i < newKey.size() - 1; i++) { +// tmp.add(newKey.get(i)); +// } + + this.newKey = newKey; + } + + @Override + public Map transform(Map event) { + + Map objectToMove = (Map) ((HashMap) getItem(event, oldKey)).clone(); + + + MapresultEvent = addItem(event, newKey, objectToMove); + resultEvent = deleteItem(event, oldKey); + + return resultEvent; + } + + private Map addItem(Map event, List keys, Map movedObject) { + if (keys.size() == 0 || (keys.size() == 1 && keys.get(0).equals(""))) { + String key = (String) movedObject.keySet().toArray()[0]; + event.put(key, movedObject.get(key)); + return event; + } else if (keys.size() == 1) { + event.put(keys.get(0), movedObject); + return event; + } else { + String key = keys.get(0); + List newKeysTmpList = keys.subList(1, keys.size()); + + Map newSubEvent = + addItem((Map) event.get(keys.get(0)), newKeysTmpList, movedObject); + + event.remove(key); + event.put(key, newSubEvent); + return event; + } + } + + + private Map getItem(Map event, List keys) { + if (keys.size() == 1) { + Map res = new HashMap<>(); + res.put(keys.get(0), event.get(keys.get(0))); + return res; + } else { + List newKeysTmpList = keys.subList(1, keys.size()); + + return getItem((Map) event.get(keys.get(0)), newKeysTmpList); + } + } + + private Map deleteItem(Map event, List keys) { + if (keys.size() == 1) { + + event.remove(keys.get(0)); + return event; + } else { + String key = keys.get(0); + List newKeysTmpList = keys.subList(1, keys.size()); + + Map newSubEvent = + deleteItem((Map) event.get(keys.get(0)), newKeysTmpList); + + event.remove(key); + event.put(key, newSubEvent); + return event; + } + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/transform/RenameTransformationRule.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/transform/RenameTransformationRule.java new file mode 100644 index 0000000000..bb3f2627bf --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/transform/RenameTransformationRule.java @@ -0,0 +1,58 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.transform; + +import java.util.List; +import java.util.Map; + +public class RenameTransformationRule implements TransformationRule { + private List oldKey; + private String newKey; + + public RenameTransformationRule(List oldKey, String newKey) { + this.oldKey = oldKey; + this.newKey = newKey; + } + + @Override + public Map transform(Map event) { + Map nestedEvent = event; + + return transform(event, oldKey); + } + + private Map transform(Map event, List keys) { + if (keys.size() == 1) { + Object o = event.get(keys.get(0)); + event.remove(keys.get(0)); + event.put(newKey, o); + + } else { + String key = keys.get(0); + List newKeysTmpList = keys.subList(1, keys.size()); + Map newSubEvent = + transform((Map) event.get(key), newKeysTmpList); + + event.remove(key); + event.put(key, newSubEvent); + } + + return event; + } +} + diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/transform/TransformationRule.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/transform/TransformationRule.java new file mode 100644 index 0000000000..2407667305 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/generic/transform/TransformationRule.java @@ -0,0 +1,24 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.generic.transform; + +import java.util.Map; + +public interface TransformationRule { + public Map transform(Map event); +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/PullAdapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/PullAdapter.java new file mode 100644 index 0000000000..717fd29b68 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/PullAdapter.java @@ -0,0 +1,100 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.specific; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.connect.adapter.generic.pipeline.AdapterPipeline; +import org.streampipes.connect.adapter.generic.pipeline.AdapterPipelineElement; +import org.streampipes.connect.adapter.generic.pipeline.elements.SendToKafkaAdapterSink; +import org.streampipes.connect.adapter.generic.pipeline.elements.TransformSchemaAdapterPipelineElement; +import org.streampipes.connect.adapter.util.PollingSettings; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.model.connect.adapter.AdapterDescription; +import org.streampipes.model.connect.adapter.SpecificAdapterStreamDescription; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +public abstract class PullAdapter extends SpecificDataStreamAdapter { + + protected static Logger logger = LoggerFactory.getLogger(PullAdapter.class); + private ScheduledExecutorService scheduler; + private ScheduledExecutorService errorThreadscheduler; + + protected AdapterPipeline adapterPipeline; + + public PullAdapter() { + super(); + } + + public PullAdapter(SpecificAdapterStreamDescription adapterDescription) { + super(adapterDescription); + } + + protected abstract void pullData(); + + protected abstract PollingSettings getPollingInterval(); + + @Override + public void startAdapter() throws AdapterException { + + List pipelineElements = new ArrayList<>(); + pipelineElements.add(new TransformSchemaAdapterPipelineElement(adapterDescription.getRules())); + pipelineElements.add(new SendToKafkaAdapterSink((AdapterDescription) adapterDescription)); + + adapterPipeline = new AdapterPipeline(pipelineElements); + + final Runnable errorThread = () -> { + executeAdpaterLogic(); + }; + + + scheduler = Executors.newScheduledThreadPool(1); + scheduler.schedule(errorThread, 0, TimeUnit.MILLISECONDS); + + } + + private void executeAdpaterLogic() { + final Runnable task = () -> { + + pullData(); + + }; + + scheduler = Executors.newScheduledThreadPool(1); + ScheduledFuture handle = scheduler.scheduleAtFixedRate(task, 1, + getPollingInterval().getValue(), getPollingInterval().getTimeUnit()); + + try { + handle.get(); + } catch (ExecutionException | InterruptedException e) { + logger.error("Error", e); + } + } + + @Override + public void stopAdapter() throws AdapterException { + scheduler.shutdownNow(); + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/PullRestAdapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/PullRestAdapter.java new file mode 100644 index 0000000000..0f7efefdfa --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/PullRestAdapter.java @@ -0,0 +1,53 @@ +package org.streampipes.connect.adapter.specific; + +import com.google.gson.Gson; +import org.apache.http.client.fluent.Request; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.model.connect.adapter.SpecificAdapterStreamDescription; + +public abstract class PullRestAdapter extends PullAdapter { + + public PullRestAdapter() { + super(); + } + + public PullRestAdapter(SpecificAdapterStreamDescription adapterDescription) { + super(adapterDescription); + } + + protected static String getDataFromEndpointString(String url) throws AdapterException { + String result = null; + + + logger.info("Started Request to open sensemap endpoint: " + url); + try { + result = Request.Get(url) + .connectTimeout(1000) + .socketTimeout(100000) + .execute().returnContent().asString(); + + + if (result.startsWith("ï")) { + result = result.substring(3); + } + + logger.info("Received data from request"); + + } catch (Exception e) { + String errorMessage = "Error while connecting to the open sensemap api"; + logger.error(errorMessage, e); + throw new AdapterException(errorMessage); + } + + return result; + } + + protected static T getDataFromEndpoint(String url, Class clazz) throws AdapterException { + + String rawJson = getDataFromEndpointString(url); + T all = new Gson().fromJson(rawJson, clazz); + + return all; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/SpecificDataSetAdapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/SpecificDataSetAdapter.java new file mode 100644 index 0000000000..068066a7ba --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/SpecificDataSetAdapter.java @@ -0,0 +1,32 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.specific; + +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.model.connect.adapter.SpecificAdapterSetDescription; + +public abstract class SpecificDataSetAdapter extends Adapter { + + public SpecificDataSetAdapter(SpecificAdapterSetDescription adapterDescription) { + super(adapterDescription); + } + + public SpecificDataSetAdapter(SpecificAdapterSetDescription adapterDescription, boolean debug) { + super(adapterDescription, debug); + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/SpecificDataStreamAdapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/SpecificDataStreamAdapter.java new file mode 100644 index 0000000000..4a2e0dce42 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/SpecificDataStreamAdapter.java @@ -0,0 +1,37 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.specific; + +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.model.connect.adapter.SpecificAdapterStreamDescription; + +public abstract class SpecificDataStreamAdapter extends Adapter { + + public SpecificDataStreamAdapter() { + super(); + } + + public SpecificDataStreamAdapter(SpecificAdapterStreamDescription adapterDescription) { + super(adapterDescription); + } + + public SpecificDataStreamAdapter(SpecificAdapterStreamDescription adapterDescription, boolean debug) { + super(adapterDescription, debug); + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/gdelt/GdeltAdapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/gdelt/GdeltAdapter.java new file mode 100644 index 0000000000..98b62a7fc6 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/gdelt/GdeltAdapter.java @@ -0,0 +1,165 @@ +package org.streampipes.connect.adapter.specific.gdelt; + +import com.opencsv.CSVReader; +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.connect.adapter.specific.PullAdapter; +import org.streampipes.connect.adapter.util.PollingSettings; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.model.connect.adapter.SpecificAdapterStreamDescription; +import org.streampipes.model.connect.guess.GuessSchema; +import org.streampipes.model.schema.EventProperty; +import org.streampipes.model.schema.EventSchema; +import org.streampipes.sdk.builder.PrimitivePropertyBuilder; +import org.streampipes.sdk.builder.adapter.SpecificDataStreamAdapterBuilder; +import org.streampipes.sdk.utils.Datatypes; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.zip.ZipInputStream; + +public class GdeltAdapter extends PullAdapter { + + public static final String ID = "http://streampipes.org/adapter/specific/gdelt"; + private String url = "http://data.gdeltproject.org/gdeltv2/lastupdate.txt"; + + public GdeltAdapter() { + super(); + } + + public GdeltAdapter(SpecificAdapterStreamDescription adapterDescription) { + super(adapterDescription); + } + + @Override + protected PollingSettings getPollingInterval() { + return PollingSettings.from(TimeUnit.MINUTES, 15); + } + + @Override + public SpecificAdapterStreamDescription declareModel() { + return SpecificDataStreamAdapterBuilder.create(ID, "GDELT", "Global Database of Society") + .iconUrl("gdelt.png") + .build(); + } + + public List> getEvents() { + + List> eventResults = new ArrayList<>(); + return eventResults; + } + + @Override + protected void pullData() { + try { + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new URL(url).openStream())); + String firstLine = bufferedReader.readLine(); + String[] parts = firstLine.split(" "); + URL zipFileUrl = new URL(parts[2]); + bufferedReader.close(); + ZipInputStream zipInputStream = new ZipInputStream(zipFileUrl.openStream()); + BufferedReader zipBufferedReader = new BufferedReader(new InputStreamReader(zipInputStream)); + zipInputStream.getNextEntry(); + CSVReader csvReader = new CSVReader(zipBufferedReader, '\t', '"'); + String[] nextRecord; + while ((nextRecord = csvReader.readNext()) != null) { + + Map event = new HashMap<>(); + + event.put("global_event_id", nextRecord[0]); + event.put("day", nextRecord[1]); + event.put("month_year", nextRecord[2]); + event.put("year", nextRecord[3]); + event.put("fraction_date", nextRecord[4]); + + event.put("actor_1_code", nextRecord[5]); + event.put("actor_1_name", nextRecord[6]); + event.put("actor_1_country_code", nextRecord[7]); + event.put("actor_1_known_group_code", nextRecord[8]); + event.put("actor_1_ethnic_code", nextRecord[9]); + event.put("actor_1_religion_1_code", nextRecord[10]); + event.put("actor_1_religion_2_code", nextRecord[11]); + event.put("actor_1_type_1_code", nextRecord[12]); + event.put("actor_1_type_2_code", nextRecord[13]); + event.put("actor_1_type_3_code", nextRecord[14]); + + event.put("actor_2_code", nextRecord[15]); + event.put("actor_2_name", nextRecord[16]); + event.put("actor_2_country_code", nextRecord[17]); + event.put("actor_2_known_group_code", nextRecord[18]); + event.put("actor_2_ethnic_code", nextRecord[19]); + event.put("actor_2_religion_1_code", nextRecord[20]); + event.put("actor_2_religion_2_code", nextRecord[21]); + event.put("actor_2_type_1_code", nextRecord[22]); + event.put("actor_2_type_2_code", nextRecord[23]); + event.put("actor_2_type_3_code", nextRecord[24]); + + adapterPipeline.process(event); + + } + } catch (Exception e) { + e.printStackTrace(); + } + + } + + @Override + public void stopAdapter() throws AdapterException { + + } + + @Override + public Adapter getInstance(SpecificAdapterStreamDescription adapterDescription) { + return new GdeltAdapter(adapterDescription); + } + + @Override + public GuessSchema getSchema(SpecificAdapterStreamDescription adapterDescription) { + GuessSchema guessSchema = new GuessSchema(); + EventSchema eventSchema = new EventSchema(); + + List allProperties = new ArrayList<>(); + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.Integer, "global_event_id").label("Global Event ID").description("Globally unique identifier").build()); + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.Integer, "day").label("Day").description("Date the event took place in YYYYMMDD format").build()); + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.Integer, "month_year").label("MonthYear").description("Date the event took place in YYYYMM format").build()); + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.Integer, "year").label("Year").description("Date the event took place in YYYY format").build()); + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.Float, "fraction_date").label("FractionDate").description("Date the event took place in YYYY.FFFF format (where FFFF is the percentage of the year completed)").build()); + + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.String, "actor_1_code").label("Actor1Code").description("The complete raw CAMEO code for Actor1").build()); + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.String, "actor_1_name").label("Actor1Name").description("The actual name of the Actor1").build()); + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.String, "actor_1_country_code").label("Actor1CountryCode").description("The 3-character CAMEO code for the country affiliation of Actor1").build()); + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.String, "actor_1_known_group_code").label("Actor1KnownGroupCode").description("The CAMEO code if Actor 1 is a known IGO/NGO/rebel organization").build()); + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.String, "actor_1_ethnic_code").label("Actor1EthnicCode").description("The CAMEO code if the source document specifies the ethnic affiliation of Actor1").build()); + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.String, "actor_1_religion_1_code").label("Actor1Religion1Code").description("The CAMEO code if the source document specifies the religious affiliation of Actor1").build()); + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.String, "actor_1_religion_2_code").label("Actor1Religion2Code").description("The CAMEO code if the source document specifies multiple religious affiliations of Actor1").build()); + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.String, "actor_1_type_1_code").label("Actor1Type1Code").description("The 3-character CAMEO code of the CAMEO “type” or “role” of Actor1").build()); + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.String, "actor_1_type_2_code").label("Actor1Type2Code").description("The 3-character CAMEO code of the CAMEO “type” or “role” of Actor1").build()); + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.String, "actor_1_type_3_code").label("Actor1Type3Code").description("The 3-character CAMEO code of the CAMEO “type” or “role” of Actor1").build()); + + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.String, "actor_2_code").label("Actor2Code").description("The complete raw CAMEO code for Actor2").build()); + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.String, "actor_2_name").label("Actor2Name").description("The actual name of the Actor2").build()); + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.String, "actor_2_country_code").label("Actor2CountryCode").description("The 3-character CAMEO code for the country affiliation of Actor2").build()); + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.String, "actor_2_known_group_code").label("Actor2KnownGroupCode").description("The CAMEO code if Actor 1 is a known IGO/NGO/rebel organization").build()); + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.String, "actor_2_ethnic_code").label("Actor2EthnicCode").description("The CAMEO code if the source document specifies the ethnic affiliation of Actor2").build()); + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.String, "actor_2_religion_1_code").label("Actor2Religion1Code").description("The CAMEO code if the source document specifies the religious affiliation of Actor2").build()); + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.String, "actor_2_religion_2_code").label("Actor2Religion2Code").description("The CAMEO code if the source document specifies multiple religious affiliations of Actor2").build()); + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.String, "actor_2_type_1_code").label("Actor2Type1Code").description("The 3-character CAMEO code of the CAMEO “type” or “role” of Actor2").build()); + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.String, "actor_2_type_2_code").label("Actor2Type2Code").description("The 3-character CAMEO code of the CAMEO “type” or “role” of Actor2").build()); + allProperties.add(PrimitivePropertyBuilder.create(Datatypes.String, "actor_2_type_3_code").label("Actor2Type3Code").description("The 3-character CAMEO code of the CAMEO “type” or “role” of Actor2").build()); + + eventSchema.setEventProperties(allProperties); + guessSchema.setEventSchema(eventSchema); + guessSchema.setPropertyProbabilityList(new ArrayList<>()); + return guessSchema; + } + + @Override + public String getId() { + return ID; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/CameraDataHttpExecutor.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/CameraDataHttpExecutor.java new file mode 100644 index 0000000000..95d69237d1 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/CameraDataHttpExecutor.java @@ -0,0 +1,37 @@ +/* +Copyright 2018 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.connect.adapter.specific.nswaustralia.trafficcamera; + +import org.apache.http.client.fluent.Request; + +import java.io.IOException; + +public class CameraDataHttpExecutor { + + private String imageUrl; + + public CameraDataHttpExecutor(String imageUrl) { + this.imageUrl = imageUrl; + } + + public byte[] getImageData() throws IOException { + return Request + .Get(imageUrl) + .execute() + .returnContent() + .asBytes(); + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/CameraFeatureTransformer.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/CameraFeatureTransformer.java new file mode 100644 index 0000000000..8f7a6bf8f7 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/CameraFeatureTransformer.java @@ -0,0 +1,64 @@ +/* +Copyright 2018 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.connect.adapter.specific.nswaustralia.trafficcamera; + +import org.streampipes.connect.adapter.specific.nswaustralia.trafficcamera.model.Feature; +import org.streampipes.connect.adapter.specific.sensemap.SensorNames; + +import java.io.IOException; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; + +public class CameraFeatureTransformer { + + private Feature cameraInfo; + + public CameraFeatureTransformer(Feature cameraInfo) { + this.cameraInfo = cameraInfo; + } + + public Map toMap() { + Map map = new HashMap<>(); + + map.put(SensorNames.KEY_TIMESTAMP, System.currentTimeMillis()); + map.put(TrafficCameraSensorNames.KEY_REGION, cameraInfo.getProperties().getRegion()); + map.put(TrafficCameraSensorNames.KEY_DIRECTION, cameraInfo.getProperties().getDirection()); + map.put(TrafficCameraSensorNames.KEY_VIEW, cameraInfo.getProperties().getView()); + map.put(TrafficCameraSensorNames.KEY_TITLE, cameraInfo.getProperties().getTitle()); + map.put(TrafficCameraSensorNames.KEY_LATITUDE, cameraInfo.getGeometry().getCoordinates().get + (0)); + map.put(TrafficCameraSensorNames.KEY_LONGITUDE, cameraInfo.getGeometry().getCoordinates().get + (1)); + map.put(TrafficCameraSensorNames.KEY_IMAGE, getImage(cameraInfo.getProperties().getHref())); + + return map; + } + + private String getImage(String href) { + try { + byte[] imageBytes = new CameraDataHttpExecutor(href).getImageData(); + return toBase64(imageBytes); + } catch (IOException e) { + e.printStackTrace(); + return "No image available"; + } + } + + private String toBase64(byte[] imageData) { + return Base64.getEncoder().encodeToString(imageData); + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/CameraInfoHttpExecutor.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/CameraInfoHttpExecutor.java new file mode 100644 index 0000000000..fdc2ae0cc3 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/CameraInfoHttpExecutor.java @@ -0,0 +1,45 @@ +/* +Copyright 2018 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.connect.adapter.specific.nswaustralia.trafficcamera; + +import com.google.gson.Gson; +import org.apache.http.client.fluent.Request; +import org.streampipes.connect.adapter.specific.nswaustralia.trafficcamera.model.FeatureCollection; + +import java.io.IOException; + +public class CameraInfoHttpExecutor { + + private static final String Url = "https://api.transport.nsw.gov.au/v1/live/cameras"; + + private String apiKey; + + public CameraInfoHttpExecutor(String apiKey) { + this.apiKey = apiKey; + } + + public FeatureCollection getCameraData() throws IOException { + + String response = Request + .Get(Url) + .addHeader("Authorization", "apiKey " +this.apiKey) + .execute() + .returnContent() + .asString(); + + return new Gson().fromJson(response, FeatureCollection.class); + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/NswTrafficCameraAdapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/NswTrafficCameraAdapter.java new file mode 100644 index 0000000000..6e677e687d --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/NswTrafficCameraAdapter.java @@ -0,0 +1,145 @@ +/* +Copyright 2018 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.connect.adapter.specific.nswaustralia.trafficcamera; + +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.connect.adapter.specific.PullAdapter; +import org.streampipes.connect.adapter.specific.nswaustralia.trafficcamera.model.Feature; +import org.streampipes.connect.adapter.specific.nswaustralia.trafficcamera.model.FeatureCollection; +import org.streampipes.connect.adapter.specific.sensemap.SensorNames; +import org.streampipes.connect.adapter.util.PollingSettings; +import org.streampipes.model.connect.adapter.SpecificAdapterStreamDescription; +import org.streampipes.model.connect.guess.GuessSchema; +import org.streampipes.model.schema.EventProperty; +import org.streampipes.model.schema.EventSchema; +import org.streampipes.sdk.builder.PrimitivePropertyBuilder; +import org.streampipes.sdk.builder.adapter.SpecificDataStreamAdapterBuilder; +import org.streampipes.sdk.helpers.EpProperties; +import org.streampipes.sdk.helpers.Labels; +import org.streampipes.sdk.utils.Datatypes; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class NswTrafficCameraAdapter extends PullAdapter { + + public static final String ID = "http://streampipes.org/adapter/specific/nswtrafficcamera"; + private static final String API_KEY = ""; + + public NswTrafficCameraAdapter() { + super(); + } + + public NswTrafficCameraAdapter(SpecificAdapterStreamDescription adapterDescription) { + super(adapterDescription); + } + + @Override + protected void pullData() { + List> events = getEvents(); + + for (Map event : events) { + adapterPipeline.process(event); + } + } + + @Override + protected PollingSettings getPollingInterval() { + return PollingSettings.from(TimeUnit.MINUTES, 5); + } + + private List> getEvents() { + List> events = new ArrayList<>(); + + try { + FeatureCollection cameras = new CameraInfoHttpExecutor(API_KEY).getCameraData(); + + for (Feature cameraInfo : cameras.getFeatures()) { + events.add(new CameraFeatureTransformer(cameraInfo).toMap()); + } + } catch (IOException e) { + e.printStackTrace(); + } + + return events; + } + + @Override + public SpecificAdapterStreamDescription declareModel() { + return SpecificDataStreamAdapterBuilder.create(ID, "NSW Traffic Cameras", "Traffic camera " + + "images produced by NSW Australia") + .requiredTextParameter(Labels.from("api-key", "API Key", "The TfNSW " + + "API key")) + .iconUrl("nsw.png") + .build(); + } + + @Override + public Adapter getInstance(SpecificAdapterStreamDescription adapterDescription) { + return new NswTrafficCameraAdapter(adapterDescription); + } + + @Override + public GuessSchema getSchema(SpecificAdapterStreamDescription adapterDescription) { + + EventSchema schema = new EventSchema(); + + List allProperties = new ArrayList<>(); + + allProperties.add(EpProperties.timestampProperty(SensorNames.KEY_TIMESTAMP)); + + allProperties.add(PrimitivePropertyBuilder + .create(Datatypes.String, TrafficCameraSensorNames.KEY_REGION) + .label(TrafficCameraSensorNames.LABEL_REGION) + .description("The region") + .build()); + + allProperties.add(PrimitivePropertyBuilder + .create(Datatypes.String, TrafficCameraSensorNames.KEY_VIEW) + .label(TrafficCameraSensorNames.LABEL_VIEW) + .description("The view") + .build()); + + allProperties.add(PrimitivePropertyBuilder + .create(Datatypes.String, TrafficCameraSensorNames.KEY_DIRECTION) + .label(TrafficCameraSensorNames.LABEL_DIRECTION) + .description("The region") + .build()); + + allProperties.add(PrimitivePropertyBuilder + .create(Datatypes.String, TrafficCameraSensorNames.KEY_IMAGE) + .label(TrafficCameraSensorNames.LABEL_IMAGE) + .description("The image") + .build()); + + schema.setEventProperties(allProperties); + + GuessSchema guessSchema = new GuessSchema(); + guessSchema.setEventSchema(schema); + guessSchema.setPropertyProbabilityList(Collections.emptyList()); + + return guessSchema; + } + + @Override + public String getId() { + return ID; + } +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/TrafficCameraSensorNames.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/TrafficCameraSensorNames.java new file mode 100644 index 0000000000..99a9ee8342 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/TrafficCameraSensorNames.java @@ -0,0 +1,40 @@ +/* +Copyright 2018 FZI Forschungszentrum Informatik + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package org.streampipes.connect.adapter.specific.nswaustralia.trafficcamera; + +public class TrafficCameraSensorNames { + + public static final String KEY_REGION = "region"; + public static final String LABEL_REGION = "Region"; + + public static final String KEY_VIEW = "view"; + public static final String LABEL_VIEW = "View"; + + public static final String KEY_DIRECTION = "direction"; + public static final String LABEL_DIRECTION = "Direction"; + + public static final String KEY_IMAGE = "image"; + public static final String LABEL_IMAGE = "Image"; + + public static final String KEY_TITLE = "title"; + public static final String LABEL_TITLE = "Title"; + + public static final String KEY_LATITUDE = "latitude"; + public static final String LABEL_LATITUDE = "Latitude"; + + public static final String KEY_LONGITUDE = "longitude"; + public static final String LABEL_LONGITUDE = "Longitude"; +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/model/Feature.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/model/Feature.java new file mode 100644 index 0000000000..c9fcee97ba --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/model/Feature.java @@ -0,0 +1,52 @@ + +package org.streampipes.connect.adapter.specific.nswaustralia.trafficcamera.model; + +import javax.annotation.Generated; +import com.google.gson.annotations.Expose; + +@Generated("net.hexar.json2pojo") +@SuppressWarnings("unused") +public class Feature { + + @Expose + private Geometry geometry; + @Expose + private String id; + @Expose + private Properties properties; + @Expose + private String type; + + public Geometry getGeometry() { + return geometry; + } + + public void setGeometry(Geometry geometry) { + this.geometry = geometry; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public Properties getProperties() { + return properties; + } + + public void setProperties(Properties properties) { + this.properties = properties; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/model/FeatureCollection.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/model/FeatureCollection.java new file mode 100644 index 0000000000..df0c902701 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/model/FeatureCollection.java @@ -0,0 +1,43 @@ + +package org.streampipes.connect.adapter.specific.nswaustralia.trafficcamera.model; + +import java.util.List; +import javax.annotation.Generated; +import com.google.gson.annotations.Expose; + +@Generated("net.hexar.json2pojo") +@SuppressWarnings("unused") +public class FeatureCollection { + + @Expose + private List features; + @Expose + private Rights rights; + @Expose + private String type; + + public List getFeatures() { + return features; + } + + public void setFeatures(List features) { + this.features = features; + } + + public Rights getRights() { + return rights; + } + + public void setRights(Rights rights) { + this.rights = rights; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/model/Geometry.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/model/Geometry.java new file mode 100644 index 0000000000..8935cd7b48 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/model/Geometry.java @@ -0,0 +1,33 @@ + +package org.streampipes.connect.adapter.specific.nswaustralia.trafficcamera.model; + +import java.util.List; +import javax.annotation.Generated; +import com.google.gson.annotations.Expose; + +@Generated("net.hexar.json2pojo") +@SuppressWarnings("unused") +public class Geometry { + + @Expose + private List coordinates; + @Expose + private String type; + + public List getCoordinates() { + return coordinates; + } + + public void setCoordinates(List coordinates) { + this.coordinates = coordinates; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/model/Properties.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/model/Properties.java new file mode 100644 index 0000000000..c60c40666b --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/model/Properties.java @@ -0,0 +1,62 @@ + +package org.streampipes.connect.adapter.specific.nswaustralia.trafficcamera.model; + +import javax.annotation.Generated; +import com.google.gson.annotations.Expose; + +@Generated("net.hexar.json2pojo") +@SuppressWarnings("unused") +public class Properties { + + @Expose + private String direction; + @Expose + private String href; + @Expose + private String region; + @Expose + private String title; + @Expose + private String view; + + public String getDirection() { + return direction; + } + + public void setDirection(String direction) { + this.direction = direction; + } + + public String getHref() { + return href; + } + + public void setHref(String href) { + this.href = href; + } + + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getView() { + return view; + } + + public void setView(String view) { + this.view = view; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/model/Rights.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/model/Rights.java new file mode 100644 index 0000000000..bc09b44c99 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/nswaustralia/trafficcamera/model/Rights.java @@ -0,0 +1,32 @@ + +package org.streampipes.connect.adapter.specific.nswaustralia.trafficcamera.model; + +import javax.annotation.Generated; +import com.google.gson.annotations.Expose; + +@Generated("net.hexar.json2pojo") +@SuppressWarnings("unused") +public class Rights { + + @Expose + private String copyright; + @Expose + private String licence; + + public String getCopyright() { + return copyright; + } + + public void setCopyright(String copyright) { + this.copyright = copyright; + } + + public String getLicence() { + return licence; + } + + public void setLicence(String licence) { + this.licence = licence; + } + +} diff --git a/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/sensemap/OpenSenseMapAdapter.java b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/sensemap/OpenSenseMapAdapter.java new file mode 100644 index 0000000000..d8fe8ee043 --- /dev/null +++ b/streampipes-connect/src/main/java/org/streampipes/connect/adapter/specific/sensemap/OpenSenseMapAdapter.java @@ -0,0 +1,400 @@ +/* + * Copyright 2018 FZI Forschungszentrum Informatik + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.streampipes.connect.adapter.specific.sensemap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.streampipes.connect.adapter.Adapter; +import org.streampipes.connect.adapter.specific.PullRestAdapter; +import org.streampipes.connect.adapter.specific.sensemap.model.CurrentLocation; +import org.streampipes.connect.adapter.specific.sensemap.model.SenseBox; +import org.streampipes.connect.adapter.specific.sensemap.model.Sensor; +import org.streampipes.connect.adapter.util.PollingSettings; +import org.streampipes.connect.exception.AdapterException; +import org.streampipes.model.connect.adapter.SpecificAdapterStreamDescription; +import org.streampipes.model.connect.guess.GuessSchema; +import org.streampipes.model.schema.EventProperty; +import org.streampipes.model.schema.EventPropertyPrimitive; +import org.streampipes.model.schema.EventSchema; +import org.streampipes.model.staticproperty.AnyStaticProperty; +import org.streampipes.model.staticproperty.Option; +import org.streampipes.sdk.builder.PrimitivePropertyBuilder; +import org.streampipes.sdk.builder.adapter.SpecificDataStreamAdapterBuilder; +import org.streampipes.sdk.helpers.EpProperties; +import org.streampipes.sdk.helpers.Labels; +import org.streampipes.sdk.utils.Datatypes; +import org.streampipes.vocabulary.XSD; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class OpenSenseMapAdapter extends PullRestAdapter { + + private Logger logger = LoggerFactory.getLogger(OpenSenseMapAdapter.class); + + public static final String ID = "http://streampipes.org/adapter/specific/opensensemap"; + public static final int POLLING_INTERVALL = 5; + + private List selectedSensors; + + private String standartKeys[] = {"id", "timestamp", "model", "latitude", "longitude"}; + + private String url = "https://api.opensensemap.org/boxes"; +// private String url = "http://localhost:3001/opensensemap"; +// private String url = "http://test-connect-datasources-rest:3001/opensensemap"; + + + public OpenSenseMapAdapter() { + super(); + } + + public OpenSenseMapAdapter(SpecificAdapterStreamDescription adapterDescription) { + super(adapterDescription); + + + } + + @Override + public SpecificAdapterStreamDescription declareModel() { + return SpecificDataStreamAdapterBuilder.create(ID, "OpenSenseMap", "Environment Sensors") + .iconUrl("openSenseMap.png") + .requiredMultiValueSelection(Labels.from("sensors", "Sensors", "Select the " + + "sensors that are included in the data stream"), Stream + .of(SensorNames.ALL_SENSOR_LABELS) + .map(s -> new Option(s, SensorNames.getKeyFromLabel(s))) + .collect(Collectors.toList())) + .build(); + } + + @Override + public Adapter getInstance(SpecificAdapterStreamDescription adapterDescription) { + return new OpenSenseMapAdapter(adapterDescription); + } + + @Override + public GuessSchema getSchema(SpecificAdapterStreamDescription adapterDescription) { + GuessSchema guessSchema = new GuessSchema(); + + EventSchema eventSchema = new EventSchema(); + EventPropertyPrimitive eventPropertyPrimitive = new EventPropertyPrimitive(); + eventPropertyPrimitive.setRuntimeType(XSD._double.toString()); + + List allProperties = new ArrayList<>(); + + List