Skip to content

Commit

Permalink
fix: skip broken records in export
Browse files Browse the repository at this point in the history
  • Loading branch information
@lvca
lvca committed Feb 4, 2025
1 parent 1771c9b commit 3fca7b1
Show file tree
Hide file tree
Showing 2 changed files with 50 additions and 44 deletions.
2 changes: 1 addition & 1 deletion integration/src/main/java/com/arcadedb/integration/exporter/Exporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ public Map<String, Object> exportDatabase() {
final Map<String, Object> result = new LinkedHashMap<>();
if (totalRecords > 0)
result.put("totalRecords", totalRecords);
result.put("elapsedInSecs", totalRecords);
result.put("elapsedInSecs", elapsedInSecs);
if (context.documents.get() > 0)
result.put("documents", context.documents.get());
if (context.vertices.get() > 0)
Expand Down
92 changes: 49 additions & 43 deletions integration/src/main/java/com/arcadedb/integration/exporter/format/JsonlExporterFormat.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,32 +29,26 @@
import com.arcadedb.integration.exporter.ExporterContext;
import com.arcadedb.integration.exporter.ExporterSettings;
import com.arcadedb.integration.importer.ConsoleLogger;
import com.arcadedb.log.LogManager;
import com.arcadedb.schema.DocumentType;
import com.arcadedb.schema.LocalEdgeType;
import com.arcadedb.schema.LocalSchema;
import com.arcadedb.schema.LocalVertexType;
import com.arcadedb.serializer.JsonGraphSerializer;
import com.arcadedb.serializer.json.JSONObject;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.zip.GZIPOutputStream;
import java.io.*;
import java.util.*;
import java.util.logging.*;
import java.util.zip.*;

public class JsonlExporterFormat extends AbstractExporterFormat {
public static final String NAME = "jsonl";
protected final JSONObject sharedJson = new JSONObject();
private OutputStreamWriter writer;
private final static int VERSION = 1;

public JsonlExporterFormat(
final DatabaseInternal database,
final ExporterSettings settings,
final ExporterContext context,
public JsonlExporterFormat(final DatabaseInternal database, final ExporterSettings settings, final ExporterContext context,
final ConsoleLogger logger) {
super(database, settings, context, logger);
}
Expand Down Expand Up @@ -88,18 +82,12 @@ public void exportDatabase() throws Exception {
DatabaseFactory.getDefaultCharset())) {
writer = fileWriter;

writeJsonLine("info", new JSONObject()
.put("description", "ArcadeDB Database Export")
.put("exporterVersion", VERSION)//
.put("dbVersion", Constants.getRawVersion())
.put("dbBranch", Constants.getBranch())
.put("dbBuild", Constants.getBuildNumber())
.put("dbTimestamp", Constants.getTimestamp()));
writeJsonLine("info", new JSONObject().put("description", "ArcadeDB Database Export").put("exporterVersion", VERSION)//
.put("dbVersion", Constants.getRawVersion()).put("dbBranch", Constants.getBranch())
.put("dbBuild", Constants.getBuildNumber()).put("dbTimestamp", Constants.getTimestamp()));

final long now = System.currentTimeMillis();
writeJsonLine("db", new JSONObject()
.put("name", database.getName())
.put("executedOn", dateFormat.format(now))
writeJsonLine("db", new JSONObject().put("name", database.getName()).put("executedOn", dateFormat.format(now))
.put("executedOnTimestamp", now));

writeJsonLine("schema", ((LocalSchema) database.getSchema()).toJSON());
Expand Down Expand Up @@ -142,41 +130,59 @@ public String getName() {
private void exportVertices(final List<String> vertexTypes, final JsonGraphSerializer graphSerializer) throws IOException {
for (final String type : vertexTypes) {
for (final Iterator<Record> cursor = database.iterateType(type, false); cursor.hasNext(); ) {
final Vertex record = cursor.next().asVertex(true);

if (settings.includeRecords != null && !settings.includeRecords.contains(record.getIdentity().toString()))
continue;

writeJsonLine("v", graphSerializer.serializeGraphElement(record));
context.vertices.incrementAndGet();
Vertex record = null;
try {
record = cursor.next().asVertex(true);

if (settings.includeRecords != null && !settings.includeRecords.contains(record.getIdentity().toString()))
continue;

writeJsonLine("v", graphSerializer.serializeGraphElement(record));
context.vertices.incrementAndGet();
} catch (Exception e) {
LogManager.instance()
.log(this, Level.SEVERE, "Error on exporting vertex %s", e, record != null ? record.getIdentity() : null);
}
}
}
}

private void exportEdges(final List<String> edgeTypes, final JsonGraphSerializer graphSerializer) throws IOException {
for (final String type : edgeTypes) {
for (final Iterator<Record> cursor = database.iterateType(type, false); cursor.hasNext(); ) {
final Edge record = cursor.next().asEdge(true);

if (settings.includeRecords != null && !settings.includeRecords.contains(record.getIdentity().toString()))
continue;

writeJsonLine("e", graphSerializer.serializeGraphElement(record));
context.edges.incrementAndGet();
Edge record = null;
try {
record = cursor.next().asEdge(true);

if (settings.includeRecords != null && !settings.includeRecords.contains(record.getIdentity().toString()))
continue;

writeJsonLine("e", graphSerializer.serializeGraphElement(record));
context.edges.incrementAndGet();
} catch (Exception e) {
LogManager.instance()
.log(this, Level.SEVERE, "Error on exporting vertex %s", e, record != null ? record.getIdentity() : null);
}
}
}
}

private void exportDocuments(final List<String> documentTypes, final JsonGraphSerializer graphSerializer) throws IOException {
for (final String type : documentTypes) {
for (final Iterator<Record> cursor = database.iterateType(type, false); cursor.hasNext(); ) {
final Document record = cursor.next().asDocument(true);

if (settings.includeRecords != null && !settings.includeRecords.contains(record.getIdentity().toString()))
continue;

writeJsonLine("d", graphSerializer.serializeGraphElement(record));
context.documents.incrementAndGet();
Document record = null;
try {
record = cursor.next().asDocument(true);

if (settings.includeRecords != null && !settings.includeRecords.contains(record.getIdentity().toString()))
continue;

writeJsonLine("d", graphSerializer.serializeGraphElement(record));
context.documents.incrementAndGet();
} catch (Exception e) {
LogManager.instance()
.log(this, Level.SEVERE, "Error on exporting vertex %s", e, record != null ? record.getIdentity() : null);
}
}
}
}
Expand Down

0 comments on commit 3fca7b1

Please sign in to comment.