Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Persist denormalized keys before writing records #50

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Persist denormalized keys before writing records #50

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a7ad2b2
Use ByteArraySet consistently in favor of Set<ByteArray>
Aug 5, 2019
273161a
persist denormalized keys to create before creating
Aug 5, 2019
77a45ad
fix tests
Aug 5, 2019
58f91b7
add test
Aug 9, 2019
94ce90f
Merge branch 'master' into eric-persist-denormalized-keys-before-writ…
Sep 14, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 20 additions & 18 deletions src/main/java/com/jwplayer/southpaw/Southpaw.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ public class Southpaw {
* records) and join indices (points at the child records). The key is the index name. Multiple offsets
* can be stored per key.
*/
protected final Map<String, BaseIndex<BaseRecord, BaseRecord, Set<ByteArray>>> fkIndices = new HashMap<>();
protected final Map<String, BaseIndex<BaseRecord, BaseRecord, ByteArraySet>> fkIndices = new HashMap<>();
/**
* A map of all input topics needed by Southpaw. The key is the short name of the topic.
*/
Expand Down Expand Up @@ -258,38 +258,38 @@ protected void build(int runTimeS) {
ConsumerRecord<BaseRecord, BaseRecord> newRecord = records.next();
ByteArray primaryKey = newRecord.key().toByteArray();
for (Relation root : relations) {
Set<ByteArray> dePrimaryKeys = dePKsByType.get(root);
ByteArraySet dePrimaryKeys = dePKsByType.get(root);
if (root.getEntity().equals(entity)) {
// The top level relation is the relation of the input record
dePrimaryKeys.add(primaryKey);
} else {
// Check the child relations instead
AbstractMap.SimpleEntry<Relation, Relation> child = getRelation(root, entity);
if (child != null && child.getValue() != null) {
BaseIndex<BaseRecord, BaseRecord, Set<ByteArray>> parentIndex =
BaseIndex<BaseRecord, BaseRecord, ByteArraySet> parentIndex =
fkIndices.get(createParentIndexName(root, child.getKey(), child.getValue()));
ByteArray newParentKey = null;
Set<ByteArray> oldParentKeys;
ByteArraySet oldParentKeys;
if (newRecord.value() != null) {
newParentKey = ByteArray.toByteArray(newRecord.value().get(child.getValue().getJoinKey()));
}
BaseIndex<BaseRecord, BaseRecord, Set<ByteArray>> joinIndex =
BaseIndex<BaseRecord, BaseRecord, ByteArraySet> joinIndex =
fkIndices.get(createJoinIndexName(child.getValue()));
oldParentKeys = ((Reversible) joinIndex).getForeignKeys(primaryKey);

// Create the denormalized records
if (oldParentKeys != null) {
for (ByteArray oldParentKey : oldParentKeys) {
if (!ObjectUtils.equals(oldParentKey, newParentKey)) {
Set<ByteArray> primaryKeys = parentIndex.getIndexEntry(oldParentKey);
ByteArraySet primaryKeys = parentIndex.getIndexEntry(oldParentKey);
if (primaryKeys != null) {
dePrimaryKeys.addAll(primaryKeys);
}
}
}
}
if (newParentKey != null) {
Set<ByteArray> primaryKeys = parentIndex.getIndexEntry(newParentKey);
ByteArraySet primaryKeys = parentIndex.getIndexEntry(newParentKey);
if (primaryKeys != null) {
dePrimaryKeys.addAll(primaryKeys);
}
Expand All @@ -300,6 +300,8 @@ protected void build(int runTimeS) {
}
int size = dePrimaryKeys.size();
if(size > config.createRecordsTrigger) {
state.put(METADATA_KEYSPACE, createDePKEntryName(root).getBytes(), dePrimaryKeys.serialize());
state.flush(METADATA_KEYSPACE);
createDenormalizedRecords(root, dePrimaryKeys);
dePrimaryKeys.clear();
}
Expand Down Expand Up @@ -393,7 +395,7 @@ public void commit() {
for(Map.Entry<String, BaseTopic<byte[], DenormalizedRecord>> topic: outputTopics.entrySet()) {
topic.getValue().flush();
}
for(Map.Entry<String, BaseIndex<BaseRecord, BaseRecord, Set<ByteArray>>> index: fkIndices.entrySet()) {
for(Map.Entry<String, BaseIndex<BaseRecord, BaseRecord, ByteArraySet>> index: fkIndices.entrySet()) {
index.getValue().flush();
}
for(Map.Entry<Relation, ByteArraySet> entry: dePKsByType.entrySet()) {
Expand Down Expand Up @@ -458,8 +460,8 @@ protected DenormalizedRecord createDenormalizedRecord(
updateParentIndex(root, relation, child, rootPrimaryKey, newParentKey);
Map<ByteArray, DenormalizedRecord> records = new TreeMap<>();
if (newParentKey != null) {
BaseIndex<BaseRecord, BaseRecord, Set<ByteArray>> joinIndex = fkIndices.get(createJoinIndexName(child));
Set<ByteArray> childPKs = joinIndex.getIndexEntry(newParentKey);
BaseIndex<BaseRecord, BaseRecord, ByteArraySet> joinIndex = fkIndices.get(createJoinIndexName(child));
ByteArraySet childPKs = joinIndex.getIndexEntry(newParentKey);
if (childPKs != null) {
for (ByteArray childPK : childPKs) {
DenormalizedRecord deChildRecord = createDenormalizedRecord(root, child, rootPrimaryKey, childPK);
Expand All @@ -481,7 +483,7 @@ protected DenormalizedRecord createDenormalizedRecord(
*/
protected void createDenormalizedRecords(
Relation root,
Set<ByteArray> rootRecordPKs) {
ByteArraySet rootRecordPKs) {
for(ByteArray dePrimaryKey: rootRecordPKs) {
if(dePrimaryKey != null) {
BaseTopic<byte[], DenormalizedRecord> outputTopic = outputTopics.get(root.getDenormalizedName());
Expand Down Expand Up @@ -528,7 +530,7 @@ protected String createDePKEntryName(Relation root) {
* @param indexedTopicName - The name of the indexed topic
* @return A brand new, shiny index
*/
protected BaseIndex<BaseRecord, BaseRecord, Set<ByteArray>> createFkIndex(
protected BaseIndex<BaseRecord, BaseRecord, ByteArraySet> createFkIndex(
String indexName,
String indexedTopicName) {
MultiIndex<BaseRecord, BaseRecord> index = new MultiIndex<>();
Expand Down Expand Up @@ -887,9 +889,9 @@ protected void scrubParentIndices(Relation root, Relation parent, ByteArray root

if(parent.getChildren() != null && rootPrimaryKey != null) {
for(Relation child: parent.getChildren()) {
BaseIndex<BaseRecord, BaseRecord, Set<ByteArray>> parentIndex =
BaseIndex<BaseRecord, BaseRecord, ByteArraySet> parentIndex =
fkIndices.get(createParentIndexName(root, parent, child));
Set<ByteArray> oldForeignKeys = ((Reversible) parentIndex).getForeignKeys(rootPrimaryKey);
ByteArraySet oldForeignKeys = ((Reversible) parentIndex).getForeignKeys(rootPrimaryKey);
if(oldForeignKeys != null) {
for(ByteArray oldForeignKey: ImmutableSet.copyOf(oldForeignKeys)) {
parentIndex.remove(oldForeignKey, rootPrimaryKey);
Expand All @@ -914,8 +916,8 @@ protected void updateJoinIndex(
ConsumerRecord<BaseRecord, BaseRecord> newRecord) {
Preconditions.checkNotNull(relation.getJoinKey());
Preconditions.checkNotNull(newRecord);
BaseIndex<BaseRecord, BaseRecord, Set<ByteArray>> joinIndex = fkIndices.get(createJoinIndexName(relation));
Set<ByteArray> oldJoinKeys = ((Reversible) joinIndex).getForeignKeys(primaryKey);
BaseIndex<BaseRecord, BaseRecord, ByteArraySet> joinIndex = fkIndices.get(createJoinIndexName(relation));
ByteArraySet oldJoinKeys = ((Reversible) joinIndex).getForeignKeys(primaryKey);
ByteArray newJoinKey = null;
if(newRecord.value() != null) {
newJoinKey = ByteArray.toByteArray(newRecord.value().get(relation.getJoinKey()));
Expand Down Expand Up @@ -952,7 +954,7 @@ protected void updateParentIndex(
Preconditions.checkNotNull(child);
Preconditions.checkNotNull(rootPrimaryKey);

BaseIndex<BaseRecord, BaseRecord, Set<ByteArray>> parentIndex =
BaseIndex<BaseRecord, BaseRecord, ByteArraySet> parentIndex =
fkIndices.get(createParentIndexName(root, parent, child));
if (newParentKey != null) parentIndex.add(newParentKey, rootPrimaryKey);
}
Expand Down Expand Up @@ -1007,7 +1009,7 @@ protected static void validateRootRelations(Relation[] relations) {
* <b>Note: this requires a full scan of each index dataset. This could be an expensive operation on larger datasets</b>
*/
protected void verifyState() {
for(Map.Entry<String, BaseIndex<BaseRecord, BaseRecord, Set<ByteArray>>> index: fkIndices.entrySet()) {
for(Map.Entry<String, BaseIndex<BaseRecord, BaseRecord, ByteArraySet>> index: fkIndices.entrySet()) {
logger.info("Verifying index state integrity: " + index.getValue().getIndexedTopic().getShortName());
Set<String> missingIndexKeys = ((MultiIndex)index.getValue()).verifyIndexState();
if(missingIndexKeys.isEmpty()){
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/com/jwplayer/southpaw/index/MultiIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
* @param <K> - The type of the key stored in the indexed topic
* @param <V> - The type of the value stored in the indexed topic
*/
public class MultiIndex<K, V> extends BaseIndex<K, V, Set<ByteArray>> implements Reversible {
public class MultiIndex<K, V> extends BaseIndex<K, V, ByteArraySet> implements Reversible {
/**
* Size of the LRU cache for storing the index entries containing more than one key
*/
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/com/jwplayer/southpaw/index/Reversible.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package com.jwplayer.southpaw.index;

import com.jwplayer.southpaw.util.ByteArray;
import com.jwplayer.southpaw.util.ByteArraySet;

import java.util.Set;

Expand All @@ -28,5 +29,5 @@ public interface Reversible {
* @param primaryKey - The primary key to lookup
* @return The keys for the given primary key or null if no corresponding entry exists
*/
Set<ByteArray> getForeignKeys(ByteArray primaryKey);
ByteArraySet getForeignKeys(ByteArray primaryKey);
}
5 changes: 3 additions & 2 deletions src/test/java/com/jwplayer/southpaw/MockSouthpaw.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.jwplayer.southpaw.record.BaseRecord;
import com.jwplayer.southpaw.topic.BaseTopic;
import com.jwplayer.southpaw.util.ByteArray;
import com.jwplayer.southpaw.util.ByteArraySet;

import java.io.IOException;
import java.net.URI;
Expand All @@ -41,7 +42,7 @@ public MockSouthpaw(Map<String, Object> config, List<URI> uris)

public void createDenormalizedRecords(
Relation root,
Set<ByteArray> rootRecordPKs) {
ByteArraySet rootRecordPKs) {
super.createDenormalizedRecords(root, rootRecordPKs);
}

Expand All @@ -61,7 +62,7 @@ public AbstractMap.SimpleEntry<Relation, Relation> getRelation(Relation relation
* Accessor for the FK indices used by Southpaw
* @return Southpaw's FK indices
*/
public Map<String, BaseIndex<BaseRecord, BaseRecord, Set<ByteArray>>> getFkIndices() {
public Map<String, BaseIndex<BaseRecord, BaseRecord, ByteArraySet>> getFkIndices() {
return fkIndices;
}

Expand Down
Loading