Skip to content
This repository has been archived by the owner on Feb 29, 2020. It is now read-only.

Commit

Permalink
[client][managed][offline] Implement true upsert
Browse files Browse the repository at this point in the history
  • Loading branch information
@hasankhan
hasankhan committed Nov 7, 2014
1 parent f02584a commit c5b0b38
Show file tree
Hide file tree
Showing 3 changed files with 156 additions and 65 deletions.
176 changes: 124 additions & 52 deletions ...Managed/src/Microsoft.WindowsAzure.MobileServices.SQLiteStore/MobileServiceSQLiteStore.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ public class MobileServiceSQLiteStore : MobileServiceLocalStore
/// Note: The default maximum number of parameters allowed by sqlite is 999
/// See: http://www.sqlite.org/limits.html#max_variable_number
/// </summary>
private const int MaxParametersPerUpsertQuery = 800;
private const int MaxParametersPerQuery = 800;

private Dictionary<string, TableDefinition> tableMap = new Dictionary<string, TableDefinition>(StringComparer.OrdinalIgnoreCase);
private SQLiteConnection connection;
Expand Down Expand Up @@ -183,64 +183,17 @@ private Task UpsertAsyncInternal(string tableName, IEnumerable<JObject> items, b
return Task.FromResult(0);
}

// Generate the prepared insert statement
string sqlBase = String.Format(
"INSERT OR REPLACE INTO {0} ({1}) VALUES ",
SqlHelpers.FormatTableName(tableName),
String.Join(", ", columns.Select(c => c.Name).Select(SqlHelpers.FormatMember))
);

// Use int division to calculate how many times this record will fit into our parameter quota
int batchSize = MaxParametersPerUpsertQuery / columns.Count;
if (batchSize == 0)
{
throw new InvalidOperationException(string.Format(Properties.Resources.SQLiteStore_TooManyColumns, MaxParametersPerUpsertQuery));
}

foreach (var batch in items.Split(maxLength: batchSize))
{
var sql = new StringBuilder(sqlBase);
var parameters = new Dictionary<string, object>();
this.ExecuteNonQuery("BEGIN TRANSACTION", null);

foreach (JObject item in batch)
{
AppendInsertValuesSql(sql, parameters, columns, item);
sql.Append(",");
}
BatchInsert(tableName, items, columns.Where(c => c.Name.Equals(MobileServiceSystemColumns.Id)).Take(1).ToList());
BatchUpdate(tableName, items, columns);

if (parameters.Any())
{
sql.Remove(sql.Length - 1, 1); // remove the trailing comma
this.ExecuteNonQuery(sql.ToString(), parameters);
}
}
this.ExecuteNonQuery("COMMIT TRANSACTION", null);

return Task.FromResult(0);
}

private static void AppendInsertValuesSql(StringBuilder sql, Dictionary<string, object> parameters, List<ColumnDefinition> columns, JObject item)
{
sql.Append("(");
int colCount = 0;
foreach (var column in columns)
{
if (colCount > 0)
sql.Append(",");

JToken rawValue = item.GetValue(column.Name, StringComparison.OrdinalIgnoreCase);
object value = SqlHelpers.SerializeValue(rawValue, column.StoreType, column.JsonType);

//The paramname for this field must be unique within this statement
string paramName = "@p" + parameters.Count;

sql.Append(paramName);
parameters[paramName] = value;

colCount++;
}
sql.Append(")");
}

/// <summary>
/// Deletes items from local table that match the given query.
/// </summary>
Expand Down Expand Up @@ -373,6 +326,110 @@ private void CreateAllTables()
}
}

private void BatchUpdate(string tableName, IEnumerable<JObject> items, List<ColumnDefinition> columns)
{
if (columns.Count <= 1)
{
return; // For update to work there has to be at least once column besides Id that needs to be updated
}

ValidateParameterCount(columns.Count);

string sqlBase = String.Format("UPDATE {0} SET ", SqlHelpers.FormatTableName(tableName));

foreach (JObject item in items)
{
var sql = new StringBuilder(sqlBase);
var parameters = new Dictionary<string, object>();

ColumnDefinition idColumn = columns.FirstOrDefault(c => c.Name.Equals(MobileServiceSystemColumns.Id));
if (idColumn == null)
{
continue;
}

foreach (var column in columns.Where(c => c != idColumn))
{
string paramName = AddParameter(item, parameters, column);

sql.AppendFormat("{0} = {1}", SqlHelpers.FormatMember(column.Name), paramName);
sql.Append(",");
}

if (parameters.Any())
{
sql.Remove(sql.Length - 1, 1); // remove the trailing comma

}

sql.AppendFormat(" WHERE {0} = {1}", SqlHelpers.FormatMember(MobileServiceSystemColumns.Id), AddParameter(item, parameters, idColumn));

this.ExecuteNonQuery(sql.ToString(), parameters);
}
}

private void BatchInsert(string tableName, IEnumerable<JObject> items, List<ColumnDefinition> columns)
{
if (columns.Count == 0) // we need to have some columns to insert the item
{
return;
}

// Generate the prepared insert statement
string sqlBase = String.Format(
"INSERT OR IGNORE INTO {0} ({1}) VALUES ",
SqlHelpers.FormatTableName(tableName),
String.Join(", ", columns.Select(c => c.Name).Select(SqlHelpers.FormatMember))
);

// Use int division to calculate how many times this record will fit into our parameter quota
int batchSize = ValidateParameterCount(columns.Count);

foreach (var batch in items.Split(maxLength: batchSize))
{
var sql = new StringBuilder(sqlBase);
var parameters = new Dictionary<string, object>();

foreach (JObject item in batch)
{
AppendInsertValuesSql(sql, parameters, columns, item);
sql.Append(",");
}

if (parameters.Any())
{
sql.Remove(sql.Length - 1, 1); // remove the trailing comma
this.ExecuteNonQuery(sql.ToString(), parameters);
}
}
}

private static int ValidateParameterCount(int parametersCount)
{
int batchSize = MaxParametersPerQuery / parametersCount;
if (batchSize == 0)
{
throw new InvalidOperationException(string.Format(Properties.Resources.SQLiteStore_TooManyColumns, MaxParametersPerQuery));
}
return batchSize;
}

private static void AppendInsertValuesSql(StringBuilder sql, Dictionary<string, object> parameters, List<ColumnDefinition> columns, JObject item)
{
sql.Append("(");
int colCount = 0;
foreach (var column in columns)
{
if (colCount > 0)
sql.Append(",");

sql.Append(AddParameter(item, parameters, column));

colCount++;
}
sql.Append(")");
}

internal virtual void CreateTableFromObject(string tableName, IEnumerable<ColumnDefinition> columns)
{
ColumnDefinition idColumn = columns.FirstOrDefault(c => c.Name.Equals(MobileServiceSystemColumns.Id));
Expand Down Expand Up @@ -404,6 +461,21 @@ internal virtual void CreateTableFromObject(string tableName, IEnumerable<Column
// NOTE: In SQLite you cannot drop columns, only add them.
}

private static string AddParameter(JObject item, Dictionary<string, object> parameters, ColumnDefinition column)
{
JToken rawValue = item.GetValue(column.Name, StringComparison.OrdinalIgnoreCase);
object value = SqlHelpers.SerializeValue(rawValue, column.StoreType, column.JsonType);
string paramName = CreateParameter(parameters, value);
return paramName;
}

private static string CreateParameter(Dictionary<string, object> parameters, object value)
{
string paramName = "@p" + parameters.Count;
parameters[paramName] = value;
return paramName;
}

/// <summary>
/// Executes a sql statement on a given table in local SQLite database.
/// </summary>
Expand Down
18 changes: 8 additions & 10 deletions ...icrosoft.WindowsAzure.MobileServices.SQLiteStore.Test/UnitTests/SQLiteStoreTests.Query.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -274,13 +274,19 @@ private static async Task<MobileServiceSQLiteStore> SetupMathTestTable(JObject[]
var store = new MobileServiceSQLiteStore(TestDbName);
store.DefineTable(MathTestTable, new JObject()
{
{ "id", String.Empty },
{ "val", 0f },
{ "expected", 0f }
});

await store.InitializeAsync();

await InsertAll(store, MathTestTable, mathTestData);
foreach (JObject item in mathTestData)
{
item[MobileServiceSystemColumns.Id] = Guid.NewGuid().ToString();
}

await store.UpsertAsync(MathTestTable, mathTestData, fromServer: false);

return store;
}
Expand Down Expand Up @@ -308,7 +314,7 @@ private static async Task<MobileServiceSQLiteStore> SetupTestTable()

if (!queryTableInitialized)
{
await InsertAll(store, TestTable, testData);
await store.UpsertAsync(TestTable, testData, fromServer: false);
}

queryTableInitialized = true;
Expand Down Expand Up @@ -349,13 +355,5 @@ private static async Task<T> Query<T>(MobileServiceSQLiteStore store, string tab
{
return (T)await store.ReadAsync(MobileServiceTableQueryDescription.Parse(tableName, query));
}

private async static Task InsertAll(MobileServiceSQLiteStore store, string tableName, JObject[] items)
{
foreach (JObject item in items)
{
await store.UpsertAsync(tableName, new[] { item }, fromServer: false);
}
}
}
}
27 changes: 24 additions & 3 deletions ...test/Microsoft.WindowsAzure.MobileServices.SQLiteStore.Test/UnitTests/SQLiteStoreTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ public class SQLiteStoreTests : TestBase
[AsyncTestMethod]
public async Task InitializeAsync_InitializesTheStore()
{
TestUtilities.DropTestTable(TestDbName, TestTable);

var store = new MobileServiceSQLiteStore(TestDbName);
store.DefineTable(TestTable, new JObject()
{
Expand Down Expand Up @@ -221,6 +223,8 @@ public void UpsertAsync_Throws_WhenStoreIsNotInitialized()
[AsyncTestMethod]
public async Task UpsertAsync_Throws_WhenColumnInItemIsNotDefinedAndItIsLocal()
{
TestUtilities.DropTestTable(TestDbName, TestTable);

using (var store = new MobileServiceSQLiteStore(TestDbName))
{
store.DefineTable(TestTable, new JObject()
Expand All @@ -240,6 +244,8 @@ public async Task UpsertAsync_Throws_WhenColumnInItemIsNotDefinedAndItIsLocal()
[AsyncTestMethod]
public async Task UpsertAsync_DoesNotThrow_WhenColumnInItemIsNotDefinedAndItIsFromServer()
{
TestUtilities.DropTestTable(TestDbName, TestTable);

using (var store = new MobileServiceSQLiteStore(TestDbName))
{
store.DefineTable(TestTable, new JObject()
Expand All @@ -250,13 +256,20 @@ public async Task UpsertAsync_DoesNotThrow_WhenColumnInItemIsNotDefinedAndItIsFr

await store.InitializeAsync();

await store.UpsertAsync(TestTable, new[] { new JObject() { { "notDefined", "okok" }, { "dob", DateTime.UtcNow } } }, fromServer: true);
await store.UpsertAsync(TestTable, new[] { new JObject()
{
{ "id", "abc" },
{ "notDefined", "okok" },
{ "dob", DateTime.UtcNow }
} }, fromServer: true);
}
}

[AsyncTestMethod]
public async Task UpsertAsync_DoesNotThrow_WhenItemIsEmpty()
{
TestUtilities.DropTestTable(TestDbName, TestTable);

using (var store = new MobileServiceSQLiteStore(TestDbName))
{
store.DefineTable(TestTable, new JObject()
Expand Down Expand Up @@ -347,6 +360,7 @@ public async Task UpsertAsync_UpdatesTheRow_WhenItExists()
await store.UpsertAsync(TestTable, new[]{new JObject()
{
{ "id", "abc" },
{ "text", "xyz" },
{ "__createdAt", DateTime.Now }
}}, fromServer: false);

Expand All @@ -355,6 +369,12 @@ await store.UpsertAsync(TestTable, new[]{new JObject()
{ "id", "abc" },
{ "__createdAt", new DateTime(200,1,1) }
}}, fromServer: false);

JObject result = await store.LookupAsync(TestTable, "abc");

Assert.AreEqual(result.Value<string>("id"), "abc");
Assert.AreEqual(result.Value<string>("text"), "xyz");
Assert.AreEqual(result.Value<string>("__createdAt"), "01/01/0200 00:00:00");
}
long count = TestUtilities.CountRows(TestDbName, TestTable);
Assert.AreEqual(count, 1L);
Expand Down Expand Up @@ -532,8 +552,9 @@ public static void DefineTestTable(MobileServiceSQLiteStore store)
{
store.DefineTable(TestTable, new JObject()
{
{"id", String.Empty },
{"__createdAt", DateTime.Now}
{ "id", String.Empty },
{ "text", String.Empty },
{ "__createdAt", DateTime.Now }
});
}
}
Expand Down

0 comments on commit c5b0b38

Please sign in to comment.