From c5b0b38b391b89b314cfeae742aff332d8c32e61 Mon Sep 17 00:00:00 2001 From: Hasan Khan Date: Thu, 6 Nov 2014 14:00:16 -0800 Subject: [PATCH] [client][managed][offline] Implement true upsert --- .../MobileServiceSQLiteStore.cs | 176 ++++++++++++------ .../UnitTests/SQLiteStoreTests.Query.cs | 18 +- .../UnitTests/SQLiteStoreTests.cs | 27 ++- 3 files changed, 156 insertions(+), 65 deletions(-) diff --git a/sdk/Managed/src/Microsoft.WindowsAzure.MobileServices.SQLiteStore/MobileServiceSQLiteStore.cs b/sdk/Managed/src/Microsoft.WindowsAzure.MobileServices.SQLiteStore/MobileServiceSQLiteStore.cs index cd4ad5e3d..e5152b7a5 100644 --- a/sdk/Managed/src/Microsoft.WindowsAzure.MobileServices.SQLiteStore/MobileServiceSQLiteStore.cs +++ b/sdk/Managed/src/Microsoft.WindowsAzure.MobileServices.SQLiteStore/MobileServiceSQLiteStore.cs @@ -25,7 +25,7 @@ public class MobileServiceSQLiteStore : MobileServiceLocalStore /// Note: The default maximum number of parameters allowed by sqlite is 999 /// See: http://www.sqlite.org/limits.html#max_variable_number /// - private const int MaxParametersPerUpsertQuery = 800; + private const int MaxParametersPerQuery = 800; private Dictionary tableMap = new Dictionary(StringComparer.OrdinalIgnoreCase); private SQLiteConnection connection; @@ -183,64 +183,17 @@ private Task UpsertAsyncInternal(string tableName, IEnumerable items, b return Task.FromResult(0); } - // Generate the prepared insert statement - string sqlBase = String.Format( - "INSERT OR REPLACE INTO {0} ({1}) VALUES ", - SqlHelpers.FormatTableName(tableName), - String.Join(", ", columns.Select(c => c.Name).Select(SqlHelpers.FormatMember)) - ); - - // Use int division to calculate how many times this record will fit into our parameter quota - int batchSize = MaxParametersPerUpsertQuery / columns.Count; - if (batchSize == 0) - { - throw new InvalidOperationException(string.Format(Properties.Resources.SQLiteStore_TooManyColumns, MaxParametersPerUpsertQuery)); - } - foreach (var batch in items.Split(maxLength: batchSize)) - { - var sql = new StringBuilder(sqlBase); - var parameters = new Dictionary(); + this.ExecuteNonQuery("BEGIN TRANSACTION", null); - foreach (JObject item in batch) - { - AppendInsertValuesSql(sql, parameters, columns, item); - sql.Append(","); - } + BatchInsert(tableName, items, columns.Where(c => c.Name.Equals(MobileServiceSystemColumns.Id)).Take(1).ToList()); + BatchUpdate(tableName, items, columns); - if (parameters.Any()) - { - sql.Remove(sql.Length - 1, 1); // remove the trailing comma - this.ExecuteNonQuery(sql.ToString(), parameters); - } - } + this.ExecuteNonQuery("COMMIT TRANSACTION", null); return Task.FromResult(0); } - private static void AppendInsertValuesSql(StringBuilder sql, Dictionary parameters, List columns, JObject item) - { - sql.Append("("); - int colCount = 0; - foreach (var column in columns) - { - if (colCount > 0) - sql.Append(","); - - JToken rawValue = item.GetValue(column.Name, StringComparison.OrdinalIgnoreCase); - object value = SqlHelpers.SerializeValue(rawValue, column.StoreType, column.JsonType); - - //The paramname for this field must be unique within this statement - string paramName = "@p" + parameters.Count; - - sql.Append(paramName); - parameters[paramName] = value; - - colCount++; - } - sql.Append(")"); - } - /// /// Deletes items from local table that match the given query. /// @@ -373,6 +326,110 @@ private void CreateAllTables() } } + private void BatchUpdate(string tableName, IEnumerable items, List columns) + { + if (columns.Count <= 1) + { + return; // For update to work there has to be at least once column besides Id that needs to be updated + } + + ValidateParameterCount(columns.Count); + + string sqlBase = String.Format("UPDATE {0} SET ", SqlHelpers.FormatTableName(tableName)); + + foreach (JObject item in items) + { + var sql = new StringBuilder(sqlBase); + var parameters = new Dictionary(); + + ColumnDefinition idColumn = columns.FirstOrDefault(c => c.Name.Equals(MobileServiceSystemColumns.Id)); + if (idColumn == null) + { + continue; + } + + foreach (var column in columns.Where(c => c != idColumn)) + { + string paramName = AddParameter(item, parameters, column); + + sql.AppendFormat("{0} = {1}", SqlHelpers.FormatMember(column.Name), paramName); + sql.Append(","); + } + + if (parameters.Any()) + { + sql.Remove(sql.Length - 1, 1); // remove the trailing comma + + } + + sql.AppendFormat(" WHERE {0} = {1}", SqlHelpers.FormatMember(MobileServiceSystemColumns.Id), AddParameter(item, parameters, idColumn)); + + this.ExecuteNonQuery(sql.ToString(), parameters); + } + } + + private void BatchInsert(string tableName, IEnumerable items, List columns) + { + if (columns.Count == 0) // we need to have some columns to insert the item + { + return; + } + + // Generate the prepared insert statement + string sqlBase = String.Format( + "INSERT OR IGNORE INTO {0} ({1}) VALUES ", + SqlHelpers.FormatTableName(tableName), + String.Join(", ", columns.Select(c => c.Name).Select(SqlHelpers.FormatMember)) + ); + + // Use int division to calculate how many times this record will fit into our parameter quota + int batchSize = ValidateParameterCount(columns.Count); + + foreach (var batch in items.Split(maxLength: batchSize)) + { + var sql = new StringBuilder(sqlBase); + var parameters = new Dictionary(); + + foreach (JObject item in batch) + { + AppendInsertValuesSql(sql, parameters, columns, item); + sql.Append(","); + } + + if (parameters.Any()) + { + sql.Remove(sql.Length - 1, 1); // remove the trailing comma + this.ExecuteNonQuery(sql.ToString(), parameters); + } + } + } + + private static int ValidateParameterCount(int parametersCount) + { + int batchSize = MaxParametersPerQuery / parametersCount; + if (batchSize == 0) + { + throw new InvalidOperationException(string.Format(Properties.Resources.SQLiteStore_TooManyColumns, MaxParametersPerQuery)); + } + return batchSize; + } + + private static void AppendInsertValuesSql(StringBuilder sql, Dictionary parameters, List columns, JObject item) + { + sql.Append("("); + int colCount = 0; + foreach (var column in columns) + { + if (colCount > 0) + sql.Append(","); + + sql.Append(AddParameter(item, parameters, column)); + + colCount++; + } + sql.Append(")"); + } + internal virtual void CreateTableFromObject(string tableName, IEnumerable columns) { ColumnDefinition idColumn = columns.FirstOrDefault(c => c.Name.Equals(MobileServiceSystemColumns.Id)); @@ -404,6 +461,21 @@ internal virtual void CreateTableFromObject(string tableName, IEnumerable parameters, ColumnDefinition column) + { + JToken rawValue = item.GetValue(column.Name, StringComparison.OrdinalIgnoreCase); + object value = SqlHelpers.SerializeValue(rawValue, column.StoreType, column.JsonType); + string paramName = CreateParameter(parameters, value); + return paramName; + } + + private static string CreateParameter(Dictionary parameters, object value) + { + string paramName = "@p" + parameters.Count; + parameters[paramName] = value; + return paramName; + } + /// /// Executes a sql statement on a given table in local SQLite database. /// diff --git a/sdk/Managed/test/Microsoft.WindowsAzure.MobileServices.SQLiteStore.Test/UnitTests/SQLiteStoreTests.Query.cs b/sdk/Managed/test/Microsoft.WindowsAzure.MobileServices.SQLiteStore.Test/UnitTests/SQLiteStoreTests.Query.cs index 99ed834e9..7ed670985 100644 --- a/sdk/Managed/test/Microsoft.WindowsAzure.MobileServices.SQLiteStore.Test/UnitTests/SQLiteStoreTests.Query.cs +++ b/sdk/Managed/test/Microsoft.WindowsAzure.MobileServices.SQLiteStore.Test/UnitTests/SQLiteStoreTests.Query.cs @@ -274,13 +274,19 @@ private static async Task SetupMathTestTable(JObject[] var store = new MobileServiceSQLiteStore(TestDbName); store.DefineTable(MathTestTable, new JObject() { + { "id", String.Empty }, { "val", 0f }, { "expected", 0f } }); await store.InitializeAsync(); - await InsertAll(store, MathTestTable, mathTestData); + foreach (JObject item in mathTestData) + { + item[MobileServiceSystemColumns.Id] = Guid.NewGuid().ToString(); + } + + await store.UpsertAsync(MathTestTable, mathTestData, fromServer: false); return store; } @@ -308,7 +314,7 @@ private static async Task SetupTestTable() if (!queryTableInitialized) { - await InsertAll(store, TestTable, testData); + await store.UpsertAsync(TestTable, testData, fromServer: false); } queryTableInitialized = true; @@ -349,13 +355,5 @@ private static async Task Query(MobileServiceSQLiteStore store, string tab { return (T)await store.ReadAsync(MobileServiceTableQueryDescription.Parse(tableName, query)); } - - private async static Task InsertAll(MobileServiceSQLiteStore store, string tableName, JObject[] items) - { - foreach (JObject item in items) - { - await store.UpsertAsync(tableName, new[] { item }, fromServer: false); - } - } } } diff --git a/sdk/Managed/test/Microsoft.WindowsAzure.MobileServices.SQLiteStore.Test/UnitTests/SQLiteStoreTests.cs b/sdk/Managed/test/Microsoft.WindowsAzure.MobileServices.SQLiteStore.Test/UnitTests/SQLiteStoreTests.cs index c4a920b54..966a59b2d 100644 --- a/sdk/Managed/test/Microsoft.WindowsAzure.MobileServices.SQLiteStore.Test/UnitTests/SQLiteStoreTests.cs +++ b/sdk/Managed/test/Microsoft.WindowsAzure.MobileServices.SQLiteStore.Test/UnitTests/SQLiteStoreTests.cs @@ -24,6 +24,8 @@ public class SQLiteStoreTests : TestBase [AsyncTestMethod] public async Task InitializeAsync_InitializesTheStore() { + TestUtilities.DropTestTable(TestDbName, TestTable); + var store = new MobileServiceSQLiteStore(TestDbName); store.DefineTable(TestTable, new JObject() { @@ -221,6 +223,8 @@ public void UpsertAsync_Throws_WhenStoreIsNotInitialized() [AsyncTestMethod] public async Task UpsertAsync_Throws_WhenColumnInItemIsNotDefinedAndItIsLocal() { + TestUtilities.DropTestTable(TestDbName, TestTable); + using (var store = new MobileServiceSQLiteStore(TestDbName)) { store.DefineTable(TestTable, new JObject() @@ -240,6 +244,8 @@ public async Task UpsertAsync_Throws_WhenColumnInItemIsNotDefinedAndItIsLocal() [AsyncTestMethod] public async Task UpsertAsync_DoesNotThrow_WhenColumnInItemIsNotDefinedAndItIsFromServer() { + TestUtilities.DropTestTable(TestDbName, TestTable); + using (var store = new MobileServiceSQLiteStore(TestDbName)) { store.DefineTable(TestTable, new JObject() @@ -250,13 +256,20 @@ public async Task UpsertAsync_DoesNotThrow_WhenColumnInItemIsNotDefinedAndItIsFr await store.InitializeAsync(); - await store.UpsertAsync(TestTable, new[] { new JObject() { { "notDefined", "okok" }, { "dob", DateTime.UtcNow } } }, fromServer: true); + await store.UpsertAsync(TestTable, new[] { new JObject() + { + { "id", "abc" }, + { "notDefined", "okok" }, + { "dob", DateTime.UtcNow } + } }, fromServer: true); } } [AsyncTestMethod] public async Task UpsertAsync_DoesNotThrow_WhenItemIsEmpty() { + TestUtilities.DropTestTable(TestDbName, TestTable); + using (var store = new MobileServiceSQLiteStore(TestDbName)) { store.DefineTable(TestTable, new JObject() @@ -347,6 +360,7 @@ public async Task UpsertAsync_UpdatesTheRow_WhenItExists() await store.UpsertAsync(TestTable, new[]{new JObject() { { "id", "abc" }, + { "text", "xyz" }, { "__createdAt", DateTime.Now } }}, fromServer: false); @@ -355,6 +369,12 @@ await store.UpsertAsync(TestTable, new[]{new JObject() { "id", "abc" }, { "__createdAt", new DateTime(200,1,1) } }}, fromServer: false); + + JObject result = await store.LookupAsync(TestTable, "abc"); + + Assert.AreEqual(result.Value("id"), "abc"); + Assert.AreEqual(result.Value("text"), "xyz"); + Assert.AreEqual(result.Value("__createdAt"), "01/01/0200 00:00:00"); } long count = TestUtilities.CountRows(TestDbName, TestTable); Assert.AreEqual(count, 1L); @@ -532,8 +552,9 @@ public static void DefineTestTable(MobileServiceSQLiteStore store) { store.DefineTable(TestTable, new JObject() { - {"id", String.Empty }, - {"__createdAt", DateTime.Now} + { "id", String.Empty }, + { "text", String.Empty }, + { "__createdAt", DateTime.Now } }); } }