From ad00dc4879c72dea85595caf19095c5c1de77e45 Mon Sep 17 00:00:00 2001
From: Ricardo Pereira <ricardopereira@whitesmith.co>
Date: Wed, 29 Aug 2018 10:15:13 +0100
Subject: [PATCH 01/19] RSL1k

---
 Spec/RestClientChannel.swift | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Spec/RestClientChannel.swift b/Spec/RestClientChannel.swift
index 2e6d407a1..86907a0c1 100644
--- a/Spec/RestClientChannel.swift
+++ b/Spec/RestClientChannel.swift
@@ -365,6 +365,11 @@ class RestClientChannel: QuickSpec {
                     }
                 }
             }
+
+            // RSL1k
+            context("idempotent publishing") {
+                
+            }
         }
 
         // RSL2

From 3f57d0956e966885fb88b087388f95e1e0b35c6a Mon Sep 17 00:00:00 2001
From: Ricardo Pereira <ricardopereira@whitesmith.co>
Date: Wed, 29 Aug 2018 10:43:00 +0100
Subject: [PATCH 02/19] TO3n

---
 Source/ARTClientOptions.h    | 9 +++++++++
 Spec/RestClientChannel.swift | 9 ++++++++-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/Source/ARTClientOptions.h b/Source/ARTClientOptions.h
index 447d69c64..10e34a34b 100644
--- a/Source/ARTClientOptions.h
+++ b/Source/ARTClientOptions.h
@@ -109,6 +109,15 @@ NS_ASSUME_NONNULL_BEGIN
  */
 @property (readwrite, strong, nonatomic) dispatch_queue_t internalDispatchQueue;
 
+/**
+ True when idempotent publishing is enabled for all messages published via REST.
+
+ When this feature is enabled, the client library will add a unique ID to every published message (without an ID) ensuring any failed published attempts (due to failures such as HTTP requests failing mid-flight) that are automatically retried will not result in duplicate messages being published to the Ably platform.
+
+ Note: This is a beta unsupported feature!
+ */
+@property (readwrite, assign, nonatomic) BOOL idempotentRestPublishing;
+
 - (BOOL)isBasicAuth;
 - (NSURL *)restUrl;
 - (NSURL *)realtimeUrl;
diff --git a/Spec/RestClientChannel.swift b/Spec/RestClientChannel.swift
index 86907a0c1..27e0bdcee 100644
--- a/Spec/RestClientChannel.swift
+++ b/Spec/RestClientChannel.swift
@@ -368,7 +368,14 @@ class RestClientChannel: QuickSpec {
 
             // RSL1k
             context("idempotent publishing") {
-                
+
+                // TO3n
+                it("idempotentRestPublishing option") {
+                    // Current version
+                    let options = AblyTests.clientOptions()
+                    expect(options.idempotentRestPublishing) == true
+                }
+
             }
         }
 

From ba36018d1bf070a55ef54076f322070f524d8751 Mon Sep 17 00:00:00 2001
From: Ricardo Pereira <ricardopereira@whitesmith.co>
Date: Wed, 29 Aug 2018 10:43:24 +0100
Subject: [PATCH 03/19] Implement TO3n

---
 Source/ARTClientOptions+Private.h |  1 +
 Source/ARTClientOptions.m         | 10 ++++++++++
 Spec/RestClientChannel.swift      | 10 ++++++++++
 3 files changed, 21 insertions(+)

diff --git a/Source/ARTClientOptions+Private.h b/Source/ARTClientOptions+Private.h
index dc5873c60..a4b186a70 100644
--- a/Source/ARTClientOptions+Private.h
+++ b/Source/ARTClientOptions+Private.h
@@ -11,6 +11,7 @@
 @interface ARTClientOptions ()
 
 + (void)setDefaultEnvironment:(NSString *_Nullable)environment;
++ (BOOL)getDefaultIdempotentRestPublishingForVersion:(NSString *_Nonnull)version;
 - (NSURLComponents *_Nonnull)restUrlComponents;
 
 @end
diff --git a/Source/ARTClientOptions.m b/Source/ARTClientOptions.m
index 54fce868e..032721e72 100644
--- a/Source/ARTClientOptions.m
+++ b/Source/ARTClientOptions.m
@@ -49,6 +49,7 @@ - (instancetype)initDefaults {
     _dispatchQueue = dispatch_get_main_queue();
     _internalDispatchQueue = dispatch_queue_create("io.ably.main", DISPATCH_QUEUE_SERIAL);
     _pushFullWait = false;
+    _idempotentRestPublishing = [ARTClientOptions getDefaultIdempotentRestPublishingForVersion:[ARTDefault version]];
     return self;
 }
 
@@ -179,4 +180,13 @@ - (void)setDefaultTokenParams:(ARTTokenParams *)value {
     _defaultTokenParams = [[ARTTokenParams alloc] initWithTokenParams:value];
 }
 
++ (BOOL)getDefaultIdempotentRestPublishingForVersion:(NSString *)version {
+    if ([@"1.1" compare:version options:NSNumericSearch] == NSOrderedDescending) {
+        return false;
+    }
+    else {
+        return true;
+    }
+}
+
 @end
diff --git a/Spec/RestClientChannel.swift b/Spec/RestClientChannel.swift
index 27e0bdcee..b85f01871 100644
--- a/Spec/RestClientChannel.swift
+++ b/Spec/RestClientChannel.swift
@@ -371,6 +371,16 @@ class RestClientChannel: QuickSpec {
 
                 // TO3n
                 it("idempotentRestPublishing option") {
+                    expect(ARTClientOptions.getDefaultIdempotentRestPublishing(forVersion: "2")) == true
+                    expect(ARTClientOptions.getDefaultIdempotentRestPublishing(forVersion: "2.0.0")) == true
+                    expect(ARTClientOptions.getDefaultIdempotentRestPublishing(forVersion: "1.1")) == true
+                    expect(ARTClientOptions.getDefaultIdempotentRestPublishing(forVersion: "1.1.2")) == true
+                    expect(ARTClientOptions.getDefaultIdempotentRestPublishing(forVersion: "1.2")) == true
+                    expect(ARTClientOptions.getDefaultIdempotentRestPublishing(forVersion: "1.0")) == false
+                    expect(ARTClientOptions.getDefaultIdempotentRestPublishing(forVersion: "1.0.5")) == false
+                    expect(ARTClientOptions.getDefaultIdempotentRestPublishing(forVersion: "0.9")) == false
+                    expect(ARTClientOptions.getDefaultIdempotentRestPublishing(forVersion: "0.9.1")) == false
+
                     // Current version
                     let options = AblyTests.clientOptions()
                     expect(options.idempotentRestPublishing) == true

From bbcadac1afd2f5ef2800a682b1d2f3d1d93fa64b Mon Sep 17 00:00:00 2001
From: Ricardo Pereira <ricardopereira@whitesmith.co>
Date: Fri, 24 Aug 2018 15:27:55 +0100
Subject: [PATCH 04/19] Test suite: improve msgpackToJSON method

---
 Spec/RestClient.swift        |  2 +-
 Spec/RestClientChannel.swift | 14 +++++++-------
 Spec/TestUtilities.swift     |  4 ++--
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/Spec/RestClient.swift b/Spec/RestClient.swift
index a19074c37..92298e0f2 100644
--- a/Spec/RestClient.swift
+++ b/Spec/RestClient.swift
@@ -1252,7 +1252,7 @@ class RestClient: QuickSpec {
                 }
 
                 let transport = realtime.transport as! TestProxyTransport
-                let object = AblyTests.msgpackToJSON(transport.rawDataSent.last! as NSData)
+                let object = AblyTests.msgpackToJSON(transport.rawDataSent.last!)
                 expect(object["messages"][0]["data"].string).to(equal("message"))
             }
 
diff --git a/Spec/RestClientChannel.swift b/Spec/RestClientChannel.swift
index b85f01871..e212038e3 100644
--- a/Spec/RestClientChannel.swift
+++ b/Spec/RestClientChannel.swift
@@ -730,7 +730,7 @@ class RestClientChannel: QuickSpec {
                                 XCTFail("HTTPBody is nil");
                                 done(); return
                             }
-                            var json = AblyTests.msgpackToJSON(httpBody as NSData)
+                            var json = AblyTests.msgpackToJSON(httpBody)
                             if let s = json["data"].string, let data = try? JSONSerialization.jsonObject(with: s.data(using: .utf8)!) {
                                 // Make sure the formatting is the same by parsing
                                 // and reformatting in the same way as the test
@@ -772,7 +772,7 @@ class RestClientChannel: QuickSpec {
                                 XCTFail("HTTPBody is nil");
                                 done(); return
                             }
-                            expect(AblyTests.msgpackToJSON(httpBody as NSData)["encoding"]).to(equal(caseItem.expected))
+                            expect(AblyTests.msgpackToJSON(httpBody)["encoding"]).to(equal(caseItem.expected))
                             done()
                         })
                     }
@@ -791,7 +791,7 @@ class RestClientChannel: QuickSpec {
                                 done(); return
                             }
                             // Binary
-                            let json = AblyTests.msgpackToJSON(httpBody as NSData)
+                            let json = AblyTests.msgpackToJSON(httpBody)
                             expect(json["data"].string).to(equal(binaryData.toBase64))
                             expect(json["encoding"]).to(equal("base64"))
                             done()
@@ -808,7 +808,7 @@ class RestClientChannel: QuickSpec {
 
                             if let request = testHTTPExecutor.requests.last, let http = request.httpBody {
                                 // String (UTF-8)
-                                let json = AblyTests.msgpackToJSON(http as NSData)
+                                let json = AblyTests.msgpackToJSON(http)
                                 expect(json["data"].string).to(equal(text))
                                 expect(json["encoding"].string).to(beNil())
                             }
@@ -832,7 +832,7 @@ class RestClientChannel: QuickSpec {
 
                                 if let request = testHTTPExecutor.requests.last, let http = request.httpBody {
                                     // Array
-                                    let json = AblyTests.msgpackToJSON(http as NSData)
+                                    let json = AblyTests.msgpackToJSON(http)
                                     expect(JSON(data: json["data"].stringValue.data(using: String.Encoding.utf8)!).asArray).to(equal(array as NSArray?))
                                     expect(json["encoding"].string).to(equal("json"))
                                 }
@@ -853,7 +853,7 @@ class RestClientChannel: QuickSpec {
 
                                 if let request = testHTTPExecutor.requests.last, let http = request.httpBody {
                                     // Dictionary
-                                    let json = AblyTests.msgpackToJSON(http as NSData)
+                                    let json = AblyTests.msgpackToJSON(http)
                                     expect(JSON(data: json["data"].stringValue.data(using: String.Encoding.utf8)!).asDictionary).to(equal(dictionary as NSDictionary?))
                                     expect(json["encoding"].string).to(equal("json"))
                                 }
@@ -949,7 +949,7 @@ class RestClientChannel: QuickSpec {
                             fail("HTTPBody is empty")
                             return
                         }
-                        let httpBodyAsJSON = AblyTests.msgpackToJSON(httpBody as NSData)
+                        let httpBodyAsJSON = AblyTests.msgpackToJSON(httpBody)
                         expect(httpBodyAsJSON["encoding"].string).to(equal("utf-8/cipher+aes-\(encryptionKeyLength)-cbc/base64"))
                         expect(httpBodyAsJSON["name"].string).to(equal("test"))
                         expect(httpBodyAsJSON["data"].string).toNot(equal("message1"))
diff --git a/Spec/TestUtilities.swift b/Spec/TestUtilities.swift
index 2d7494579..22b1900c3 100644
--- a/Spec/TestUtilities.swift
+++ b/Spec/TestUtilities.swift
@@ -52,8 +52,8 @@ class AblyTests {
         return Data(base64Encoded: base64, options: NSData.Base64DecodingOptions(rawValue: 0))!
     }
 
-    class func msgpackToJSON(_ data: NSData) -> JSON {
-        let decoded = try! ARTMsgPackEncoder().decode(data as Data)
+    class func msgpackToJSON(_ data: Data) -> JSON {
+        let decoded = try! ARTMsgPackEncoder().decode(data)
         let encoded = try! ARTJsonEncoder().encode(decoded)
         return JSON(data: encoded)
     }

From 26a46d66a585e56a9a41ce491274a070c0de9e93 Mon Sep 17 00:00:00 2001
From: Ricardo Pereira <ricardopereira@whitesmith.co>
Date: Wed, 29 Aug 2018 12:18:27 +0100
Subject: [PATCH 05/19] RSL1k1

---
 Spec/RestClientChannel.swift | 83 ++++++++++++++++++++++++++++++++++++
 1 file changed, 83 insertions(+)

diff --git a/Spec/RestClientChannel.swift b/Spec/RestClientChannel.swift
index e212038e3..b45be97da 100644
--- a/Spec/RestClientChannel.swift
+++ b/Spec/RestClientChannel.swift
@@ -386,6 +386,89 @@ class RestClientChannel: QuickSpec {
                     expect(options.idempotentRestPublishing) == true
                 }
 
+                func assertMessagePayloadId(id: String?, expectedSerial: String) {
+                    guard let id = id else {
+                        fail("Message.id from payload is nil"); return
+                    }
+
+                    let idParts = id.split(separator: ":")
+
+                    if idParts.count != 2 {
+                        fail("Message.id from payload should have baseId and serial separated by a colon"); return
+                    }
+
+                    let baseId = String(idParts[0])
+                    let serial = String(idParts[1])
+
+                    guard let baseIdData = Data(base64Encoded: baseId) else {
+                        fail("BaseId should be a base64 encoded string"); return
+                    }
+
+                    expect(baseIdData.bytes.count) == 9
+                    expect(serial).to(equal(expectedSerial))
+                }
+
+                // RSL1k1
+                context("random idempotent publish id") {
+
+                    it("should generate for one message with empty id") {
+                        let message = ARTMessage(name: nil, data: "foo")
+                        expect(message.id).to(beNil())
+
+                        let rest = ARTRest(key: "xxxx:xxxx")
+                        expect(rest.options.idempotentRestPublishing) == true
+                        let mockHTTPExecutor = MockHTTPExecutor()
+                        rest.httpExecutor = mockHTTPExecutor
+                        let channel = rest.channels.get("idempotent")
+
+                        waitUntil(timeout: testTimeout) { done in
+                            channel.publish([message]) { error in
+                                expect(error).to(beNil())
+                                done()
+                            }
+                        }
+
+                        guard let encodedBody = mockHTTPExecutor.requests.last?.httpBody else {
+                            fail("Body from the last request is empty"); return
+                        }
+
+                        let json = AblyTests.msgpackToJSON(encodedBody)
+                        assertMessagePayloadId(id: json.arrayValue.first?["id"].string, expectedSerial: "0")
+                        expect(message.id).to(beNil())
+                    }
+
+                    it("should generate for multiple messages with empty id") {
+                        let message1 = ARTMessage(name: nil, data: "foo1")
+                        expect(message1.id).to(beNil())
+                        let message2 = ARTMessage(name: "john", data: "foo2")
+                        expect(message2.id).to(beNil())
+
+                        let rest = ARTRest(key: "xxxx:xxxx")
+                        let mockHTTPExecutor = MockHTTPExecutor()
+                        rest.httpExecutor = mockHTTPExecutor
+                        let channel = rest.channels.get("idempotent")
+
+                        waitUntil(timeout: testTimeout) { done in
+                            channel.publish([message1, message2]) { error in
+                                expect(error).to(beNil())
+                                done()
+                            }
+                        }
+
+                        guard let encodedBody = mockHTTPExecutor.requests.last?.httpBody else {
+                            fail("Body from the last request is empty"); return
+                        }
+
+                        let json = AblyTests.msgpackToJSON(encodedBody)
+                        let id1 = json.arrayValue.first?["id"].string
+                        assertMessagePayloadId(id: id1, expectedSerial: "0")
+                        let id2 = json.arrayValue.last?["id"].string
+                        assertMessagePayloadId(id: id2, expectedSerial: "1")
+
+                        // Same Base ID
+                        expect(id1?.split(separator: ":").first).to(equal(id2?.split(separator: ":").first))
+                    }
+                }
             }
         }
 

From cca8e5a2215a7db818f821ebfd94acf7521e8915 Mon Sep 17 00:00:00 2001
From: Ricardo Pereira <ricardopereira@whitesmith.co>
Date: Wed, 29 Aug 2018 15:32:51 +0100
Subject: [PATCH 06/19] Implement idempotent publish id

---
 Source/ARTBaseMessage+Private.h |  2 ++
 Source/ARTBaseMessage.m         |  4 ++++
 Source/ARTJsonLikeEncoder.m     |  5 ++++-
 Source/ARTRestChannel.m         | 33 +++++++++++++++++++++++++++++++--
 4 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/Source/ARTBaseMessage+Private.h b/Source/ARTBaseMessage+Private.h
index 5d34f1a10..8cbfd7e19 100644
--- a/Source/ARTBaseMessage+Private.h
+++ b/Source/ARTBaseMessage+Private.h
@@ -15,6 +15,8 @@ ART_ASSUME_NONNULL_BEGIN
 
 @interface ARTBaseMessage ()
 
+@property (nonatomic, assign, readonly) BOOL isIdEmpty;
+
 - (id __art_nonnull)decodeWithEncoder:(ARTDataEncoder*)encoder error:(NSError *__art_nullable*__art_nullable)error;
 - (id __art_nonnull)encodeWithEncoder:(ARTDataEncoder*)encoder error:(NSError *__art_nullable*__art_nullable)error;
 
diff --git a/Source/ARTBaseMessage.m b/Source/ARTBaseMessage.m
index d309e2d19..d17e74f1b 100644
--- a/Source/ARTBaseMessage.m
+++ b/Source/ARTBaseMessage.m
@@ -93,4 +93,8 @@ - (NSInteger)messageSize {
     return finalResult;
 }
 
+- (BOOL)isIdEmpty {
+    return self.id == nil || [self.id isEqualToString:@""];
+}
+
 @end
diff --git a/Source/ARTJsonLikeEncoder.m b/Source/ARTJsonLikeEncoder.m
index d017e6b22..c5012940e 100644
--- a/Source/ARTJsonLikeEncoder.m
+++ b/Source/ARTJsonLikeEncoder.m
@@ -373,7 +373,10 @@ - (NSArray *)presenceMessagesFromArray:(NSArray *)input {
 
 - (NSDictionary *)messageToDictionary:(ARTMessage *)message {
     NSMutableDictionary *output = [NSMutableDictionary dictionary];
-    
+    if (message.id) {
+        [output setObject:message.id forKey:@"id"];
+    }
+
     if (message.timestamp) {
         [output setObject:[message.timestamp artToNumberMs] forKey:@"timestamp"];
     }
diff --git a/Source/ARTRestChannel.m b/Source/ARTRestChannel.m
index de7c48269..c87704a73 100644
--- a/Source/ARTRestChannel.m
+++ b/Source/ARTRestChannel.m
@@ -21,6 +21,9 @@
 #import "ARTTokenDetails.h"
 #import "ARTNSArray+ARTFunctional.h"
 #import "ARTPushChannel.h"
+#import "ARTCrypto+Private.h"
+
+static const NSUInteger kIdempotentLibraryGeneratedIdLength = 9; //bytes
 
 @implementation ARTRestChannel {
 @private
@@ -165,6 +168,14 @@ - (void)internalPostMessages:(id)data callback:(void (^)(ARTErrorInfo *__art_nul
     
     if ([data isKindOfClass:[ARTMessage class]]) {
         ARTMessage *message = (ARTMessage *)data;
+
+        NSString *baseId = nil;
+        if (self.rest.options.idempotentRestPublishing && message.isIdEmpty) {
+            NSData *baseIdData = [ARTCrypto generateSecureRandomData:kIdempotentLibraryGeneratedIdLength];
+            baseId = [baseIdData base64EncodedStringWithOptions:0];
+            message.id = [NSString stringWithFormat:@"%@:0", baseId];
+        }
+
         if (message.clientId && self.rest.auth.clientId_nosync && ![message.clientId isEqualToString:self.rest.auth.clientId_nosync]) {
             callback([ARTErrorInfo createWithCode:ARTStateMismatchedClientId message:@"attempted to publish message with an invalid clientId"]);
             return;
@@ -172,20 +183,38 @@ - (void)internalPostMessages:(id)data callback:(void (^)(ARTErrorInfo *__art_nul
         else {
             message.clientId = self.rest.auth.clientId_nosync;
         }
+
         NSError *encodeError = nil;
         encodedMessage = [self.rest.defaultEncoder encodeMessage:message error:&encodeError];
         if (encodeError) {
             callback([ARTErrorInfo createFromNSError:encodeError]);
             return;
         }
-    } else if ([data isKindOfClass:[NSArray class]]) {
-        __GENERIC(NSArray, ARTMessage *) *messages = (NSArray *)data;
+    }
+    else if ([data isKindOfClass:[NSArray class]]) {
+        NSArray<ARTMessage *> *messages = (NSArray *)data;
+
+        NSString *baseId = nil;
+        if (self.rest.options.idempotentRestPublishing) {
+            BOOL messagesHaveEmptyId = [messages artFilter:^BOOL(ARTMessage *m) { return !m.isIdEmpty; }].count <= 0;
+            if (messagesHaveEmptyId) {
+                NSData *baseIdData = [ARTCrypto generateSecureRandomData:kIdempotentLibraryGeneratedIdLength];
+                baseId = [baseIdData base64EncodedStringWithOptions:0];
+            }
+        }
+
+        NSInteger serial = 0;
         for (ARTMessage *message in messages) {
             if (message.clientId && self.rest.auth.clientId_nosync && ![message.clientId isEqualToString:self.rest.auth.clientId_nosync]) {
                 callback([ARTErrorInfo createWithCode:ARTStateMismatchedClientId message:@"attempted to publish message with an invalid clientId"]);
                 return;
             }
+            if (baseId) {
+                message.id = [NSString stringWithFormat:@"%@:%ld", baseId, (long)serial];
+            }
+            serial += 1;
         }
+
         NSError *encodeError = nil;
         encodedMessage = [self.rest.defaultEncoder encodeMessages:data error:&encodeError];
         if (encodeError) {

From accd6251b81ad1e0e863038bee01f7811a6174d3 Mon Sep 17 00:00:00 2001
From: Ricardo Pereira <ricardopereira@whitesmith.co>
Date: Wed, 29 Aug 2018 15:44:04 +0100
Subject: [PATCH 07/19] RSL1k2

---
 Spec/RestClientChannel.swift | 46 ++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/Spec/RestClientChannel.swift b/Spec/RestClientChannel.swift
index b45be97da..9bb748447 100644
--- a/Spec/RestClientChannel.swift
+++ b/Spec/RestClientChannel.swift
@@ -469,6 +469,52 @@ class RestClientChannel: QuickSpec {
                         expect(id1?.split(separator: ":").first).to(equal(id2?.split(separator: ":").first))
                     }
                 }
+
+                // RSL1k2
+                it("should not generate for message with a non empty id") {
+                    let message = ARTMessage(name: nil, data: "foo")
+                    message.id = "123"
+
+                    let rest = ARTRest(key: "xxxx:xxxx")
+                    let mockHTTPExecutor = MockHTTPExecutor()
+                    rest.httpExecutor = mockHTTPExecutor
+                    let channel = rest.channels.get("idempotent")
+
+                    waitUntil(timeout: testTimeout) { done in
+                        channel.publish([message]) { error in
+                            expect(error).to(beNil())
+                            done()
+                        }
+                    }
+
+                    guard let encodedBody = mockHTTPExecutor.requests.last?.httpBody else {
+                        fail("Body from the last request is empty"); return
+                    }
+
+                    let json = AblyTests.msgpackToJSON(encodedBody)
+                    expect(json.arrayValue.first?["id"].string).to(equal("123"))
+                }
+
+                it("should generate for internal message that is created in publish(name:data:) method") {
+                    let rest = ARTRest(key: "xxxx:xxxx")
+                    let mockHTTPExecutor = MockHTTPExecutor()
+                    rest.httpExecutor = mockHTTPExecutor
+                    let channel = rest.channels.get("idempotent")
+
+                    waitUntil(timeout: testTimeout) { done in
+                        channel.publish("john", data: "foo") { error in
+                            expect(error).to(beNil())
+                            done()
+                        }
+                    }
+
+                    guard let encodedBody = mockHTTPExecutor.requests.last?.httpBody else {
+                        fail("Body from the last request is empty"); return
+                    }
+
+                    let json = AblyTests.msgpackToJSON(encodedBody)
+                    assertMessagePayloadId(id: json["id"].string, expectedSerial: "0")
+                }
             }
         }
 

From 97feaafc685f2e80dae9b5e650970dab190c8972 Mon Sep 17 00:00:00 2001
From: Ricardo Pereira <ricardopereira@whitesmith.co>
Date: Wed, 29 Aug 2018 15:44:21 +0100
Subject: [PATCH 08/19] RSL1k3

---
 Spec/RestClientChannel.swift | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/Spec/RestClientChannel.swift b/Spec/RestClientChannel.swift
index 9bb748447..684766763 100644
--- a/Spec/RestClientChannel.swift
+++ b/Spec/RestClientChannel.swift
@@ -515,6 +515,34 @@ class RestClientChannel: QuickSpec {
                     let json = AblyTests.msgpackToJSON(encodedBody)
                     assertMessagePayloadId(id: json["id"].string, expectedSerial: "0")
                 }
+
+                // RSL1k3
+                it("should not generate for multiple messages with a non empty id") {
+                    let message1 = ARTMessage(name: nil, data: "foo1")
+                    expect(message1.id).to(beNil())
+                    let message2 = ARTMessage(name: "john", data: "foo2")
+                    message2.id = "123"
+
+                    let rest = ARTRest(key: "xxxx:xxxx")
+                    let mockHTTPExecutor = MockHTTPExecutor()
+                    rest.httpExecutor = mockHTTPExecutor
+                    let channel = rest.channels.get("idempotent")
+
+                    waitUntil(timeout: testTimeout) { done in
+                        channel.publish([message1, message2]) { error in
+                            expect(error).to(beNil())
+                            done()
+                        }
+                    }
+
+                    guard let encodedBody = mockHTTPExecutor.requests.last?.httpBody else {
+                        fail("Body from the last request is empty"); return
+                    }
+
+                    let json = AblyTests.msgpackToJSON(encodedBody)
+                    expect(json.arrayValue.first?["id"].string).to(beNil())
+                    expect(json.arrayValue.last?["id"].string).to(equal("123"))
+                }
             }
         }
 

From ccaa6418bcf05964341e7789dc7bde2a6448fc34 Mon Sep 17 00:00:00 2001
From: Ricardo Pereira <ricardopereira@whitesmith.co>
Date: Wed, 29 Aug 2018 16:24:24 +0100
Subject: [PATCH 09/19] [fix] should not generate when idempotentRestPublishing
 flag is off

---
 Source/ARTClientOptions.m    |  1 +
 Spec/RestClientChannel.swift | 30 ++++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+)

diff --git a/Source/ARTClientOptions.m b/Source/ARTClientOptions.m
index 032721e72..b646f0adc 100644
--- a/Source/ARTClientOptions.m
+++ b/Source/ARTClientOptions.m
@@ -128,6 +128,7 @@ - (id)copyWithZone:(NSZone *)zone {
     options.dispatchQueue = self.dispatchQueue;
     options.internalDispatchQueue = self.internalDispatchQueue;
     options.pushFullWait = self.pushFullWait;
+    options.idempotentRestPublishing = self.idempotentRestPublishing;
 
     return options;
 }
diff --git a/Spec/RestClientChannel.swift b/Spec/RestClientChannel.swift
index 684766763..2bfaec623 100644
--- a/Spec/RestClientChannel.swift
+++ b/Spec/RestClientChannel.swift
@@ -543,6 +543,36 @@ class RestClientChannel: QuickSpec {
                     expect(json.arrayValue.first?["id"].string).to(beNil())
                     expect(json.arrayValue.last?["id"].string).to(equal("123"))
                 }
+
+                it("should not generate when idempotentRestPublishing flag is off") {
+                    let options = ARTClientOptions(key: "xxxx:xxxx")
+                    options.idempotentRestPublishing = false
+
+                    let message1 = ARTMessage(name: nil, data: "foo1")
+                    expect(message1.id).to(beNil())
+                    let message2 = ARTMessage(name: "john", data: "foo2")
+                    expect(message2.id).to(beNil())
+
+                    let rest = ARTRest(options: options)
+                    let mockHTTPExecutor = MockHTTPExecutor()
+                    rest.httpExecutor = mockHTTPExecutor
+                    let channel = rest.channels.get("idempotent")
+
+                    waitUntil(timeout: testTimeout) { done in
+                        channel.publish([message1, message2]) { error in
+                            expect(error).to(beNil())
+                            done()
+                        }
+                    }
+
+                    guard let encodedBody = mockHTTPExecutor.requests.last?.httpBody else {
+                        fail("Body from the last request is empty"); return
+                    }
+
+                    let json = AblyTests.msgpackToJSON(encodedBody)
+                    expect(json.arrayValue.first?["id"].string).to(beNil())
+                    expect(json.arrayValue.last?["id"].string).to(beNil())
+                }
             }
         }
 

From da5fb6393ab5020c2612c127421c505399313e25 Mon Sep 17 00:00:00 2001
From: Ricardo Pereira <ricardopereira@whitesmith.co>
Date: Wed, 29 Aug 2018 19:15:59 +0100
Subject: [PATCH 10/19] RSL1k4

---
 Spec/RestClientChannel.swift | 42 ++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/Spec/RestClientChannel.swift b/Spec/RestClientChannel.swift
index 2bfaec623..86d3d0683 100644
--- a/Spec/RestClientChannel.swift
+++ b/Spec/RestClientChannel.swift
@@ -573,6 +573,48 @@ class RestClientChannel: QuickSpec {
                     expect(json.arrayValue.first?["id"].string).to(beNil())
                     expect(json.arrayValue.last?["id"].string).to(beNil())
                 }
+
+                // RSL1k4
+                it("should have only one published message") {
+                    client.httpExecutor = testHTTPExecutor
+                    client.options.fallbackHostsUseDefault = true
+
+                    let forceRetryError = ErrorSimulator(
+                        value: 50000,
+                        description: "force retry",
+                        statusCode: 500,
+                        shouldPerformRequest: true,
+                        stubData: nil
+                    )
+
+                    testHTTPExecutor.simulateIncomingServerErrorOnNextRequest(forceRetryError)
+
+                    let messages = [
+                        ARTMessage(name: nil, data: "test1"),
+                        ARTMessage(name: nil, data: "test2"),
+                        ARTMessage(name: nil, data: "test3"),
+                    ]
+
+                    waitUntil(timeout: testTimeout) { done in
+                        channel.publish(messages) { error in
+                            expect(error).toNot(beNil())
+                            done()
+                        }
+                    }
+
+                    expect(testHTTPExecutor.requests.count) == 2
+
+                    waitUntil(timeout: testTimeout) { done in
+                        channel.history { result, error in
+                            expect(error).to(beNil())
+                            guard let result = result else {
+                                fail("No result"); done(); return
+                            }
+                            expect(result.items.count) == 3
+                            done()
+                        }
+                    }
+                }
             }
         }
 

From 666a32b53571f1bd1f43af6b5b9910a926e11334 Mon Sep 17 00:00:00 2001
From: Ricardo Pereira <ricardopereira@whitesmith.co>
Date: Wed, 29 Aug 2018 19:17:04 +0100
Subject: [PATCH 11/19] TestSuite: ErrorSimulator option to call the original
 request

---
 Spec/TestUtilities.swift | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/Spec/TestUtilities.swift b/Spec/TestUtilities.swift
index 22b1900c3..36aa0e5d5 100644
--- a/Spec/TestUtilities.swift
+++ b/Spec/TestUtilities.swift
@@ -733,11 +733,12 @@ class MockDeviceStorage: NSObject, ARTDeviceStorage {
 
 }
 
-fileprivate struct ErrorSimulator {
+struct ErrorSimulator {
     let value: Int
     let description: String
     let serverId = "server-test-suite"
     var statusCode: Int = 401
+    var shouldPerformRequest: Bool = false
 
     mutating func stubResponse(_ url: URL) -> HTTPURLResponse? {
         return HTTPURLResponse(url: url, statusCode: statusCode, httpVersion: "HTTP/1.1", headerFields: [
@@ -844,8 +845,17 @@ class TestProxyHTTPExecutor: NSObject, ARTHTTPExecutor {
         self.requests.append(request)
 
         if var simulatedError = errorSimulator, var requestURL = request.url {
-            defer { errorSimulator = nil }
-            callback?(simulatedError.stubResponse(requestURL), simulatedError.stubData, nil)
+            defer {
+                errorSimulator = nil
+            }
+            if simulatedError.shouldPerformRequest {
+                http.execute(request, completion: { response, data, error in
+                    callback?(simulatedError.stubResponse(requestURL), simulatedError.stubData, nil)
+                })
+            }
+            else {
+                callback?(simulatedError.stubResponse(requestURL), simulatedError.stubData, nil)
+            }
             return
         }
 
@@ -869,11 +879,15 @@ class TestProxyHTTPExecutor: NSObject, ARTHTTPExecutor {
     }
 
     func simulateIncomingServerErrorOnNextRequest(_ errorValue: Int, description: String) {
-        errorSimulator = ErrorSimulator(value: errorValue, description: description, statusCode: 401, stubData: nil)
+        errorSimulator = ErrorSimulator(value: errorValue, description: description, statusCode: 401, shouldPerformRequest: false, stubData: nil)
+    }
+
+    func simulateIncomingServerErrorOnNextRequest(_ error: ErrorSimulator) {
+        errorSimulator = error
     }
 
     func simulateIncomingPayloadOnNextRequest(_ data: Data) {
-        errorSimulator = ErrorSimulator(value: 0, description: "", statusCode: 200, stubData: data)
+        errorSimulator = ErrorSimulator(value: 0, description: "", statusCode: 200, shouldPerformRequest: false, stubData: data)
     }
 
 }

From a2b70b33d09733792cb6a319bea44146f88e44db Mon Sep 17 00:00:00 2001
From: Ricardo Pereira <ricardopereira@whitesmith.co>
Date: Wed, 29 Aug 2018 19:17:33 +0100
Subject: [PATCH 12/19] TestSuite: ARTMessage extension to accept the ID in the
 initializer

---
 Spec/TestUtilities.swift | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/Spec/TestUtilities.swift b/Spec/TestUtilities.swift
index 36aa0e5d5..1dc5a00f7 100644
--- a/Spec/TestUtilities.swift
+++ b/Spec/TestUtilities.swift
@@ -1326,6 +1326,15 @@ extension ARTPresenceMessage {
 
 }
 
+extension ARTMessage {
+
+    convenience init(id: String, name: String? = nil, data: Any) {
+        self.init(name: name, data: data)
+        self.id = id
+    }
+
+}
+
 extension ARTRealtimeConnectionState : CustomStringConvertible {
     public var description : String {
         return ARTRealtimeConnectionStateToStr(self)

From 77c58e4f4d7640c6214142045f6ccdd0af3431dd Mon Sep 17 00:00:00 2001
From: Ricardo Pereira <ricardopereira@whitesmith.co>
Date: Wed, 29 Aug 2018 19:17:39 +0100
Subject: [PATCH 13/19] RSL1k5

---
 Spec/RestClientChannel.swift | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/Spec/RestClientChannel.swift b/Spec/RestClientChannel.swift
index 86d3d0683..b9ed3accb 100644
--- a/Spec/RestClientChannel.swift
+++ b/Spec/RestClientChannel.swift
@@ -615,6 +615,37 @@ class RestClientChannel: QuickSpec {
                         }
                     }
                 }
+
+                // RSL1k5
+                it("should publish a message with implicit Id only once") {
+                    let options = AblyTests.commonAppSetup()
+                    let rest = ARTRest(options: options)
+                    let channel = rest.channels.get("idempotent")
+
+                    let message = ARTMessage(name: "unique", data: "foo")
+                    message.id = "123"
+
+                    for _ in 1...4 {
+                        waitUntil(timeout: testTimeout) { done in
+                            channel.publish([message]) { error in
+                                expect(error).to(beNil())
+                                done()
+                            }
+                        }
+                    }
+
+                    waitUntil(timeout: testTimeout) { done in
+                        channel.history { result, error in
+                            expect(error).to(beNil())
+                            guard let result = result else {
+                                fail("No result"); done(); return
+                            }
+                            expect(result.items.count) == 1
+                            expect(result.items.first?.id).to(equal("123"))
+                            done()
+                        }
+                    }
+                }
             }
         }
 

From 29b82d4e7a5156c86c37d1d64df326e59a1d1daf Mon Sep 17 00:00:00 2001
From: Ricardo Pereira <ricardopereira@whitesmith.co>
Date: Thu, 30 Aug 2018 18:37:33 +0100
Subject: [PATCH 14/19] [fix] RSL4a: spec was failing because it compares the
 exact payload

so, I disabled idempotentRestPublishing explicitly
---
 Spec/RestClientChannel.swift | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Spec/RestClientChannel.swift b/Spec/RestClientChannel.swift
index b9ed3accb..3b86091c8 100644
--- a/Spec/RestClientChannel.swift
+++ b/Spec/RestClientChannel.swift
@@ -980,6 +980,7 @@ class RestClientChannel: QuickSpec {
                     TestCase(value: binaryData, expected: JSON(["data": binaryData.toBase64, "encoding": "base64"])),
                 ]
 
+                client.options.idempotentRestPublishing = false
                 client.httpExecutor = testHTTPExecutor
 
                 validCases.forEach { caseTest in

From ab82fe2292ebe97d1b4cd2d2a435c6dd68a4d48c Mon Sep 17 00:00:00 2001
From: Ricardo Pereira <ricardopereira@whitesmith.co>
Date: Thu, 20 Sep 2018 19:51:35 +0100
Subject: [PATCH 15/19] Update RTN16b

---
 Source/ARTConnection.m              | 2 +-
 Spec/RealtimeClientConnection.swift | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/Source/ARTConnection.m b/Source/ARTConnection.m
index fe37b92ef..ee4e61b67 100644
--- a/Source/ARTConnection.m
+++ b/Source/ARTConnection.m
@@ -175,7 +175,7 @@ - (NSString *)recoveryKey_nosync {
             if (recStr == nil) {
                 return nil;
             }
-            NSString *str = [recStr stringByAppendingString:[NSString stringWithFormat:@":%ld", (long)self.serial_nosync]];
+            NSString *str = [recStr stringByAppendingString:[NSString stringWithFormat:@":%ld:%ld", (long)self.serial_nosync, (long)_realtime.msgSerial]];
             return str;
         } default:
             return nil;
diff --git a/Spec/RealtimeClientConnection.swift b/Spec/RealtimeClientConnection.swift
index 4c1edebf5..2205b86a6 100644
--- a/Spec/RealtimeClientConnection.swift
+++ b/Spec/RealtimeClientConnection.swift
@@ -2937,7 +2937,7 @@ class RealtimeClientConnection: QuickSpec {
                 }
 
                 // RTN16b
-                it("Connection#recoveryKey should be composed with the connection key and latest serial received") {
+                it("Connection#recoveryKey should be composed with the connection key and latest serial received and msgSerial") {
                     let options = AblyTests.commonAppSetup()
                     let client = ARTRealtime(options: options)
                     defer { client.dispose(); client.close() }
@@ -2946,7 +2946,7 @@ class RealtimeClientConnection: QuickSpec {
                         let partialDone = AblyTests.splitDone(2, done: done)
                         client.connection.once(.connected) { _ in
                             expect(client.connection.serial).to(equal(-1))
-                            expect(client.connection.recoveryKey).to(equal("\(client.connection.key!):\(client.connection.serial)"))
+                            expect(client.connection.recoveryKey).to(equal("\(client.connection.key!):\(client.connection.serial):\(client.msgSerial)"))
                         }
                         channel.publish(nil, data: "message") { error in
                             expect(error).to(beNil())
@@ -2959,6 +2959,8 @@ class RealtimeClientConnection: QuickSpec {
                             partialDone()
                         }
                     }
+                    expect(client.msgSerial) == 1
+                    expect(client.connection.recoveryKey).to(equal("\(client.connection.key!):\(client.connection.serial):\(client.msgSerial)"))
                 }
 
             }

From 04ed83ca3dc266511581be60a65ea774bfc6c913 Mon Sep 17 00:00:00 2001
From: Ricardo Pereira <ricardopereira@whitesmith.co>
Date: Sun, 7 Oct 2018 11:56:51 +0100
Subject: [PATCH 16/19] idempotent-dev env

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index c012ba15e..7080b1507 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,7 +23,7 @@ install:
 before_script:
   - xcrun simctl erase all
 script:
-  - export ABLY_ENV="sandbox"
+  - export ABLY_ENV="idempotent-dev"
   - fastlane $LANE
 after_success:
   - sleep 5

From 69aa35861bf60c0eb71c392d343a440add5f485c Mon Sep 17 00:00:00 2001
From: Ricardo Pereira <ricardopereira@whitesmith.co>
Date: Fri, 4 Jan 2019 02:04:25 +0000
Subject: [PATCH 17/19] Change to 1.2

---
 Source/ARTClientOptions.m    | 2 +-
 Spec/RestClientChannel.swift | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/Source/ARTClientOptions.m b/Source/ARTClientOptions.m
index b646f0adc..2501e95eb 100644
--- a/Source/ARTClientOptions.m
+++ b/Source/ARTClientOptions.m
@@ -182,7 +182,7 @@ - (void)setDefaultTokenParams:(ARTTokenParams *)value {
 }
 
 + (BOOL)getDefaultIdempotentRestPublishingForVersion:(NSString *)version {
-    if ([@"1.1" compare:version options:NSNumericSearch] == NSOrderedDescending) {
+    if ([@"1.2" compare:version options:NSNumericSearch] == NSOrderedDescending) {
         return false;
     }
     else {
diff --git a/Spec/RestClientChannel.swift b/Spec/RestClientChannel.swift
index 4299ca728..518c49e1e 100644
--- a/Spec/RestClientChannel.swift
+++ b/Spec/RestClientChannel.swift
@@ -414,9 +414,10 @@ class RestClientChannel: QuickSpec {
                 it("idempotentRestPublishing option") {
                     expect(ARTClientOptions.getDefaultIdempotentRestPublishing(forVersion: "2")) == true
                     expect(ARTClientOptions.getDefaultIdempotentRestPublishing(forVersion: "2.0.0")) == true
-                    expect(ARTClientOptions.getDefaultIdempotentRestPublishing(forVersion: "1.1")) == true
-                    expect(ARTClientOptions.getDefaultIdempotentRestPublishing(forVersion: "1.1.2")) == true
+                    expect(ARTClientOptions.getDefaultIdempotentRestPublishing(forVersion: "1.1")) == false
+                    expect(ARTClientOptions.getDefaultIdempotentRestPublishing(forVersion: "1.1.2")) == false
                     expect(ARTClientOptions.getDefaultIdempotentRestPublishing(forVersion: "1.2")) == true
+                    expect(ARTClientOptions.getDefaultIdempotentRestPublishing(forVersion: "1.2.2")) == true
                     expect(ARTClientOptions.getDefaultIdempotentRestPublishing(forVersion: "1.0")) == false
                     expect(ARTClientOptions.getDefaultIdempotentRestPublishing(forVersion: "1.0.5")) == false
                     expect(ARTClientOptions.getDefaultIdempotentRestPublishing(forVersion: "0.9")) == false

From f6a7a40c98e7d3dbbf0c624ac27ccd2a079b1acc Mon Sep 17 00:00:00 2001
From: Ricardo Pereira <ricardopereira@whitesmith.co>
Date: Fri, 4 Jan 2019 19:05:57 +0000
Subject: [PATCH 18/19] Since Idempotent Publishing is not enabled by default
 in 1.1

---
 Spec/RestClientChannel.swift | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/Spec/RestClientChannel.swift b/Spec/RestClientChannel.swift
index 518c49e1e..70e0c7d17 100644
--- a/Spec/RestClientChannel.swift
+++ b/Spec/RestClientChannel.swift
@@ -425,7 +425,7 @@ class RestClientChannel: QuickSpec {
 
                     // Current version
                     let options = AblyTests.clientOptions()
-                    expect(options.idempotentRestPublishing) == true
+                    expect(options.idempotentRestPublishing) == false
                 }
 
                 func assertMessagePayloadId(id: String?, expectedSerial: String) {
@@ -458,7 +458,7 @@ class RestClientChannel: QuickSpec {
                         expect(message.id).to(beNil())
 
                         let rest = ARTRest(key: "xxxx:xxxx")
-                        expect(rest.options.idempotentRestPublishing) == true
+                        rest.options.idempotentRestPublishing = true
                         let mockHTTPExecutor = MockHTTPExecutor()
                         rest.httpExecutor = mockHTTPExecutor
                         let channel = rest.channels.get("idempotent")
@@ -486,6 +486,7 @@ class RestClientChannel: QuickSpec {
                         expect(message2.id).to(beNil())
 
                         let rest = ARTRest(key: "xxxx:xxxx")
+                        rest.options.idempotentRestPublishing = true
                         let mockHTTPExecutor = MockHTTPExecutor()
                         rest.httpExecutor = mockHTTPExecutor
                         let channel = rest.channels.get("idempotent")
@@ -518,6 +519,7 @@ class RestClientChannel: QuickSpec {
                     message.id = "123"
 
                     let rest = ARTRest(key: "xxxx:xxxx")
+                    rest.options.idempotentRestPublishing = true
                     let mockHTTPExecutor = MockHTTPExecutor()
                     rest.httpExecutor = mockHTTPExecutor
                     let channel = rest.channels.get("idempotent")
@@ -539,6 +541,7 @@ class RestClientChannel: QuickSpec {
 
                 it("should generate for internal message that is created in publish(name:data:) method") {
                     let rest = ARTRest(key: "xxxx:xxxx")
+                    rest.options.idempotentRestPublishing = true
                     let mockHTTPExecutor = MockHTTPExecutor()
                     rest.httpExecutor = mockHTTPExecutor
                     let channel = rest.channels.get("idempotent")
@@ -566,6 +569,7 @@ class RestClientChannel: QuickSpec {
                     message2.id = "123"
 
                     let rest = ARTRest(key: "xxxx:xxxx")
+                    rest.options.idempotentRestPublishing = true
                     let mockHTTPExecutor = MockHTTPExecutor()
                     rest.httpExecutor = mockHTTPExecutor
                     let channel = rest.channels.get("idempotent")
@@ -618,6 +622,7 @@ class RestClientChannel: QuickSpec {
 
                 // RSL1k4
                 it("should have only one published message") {
+                    client.options.idempotentRestPublishing = true
                     client.httpExecutor = testHTTPExecutor
                     client.options.fallbackHostsUseDefault = true
 
@@ -662,6 +667,7 @@ class RestClientChannel: QuickSpec {
                 it("should publish a message with implicit Id only once") {
                     let options = AblyTests.commonAppSetup()
                     let rest = ARTRest(options: options)
+                    rest.options.idempotentRestPublishing = true
                     let channel = rest.channels.get("idempotent")
 
                     let message = ARTMessage(name: "unique", data: "foo")

From c9a9c36d71d089abaf7c2b7ee5300061277c2a1b Mon Sep 17 00:00:00 2001
From: Ricardo Pereira <ricardopereira@whitesmith.co>
Date: Fri, 5 Apr 2019 17:03:49 +0100
Subject: [PATCH 19/19] Revert "idempotent-dev env"

This reverts commit 04ed83ca3dc266511581be60a65ea774bfc6c913.
---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index c0ebaea2c..9e0d96752 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -22,7 +22,7 @@ install:
 before_script:
   - xcrun simctl erase all
 script:
-  - export ABLY_ENV="idempotent-dev"
+  - export ABLY_ENV="sandbox"
   - fastlane $LANE
 after_success:
   - sleep 5