feat: implement retry mechanism for Discourse API requests (#1307)

rs/cli/src/forum/impls.rs

@@ -361,6 +361,7 @@ struct DiscourseClientImp {
 #[derive(Debug)]
 enum DiscourseClientImpError {
     NotFound,
+    CloneFailed,
     ExpectedPayload,
     DeserializeFailed(String),
     OtherReqwestError(reqwest::Error),
@@ -419,25 +420,49 @@ impl DiscourseClientImp {
             request = request.body(payload);
         }
 
-        match request.send().await {
-            Ok(r) => {
-                let rstatus = r.status();
-                match r.error_for_status() {
-                    Ok(t) => t
-                        .json()
-                        .await
-                        .map_err(|e| DiscourseClientImpError::DeserializeFailed(format!("Deserialization of response failed: {}", e))),
-                    Err(e) => {
-                        if rstatus == reqwest::StatusCode::NOT_FOUND {
-                            Err(DiscourseClientImpError::NotFound)
-                        } else {
-                            Err(DiscourseClientImpError::OtherReqwestError(e))
+        let sleep_secs = 5;
+        let retries_max = 5;
+
+        for retry_num in 1..=retries_max {
+            let retry_secs = sleep_secs * retry_num;
+
+            // Properly handle potential cloning failure.
+            let cloned_request = request.try_clone().ok_or(DiscourseClientImpError::CloneFailed)?;
+            match cloned_request.send().await {
+                Ok(response) => {
+                    let response_status = response.status();
+                    match response.error_for_status() {
+                        Ok(success_response) => {
+                            return success_response
+                                .json()
+                                .await
+                                .map_err(|e| DiscourseClientImpError::DeserializeFailed(format!("Deserialization of response failed: {}", e)));
+                        }
+                        Err(error) => {
+                            if response_status == reqwest::StatusCode::TOO_MANY_REQUESTS {
+                                warn!("Rate limited by Discourse, retrying in {} seconds", retry_secs);
+                                tokio::time::sleep(std::time::Duration::from_secs(retry_secs)).await;
+                                continue;
+                            } else if response_status == reqwest::StatusCode::NOT_FOUND {
+                                return Err(DiscourseClientImpError::NotFound);
+                            } else {
+                                return Err(DiscourseClientImpError::OtherReqwestError(error));
+                            }
                         }
                     }
                 }
+                Err(error) => {
+                    if retry_num == retries_max {
+                        return Err(DiscourseClientImpError::OtherReqwestError(error));
+                    }
+                    warn!("Error while sending request to Discourse: {}. Retrying in {} seconds", error, retry_secs);
+                    tokio::time::sleep(std::time::Duration::from_secs(retry_secs)).await;
+                }
             }
-            Err(e) => Err(DiscourseClientImpError::OtherReqwestError(e)),
         }
+
+        // This should never be reached due to the return in the last retry iteration
+        unreachable!("Should have either succeeded or returned an error by now");
     }
 
     async fn get_category_id(&self, category_name: String) -> anyhow::Result<u64> {