Additional logging and add timeouts for flaky tests for Concurrent Se…

…ssion limit for users (elastic#174748)

## Summary

Closes elastic#149091 

This PR addresses the potential issue of a session not being found in
the session index by introducing a timeout before attempting to write
the next one. Passing these [changes through
FTR](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4854)
make it pass 100% of the time with 400 test runs.

x-pack/plugins/security/server/session_management/session_index.ts

@@ -276,6 +276,11 @@ export class SessionIndex {
         ({ body, statusCode } = await this.writeNewSessionDocument(sessionValue, {
           ignore404: false,
         }));
+        if (statusCode !== 201) {
+          this.options.logger.error(
+            `Failed to write a new session (status code: ${statusCode}): ${JSON.stringify(body)}.`
+          );
+        }
       }
 
       return {

x-pack/test/security_api_integration/tests/session_concurrent_limit/cleanup.ts

@@ -240,12 +240,30 @@ export default function ({ getService }: FtrProviderContext) {
       log.debug(`Log in as ${testUser.username} and SAML user 3 times each with a 0.5s delay.`);
 
       const basicSessionCookieOne = await loginWithBasic(testUser);
+      await retry.tryForTime(20000, async () => {
+        expect(await getNumberOfSessionDocuments()).to.be(1);
+      });
+
       const samlSessionCookieOne = await loginWithSAML();
-      await setTimeoutAsync(500);
+      await retry.tryForTime(20000, async () => {
+        expect(await getNumberOfSessionDocuments()).to.be(2);
+      });
+
       const basicSessionCookieTwo = await loginWithBasic(testUser);
+      await retry.tryForTime(20000, async () => {
+        expect(await getNumberOfSessionDocuments()).to.be(3);
+      });
+
       const samlSessionCookieTwo = await loginWithSAML();
-      await setTimeoutAsync(500);
+      await retry.tryForTime(20000, async () => {
+        expect(await getNumberOfSessionDocuments()).to.be(4);
+      });
+
       const basicSessionCookieThree = await loginWithBasic(testUser);
+      await retry.tryForTime(20000, async () => {
+        expect(await getNumberOfSessionDocuments()).to.be(5);
+      });
+
       const samlSessionCookieThree = await loginWithSAML();
 
       log.debug('Waiting for all sessions to be persisted...');