From 2afc4b17f004093f0c807c7ed2aeee63ec773af6 Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Tue, 25 Jul 2023 13:22:24 +0100 Subject: [PATCH 01/32] in progress (+9 squashed commits) Squashed commits: [0e51c10685] In progress. [fdecb78889] new base copy [0cd4b10502] package lock [401a7b415a] php lint fixes [62c4829170] endpoint updates [8c95d06ba2] fixes and remove unlimited connection [05d5134c0b] package lock & lint fixes [9a85abdd68] Working signaling server using wp ajax admin [f5f13f7d57] Base y-webrtc copied --- lib/sync/endpoint.php | 210 ++++++ packages/sync/package.json | 6 + packages/sync/src/connect-webrtc.js | 15 +- .../sync/src/webrtc-http-stream-signaling.js | 409 +++++++++++ packages/sync/src/y-webrtc/crypto.js | 118 +++ packages/sync/src/y-webrtc/y-webrtc.js | 676 ++++++++++++++++++ packages/sync/tsconfig.json | 3 +- 7 files changed, 1433 insertions(+), 4 deletions(-) create mode 100644 lib/sync/endpoint.php create mode 100644 packages/sync/src/webrtc-http-stream-signaling.js create mode 100644 packages/sync/src/y-webrtc/crypto.js create mode 100644 packages/sync/src/y-webrtc/y-webrtc.js diff --git a/lib/sync/endpoint.php b/lib/sync/endpoint.php new file mode 100644 index 0000000000000..1e69752c22d7d --- /dev/null +++ b/lib/sync/endpoint.php @@ -0,0 +1,210 @@ + array( 'message hello','handshake message' ) ). + + $topics_to_subscribers_path = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'topics_to_subscribers.txt'; + // Example: array( 'doc1: array( 2323232121 ), 'doc2: array( 2323232123, 2323232121 ) ). + + $subscribers_to_last_connection_path = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'subscribers_to_last_connection.txt'; + // Example: array( 2323232121 => 34343433323(timestamp) ). + + $subscriber_id = $_REQUEST['unique']; + if ( ! $subscriber_id ) { + die( 'no identifier' ); + } + + if ( 'GET' === $_SERVER['REQUEST_METHOD'] ) { + header( 'Content-Type: text/event-stream' ); + header( 'Cache-Control: no-cache' ); + echo 'retry: 3000' . PHP_EOL; + $fd = fopen( $subscriber_to_messages_path, 'c+' ); + flock( $fd, LOCK_EX ); + $subscriber_to_messages = _gutenberg_get_contents_from_file_descriptor( $fd ); + if ( isset( $subscriber_to_messages[ $subscriber_id ] ) && count( $subscriber_to_messages[ $subscriber_id ] ) > 0 ) { + $messages = array_map( 'json_encode', $subscriber_to_messages[ $subscriber_id ] ); + $subscriber_to_messages[ $subscriber_id ] = array(); + $data = null; + if ( count( $messages ) > 1 ) { + $data = implode( '|MULTIPLE|', $messages ); + } else { + $data = $messages[0]; + } + if ( $data ) { + echo 'id: ' . time() . PHP_EOL; + echo 'event: message' . PHP_EOL; + echo 'data: ' . $data . PHP_EOL . PHP_EOL; + } + _gutenberg_save_contents_to_file_descriptor( $fd, $subscriber_to_messages ); + + } else { + echo PHP_EOL; + } + flock( $fd, LOCK_UN ); + fclose( $fd ); + } else { + $raw_data = $_POST['data']; + $message = json_decode( wp_unslash( $raw_data ), true ); + if ( ! $message ) { + die( 'no message' ); + } + $fd_topics_subscriber = fopen( $topics_to_subscribers_path, 'c+' ); + flock( $fd_topics_subscriber, LOCK_EX ); + $topics_to_subscribers = _gutenberg_get_contents_from_file_descriptor( $fd_topics_subscriber ); + + switch ( $message['type'] ) { + case 'subscribe': + $topics = $message['topics']; + foreach ( $topics as $topic ) { + if ( ! $topics_to_subscribers[ $topic ] ) { + $topics_to_subscribers[ $topic ] = array(); + } + $topics_to_subscribers[ $topic ] = array_unique( array_merge( $topics_to_subscribers[ $topic ], array( $subscriber_id ) ) ); + } + _gutenberg_save_contents_to_file_descriptor( $fd_topics_subscriber, $topics_to_subscribers ); + break; + case 'unsubscribe': + $topics = $message['topics']; + foreach ( $topics as $topic ) { + if ( $topics_to_subscribers[ $topic ] ) { + $topics_to_subscribers[ $topic ] = array_diff( $topics_to_subscribers[ $topic ], array( $subscriber_id ) ); + } + } + _gutenberg_save_contents_to_file_descriptor( $fd_topics_subscriber, $topics_to_subscribers ); + break; + case 'publish': + $fd_subscriber_messages = fopen( $subscriber_to_messages_path, 'c+' ); + flock( $fd_subscriber_messages, LOCK_EX ); + $subscriber_to_messages = _gutenberg_get_contents_from_file_descriptor( $fd_subscriber_messages ); + $topic = $message['topic']; + $receivers = $topics_to_subscribers[ $topic ]; + if ( $receivers ) { + $message['clients'] = count( $receivers ); + foreach ( $receivers as $receiver ) { + if ( ! $subscriber_to_messages[ $receiver ] ) { + $subscriber_to_messages[ $receiver ] = array(); + } + $subscriber_to_messages[ $receiver ][] = $message; + } + _gutenberg_save_contents_to_file_descriptor( $fd_subscriber_messages, $subscriber_to_messages ); + } + flock( $fd_subscriber_messages, LOCK_UN ); + fclose( $fd_subscriber_messages ); + break; + case 'ping': + $fd_subscriber_messages = fopen( $subscriber_to_messages_path, 'c+' ); + flock( $fd_subscriber_messages, LOCK_EX ); + $subscriber_to_messages = _gutenberg_get_contents_from_file_descriptor( $fd_subscriber_messages ); + if ( ! $subscriber_to_messages[ $subscriber_id ] ) { + $subscriber_to_messages[ $subscriber_id ] = array(); + } + $subscriber_to_messages[ $subscriber_id ][] = array( 'type' => 'pong' ); + _gutenberg_save_contents_to_file_descriptor( $fd_subscriber_messages, $subscriber_to_messages ); + flock( $fd_subscriber_messages, LOCK_UN ); + fclose( $fd_subscriber_messages ); + break; + } + flock( $fd_topics_subscriber, LOCK_UN ); + fclose( $fd_topics_subscriber ); + echo json_encode( array( 'result' => 'ok' ) ), PHP_EOL, PHP_EOL; + } + + $fd_subscribers_last_connection = fopen( $subscribers_to_last_connection_path, 'c+' ); + flock( $fd_subscribers_last_connection, LOCK_EX ); + $subscribers_to_last_connection_time = _gutenberg_get_contents_from_file_descriptor( $fd_subscribers_last_connection ); + $subscribers_to_last_connection_time[ $subscriber_id ] = time(); + $needs_cleanup = false; + foreach ( $subscribers_to_last_connection_time as $subscriber_id => $last_connection_time ) { + // cleanup connections older than 24 hours. + if ( $last_connection_time < time() - 24 * 60 * 60 ) { + unset( $subscribers_to_last_connection_time[ $subscriber_id ] ); + $needs_cleanup = true; + } + } + if ( $needs_cleanup ) { + $fd_subscriber_messages = fopen( $subscriber_to_messages_path, 'c+' ); + flock( $fd_subscriber_messages, LOCK_EX ); + $subscriber_to_messages = _gutenberg_get_contents_from_file_descriptor( $fd_subscriber_messages ); + foreach ( $subscriber_to_messages as $subscriber_id => $messages ) { + if ( ! isset( $subscribers_to_last_connection_time[ $subscriber_id ] ) ) { + unset( $subscriber_to_messages[ $subscriber_id ] ); + } + } + _gutenberg_save_contents_to_file_descriptor( $fd_subscriber_messages, $subscriber_to_messages ); + flock( $fd_subscriber_messages, LOCK_UN ); + fclose( $fd_subscriber_messages ); + + $fd_topics_subscriber = fopen( $topics_to_subscribers_path, 'c+' ); + flock( $fd_topics_subscriber, LOCK_EX ); + $topics_to_subscribers = _gutenberg_get_contents_from_file_descriptor( $fd_topics_subscriber ); + foreach ( $topics_to_subscribers as $topic => $subscribers ) { + foreach ( $subscribers as $subscriber_id ) { + if ( ! isset( $subscribers_to_last_connection_time[ $subscriber_id ] ) ) { + $topics_to_subscribers[ $topic ] = array_diff( $topics_to_subscribers[ $topic ], array( $subscriber_id ) ); + } + } + } + _gutenberg_save_contents_to_file_descriptor( $fd_topics_subscriber, $topics_to_subscribers ); + flock( $fd_topics_subscriber, LOCK_UN ); + fclose( $fd_topics_subscriber ); + } + + _gutenberg_save_contents_to_file_descriptor( $fd_subscribers_last_connection, $subscribers_to_last_connection_time ); + flock( $fd_subscribers_last_connection, LOCK_UN ); + fclose( $fd_subscribers_last_connection ); + exit; +} + + + +add_action( 'wp_ajax_gutenberg_signaling_server', '_gutenberg_wp_ajax_signaling_server' ); diff --git a/packages/sync/package.json b/packages/sync/package.json index 34ce268433f31..e73983f2a42fd 100644 --- a/packages/sync/package.json +++ b/packages/sync/package.json @@ -28,7 +28,13 @@ "sideEffects": false, "dependencies": { "@babel/runtime": "^7.16.0", + "@types/simple-peer": "^9.11.5", + "@wordpress/url": "file:../url", + "import-locals": "^2.0.0", + "lib0": "^0.2.42", + "simple-peer": "^9.11.0", "y-indexeddb": "~9.0.11", + "y-protocols": "^1.0.5", "y-webrtc": "~10.2.5", "yjs": "~13.6.6" }, diff --git a/packages/sync/src/connect-webrtc.js b/packages/sync/src/connect-webrtc.js index fefe3fddbb8a5..1801212c8c2bf 100644 --- a/packages/sync/src/connect-webrtc.js +++ b/packages/sync/src/connect-webrtc.js @@ -1,8 +1,12 @@ /** * External dependencies */ -// @ts-ignore -import { WebrtcProvider } from 'y-webrtc'; +// import { WebrtcProvider } from 'y-webrtc'; + +/** + * Internal dependencies + */ +import { WebrtcProviderWithHttpSignaling } from './webrtc-http-stream-signaling'; /** @typedef {import('./types').ObjectType} ObjectType */ /** @typedef {import('./types').ObjectID} ObjectID */ @@ -19,7 +23,12 @@ import { WebrtcProvider } from 'y-webrtc'; */ export function connectWebRTC( objectId, objectType, doc ) { const roomName = `${ objectType }-${ objectId }`; - new WebrtcProvider( roomName, doc, { + new WebrtcProviderWithHttpSignaling( roomName, doc, { + signaling: [ + //'ws://localhost:4444', + // @ts-ignore + window.wp.ajax.settings.url, + ], // @ts-ignore password: window.__experimentalCollaborativeEditingSecret, } ); diff --git a/packages/sync/src/webrtc-http-stream-signaling.js b/packages/sync/src/webrtc-http-stream-signaling.js new file mode 100644 index 0000000000000..efad9af3039e6 --- /dev/null +++ b/packages/sync/src/webrtc-http-stream-signaling.js @@ -0,0 +1,409 @@ +/** + * External dependencies + */ +/** + * Internal dependencies + */ +import { + WebrtcProvider, + SignalingConn, + WebrtcConn, + signalingConns, + rooms, + publishSignalingMessage, + log, +} from './y-webrtc/y-webrtc'; +import * as cryptoutils from './y-webrtc/crypto'; + +import * as map from 'lib0/map'; +import { Observable } from 'lib0/observable'; +import * as buffer from 'lib0/buffer'; + +/** + * WordPress dependencies + */ +import { addQueryArgs } from '@wordpress/url'; + +/** + * Method copied as is from the SignalingConn constructor. + * Setups the needed event handlers for an http signaling connection. + * + * @param {HttpSignalingConn} signalCon The signaling connection. + * @param {string} url The url. + */ +function setupSignalEventHandlers( signalCon, url ) { + signalCon.on( 'connect', () => { + log( `connected (${ url })` ); + const topics = Array.from( rooms.keys() ); + signalCon.send( { type: 'subscribe', topics } ); + rooms.forEach( ( room ) => + publishSignalingMessage( signalCon, room, { + type: 'announce', + from: room.peerId, + } ) + ); + } ); + signalCon.on( + 'message', + ( /** @type {{ type: any; topic: any; data: string; }} */ m ) => { + switch ( m.type ) { + case 'publish': { + const roomName = m.topic; + const room = rooms.get( roomName ); + if ( + room === null || + typeof roomName !== 'string' || + room === undefined + ) { + return; + } + const execMessage = ( /** @type {any} */ data ) => { + const webrtcConns = room.webrtcConns; + const peerId = room.peerId; + if ( + data === null || + data.from === peerId || + ( data.to !== undefined && data.to !== peerId ) || + room.bcConns.has( data.from ) + ) { + // ignore messages that are not addressed to this conn, or from clients that are connected via broadcastchannel + return; + } + const emitPeerChange = webrtcConns.has( data.from ) + ? () => {} + : () => + room.provider.emit( 'peers', [ + { + removed: [], + added: [ data.from ], + webrtcPeers: Array.from( + room.webrtcConns.keys() + ), + bcPeers: Array.from( room.bcConns ), + }, + ] ); + switch ( data.type ) { + case 'announce': + if ( + webrtcConns.size < room.provider.maxConns + ) { + map.setIfUndefined( + webrtcConns, + data.from, + () => + new WebrtcConn( + signalCon, + true, + data.from, + room + ) + ); + emitPeerChange(); + } + break; + case 'signal': + if ( data.signal.type === 'offer' ) { + const existingConn = webrtcConns.get( + data.from + ); + if ( existingConn ) { + const remoteToken = data.token; + const localToken = + existingConn.glareToken; + if ( + localToken && + localToken > remoteToken + ) { + log( + 'offer rejected: ', + data.from + ); + return; + } + // if we don't reject the offer, we will be accepting it and answering it + existingConn.glareToken = undefined; + } + } + if ( data.signal.type === 'answer' ) { + log( 'offer answered by: ', data.from ); + const existingConn = webrtcConns.get( + data.from + ); + if ( existingConn ) { + existingConn.glareToken = undefined; + } + } + if ( data.to === peerId ) { + map.setIfUndefined( + webrtcConns, + data.from, + () => + new WebrtcConn( + signalCon, + false, + data.from, + room + ) + ).peer.signal( data.signal ); + emitPeerChange(); + } + break; + } + }; + if ( room.key ) { + if ( typeof m.data === 'string' ) { + cryptoutils + .decryptJson( + buffer.fromBase64( m.data ), + room.key + ) + .then( execMessage ); + } + } else { + execMessage( m.data ); + } + } + } + } + ); + signalCon.on( 'disconnect', () => log( `disconnect (${ url })` ) ); +} + +/** + * Method that instantiates the http signaling connection. + * Tries to implement the same methods a websocket provides using ajax requests + * to send messages and EventSource to retrieve messages. + * + * @param {HttpSignalingConn} httpClient The signaling connection. + */ +function setupHttpSignal( httpClient ) { + if ( httpClient.shouldConnect && httpClient.ws === null ) { + // eslint-disable-next-line no-restricted-syntax + const unique = Math.floor( 100000 + Math.random() * 900000 ); + const url = httpClient.url; + const eventSource = new window.EventSource( + addQueryArgs( url, { + unique, + action: 'gutenberg_signaling_server', + } ) + ); + /** + * @type {any} + */ + let pingTimeout = null; + eventSource.onmessage = ( event ) => { + httpClient.lastMessageReceived = Date.now(); + const data = event.data; + + if ( data.includes( '|MULTIPLE|' ) ) { + const messages = data.split( '|MULTIPLE|' ); + messages.forEach( onSingleMessage ); + } else { + onSingleMessage( data ); + } + }; + // @ts-ignore + httpClient.ws = eventSource; + httpClient.connecting = true; + httpClient.connected = false; + const onSingleMessage = ( /** @type {string} */ data ) => { + const message = + typeof data === 'string' ? JSON.parse( data ) : data; + //console.log( 'receive', message ); + if ( message && message.type === 'pong' ) { + clearTimeout( pingTimeout ); + pingTimeout = setTimeout( + sendPing, + messageReconnectTimeout / 2 + ); + } + httpClient.emit( 'message', [ message, httpClient ] ); + }; + + /** + * @param {any} error + */ + const onclose = ( error ) => { + if ( httpClient.ws !== null ) { + httpClient.ws.close(); + httpClient.ws = null; + httpClient.connecting = false; + if ( httpClient.connected ) { + httpClient.connected = false; + httpClient.emit( 'disconnect', [ + { type: 'disconnect', error }, + httpClient, + ] ); + } else { + httpClient.unsuccessfulReconnects++; + } + } + clearTimeout( pingTimeout ); + }; + const sendPing = () => { + if ( + httpClient.ws && + httpClient.ws.readyState === window.EventSource.OPEN + ) { + httpClient.send( { + type: 'ping', + } ); + } + }; + if ( httpClient.ws ) { + httpClient.ws.onclose = () => { + onclose( null ); + }; + httpClient.ws.send = function send( + /** @type {string} */ message + ) { + //console.log( 'send', message ); + const xhttp = new window.XMLHttpRequest(); + xhttp.onreadystatechange = function () { + if ( this.readyState !== 4 ) { + return; + } + if ( this.status !== 200 ) { + log( + 'Error sending to server with message: ' + message + ); + } + }; + xhttp.open( 'POST', url, true ); + const dataToSend = new URLSearchParams(); + dataToSend.set( 'unique', unique.toString() ); + dataToSend.set( 'data', message ); + dataToSend.set( 'action', 'gutenberg_signaling_server' ); + xhttp.send( dataToSend ); + }; + } + eventSource.onerror = () => { + // Todo: add an error handler + }; + eventSource.onopen = () => { + if ( httpClient.connected ) { + return; + } + if ( eventSource.readyState === window.EventSource.OPEN ) { + httpClient.lastMessageReceived = Date.now(); + httpClient.connecting = false; + httpClient.connected = true; + httpClient.unsuccessfulReconnects = 0; + httpClient.emit( 'connect', [ + { type: 'connect' }, + httpClient, + ] ); + // set ping + pingTimeout = setTimeout( + sendPing, + messageReconnectTimeout / 2 + ); + } + }; + } +} +const messageReconnectTimeout = 30000; + +/** + * @augments Observable + */ export class HttpSignalingConn extends Observable { + /** + * @param {string} url + */ + constructor( url ) { + super(); + + //WebsocketClient from lib0/websocket.js + this.url = url; + /** + * @type {WebSocket?} + */ + this.ws = null; + // @ts-ignore + this.binaryType = null; // this.binaryType = binaryType + this.connected = false; + this.connecting = false; + this.unsuccessfulReconnects = 0; + this.lastMessageReceived = 0; + /** + * Whether to connect to other peers or not + * + * @type {boolean} + */ + this.shouldConnect = true; + this._checkInterval = setInterval( () => { + if ( + this.connected && + messageReconnectTimeout < + Date.now() - this.lastMessageReceived && + this.ws + ) { + // no message received in a long time - not even your own awareness + // updates (which are updated every 15 seconds) + this.ws.close(); + } + }, messageReconnectTimeout / 2 ); + //setupWS( this ); + setupHttpSignal( this ); + + // From SignalingConn + /** + * @type {Set} + */ + this.providers = new Set(); + + setupSignalEventHandlers( this, url ); + } + + /** + * @param {any} message + */ + send( message ) { + if ( this.ws ) { + this.ws.send( JSON.stringify( message ) ); + } + } + + destroy() { + clearInterval( this._checkInterval ); + this.disconnect(); + super.destroy(); + } + + disconnect() { + this.shouldConnect = false; + if ( this.ws !== null ) { + this.ws.close(); + } + } + + connect() { + this.shouldConnect = true; + if ( ! this.connected && this.ws === null ) { + setupHttpSignal( this ); + } + } +} + +export class WebrtcProviderWithHttpSignaling extends WebrtcProvider { + connect() { + this.shouldConnect = true; + this.signalingUrls.forEach( ( /** @type {string} */ url ) => { + const signalingConn = map.setIfUndefined( + signalingConns, + url, + // Only this conditional logic to create a normal websocket connection or + // an http signaling connection was added to the constructor when compared + // with the base class. + url.startsWith( 'ws://' ) || url.startsWith( 'wss://' ) + ? () => new SignalingConn( url ) + : () => new HttpSignalingConn( url ) + ); + this.signalingConns.push( signalingConn ); + signalingConn.providers.add( this ); + } ); + if ( this.room ) { + this.room.connect(); + } + } +} diff --git a/packages/sync/src/y-webrtc/crypto.js b/packages/sync/src/y-webrtc/crypto.js new file mode 100644 index 0000000000000..b9a15e85e8e29 --- /dev/null +++ b/packages/sync/src/y-webrtc/crypto.js @@ -0,0 +1,118 @@ +// File copied as is from the y-webrtc package. +/* eslint-disable eslint-comments/disable-enable-pair */ +/* eslint-disable eslint-comments/no-unlimited-disable */ +/* eslint-disable */ +// @ts-nocheck +/* eslint-env browser */ + +import * as encoding from 'lib0/encoding' +import * as decoding from 'lib0/decoding' +import * as promise from 'lib0/promise' +import * as error from 'lib0/error' +import * as string from 'lib0/string' + +/** + * @param {string} secret + * @param {string} roomName + * @return {PromiseLike} + */ +export const deriveKey = (secret, roomName) => { + const secretBuffer = string.encodeUtf8(secret).buffer + const salt = string.encodeUtf8(roomName).buffer + return crypto.subtle.importKey( + 'raw', + secretBuffer, + 'PBKDF2', + false, + ['deriveKey'] + ).then(keyMaterial => + crypto.subtle.deriveKey( + { + name: 'PBKDF2', + salt, + iterations: 100000, + hash: 'SHA-256' + }, + keyMaterial, + { + name: 'AES-GCM', + length: 256 + }, + true, + ['encrypt', 'decrypt'] + ) + ) +} + +/** + * @param {Uint8Array} data data to be encrypted + * @param {CryptoKey?} key + * @return {PromiseLike} encrypted, base64 encoded message + */ +export const encrypt = (data, key) => { + if (!key) { + return /** @type {PromiseLike} */ (promise.resolve(data)) + } + const iv = crypto.getRandomValues(new Uint8Array(12)) + return crypto.subtle.encrypt( + { + name: 'AES-GCM', + iv + }, + key, + data + ).then(cipher => { + const encryptedDataEncoder = encoding.createEncoder() + encoding.writeVarString(encryptedDataEncoder, 'AES-GCM') + encoding.writeVarUint8Array(encryptedDataEncoder, iv) + encoding.writeVarUint8Array(encryptedDataEncoder, new Uint8Array(cipher)) + return encoding.toUint8Array(encryptedDataEncoder) + }) +} + +/** + * @param {Object} data data to be encrypted + * @param {CryptoKey?} key + * @return {PromiseLike} encrypted data, if key is provided + */ +export const encryptJson = (data, key) => { + const dataEncoder = encoding.createEncoder() + encoding.writeAny(dataEncoder, data) + return encrypt(encoding.toUint8Array(dataEncoder), key) +} + +/** + * @param {Uint8Array} data + * @param {CryptoKey?} key + * @return {PromiseLike} decrypted buffer + */ +export const decrypt = (data, key) => { + if (!key) { + return /** @type {PromiseLike} */ (promise.resolve(data)) + } + const dataDecoder = decoding.createDecoder(data) + const algorithm = decoding.readVarString(dataDecoder) + if (algorithm !== 'AES-GCM') { + promise.reject(error.create('Unknown encryption algorithm')) + } + const iv = decoding.readVarUint8Array(dataDecoder) + const cipher = decoding.readVarUint8Array(dataDecoder) + return crypto.subtle.decrypt( + { + name: 'AES-GCM', + iv + }, + key, + cipher + ).then(data => new Uint8Array(data)) +} + +/** + * @param {Uint8Array} data + * @param {CryptoKey?} key + * @return {PromiseLike} decrypted object + */ +export const decryptJson = (data, key) => + decrypt(data, key).then(decryptedValue => + decoding.readAny(decoding.createDecoder(new Uint8Array(decryptedValue))) + ) \ No newline at end of file diff --git a/packages/sync/src/y-webrtc/y-webrtc.js b/packages/sync/src/y-webrtc/y-webrtc.js new file mode 100644 index 0000000000000..15b7ea5e0892a --- /dev/null +++ b/packages/sync/src/y-webrtc/y-webrtc.js @@ -0,0 +1,676 @@ +// File copied as is from the y-webrtc package with only exports +// added to the following vars/functions: signalingConns,rooms, publishSignalingMessage, log. +/* eslint-disable eslint-comments/disable-enable-pair */ +/* eslint-disable eslint-comments/no-unlimited-disable */ +/* eslint-disable */ +// @ts-nocheck + +import * as ws from 'lib0/websocket' +import * as map from 'lib0/map' +import * as error from 'lib0/error' +import * as random from 'lib0/random' +import * as encoding from 'lib0/encoding' +import * as decoding from 'lib0/decoding' +import { Observable } from 'lib0/observable' +import * as logging from 'lib0/logging' +import * as promise from 'lib0/promise' +import * as bc from 'lib0/broadcastchannel' +import * as buffer from 'lib0/buffer' +import * as math from 'lib0/math' +import { createMutex } from 'lib0/mutex' + +import * as Y from 'yjs' // eslint-disable-line +import Peer from 'simple-peer/simplepeer.min.js'; + +import * as syncProtocol from 'y-protocols/sync' +import * as awarenessProtocol from 'y-protocols/awareness' + +import * as cryptoutils from './crypto.js' + +export const log = logging.createModuleLogger('y-webrtc') + +const messageSync = 0 +const messageQueryAwareness = 3 +const messageAwareness = 1 +const messageBcPeerId = 4 + +/** + * @type {Map} + */ +export const signalingConns = new Map() + +/** + * @type {Map} + */ +export const rooms = new Map() + +/** + * @param {Room} room + */ +const checkIsSynced = room => { + let synced = true + room.webrtcConns.forEach(peer => { + if (!peer.synced) { + synced = false + } + }) + if ((!synced && room.synced) || (synced && !room.synced)) { + room.synced = synced + room.provider.emit('synced', [{ synced }]) + log('synced ', logging.BOLD, room.name, logging.UNBOLD, ' with all peers') + } +} + +/** + * @param {Room} room + * @param {Uint8Array} buf + * @param {function} syncedCallback + * @return {encoding.Encoder?} + */ +const readMessage = (room, buf, syncedCallback) => { + const decoder = decoding.createDecoder(buf) + const encoder = encoding.createEncoder() + const messageType = decoding.readVarUint(decoder) + if (room === undefined) { + return null + } + const awareness = room.awareness + const doc = room.doc + let sendReply = false + switch (messageType) { + case messageSync: { + encoding.writeVarUint(encoder, messageSync) + const syncMessageType = syncProtocol.readSyncMessage(decoder, encoder, doc, room) + if (syncMessageType === syncProtocol.messageYjsSyncStep2 && !room.synced) { + syncedCallback() + } + if (syncMessageType === syncProtocol.messageYjsSyncStep1) { + sendReply = true + } + break + } + case messageQueryAwareness: + encoding.writeVarUint(encoder, messageAwareness) + encoding.writeVarUint8Array(encoder, awarenessProtocol.encodeAwarenessUpdate(awareness, Array.from(awareness.getStates().keys()))) + sendReply = true + break + case messageAwareness: + awarenessProtocol.applyAwarenessUpdate(awareness, decoding.readVarUint8Array(decoder), room) + break + case messageBcPeerId: { + const add = decoding.readUint8(decoder) === 1 + const peerName = decoding.readVarString(decoder) + if (peerName !== room.peerId && ((room.bcConns.has(peerName) && !add) || (!room.bcConns.has(peerName) && add))) { + const removed = [] + const added = [] + if (add) { + room.bcConns.add(peerName) + added.push(peerName) + } else { + room.bcConns.delete(peerName) + removed.push(peerName) + } + room.provider.emit('peers', [{ + added, + removed, + webrtcPeers: Array.from(room.webrtcConns.keys()), + bcPeers: Array.from(room.bcConns) + }]) + broadcastBcPeerId(room) + } + break + } + default: + console.error('Unable to compute message') + return encoder + } + if (!sendReply) { + // nothing has been written, no answer created + return null + } + return encoder +} + +/** + * @param {WebrtcConn} peerConn + * @param {Uint8Array} buf + * @return {encoding.Encoder?} + */ +const readPeerMessage = (peerConn, buf) => { + const room = peerConn.room + log('received message from ', logging.BOLD, peerConn.remotePeerId, logging.GREY, ' (', room.name, ')', logging.UNBOLD, logging.UNCOLOR) + return readMessage(room, buf, () => { + peerConn.synced = true + log('synced ', logging.BOLD, room.name, logging.UNBOLD, ' with ', logging.BOLD, peerConn.remotePeerId) + checkIsSynced(room) + }) +} + +/** + * @param {WebrtcConn} webrtcConn + * @param {encoding.Encoder} encoder + */ +const sendWebrtcConn = (webrtcConn, encoder) => { + log('send message to ', logging.BOLD, webrtcConn.remotePeerId, logging.UNBOLD, logging.GREY, ' (', webrtcConn.room.name, ')', logging.UNCOLOR) + try { + webrtcConn.peer.send(encoding.toUint8Array(encoder)) + } catch (e) {} +} + +/** + * @param {Room} room + * @param {Uint8Array} m + */ +const broadcastWebrtcConn = (room, m) => { + log('broadcast message in ', logging.BOLD, room.name, logging.UNBOLD) + room.webrtcConns.forEach(conn => { + try { + conn.peer.send(m) + } catch (e) {} + }) +} + +export class WebrtcConn { + /** + * @param {SignalingConn} signalingConn + * @param {boolean} initiator + * @param {string} remotePeerId + * @param {Room} room + */ + constructor (signalingConn, initiator, remotePeerId, room) { + log('establishing connection to ', logging.BOLD, remotePeerId) + this.room = room + this.remotePeerId = remotePeerId + this.glareToken = undefined + this.closed = false + this.connected = false + this.synced = false + /** + * @type {any} + */ + this.peer = new Peer({ initiator, ...room.provider.peerOpts }) + this.peer.on('signal', signal => { + if (this.glareToken === undefined) { + // add some randomness to the timestamp of the offer + this.glareToken = Date.now() + Math.random() + } + publishSignalingMessage(signalingConn, room, { to: remotePeerId, from: room.peerId, type: 'signal', token: this.glareToken, signal }) + }) + this.peer.on('connect', () => { + log('connected to ', logging.BOLD, remotePeerId) + this.connected = true + // send sync step 1 + const provider = room.provider + const doc = provider.doc + const awareness = room.awareness + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, messageSync) + syncProtocol.writeSyncStep1(encoder, doc) + sendWebrtcConn(this, encoder) + const awarenessStates = awareness.getStates() + if (awarenessStates.size > 0) { + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, messageAwareness) + encoding.writeVarUint8Array(encoder, awarenessProtocol.encodeAwarenessUpdate(awareness, Array.from(awarenessStates.keys()))) + sendWebrtcConn(this, encoder) + } + }) + this.peer.on('close', () => { + this.connected = false + this.closed = true + if (room.webrtcConns.has(this.remotePeerId)) { + room.webrtcConns.delete(this.remotePeerId) + room.provider.emit('peers', [{ + removed: [this.remotePeerId], + added: [], + webrtcPeers: Array.from(room.webrtcConns.keys()), + bcPeers: Array.from(room.bcConns) + }]) + } + checkIsSynced(room) + this.peer.destroy() + log('closed connection to ', logging.BOLD, remotePeerId) + announceSignalingInfo(room) + }) + this.peer.on('error', err => { + log('Error in connection to ', logging.BOLD, remotePeerId, ': ', err) + announceSignalingInfo(room) + }) + this.peer.on('data', data => { + const answer = readPeerMessage(this, data) + if (answer !== null) { + sendWebrtcConn(this, answer) + } + }) + } + + destroy () { + this.peer.destroy() + } +} + +/** + * @param {Room} room + * @param {Uint8Array} m + */ +const broadcastBcMessage = (room, m) => cryptoutils.encrypt(m, room.key).then(data => + room.mux(() => + bc.publish(room.name, data) + ) +) + +/** + * @param {Room} room + * @param {Uint8Array} m + */ +const broadcastRoomMessage = (room, m) => { + if (room.bcconnected) { + broadcastBcMessage(room, m) + } + broadcastWebrtcConn(room, m) +} + +/** + * @param {Room} room + */ +const announceSignalingInfo = room => { + signalingConns.forEach(conn => { + // only subscribe if connection is established, otherwise the conn automatically subscribes to all rooms + if (conn.connected) { + conn.send({ type: 'subscribe', topics: [room.name] }) + if (room.webrtcConns.size < room.provider.maxConns) { + publishSignalingMessage(conn, room, { type: 'announce', from: room.peerId }) + } + } + }) +} + +/** + * @param {Room} room + */ +const broadcastBcPeerId = room => { + if (room.provider.filterBcConns) { + // broadcast peerId via broadcastchannel + const encoderPeerIdBc = encoding.createEncoder() + encoding.writeVarUint(encoderPeerIdBc, messageBcPeerId) + encoding.writeUint8(encoderPeerIdBc, 1) + encoding.writeVarString(encoderPeerIdBc, room.peerId) + broadcastBcMessage(room, encoding.toUint8Array(encoderPeerIdBc)) + } +} + +export class Room { + /** + * @param {Y.Doc} doc + * @param {WebrtcProvider} provider + * @param {string} name + * @param {CryptoKey|null} key + */ + constructor (doc, provider, name, key) { + /** + * Do not assume that peerId is unique. This is only meant for sending signaling messages. + * + * @type {string} + */ + this.peerId = random.uuidv4() + this.doc = doc + /** + * @type {awarenessProtocol.Awareness} + */ + this.awareness = provider.awareness + this.provider = provider + this.synced = false + this.name = name + // @todo make key secret by scoping + this.key = key + /** + * @type {Map} + */ + this.webrtcConns = new Map() + /** + * @type {Set} + */ + this.bcConns = new Set() + this.mux = createMutex() + this.bcconnected = false + /** + * @param {ArrayBuffer} data + */ + this._bcSubscriber = data => + cryptoutils.decrypt(new Uint8Array(data), key).then(m => + this.mux(() => { + const reply = readMessage(this, m, () => {}) + if (reply) { + broadcastBcMessage(this, encoding.toUint8Array(reply)) + } + }) + ) + /** + * Listens to Yjs updates and sends them to remote peers + * + * @param {Uint8Array} update + * @param {any} origin + */ + this._docUpdateHandler = (update, origin) => { + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, messageSync) + syncProtocol.writeUpdate(encoder, update) + broadcastRoomMessage(this, encoding.toUint8Array(encoder)) + } + /** + * Listens to Awareness updates and sends them to remote peers + * + * @param {any} changed + * @param {any} origin + */ + this._awarenessUpdateHandler = ({ added, updated, removed }, origin) => { + const changedClients = added.concat(updated).concat(removed) + const encoderAwareness = encoding.createEncoder() + encoding.writeVarUint(encoderAwareness, messageAwareness) + encoding.writeVarUint8Array(encoderAwareness, awarenessProtocol.encodeAwarenessUpdate(this.awareness, changedClients)) + broadcastRoomMessage(this, encoding.toUint8Array(encoderAwareness)) + } + + this._beforeUnloadHandler = () => { + awarenessProtocol.removeAwarenessStates(this.awareness, [doc.clientID], 'window unload') + rooms.forEach(room => { + room.disconnect() + }) + } + + if (typeof window !== 'undefined') { + window.addEventListener('beforeunload', this._beforeUnloadHandler) + } else if (typeof process !== 'undefined') { + process.on('exit', this._beforeUnloadHandler) + } + } + + connect () { + this.doc.on('update', this._docUpdateHandler) + this.awareness.on('update', this._awarenessUpdateHandler) + // signal through all available signaling connections + announceSignalingInfo(this) + const roomName = this.name + bc.subscribe(roomName, this._bcSubscriber) + this.bcconnected = true + // broadcast peerId via broadcastchannel + broadcastBcPeerId(this) + // write sync step 1 + const encoderSync = encoding.createEncoder() + encoding.writeVarUint(encoderSync, messageSync) + syncProtocol.writeSyncStep1(encoderSync, this.doc) + broadcastBcMessage(this, encoding.toUint8Array(encoderSync)) + // broadcast local state + const encoderState = encoding.createEncoder() + encoding.writeVarUint(encoderState, messageSync) + syncProtocol.writeSyncStep2(encoderState, this.doc) + broadcastBcMessage(this, encoding.toUint8Array(encoderState)) + // write queryAwareness + const encoderAwarenessQuery = encoding.createEncoder() + encoding.writeVarUint(encoderAwarenessQuery, messageQueryAwareness) + broadcastBcMessage(this, encoding.toUint8Array(encoderAwarenessQuery)) + // broadcast local awareness state + const encoderAwarenessState = encoding.createEncoder() + encoding.writeVarUint(encoderAwarenessState, messageAwareness) + encoding.writeVarUint8Array(encoderAwarenessState, awarenessProtocol.encodeAwarenessUpdate(this.awareness, [this.doc.clientID])) + broadcastBcMessage(this, encoding.toUint8Array(encoderAwarenessState)) + } + + disconnect () { + // signal through all available signaling connections + signalingConns.forEach(conn => { + if (conn.connected) { + conn.send({ type: 'unsubscribe', topics: [this.name] }) + } + }) + awarenessProtocol.removeAwarenessStates(this.awareness, [this.doc.clientID], 'disconnect') + // broadcast peerId removal via broadcastchannel + const encoderPeerIdBc = encoding.createEncoder() + encoding.writeVarUint(encoderPeerIdBc, messageBcPeerId) + encoding.writeUint8(encoderPeerIdBc, 0) // remove peerId from other bc peers + encoding.writeVarString(encoderPeerIdBc, this.peerId) + broadcastBcMessage(this, encoding.toUint8Array(encoderPeerIdBc)) + + bc.unsubscribe(this.name, this._bcSubscriber) + this.bcconnected = false + this.doc.off('update', this._docUpdateHandler) + this.awareness.off('update', this._awarenessUpdateHandler) + this.webrtcConns.forEach(conn => conn.destroy()) + } + + destroy () { + this.disconnect() + if (typeof window !== 'undefined') { + window.removeEventListener('beforeunload', this._beforeUnloadHandler) + } else if (typeof process !== 'undefined') { + process.off('exit', this._beforeUnloadHandler) + } + } +} + +/** + * @param {Y.Doc} doc + * @param {WebrtcProvider} provider + * @param {string} name + * @param {CryptoKey|null} key + * @return {Room} + */ +const openRoom = (doc, provider, name, key) => { + // there must only be one room + if (rooms.has(name)) { + throw error.create(`A Yjs Doc connected to room "${name}" already exists!`) + } + const room = new Room(doc, provider, name, key) + rooms.set(name, /** @type {Room} */ (room)) + return room +} + +/** + * @param {SignalingConn} conn + * @param {Room} room + * @param {any} data + */ +export const publishSignalingMessage = (conn, room, data) => { + if (room.key) { + cryptoutils.encryptJson(data, room.key).then(data => { + conn.send({ type: 'publish', topic: room.name, data: buffer.toBase64(data) }) + }) + } else { + conn.send({ type: 'publish', topic: room.name, data }) + } +} + +export class SignalingConn extends ws.WebsocketClient { + constructor (url) { + super(url) + /** + * @type {Set} + */ + this.providers = new Set() + this.on('connect', () => { + log(`connected (${url})`) + const topics = Array.from(rooms.keys()) + this.send({ type: 'subscribe', topics }) + rooms.forEach(room => + publishSignalingMessage(this, room, { type: 'announce', from: room.peerId }) + ) + }) + this.on('message', m => { + switch (m.type) { + case 'publish': { + const roomName = m.topic + const room = rooms.get(roomName) + if (room == null || typeof roomName !== 'string') { + return + } + const execMessage = data => { + const webrtcConns = room.webrtcConns + const peerId = room.peerId + if (data == null || data.from === peerId || (data.to !== undefined && data.to !== peerId) || room.bcConns.has(data.from)) { + // ignore messages that are not addressed to this conn, or from clients that are connected via broadcastchannel + return + } + const emitPeerChange = webrtcConns.has(data.from) + ? () => {} + : () => + room.provider.emit('peers', [{ + removed: [], + added: [data.from], + webrtcPeers: Array.from(room.webrtcConns.keys()), + bcPeers: Array.from(room.bcConns) + }]) + switch (data.type) { + case 'announce': + if (webrtcConns.size < room.provider.maxConns) { + map.setIfUndefined(webrtcConns, data.from, () => new WebrtcConn(this, true, data.from, room)) + emitPeerChange() + } + break + case 'signal': + if (data.signal.type === 'offer') { + const existingConn = webrtcConns.get(data.from) + if (existingConn) { + const remoteToken = data.token + const localToken = existingConn.glareToken + if (localToken && localToken > remoteToken) { + log('offer rejected: ', data.from) + return + } + // if we don't reject the offer, we will be accepting it and answering it + existingConn.glareToken = undefined + } + } + if (data.signal.type === 'answer') { + log('offer answered by: ', data.from) + const existingConn = webrtcConns.get(data.from) + existingConn.glareToken = undefined + } + if (data.to === peerId) { + map.setIfUndefined(webrtcConns, data.from, () => new WebrtcConn(this, false, data.from, room)).peer.signal(data.signal) + emitPeerChange() + } + break + } + } + if (room.key) { + if (typeof m.data === 'string') { + cryptoutils.decryptJson(buffer.fromBase64(m.data), room.key).then(execMessage) + } + } else { + execMessage(m.data) + } + } + } + }) + this.on('disconnect', () => log(`disconnect (${url})`)) + } +} + +/** + * @typedef {Object} ProviderOptions + * @property {Array} [signaling] + * @property {string} [password] + * @property {awarenessProtocol.Awareness} [awareness] + * @property {number} [maxConns] + * @property {boolean} [filterBcConns] + * @property {any} [peerOpts] + */ + +/** + * @extends Observable + */ +export class WebrtcProvider extends Observable { + /** + * @param {string} roomName + * @param {Y.Doc} doc + * @param {ProviderOptions?} opts + */ + constructor ( + roomName, + doc, + { + signaling = ['wss://y-webrtc-eu.fly.dev'], + password = null, + awareness = new awarenessProtocol.Awareness(doc), + maxConns = 20 + math.floor(random.rand() * 15), // the random factor reduces the chance that n clients form a cluster + filterBcConns = true, + peerOpts = {} // simple-peer options. See https://github.com/feross/simple-peer#peer--new-peeropts + } = {} + ) { + super() + this.roomName = roomName + this.doc = doc + this.filterBcConns = filterBcConns + /** + * @type {awarenessProtocol.Awareness} + */ + this.awareness = awareness + this.shouldConnect = false + this.signalingUrls = signaling + this.signalingConns = [] + this.maxConns = maxConns + this.peerOpts = peerOpts + /** + * @type {PromiseLike} + */ + this.key = password ? cryptoutils.deriveKey(password, roomName) : /** @type {PromiseLike} */ (promise.resolve(null)) + /** + * @type {Room|null} + */ + this.room = null + this.key.then(key => { + this.room = openRoom(doc, this, roomName, key) + if (this.shouldConnect) { + this.room.connect() + } else { + this.room.disconnect() + } + }) + this.connect() + this.destroy = this.destroy.bind(this) + doc.on('destroy', this.destroy) + } + + /** + * @type {boolean} + */ + get connected () { + return this.room !== null && this.shouldConnect + } + + connect () { + this.shouldConnect = true + this.signalingUrls.forEach(url => { + const signalingConn = map.setIfUndefined(signalingConns, url, () => new SignalingConn(url)) + this.signalingConns.push(signalingConn) + signalingConn.providers.add(this) + }) + if (this.room) { + this.room.connect() + } + } + + disconnect () { + this.shouldConnect = false + this.signalingConns.forEach(conn => { + conn.providers.delete(this) + if (conn.providers.size === 0) { + conn.destroy() + signalingConns.delete(conn.url) + } + }) + if (this.room) { + this.room.disconnect() + } + } + + destroy () { + this.doc.off('destroy', this.destroy) + // need to wait for key before deleting room + this.key.then(() => { + /** @type {Room} */ (this.room).destroy() + rooms.delete(this.roomName) + }) + super.destroy() + } +} \ No newline at end of file diff --git a/packages/sync/tsconfig.json b/packages/sync/tsconfig.json index d2d94e16acd2c..83a2163230eae 100644 --- a/packages/sync/tsconfig.json +++ b/packages/sync/tsconfig.json @@ -5,5 +5,6 @@ "declarationDir": "build-types", "types": [ "node" ] }, - "include": [ "src/**/*" ] + "include": [ "src/**/*" ], + "references": [ { "path": "../url" } ] } From c5acd3ae25f0c17d4e32f2ccd1f93f3777e11e9d Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Fri, 8 Sep 2023 18:31:01 +0100 Subject: [PATCH 02/32] Update lib/sync/endpoint.php Co-authored-by: Anton Vlasenko <43744263+anton-vlasenko@users.noreply.github.com> --- lib/sync/endpoint.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sync/endpoint.php b/lib/sync/endpoint.php index 1e69752c22d7d..0f822e36a90eb 100644 --- a/lib/sync/endpoint.php +++ b/lib/sync/endpoint.php @@ -1,6 +1,6 @@ Date: Fri, 8 Sep 2023 18:31:07 +0100 Subject: [PATCH 03/32] Update lib/sync/endpoint.php Co-authored-by: Anton Vlasenko <43744263+anton-vlasenko@users.noreply.github.com> --- lib/sync/endpoint.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sync/endpoint.php b/lib/sync/endpoint.php index 0f822e36a90eb..475a2f5807b54 100644 --- a/lib/sync/endpoint.php +++ b/lib/sync/endpoint.php @@ -11,7 +11,7 @@ * @access private * @internal * - * @param integer $fd A file descriptor. + * @param resource $fd A file descriptor. * * @return array Unserialized contents of fd. */ From a41b9624c4dcae34f827e6c977b0798755223376 Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Fri, 8 Sep 2023 18:31:13 +0100 Subject: [PATCH 04/32] Update lib/sync/endpoint.php Co-authored-by: Anton Vlasenko <43744263+anton-vlasenko@users.noreply.github.com> --- lib/sync/endpoint.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/sync/endpoint.php b/lib/sync/endpoint.php index 475a2f5807b54..d5c1b3f172fa7 100644 --- a/lib/sync/endpoint.php +++ b/lib/sync/endpoint.php @@ -32,8 +32,8 @@ function _gutenberg_get_contents_from_file_descriptor( $fd ) { * @access private * @internal * - * @param integer $fd A file descriptor. - * @param array $content Content to be serialized and written in a file descriptor. + * @param resource $fd A file descriptor. + * @param array $content Content to be serialized and written in a file descriptor. */ function _gutenberg_save_contents_to_file_descriptor( $fd, $content ) { rewind( $fd ); From 758c3a8f2cb022070d2198814c4256ce738aa9da Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Fri, 8 Sep 2023 18:43:37 +0100 Subject: [PATCH 05/32] Refactor to a class. --- lib/sync/endpoint.php | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/sync/endpoint.php b/lib/sync/endpoint.php index d5c1b3f172fa7..42c9afff00c3b 100644 --- a/lib/sync/endpoint.php +++ b/lib/sync/endpoint.php @@ -74,7 +74,6 @@ function _gutenberg_wp_ajax_signaling_server() { if ( isset( $subscriber_to_messages[ $subscriber_id ] ) && count( $subscriber_to_messages[ $subscriber_id ] ) > 0 ) { $messages = array_map( 'json_encode', $subscriber_to_messages[ $subscriber_id ] ); $subscriber_to_messages[ $subscriber_id ] = array(); - $data = null; if ( count( $messages ) > 1 ) { $data = implode( '|MULTIPLE|', $messages ); } else { From baa1f265e7247f9d768f56e066efc8bda36b89ba Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Mon, 11 Sep 2023 18:08:58 +0100 Subject: [PATCH 06/32] Use fetch instead of XMLHttpRequest. --- .../class-gutenberg-http-signaling-server.php | 322 ++++++++++++++++++ lib/sync/endpoint.php | 209 ------------ 2 files changed, 322 insertions(+), 209 deletions(-) create mode 100644 lib/experimental/sync/class-gutenberg-http-signaling-server.php delete mode 100644 lib/sync/endpoint.php diff --git a/lib/experimental/sync/class-gutenberg-http-signaling-server.php b/lib/experimental/sync/class-gutenberg-http-signaling-server.php new file mode 100644 index 0000000000000..476683292eb46 --- /dev/null +++ b/lib/experimental/sync/class-gutenberg-http-signaling-server.php @@ -0,0 +1,322 @@ + array( 'message hello','handshake message' ) ). + * + * @access private + * @var array + */ + private static $subscriber_to_messages_path; + + /** + * Contains the path of the topics to subscriber file. + * The file contains a data-structure similar to the following example: + * array( 'doc1: array( 2323232121 ), 'doc2: array( 2323232123, 2323232121 ) ). + * + * @access private + * @var array + */ + private static $topics_to_subscribers_path; + + /** + * Contains the path of the subscribers to last connection file. + * The file contains a data-structure similar to the following example: + * array( 2323232121 => 34343433323(timestamp) ). + * + * @access private + * @var array + */ + private static $subscribers_to_last_connection_path; + + /** + * Contains the subscriber id of the client reading or sending messages. + * + * @access private + * @var Integer + */ + private static $subscriber_id; + + /** + * Adds a wp_ajax action to handle the signaling server requests. + */ + public static function init() { + add_action( 'wp_ajax_gutenberg_signaling_server', array( __CLASS__, 'do_wp_ajax_action' ) ); + } + + /** + * Handles a wp_ajax signaling server request. + */ + public static function do_wp_ajax_action() { + error_reporting( E_ALL ); + ini_set( 'display_errors', '1' ); + session_write_close(); + + static::initialize_paths(); + + if ( empty( $_REQUEST ) || empty( $_REQUEST['unique'] ) ) { + die( 'no identifier' ); + } + static::$subscriber_id = $_REQUEST['unique']; + + if ( 'GET' === $_SERVER['REQUEST_METHOD'] ) { + static::handle_message_read_request(); + } else { + if ( empty( $_POST ) || empty( $_POST['data'] ) ) { + die( 'no message' ); + } + $message = json_decode( wp_unslash( $_POST['data'] ), true ); + if ( ! $message ) { + die( 'no message' ); + } + static::handle_message_operation( $message ); + } + + static::clean_up_old_connections(); + exit; + } + + /** + * Initializes the paths of the temporary files used. + */ + private static function initialize_paths() { + static::$subscriber_to_messages_path = get_temp_dir() . DIRECTORY_SEPARATOR . 'subscribers_to_messages.txt'; + // Example: array( 2323232121 => array( 'message hello','handshake message' ) ). + + static::$topics_to_subscribers_path = get_temp_dir() . DIRECTORY_SEPARATOR . 'topics_to_subscribers.txt'; + // Example: array( 'doc1: array( 2323232121 ), 'doc2: array( 2323232123, 2323232121 ) ). + + static::$subscribers_to_last_connection_path = get_temp_dir() . DIRECTORY_SEPARATOR . 'subscribers_to_last_connection.txt'; + // Example: array( 2323232121 => 34343433323(timestamp) ). + } + + /** + * Reads the contents of $fd and returns them unserialized. + * + * @access private + * @internal + * + * @param resource $fd A file descriptor. + * + * @return array Unserialized contents of fd. + */ + private static function get_contents_from_file_descriptor( $fd ) { + $contents_raw = stream_get_contents( $fd ); + $result = array(); + if ( $contents_raw ) { + $result = unserialize( $contents_raw ); + } + return $result; + } + + + /** + * Makes the file descriptor content of $fd equal to the serialization of content. + * Overwrites what was previously in $fd. + * + * @access private + * @internal + * + * @param resource $fd A file descriptor. + * @param array $content Content to be serialized and written in a file descriptor. + */ + private static function save_contents_to_file_descriptor( $fd, $content ) { + rewind( $fd ); + $data = serialize( $content ); + fwrite( $fd, $data ); + ftruncate( $fd, strlen( $data ) ); + } + + /** + * Handles a wp_ajax signaling server request of client that wants to retrieve its messages. + */ + private static function handle_message_read_request() { + header( 'Content-Type: text/event-stream' ); + header( 'Cache-Control: no-cache' ); + echo 'retry: 3000' . PHP_EOL; + $fd = fopen( static::$subscriber_to_messages_path, 'c+' ); + flock( $fd, LOCK_EX ); + $subscriber_to_messages = static::get_contents_from_file_descriptor( $fd ); + if ( isset( $subscriber_to_messages[ static::$subscriber_id ] ) && count( $subscriber_to_messages[ static::$subscriber_id ] ) > 0 ) { + $messages = array_map( 'wp_json_encode', $subscriber_to_messages[ static::$subscriber_id ] ); + $subscriber_to_messages[ static::$subscriber_id ] = array(); + if ( count( $messages ) > 1 ) { + $data = implode( '|MULTIPLE|', $messages ); + } else { + $data = $messages[0]; + } + if ( $data ) { + echo 'id: ' . time() . PHP_EOL; + echo 'event: message' . PHP_EOL; + echo 'data: ' . $data . PHP_EOL . PHP_EOL; + } + static::save_contents_to_file_descriptor( $fd, $subscriber_to_messages ); + + } else { + echo PHP_EOL; + } + flock( $fd, LOCK_UN ); + fclose( $fd ); + } + + /** + * Receives a $topics_to_subscribers data-structure and an array of topics, + * and returns a new $topics_to_subscribers data-structure where the current subscriber is subscribed to the topics. + * + * @access private + * @internal + * + * @param array $topics_to_subscribers Topics to subscribers data-structure. + * @param array $topics An array of topics e.g: array( 'doc1', 'doc2' ). + */ + private static function subscribe_to_topics( $topics_to_subscribers, $topics ) { + foreach ( $topics as $topic ) { + if ( ! $topics_to_subscribers[ $topic ] ) { + $topics_to_subscribers[ $topic ] = array(); + } + $topics_to_subscribers[ $topic ] = array_unique( array_merge( $topics_to_subscribers[ $topic ], array( static::$subscriber_id ) ) ); + } + return $topics_to_subscribers; + } + + /** + * Receives a $topics_to_subscribers data-structure and an array of topics, + * and returns a new $topics_to_subscribers data-structure where the current subscriber is not subscribed to the topics. + * + * @access private + * @internal + * + * @param array $topics_to_subscribers Topics to subscribers data-structure. + * @param array $topics An array of topics e.g: array( 'doc1', 'doc2' ). + */ + private static function unsubscribe_from_topics( $topics_to_subscribers, $topics ) { + foreach ( $topics as $topic ) { + if ( $topics_to_subscribers[ $topic ] ) { + $topics_to_subscribers[ $topic ] = array_diff( $topics_to_subscribers[ $topic ], array( static::$subscriber_id ) ); + } + } + return $topics_to_subscribers; + } + + /** + * Handles a wp_ajax signaling server request of client that is performing an operation. + * An operation can be a ping to say the client is alive, sending a message, or subscribing/unscribing to a set of topics. + * + * @param array $message An array of topics e.g: array( 'doc1', 'doc2' ). + */ + private static function handle_message_operation( $message ) { + $fd_topics_subscriber = fopen( static::$topics_to_subscribers_path, 'c+' ); + flock( $fd_topics_subscriber, LOCK_EX ); + $topics_to_subscribers = static::get_contents_from_file_descriptor( $fd_topics_subscriber ); + + switch ( $message['type'] ) { + case 'subscribe': + static::save_contents_to_file_descriptor( $fd_topics_subscriber, static::subscribe_to_topics( $topics_to_subscribers, $message['topics'] ) ); + break; + case 'unsubscribe': + static::save_contents_to_file_descriptor( $fd_topics_subscriber, static::unsubscribe_from_topics( $topics_to_subscribers, $message['topics'] ) ); + break; + case 'publish': + $fd_subscriber_messages = fopen( static::$subscriber_to_messages_path, 'c+' ); + flock( $fd_subscriber_messages, LOCK_EX ); + $subscriber_to_messages = static::get_contents_from_file_descriptor( $fd_subscriber_messages ); + $topic = $message['topic']; + $receivers = $topics_to_subscribers[ $topic ]; + if ( $receivers ) { + $message['clients'] = count( $receivers ); + foreach ( $receivers as $receiver ) { + if ( ! $subscriber_to_messages[ $receiver ] ) { + $subscriber_to_messages[ $receiver ] = array(); + } + $subscriber_to_messages[ $receiver ][] = $message; + } + static::save_contents_to_file_descriptor( $fd_subscriber_messages, $subscriber_to_messages ); + } + flock( $fd_subscriber_messages, LOCK_UN ); + fclose( $fd_subscriber_messages ); + break; + case 'ping': + $fd_subscriber_messages = fopen( static::$subscriber_to_messages_path, 'c+' ); + flock( $fd_subscriber_messages, LOCK_EX ); + $subscriber_to_messages = static::get_contents_from_file_descriptor( $fd_subscriber_messages ); + if ( ! $subscriber_to_messages[ static::$subscriber_id ] ) { + $subscriber_to_messages[ static::$subscriber_id ] = array(); + } + $subscriber_to_messages[ static::$subscriber_id ][] = array( 'type' => 'pong' ); + static::save_contents_to_file_descriptor( $fd_subscriber_messages, $subscriber_to_messages ); + flock( $fd_subscriber_messages, LOCK_UN ); + fclose( $fd_subscriber_messages ); + break; + } + flock( $fd_topics_subscriber, LOCK_UN ); + fclose( $fd_topics_subscriber ); + echo wp_json_encode( array( 'result' => 'ok' ) ), PHP_EOL, PHP_EOL; + } + + /** + * Deletes messages and subcriber information of clients that have not interacted with the signaling server in a long time. + */ + private static function clean_up_old_connections() { + $fd_subscribers_last_connection = fopen( static::$subscribers_to_last_connection_path, 'c+' ); + flock( $fd_subscribers_last_connection, LOCK_EX ); + $subscribers_to_last_connection_time = static::get_contents_from_file_descriptor( $fd_subscribers_last_connection ); + $subscribers_to_last_connection_time[ static::$subscriber_id ] = time(); + $needs_cleanup = false; + foreach ( $subscribers_to_last_connection_time as $subscriber_id => $last_connection_time ) { + // cleanup connections older than 24 hours. + if ( $last_connection_time < time() - 24 * 60 * 60 ) { + unset( $subscribers_to_last_connection_time[ $subscriber_id ] ); + $needs_cleanup = true; + } + } + if ( $needs_cleanup ) { + $fd_subscriber_messages = fopen( static::$subscriber_to_messages_path, 'c+' ); + flock( $fd_subscriber_messages, LOCK_EX ); + $subscriber_to_messages = static::get_contents_from_file_descriptor( $fd_subscriber_messages ); + foreach ( $subscriber_to_messages as $subscriber_id => $messages ) { + if ( ! isset( $subscribers_to_last_connection_time[ static::$subscriber_id ] ) ) { + unset( $subscriber_to_messages[ $subscriber_id ] ); + } + } + static::save_contents_to_file_descriptor( $fd_subscriber_messages, $subscriber_to_messages ); + flock( $fd_subscriber_messages, LOCK_UN ); + fclose( $fd_subscriber_messages ); + + $fd_topics_subscriber = fopen( static::$topics_to_subscribers_path, 'c+' ); + flock( $fd_topics_subscriber, LOCK_EX ); + $topics_to_subscribers = static::get_contents_from_file_descriptor( $fd_topics_subscriber ); + foreach ( $topics_to_subscribers as $topic => $subscribers ) { + foreach ( $subscribers as $subscriber_id ) { + if ( ! isset( $subscribers_to_last_connection_time[ static::$subscriber_id ] ) ) { + $topics_to_subscribers[ $topic ] = array_diff( $topics_to_subscribers[ $topic ], array( static::$subscriber_id ) ); + } + } + } + static::save_contents_to_file_descriptor( $fd_topics_subscriber, $topics_to_subscribers ); + flock( $fd_topics_subscriber, LOCK_UN ); + fclose( $fd_topics_subscriber ); + } + + static::save_contents_to_file_descriptor( $fd_subscribers_last_connection, $subscribers_to_last_connection_time ); + flock( $fd_subscribers_last_connection, LOCK_UN ); + fclose( $fd_subscribers_last_connection ); + } + + +} + +Gutenberg_HTTP_Signaling_Server::init(); diff --git a/lib/sync/endpoint.php b/lib/sync/endpoint.php deleted file mode 100644 index 42c9afff00c3b..0000000000000 --- a/lib/sync/endpoint.php +++ /dev/null @@ -1,209 +0,0 @@ - array( 'message hello','handshake message' ) ). - - $topics_to_subscribers_path = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'topics_to_subscribers.txt'; - // Example: array( 'doc1: array( 2323232121 ), 'doc2: array( 2323232123, 2323232121 ) ). - - $subscribers_to_last_connection_path = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'subscribers_to_last_connection.txt'; - // Example: array( 2323232121 => 34343433323(timestamp) ). - - $subscriber_id = $_REQUEST['unique']; - if ( ! $subscriber_id ) { - die( 'no identifier' ); - } - - if ( 'GET' === $_SERVER['REQUEST_METHOD'] ) { - header( 'Content-Type: text/event-stream' ); - header( 'Cache-Control: no-cache' ); - echo 'retry: 3000' . PHP_EOL; - $fd = fopen( $subscriber_to_messages_path, 'c+' ); - flock( $fd, LOCK_EX ); - $subscriber_to_messages = _gutenberg_get_contents_from_file_descriptor( $fd ); - if ( isset( $subscriber_to_messages[ $subscriber_id ] ) && count( $subscriber_to_messages[ $subscriber_id ] ) > 0 ) { - $messages = array_map( 'json_encode', $subscriber_to_messages[ $subscriber_id ] ); - $subscriber_to_messages[ $subscriber_id ] = array(); - if ( count( $messages ) > 1 ) { - $data = implode( '|MULTIPLE|', $messages ); - } else { - $data = $messages[0]; - } - if ( $data ) { - echo 'id: ' . time() . PHP_EOL; - echo 'event: message' . PHP_EOL; - echo 'data: ' . $data . PHP_EOL . PHP_EOL; - } - _gutenberg_save_contents_to_file_descriptor( $fd, $subscriber_to_messages ); - - } else { - echo PHP_EOL; - } - flock( $fd, LOCK_UN ); - fclose( $fd ); - } else { - $raw_data = $_POST['data']; - $message = json_decode( wp_unslash( $raw_data ), true ); - if ( ! $message ) { - die( 'no message' ); - } - $fd_topics_subscriber = fopen( $topics_to_subscribers_path, 'c+' ); - flock( $fd_topics_subscriber, LOCK_EX ); - $topics_to_subscribers = _gutenberg_get_contents_from_file_descriptor( $fd_topics_subscriber ); - - switch ( $message['type'] ) { - case 'subscribe': - $topics = $message['topics']; - foreach ( $topics as $topic ) { - if ( ! $topics_to_subscribers[ $topic ] ) { - $topics_to_subscribers[ $topic ] = array(); - } - $topics_to_subscribers[ $topic ] = array_unique( array_merge( $topics_to_subscribers[ $topic ], array( $subscriber_id ) ) ); - } - _gutenberg_save_contents_to_file_descriptor( $fd_topics_subscriber, $topics_to_subscribers ); - break; - case 'unsubscribe': - $topics = $message['topics']; - foreach ( $topics as $topic ) { - if ( $topics_to_subscribers[ $topic ] ) { - $topics_to_subscribers[ $topic ] = array_diff( $topics_to_subscribers[ $topic ], array( $subscriber_id ) ); - } - } - _gutenberg_save_contents_to_file_descriptor( $fd_topics_subscriber, $topics_to_subscribers ); - break; - case 'publish': - $fd_subscriber_messages = fopen( $subscriber_to_messages_path, 'c+' ); - flock( $fd_subscriber_messages, LOCK_EX ); - $subscriber_to_messages = _gutenberg_get_contents_from_file_descriptor( $fd_subscriber_messages ); - $topic = $message['topic']; - $receivers = $topics_to_subscribers[ $topic ]; - if ( $receivers ) { - $message['clients'] = count( $receivers ); - foreach ( $receivers as $receiver ) { - if ( ! $subscriber_to_messages[ $receiver ] ) { - $subscriber_to_messages[ $receiver ] = array(); - } - $subscriber_to_messages[ $receiver ][] = $message; - } - _gutenberg_save_contents_to_file_descriptor( $fd_subscriber_messages, $subscriber_to_messages ); - } - flock( $fd_subscriber_messages, LOCK_UN ); - fclose( $fd_subscriber_messages ); - break; - case 'ping': - $fd_subscriber_messages = fopen( $subscriber_to_messages_path, 'c+' ); - flock( $fd_subscriber_messages, LOCK_EX ); - $subscriber_to_messages = _gutenberg_get_contents_from_file_descriptor( $fd_subscriber_messages ); - if ( ! $subscriber_to_messages[ $subscriber_id ] ) { - $subscriber_to_messages[ $subscriber_id ] = array(); - } - $subscriber_to_messages[ $subscriber_id ][] = array( 'type' => 'pong' ); - _gutenberg_save_contents_to_file_descriptor( $fd_subscriber_messages, $subscriber_to_messages ); - flock( $fd_subscriber_messages, LOCK_UN ); - fclose( $fd_subscriber_messages ); - break; - } - flock( $fd_topics_subscriber, LOCK_UN ); - fclose( $fd_topics_subscriber ); - echo json_encode( array( 'result' => 'ok' ) ), PHP_EOL, PHP_EOL; - } - - $fd_subscribers_last_connection = fopen( $subscribers_to_last_connection_path, 'c+' ); - flock( $fd_subscribers_last_connection, LOCK_EX ); - $subscribers_to_last_connection_time = _gutenberg_get_contents_from_file_descriptor( $fd_subscribers_last_connection ); - $subscribers_to_last_connection_time[ $subscriber_id ] = time(); - $needs_cleanup = false; - foreach ( $subscribers_to_last_connection_time as $subscriber_id => $last_connection_time ) { - // cleanup connections older than 24 hours. - if ( $last_connection_time < time() - 24 * 60 * 60 ) { - unset( $subscribers_to_last_connection_time[ $subscriber_id ] ); - $needs_cleanup = true; - } - } - if ( $needs_cleanup ) { - $fd_subscriber_messages = fopen( $subscriber_to_messages_path, 'c+' ); - flock( $fd_subscriber_messages, LOCK_EX ); - $subscriber_to_messages = _gutenberg_get_contents_from_file_descriptor( $fd_subscriber_messages ); - foreach ( $subscriber_to_messages as $subscriber_id => $messages ) { - if ( ! isset( $subscribers_to_last_connection_time[ $subscriber_id ] ) ) { - unset( $subscriber_to_messages[ $subscriber_id ] ); - } - } - _gutenberg_save_contents_to_file_descriptor( $fd_subscriber_messages, $subscriber_to_messages ); - flock( $fd_subscriber_messages, LOCK_UN ); - fclose( $fd_subscriber_messages ); - - $fd_topics_subscriber = fopen( $topics_to_subscribers_path, 'c+' ); - flock( $fd_topics_subscriber, LOCK_EX ); - $topics_to_subscribers = _gutenberg_get_contents_from_file_descriptor( $fd_topics_subscriber ); - foreach ( $topics_to_subscribers as $topic => $subscribers ) { - foreach ( $subscribers as $subscriber_id ) { - if ( ! isset( $subscribers_to_last_connection_time[ $subscriber_id ] ) ) { - $topics_to_subscribers[ $topic ] = array_diff( $topics_to_subscribers[ $topic ], array( $subscriber_id ) ); - } - } - } - _gutenberg_save_contents_to_file_descriptor( $fd_topics_subscriber, $topics_to_subscribers ); - flock( $fd_topics_subscriber, LOCK_UN ); - fclose( $fd_topics_subscriber ); - } - - _gutenberg_save_contents_to_file_descriptor( $fd_subscribers_last_connection, $subscribers_to_last_connection_time ); - flock( $fd_subscribers_last_connection, LOCK_UN ); - fclose( $fd_subscribers_last_connection ); - exit; -} - - - -add_action( 'wp_ajax_gutenberg_signaling_server', '_gutenberg_wp_ajax_signaling_server' ); From b6bdf0ec23c85e64dbecf3caaa86d0083606fc23 Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Wed, 13 Sep 2023 17:00:58 +0100 Subject: [PATCH 07/32] Fixed possible race condition and added file access error handling. --- .../class-gutenberg-http-signaling-server.php | 36 ++++++++++++++----- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/lib/experimental/sync/class-gutenberg-http-signaling-server.php b/lib/experimental/sync/class-gutenberg-http-signaling-server.php index 476683292eb46..44d5cb034d656 100644 --- a/lib/experimental/sync/class-gutenberg-http-signaling-server.php +++ b/lib/experimental/sync/class-gutenberg-http-signaling-server.php @@ -62,10 +62,6 @@ public static function init() { * Handles a wp_ajax signaling server request. */ public static function do_wp_ajax_action() { - error_reporting( E_ALL ); - ini_set( 'display_errors', '1' ); - session_write_close(); - static::initialize_paths(); if ( empty( $_REQUEST ) || empty( $_REQUEST['unique'] ) ) { @@ -149,6 +145,9 @@ private static function handle_message_read_request() { header( 'Cache-Control: no-cache' ); echo 'retry: 3000' . PHP_EOL; $fd = fopen( static::$subscriber_to_messages_path, 'c+' ); + if( ! $fd ) { + die( 'Could not open required file.' ); + } flock( $fd, LOCK_EX ); $subscriber_to_messages = static::get_contents_from_file_descriptor( $fd ); if ( isset( $subscriber_to_messages[ static::$subscriber_id ] ) && count( $subscriber_to_messages[ static::$subscriber_id ] ) > 0 ) { @@ -220,6 +219,9 @@ private static function unsubscribe_from_topics( $topics_to_subscribers, $topics */ private static function handle_message_operation( $message ) { $fd_topics_subscriber = fopen( static::$topics_to_subscribers_path, 'c+' ); + if( ! $fd_topics_subscriber ) { + die( 'Could not open required file.' ); + } flock( $fd_topics_subscriber, LOCK_EX ); $topics_to_subscribers = static::get_contents_from_file_descriptor( $fd_topics_subscriber ); @@ -232,6 +234,9 @@ private static function handle_message_operation( $message ) { break; case 'publish': $fd_subscriber_messages = fopen( static::$subscriber_to_messages_path, 'c+' ); + if( ! $fd_subscriber_messages ) { + die( 'Could not open required file.' ); + } flock( $fd_subscriber_messages, LOCK_EX ); $subscriber_to_messages = static::get_contents_from_file_descriptor( $fd_subscriber_messages ); $topic = $message['topic']; @@ -251,6 +256,9 @@ private static function handle_message_operation( $message ) { break; case 'ping': $fd_subscriber_messages = fopen( static::$subscriber_to_messages_path, 'c+' ); + if( ! $fd_subscriber_messages ) { + die( 'Could not open required file.' ); + } flock( $fd_subscriber_messages, LOCK_EX ); $subscriber_to_messages = static::get_contents_from_file_descriptor( $fd_subscriber_messages ); if ( ! $subscriber_to_messages[ static::$subscriber_id ] ) { @@ -272,6 +280,9 @@ private static function handle_message_operation( $message ) { */ private static function clean_up_old_connections() { $fd_subscribers_last_connection = fopen( static::$subscribers_to_last_connection_path, 'c+' ); + if( ! $fd_subscribers_last_connection ) { + die( 'Could not open required file.' ); + } flock( $fd_subscribers_last_connection, LOCK_EX ); $subscribers_to_last_connection_time = static::get_contents_from_file_descriptor( $fd_subscribers_last_connection ); $subscribers_to_last_connection_time[ static::$subscriber_id ] = time(); @@ -283,8 +294,13 @@ private static function clean_up_old_connections() { $needs_cleanup = true; } } + static::save_contents_to_file_descriptor( $fd_subscribers_last_connection, $subscribers_to_last_connection_time ); + if ( $needs_cleanup ) { $fd_subscriber_messages = fopen( static::$subscriber_to_messages_path, 'c+' ); + if( ! $fd_subscriber_messages ) { + die( 'Could not open required file.' ); + } flock( $fd_subscriber_messages, LOCK_EX ); $subscriber_to_messages = static::get_contents_from_file_descriptor( $fd_subscriber_messages ); foreach ( $subscriber_to_messages as $subscriber_id => $messages ) { @@ -293,10 +309,11 @@ private static function clean_up_old_connections() { } } static::save_contents_to_file_descriptor( $fd_subscriber_messages, $subscriber_to_messages ); - flock( $fd_subscriber_messages, LOCK_UN ); - fclose( $fd_subscriber_messages ); $fd_topics_subscriber = fopen( static::$topics_to_subscribers_path, 'c+' ); + if( ! $fd_topics_subscriber ) { + die( 'Could not open required file.' ); + } flock( $fd_topics_subscriber, LOCK_EX ); $topics_to_subscribers = static::get_contents_from_file_descriptor( $fd_topics_subscriber ); foreach ( $topics_to_subscribers as $topic => $subscribers ) { @@ -307,11 +324,14 @@ private static function clean_up_old_connections() { } } static::save_contents_to_file_descriptor( $fd_topics_subscriber, $topics_to_subscribers ); + + flock( $fd_subscriber_messages, LOCK_UN ); + fclose( $fd_subscriber_messages ); + flock( $fd_topics_subscriber, LOCK_UN ); fclose( $fd_topics_subscriber ); } - - static::save_contents_to_file_descriptor( $fd_subscribers_last_connection, $subscribers_to_last_connection_time ); + flock( $fd_subscribers_last_connection, LOCK_UN ); fclose( $fd_subscribers_last_connection ); } From d20d38bf4fcf3a70fba100fc8f8521abb394a44c Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Fri, 15 Sep 2023 18:42:43 +0100 Subject: [PATCH 08/32] Refactor to createWebRTCConnection. --- packages/core-data/src/sync.js | 13 ++++++- packages/sync/src/create-webrtc-connection.js | 39 +++++++++++++++++++ packages/sync/src/index.js | 2 +- 3 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 packages/sync/src/create-webrtc-connection.js diff --git a/packages/core-data/src/sync.js b/packages/core-data/src/sync.js index 89ebdd605208d..140bcd8f04a6b 100644 --- a/packages/core-data/src/sync.js +++ b/packages/core-data/src/sync.js @@ -4,14 +4,23 @@ import { createSyncProvider, connectIndexDb, - connectWebRTC, + createWebRTCConnection, } from '@wordpress/sync'; let syncProvider; export function getSyncProvider() { if ( ! syncProvider ) { - syncProvider = createSyncProvider( connectIndexDb, connectWebRTC ); + syncProvider = createSyncProvider( + connectIndexDb, + createWebRTCConnection( { + signaling: [ + //'ws://localhost:4444', + window.wp.ajax.settings.url, + ], + password: window.__experimentalCollaborativeEditingSecret, + } ) + ); } return syncProvider; diff --git a/packages/sync/src/create-webrtc-connection.js b/packages/sync/src/create-webrtc-connection.js new file mode 100644 index 0000000000000..8cd6c2c3b3b1e --- /dev/null +++ b/packages/sync/src/create-webrtc-connection.js @@ -0,0 +1,39 @@ +/** + * External dependencies + */ +// import { WebrtcProvider } from 'y-webrtc'; + +/** + * Internal dependencies + */ +import { WebrtcProviderWithHttpSignaling } from './webrtc-http-stream-signaling'; + +/** @typedef {import('./types').ObjectType} ObjectType */ +/** @typedef {import('./types').ObjectID} ObjectID */ +/** @typedef {import('./types').CRDTDoc} CRDTDoc */ + +/** + * Function that creates a new WebRTC Connection. + * + * @param {Object} config The object ID. + * + * @param {Array} config.signaling + * @param {string} config.password + * @return {Function} Promise that resolves when the connection is established. + */ +export function createWebRTCConnection( { signaling, password } ) { + return function ( + /** @type {string} */ objectId, + /** @type {string} */ objectType, + /** @type {import("yjs").Doc} */ doc + ) { + const docName = `${ objectType }-${ objectId }`; + new WebrtcProviderWithHttpSignaling( docName, doc, { + signaling, + // @ts-ignore + password, + } ); + + return Promise.resolve( () => true ); + }; +} diff --git a/packages/sync/src/index.js b/packages/sync/src/index.js index 975fb52989f5d..6c2b6899ffb61 100644 --- a/packages/sync/src/index.js +++ b/packages/sync/src/index.js @@ -1,3 +1,3 @@ export { connectIndexDb } from './connect-indexdb'; -export { connectWebRTC } from './connect-webrtc'; +export { createWebRTCConnection } from './create-webrtc-connection'; export { createSyncProvider } from './provider'; From fc44534f4705e4e2b17813595b598532d0e0d102 Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Fri, 15 Sep 2023 20:29:08 +0100 Subject: [PATCH 09/32] Rename unique to subscriber_id. And improve code sending messages. --- .../class-gutenberg-http-signaling-server.php | 8 ++++---- .../sync/src/webrtc-http-stream-signaling.js | 16 +++++++++------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/lib/experimental/sync/class-gutenberg-http-signaling-server.php b/lib/experimental/sync/class-gutenberg-http-signaling-server.php index 44d5cb034d656..ab1173df6264e 100644 --- a/lib/experimental/sync/class-gutenberg-http-signaling-server.php +++ b/lib/experimental/sync/class-gutenberg-http-signaling-server.php @@ -64,18 +64,18 @@ public static function init() { public static function do_wp_ajax_action() { static::initialize_paths(); - if ( empty( $_REQUEST ) || empty( $_REQUEST['unique'] ) ) { + if ( empty( $_REQUEST ) || empty( $_REQUEST['subscriber_id'] ) ) { die( 'no identifier' ); } - static::$subscriber_id = $_REQUEST['unique']; + static::$subscriber_id = $_REQUEST['subscriber_id']; if ( 'GET' === $_SERVER['REQUEST_METHOD'] ) { static::handle_message_read_request(); } else { - if ( empty( $_POST ) || empty( $_POST['data'] ) ) { + if ( empty( $_POST ) || empty( $_POST['message'] ) ) { die( 'no message' ); } - $message = json_decode( wp_unslash( $_POST['data'] ), true ); + $message = json_decode( wp_unslash( $_POST['message'] ), true ); if ( ! $message ) { die( 'no message' ); } diff --git a/packages/sync/src/webrtc-http-stream-signaling.js b/packages/sync/src/webrtc-http-stream-signaling.js index efad9af3039e6..de04e9140911f 100644 --- a/packages/sync/src/webrtc-http-stream-signaling.js +++ b/packages/sync/src/webrtc-http-stream-signaling.js @@ -179,11 +179,11 @@ function setupSignalEventHandlers( signalCon, url ) { function setupHttpSignal( httpClient ) { if ( httpClient.shouldConnect && httpClient.ws === null ) { // eslint-disable-next-line no-restricted-syntax - const unique = Math.floor( 100000 + Math.random() * 900000 ); + const subscriberId = Math.floor( 100000 + Math.random() * 900000 ); const url = httpClient.url; const eventSource = new window.EventSource( addQueryArgs( url, { - unique, + subscriber_id: subscriberId, action: 'gutenberg_signaling_server', } ) ); @@ -270,11 +270,13 @@ function setupHttpSignal( httpClient ) { } }; xhttp.open( 'POST', url, true ); - const dataToSend = new URLSearchParams(); - dataToSend.set( 'unique', unique.toString() ); - dataToSend.set( 'data', message ); - dataToSend.set( 'action', 'gutenberg_signaling_server' ); - xhttp.send( dataToSend ); + xhttp.send( + new URLSearchParams( { + subscriber_id: subscriberId.toString(), + message, + action: 'gutenberg_signaling_server', + } ) + ); }; } eventSource.onerror = () => { From 592fb910c3a92d6470da85a0c15e566e2d6e3b9b Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Fri, 15 Sep 2023 20:58:38 +0100 Subject: [PATCH 10/32] Remove use of an hack for multiple messages. --- .../class-gutenberg-http-signaling-server.php | 15 +++------------ .../sync/src/webrtc-http-stream-signaling.js | 16 ++++++---------- 2 files changed, 9 insertions(+), 22 deletions(-) diff --git a/lib/experimental/sync/class-gutenberg-http-signaling-server.php b/lib/experimental/sync/class-gutenberg-http-signaling-server.php index ab1173df6264e..5b3b216414273 100644 --- a/lib/experimental/sync/class-gutenberg-http-signaling-server.php +++ b/lib/experimental/sync/class-gutenberg-http-signaling-server.php @@ -151,20 +151,11 @@ private static function handle_message_read_request() { flock( $fd, LOCK_EX ); $subscriber_to_messages = static::get_contents_from_file_descriptor( $fd ); if ( isset( $subscriber_to_messages[ static::$subscriber_id ] ) && count( $subscriber_to_messages[ static::$subscriber_id ] ) > 0 ) { - $messages = array_map( 'wp_json_encode', $subscriber_to_messages[ static::$subscriber_id ] ); + echo 'id: ' . time() . PHP_EOL; + echo 'event: message' . PHP_EOL; + echo 'data: ' . wp_json_encode( $subscriber_to_messages[ static::$subscriber_id ] ) . PHP_EOL . PHP_EOL; $subscriber_to_messages[ static::$subscriber_id ] = array(); - if ( count( $messages ) > 1 ) { - $data = implode( '|MULTIPLE|', $messages ); - } else { - $data = $messages[0]; - } - if ( $data ) { - echo 'id: ' . time() . PHP_EOL; - echo 'event: message' . PHP_EOL; - echo 'data: ' . $data . PHP_EOL . PHP_EOL; - } static::save_contents_to_file_descriptor( $fd, $subscriber_to_messages ); - } else { echo PHP_EOL; } diff --git a/packages/sync/src/webrtc-http-stream-signaling.js b/packages/sync/src/webrtc-http-stream-signaling.js index de04e9140911f..877b29b7c84b3 100644 --- a/packages/sync/src/webrtc-http-stream-signaling.js +++ b/packages/sync/src/webrtc-http-stream-signaling.js @@ -194,22 +194,18 @@ function setupHttpSignal( httpClient ) { eventSource.onmessage = ( event ) => { httpClient.lastMessageReceived = Date.now(); const data = event.data; - - if ( data.includes( '|MULTIPLE|' ) ) { - const messages = data.split( '|MULTIPLE|' ); - messages.forEach( onSingleMessage ); - } else { - onSingleMessage( data ); + if ( data ) { + const messages = JSON.parse( data ); + if ( Array.isArray( messages ) ) { + messages.forEach( onSingleMessage ); + } } }; // @ts-ignore httpClient.ws = eventSource; httpClient.connecting = true; httpClient.connected = false; - const onSingleMessage = ( /** @type {string} */ data ) => { - const message = - typeof data === 'string' ? JSON.parse( data ) : data; - //console.log( 'receive', message ); + const onSingleMessage = ( /** @type {any} */ message ) => { if ( message && message.type === 'pong' ) { clearTimeout( pingTimeout ); pingTimeout = setTimeout( From fe554946850383e3dedc9c6d79d45b11afdf1eec Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Fri, 15 Sep 2023 23:04:41 +0100 Subject: [PATCH 11/32] Use fetch instead of XMLHttpRequest --- .../sync/src/webrtc-http-stream-signaling.js | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/packages/sync/src/webrtc-http-stream-signaling.js b/packages/sync/src/webrtc-http-stream-signaling.js index 877b29b7c84b3..14f8aa0de221b 100644 --- a/packages/sync/src/webrtc-http-stream-signaling.js +++ b/packages/sync/src/webrtc-http-stream-signaling.js @@ -253,26 +253,20 @@ function setupHttpSignal( httpClient ) { httpClient.ws.send = function send( /** @type {string} */ message ) { - //console.log( 'send', message ); - const xhttp = new window.XMLHttpRequest(); - xhttp.onreadystatechange = function () { - if ( this.readyState !== 4 ) { - return; - } - if ( this.status !== 200 ) { + window + .fetch( url, { + body: new URLSearchParams( { + subscriber_id: subscriberId.toString(), + action: 'gutenberg_signaling_server', + message, + } ), + method: 'POST', + } ) + .catch( () => { log( 'Error sending to server with message: ' + message ); - } - }; - xhttp.open( 'POST', url, true ); - xhttp.send( - new URLSearchParams( { - subscriber_id: subscriberId.toString(), - message, - action: 'gutenberg_signaling_server', - } ) - ); + } ); }; } eventSource.onerror = () => { From f48cfef790fe1c736d09ffcd48a23a383bc8997c Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Fri, 15 Sep 2023 23:05:04 +0100 Subject: [PATCH 12/32] Add documentation for the signalling server --- lib/experimental/sync/README.md | 175 ++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 lib/experimental/sync/README.md diff --git a/lib/experimental/sync/README.md b/lib/experimental/sync/README.md new file mode 100644 index 0000000000000..b20f0b06936f0 --- /dev/null +++ b/lib/experimental/sync/README.md @@ -0,0 +1,175 @@ +# Signaling Server Documentation + +The signaling server allows multiple clients to exchange messages with each other through various communication topics. Clients can subscribe and unsubscribe to these topics. +To retrieve messages, clients can send GET requests. They can also perform actions such as subscribing to topics, unsubscribing from topics, publishing messages in specific topics, or pinging the server by sending POST requests. +Every client must have a unique identifier (which can be randomly generated). This identifier should be included as a parameter named "subscriber_id" in every GET or POST request. + +## Sending messages to the server + +To send a message to the server, the client needs to send a POST request with its `subscriber_id` and the `message` to be sent. +The message should be a JSON string that includes its type, topics (or topic, depending on the type), and the data to be transmitted for messages of type publish. The type can be one of the following options: `subscribe`, `unsubscribe`, `publish`, `ping`, depending on the desired action the client wants to execute. +The details of each action are documented below. If the action is executed successfully by the server, the server will respond with `{"result":"ok"}`. + +### Subscribe to topics + +To subscribe to a set of topics, a client must send a POST request with the following parameters: + +- `subscriber_id`: Subscriber ID of the client. +- `message`: + - `type`: Should be set as `subscribe`. + - `topics`: An array of topics that the client is no longer interested in reading messages from, e.g., ["WordPress"]. + + +#### Sample request + +```js + +await ( + await fetch( 'http://localhost:7888/site-wp-dev/wp-admin/admin-ajax.php', { + body: new URLSearchParams( { + subscriber_id: '1', + action: 'gutenberg_signaling_server', + message: JSON.stringify( { + type: 'subscribe', + topics: [ 'WordPress', 'Drupal' ], + } ), + } ), + method: 'POST', + } ) + ).text(); + +``` + +### Publish a message + + + +To publish a message in a specific topic, a client must send a POST request with the following parameters: + +- `subscriber_id`: Subscriber ID of the client. +- `message`: + - `type`: Should be set as `publish`. + - `topic`: The topic where the message should be published, e.g., "WordPress". + - `data`: The data to be published to every client that subscribed to that topic. The data can be any string and may be encrypted to prevent the server from reading the messages. + + +#### Sample request + +```js + +await ( + await fetch( 'http://localhost:7888/site-wp-dev/wp-admin/admin-ajax.php', { + body: new URLSearchParams( { + subscriber_id: '1', + action: 'gutenberg_signaling_server', + message: JSON.stringify( { + type: 'publish', + topic: 'WordPress', + data: 'hello I am client 1!', + } ), + } ), + method: 'POST', + } ) + ).text(); + +``` + + + +### Unsubscribe to a set of topics + +To unsubscribe from a set of topics, a client must send a POST request with the following parameters: + +- `subscriber_id`: Subscriber ID of the client. +- `message`: + - `type`: Should be set as `unsubscribe`. + - `topics`: An array of topics that the client is no longer interested in reading messages from, e.g., ["WordPress", "Drupal"]. + + + + +#### Sample request + +```js +await ( + await fetch( 'http://localhost:7888/site-wp-dev/wp-admin/admin-ajax.php', { + body: new URLSearchParams( { + subscriber_id: '1', + action: 'gutenberg_signaling_server', + message: JSON.stringify( { + type: 'unsubscribe', + topics: [ 'WordPress', 'Drupal' ], + } ), + } ), + method: 'POST', + } ) + ).text(); +``` + + + +### Ping the server + + +To ensure that the server is listening and to indicate that a client is still alive, the client can periodically send a ping to the server. When the server receives a ping from a client, it will respond with a message containing `pong`. The client will receive this `pong` message when it asks the server for new messages. + +To send a ping, the client should send a message with the following parameters: +- `subscriber_id`: Subscriber ID of the client. +- `message`: + - `type`: Should be set as `ping`. + + +#### Sample request + +```js +await ( + await fetch( 'http://localhost:7888/site-wp-dev/wp-admin/admin-ajax.php', { + body: new URLSearchParams( { + subscriber_id: '1', + action: 'gutenberg_signaling_server', + message: JSON.stringify( { + type: 'ping', + } ), + } ), + method: 'POST', + } ) + ).text(); + +``` + + + +## Read messages + +In order for a client to read its pending messages, the client simply needs to initiate a GET request with its subscriber_id. The server responds with a content type of `text/event-stream;charset=UTF-8`, along with a retry value indicating the number of milliseconds after which the client should check again for new messages (e.g., `retry: 3000`). + +If there are no pending messages, the server's response ends there. However, if there are pending messages, the server continues by providing additional information in its response. This includes an 'id', which serves as a unique identifier for this response, an 'event' which, at present, is always set to 'message', and a 'data' field. The 'data' field is a JSON encoded string containing an array of messages that the given client is supposed to receive. Each message is similar to the published message object but includes an additional property named 'clients'. This property specifies the number of clients for which the message was sent (note: it does not indicate whether they have already received/requested it). + + +### Sample request + +```js +await ( + await fetch( + window.wp.url.addQueryArgs( window.wp.ajax.settings.url, { + subscriber_id: '1', + action: 'gutenberg_signaling_server', + } ) + ) + ).text(); +``` + + + +Sample answer when no messages are present: +``` +retry: 3000 +``` + +Sample answer when there are messages: +``` +retry: 3000 +id: 1694809781 +event: message +data: [{"type":"publish","topic":"WordPress","data":"hello I am client 1!","clients":2},{"type":"publish","topic":"WordPress","data":"Hi client 1 I am client 2","clients":2}] +`````` \ No newline at end of file From 5be652823a91a23f73e783aa64222b5139fac124 Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Mon, 18 Sep 2023 17:17:37 +0100 Subject: [PATCH 13/32] Add signalling server loading back --- lib/load.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/load.php b/lib/load.php index 13636b2587130..aa2f0d3f964ac 100644 --- a/lib/load.php +++ b/lib/load.php @@ -70,6 +70,11 @@ function gutenberg_is_experiment_enabled( $name ) { require_once __DIR__ . '/experimental/kses-allowed-html.php'; } +// Experimental signaling server. +if ( ! class_exists( 'Gutenberg_HTTP_Singling_Server' ) ) { + require_once __DIR__ . '/experimental/sync/class-gutenberg-http-signaling-server.php'; +} + require __DIR__ . '/experimental/editor-settings.php'; // Gutenberg plugin compat. From 762abd685fed13ece501627ac1cbfb893d8515f7 Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Mon, 18 Sep 2023 17:22:10 +0100 Subject: [PATCH 14/32] Remove hardcoded url from the docs. --- lib/experimental/sync/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/experimental/sync/README.md b/lib/experimental/sync/README.md index b20f0b06936f0..93e365e653221 100644 --- a/lib/experimental/sync/README.md +++ b/lib/experimental/sync/README.md @@ -25,7 +25,7 @@ To subscribe to a set of topics, a client must send a POST request with the foll ```js await ( - await fetch( 'http://localhost:7888/site-wp-dev/wp-admin/admin-ajax.php', { + await fetch( window.wp.ajax.settings.url, { body: new URLSearchParams( { subscriber_id: '1', action: 'gutenberg_signaling_server', @@ -58,7 +58,7 @@ To publish a message in a specific topic, a client must send a POST request with ```js await ( - await fetch( 'http://localhost:7888/site-wp-dev/wp-admin/admin-ajax.php', { + await fetch( window.wp.ajax.settings.url, { body: new URLSearchParams( { subscriber_id: '1', action: 'gutenberg_signaling_server', @@ -92,7 +92,7 @@ To unsubscribe from a set of topics, a client must send a POST request with the ```js await ( - await fetch( 'http://localhost:7888/site-wp-dev/wp-admin/admin-ajax.php', { + await fetch( window.wp.ajax.settings.url, { body: new URLSearchParams( { subscriber_id: '1', action: 'gutenberg_signaling_server', @@ -123,7 +123,7 @@ To send a ping, the client should send a message with the following parameters: ```js await ( - await fetch( 'http://localhost:7888/site-wp-dev/wp-admin/admin-ajax.php', { + await fetch( window.wp.ajax.settings.url, { body: new URLSearchParams( { subscriber_id: '1', action: 'gutenberg_signaling_server', From 11669ed6acbc69542a381920613a9ca2db9edf34 Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Thu, 21 Sep 2023 11:28:02 +0300 Subject: [PATCH 15/32] Updated sync readme. --- packages/sync/README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/sync/README.md b/packages/sync/README.md index 62fe20af4f2fb..f15d61b5a1eb5 100644 --- a/packages/sync/README.md +++ b/packages/sync/README.md @@ -28,32 +28,32 @@ _Returns_ - `Promise<() => void>`: Promise that resolves when the connection is established. -### connectWebRTC +### createSyncProvider -Connect function to the WebRTC provider. +Create a sync provider. _Parameters_ -- _objectId_ `ObjectID`: The object ID. -- _objectType_ `ObjectType`: The object type. -- _doc_ `CRDTDoc`: The CRDT document. +- _connectLocal_ `ConnectDoc`: Connect the document to a local database. +- _connectRemote_ `ConnectDoc`: Connect the document to a remote sync connection. _Returns_ -- `Promise<() => void>`: Promise that resolves when the connection is established. +- `SyncProvider`: Sync provider. -### createSyncProvider +### createWebRTCConnection -Create a sync provider. +Function that creates a new WebRTC Connection. _Parameters_ -- _connectLocal_ `ConnectDoc`: Connect the document to a local database. -- _connectRemote_ `ConnectDoc`: Connect the document to a remote sync connection. +- _config_ `Object`: The object ID. +- _config.signaling_ `Array`: +- _config.password_ `string`: _Returns_ -- `SyncProvider`: Sync provider. +- `Function`: Promise that resolves when the connection is established. From 584fe0e9b1574fce42d8615a7fbfddf7ffe38bfc Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Thu, 21 Sep 2023 12:00:16 +0300 Subject: [PATCH 16/32] Fix undefined error on test. --- packages/core-data/src/sync.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core-data/src/sync.js b/packages/core-data/src/sync.js index 140bcd8f04a6b..fdc421a6bd70e 100644 --- a/packages/core-data/src/sync.js +++ b/packages/core-data/src/sync.js @@ -16,9 +16,9 @@ export function getSyncProvider() { createWebRTCConnection( { signaling: [ //'ws://localhost:4444', - window.wp.ajax.settings.url, + window?.wp?.ajax?.settings?.url, ], - password: window.__experimentalCollaborativeEditingSecret, + password: window?.__experimentalCollaborativeEditingSecret, } ) ); } From 8eb88f73eab9859424ea5815e07f955e39b463ba Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Wed, 27 Sep 2023 16:49:54 +0100 Subject: [PATCH 17/32] Update package lock. --- package-lock.json | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/package-lock.json b/package-lock.json index 9f394be2011ab..ed33a0a19a48e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14735,6 +14735,14 @@ "@types/node": "*" } }, + "node_modules/@types/simple-peer": { + "version": "9.11.6", + "resolved": "https://registry.npmjs.org/@types/simple-peer/-/simple-peer-9.11.6.tgz", + "integrity": "sha512-6kD6yWzHJrVwx+N3RcC+5H9x/rZ7vRTDMrLl40eowvxXEKwvOW2W/ST5GlSQ/6j9yubSzaMj02LdNFTTfscQ3w==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/sockjs": { "version": "0.3.33", "resolved": "https://registry.npmjs.org/@types/sockjs/-/sockjs-0.3.33.tgz", @@ -33709,6 +33717,11 @@ "node": ">=8" } }, + "node_modules/import-locals": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/import-locals/-/import-locals-2.0.0.tgz", + "integrity": "sha512-1/bPE89IZhyf7dr5Pkz7b4UyVXy5pEt7PTEfye15UEn3AK8+2zwcDCfKk9Pwun4ltfhOSszOrReSsFcDKw/yoA==" + }, "node_modules/imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", @@ -58020,7 +58033,13 @@ "license": "GPL-2.0-or-later", "dependencies": { "@babel/runtime": "^7.16.0", + "@types/simple-peer": "^9.11.5", + "@wordpress/url": "file:../url", + "import-locals": "^2.0.0", + "lib0": "^0.2.42", + "simple-peer": "^9.11.0", "y-indexeddb": "~9.0.11", + "y-protocols": "^1.0.5", "y-webrtc": "~10.2.5", "yjs": "~13.6.6" }, @@ -68613,6 +68632,14 @@ "@types/node": "*" } }, + "@types/simple-peer": { + "version": "9.11.6", + "resolved": "https://registry.npmjs.org/@types/simple-peer/-/simple-peer-9.11.6.tgz", + "integrity": "sha512-6kD6yWzHJrVwx+N3RcC+5H9x/rZ7vRTDMrLl40eowvxXEKwvOW2W/ST5GlSQ/6j9yubSzaMj02LdNFTTfscQ3w==", + "requires": { + "@types/node": "*" + } + }, "@types/sockjs": { "version": "0.3.33", "resolved": "https://registry.npmjs.org/@types/sockjs/-/sockjs-0.3.33.tgz", @@ -70769,7 +70796,13 @@ "version": "file:packages/sync", "requires": { "@babel/runtime": "^7.16.0", + "@types/simple-peer": "^9.11.5", + "@wordpress/url": "file:../url", + "import-locals": "^2.0.0", + "lib0": "^0.2.42", + "simple-peer": "^9.11.0", "y-indexeddb": "~9.0.11", + "y-protocols": "^1.0.5", "y-webrtc": "~10.2.5", "yjs": "~13.6.6" } @@ -85970,6 +86003,11 @@ } } }, + "import-locals": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/import-locals/-/import-locals-2.0.0.tgz", + "integrity": "sha512-1/bPE89IZhyf7dr5Pkz7b4UyVXy5pEt7PTEfye15UEn3AK8+2zwcDCfKk9Pwun4ltfhOSszOrReSsFcDKw/yoA==" + }, "imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", From 0efb27e2f4a30aef1507473bd376e6a5fce3acfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Maneiro?= <583546+oandregal@users.noreply.github.com> Date: Fri, 29 Sep 2023 17:42:34 +0200 Subject: [PATCH 18/32] Document the Event stream format --- .../sync/class-gutenberg-http-signaling-server.php | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/experimental/sync/class-gutenberg-http-signaling-server.php b/lib/experimental/sync/class-gutenberg-http-signaling-server.php index 5b3b216414273..eda102155fbe4 100644 --- a/lib/experimental/sync/class-gutenberg-http-signaling-server.php +++ b/lib/experimental/sync/class-gutenberg-http-signaling-server.php @@ -139,6 +139,16 @@ private static function save_contents_to_file_descriptor( $fd, $content ) { /** * Handles a wp_ajax signaling server request of client that wants to retrieve its messages. + * + * It returns the client a response following the + * {@link https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#event_stream_format Event stream format}. + * + * ``` + * id: + * retry: + * event: + * data: + * ``` */ private static function handle_message_read_request() { header( 'Content-Type: text/event-stream' ); From 724b0e7ce75cc5fb7b71c263db1c36a0b68bbb74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Maneiro?= <583546+oandregal@users.noreply.github.com> Date: Fri, 29 Sep 2023 18:40:52 +0200 Subject: [PATCH 19/32] Update docs for signaling server --- lib/experimental/sync/README.md | 168 +++++++++++++++++--------------- 1 file changed, 87 insertions(+), 81 deletions(-) diff --git a/lib/experimental/sync/README.md b/lib/experimental/sync/README.md index 93e365e653221..cd764d2e7147a 100644 --- a/lib/experimental/sync/README.md +++ b/lib/experimental/sync/README.md @@ -1,94 +1,62 @@ # Signaling Server Documentation -The signaling server allows multiple clients to exchange messages with each other through various communication topics. Clients can subscribe and unsubscribe to these topics. -To retrieve messages, clients can send GET requests. They can also perform actions such as subscribing to topics, unsubscribing from topics, publishing messages in specific topics, or pinging the server by sending POST requests. -Every client must have a unique identifier (which can be randomly generated). This identifier should be included as a parameter named "subscriber_id" in every GET or POST request. +The signaling server allows multiple clients to exchange messages with each other through various communication topics. -## Sending messages to the server +Topics are not defined upfront, but clients define them by subscribing to them. By subscribing to a given topic, the client tells the server to keep track of its unread messages in the given topic. By unsubscribing from a topic, the client tells the server to free the bookeeping it maintains for the given client and topic. -To send a message to the server, the client needs to send a POST request with its `subscriber_id` and the `message` to be sent. -The message should be a JSON string that includes its type, topics (or topic, depending on the type), and the data to be transmitted for messages of type publish. The type can be one of the following options: `subscribe`, `unsubscribe`, `publish`, `ping`, depending on the desired action the client wants to execute. -The details of each action are documented below. If the action is executed successfully by the server, the server will respond with `{"result":"ok"}`. - -### Subscribe to topics +Every client communicates with the server via `GET` or `POST`. Clients must have a unique identifier, which can be randomly generated. This identifier should be included as a parameter named `subscriber_id` in every request. -To subscribe to a set of topics, a client must send a POST request with the following parameters: +Available operations: -- `subscriber_id`: Subscriber ID of the client. -- `message`: - - `type`: Should be set as `subscribe`. - - `topics`: An array of topics that the client is no longer interested in reading messages from, e.g., ["WordPress"]. - - -#### Sample request - -```js +- Subscribe to topics +- Unsubscribe from topics +- Publish a message +- Read pending messages +- Ping the server -await ( - await fetch( window.wp.ajax.settings.url, { - body: new URLSearchParams( { - subscriber_id: '1', - action: 'gutenberg_signaling_server', - message: JSON.stringify( { - type: 'subscribe', - topics: [ 'WordPress', 'Drupal' ], - } ), - } ), - method: 'POST', - } ) - ).text(); - -``` +## Subscribe to topics -### Publish a message - - - -To publish a message in a specific topic, a client must send a POST request with the following parameters: +To subscribe to a set of topics, a client must send a `POST` request with the following parameters: - `subscriber_id`: Subscriber ID of the client. -- `message`: - - `type`: Should be set as `publish`. - - `topic`: The topic where the message should be published, e.g., "WordPress". - - `data`: The data to be published to every client that subscribed to that topic. The data can be any string and may be encrypted to prevent the server from reading the messages. +- `action`: should be set to `gutenberg_signaling_server`. +- `message`: + - `type`: should be set to `subscribe`. + - `topics`: array of topics that the client is interested in reading messages from, e.g., `[ 'WordPress', 'Drupal' ]`. - -#### Sample request +If the action is executed successfully by the server, the server will respond with `{"result":"ok"}`. -```js +### Sample request +```js await ( await fetch( window.wp.ajax.settings.url, { body: new URLSearchParams( { subscriber_id: '1', action: 'gutenberg_signaling_server', message: JSON.stringify( { - type: 'publish', - topic: 'WordPress', - data: 'hello I am client 1!', + type: 'subscribe', + topics: [ 'WordPress', 'Drupal' ], } ), } ), method: 'POST', } ) ).text(); - ``` - - -### Unsubscribe to a set of topics +## Unsubscribe from topics -To unsubscribe from a set of topics, a client must send a POST request with the following parameters: +To unsubscribe from a set of topics, a client must send a `POST` request with the following parameters: -- `subscriber_id`: Subscriber ID of the client. +- `subscriber_id`: subscriber ID of the client. +- `action`: should be set to `gutenberg_signaling_server`. - `message`: - - `type`: Should be set as `unsubscribe`. - - `topics`: An array of topics that the client is no longer interested in reading messages from, e.g., ["WordPress", "Drupal"]. + - `type`: should be set as `unsubscribe`. + - `topics`: an array of topics that the client is no longer interested in reading messages from, e.g., `['WordPress', 'Drupal']`. - - +If the action is executed successfully by the server, the server will respond with `{"result":"ok"}`. -#### Sample request +### Sample request ```js await ( @@ -106,20 +74,20 @@ await ( ).text(); ``` - +## Publish a message -### Ping the server - - -To ensure that the server is listening and to indicate that a client is still alive, the client can periodically send a ping to the server. When the server receives a ping from a client, it will respond with a message containing `pong`. The client will receive this `pong` message when it asks the server for new messages. +To publish a message in a specific topic, a client must send a `POST` request with the following parameters: -To send a ping, the client should send a message with the following parameters: -- `subscriber_id`: Subscriber ID of the client. +- `subscriber_id`: subscriber ID of the client. +- `action`: should be set to `gutenberg_signaling_server`. - `message`: - - `type`: Should be set as `ping`. - + - `type`: should be set as `publish`. + - `topic`: the topic where the message should be published, e.g., `WordPress`. + - `data`: The data to be broadcasted to every client that subscribed to the topic. The data can be any string and may be encrypted to prevent the server from reading the messages. -#### Sample request +If the action is executed successfully by the server, the server will respond with `{"result":"ok"}`. + +### Sample request ```js await ( @@ -128,24 +96,32 @@ await ( subscriber_id: '1', action: 'gutenberg_signaling_server', message: JSON.stringify( { - type: 'ping', + type: 'publish', + topic: 'WordPress', + data: 'hello I am client 1!', } ), } ), method: 'POST', } ) ).text(); - ``` - +## Read pending messages -## Read messages +To read pending messages, the client should send a `GET` request with the following parameters: -In order for a client to read its pending messages, the client simply needs to initiate a GET request with its subscriber_id. The server responds with a content type of `text/event-stream;charset=UTF-8`, along with a retry value indicating the number of milliseconds after which the client should check again for new messages (e.g., `retry: 3000`). +- `subscriber_id`: Subscriber ID of the client. +- `action`: should be set to `gutenberg_signaling_server`. -If there are no pending messages, the server's response ends there. However, if there are pending messages, the server continues by providing additional information in its response. This includes an 'id', which serves as a unique identifier for this response, an 'event' which, at present, is always set to 'message', and a 'data' field. The 'data' field is a JSON encoded string containing an array of messages that the given client is supposed to receive. Each message is similar to the published message object but includes an additional property named 'clients'. This property specifies the number of clients for which the message was sent (note: it does not indicate whether they have already received/requested it). +The server will respond using the [Event stream format](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#event_stream_format), whose content type is set to `text/event-stream;charset=UTF-8`. The Event stream format defines the following fields: + +- `retry`: the reconnection time, in ms. The time after which the client should check again for messages. +- `id`: unique identifier for the server response. +- `event`: always set to `message`. +- `data`: a JSON encoded string containing an array of messages that the given client has not read yet. Each message is similar to the published message object but includes an additional property named `clients`. This property specifies the number of clients for which the message was sent. Note it does not indicate whether they have already received/requested it. + +If there are no pending messages, the server's response only contains the `retry:` field. If there are pending messages, the server will respond including all the fields. - ### Sample request ```js @@ -159,17 +135,47 @@ await ( ).text(); ``` - +Sample answer from the server when there are no unread messages: -Sample answer when no messages are present: ``` retry: 3000 ``` -Sample answer when there are messages: +Sample answer from the server when there are unread messages: + ``` retry: 3000 id: 1694809781 event: message data: [{"type":"publish","topic":"WordPress","data":"hello I am client 1!","clients":2},{"type":"publish","topic":"WordPress","data":"Hi client 1 I am client 2","clients":2}] -`````` \ No newline at end of file +``` + +## Ping the server + +To ensure that the server is listening and to indicate that a client is still alive, the client can periodically send a ping to the server. When the server receives a ping from a client, it will respond with a message containing `pong`. The client will receive this `pong` message when it asks the server for new messages. + +To send a ping, the client should send a `POST` request with the following parameters: + +- `subscriber_id`: Subscriber ID of the client. +- `action`: should be set to `gutenberg_signaling_server`. +- `message`: + - `type`: Should be set as `ping`. + +If the action is executed successfully by the server, the server will respond with `{"result":"ok"}`. + +#### Sample request + +```js +await ( + await fetch( window.wp.ajax.settings.url, { + body: new URLSearchParams( { + subscriber_id: '1', + action: 'gutenberg_signaling_server', + message: JSON.stringify( { + type: 'ping', + } ), + } ), + method: 'POST', + } ) + ).text(); +``` From f9a639a41da78ce6bad850fc26742390a9c425ef Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Fri, 6 Oct 2023 11:43:46 +0100 Subject: [PATCH 20/32] Update lib/experimental/sync/class-gutenberg-http-signaling-server.php MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: André <583546+oandregal@users.noreply.github.com> --- .../sync/class-gutenberg-http-signaling-server.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/experimental/sync/class-gutenberg-http-signaling-server.php b/lib/experimental/sync/class-gutenberg-http-signaling-server.php index eda102155fbe4..771ca99a2ba04 100644 --- a/lib/experimental/sync/class-gutenberg-http-signaling-server.php +++ b/lib/experimental/sync/class-gutenberg-http-signaling-server.php @@ -188,7 +188,8 @@ private static function subscribe_to_topics( $topics_to_subscribers, $topics ) { if ( ! $topics_to_subscribers[ $topic ] ) { $topics_to_subscribers[ $topic ] = array(); } - $topics_to_subscribers[ $topic ] = array_unique( array_merge( $topics_to_subscribers[ $topic ], array( static::$subscriber_id ) ) ); + $topics_to_subscribers[ $topic ][] = static::$subscriber_id; + $topics_to_subscribers[ $topic ] = array_unique( $topics_to_subscribers[ $topic ] ); } return $topics_to_subscribers; } From 8cc43445c78ccad162deb7d684df7fc8fdc81c54 Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Fri, 6 Oct 2023 15:22:49 +0100 Subject: [PATCH 21/32] refactor in progress --- .../class-gutenberg-http-signaling-server.php | 174 ++++++++++-------- 1 file changed, 96 insertions(+), 78 deletions(-) diff --git a/lib/experimental/sync/class-gutenberg-http-signaling-server.php b/lib/experimental/sync/class-gutenberg-http-signaling-server.php index 771ca99a2ba04..7aba02bd49680 100644 --- a/lib/experimental/sync/class-gutenberg-http-signaling-server.php +++ b/lib/experimental/sync/class-gutenberg-http-signaling-server.php @@ -62,15 +62,17 @@ public static function init() { * Handles a wp_ajax signaling server request. */ public static function do_wp_ajax_action() { - static::initialize_paths(); - if ( empty( $_REQUEST ) || empty( $_REQUEST['subscriber_id'] ) ) { die( 'no identifier' ); } + + static::initialize_paths(); + + static::$subscriber_id = $_REQUEST['subscriber_id']; if ( 'GET' === $_SERVER['REQUEST_METHOD'] ) { - static::handle_message_read_request(); + static::handle_read_pending_messages( static::$subscriber_id, static::$subscriber_to_messages_path ); } else { if ( empty( $_POST ) || empty( $_POST['message'] ) ) { die( 'no message' ); @@ -79,7 +81,63 @@ public static function do_wp_ajax_action() { if ( ! $message ) { die( 'no message' ); } - static::handle_message_operation( $message ); + + $fd_topics_subscriber = fopen( static::$topics_to_subscribers_path, 'c+' ); + if ( ! $fd_topics_subscriber ) { + die( 'Could not open required file.' ); + } + + flock( $fd_topics_subscriber, LOCK_EX ); + $topics_to_subscribers = static::get_contents_from_file_descriptor( $fd_topics_subscriber ); + + switch ( $message['type'] ) { + case 'subscribe': + static::handle_subscribe_to_topics( static::$topics_to_subscribers_path, $subscriber_id, $message['topics'] ); + break; + case 'unsubscribe': + static::handle_unsubscribe_from_topics( static::$topics_to_subscribers_path, $subscriber_id, $message['topics'] ); + break; + case 'publish': + $fd_subscriber_messages = fopen( static::$subscriber_to_messages_path, 'c+' ); + if( ! $fd_subscriber_messages ) { + die( 'Could not open required file.' ); + } + flock( $fd_subscriber_messages, LOCK_EX ); + $subscriber_to_messages = static::get_contents_from_file_descriptor( $fd_subscriber_messages ); + $topic = $message['topic']; + $receivers = $topics_to_subscribers[ $topic ]; + if ( $receivers ) { + $message['clients'] = count( $receivers ); + foreach ( $receivers as $receiver ) { + if ( ! $subscriber_to_messages[ $receiver ] ) { + $subscriber_to_messages[ $receiver ] = array(); + } + $subscriber_to_messages[ $receiver ][] = $message; + } + static::save_contents_to_file_descriptor( $fd_subscriber_messages, $subscriber_to_messages ); + } + flock( $fd_subscriber_messages, LOCK_UN ); + fclose( $fd_subscriber_messages ); + break; + case 'ping': + $fd_subscriber_messages = fopen( static::$subscriber_to_messages_path, 'c+' ); + if( ! $fd_subscriber_messages ) { + die( 'Could not open required file.' ); + } + flock( $fd_subscriber_messages, LOCK_EX ); + $subscriber_to_messages = static::get_contents_from_file_descriptor( $fd_subscriber_messages ); + if ( ! $subscriber_to_messages[ static::$subscriber_id ] ) { + $subscriber_to_messages[ static::$subscriber_id ] = array(); + } + $subscriber_to_messages[ static::$subscriber_id ][] = array( 'type' => 'pong' ); + static::save_contents_to_file_descriptor( $fd_subscriber_messages, $subscriber_to_messages ); + flock( $fd_subscriber_messages, LOCK_UN ); + fclose( $fd_subscriber_messages ); + break; + } + flock( $fd_topics_subscriber, LOCK_UN ); + fclose( $fd_topics_subscriber ); + echo wp_json_encode( array( 'result' => 'ok' ) ), PHP_EOL, PHP_EOL; } static::clean_up_old_connections(); @@ -100,6 +158,21 @@ private static function initialize_paths() { // Example: array( 2323232121 => 34343433323(timestamp) ). } + private static function get_contents_and_lock_file( $path ) { + $fd = fopen( $path, 'c+' ); + if ( ! $fd ) { + return array( $fd, null ); + } + flock( $fd, LOCK_EX ); + return array( $fd, static::get_contents_from_file_descriptor( $fd ) ); + } + + private static function save_contents_and_unlock_file( $fd, $content ) { + static::save_contents_to_file_descriptor( $fd, $content ); + flock( $fd, LOCK_UN ); + fclose( $fd ); + } + /** * Reads the contents of $fd and returns them unserialized. * @@ -150,21 +223,21 @@ private static function save_contents_to_file_descriptor( $fd, $content ) { * data: * ``` */ - private static function handle_message_read_request() { + private static function handle_read_pending_messages( $subscriber_id, $subscriber_to_messages_path ) { header( 'Content-Type: text/event-stream' ); header( 'Cache-Control: no-cache' ); echo 'retry: 3000' . PHP_EOL; - $fd = fopen( static::$subscriber_to_messages_path, 'c+' ); - if( ! $fd ) { + $fd = fopen( $subscriber_to_messages_path, 'c+' ); + if ( ! $fd ) { die( 'Could not open required file.' ); } flock( $fd, LOCK_EX ); $subscriber_to_messages = static::get_contents_from_file_descriptor( $fd ); - if ( isset( $subscriber_to_messages[ static::$subscriber_id ] ) && count( $subscriber_to_messages[ static::$subscriber_id ] ) > 0 ) { + if ( isset( $subscriber_to_messages[ $subscriber_id ] ) && count( $subscriber_to_messages[ $subscriber_id ] ) > 0 ) { echo 'id: ' . time() . PHP_EOL; echo 'event: message' . PHP_EOL; - echo 'data: ' . wp_json_encode( $subscriber_to_messages[ static::$subscriber_id ] ) . PHP_EOL . PHP_EOL; - $subscriber_to_messages[ static::$subscriber_id ] = array(); + echo 'data: ' . wp_json_encode( $subscriber_to_messages[ $subscriber_id ] ) . PHP_EOL . PHP_EOL; + $subscriber_to_messages[ $subscriber_id ] = array(); static::save_contents_to_file_descriptor( $fd, $subscriber_to_messages ); } else { echo PHP_EOL; @@ -183,15 +256,19 @@ private static function handle_message_read_request() { * @param array $topics_to_subscribers Topics to subscribers data-structure. * @param array $topics An array of topics e.g: array( 'doc1', 'doc2' ). */ - private static function subscribe_to_topics( $topics_to_subscribers, $topics ) { + private static function handle_subscribe_to_topics( $topics_to_subscribers_path, $subscriber_id, $topics ) { + list( $fd, $topics_to_subscribers ) = static::get_contents_and_lock_file( $topics_to_subscribers_path ); + if ( ! $fd ) { + die( 'Could not open required file.' ); + } foreach ( $topics as $topic ) { if ( ! $topics_to_subscribers[ $topic ] ) { $topics_to_subscribers[ $topic ] = array(); } $topics_to_subscribers[ $topic ][] = static::$subscriber_id; - $topics_to_subscribers[ $topic ] = array_unique( $topics_to_subscribers[ $topic ] ); + $topics_to_subscribers[ $topic ] = array_unique( $topics_to_subscribers[ $topic ] ); } - return $topics_to_subscribers; + static::save_contents_and_unlock_file( $fd, $topics_to_subscribers ); } /** @@ -204,78 +281,19 @@ private static function subscribe_to_topics( $topics_to_subscribers, $topics ) { * @param array $topics_to_subscribers Topics to subscribers data-structure. * @param array $topics An array of topics e.g: array( 'doc1', 'doc2' ). */ - private static function unsubscribe_from_topics( $topics_to_subscribers, $topics ) { + private static function handle_unsubscribe_from_topics( $topics_to_subscribers_path, $subscriber_id, $topics ) { + list( $fd, $topics_to_subscribers ) = static::get_contents_and_lock_file( $topics_to_subscribers_path ); + if ( ! $fd ) { + die( 'Could not open required file.' ); + } foreach ( $topics as $topic ) { if ( $topics_to_subscribers[ $topic ] ) { $topics_to_subscribers[ $topic ] = array_diff( $topics_to_subscribers[ $topic ], array( static::$subscriber_id ) ); } } - return $topics_to_subscribers; + static::save_contents_and_unlock_file( $fd, $topics_to_subscribers ); } - /** - * Handles a wp_ajax signaling server request of client that is performing an operation. - * An operation can be a ping to say the client is alive, sending a message, or subscribing/unscribing to a set of topics. - * - * @param array $message An array of topics e.g: array( 'doc1', 'doc2' ). - */ - private static function handle_message_operation( $message ) { - $fd_topics_subscriber = fopen( static::$topics_to_subscribers_path, 'c+' ); - if( ! $fd_topics_subscriber ) { - die( 'Could not open required file.' ); - } - flock( $fd_topics_subscriber, LOCK_EX ); - $topics_to_subscribers = static::get_contents_from_file_descriptor( $fd_topics_subscriber ); - - switch ( $message['type'] ) { - case 'subscribe': - static::save_contents_to_file_descriptor( $fd_topics_subscriber, static::subscribe_to_topics( $topics_to_subscribers, $message['topics'] ) ); - break; - case 'unsubscribe': - static::save_contents_to_file_descriptor( $fd_topics_subscriber, static::unsubscribe_from_topics( $topics_to_subscribers, $message['topics'] ) ); - break; - case 'publish': - $fd_subscriber_messages = fopen( static::$subscriber_to_messages_path, 'c+' ); - if( ! $fd_subscriber_messages ) { - die( 'Could not open required file.' ); - } - flock( $fd_subscriber_messages, LOCK_EX ); - $subscriber_to_messages = static::get_contents_from_file_descriptor( $fd_subscriber_messages ); - $topic = $message['topic']; - $receivers = $topics_to_subscribers[ $topic ]; - if ( $receivers ) { - $message['clients'] = count( $receivers ); - foreach ( $receivers as $receiver ) { - if ( ! $subscriber_to_messages[ $receiver ] ) { - $subscriber_to_messages[ $receiver ] = array(); - } - $subscriber_to_messages[ $receiver ][] = $message; - } - static::save_contents_to_file_descriptor( $fd_subscriber_messages, $subscriber_to_messages ); - } - flock( $fd_subscriber_messages, LOCK_UN ); - fclose( $fd_subscriber_messages ); - break; - case 'ping': - $fd_subscriber_messages = fopen( static::$subscriber_to_messages_path, 'c+' ); - if( ! $fd_subscriber_messages ) { - die( 'Could not open required file.' ); - } - flock( $fd_subscriber_messages, LOCK_EX ); - $subscriber_to_messages = static::get_contents_from_file_descriptor( $fd_subscriber_messages ); - if ( ! $subscriber_to_messages[ static::$subscriber_id ] ) { - $subscriber_to_messages[ static::$subscriber_id ] = array(); - } - $subscriber_to_messages[ static::$subscriber_id ][] = array( 'type' => 'pong' ); - static::save_contents_to_file_descriptor( $fd_subscriber_messages, $subscriber_to_messages ); - flock( $fd_subscriber_messages, LOCK_UN ); - fclose( $fd_subscriber_messages ); - break; - } - flock( $fd_topics_subscriber, LOCK_UN ); - fclose( $fd_topics_subscriber ); - echo wp_json_encode( array( 'result' => 'ok' ) ), PHP_EOL, PHP_EOL; - } /** * Deletes messages and subcriber information of clients that have not interacted with the signaling server in a long time. From 3e2e03b385a2ab1f0913b4c7cca3d1055502d15d Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Mon, 9 Oct 2023 14:30:47 +0100 Subject: [PATCH 22/32] tmp --- .../class-gutenberg-http-signaling-server.php | 222 ++++++++---------- 1 file changed, 98 insertions(+), 124 deletions(-) diff --git a/lib/experimental/sync/class-gutenberg-http-signaling-server.php b/lib/experimental/sync/class-gutenberg-http-signaling-server.php index 7aba02bd49680..4c2a7c2b3397f 100644 --- a/lib/experimental/sync/class-gutenberg-http-signaling-server.php +++ b/lib/experimental/sync/class-gutenberg-http-signaling-server.php @@ -1,4 +1,7 @@ array( 'message hello','handshake message' ) ). - * - * @access private - * @var array - */ - private static $subscriber_to_messages_path; - - /** - * Contains the path of the topics to subscriber file. - * The file contains a data-structure similar to the following example: - * array( 'doc1: array( 2323232121 ), 'doc2: array( 2323232123, 2323232121 ) ). - * - * @access private - * @var array - */ - private static $topics_to_subscribers_path; - - /** - * Contains the path of the subscribers to last connection file. - * The file contains a data-structure similar to the following example: - * array( 2323232121 => 34343433323(timestamp) ). - * - * @access private - * @var array - */ - private static $subscribers_to_last_connection_path; - - /** - * Contains the subscriber id of the client reading or sending messages. - * - * @access private - * @var Integer - */ - private static $subscriber_id; /** * Adds a wp_ajax action to handle the signaling server requests. @@ -66,13 +32,18 @@ public static function do_wp_ajax_action() { die( 'no identifier' ); } - static::initialize_paths(); + // Contains the subscriber id of the client reading or sending messages. + $subscriber_id = $_REQUEST['subscriber_id']; + // Example inside file: array( 2323232121 => array( 'message hello','handshake message' ) ). + $subscriber_to_messages_path = get_temp_dir() . DIRECTORY_SEPARATOR . 'subscribers_to_messages.txt'; + + // Example inside file: array( 'doc1: array( 2323232121 ), 'doc2: array( 2323232123, 2323232121 ) ). + $topics_to_subscribers_path = get_temp_dir() . DIRECTORY_SEPARATOR . 'topics_to_subscribers.txt'; - static::$subscriber_id = $_REQUEST['subscriber_id']; if ( 'GET' === $_SERVER['REQUEST_METHOD'] ) { - static::handle_read_pending_messages( static::$subscriber_id, static::$subscriber_to_messages_path ); + static::handle_read_pending_messages( $subscriber_id, $subscriber_to_messages_path ); } else { if ( empty( $_POST ) || empty( $_POST['message'] ) ) { die( 'no message' ); @@ -82,97 +53,27 @@ public static function do_wp_ajax_action() { die( 'no message' ); } - $fd_topics_subscriber = fopen( static::$topics_to_subscribers_path, 'c+' ); - if ( ! $fd_topics_subscriber ) { - die( 'Could not open required file.' ); - } - - flock( $fd_topics_subscriber, LOCK_EX ); - $topics_to_subscribers = static::get_contents_from_file_descriptor( $fd_topics_subscriber ); - switch ( $message['type'] ) { case 'subscribe': - static::handle_subscribe_to_topics( static::$topics_to_subscribers_path, $subscriber_id, $message['topics'] ); + static::handle_subscribe_to_topics( $topics_to_subscribers_path, $subscriber_id, $message['topics'] ); break; case 'unsubscribe': - static::handle_unsubscribe_from_topics( static::$topics_to_subscribers_path, $subscriber_id, $message['topics'] ); + static::handle_unsubscribe_from_topics( $topics_to_subscribers_path, $subscriber_id, $message['topics'] ); break; case 'publish': - $fd_subscriber_messages = fopen( static::$subscriber_to_messages_path, 'c+' ); - if( ! $fd_subscriber_messages ) { - die( 'Could not open required file.' ); - } - flock( $fd_subscriber_messages, LOCK_EX ); - $subscriber_to_messages = static::get_contents_from_file_descriptor( $fd_subscriber_messages ); - $topic = $message['topic']; - $receivers = $topics_to_subscribers[ $topic ]; - if ( $receivers ) { - $message['clients'] = count( $receivers ); - foreach ( $receivers as $receiver ) { - if ( ! $subscriber_to_messages[ $receiver ] ) { - $subscriber_to_messages[ $receiver ] = array(); - } - $subscriber_to_messages[ $receiver ][] = $message; - } - static::save_contents_to_file_descriptor( $fd_subscriber_messages, $subscriber_to_messages ); - } - flock( $fd_subscriber_messages, LOCK_UN ); - fclose( $fd_subscriber_messages ); + static::handle_publish_message( static::$topics_to_subscribers_path, $subscriber_id, $message ); break; case 'ping': - $fd_subscriber_messages = fopen( static::$subscriber_to_messages_path, 'c+' ); - if( ! $fd_subscriber_messages ) { - die( 'Could not open required file.' ); - } - flock( $fd_subscriber_messages, LOCK_EX ); - $subscriber_to_messages = static::get_contents_from_file_descriptor( $fd_subscriber_messages ); - if ( ! $subscriber_to_messages[ static::$subscriber_id ] ) { - $subscriber_to_messages[ static::$subscriber_id ] = array(); - } - $subscriber_to_messages[ static::$subscriber_id ][] = array( 'type' => 'pong' ); - static::save_contents_to_file_descriptor( $fd_subscriber_messages, $subscriber_to_messages ); - flock( $fd_subscriber_messages, LOCK_UN ); - fclose( $fd_subscriber_messages ); + static::handle_ping( $subscriber_to_messages_path, $subscriber_id ); break; } - flock( $fd_topics_subscriber, LOCK_UN ); - fclose( $fd_topics_subscriber ); echo wp_json_encode( array( 'result' => 'ok' ) ), PHP_EOL, PHP_EOL; } - static::clean_up_old_connections(); + static::clean_up_old_connections( $subscriber_id, $subscriber_to_messages_path, $topics_to_subscribers_path ); exit; } - /** - * Initializes the paths of the temporary files used. - */ - private static function initialize_paths() { - static::$subscriber_to_messages_path = get_temp_dir() . DIRECTORY_SEPARATOR . 'subscribers_to_messages.txt'; - // Example: array( 2323232121 => array( 'message hello','handshake message' ) ). - - static::$topics_to_subscribers_path = get_temp_dir() . DIRECTORY_SEPARATOR . 'topics_to_subscribers.txt'; - // Example: array( 'doc1: array( 2323232121 ), 'doc2: array( 2323232123, 2323232121 ) ). - - static::$subscribers_to_last_connection_path = get_temp_dir() . DIRECTORY_SEPARATOR . 'subscribers_to_last_connection.txt'; - // Example: array( 2323232121 => 34343433323(timestamp) ). - } - - private static function get_contents_and_lock_file( $path ) { - $fd = fopen( $path, 'c+' ); - if ( ! $fd ) { - return array( $fd, null ); - } - flock( $fd, LOCK_EX ); - return array( $fd, static::get_contents_from_file_descriptor( $fd ) ); - } - - private static function save_contents_and_unlock_file( $fd, $content ) { - static::save_contents_to_file_descriptor( $fd, $content ); - flock( $fd, LOCK_UN ); - fclose( $fd ); - } - /** * Reads the contents of $fd and returns them unserialized. * @@ -192,6 +93,20 @@ private static function get_contents_from_file_descriptor( $fd ) { return $result; } + private static function get_contents_and_lock_file( $path ) { + $fd = fopen( $path, 'c+' ); + if ( ! $fd ) { + return array( $fd, null ); + } + flock( $fd, LOCK_EX ); + return array( $fd, static::get_contents_from_file_descriptor( $fd ) ); + } + + private static function save_contents_and_unlock_file( $fd, $content ) { + static::save_contents_to_file_descriptor( $fd, $content ); + flock( $fd, LOCK_UN ); + fclose( $fd ); + } /** * Makes the file descriptor content of $fd equal to the serialization of content. @@ -294,18 +209,77 @@ private static function handle_unsubscribe_from_topics( $topics_to_subscribers_p static::save_contents_and_unlock_file( $fd, $topics_to_subscribers ); } + /** + * Receives a $topics_to_subscribers data-structure and an array of topics, + * and returns a new $topics_to_subscribers data-structure where the current subscriber is subscribed to the topics. + * + * @access private + * @internal + * + * @param array $topics_to_subscribers Topics to subscribers data-structure. + * @param array $topics An array of topics e.g: array( 'doc1', 'doc2' ). + */ + private static function handle_publish_message( $topics_to_subscribers_path, $subscriber_to_messages_path, $subscriber_id, $message ) { + list( $fd_topics_subscriber, $topics_to_subscribers ) = static::get_contents_and_lock_file( $topics_to_subscribers_path ); + list( $fd_subscriber_to_messages, $subscriber_to_messages ) = static::get_contents_and_lock_file( $subscriber_to_messages_path ); + if ( ! $fd_topics_subscriber || ! $fd_subscriber_to_messages ) { + die( 'Could not open required file.' ); + } + $topic = $message['topic']; + $receivers = $topics_to_subscribers[ $topic ]; + if ( $receivers && count( $receivers ) > 0 ) { + $message['clients'] = count( $receivers ); + foreach ( $receivers as $receiver ) { + if ( ! $subscriber_to_messages[ $receiver ] ) { + $subscriber_to_messages[ $receiver ] = array(); + } + $subscriber_to_messages[ $receiver ][] = $message; + } + static::save_contents_to_file_descriptor( $fd_subscriber_to_messages, $subscriber_to_messages ); + } + flock( $fd_subscriber_messages, LOCK_UN ); + fclose( $fd_subscriber_messages ); + flock( $fd_topics_subscriber, LOCK_UN ); + fclose( $fd_topics_subscriber ); + } /** - * Deletes messages and subcriber information of clients that have not interacted with the signaling server in a long time. + * Receives a $topics_to_subscribers data-structure and an array of topics, + * and returns a new $topics_to_subscribers data-structure where the current subscriber is subscribed to the topics. + * + * @access private + * @internal + * + * @param array $topics_to_subscribers Topics to subscribers data-structure. + * @param array $topics An array of topics e.g: array( 'doc1', 'doc2' ). + */ + private static function handle_ping( $subscriber_to_messages_path, $subscriber_id ) { + list( $fd_subscriber_to_messages, $subscriber_to_messages ) = static::get_contents_and_lock_file( $subscriber_to_messages_path ); + if ( ! $fd_subscriber_to_messages ) { + die( 'Could not open required file.' ); + } + if ( ! $subscriber_to_messages[ static::$subscriber_id ] ) { + $subscriber_to_messages[ $subscriber_id ] = array(); + } + $subscriber_to_messages[ $subscriber_id ][] = array( 'type' => 'pong' ); + static::save_contents_and_unlock_file( $fd_subscriber_to_messages, $subscriber_to_messages ); + } + + + /** + * Deletes messages and subscriber information of clients that have not interacted with the signaling server in a long time. */ - private static function clean_up_old_connections() { - $fd_subscribers_last_connection = fopen( static::$subscribers_to_last_connection_path, 'c+' ); + private static function clean_up_old_connections( $connected_subscriber_id, $subscriber_to_messages_path, $topics_to_subscribers_path ) { + $subscribers_to_last_connection_path = get_temp_dir() . DIRECTORY_SEPARATOR . 'subscribers_to_last_connection.txt'; + // Example: array( 2323232121 => 34343433323(timestamp) ). + + $fd_subscribers_last_connection = fopen( $subscribers_to_last_connection_path, 'c+' ); if( ! $fd_subscribers_last_connection ) { die( 'Could not open required file.' ); } flock( $fd_subscribers_last_connection, LOCK_EX ); $subscribers_to_last_connection_time = static::get_contents_from_file_descriptor( $fd_subscribers_last_connection ); - $subscribers_to_last_connection_time[ static::$subscriber_id ] = time(); + $subscribers_to_last_connection_time[ $connected_subscriber_id ] = time(); $needs_cleanup = false; foreach ( $subscribers_to_last_connection_time as $subscriber_id => $last_connection_time ) { // cleanup connections older than 24 hours. @@ -317,20 +291,20 @@ private static function clean_up_old_connections() { static::save_contents_to_file_descriptor( $fd_subscribers_last_connection, $subscribers_to_last_connection_time ); if ( $needs_cleanup ) { - $fd_subscriber_messages = fopen( static::$subscriber_to_messages_path, 'c+' ); + $fd_subscriber_messages = fopen( $subscriber_to_messages_path, 'c+' ); if( ! $fd_subscriber_messages ) { die( 'Could not open required file.' ); } flock( $fd_subscriber_messages, LOCK_EX ); $subscriber_to_messages = static::get_contents_from_file_descriptor( $fd_subscriber_messages ); foreach ( $subscriber_to_messages as $subscriber_id => $messages ) { - if ( ! isset( $subscribers_to_last_connection_time[ static::$subscriber_id ] ) ) { + if ( ! isset( $subscribers_to_last_connection_time[ $subscriber_id ] ) ) { unset( $subscriber_to_messages[ $subscriber_id ] ); } } static::save_contents_to_file_descriptor( $fd_subscriber_messages, $subscriber_to_messages ); - $fd_topics_subscriber = fopen( static::$topics_to_subscribers_path, 'c+' ); + $fd_topics_subscriber = fopen( $topics_to_subscribers_path, 'c+' ); if( ! $fd_topics_subscriber ) { die( 'Could not open required file.' ); } @@ -338,8 +312,8 @@ private static function clean_up_old_connections() { $topics_to_subscribers = static::get_contents_from_file_descriptor( $fd_topics_subscriber ); foreach ( $topics_to_subscribers as $topic => $subscribers ) { foreach ( $subscribers as $subscriber_id ) { - if ( ! isset( $subscribers_to_last_connection_time[ static::$subscriber_id ] ) ) { - $topics_to_subscribers[ $topic ] = array_diff( $topics_to_subscribers[ $topic ], array( static::$subscriber_id ) ); + if ( ! isset( $subscribers_to_last_connection_time[ $subscriber_id ] ) ) { + $topics_to_subscribers[ $topic ] = array_diff( $topics_to_subscribers[ $topic ], array( $subscriber_id ) ); } } } From 08390dfcafa4dc8970da0642ca28fb13c7f47f92 Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Tue, 10 Oct 2023 17:13:26 +0100 Subject: [PATCH 23/32] fix some bugs after refactoring --- .../class-gutenberg-http-signaling-server.php | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/experimental/sync/class-gutenberg-http-signaling-server.php b/lib/experimental/sync/class-gutenberg-http-signaling-server.php index 4c2a7c2b3397f..03c7f9dbc79ce 100644 --- a/lib/experimental/sync/class-gutenberg-http-signaling-server.php +++ b/lib/experimental/sync/class-gutenberg-http-signaling-server.php @@ -61,7 +61,7 @@ public static function do_wp_ajax_action() { static::handle_unsubscribe_from_topics( $topics_to_subscribers_path, $subscriber_id, $message['topics'] ); break; case 'publish': - static::handle_publish_message( static::$topics_to_subscribers_path, $subscriber_id, $message ); + static::handle_publish_message( $topics_to_subscribers_path, $subscriber_to_messages_path, $subscriber_id, $message ); break; case 'ping': static::handle_ping( $subscriber_to_messages_path, $subscriber_id ); @@ -177,10 +177,10 @@ private static function handle_subscribe_to_topics( $topics_to_subscribers_path, die( 'Could not open required file.' ); } foreach ( $topics as $topic ) { - if ( ! $topics_to_subscribers[ $topic ] ) { + if ( ! isset( $topics_to_subscribers[ $topic ] ) ) { $topics_to_subscribers[ $topic ] = array(); } - $topics_to_subscribers[ $topic ][] = static::$subscriber_id; + $topics_to_subscribers[ $topic ][] = $subscriber_id; $topics_to_subscribers[ $topic ] = array_unique( $topics_to_subscribers[ $topic ] ); } static::save_contents_and_unlock_file( $fd, $topics_to_subscribers ); @@ -203,7 +203,7 @@ private static function handle_unsubscribe_from_topics( $topics_to_subscribers_p } foreach ( $topics as $topic ) { if ( $topics_to_subscribers[ $topic ] ) { - $topics_to_subscribers[ $topic ] = array_diff( $topics_to_subscribers[ $topic ], array( static::$subscriber_id ) ); + $topics_to_subscribers[ $topic ] = array_diff( $topics_to_subscribers[ $topic ], array( $subscriber_id ) ); } } static::save_contents_and_unlock_file( $fd, $topics_to_subscribers ); @@ -220,7 +220,7 @@ private static function handle_unsubscribe_from_topics( $topics_to_subscribers_p * @param array $topics An array of topics e.g: array( 'doc1', 'doc2' ). */ private static function handle_publish_message( $topics_to_subscribers_path, $subscriber_to_messages_path, $subscriber_id, $message ) { - list( $fd_topics_subscriber, $topics_to_subscribers ) = static::get_contents_and_lock_file( $topics_to_subscribers_path ); + list( $fd_topics_subscriber, $topics_to_subscribers ) = static::get_contents_and_lock_file( $topics_to_subscribers_path ); list( $fd_subscriber_to_messages, $subscriber_to_messages ) = static::get_contents_and_lock_file( $subscriber_to_messages_path ); if ( ! $fd_topics_subscriber || ! $fd_subscriber_to_messages ) { die( 'Could not open required file.' ); @@ -230,15 +230,15 @@ private static function handle_publish_message( $topics_to_subscribers_path, $su if ( $receivers && count( $receivers ) > 0 ) { $message['clients'] = count( $receivers ); foreach ( $receivers as $receiver ) { - if ( ! $subscriber_to_messages[ $receiver ] ) { + if ( ! isset( $subscriber_to_messages[ $receiver ] ) ) { $subscriber_to_messages[ $receiver ] = array(); } $subscriber_to_messages[ $receiver ][] = $message; } static::save_contents_to_file_descriptor( $fd_subscriber_to_messages, $subscriber_to_messages ); } - flock( $fd_subscriber_messages, LOCK_UN ); - fclose( $fd_subscriber_messages ); + flock( $fd_subscriber_to_messages, LOCK_UN ); + fclose( $fd_subscriber_to_messages ); flock( $fd_topics_subscriber, LOCK_UN ); fclose( $fd_topics_subscriber ); } @@ -258,7 +258,7 @@ private static function handle_ping( $subscriber_to_messages_path, $subscriber_i if ( ! $fd_subscriber_to_messages ) { die( 'Could not open required file.' ); } - if ( ! $subscriber_to_messages[ static::$subscriber_id ] ) { + if ( ! $subscriber_to_messages[ $subscriber_id ] ) { $subscriber_to_messages[ $subscriber_id ] = array(); } $subscriber_to_messages[ $subscriber_id ][] = array( 'type' => 'pong' ); From e3ff7b1b1a228fe07aa918763341e879916e5b20 Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Tue, 10 Oct 2023 18:43:44 +0100 Subject: [PATCH 24/32] Added retry mechanism with error events. --- .../class-gutenberg-http-signaling-server.php | 79 +++++++++++++------ 1 file changed, 55 insertions(+), 24 deletions(-) diff --git a/lib/experimental/sync/class-gutenberg-http-signaling-server.php b/lib/experimental/sync/class-gutenberg-http-signaling-server.php index 03c7f9dbc79ce..9959e357d7e2d 100644 --- a/lib/experimental/sync/class-gutenberg-http-signaling-server.php +++ b/lib/experimental/sync/class-gutenberg-http-signaling-server.php @@ -1,7 +1,7 @@ array( 'message hello','handshake message' ) ). $subscriber_to_messages_path = get_temp_dir() . DIRECTORY_SEPARATOR . 'subscribers_to_messages.txt'; - + // Example inside file: array( 'doc1: array( 2323232121 ), 'doc2: array( 2323232123, 2323232121 ) ). $topics_to_subscribers_path = get_temp_dir() . DIRECTORY_SEPARATOR . 'topics_to_subscribers.txt'; - if ( 'GET' === $_SERVER['REQUEST_METHOD'] ) { static::handle_read_pending_messages( $subscriber_id, $subscriber_to_messages_path ); } else { @@ -75,14 +74,14 @@ public static function do_wp_ajax_action() { } /** - * Reads the contents of $fd and returns them unserialized. + * Reads the contents of $fd and returns them deserialized. * * @access private * @internal * * @param resource $fd A file descriptor. * - * @return array Unserialized contents of fd. + * @return array Deserialized contents of fd. */ private static function get_contents_from_file_descriptor( $fd ) { $contents_raw = stream_get_contents( $fd ); @@ -93,6 +92,16 @@ private static function get_contents_from_file_descriptor( $fd ) { return $result; } + /** + * Locks a file with an exclusive lock and returns its contents deserialized. + * + * @access private + * @internal + * + * @param string $path The path of a file. + * + * @return array File descriptor as the first element of the array and the contents of the file as the second element. + */ private static function get_contents_and_lock_file( $path ) { $fd = fopen( $path, 'c+' ); if ( ! $fd ) { @@ -102,6 +111,17 @@ private static function get_contents_and_lock_file( $path ) { return array( $fd, static::get_contents_from_file_descriptor( $fd ) ); } + /** + * Makes the file descriptor content of $fd equal to the serialization of content. + * Overwrites what was previously in $fd. + * Unlocks the file descriptor and closes it. + * + * @access private + * @internal + * + * @param resource $fd A file descriptor. + * @param array $content An array with the contents to serialized and written to the file. + */ private static function save_contents_and_unlock_file( $fd, $content ) { static::save_contents_to_file_descriptor( $fd, $content ); flock( $fd, LOCK_UN ); @@ -141,13 +161,25 @@ private static function save_contents_to_file_descriptor( $fd, $content ) { private static function handle_read_pending_messages( $subscriber_id, $subscriber_to_messages_path ) { header( 'Content-Type: text/event-stream' ); header( 'Cache-Control: no-cache' ); - echo 'retry: 3000' . PHP_EOL; - $fd = fopen( $subscriber_to_messages_path, 'c+' ); + list( $fd, $subscriber_to_messages ) = static::get_contents_and_lock_file( $subscriber_to_messages_path ); if ( ! $fd ) { - die( 'Could not open required file.' ); + $retries = isset( $_COOKIE['signaling_server_retries'] ) ? intval( $_COOKIE['signaling_server_retries'] ) : 0; + $secure = ( 'https' === parse_url( home_url(), PHP_URL_SCHEME ) ); + setcookie( 'signaling_server_retries', $retries + 1, time() + DAY_IN_SECONDS, SITECOOKIEPATH, '', $secure ); + echo 'id: ' . time() . PHP_EOL; + echo 'event: error' . PHP_EOL; + echo 'data: ' . 'Could not open required file.' . PHP_EOL . PHP_EOL; + echo 'retry: ' . 3000 * pow( 2, $retries ) . PHP_EOL; + exit; } - flock( $fd, LOCK_EX ); - $subscriber_to_messages = static::get_contents_from_file_descriptor( $fd ); + + if ( isset( $_COOKIE['signaling_server_retries'] ) ) { + $secure = ( 'https' === parse_url( home_url(), PHP_URL_SCHEME ) ); + // unset the cookie using a past expiration date. + setcookie( 'signaling_server_retries', 0, time() - DAY_IN_SECONDS, SITECOOKIEPATH, '', $secure ); + } + + echo 'retry: 3000' . PHP_EOL; if ( isset( $subscriber_to_messages[ $subscriber_id ] ) && count( $subscriber_to_messages[ $subscriber_id ] ) > 0 ) { echo 'id: ' . time() . PHP_EOL; echo 'event: message' . PHP_EOL; @@ -181,7 +213,7 @@ private static function handle_subscribe_to_topics( $topics_to_subscribers_path, $topics_to_subscribers[ $topic ] = array(); } $topics_to_subscribers[ $topic ][] = $subscriber_id; - $topics_to_subscribers[ $topic ] = array_unique( $topics_to_subscribers[ $topic ] ); + $topics_to_subscribers[ $topic ] = array_unique( $topics_to_subscribers[ $topic ] ); } static::save_contents_and_unlock_file( $fd, $topics_to_subscribers ); } @@ -193,8 +225,9 @@ private static function handle_subscribe_to_topics( $topics_to_subscribers_path, * @access private * @internal * - * @param array $topics_to_subscribers Topics to subscribers data-structure. - * @param array $topics An array of topics e.g: array( 'doc1', 'doc2' ). + * @param string $topics_to_subscribers_path Topics to subscribers path. + * @param string $subscriber_id The subscriber id. + * @param array $topics An array of topics e.g: array( 'doc1', 'doc2' ). */ private static function handle_unsubscribe_from_topics( $topics_to_subscribers_path, $subscriber_id, $topics ) { list( $fd, $topics_to_subscribers ) = static::get_contents_and_lock_file( $topics_to_subscribers_path ); @@ -274,16 +307,16 @@ private static function clean_up_old_connections( $connected_subscriber_id, $sub // Example: array( 2323232121 => 34343433323(timestamp) ). $fd_subscribers_last_connection = fopen( $subscribers_to_last_connection_path, 'c+' ); - if( ! $fd_subscribers_last_connection ) { + if ( ! $fd_subscribers_last_connection ) { die( 'Could not open required file.' ); } flock( $fd_subscribers_last_connection, LOCK_EX ); - $subscribers_to_last_connection_time = static::get_contents_from_file_descriptor( $fd_subscribers_last_connection ); + $subscribers_to_last_connection_time = static::get_contents_from_file_descriptor( $fd_subscribers_last_connection ); $subscribers_to_last_connection_time[ $connected_subscriber_id ] = time(); $needs_cleanup = false; foreach ( $subscribers_to_last_connection_time as $subscriber_id => $last_connection_time ) { - // cleanup connections older than 24 hours. - if ( $last_connection_time < time() - 24 * 60 * 60 ) { + // cleanup connections older than 1 hour. + if ( $last_connection_time < time() - 1 * 60 * 60 ) { unset( $subscribers_to_last_connection_time[ $subscriber_id ] ); $needs_cleanup = true; } @@ -292,7 +325,7 @@ private static function clean_up_old_connections( $connected_subscriber_id, $sub if ( $needs_cleanup ) { $fd_subscriber_messages = fopen( $subscriber_to_messages_path, 'c+' ); - if( ! $fd_subscriber_messages ) { + if ( ! $fd_subscriber_messages ) { die( 'Could not open required file.' ); } flock( $fd_subscriber_messages, LOCK_EX ); @@ -305,7 +338,7 @@ private static function clean_up_old_connections( $connected_subscriber_id, $sub static::save_contents_to_file_descriptor( $fd_subscriber_messages, $subscriber_to_messages ); $fd_topics_subscriber = fopen( $topics_to_subscribers_path, 'c+' ); - if( ! $fd_topics_subscriber ) { + if ( ! $fd_topics_subscriber ) { die( 'Could not open required file.' ); } flock( $fd_topics_subscriber, LOCK_EX ); @@ -325,12 +358,10 @@ private static function clean_up_old_connections( $connected_subscriber_id, $sub flock( $fd_topics_subscriber, LOCK_UN ); fclose( $fd_topics_subscriber ); } - + flock( $fd_subscribers_last_connection, LOCK_UN ); fclose( $fd_subscribers_last_connection ); } - - } Gutenberg_HTTP_Signaling_Server::init(); From 53070baa178cfa4c8a4b7a286ce67280711ab39c Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Wed, 11 Oct 2023 10:54:12 +0100 Subject: [PATCH 25/32] Document clean_up_old_connections. --- .../sync/class-gutenberg-http-signaling-server.php | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/experimental/sync/class-gutenberg-http-signaling-server.php b/lib/experimental/sync/class-gutenberg-http-signaling-server.php index 9959e357d7e2d..beea13989a653 100644 --- a/lib/experimental/sync/class-gutenberg-http-signaling-server.php +++ b/lib/experimental/sync/class-gutenberg-http-signaling-server.php @@ -157,6 +157,12 @@ private static function save_contents_to_file_descriptor( $fd, $content ) { * event: * data: * ``` + * + * @access private + * @internal + * + * @param string $subscriber_id The subscriber id. + * @param string $subscriber_to_messages_path The path to the file that contains the messages of the subscribers. */ private static function handle_read_pending_messages( $subscriber_id, $subscriber_to_messages_path ) { header( 'Content-Type: text/event-stream' ); @@ -301,6 +307,13 @@ private static function handle_ping( $subscriber_to_messages_path, $subscriber_i /** * Deletes messages and subscriber information of clients that have not interacted with the signaling server in a long time. + * + * @access private + * @internal + * + * @param string $connected_subscriber_id The subscriber id. + * @param string $subscriber_to_messages_path The path to the file that contains the messages of the subscribers. + * @param string $topics_to_subscribers_path The path to the file that contains the subscribers of the topics. */ private static function clean_up_old_connections( $connected_subscriber_id, $subscriber_to_messages_path, $topics_to_subscribers_path ) { $subscribers_to_last_connection_path = get_temp_dir() . DIRECTORY_SEPARATOR . 'subscribers_to_last_connection.txt'; From 5cb0bf21d0ba4bd2a8cc1b6a87b02df71824a170 Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Wed, 11 Oct 2023 10:59:03 +0100 Subject: [PATCH 26/32] Make parameter order consistent. --- .../sync/class-gutenberg-http-signaling-server.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/experimental/sync/class-gutenberg-http-signaling-server.php b/lib/experimental/sync/class-gutenberg-http-signaling-server.php index beea13989a653..5158b96593e49 100644 --- a/lib/experimental/sync/class-gutenberg-http-signaling-server.php +++ b/lib/experimental/sync/class-gutenberg-http-signaling-server.php @@ -69,7 +69,7 @@ public static function do_wp_ajax_action() { echo wp_json_encode( array( 'result' => 'ok' ) ), PHP_EOL, PHP_EOL; } - static::clean_up_old_connections( $subscriber_id, $subscriber_to_messages_path, $topics_to_subscribers_path ); + static::clean_up_old_connections( $topics_to_subscribers_path, $subscriber_to_messages_path, $subscriber_id ); exit; } @@ -311,11 +311,11 @@ private static function handle_ping( $subscriber_to_messages_path, $subscriber_i * @access private * @internal * - * @param string $connected_subscriber_id The subscriber id. - * @param string $subscriber_to_messages_path The path to the file that contains the messages of the subscribers. * @param string $topics_to_subscribers_path The path to the file that contains the subscribers of the topics. + * @param string $subscriber_to_messages_path The path to the file that contains the messages of the subscribers. + * @param string $connected_subscriber_id The subscriber id. */ - private static function clean_up_old_connections( $connected_subscriber_id, $subscriber_to_messages_path, $topics_to_subscribers_path ) { + private static function clean_up_old_connections( $topics_to_subscribers_path, $subscriber_to_messages_path, $connected_subscriber_id, ) { $subscribers_to_last_connection_path = get_temp_dir() . DIRECTORY_SEPARATOR . 'subscribers_to_last_connection.txt'; // Example: array( 2323232121 => 34343433323(timestamp) ). From ac3e8ba8c581b760f792c101f3c9363a70d413e0 Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Wed, 11 Oct 2023 11:50:12 +0100 Subject: [PATCH 27/32] Document remaining parameters. --- .../class-gutenberg-http-signaling-server.php | 39 +++++++++---------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/lib/experimental/sync/class-gutenberg-http-signaling-server.php b/lib/experimental/sync/class-gutenberg-http-signaling-server.php index 5158b96593e49..97a3b09d146c2 100644 --- a/lib/experimental/sync/class-gutenberg-http-signaling-server.php +++ b/lib/experimental/sync/class-gutenberg-http-signaling-server.php @@ -42,7 +42,7 @@ public static function do_wp_ajax_action() { $topics_to_subscribers_path = get_temp_dir() . DIRECTORY_SEPARATOR . 'topics_to_subscribers.txt'; if ( 'GET' === $_SERVER['REQUEST_METHOD'] ) { - static::handle_read_pending_messages( $subscriber_id, $subscriber_to_messages_path ); + static::handle_read_pending_messages( $subscriber_to_messages_path, $subscriber_id ); } else { if ( empty( $_POST ) || empty( $_POST['message'] ) ) { die( 'no message' ); @@ -161,10 +161,10 @@ private static function save_contents_to_file_descriptor( $fd, $content ) { * @access private * @internal * - * @param string $subscriber_id The subscriber id. * @param string $subscriber_to_messages_path The path to the file that contains the messages of the subscribers. + * @param string $subscriber_id The subscriber id. */ - private static function handle_read_pending_messages( $subscriber_id, $subscriber_to_messages_path ) { + private static function handle_read_pending_messages( $subscriber_to_messages_path, $subscriber_id ) { header( 'Content-Type: text/event-stream' ); header( 'Cache-Control: no-cache' ); list( $fd, $subscriber_to_messages ) = static::get_contents_and_lock_file( $subscriber_to_messages_path ); @@ -200,14 +200,14 @@ private static function handle_read_pending_messages( $subscriber_id, $subscribe } /** - * Receives a $topics_to_subscribers data-structure and an array of topics, - * and returns a new $topics_to_subscribers data-structure where the current subscriber is subscribed to the topics. + * Handles a wp_ajax signaling server request of client that wants to subscribe to a set of topics its messages. * * @access private * @internal * - * @param array $topics_to_subscribers Topics to subscribers data-structure. - * @param array $topics An array of topics e.g: array( 'doc1', 'doc2' ). + * @param string $topics_to_subscribers_path Topics to subscribers path. + * @param string $subscriber_id The subscriber id. + * @param array $topics An array of topics e.g: array( 'doc1', 'doc2' ). */ private static function handle_subscribe_to_topics( $topics_to_subscribers_path, $subscriber_id, $topics ) { list( $fd, $topics_to_subscribers ) = static::get_contents_and_lock_file( $topics_to_subscribers_path ); @@ -225,15 +225,14 @@ private static function handle_subscribe_to_topics( $topics_to_subscribers_path, } /** - * Receives a $topics_to_subscribers data-structure and an array of topics, - * and returns a new $topics_to_subscribers data-structure where the current subscriber is not subscribed to the topics. + * Handles a wp_ajax signaling server request of client that wants to unsubscribe from a set of topics its messages. * * @access private * @internal * - * @param string $topics_to_subscribers_path Topics to subscribers path. - * @param string $subscriber_id The subscriber id. - * @param array $topics An array of topics e.g: array( 'doc1', 'doc2' ). + * @param string $topics_to_subscribers_path Topics to subscribers path. + * @param string $subscriber_id The subscriber id. + * @param array $topics An array of topics e.g: array( 'doc1', 'doc2' ). */ private static function handle_unsubscribe_from_topics( $topics_to_subscribers_path, $subscriber_id, $topics ) { list( $fd, $topics_to_subscribers ) = static::get_contents_and_lock_file( $topics_to_subscribers_path ); @@ -249,14 +248,15 @@ private static function handle_unsubscribe_from_topics( $topics_to_subscribers_p } /** - * Receives a $topics_to_subscribers data-structure and an array of topics, - * and returns a new $topics_to_subscribers data-structure where the current subscriber is subscribed to the topics. + * Handles a wp_ajax signaling server request of client that wants to publish a message. * * @access private * @internal * - * @param array $topics_to_subscribers Topics to subscribers data-structure. - * @param array $topics An array of topics e.g: array( 'doc1', 'doc2' ). + * @param string $topics_to_subscribers_path Topics to subscribers path. + * @param string $subscriber_to_messages_path The path to the file that contains the messages of the subscribers. + * @param string $subscriber_id The subscriber id. + * @param array $message The message associative array. */ private static function handle_publish_message( $topics_to_subscribers_path, $subscriber_to_messages_path, $subscriber_id, $message ) { list( $fd_topics_subscriber, $topics_to_subscribers ) = static::get_contents_and_lock_file( $topics_to_subscribers_path ); @@ -283,14 +283,13 @@ private static function handle_publish_message( $topics_to_subscribers_path, $su } /** - * Receives a $topics_to_subscribers data-structure and an array of topics, - * and returns a new $topics_to_subscribers data-structure where the current subscriber is subscribed to the topics. + * Handles a wp_ajax signaling server request for the ping of a client. * * @access private * @internal * - * @param array $topics_to_subscribers Topics to subscribers data-structure. - * @param array $topics An array of topics e.g: array( 'doc1', 'doc2' ). + * @param string $subscriber_to_messages_path The path to the file that contains the messages of the subscribers. + * @param string $subscriber_id The subscriber id. */ private static function handle_ping( $subscriber_to_messages_path, $subscriber_id ) { list( $fd_subscriber_to_messages, $subscriber_to_messages ) = static::get_contents_and_lock_file( $subscriber_to_messages_path ); From f179c0eae2dade0c7217b0678715cb18f1a8259e Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Wed, 11 Oct 2023 12:29:15 +0100 Subject: [PATCH 28/32] Follow y-webrtc room name convention --- packages/sync/src/create-webrtc-connection.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/sync/src/create-webrtc-connection.js b/packages/sync/src/create-webrtc-connection.js index 8cd6c2c3b3b1e..97fcddc727d02 100644 --- a/packages/sync/src/create-webrtc-connection.js +++ b/packages/sync/src/create-webrtc-connection.js @@ -27,8 +27,8 @@ export function createWebRTCConnection( { signaling, password } ) { /** @type {string} */ objectType, /** @type {import("yjs").Doc} */ doc ) { - const docName = `${ objectType }-${ objectId }`; - new WebrtcProviderWithHttpSignaling( docName, doc, { + const roomName = `${ objectType }-${ objectId }`; + new WebrtcProviderWithHttpSignaling( roomName, doc, { signaling, // @ts-ignore password, From a56a392cb8a008b2ba79acfb57f9d84fd584d87a Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Wed, 11 Oct 2023 14:21:02 +0100 Subject: [PATCH 29/32] Conditionally enable endpoint depending if setting is set or not. --- .../sync/class-gutenberg-http-signaling-server.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/experimental/sync/class-gutenberg-http-signaling-server.php b/lib/experimental/sync/class-gutenberg-http-signaling-server.php index 97a3b09d146c2..7250555f6aafa 100644 --- a/lib/experimental/sync/class-gutenberg-http-signaling-server.php +++ b/lib/experimental/sync/class-gutenberg-http-signaling-server.php @@ -21,6 +21,10 @@ class Gutenberg_HTTP_Signaling_Server { * Adds a wp_ajax action to handle the signaling server requests. */ public static function init() { + $gutenberg_experiments = get_option( 'gutenberg-experiments' ); + if ( ! $gutenberg_experiments || ! array_key_exists( 'gutenberg-sync-collaboration', $gutenberg_experiments ) ) { + return; + } add_action( 'wp_ajax_gutenberg_signaling_server', array( __CLASS__, 'do_wp_ajax_action' ) ); } From 7dbd59f3c7ccb0ab67a6af3c9881454f35a86a3a Mon Sep 17 00:00:00 2001 From: Jorge Costa Date: Wed, 11 Oct 2023 14:35:48 +0100 Subject: [PATCH 30/32] Remove debug code. --- .../sync/class-gutenberg-http-signaling-server.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/experimental/sync/class-gutenberg-http-signaling-server.php b/lib/experimental/sync/class-gutenberg-http-signaling-server.php index 7250555f6aafa..26832e3473039 100644 --- a/lib/experimental/sync/class-gutenberg-http-signaling-server.php +++ b/lib/experimental/sync/class-gutenberg-http-signaling-server.php @@ -1,7 +1,4 @@ Date: Thu, 19 Oct 2023 12:20:49 +0200 Subject: [PATCH 31/32] Fix rebase: remove no longer existing file --- packages/sync/src/connect-webrtc.js | 37 ----------------------------- 1 file changed, 37 deletions(-) delete mode 100644 packages/sync/src/connect-webrtc.js diff --git a/packages/sync/src/connect-webrtc.js b/packages/sync/src/connect-webrtc.js deleted file mode 100644 index 1801212c8c2bf..0000000000000 --- a/packages/sync/src/connect-webrtc.js +++ /dev/null @@ -1,37 +0,0 @@ -/** - * External dependencies - */ -// import { WebrtcProvider } from 'y-webrtc'; - -/** - * Internal dependencies - */ -import { WebrtcProviderWithHttpSignaling } from './webrtc-http-stream-signaling'; - -/** @typedef {import('./types').ObjectType} ObjectType */ -/** @typedef {import('./types').ObjectID} ObjectID */ -/** @typedef {import('./types').CRDTDoc} CRDTDoc */ - -/** - * Connect function to the WebRTC provider. - * - * @param {ObjectID} objectId The object ID. - * @param {ObjectType} objectType The object type. - * @param {CRDTDoc} doc The CRDT document. - * - * @return {Promise<() => void>} Promise that resolves when the connection is established. - */ -export function connectWebRTC( objectId, objectType, doc ) { - const roomName = `${ objectType }-${ objectId }`; - new WebrtcProviderWithHttpSignaling( roomName, doc, { - signaling: [ - //'ws://localhost:4444', - // @ts-ignore - window.wp.ajax.settings.url, - ], - // @ts-ignore - password: window.__experimentalCollaborativeEditingSecret, - } ); - - return Promise.resolve( () => true ); -} From b42e5f1ec870d655674196f526d8f5c59a0cf3db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Maneiro?= <583546+oandregal@users.noreply.github.com> Date: Thu, 19 Oct 2023 12:50:49 +0200 Subject: [PATCH 32/32] Update docs --- lib/experimental/sync/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/experimental/sync/README.md b/lib/experimental/sync/README.md index cd764d2e7147a..83a105adddf7a 100644 --- a/lib/experimental/sync/README.md +++ b/lib/experimental/sync/README.md @@ -117,8 +117,10 @@ The server will respond using the [Event stream format](https://developer.mozill - `retry`: the reconnection time, in ms. The time after which the client should check again for messages. - `id`: unique identifier for the server response. -- `event`: always set to `message`. -- `data`: a JSON encoded string containing an array of messages that the given client has not read yet. Each message is similar to the published message object but includes an additional property named `clients`. This property specifies the number of clients for which the message was sent. Note it does not indicate whether they have already received/requested it. +- `event`: one of `message` or `error`. +- `data`: + - If `event` is `message`, data is a JSON encoded string containing an array of messages that the given client has not read yet. Each message is similar to the published message object but includes an additional property named `clients`. This property specifies the number of clients for which the message was sent. Note it does not indicate whether they have already received/requested it. + - If `event` is `error`, data is a description of the error. If there are no pending messages, the server's response only contains the `retry:` field. If there are pending messages, the server will respond including all the fields.