[Yjs Collab] Reliable sync with the backend

lib/experimental/editor-settings.php

@@ -13,6 +13,12 @@ function gutenberg_enable_experiments() {
 	if ( $gutenberg_experiments && array_key_exists( 'gutenberg-sync-collaboration', $gutenberg_experiments ) ) {
 		wp_add_inline_script( 'wp-block-editor', 'window.__experimentalEnableSync = true', 'before' );
 	}
+	if ( $gutenberg_experiments && array_key_exists( 'gutenberg-sync-heartbeat-collaboration', $gutenberg_experiments ) ) {
+		wp_add_inline_script( 'wp-block-editor', 'window.__experimentalEnableHeartbeatSync = true', 'before' );
+	}
+	if ( $gutenberg_experiments && array_key_exists( 'gutenberg-sync-webrtc-collaboration', $gutenberg_experiments ) ) {
+		wp_add_inline_script( 'wp-block-editor', 'window.__experimentalEnableWebrtcSync = true', 'before' );
+	}
 	if ( $gutenberg_experiments && array_key_exists( 'gutenberg-custom-dataviews', $gutenberg_experiments ) ) {
 		wp_add_inline_script( 'wp-block-editor', 'window.__experimentalCustomViews = true', 'before' );
 	}

lib/experimental/sync/class-gutenberg-http-signaling-server.php

@@ -19,7 +19,7 @@ class Gutenberg_HTTP_Signaling_Server {
 	 */
 	public static function init() {
 		$gutenberg_experiments = get_option( 'gutenberg-experiments' );
-		if ( ! $gutenberg_experiments || ! array_key_exists( 'gutenberg-sync-collaboration', $gutenberg_experiments ) ) {
+		if ( ! $gutenberg_experiments || ! array_key_exists( 'gutenberg-sync-webrtc-collaboration', $gutenberg_experiments ) ) {
 			return;
 		}
 		add_action( 'wp_ajax_gutenberg_signaling_server', array( __CLASS__, 'do_wp_ajax_action' ) );

lib/experimental/synchronization.php

@@ -18,7 +18,155 @@ function gutenberg_rest_api_init_collaborative_editing() {
 		$collaborative_editing_secret = wp_generate_password( 64, false );
 	}
 	add_site_option( 'collaborative_editing_secret', $collaborative_editing_secret );
-
 	wp_add_inline_script( 'wp-sync', 'window.__experimentalCollaborativeEditingSecret = "' . $collaborative_editing_secret . '";', 'before' );
 }
 add_action( 'admin_init', 'gutenberg_rest_api_init_collaborative_editing' );
+
+/**
+ * Maintains the <!-- y:gutenberg [..] --> comment, which contains the Yjs document
+ * state in a base64 encoded format.
+ *
+ *   <!-- y:gutenberg version="1" state="(base64-encoded Yjs doc)" new-content-clientid="(u53)" -->
+ *
+ * The comment tag will be part of the HTML content and enables collaborative
+ * clients to exchange editing history. It is used to keep a Yjs document
+ * in-sync with the HTML content. For forwards-compatibility, we also maintain a
+ * version property that can be used in the future by clients to properly handle
+ * legacy y:gutenberg comments.
+ *
+ * The Yjs document state contains information that is needed for automatic
+ * conflict resolution to enable collaborative editing on the HTML content.
+ * Collaboration-enabled clients will try to keep the Yjs state in-sync with
+ * the HTML content.
+ *
+ * Legacy clients may manipulate the HTML state without updating the Yjs
+ * document. Ideally, they leave the y:gutenberg comment alone. Once a
+ * collaboration-enabled client recognizes that the HTML content changed and is
+ * not in-sync with the Yjs state, it will update the Yjs document.
+ *
+ * To ensure that all clients update the Yjs state in "the same way" and
+ * produce the same Yjs update, all client must use the same Yjs-clientid. This
+ * clientid must change whenever the HTML content updates, to prevent the
+ * creation of conflicting Yjs updates.
+ *
+ * Note: Yjs has a concept of clientId that is very different from the
+ * clientIds used in the block editor. Yjs' clientIds should be unique per
+ * client (i.e. each browser tab has a different clientId) and are used for
+ * conflict-resolution.
+ *
+ * It is usually not recommended to change the clientid, as this can corrup the
+ * Yjs document and make it unusable. Please consult an expert on Yjs CRDTs
+ * before changing this approach.
+ *
+ * This approach is not ideal and may - under very specific circumstances -
+ * lead to content duplication.
+ *
+ * When multiple changes to the HTML document happen (without updating the Yjs
+ * state) while multiple collaboration-enabled clients listen to changes, it
+ * may result in content duplication.
+ *
+ * Example:
+ *
+ *   - Change 1: Paragraph 1 is added to the HTML content without updating the
+ *               Yjs document.
+ *   - Change 2: Paragraph 2 is added to the HTML content without updating the
+ *               Yjs document. This change happens immediately after change 1.
+ *               So this changes also incorporates the changeset of change 1.
+ *
+ * Result:
+ *
+ *   - Clients that see change 1 will add paragraph 1 to the Yjs document.
+ *   - Clients that see change 2 will add paragraph 1 and paragraph 2 to the
+ *     Yjs document, using a different clientid.
+ *   - In total, three paragraphs are added. The clients have no way of knowing
+ *     that change 2 incorporates changes from change 2.
+ *
+ * If content duplication happens a lot, it may be necessary to increase the
+ * debounce interval between fetching document states.
+ *
+ * A real solution would be to maintain diffs in the y:gutenberg comment when changes
+ * happen without Yjs noticing. However, such an implementation will further
+ * increase the size of the y:gutenberg comment.
+ *
+ * In practice, we can accept content duplication in some edge-cases. This
+ * is better that the status quo, which overwrites existing content and can
+ * lead to data loss.
+ */
+function gutenberg_filter_post_content_ydoc( $data ) {
+	if ( 'post' !== $data['post_type'] and 'revision' !== $data['post_type'] ) {
+		return $data;
+	}
+	$gutenberg_experiments = get_option( 'gutenberg-experiments' );
+	if ( ! $gutenberg_experiments || ! array_key_exists( 'gutenberg-sync-collaboration', $gutenberg_experiments ) ) {
+		return $data;
+	}
+	$content = stripslashes( $data['post_content'] );
+	// transform $content if it contains ydoc comment tag
+	$yinfo = gutenberg_get_yinfo( $content );
+	if ( $yinfo ) {
+		$content = substr( $content, 0, $yinfo['commentStart'] ) . substr( $content, $yinfo['commentEnd'] );
+		// Always supply a new client id after any change. Generate a new clientid
+		// for updated content that can be represented as a 53bit unsigned integer
+		// (max clientid in Yjs).
+		$ynewclientid         = wp_rand( 0, 9007199254740991 ); // This is 2^53 – 1 which is `Number.MAX_SAFE_INTEGER` from JavaScript
+		$updated_yinfo        = '<!-- y:gutenberg version="' . $yinfo['version'] . '" state="' . $yinfo['state'] . '" new-content-clientid="' . $ynewclientid . '" -->';
+		$data['post_content'] = addslashes( $content . $updated_yinfo );
+	}
+	return $data;
+}
+add_filter( 'wp_insert_post_data', 'gutenberg_filter_post_content_ydoc', 10, 1 );
+
+/**
+ * Extracts the <!-- y:gutenberg .. --> comment from HTML $content and returns the encoded data.
+ */
+function gutenberg_get_yinfo( $content ) {
+	preg_match( '/<!-- y:gutenberg version=\"([a-zA-Z0-9]*)\" state=\"([a-zA-Z0-9+\/]*={0,3})\" new-content-clientid=\"([0-9]*)\" -->/', $content, $match, PREG_OFFSET_CAPTURE );
+	if ( $match ) {
+		return array(
+			'comment'              => $match[0][0],
+			'version'              => $match[1][0],
+			'state'                => $match[2][0],
+			'new-content-clientid' => $match[3][0],
+			'commentStart'         => $match[0][1],
+			'commentEnd'           => $match[0][1] + strlen( $match[0][0] ),
+		);
+	}
+	return null;
+}
+
+/**
+ * The client may request Yjs updates via the heartbeat api. It requests by
+ * supplying the last known "new-content-clientid", which changes whenever the
+ * document is written to the database. If the requested document has the same
+ * "new-content-clientid", then no update will be returned.
+ */
+function gutenberg_sync_heartbeat( array $response, array $data ) {
+	if ( empty( $data['y-sync'] ) ) {
+		return $response;
+	}
+	$updated_documents = array();
+
+	foreach ( $data['y-sync'] as $posttype => $requested_docs ) {
+		if ( strcmp( $posttype, 'postType/Posts' ) === 0 ) {
+			$docs = array();
+			foreach ( $requested_docs as $postid => $expected_client_id ) {
+				$post = wp_get_post_autosave( $postid );
+				if ( $post ) {
+					$postcontent = stripslashes( $post->post_content );
+					$yinfo       = gutenberg_get_yinfo( $postcontent );
+					if ( $yinfo and strcmp( $yinfo['new-content-clientid'], strval( $expected_client_id ) ) !== 0 ) {
+						$docs[ $postid ] = array(
+							'contentClientId' => $yinfo['new-content-clientid'],
+							'state'           => $yinfo['state'],
+						);
+					}
+				}
+			}
+			$updated_documents[ $posttype ] = $docs;
+		}
+	}
+	$response['y-sync'] = $updated_documents;
+	return $response;
+}
+
+add_filter( 'heartbeat_received', 'gutenberg_sync_heartbeat', 10, 2 );

lib/experiments-page.php

@@ -117,16 +117,40 @@ function gutenberg_initialize_experiments_settings() {
 
 	add_settings_field(
 		'gutenberg-sync-collaboration',
-		__( 'Collaboration: add real time editing', 'gutenberg' ),
+		__( 'Collaboration: add automatic conflict resolution on save / autosave.', 'gutenberg' ),
 		'gutenberg_display_experiment_field',
 		'gutenberg-experiments',
 		'gutenberg_experiments_section',
 		array(
-			'label' => __( 'Enables live collaboration and offline persistence between peers.', 'gutenberg' ),
+			'label' => __( 'Enables automatic conflict resolution on save / autosave.', 'gutenberg' ),
 			'id'    => 'gutenberg-sync-collaboration',
 		)
 	);
 
+	add_settings_field(
+		'gutenberg-sync-heartbeat-collaboration',
+		__( 'Collaboration: add (almost) real time editing using heartbeat API', 'gutenberg' ),
+		'gutenberg_display_experiment_field',
+		'gutenberg-experiments',
+		'gutenberg_experiments_section',
+		array(
+			'label' => __( 'Enables (almost) live collaboration using heartbeat API.', 'gutenberg' ),
+			'id'    => 'gutenberg-sync-heartbeat-collaboration',
+		)
+	);
+
+	add_settings_field(
+		'gutenberg-sync-webrtc-collaboration',
+		__( 'Collaboration: add real time editing using webrtc', 'gutenberg' ),
+		'gutenberg_display_experiment_field',
+		'gutenberg-experiments',
+		'gutenberg_experiments_section',
+		array(
+			'label' => __( 'Enables live collaboration using webrtc.', 'gutenberg' ),
+			'id'    => 'gutenberg-sync-webrtc-collaboration',
+		)
+	);
+
 	add_settings_field(
 		'gutenberg-color-randomizer',
 		__( 'Color randomizer', 'gutenberg' ),

package.json

@@ -178,9 +178,9 @@
 		"jsdom": "25.0.1"
 	},
 	"scripts": {
-		"build": "npm run build:packages && wp-scripts build",
+		"build": "cross-env NODE_OPTIONS='--max-old-space-size=8192' npm run build:packages && wp-scripts build",
 		"build:analyze-bundles": "npm run build -- --webpack-bundle-analyzer",
-		"build:package-types": "node ./bin/packages/validate-typescript-version.js && ( tsc --build || ( echo 'tsc failed. Try cleaning up first: `npm run clean:package-types`'; exit 1 ) ) && node ./bin/packages/check-build-type-declaration-files.js",
+		"build:package-types": "cross-env NODE_OPTIONS='--max-old-space-size=13192' node ./bin/packages/validate-typescript-version.js && ( tsc --build || ( echo 'tsc failed. Try cleaning up first: `npm run clean:package-types`'; exit 1 ) ) && node ./bin/packages/check-build-type-declaration-files.js",
 		"build:profile-types": "rimraf ./ts-traces && npm run clean:package-types && node ./bin/packages/validate-typescript-version.js && ( tsc --build --extendedDiagnostics --generateTrace ./ts-traces || ( echo 'tsc failed.'; exit 1 ) ) && node ./bin/packages/check-build-type-declaration-files.js && npx --yes @typescript/analyze-trace ts-traces > ts-traces/analysis.txt && echo $'\n\nDone! Build traces saved to ts-traces/ directory.\nTrace analysis saved to ts-traces/analysis.txt.'",
 		"prebuild:packages": "npm run clean:packages && npm run --if-present --workspaces build",
 		"build:packages": "npm run --silent build:package-types && node ./bin/packages/build.js",
@@ -189,7 +189,7 @@
 		"clean:package-types": "tsc --build --clean && rimraf --glob \"./packages/*/build-types\"",
 		"clean:packages": "rimraf --glob \"./packages/*/{build,build-module,build-wp,build-style}\"",
 		"component-usage-stats": "node ./node_modules/react-scanner/bin/react-scanner -c ./react-scanner.config.js",
-		"dev": "cross-env NODE_ENV=development npm run build:packages && concurrently \"wp-scripts start\" \"npm run dev:packages\"",
+		"dev": "cross-env NODE_ENV=development NODE_OPTIONS='--max-old-space-size=8192' npm run build:packages && concurrently \"wp-scripts start\" \"npm run dev:packages\"",
 		"dev:packages": "cross-env NODE_ENV=development concurrently \"node ./bin/packages/watch.js\" \"tsc --build --watch\"",
 		"distclean": "git clean --force -d -X",
 		"docs:api-ref": "node ./bin/api-docs/update-api-docs.js",

packages/block-editor/src/components/provider/use-block-sync.js

@@ -239,7 +239,7 @@ export default function useBlockSync( {
 
 			const newIsPersistent = isLastBlockChangePersistent();
 			const newBlocks = getBlocks( clientId );
-			const areBlocksDifferent = newBlocks !== blocks;
+			const areBlocksDifferent = newBlocks !== blocks; // @todo FYI this is always true..
 			blocks = newBlocks;
 			if (
 				areBlocksDifferent &&

packages/block-editor/src/components/writing-flow/use-input.js

@@ -69,6 +69,9 @@ export default function useInput() {
 					const selectionStart = getSelectionStart();
 					const selectionEnd = getSelectionEnd();
 
+					if ( blockName === null ) {
+						return;
+					}
 					if (
 						selectionStart.attributeKey ===
 						selectionEnd.attributeKey

packages/blocks/package.json

@@ -45,12 +45,14 @@
 		"@wordpress/private-apis": "file:../private-apis",
 		"@wordpress/rich-text": "file:../rich-text",
 		"@wordpress/shortcode": "file:../shortcode",
+		"@wordpress/sync": "file:../sync",
 		"@wordpress/warning": "file:../warning",
 		"change-case": "^4.1.2",
 		"colord": "^2.7.0",
 		"fast-deep-equal": "^3.1.3",
 		"hpq": "^1.3.0",
 		"is-plain-object": "^5.0.0",
+		"lib0": "^0.2.98",
 		"memize": "^2.1.0",
 		"react-is": "^18.3.0",
 		"remove-accents": "^0.5.0",

packages/blocks/src/api/index.js

@@ -71,6 +71,7 @@ export {
 	getBlockProps as __unstableGetBlockProps,
 	getInnerBlocksProps as __unstableGetInnerBlocksProps,
 	__unstableSerializeAndClean,
+	__unstableSerializeAndCleanWithYdoc,
 } from './serializer';
 
 // Validation is the process of comparing a block source with its output before