Add URL Details endpoint to REST API to allow retrieval of info about…

… a remote URL (#18042)

* Scaffold out basic new endpoint

* Implement basic retrival of title tag from remote url

* Adds validation, sanitization and permissions checks.

* i18n fixes and docblocks

* Adds caching of remote request

* Update with feedback

* Tie up loose ends

* Remove unneed replacements

* Account for Custom Post Types in permissions check

Addresses #18042 (comment)

* Improve error handling

* Improve args checking

* Refactor to enable query of more data in future.

* Use md5

Addresses #18042 (comment)

* Extract utility functions

* Fix lint errors

* Add unit test scaffold

* Attempt to mock HTTP requests for testing. Needs work.

* Use DIR convention

* Allow opt out of cache functionality via filter

* Force tests to opt out of cache. Fix broken tests.

* Adjust 404 case to use standard response code

* Add dedicated methods to mock success/failure HTTP responses

* Add test for empty body in response.

This should trigger a failed response because we can’t read details from a URL which doesn’t provide a HTTP body.

* Add ability to filter request args. Update tests to account.

* Document cache filter

* Iniital attempt at adding schema

* Add test for unautenticated user.

* Adjust schema to use WP convention

* Adjust schema defaults

* Add endpoint args schema test

* Removes filter to bypass cache.

This already exists in core as https://core.trac.wordpress.org/browser/tags/5.6/src/wp-includes/option.php#L785

* Use existing transient filter to test cache and bypass for majority of tests

Fixes #18042 (comment)

* Use default timeout.

Addresses #18042 (comment)

* Removes unnecessary default from schema

Addresses #18042 (comment)

* Use predefine constants for response size

Resolves #18042 (comment)

* Rearrange test code order

* Adds ability to filter cache expiration

Addresses #18042 (review)

* Allow filtering the data retrived for a given URL

Addresses #18042 (comment)

* Remove @access comments

Addresses #18042 (comment)

* Prefer custom error message over get_status_header_desc

Addresses #18042 (comment)

* Utilise add_additional_fields_schema

Addresses #18042 (comment)

* Utilise add_additional_fields_to_object

Addresses #18042 (comment)

* Enable filtering of response object

Filter on both cached and uncached response.

* Add test to assert on ability to filter uncached responses

* Rename url placeholder literal

* Ensure accurately testing for correctly passed cached variable in filter args

* Update to test filtering of both cached and uncached responses

* Fixing linting

* Use array offset syntax prefered by Core

Addresses #18042 (comment)

* Remove from cache and update filter

See #18042 (comment)

* Update filter name to align with standards

Addresses #18042 (comment)

* Update response code to align with Core standards

Addresses #18042 (comment)

* Update cache ttl in docs to use variable

Addresses #18042 (comment)

* Use comment to reference canonical version of filter docs

Addresses #18042 (comment)

* Update to cache entire HTTP response body and remove additional filter

Addresses #18042 (comment)

* Tweak code comments

* Remove need to encode/decode cache value

Co-authored-by: Konstantin Obenland <[email protected]>

lib/class-wp-rest-url-details-controller.php

@@ -0,0 +1,264 @@
+<?php
+/**
+ * REST API: WP_REST_URL_Details_Controller class
+ *
+ * @package Gutenberg
+ */
+
+/**
+ * Controller which provides REST endpoint for retrieving information
+ * from a remote site's HTML response.
+ *
+ * @since 5.?.0
+ *
+ * @see WP_REST_Controller
+ */
+class WP_REST_URL_Details_Controller extends WP_REST_Controller {
+
+	/**
+	 * Constructs the controller.
+	 */
+	public function __construct() {
+		$this->namespace = '__experimental';
+		$this->rest_base = 'url-details';
+	}
+
+	/**
+	 * Registers the necessary REST API routes.
+	 */
+	public function register_routes() {
+		register_rest_route(
+			$this->namespace,
+			'/' . $this->rest_base,
+			array(
+				array(
+					'methods'             => WP_REST_Server::READABLE,
+					'callback'            => array( $this, 'parse_url_details' ),
+					'args'                => array(
+						'url' => array(
+							'required'          => true,
+							'description'       => __( 'The URL to process.', 'gutenberg' ),
+							'validate_callback' => 'wp_http_validate_url',
+							'sanitize_callback' => 'esc_url_raw',
+							'type'              => 'string',
+							'format'            => 'uri',
+						),
+					),
+					'permission_callback' => array( $this, 'permissions_check' ),
+					'schema'              => array( $this, 'get_public_item_schema' ),
+				),
+			)
+		);
+	}
+
+	/**
+	 * Get the schema for the endpoint.
+	 *
+	 * @return array the schema.
+	 */
+	public function get_item_schema() {
+
+		if ( $this->schema ) {
+			return $this->add_additional_fields_schema( $this->schema );
+		}
+
+		$schema = array(
+			'$schema'    => 'http://json-schema.org/draft-04/schema#',
+			'title'      => 'url-details',
+			'type'       => 'object',
+			'properties' => array(
+				'title' => array(
+					'description' => __( 'The contents of the <title> tag from the URL.', 'gutenberg' ),
+					'type'        => 'string',
+					'context'     => array( 'view', 'edit', 'embed' ),
+					'readonly'    => true,
+				),
+			),
+		);
+
+		$this->schema = $schema;
+
+		return $this->add_additional_fields_schema( $this->schema );
+	}
+
+	/**
+	 * Retrieves the contents of the <title> tag from the HTML
+	 * response.
+	 *
+	 * @param WP_REST_REQUEST $request Full details about the request.
+	 * @return WP_REST_Response|WP_Error The parsed details as a response object or an error.
+	 */
+	public function parse_url_details( $request ) {
+
+		$url = untrailingslashit( $request['url'] );
+
+		if ( empty( $url ) ) {
+			return new WP_Error( 'rest_invalid_url', __( 'Invalid URL', 'gutenberg' ), array( 'status' => 404 ) );
+		}
+
+		// Transient per URL.
+		$cache_key = $this->build_cache_key_for_url( $url );
+
+		// Attempt to retrieve cached response.
+		$cached_response = $this->get_cache( $cache_key );
+
+		if ( ! empty( $cached_response ) ) {
+			$remote_url_response = $cached_response;
+		} else {
+			$remote_url_response = $this->get_remote_url( $url );
+
+			// Exit if we don't have a valid body or it's empty.
+			if ( is_wp_error( $remote_url_response ) || empty( $remote_url_response ) ) {
+				return $remote_url_response;
+			}
+
+			// Cache the valid response.
+			$this->set_cache( $cache_key, $remote_url_response );
+		}
+
+		$data = $this->add_additional_fields_to_object(
+			array(
+				'title' => $this->get_title( $remote_url_response ),
+			),
+			$request
+		);
+
+		// Wrap the data in a response object.
+		$response = rest_ensure_response( $data );
+
+		/**
+		 * Filters the URL data for the response.
+		 *
+		 * @param WP_REST_Response $response The response object.
+		 * @param string           $url      The requested URL.
+		 * @param WP_REST_Request  $request  Request object.
+		 * @param array            $remote_url_response HTTP response body from the remote URL.
+		 */
+		return apply_filters( 'rest_prepare_url_details', $response, $url, $request, $remote_url_response );
+	}
+
+	/**
+	 * Checks whether a given request has permission to read remote urls.
+	 *
+	 * @return WP_Error|bool True if the request has access, WP_Error object otherwise.
+	 */
+	public function permissions_check() {
+		if ( current_user_can( 'edit_posts' ) ) {
+			return true;
+		}
+
+		foreach ( get_post_types( array( 'show_in_rest' => true ), 'objects' ) as $post_type ) {
+			if ( current_user_can( $post_type->cap->edit_posts ) ) {
+				return true;
+			}
+		}
+
+		return new WP_Error(
+			'rest_cannot_view_url_details',
+			__( 'Sorry, you are not allowed to process remote urls.', 'gutenberg' ),
+			array( 'status' => rest_authorization_required_code() )
+		);
+	}
+
+	/**
+	 * Retrieves the document title from a remote URL.
+	 *
+	 * @param string $url The website url whose HTML we want to access.
+	 * @return array|WP_Error the HTTP response from the remote URL or error.
+	 */
+	private function get_remote_url( $url ) {
+
+		$args = array(
+			'limit_response_size' => 150 * KB_IN_BYTES,
+		);
+
+		/**
+		 * Filters the HTTP request args for URL data retrieval.
+		 *
+		 * Can be used to adjust response size limit and other WP_Http::request args.
+		 *
+		 * @param array $args Arguments used for the HTTP request
+		 * @param string $url The attempted URL.
+		 */
+		$args = apply_filters( 'rest_url_details_http_request_args', $args, $url );
+
+		$response = wp_safe_remote_get(
+			$url,
+			$args
+		);
+
+		if ( WP_Http::OK !== wp_remote_retrieve_response_code( $response ) ) {
+			// Not saving the error response to cache since the error might be temporary.
+			return new WP_Error( 'no_response', __( 'URL not found. Response returned a non-200 status code for this URL.', 'gutenberg' ), array( 'status' => WP_Http::NOT_FOUND ) );
+		}
+
+		$remote_body = wp_remote_retrieve_body( $response );
+
+		if ( empty( $remote_body ) ) {
+			return new WP_Error( 'no_content', __( 'Unable to retrieve body from response at this URL.', 'gutenberg' ), array( 'status' => WP_Http::NOT_FOUND ) );
+		}
+
+		return $remote_body;
+	}
+
+	/**
+	 * Parses the <title> contents from the provided HTML
+	 *
+	 * @param string $html the HTML from the remote website at URL.
+	 * @return string the title tag contents (maybe empty).
+	 */
+	private function get_title( $html ) {
+		preg_match( '|<title>([^<]*?)</title>|is', $html, $match_title );
+
+		$title = isset( $match_title[1] ) ? trim( $match_title[1] ) : '';
+
+		return $title;
+	}
+
+	/**
+	 * Utility function to build cache key for a given URL.
+	 *
+	 * @param string $url the URL for which to build a cache key.
+	 * @return string the cache key.
+	 */
+	private function build_cache_key_for_url( $url ) {
+		return 'g_url_details_response_' . md5( $url );
+	}
+
+	/**
+	 * Utility function to retrieve a value from the cache at a given key.
+	 *
+	 * @param string $key the cache key.
+	 * @return string the value from the cache.
+	 */
+	private function get_cache( $key ) {
+		return get_transient( $key );
+	}
+
+	/**
+	 * Utility function to cache a given data set at a given cache key.
+	 *
+	 * @param string $key the cache key under which to store the value.
+	 * @param string $data the data to be stored at the given cache key.
+	 * @return void
+	 */
+	private function set_cache( $key, $data = '' ) {
+		if ( ! is_array( $data ) ) {
+			return;
+		}
+
+		$ttl = HOUR_IN_SECONDS;
+
+		/**
+		 * Filters the cache expiration.
+		 *
+		 * Can be used to adjust the time until expiration in seconds for the cache
+		 * of the data retrieved for the given URL.
+		 *
+		 * @param int $ttl the time until cache expiration in seconds.
+		 */
+		$cache_expiration = apply_filters( 'rest_url_details_cache_expiration', $ttl );
+
+		return set_transient( $key, $data, $cache_expiration );
+	}
+}

lib/load.php

@@ -71,6 +71,10 @@ function gutenberg_is_experiment_enabled( $name ) {
 	* End: Include for phase 2
 	*/
 
+	if ( ! class_exists( 'WP_REST_URL_Details_Controller' ) ) {
+		require_once __DIR__ . '/class-wp-rest-url-details-controller.php';
+	}
+
 	require __DIR__ . '/rest-api.php';
 }
 

lib/rest-api.php

@@ -10,6 +10,18 @@
 	die( 'Silence is golden.' );
 }
 
+
+/**
+ * Registers the REST API routes for URL Details.
+ *
+ * @since 5.0.0
+ */
+function gutenberg_register_url_details_routes() {
+	$url_details_controller = new WP_REST_URL_Details_Controller();
+	$url_details_controller->register_routes();
+}
+add_action( 'rest_api_init', 'gutenberg_register_url_details_routes' );
+
 /**
  * Registers the block pattern directory.
  */