feat(inspector-ui)!: show json preview for the feed (#64)

Fixes #53.

This pull request adds the feature to view the json representation of
the feed, which can be manipulated in the `js` filter.


![image](https://github.com/shouya/rss-funnel/assets/526598/f9ddd95d-8361-4bf5-84c8-4e445f2fbd81)


BREAKING CHANGE: the `pp=1` (pretty-print) query parameter on feed
endpoints are no longer respected. The serialized feed will always be
pretty-printed.

---------

Co-authored-by: shouya <[email protected]>

Cargo.toml

@@ -57,7 +57,6 @@ ego-tree = "0.6.2"
 readability = { version = "0.3.0", default-features = false }
 html5ever = "0.26.0"
 htmlescape = "0.3.1"
-xmlem = "0.2.3"
 
 # JS runtime crates
 rquickjs = { version = "0.5.1", features = ["loader", "parallel", "macro", "futures", "exports", "either"] }

inspector/package.json

@@ -20,6 +20,7 @@
   },
   "dependencies": {
     "@codemirror/lang-xml": "^6.0.2",
+    "@codemirror/lang-json": "^6.0.1",
     "@codemirror/language": "^6.10.1",
     "@codemirror/state": "^6.4.0",
     "@codemirror/view": "^6.24.0",

inspector/src/FeedInspector.js

@@ -1,8 +1,9 @@
-import { elt, $ } from "./util.js";
+import { elt, $, $$ } from "./util.js";
 import { Filter } from "./Filter.js";
 import { basicSetup, EditorView } from "codemirror";
 import { EditorState } from "@codemirror/state";
 import { xml } from "@codemirror/lang-xml";
+import { json } from "@codemirror/lang-json";
 import Split from "split.js";
 import HtmlSanitizer from "jitbit-html-sanitizer";
 import JSONSchemaView from "json-schema-view-js";
@@ -15,13 +16,15 @@ export class FeedInspector {
     this.feed_error = null;
     this.filter_schema = null;
     this.current_endpoint = null;
-    this.current_preview = null;
     this.raw_editor = null;
-    this.raw_feed_xml = null;
+    this.raw_feed_content = null;
+    this.json_preview_editor = null;
+    this.json_preview_content = null;
   }
 
   async init() {
     this.setup_raw_editor();
+    this.setup_json_preview_editor();
     this.setup_splitter();
     this.setup_view_mode_selector();
     this.setup_reload_config_handler();
@@ -87,13 +90,9 @@ export class FeedInspector {
   }
 
   async setup_view_mode_selector() {
-    $("#view-mode-selector #rendered-radio").addEventListener("change", () => {
-      this.render_feed();
-    });
-
-    $("#view-mode-selector #raw-radio").addEventListener("change", () => {
-      this.render_feed();
-    });
+    for (const node of $$("#view-mode-selector input")) {
+      node.addEventListener("change", () => this.render_feed());
+    }
   }
 
   async setup_param() {
@@ -130,6 +129,18 @@ export class FeedInspector {
     });
   }
 
+  async setup_json_preview_editor() {
+    this.json_preview_editor = new EditorView({
+      extensions: [
+        basicSetup,
+        json(),
+        EditorState.readOnly.of(true),
+        EditorView.lineWrapping,
+      ],
+      parent: $("#feed-preview #json"),
+    });
+  }
+
   async setup_splitter() {
     Split(["#sidebar-panel", "#main-panel"], {
       sizes: [20, 80],
@@ -202,24 +213,25 @@ export class FeedInspector {
   }
 
   async fetch_and_render_feed() {
-    await this.fetch_feed();
+    await this.fetch_feed_preview();
     this.render_feed();
   }
 
   async render_feed() {
     const view_mode =
       ($("#view-mode-selector #rendered-radio-input").checked && "rendered") ||
       ($("#view-mode-selector #raw-radio-input").checked && "raw") ||
+      ($("#view-mode-selector #json-radio-input").checked && "json") ||
       "rendered";
 
-    ["rendered", "raw"].forEach((mode) => {
+    ["rendered", "raw", "json"].forEach((mode) => {
       if (mode === view_mode) {
         $(`#feed-preview #${mode}`).classList.remove("hidden");
       } else {
         $(`#feed-preview #${mode}`).classList.add("hidden");
       }
 
-      const raw_feed_xml_xml = this.raw_feed_xml;
+      const raw_feed_xml_xml = this.raw_feed_content;
       const function_name = `render_feed_${mode}`;
       if (this[function_name]) {
         this[function_name](raw_feed_xml_xml);
@@ -269,6 +281,20 @@ export class FeedInspector {
     });
   }
 
+  async render_feed_json(_unused) {
+    const json = JSON.stringify(this.json_preview_content, null, 2);
+    if (this.json_preview_editor.state.doc.toString() === json) {
+      return;
+    }
+    this.json_preview_editor.dispatch({
+      changes: {
+        from: 0,
+        to: this.json_preview_editor.state.doc.length,
+        insert: json,
+      },
+    });
+  }
+
   update_request_param_controls() {
     if (!this.current_endpoint) return;
 
@@ -323,7 +349,7 @@ export class FeedInspector {
     $("#main-panel").classList.remove("hidden");
 
     // show feed preview
-    await this.fetch_feed();
+    await this.fetch_feed_preview();
     this.render_feed();
   }
 
@@ -378,29 +404,29 @@ export class FeedInspector {
     }
   }
 
-  async fetch_feed() {
+  async fetch_feed_preview() {
     if (!this.current_endpoint) return;
     const { path } = this.current_endpoint;
 
     const params = this.feed_request_param();
     $("#feed-preview").classList.add("loading");
 
     const time_start = performance.now();
-    const request_path = `${path}?${params}`;
-    $("#fetch-status").innerText = `Fetching ${request_path}...`;
-    const resp = await fetch(`${path}?${params}`);
+    $("#fetch-status").innerText = `Fetching preview for ${path}...`;
+
+    const resp = await fetch(`/_inspector/preview?endpoint=${path}&${params}`);
     let status_text = "";
 
     if (resp.status != 200) {
-      status_text = `Failed fetching ${request_path}`;
+      status_text = `Failed fetching ${path}`;
       status_text += ` (status: ${resp.status} ${resp.statusText})`;
       this.update_feed_error(await resp.text());
     } else {
       this.update_feed_error(null);
-      const text = await resp.text();
-      this.raw_feed_xml = text;
-      status_text = `Fetched ${request_path} `;
-      status_text += `(content-type: ${resp.headers.get("content-type")})`;
+      const preview = await resp.json();
+      this.raw_feed_content = preview.content;
+      this.json_preview_content = preview.json;
+      status_text = `Fetched ${path} (${preview.content_type})`;
     }
 
     status_text += ` in ${performance.now() - time_start}ms.`;
@@ -429,11 +455,11 @@ export class FeedInspector {
       $("#limit-filters", parent).value;
 
     const params = [];
-    if (source) params.push(`source=${source}`);
+    if (!this.current_endpoint.source && source)
+      params.push(`source=${source}`);
     if (limit_posts) params.push(`limit_posts=${limit_posts}`);
     if (limit_filters) params.push(`limit_filters=${limit_filters}`);
 
-    params.push("pp=1");
     return params.join("&");
   }
 

inspector/src/index.html

@@ -96,9 +96,14 @@ <h4>Error detected while processing feed</h4>
               <input type="radio" id="raw-radio-input" name="view" />
               <label for="raw-radio-input">Raw</label>
             </div>
+            <div id="json-radio" class="radio-button">
+              <input type="radio" id="json-radio-input" name="view" />
+              <label for="json-radio-input">Json</label>
+            </div>
           </header>
           <div id="rendered"></div>
           <div id="raw"></div>
+          <div id="json"></div>
         </div>
         <div id="fetch-status"></div>
       </div>

inspector/src/style.css

@@ -209,7 +209,8 @@ ul#filter-list {
       }
       pre.js-code {
         /* allow wrapping in js code */
-        white-space: normal;
+        white-space: pre-wrap;
+        word-wrap: break-word;
       }
 
       ul {

inspector/src/util.js

@@ -35,3 +35,6 @@ export function isBlankValue(value) {
 
 export const $ = (selector, parent = document) =>
   parent.querySelector(selector);
+
+export const $$ = (selector, parent = document) =>
+  Array.from(parent.querySelectorAll(selector));

src/cli.rs

@@ -3,7 +3,6 @@ use std::path::{Path, PathBuf};
 use clap::Parser;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
-use tower::Service;
 use url::Url;
 
 use crate::{
@@ -42,9 +41,6 @@ struct TestConfig {
   /// Limit the number of items in the feed
   #[clap(long, short('n'))]
   limit_posts: Option<usize>,
-  /// Whether to compact the XML output (opposite of pretty-print)
-  #[clap(long, short)]
-  compact_output: bool,
   /// Don't print XML output (Useful for checking console.log in JS filters)
   #[clap(long, short)]
   quiet: bool,
@@ -59,7 +55,6 @@ impl TestConfig {
       self.source.as_ref().cloned(),
       self.limit_filters,
       self.limit_posts,
-      !self.compact_output,
       self.base.clone(),
     )
   }
@@ -114,17 +109,17 @@ async fn test_endpoint(feed_defn: RootConfig, test_config: &TestConfig) {
     );
     return;
   };
-  let mut endpoint_service = endpoint_conf
+  let endpoint_service = endpoint_conf
     .build()
     .await
     .expect("failed to build endpoint service");
   let endpoint_param = test_config.to_endpoint_param();
-  let outcome = endpoint_service
-    .call(endpoint_param)
+  let feed = endpoint_service
+    .run(endpoint_param)
     .await
     .expect("failed to call endpoint service");
 
   if !test_config.quiet {
-    println!("{}", outcome.feed_xml());
+    println!("{}", feed.serialize(true).expect("failed serializing feed"));
   }
 }

src/feed.rs

@@ -6,7 +6,6 @@ use url::Url;
 
 use crate::html::convert_relative_url;
 use crate::html::html_body;
-use crate::server::EndpointOutcome;
 use crate::source::FromScratch;
 use crate::util::Error;
 use crate::util::Result;
@@ -56,6 +55,44 @@ impl Feed {
       .or_else(|_| Feed::from_atom_content(content))
   }
 
+  pub fn content_type(&self) -> &'static str {
+    match self {
+      Feed::Rss(_) => "application/rss+xml",
+      Feed::Atom(_) => "application/atom+xml",
+    }
+  }
+
+  pub fn serialize(&self, pretty: bool) -> Result<String> {
+    let mut buffer = vec![];
+
+    match self {
+      Feed::Rss(channel) => {
+        if pretty {
+          channel.pretty_write_to(&mut buffer, b' ', 2)?;
+        } else {
+          channel.write_to(&mut buffer)?;
+        }
+      }
+      Feed::Atom(feed) => {
+        let mut feed = feed.clone();
+        fix_escaping_in_extension_attr(&mut feed);
+        let mut conf = atom_syndication::WriteConfig {
+          indent_size: None,
+          write_document_declaration: true,
+        };
+
+        if pretty {
+          conf.indent_size = Some(2);
+        }
+
+        feed.write_with_config(&mut buffer, conf)?;
+      }
+    };
+
+    let s = String::from_utf8_lossy(&buffer).into_owned();
+    Ok(s)
+  }
+
   #[allow(clippy::field_reassign_with_default)]
   pub fn from_html_content(content: &str, url: &Url) -> Result<Self> {
     let item = Post::from_html_content(content, url)?;
@@ -70,20 +107,6 @@ impl Feed {
     Ok(feed)
   }
 
-  pub fn into_outcome(self) -> Result<EndpointOutcome> {
-    match self {
-      Feed::Rss(channel) => {
-        let body = channel.to_string();
-        Ok(EndpointOutcome::new(body, "application/rss+xml"))
-      }
-      Feed::Atom(mut feed) => {
-        fix_escaping_in_extension_attr(&mut feed);
-        let body = feed.to_string();
-        Ok(EndpointOutcome::new(body, "application/atom+xml"))
-      }
-    }
-  }
-
   pub fn take_posts(&mut self) -> Vec<Post> {
     match self {
       Feed::Rss(channel) => {
@@ -480,3 +503,13 @@ fn rss_item_timestamp(item: &rss::Item) -> Option<i64> {
 
   Some(date.timestamp())
 }
+
+impl axum::response::IntoResponse for Feed {
+  fn into_response(self) -> axum::response::Response {
+    let content = self.serialize(true).expect("failed serializing feed");
+    let content_type = self.content_type();
+    let headers = [("content-type", content_type)];
+
+    (http::StatusCode::OK, headers, content).into_response()
+  }
+}