Cache the format of feeds

Detecting the format of a feed accounts for up to 30% of the time spent in `parser.ParseFeed`. Cache the value once detected, as it'll never change.
miniflux · Feb 4, 2025 · 84c3a16 · 84c3a16
1 parent 7bf1dd6
commit 84c3a16
Show file tree

Hide file tree

Showing 11 changed files with 43 additions and 8 deletions.
diff --git a/internal/database/migrations.go b/internal/database/migrations.go
@@ -994,4 +994,12 @@ var migrations = []func(tx *sql.Tx, driver string) error{
 		_, err = tx.Exec(`ALTER TABLE feeds ADD COLUMN webhook_url text default '';`)
 		return err
 	},
+	func(tx *sql.Tx, _ string) (err error) {
+		sql := `
+		ALTER TABLE feeds ADD COLUMN format text default '';
+		ALTER TABLE feeds ADD COLUMN format_version text default '';
+		`
+		_, err = tx.Exec(sql)
+		return err
+	},
 }
diff --git a/internal/model/feed.go b/internal/model/feed.go
@@ -56,6 +56,8 @@ type Feed struct {
 	WebhookURL                  string    `json:"webhook_url"`
 	NtfyEnabled                 bool      `json:"ntfy_enabled"`
 	NtfyPriority                int       `json:"ntfy_priority"`
+	Format                      string    `json:"format"`
+	FormatVersion               string    `json:"format_version"`
 
 	// Non-persisted attributes
 	Category *Category `json:"category,omitempty"`

diff --git a/internal/reader/atom/atom_03_adapter.go b/internal/reader/atom/atom_03_adapter.go
@@ -25,6 +25,9 @@ func NewAtom03Adapter(atomFeed *Atom03Feed) *Atom03Adapter {
 func (a *Atom03Adapter) BuildFeed(baseURL string) *model.Feed {
 	feed := new(model.Feed)
 
+	feed.Format = "atom"
+	feed.FormatVersion = "0.3"
+
 	// Populate the feed URL.
 	feedURL := a.atomFeed.Links.firstLinkWithRelation("self")
 	if feedURL != "" {

diff --git a/internal/reader/atom/atom_10_adapter.go b/internal/reader/atom/atom_10_adapter.go
@@ -29,6 +29,9 @@ func NewAtom10Adapter(atomFeed *Atom10Feed) *Atom10Adapter {
 func (a *Atom10Adapter) BuildFeed(baseURL string) *model.Feed {
 	feed := new(model.Feed)
 
+	feed.Format = "atom"
+	feed.FormatVersion = "10"
+
 	// Populate the feed URL.
 	feedURL := a.atomFeed.Links.firstLinkWithRelation("self")
 	if feedURL != "" {

diff --git a/internal/reader/handler/handler.go b/internal/reader/handler/handler.go
@@ -275,7 +275,14 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
 			return localizedError
 		}
 
-		updatedFeed, parseErr := parser.ParseFeed(responseHandler.EffectiveURL(), bytes.NewReader(responseBody))
+		var updatedFeed *model.Feed
+		var parseErr error
+		if originalFeed.Format != "" {
+			format, version := originalFeed.Format, originalFeed.FormatVersion
+			updatedFeed, parseErr = parser.ParseFeedWithFormat(responseHandler.EffectiveURL(), bytes.NewReader(responseBody), format, version)
+		} else {
+			updatedFeed, parseErr = parser.ParseFeed(responseHandler.EffectiveURL(), bytes.NewReader(responseBody))
+		}
 		if parseErr != nil {
 			localizedError := locale.NewLocalizedErrorWrapper(parseErr, "error.unable_to_parse_feed", parseErr)
 

diff --git a/internal/reader/json/adapter.go b/internal/reader/json/adapter.go
@@ -29,6 +29,7 @@ func (j *JSONAdapter) BuildFeed(baseURL string) *model.Feed {
 		Title:   strings.TrimSpace(j.jsonFeed.Title),
 		FeedURL: strings.TrimSpace(j.jsonFeed.FeedURL),
 		SiteURL: strings.TrimSpace(j.jsonFeed.HomePageURL),
+		Format:  "json",
 	}
 
 	if feed.FeedURL == "" {

diff --git a/internal/reader/parser/parser.go b/internal/reader/parser/parser.go
@@ -16,10 +16,8 @@ import (
 
 var ErrFeedFormatNotDetected = errors.New("parser: unable to detect feed format")
 
-// ParseFeed analyzes the input data and returns a normalized feed object.
-func ParseFeed(baseURL string, r io.ReadSeeker) (*model.Feed, error) {
-	r.Seek(0, io.SeekStart)
-	format, version := DetectFeedFormat(r)
+// ParseFeedWithFormat returns a normalized feed object.
+func ParseFeedWithFormat(baseURL string, r io.ReadSeeker, format, version string) (*model.Feed, error) {
 	switch format {
 	case FormatAtom:
 		r.Seek(0, io.SeekStart)
@@ -37,3 +35,10 @@ func ParseFeed(baseURL string, r io.ReadSeeker) (*model.Feed, error) {
 		return nil, ErrFeedFormatNotDetected
 	}
 }
+
+// ParseFeed analyzes the input data and returns a normalized feed object.
+func ParseFeed(baseURL string, r io.ReadSeeker) (*model.Feed, error) {
+	r.Seek(0, io.SeekStart)
+	format, version := DetectFeedFormat(r)
+	return ParseFeedWithFormat(baseURL, r, format, version)
+}
diff --git a/internal/reader/rdf/adapter.go b/internal/reader/rdf/adapter.go
@@ -29,6 +29,7 @@ func (r *RDFAdapter) BuildFeed(baseURL string) *model.Feed {
 		Title:   stripTags(r.rdf.Channel.Title),
 		FeedURL: strings.TrimSpace(baseURL),
 		SiteURL: strings.TrimSpace(r.rdf.Channel.Link),
+		Format:  "rdf",
 	}
 
 	if feed.Title == "" {

diff --git a/internal/reader/rss/adapter.go b/internal/reader/rss/adapter.go
@@ -31,6 +31,7 @@ func (r *RSSAdapter) BuildFeed(baseURL string) *model.Feed {
 		Title:   html.UnescapeString(strings.TrimSpace(r.rss.Channel.Title)),
 		FeedURL: strings.TrimSpace(baseURL),
 		SiteURL: strings.TrimSpace(r.rss.Channel.Link),
+		Format:  "rss",
 	}
 
 	// Ensure the Site URL is absolute.

diff --git a/internal/storage/feed.go b/internal/storage/feed.go
@@ -248,10 +248,12 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
 			apprise_service_urls,
 			webhook_url,
 			disable_http2,
-			description
+			description,
+			format,
+			format_version
 		)
 		VALUES
-			($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27)
+			($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29)
 		RETURNING
 			id
 	`
@@ -284,6 +286,8 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
 		feed.WebhookURL,
 		feed.DisableHTTP2,
 		feed.Description,
+		feed.Format,
+		feed.FormatVersion,
 	).Scan(&feed.ID)
 	if err != nil {
 		return fmt.Errorf(`store: unable to create feed %q: %v`, feed.FeedURL, err)

diff --git a/internal/ui/static/css/common.css b/internal/ui/static/css/common.css
@@ -75,7 +75,7 @@ a:hover {
     padding: var(--padding-size);
     position: absolute;
     transition: translate 0.3s;
-    translate: -50% calc(-100% - calc(var(--padding-size) * 2) - calc(var(--border-size) * 2));
+    translate: -50% calc(-100% - var(--padding-size) * 2 - var(--border-size) * 2);
 }
 
 .skip-to-content-link:focus {