Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cache the format of feeds #3126

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Cache the format of feeds #3126

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions internal/database/migrations.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1008,4 +1008,12 @@ var migrations = []func(tx *sql.Tx, driver string) error{
_, err = tx.Exec(sql)
return err
},
func(tx *sql.Tx, _ string) (err error) {
sql := `
ALTER TABLE feeds ADD COLUMN format text default '';
ALTER TABLE feeds ADD COLUMN format_version text default '';
`
_, err = tx.Exec(sql)
return err
},
}
2 changes: 2 additions & 0 deletions internal/model/feed.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ type Feed struct {
NtfyPriority int `json:"ntfy_priority"`
PushoverEnabled bool `json:"pushover_enabled,omitempty"`
PushoverPriority int `json:"pushover_priority,omitempty"`
Format string `json:"format"`
FormatVersion string `json:"format_version"`

// Non-persisted attributes
Category *Category `json:"category,omitempty"`
Expand Down
3 changes: 3 additions & 0 deletions internal/reader/atom/atom_03_adapter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ func NewAtom03Adapter(atomFeed *Atom03Feed) *Atom03Adapter {
func (a *Atom03Adapter) BuildFeed(baseURL string) *model.Feed {
feed := new(model.Feed)

feed.Format = "atom"
feed.FormatVersion = "0.3"

// Populate the feed URL.
feedURL := a.atomFeed.Links.firstLinkWithRelation("self")
if feedURL != "" {
Expand Down
3 changes: 3 additions & 0 deletions internal/reader/atom/atom_10_adapter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@ func NewAtom10Adapter(atomFeed *Atom10Feed) *Atom10Adapter {
func (a *Atom10Adapter) BuildFeed(baseURL string) *model.Feed {
feed := new(model.Feed)

feed.Format = "atom"
feed.FormatVersion = "10"

// Populate the feed URL.
feedURL := a.atomFeed.Links.firstLinkWithRelation("self")
if feedURL != "" {
Expand Down
13 changes: 12 additions & 1 deletion internal/reader/handler/handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,18 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
return localizedError
}

updatedFeed, parseErr := parser.ParseFeed(responseHandler.EffectiveURL(), bytes.NewReader(responseBody))
var updatedFeed *model.Feed
var parseErr error
if originalFeed.Format != "" {
format, version := originalFeed.Format, originalFeed.FormatVersion
updatedFeed, parseErr = parser.ParseFeedWithFormat(responseHandler.EffectiveURL(), bytes.NewReader(responseBody), format, version)
if parseErr != nil { // Maybe the feed changed its format.
slog.Warn("Unable to parse feed with the given format", slog.String("feed_url", originalFeed.FeedURL), slog.String("format", format), slog.Any("error", parseErr))
updatedFeed, parseErr = parser.ParseFeed(responseHandler.EffectiveURL(), bytes.NewReader(responseBody))
}
} else {
updatedFeed, parseErr = parser.ParseFeed(responseHandler.EffectiveURL(), bytes.NewReader(responseBody))
}
if parseErr != nil {
localizedError := locale.NewLocalizedErrorWrapper(parseErr, "error.unable_to_parse_feed", parseErr)

Expand Down
1 change: 1 addition & 0 deletions internal/reader/json/adapter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ func (j *JSONAdapter) BuildFeed(baseURL string) *model.Feed {
Title: strings.TrimSpace(j.jsonFeed.Title),
FeedURL: strings.TrimSpace(j.jsonFeed.FeedURL),
SiteURL: strings.TrimSpace(j.jsonFeed.HomePageURL),
Format: "json",
}

if feed.FeedURL == "" {
Expand Down
13 changes: 9 additions & 4 deletions internal/reader/parser/parser.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,8 @@ import (

var ErrFeedFormatNotDetected = errors.New("parser: unable to detect feed format")

// ParseFeed analyzes the input data and returns a normalized feed object.
func ParseFeed(baseURL string, r io.ReadSeeker) (*model.Feed, error) {
r.Seek(0, io.SeekStart)
format, version := DetectFeedFormat(r)
// ParseFeedWithFormat returns a normalized feed object.
func ParseFeedWithFormat(baseURL string, r io.ReadSeeker, format, version string) (*model.Feed, error) {
switch format {
case FormatAtom:
r.Seek(0, io.SeekStart)
Expand All @@ -37,3 +35,10 @@ func ParseFeed(baseURL string, r io.ReadSeeker) (*model.Feed, error) {
return nil, ErrFeedFormatNotDetected
}
}

// ParseFeed analyzes the input data and returns a normalized feed object.
func ParseFeed(baseURL string, r io.ReadSeeker) (*model.Feed, error) {
r.Seek(0, io.SeekStart)
format, version := DetectFeedFormat(r)
return ParseFeedWithFormat(baseURL, r, format, version)
}
1 change: 1 addition & 0 deletions internal/reader/rdf/adapter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ func (r *RDFAdapter) BuildFeed(baseURL string) *model.Feed {
Title: stripTags(r.rdf.Channel.Title),
FeedURL: strings.TrimSpace(baseURL),
SiteURL: strings.TrimSpace(r.rdf.Channel.Link),
Format: "rdf",
}

if feed.Title == "" {
Expand Down
1 change: 1 addition & 0 deletions internal/reader/rss/adapter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ func (r *RSSAdapter) BuildFeed(baseURL string) *model.Feed {
Title: html.UnescapeString(strings.TrimSpace(r.rss.Channel.Title)),
FeedURL: strings.TrimSpace(baseURL),
SiteURL: strings.TrimSpace(r.rss.Channel.Link),
Format: "rss",
}

// Ensure the Site URL is absolute.
Expand Down
8 changes: 6 additions & 2 deletions internal/storage/feed.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,10 +248,12 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
apprise_service_urls,
webhook_url,
disable_http2,
description
description,
format,
format_version
)
VALUES
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27)
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29)
RETURNING
id
`
Expand Down Expand Up @@ -284,6 +286,8 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
feed.WebhookURL,
feed.DisableHTTP2,
feed.Description,
feed.Format,
feed.FormatVersion,
).Scan(&feed.ID)
if err != nil {
return fmt.Errorf(`store: unable to create feed %q: %v`, feed.FeedURL, err)
Expand Down
6 changes: 5 additions & 1 deletion internal/storage/feed_query_builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,9 @@ func (f *FeedQueryBuilder) GetFeeds() (model.Feeds, error) {
f.ntfy_enabled,
f.ntfy_priority,
f.pushover_enabled,
f.pushover_priority
f.pushover_priority,
f.format,
f.format_version
FROM
feeds f
LEFT JOIN
Expand Down Expand Up @@ -244,6 +246,8 @@ func (f *FeedQueryBuilder) GetFeeds() (model.Feeds, error) {
&feed.NtfyPriority,
&feed.PushoverEnabled,
&feed.PushoverPriority,
&feed.Format,
&feed.FormatVersion,
)

if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion internal/ui/static/css/common.css
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ a:hover {
padding: var(--padding-size);
position: absolute;
transition: translate 0.3s;
translate: -50% calc(-100% - calc(var(--padding-size) * 2) - calc(var(--border-size) * 2));
translate: -50% calc(-100% - var(--padding-size) * 2 - var(--border-size) * 2);
}

.skip-to-content-link:focus {
Expand Down
Loading