diff --git a/.changeset/stale-houses-shout.md b/.changeset/stale-houses-shout.md new file mode 100644 index 0000000..6c0a406 --- /dev/null +++ b/.changeset/stale-houses-shout.md @@ -0,0 +1,5 @@ +--- +"astro-loader-tweets": patch +--- + +Add data type validation for JSON-stored tweets and update docs. diff --git a/README.md b/README.md index 3aed72e..7d8b01a 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Currently available loaders (see individual READMEs for details): - [astro-loader-github-releases](https://github.com/lin-stephanie/astro-loaders/tree/main/packages/astro-loader-github-releases): Loads GitHub releases for user-related or specific repositories. - [astro-loader-github-prs](https://github.com/lin-stephanie/astro-loaders/tree/main/packages/astro-loader-github-prs): Loads GitHub pull reuqests from a given search string. -- [astro-loader-tweets](https://github.com/lin-stephanie/astro-loaders/tree/main/packages/astro-loader-tweets): Loads tweets from multiple tweet ids. +- [astro-loader-tweets](https://github.com/lin-stephanie/astro-loaders/tree/main/packages/astro-loader-tweets): Loads tweets from multiple tweet IDs. ## Resources diff --git a/packages/astro-loader-tweets/README.md b/packages/astro-loader-tweets/README.md index 035eaa3..4f31d2a 100644 --- a/packages/astro-loader-tweets/README.md +++ b/packages/astro-loader-tweets/README.md @@ -47,7 +47,7 @@ export const collections = { tweets } import { getCollection } from "astro:content" const tweets = await getCollection("tweets") -// Check the entries' Zod schema for available props below +// Check the entries' Zod schema for available fields below --- { @@ -72,8 +72,8 @@ This loader retrieves tweets via the X API V2 [`GET /2/tweets`](https://develope | Option (* required) | Type (default) | Description | | -------------------- | ----------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `ids`* | `string[]` | An array of Tweet IDs to fetch content for. | -| `storage` | `'default' \| 'custom' \| 'both'` (default: `'default'`) | The method to store the loaded tweets:
`'default'`: Store data using Astro's default KV store.
`'custom'`: Use a custom local storage path and JSON file name.
`'both'`: Use both the default store and a custom path. | -| `storePath` | `string` (default: `'src/data/tweets.json'`) | The custom local output path for storing tweets, either absolute or relative to the Astro project root. Must end with `.json`. Required if `storage` is `'custom'` or `'both'`. | +| `storage` | `'default' \| 'custom' \| 'both'` (default: `'default'`) | The method to store the loaded tweets:
`'default'`: Uses Astro's default KV store.
`'custom'`: Use a custom JSON file path.
`'both'`: Both default and custom path. | +| `storePath` | `string` (default: `'src/data/tweets.json'`) | The custom output path for storing tweets, either absolute or relative to the Astro project root. Must end with `.json`. Required if `storage` is `'custom'` or `'both'`. | | `removeTrailingUrls` | `boolean` (default: `true`) | Whether to remove trailing URLs from the tweet text in the generated `text_html` and `text_markdown`, typically used for views or referenced tweets. | | `linkTextType` | `'domain-path' \| 'display-url'` (default: `'display-url'`) | The type of text to display for links when generating `text_html` and `text_markdown`:
`'domain-path'`: Displays the link's domain and path.
`'display-url'`: Uses the link text as shown in the tweet. | | `newlineHandling` | `'none' \| 'break' \| 'paragraph'` (default: `'none'`) | The way for processing `\n` in `text_html` generation:
`'none'`: Keep as is.
`'break'`: Replace `\n` with `
`.
`'paragraph'`: Wrap paragraphs with `

` while removing standalone `\n`. | @@ -81,20 +81,18 @@ This loader retrieves tweets via the X API V2 [`GET /2/tweets`](https://develope ### About the `storage` Configuration -**Why not use the Astro loader’s default storage?** +**Why not use the Astro default store?** - Each time the `content.config.ts` file is modified, Astro clears the [store](https://docs.astro.build/en/reference/content-loader-reference/#datastore) (i.e., `.astro/data-store.json` file). - Under the X API V2 free plan, the endpoint requested are limited: only **1 request per 15 minutes** is allowed, with a maximum of **100 tweets retrievable per month**. **Benefits of custom JSON file storage** -To overcome these limits, the loader allows configuring `storage` to store tweets in a custom JSON file, offering these advantages: - - Newly loaded Tweets are appended or updated in the specified JSON file. - Data can be edited while retaining the `id` attribute and structure, but repeated requests for the same Tweet ID will overwrite existing data. -- Stored Tweets persist unless the file is manually deleted, allowing up to 100 unique Tweets to be loaded monthly. +- Stored tweets persist unless the file is manually deleted, allowing up to 100 unique Tweets to be loaded monthly. -**Creating content collection from JSON File** +**Creating content collection from JSON file** After storing tweets in a custom JSON file, you will need to define an additional content collection for rendering purposes: @@ -130,13 +128,15 @@ const savedTweets = await getCollection("savedTweets") } ``` +This is my current solution. If you have a more scalable approach, please share it to address these challenges. + ## Schema -Check the Zod schema for loaded collection entries in the [source code](https://github.com/lin-stephanie/astro-loaders/blob/main/packages/astro-loader-tweets/src/schema.ts#L269). Astro automatically applies this schema to generate TypeScript interfaces, providing full support for autocompletion and type-checking when querying the collection. +Refer to the [source code](https://github.com/lin-stephanie/astro-loaders/blob/main/packages/astro-loader-tweets/src/schema.ts#L269) for the Zod schema used for loaded collection entries. Astro automatically applies this schema to generate TypeScript interfaces, enabling autocompletion and type-checking for collection queries. -Besides the fields directly fetched from the API, the loader extends fields defined in [`TweetV2WithRichContentSchema`](https://github.com/lin-stephanie/astro-loaders/blob/main/packages/astro-loader-tweets/src/schema.ts#L124), allowing for more streamlined control over the display of tweet content. +In addition to API-fetched fields, the loader extends fields defined in the [`TweetV2ExtendedSchema`](https://github.com/lin-stephanie/astro-loaders/blob/main/packages/astro-loader-tweets/src/schema.ts#L124), simplifying the control over tweet content display. -If you need to [customize the collection schema](https://docs.astro.build/en/guides/content-collections/#defining-the-collection-schema), ensure it remains compatible with the built-in Zod schema of the loader to avoid errors. For additional fields you'd like to fetch, feel free to [open an issue](https://github.com/lin-stephanie/astro-loaders/issues). +To [customize the schema](https://docs.astro.build/en/guides/content-collections/#defining-the-collection-schema), ensure compatibility with the loader's built-in Zod schema to prevent errors. For additional fields, consider [opening an issue](https://github.com/lin-stephanie/astro-loaders/issues). [version-badge]: https://img.shields.io/npm/v/astro-loader-tweets?label=release&style=flat&colorA=080f12&colorB=ef7575 [version-link]: https://www.npmjs.com/package/astro-loader-tweets diff --git a/packages/astro-loader-tweets/package.json b/packages/astro-loader-tweets/package.json index 6370b3c..ca70cdc 100644 --- a/packages/astro-loader-tweets/package.json +++ b/packages/astro-loader-tweets/package.json @@ -1,7 +1,7 @@ { "name": "astro-loader-tweets", "version": "1.2.0", - "description": "Aatro loader for loading tweets from multiple tweet ids.", + "description": "Aatro loader for loading tweets from multiple tweet IDs.", "author": "Stephanie Lin ", "license": "MIT", "keywords": [ diff --git a/packages/astro-loader-tweets/src/config.ts b/packages/astro-loader-tweets/src/config.ts index c07bf7a..0603843 100644 --- a/packages/astro-loader-tweets/src/config.ts +++ b/packages/astro-loader-tweets/src/config.ts @@ -18,9 +18,9 @@ export const TweetsLoaderConfigSchema = z /** * The method to store the loaded tweets: - * - `'default'`: Store data using Astro's default KV store (`./store/data-store.json`). - * - `'custom'`: Use a custom local storage path and JSON file name. - * - `'both'`: Use both the default store and a custom path. + * - `'default'`: Uses Astro's default KV store (`./store/data-store.json`). + * - `'custom'`: Use a custom JSON file path. + * - `'both'`: Both default and custom path. * * @default 'default' */ @@ -29,7 +29,7 @@ export const TweetsLoaderConfigSchema = z .default(defaultConfig.storage), /** - * The custom local output path for storing tweets, either absolute or + * The custom output path for storing tweets, either absolute or * relative to the Astro project root. Must end with `.json`. * Required if `storage` is `'custom'` or `'both'`. * diff --git a/packages/astro-loader-tweets/src/index.ts b/packages/astro-loader-tweets/src/index.ts index 1b8816d..c58cae9 100644 --- a/packages/astro-loader-tweets/src/index.ts +++ b/packages/astro-loader-tweets/src/index.ts @@ -12,7 +12,7 @@ import type { Tweet } from './schema.js' const MAX_IDS_PER_REQUEST = 100 /** - * Aatro loader for loading tweets from multiple tweet ids. + * Aatro loader for loading tweets from multiple tweet IDs. * * @see https://github.com/lin-stephanie/astro-loaders/tree/main/packages/astro-loader-tweets */ @@ -35,7 +35,7 @@ Check out the configuration: ${pkg.homepage}README.md#configuration.` const token = authToken || import.meta.env.X_TOKEN if (ids.length === 0) { - logger.warn('No tweet ids provided') + logger.warn('No tweet IDs provided') return } @@ -71,28 +71,31 @@ Check out the configuration: ${pkg.homepage}README.md#configuration.` id: item.id, data: item, }) - const res = store.set({ + store.set({ id: item.id, data: parsedItem, digest: generateDigest(parsedItem), rendered: { html: item.tweet.text_html }, }) - console.log('id', item.tweet.id) - console.log('res', res) } + logger.info( + `Successfully loaded ${tweets.length} tweets into the Astro store` + ) } if (storage === 'custom' || storage === 'both') { - const result = await saveOrUpdateTweets(tweets, storePath as string) - if (!result.success) { - logger.error((result.error as Error).message) + try { + await saveOrUpdateTweets(tweets, storePath as string) + logger.info( + `Successfully loaded ${tweets.length} tweets into '${storePath}'` + ) + } catch (error) { + logger.error( + `Failed to save tweets to '${storePath}': ${(error as Error).message}` + ) return } } - - logger.info( - `Successfully loaded ${tweets.length} tweets${storage === 'custom' || storage === 'both' ? `, exported to '${storePath}'` : ''}` - ) } catch (error) { if ( error instanceof ApiResponseError && diff --git a/packages/astro-loader-tweets/src/schema.ts b/packages/astro-loader-tweets/src/schema.ts index 94f0701..954bb86 100644 --- a/packages/astro-loader-tweets/src/schema.ts +++ b/packages/astro-loader-tweets/src/schema.ts @@ -121,7 +121,7 @@ export const TweetV2Schema = z.object({ source: z.string().optional(), }) -export const TweetV2WithRichContentSchema = TweetV2Schema.extend({ +export const TweetV2ExtendedSchema = TweetV2Schema.extend({ text_html: z.string(), text_markdown: z.string(), view_type: z.enum(['none', 'media', 'link']), @@ -268,7 +268,7 @@ const ResIncludesSchema = z.object({ export const TweetSchema = z.object({ id: z.string(), - tweet: TweetV2WithRichContentSchema, + tweet: TweetV2ExtendedSchema, user: z.union([UserV2Schema, z.null()]), place: z.union([PlaceV2Schema, z.null()]), media: z.union([z.array(MediaObjectV2Schema), z.null()]), diff --git a/packages/astro-loader-tweets/src/utils.ts b/packages/astro-loader-tweets/src/utils.ts index 09ff955..ef3cbf8 100644 --- a/packages/astro-loader-tweets/src/utils.ts +++ b/packages/astro-loader-tweets/src/utils.ts @@ -1,12 +1,13 @@ import fs from 'node:fs/promises' import path from 'node:path' -import type { z } from 'astro/zod' +import { z } from 'astro/zod' + import type { TweetsLoaderConfigSchema } from './config.js' import type { ResIncludes, TweetV2Schema, - TweetV2WithRichContentSchema, + TweetV2ExtendedSchema, Tweet, } from './schema.js' @@ -60,7 +61,7 @@ export function processTweetText( z.infer, 'ids' | 'storage' | 'storePath' | 'authToken' > -): z.infer { +): z.infer { const { removeTrailingUrls, linkTextType, newlineHandling } = options // const originalText = tweet.text @@ -221,7 +222,7 @@ export function processTweetText( * into an array of tweet entries conforming to the expected entry schema. */ export function processTweets( - processedTweets: z.infer[], + processedTweets: z.infer[], includes: ResIncludes | undefined ): Tweet[] { if (!includes) { @@ -278,6 +279,14 @@ export function processTweets( }) } +const SavedTweets = z.array( + z + .object({ + id: z.string(), + }) + .passthrough() +) + /** * Saves or updates tweets to a specified JSON file. * @@ -288,15 +297,12 @@ export function processTweets( export async function saveOrUpdateTweets( tweets: Tweet[], storePath: string -): Promise<{ - success: boolean - error?: Error -}> { +): Promise { const resolvedPath = path.isAbsolute(storePath) ? storePath : path.resolve(process.cwd(), storePath) - let savedTweets: Tweet[] = [] + let savedTweets: z.infer = [] let fileExists = true // check if file exists @@ -306,20 +312,16 @@ export async function saveOrUpdateTweets( fileExists = false } - // update existing tweets if (fileExists) { - try { - const fileContent = await fs.readFile(resolvedPath, 'utf-8') - savedTweets = JSON.parse(fileContent) - // savedTweets = TweetsSchema.parse(parsedData) - } catch (error) { - return { - success: false, - error: new Error( - `Failed to process the existing tweet file at '${resolvedPath}': ${String(error)}` - ), - } - } + // update existing tweets + const fileContent = await fs.readFile(resolvedPath, 'utf-8') + const parsedContent = JSON.parse(fileContent) + const parsedData = SavedTweets.safeParse(parsedContent) + if (!parsedData.success) + throw Error( + 'Invalid JSON format. Ensure the file contains an array of objects, each with a valid `id` field as a string.' + ) + savedTweets = parsedData.data // create a map of existing tweets for efficient lookup const savedTweetsMap = new Map(savedTweets.map((t) => [t.id, t])) @@ -332,21 +334,6 @@ export async function saveOrUpdateTweets( } // write updated tweets back to file - try { - await fs.mkdir(path.dirname(resolvedPath), { recursive: true }) - await fs.writeFile( - resolvedPath, - JSON.stringify(savedTweets, null, 2), - 'utf8' - ) - } catch (error) { - return { - success: false, - error: new Error( - `Failed to save tweets to '${resolvedPath}': ${String(error)}` - ), - } - } - - return { success: true } + await fs.mkdir(path.dirname(resolvedPath), { recursive: true }) + await fs.writeFile(resolvedPath, JSON.stringify(savedTweets, null, 2), 'utf8') }