chore: release [email protected]

.changeset/stale-houses-shout.md

@@ -0,0 +1,5 @@
+---
+"astro-loader-tweets": patch
+---
+
+Add data type validation for JSON-stored tweets and update docs.

README.md

@@ -8,7 +8,7 @@ Currently available loaders (see individual READMEs for details):
 
 - [astro-loader-github-releases](https://github.com/lin-stephanie/astro-loaders/tree/main/packages/astro-loader-github-releases): Loads GitHub releases for user-related or specific repositories.
 - [astro-loader-github-prs](https://github.com/lin-stephanie/astro-loaders/tree/main/packages/astro-loader-github-prs): Loads GitHub pull reuqests from a given search string.
-- [astro-loader-tweets](https://github.com/lin-stephanie/astro-loaders/tree/main/packages/astro-loader-tweets): Loads tweets from multiple tweet ids.
+- [astro-loader-tweets](https://github.com/lin-stephanie/astro-loaders/tree/main/packages/astro-loader-tweets): Loads tweets from multiple tweet IDs.
 
 ## Resources
 

packages/astro-loader-tweets/README.md

@@ -47,7 +47,7 @@ export const collections = { tweets }
 import { getCollection } from "astro:content"
 
 const tweets = await getCollection("tweets")
-// Check the entries' Zod schema for available props below
+// Check the entries' Zod schema for available fields below
 ---
 
 {
@@ -72,29 +72,27 @@ This loader retrieves tweets via the X API V2 [`GET /2/tweets`](https://develope
 | Option (* required)  | Type (default)                                              | Description                                                                                                                                                                                                                                                                                                                                                 |
 | -------------------- | ----------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `ids`*               | `string[]`                                                  | An array of Tweet IDs to fetch content for.                                                                                                                                                                                                                                                                                                                 |
-| `storage`            | `'default' \| 'custom' \| 'both'` (default: `'default'`)    | The method to store the loaded tweets:<br>`'default'`: Store data using Astro's default KV store.<br>`'custom'`: Use a custom local storage path and JSON file name.<br>`'both'`: Use both the default store and a custom path.                                                                                                                             |
-| `storePath`          | `string` (default: `'src/data/tweets.json'`)                | The custom local output path for storing tweets, either absolute or relative to the Astro project root. Must end with `.json`. Required if `storage` is `'custom'` or `'both'`.                                                                                                                                                                             |
+| `storage`            | `'default' \| 'custom' \| 'both'` (default: `'default'`)    | The method to store the loaded tweets:<br>`'default'`: Uses Astro's default KV store.<br>`'custom'`: Use a custom JSON file path.<br>`'both'`: Both default and custom path.                                                                                                                             |
+| `storePath`          | `string` (default: `'src/data/tweets.json'`)                | The custom output path for storing tweets, either absolute or relative to the Astro project root. Must end with `.json`. Required if `storage` is `'custom'` or `'both'`.                                                                                                                                                                             |
 | `removeTrailingUrls` | `boolean` (default: `true`)                                 | Whether to remove trailing URLs from the tweet text in the generated `text_html` and `text_markdown`, typically used for views or referenced tweets.                                                                                                                                                                                                        |
 | `linkTextType`       | `'domain-path' \| 'display-url'` (default: `'display-url'`) | The type of text to display for links when generating `text_html` and `text_markdown`:<br>`'domain-path'`: Displays the link's domain and path.<br>`'display-url'`: Uses the link text as shown in the tweet.                                                                                                                                               |
 | `newlineHandling`    | `'none' \| 'break' \| 'paragraph'` (default: `'none'`)      | The way for processing `\n` in `text_html` generation:<br>`'none'`: Keep as is.<br>`'break'`: Replace `\n` with `<br>`.<br>`'paragraph'`: Wrap paragraphs with `<p>` while removing standalone `\n`.                                                                                                                                                        |
 | `authToken`          | `string` (Defaults to the `X_TOKEN` environment variable)   | The X app-only Bearer Token for authentication. **If configured here, keep confidential and avoid public exposure.** See [how to create one](https://developer.x.com/en/docs/authentication/oauth-2-0/bearer-tokens) and [configure env vars in an Astro project](https://docs.astro.build/en/guides/environment-variables/#setting-environment-variables). |
 
 ### About the `storage` Configuration
 
-**Why not use the Astro loader’s default storage?**
+**Why not use the Astro default store?**
 
 - Each time the `content.config.ts` file is modified, Astro clears the [store](https://docs.astro.build/en/reference/content-loader-reference/#datastore) (i.e., `.astro/data-store.json` file).  
 - Under the X API V2 free plan, the endpoint requested are limited: only **1 request per 15 minutes** is allowed, with a maximum of **100 tweets retrievable per month**.
 
 **Benefits of custom JSON file storage**
 
-To overcome these limits, the loader allows configuring `storage` to store tweets in a custom JSON file, offering these advantages:
-
 - Newly loaded Tweets are appended or updated in the specified JSON file.  
 - Data can be edited while retaining the `id` attribute and structure, but repeated requests for the same Tweet ID will overwrite existing data.  
-- Stored Tweets persist unless the file is manually deleted, allowing up to 100 unique Tweets to be loaded monthly.
+- Stored tweets persist unless the file is manually deleted, allowing up to 100 unique Tweets to be loaded monthly.
 
-**Creating content collection from JSON File**
+**Creating content collection from JSON file**
 
 After storing tweets in a custom JSON file, you will need to define an additional content collection for rendering purposes:
 
@@ -130,13 +128,15 @@ const savedTweets = await getCollection("savedTweets")
 }
 ```
 
+This is my current solution. If you have a more scalable approach, please share it to address these challenges.
+
 ## Schema
 
-Check the Zod schema for loaded collection entries in the [source code](https://github.com/lin-stephanie/astro-loaders/blob/main/packages/astro-loader-tweets/src/schema.ts#L269). Astro automatically applies this schema to generate TypeScript interfaces, providing full support for autocompletion and type-checking when querying the collection.
+Refer to the [source code](https://github.com/lin-stephanie/astro-loaders/blob/main/packages/astro-loader-tweets/src/schema.ts#L269) for the Zod schema used for loaded collection entries. Astro automatically applies this schema to generate TypeScript interfaces, enabling autocompletion and type-checking for collection queries.
 
-Besides the fields directly fetched from the API, the loader extends fields defined in [`TweetV2WithRichContentSchema`](https://github.com/lin-stephanie/astro-loaders/blob/main/packages/astro-loader-tweets/src/schema.ts#L124), allowing for more streamlined control over the display of tweet content.
+In addition to API-fetched fields, the loader extends fields defined in the [`TweetV2ExtendedSchema`](https://github.com/lin-stephanie/astro-loaders/blob/main/packages/astro-loader-tweets/src/schema.ts#L124), simplifying the control over tweet content display.
 
-If you need to [customize the collection schema](https://docs.astro.build/en/guides/content-collections/#defining-the-collection-schema), ensure it remains compatible with the built-in Zod schema of the loader to avoid errors. For additional fields you'd like to fetch, feel free to [open an issue](https://github.com/lin-stephanie/astro-loaders/issues).
+To [customize the schema](https://docs.astro.build/en/guides/content-collections/#defining-the-collection-schema), ensure compatibility with the loader's built-in Zod schema to prevent errors. For additional fields, consider [opening an issue](https://github.com/lin-stephanie/astro-loaders/issues).
 
 [version-badge]: https://img.shields.io/npm/v/astro-loader-tweets?label=release&style=flat&colorA=080f12&colorB=ef7575
 [version-link]: https://www.npmjs.com/package/astro-loader-tweets

packages/astro-loader-tweets/package.json

@@ -1,7 +1,7 @@
 {
   "name": "astro-loader-tweets",
   "version": "1.2.0",
-  "description": "Aatro loader for loading tweets from multiple tweet ids.",
+  "description": "Aatro loader for loading tweets from multiple tweet IDs.",
   "author": "Stephanie Lin <[email protected]>",
   "license": "MIT",
   "keywords": [

packages/astro-loader-tweets/src/config.ts

@@ -18,9 +18,9 @@ export const TweetsLoaderConfigSchema = z
 
     /**
      * The method to store the loaded tweets:
-     * - `'default'`: Store data using Astro's default KV store (`./store/data-store.json`).
-     * - `'custom'`: Use a custom local storage path and JSON file name.
-     * - `'both'`: Use both the default store and a custom path.
+     * - `'default'`: Uses Astro's default KV store (`./store/data-store.json`).
+     * - `'custom'`: Use a custom JSON file path.
+     * - `'both'`: Both default and custom path.
      *
      * @default 'default'
      */
@@ -29,7 +29,7 @@ export const TweetsLoaderConfigSchema = z
       .default(defaultConfig.storage),
 
     /**
-     * The custom local output path for storing tweets, either absolute or
+     * The custom output path for storing tweets, either absolute or
      * relative to the Astro project root. Must end with `.json`.
      * Required if `storage` is `'custom'` or `'both'`.
      *

packages/astro-loader-tweets/src/index.ts

@@ -12,7 +12,7 @@ import type { Tweet } from './schema.js'
 const MAX_IDS_PER_REQUEST = 100
 
 /**
- * Aatro loader for loading tweets from multiple tweet ids.
+ * Aatro loader for loading tweets from multiple tweet IDs.
  *
  * @see https://github.com/lin-stephanie/astro-loaders/tree/main/packages/astro-loader-tweets
  */
@@ -35,7 +35,7 @@ Check out the configuration: ${pkg.homepage}README.md#configuration.`
       const token = authToken || import.meta.env.X_TOKEN
 
       if (ids.length === 0) {
-        logger.warn('No tweet ids provided')
+        logger.warn('No tweet IDs provided')
         return
       }
 
@@ -71,28 +71,31 @@ Check out the configuration: ${pkg.homepage}README.md#configuration.`
               id: item.id,
               data: item,
             })
-            const res = store.set({
+            store.set({
               id: item.id,
               data: parsedItem,
               digest: generateDigest(parsedItem),
               rendered: { html: item.tweet.text_html },
             })
-            console.log('id', item.tweet.id)
-            console.log('res', res)
           }
+          logger.info(
+            `Successfully loaded ${tweets.length} tweets into the Astro store`
+          )
         }
 
         if (storage === 'custom' || storage === 'both') {
-          const result = await saveOrUpdateTweets(tweets, storePath as string)
-          if (!result.success) {
-            logger.error((result.error as Error).message)
+          try {
+            await saveOrUpdateTweets(tweets, storePath as string)
+            logger.info(
+              `Successfully loaded ${tweets.length} tweets into '${storePath}'`
+            )
+          } catch (error) {
+            logger.error(
+              `Failed to save tweets to '${storePath}': ${(error as Error).message}`
+            )
             return
           }
         }
-
-        logger.info(
-          `Successfully loaded ${tweets.length} tweets${storage === 'custom' || storage === 'both' ? `, exported to '${storePath}'` : ''}`
-        )
       } catch (error) {
         if (
           error instanceof ApiResponseError &&

packages/astro-loader-tweets/src/schema.ts

@@ -121,7 +121,7 @@ export const TweetV2Schema = z.object({
   source: z.string().optional(),
 })
 
-export const TweetV2WithRichContentSchema = TweetV2Schema.extend({
+export const TweetV2ExtendedSchema = TweetV2Schema.extend({
   text_html: z.string(),
   text_markdown: z.string(),
   view_type: z.enum(['none', 'media', 'link']),
@@ -268,7 +268,7 @@ const ResIncludesSchema = z.object({
 
 export const TweetSchema = z.object({
   id: z.string(),
-  tweet: TweetV2WithRichContentSchema,
+  tweet: TweetV2ExtendedSchema,
   user: z.union([UserV2Schema, z.null()]),
   place: z.union([PlaceV2Schema, z.null()]),
   media: z.union([z.array(MediaObjectV2Schema), z.null()]),

packages/astro-loader-tweets/src/utils.ts

@@ -1,12 +1,13 @@
 import fs from 'node:fs/promises'
 import path from 'node:path'
 
-import type { z } from 'astro/zod'
+import { z } from 'astro/zod'
+
 import type { TweetsLoaderConfigSchema } from './config.js'
 import type {
   ResIncludes,
   TweetV2Schema,
-  TweetV2WithRichContentSchema,
+  TweetV2ExtendedSchema,
   Tweet,
 } from './schema.js'
 
@@ -60,7 +61,7 @@ export function processTweetText(
     z.infer<typeof TweetsLoaderConfigSchema>,
     'ids' | 'storage' | 'storePath' | 'authToken'
   >
-): z.infer<typeof TweetV2WithRichContentSchema> {
+): z.infer<typeof TweetV2ExtendedSchema> {
   const { removeTrailingUrls, linkTextType, newlineHandling } = options
 
   // const originalText = tweet.text
@@ -221,7 +222,7 @@ export function processTweetText(
  * into an array of tweet entries conforming to the expected entry schema.
  */
 export function processTweets(
-  processedTweets: z.infer<typeof TweetV2WithRichContentSchema>[],
+  processedTweets: z.infer<typeof TweetV2ExtendedSchema>[],
   includes: ResIncludes | undefined
 ): Tweet[] {
   if (!includes) {
@@ -278,6 +279,14 @@ export function processTweets(
   })
 }
 
+const SavedTweets = z.array(
+  z
+    .object({
+      id: z.string(),
+    })
+    .passthrough()
+)
+
 /**
  * Saves or updates tweets to a specified JSON file.
  *
@@ -288,15 +297,12 @@ export function processTweets(
 export async function saveOrUpdateTweets(
   tweets: Tweet[],
   storePath: string
-): Promise<{
-  success: boolean
-  error?: Error
-}> {
+): Promise<void> {
   const resolvedPath = path.isAbsolute(storePath)
     ? storePath
     : path.resolve(process.cwd(), storePath)
 
-  let savedTweets: Tweet[] = []
+  let savedTweets: z.infer<typeof SavedTweets> = []
   let fileExists = true
 
   // check if file exists
@@ -306,20 +312,16 @@ export async function saveOrUpdateTweets(
     fileExists = false
   }
 
-  // update existing tweets
   if (fileExists) {
-    try {
-      const fileContent = await fs.readFile(resolvedPath, 'utf-8')
-      savedTweets = JSON.parse(fileContent)
-      // savedTweets = TweetsSchema.parse(parsedData)
-    } catch (error) {
-      return {
-        success: false,
-        error: new Error(
-          `Failed to process the existing tweet file at '${resolvedPath}': ${String(error)}`
-        ),
-      }
-    }
+    // update existing tweets
+    const fileContent = await fs.readFile(resolvedPath, 'utf-8')
+    const parsedContent = JSON.parse(fileContent)
+    const parsedData = SavedTweets.safeParse(parsedContent)
+    if (!parsedData.success)
+      throw Error(
+        'Invalid JSON format. Ensure the file contains an array of objects, each with a valid `id` field as a string.'
+      )
+    savedTweets = parsedData.data
 
     // create a map of existing tweets for efficient lookup
     const savedTweetsMap = new Map(savedTweets.map((t) => [t.id, t]))
@@ -332,21 +334,6 @@ export async function saveOrUpdateTweets(
   }
 
   // write updated tweets back to file
-  try {
-    await fs.mkdir(path.dirname(resolvedPath), { recursive: true })
-    await fs.writeFile(
-      resolvedPath,
-      JSON.stringify(savedTweets, null, 2),
-      'utf8'
-    )
-  } catch (error) {
-    return {
-      success: false,
-      error: new Error(
-        `Failed to save tweets to '${resolvedPath}': ${String(error)}`
-      ),
-    }
-  }
-
-  return { success: true }
+  await fs.mkdir(path.dirname(resolvedPath), { recursive: true })
+  await fs.writeFile(resolvedPath, JSON.stringify(savedTweets, null, 2), 'utf8')
 }