chore: update utils

DIYgod · Mar 12, 2023 · 38335f6 · 38335f6
1 parent 2adb6ac
commit 38335f6
Showing 1 changed file with 5 additions and 4 deletions.
diff --git a/lib/v2/wsj/utils.js b/lib/v2/wsj/utils.js
@@ -2,8 +2,9 @@ const asyncPool = require('tiny-async-pool');
 const cheerio = require('cheerio');
 const got = require('@/utils/got');
 const { parseDate } = require('@/utils/parse-date');
+const UA = require('@/utils/rand-user-agent')({ browser: 'chrome', os: 'android', device: 'mobile' });
 
-const chromeMobileUserAgent = 'Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/67.0.3396.87 Mobile Safari/537.36';
+// const chromeMobileUserAgent = 'Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/67.0.3396.87 Mobile Safari/537.36';
 const parseArticle = (item, ctx) =>
     ctx.cache.tryGet(item.link, async () => {
         // Fetch the AMP version
@@ -12,10 +13,10 @@ const parseArticle = (item, ctx) =>
             url,
             method: 'get',
             headers: {
-                'User-Agent': chromeMobileUserAgent,
+                'User-Agent': UA,
             },
         });
-        const html = response.body;
+        const html = response.data;
         const $ = cheerio.load(html);
         const content = $('.articleBody > section');
 
@@ -33,7 +34,7 @@ const parseArticle = (item, ctx) =>
         // Metadata (categories & updatedAt)
         const updatedAt = $('meta[itemprop="dateModified"]').attr('content');
         const publishedAt = $('meta[itemprop="datePublished"]').attr('content');
-        const author = $('.author > a[rel="author"]').html();
+        const author = $('.author > a[rel="author"]').text();
 
         const categories = $('meta[name="keywords"]')
             .attr('content')