From d599186f84184d133ac633c3457144be358e4b8f Mon Sep 17 00:00:00 2001 From: Michael Levin Date: Thu, 20 Feb 2025 15:39:39 -0500 Subject: [PATCH] [Feature] Add report for top pages by views and active users - Space out publisher jobs by 2 mins instead of 1 --- deploy/publisher.js | 18 +- reports/usa.json | 1262 +++++++++++++------------------------------ 2 files changed, 378 insertions(+), 902 deletions(-) diff --git a/deploy/publisher.js b/deploy/publisher.js index cbf1c3d5..ac404c89 100644 --- a/deploy/publisher.js +++ b/deploy/publisher.js @@ -20,26 +20,26 @@ const bree = new Bree({ timeout: "1", interval: "15m", }, - // Runs `../jobs/analytics.usa.gov_daily.js` 1 minute after the process starts and then at + // Runs `../jobs/analytics.usa.gov_daily.js` 2 minute after the process starts and then at // 10:01 AM every day going forward. { name: "analytics.usa.gov_daily", - timeout: "1m", - interval: "at 10:01 am", + timeout: "2m", + interval: "at 10:02 am", }, - // Runs `../jobs/daily.js` 2 minutes after the process starts and then at + // Runs `../jobs/daily.js` 4 minutes after the process starts and then at // 10:02 AM every day going forward. { name: "daily", - timeout: "2m", - interval: "at 10:02 am", + timeout: "4m", + interval: "at 10:04 am", }, - // Runs `../jobs/api.js` 3 minutes after the process starts and then at + // Runs `../jobs/api.js` 6 minutes after the process starts and then at // 10:03 AM every day going forward. { name: "api", - timeout: "3m", - interval: "at 10:03 am", + timeout: "6m", + interval: "at 10:06 am", }, ], }); diff --git a/reports/usa.json b/reports/usa.json index eaa9bb7d..6e73522f 100644 --- a/reports/usa.json +++ b/reports/usa.json @@ -100,41 +100,14 @@ "notExpression": { "filter": { "fieldName": "screenResolution", - "stringFilter": { - "value": "(other)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "screenResolution", - "stringFilter": { - "value": "null", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "screenResolution", - "stringFilter": { - "value": "", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "screenResolution", - "stringFilter": { - "value": "(not set)", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -200,74 +173,14 @@ "notExpression": { "filter": { "fieldName": "language", - "stringFilter": { - "value": "(other)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "language", - "stringFilter": { - "value": "null", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "language", - "stringFilter": { - "value": "", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "language", - "stringFilter": { - "value": "(not set)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "languageCode", - "stringFilter": { - "value": "(other)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "languageCode", - "stringFilter": { - "value": "null", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "languageCode", - "stringFilter": { - "value": "", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -277,8 +190,14 @@ "notExpression": { "filter": { "fieldName": "languageCode", - "stringFilter": { - "value": "(not set)", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -338,41 +257,14 @@ "notExpression": { "filter": { "fieldName": "deviceCategory", - "stringFilter": { - "value": "(other)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "deviceCategory", - "stringFilter": { - "value": "null", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "deviceCategory", - "stringFilter": { - "value": "", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "deviceCategory", - "stringFilter": { - "value": "(not set)", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -434,41 +326,14 @@ "notExpression": { "filter": { "fieldName": "deviceCategory", - "stringFilter": { - "value": "(other)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "deviceCategory", - "stringFilter": { - "value": "null", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "deviceCategory", - "stringFilter": { - "value": "", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "deviceCategory", - "stringFilter": { - "value": "(not set)", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -519,41 +384,14 @@ "notExpression": { "filter": { "fieldName": "mobileDeviceModel", - "stringFilter": { - "value": "(other)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "mobileDeviceModel", - "stringFilter": { - "value": "null", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "mobileDeviceModel", - "stringFilter": { - "value": "", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "mobileDeviceModel", - "stringFilter": { - "value": "(not set)", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -621,41 +459,14 @@ "notExpression": { "filter": { "fieldName": "operatingSystem", - "stringFilter": { - "value": "(other)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "operatingSystem", - "stringFilter": { - "value": "null", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "operatingSystem", - "stringFilter": { - "value": "", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "operatingSystem", - "stringFilter": { - "value": "(not set)", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -736,41 +547,14 @@ "notExpression": { "filter": { "fieldName": "operatingSystemVersion", - "stringFilter": { - "value": "(other)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "operatingSystemVersion", - "stringFilter": { - "value": "null", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "operatingSystemVersion", - "stringFilter": { - "value": "", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "operatingSystemVersion", - "stringFilter": { - "value": "(not set)", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -832,41 +616,14 @@ "notExpression": { "filter": { "fieldName": "browser", - "stringFilter": { - "value": "(other)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "browser", - "stringFilter": { - "value": "null", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "browser", - "stringFilter": { - "value": "", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "browser", - "stringFilter": { - "value": "(not set)", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -928,74 +685,14 @@ "notExpression": { "filter": { "fieldName": "browser", - "stringFilter": { - "value": "(other)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "browser", - "stringFilter": { - "value": "null", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "browser", - "stringFilter": { - "value": "", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "browser", - "stringFilter": { - "value": "(not set)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "operatingSystem", - "stringFilter": { - "value": "(other)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "operatingSystem", - "stringFilter": { - "value": "null", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "operatingSystem", - "stringFilter": { - "value": "", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -1005,8 +702,14 @@ "notExpression": { "filter": { "fieldName": "operatingSystem", - "stringFilter": { - "value": "(not set)", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -1075,74 +778,14 @@ "notExpression": { "filter": { "fieldName": "browser", - "stringFilter": { - "value": "(other)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "browser", - "stringFilter": { - "value": "null", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "browser", - "stringFilter": { - "value": "", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "browser", - "stringFilter": { - "value": "(not set)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "operatingSystemVersion", - "stringFilter": { - "value": "(other)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "operatingSystemVersion", - "stringFilter": { - "value": "null", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "operatingSystemVersion", - "stringFilter": { - "value": "", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -1152,8 +795,14 @@ "notExpression": { "filter": { "fieldName": "operatingSystemVersion", - "stringFilter": { - "value": "(not set)", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -1228,41 +877,14 @@ "notExpression": { "filter": { "fieldName": "city", - "stringFilter": { - "value": "(other)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "city", - "stringFilter": { - "value": "null", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "city", - "stringFilter": { - "value": "", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "city", - "stringFilter": { - "value": "(not set)", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -1321,41 +943,14 @@ "notExpression": { "filter": { "fieldName": "city", - "stringFilter": { - "value": "(other)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "city", - "stringFilter": { - "value": "null", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "city", - "stringFilter": { - "value": "", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "city", - "stringFilter": { - "value": "(not set)", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -1400,41 +995,14 @@ "notExpression": { "filter": { "fieldName": "country", - "stringFilter": { - "value": "(other)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "country", - "stringFilter": { - "value": "null", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "country", - "stringFilter": { - "value": "", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "country", - "stringFilter": { - "value": "(not set)", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -1489,41 +1057,14 @@ "notExpression": { "filter": { "fieldName": "country", - "stringFilter": { - "value": "(other)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "country", - "stringFilter": { - "value": "null", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "country", - "stringFilter": { - "value": "", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "country", - "stringFilter": { - "value": "(not set)", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -1576,8 +1117,14 @@ "notExpression": { "filter": { "fieldName": "unifiedScreenName", - "stringFilter": { - "value": "(other)", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -1588,7 +1135,74 @@ "filter": { "fieldName": "unifiedScreenName", "stringFilter": { - "value": "null", + "matchType": "FULL_REGEXP", + "value": "\/(.+)$", + "caseSensitive": false + } + } + } + } + ] + } + }, + "limit": "30" + }, + "meta": { + "name": "Top Pages (Live)", + "description": "The top 30 pages, measured by active onsite users, for all sites." + } + }, + { + "name": "top-viewed-pages", + "frequency": "daily", + "dateRanges": [ + "yesterday", + "7-days", + "30-days", + "90-days", + "current-year", + "current-fiscal-year", + "previous-year", + "previous-fiscal-year" + ], + "dateRangeChunkSize": 1, + "query": { + "dimensions": [ + { + "name": "hostName" + }, + { + "name": "pagePath" + } + ], + "metrics": [ + { + "name": "screenPageViews" + } + ], + "orderBys": [ + { + "metric": { + "metricName": "screenPageViews" + }, + "desc": true + } + ], + "dimensionFilter": { + "andGroup": { + "expressions": [ + { + "notExpression": { + "filter": { + "fieldName": "hostName", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -1597,9 +1211,80 @@ { "notExpression": { "filter": { - "fieldName": "unifiedScreenName", - "stringFilter": { - "value": "", + "fieldName": "pagePath", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ] + } + } + } + } + ] + } + }, + "limit": "10000" + }, + "meta": { + "name": "Top 10000 pages people are viewing", + "description": "The top 10000 pages, measured by screen page views for all sites." + } + }, + { + "name": "top-pages-by-active-user", + "frequency": "daily", + "dateRanges": [ + "yesterday", + "7-days", + "30-days", + "90-days", + "current-year", + "current-fiscal-year", + "previous-year", + "previous-fiscal-year" + ], + "dateRangeChunkSize": 1, + "query": { + "dimensions": [ + { + "name": "hostName" + }, + { + "name": "pagePath" + } + ], + "metrics": [ + { + "name": "activeUsers" + } + ], + "orderBys": [ + { + "metric": { + "metricName": "activeUsers" + }, + "desc": true + } + ], + "dimensionFilter": { + "andGroup": { + "expressions": [ + { + "notExpression": { + "filter": { + "fieldName": "hostName", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -1608,11 +1293,15 @@ { "notExpression": { "filter": { - "fieldName": "unifiedScreenName", - "stringFilter": { - "matchType": "FULL_REGEXP", - "value": "\/(.+)$", - "caseSensitive": false + "fieldName": "pagePath", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ] } } } @@ -1620,11 +1309,11 @@ ] } }, - "limit": "30" + "limit": "10000" }, "meta": { - "name": "Top Pages (Live)", - "description": "The top 30 pages, measured by active onsite users, for all sites." + "name": "Top 10000 pages by active users", + "description": "The top 10000 pages, measured by active users for all sites." } }, { @@ -1669,63 +1358,16 @@ "filter": { "fieldName": "unifiedScreenName", "stringFilter": { - "value": "(other)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "unifiedScreenName", - "stringFilter": { - "value": "other", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "unifiedScreenName", - "stringFilter": { - "value": "(not set)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "unifiedScreenName", - "stringFilter": { - "value": "/", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "unifiedScreenName", - "stringFilter": { - "value": "null", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "unifiedScreenName", - "stringFilter": { - "value": "", - "caseSensitive": false + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], + "caseSensitive": false + } } } } @@ -1798,30 +1440,14 @@ "notExpression": { "filter": { "fieldName": "hostName", - "stringFilter": { - "value": "(other)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "hostName", - "stringFilter": { - "value": "null", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "hostName", - "stringFilter": { - "value": "", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -1919,30 +1545,15 @@ "notExpression": { "filter": { "fieldName": "sessionDefaultChannelGroup", - "stringFilter": { - "value": "unassigned", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "sessionDefaultChannelGroup", - "stringFilter": { - "value": "paid other", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "sessionDefaultChannelGroup", - "stringFilter": { - "value": "(other)", + "inListFilter": { + "values": [ + "unassigned", + "paid other", + "(other)", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -2002,52 +1613,17 @@ "notExpression": { "filter": { "fieldName": "sessionSourceMedium", - "stringFilter": { - "value": "(not set)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "sessionSourceMedium", - "stringFilter": { - "value": "other / email", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "sessionSourceMedium", - "stringFilter": { - "value": "expected / email", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "sessionSourceMedium", - "stringFilter": { - "value": "outfordelivery / email", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "sessionSourceMedium", - "stringFilter": { - "value": "(other)", + "inListFilter": { + "values": [ + "other / email", + "expected / email", + "outfordelivery / email", + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -2153,76 +1729,24 @@ "filter": { "fieldName": "eventName", "stringFilter": { - "matchType": "FULL_REGEXP", - "value": "^(file_download|download|downloads|(outbound downloads))$", + "value": "file_download", "caseSensitive": false } }, "andGroup": { "expressions": [ - { - "notExpression": { - "filter": { - "fieldName": "fileName", - "stringFilter": { - "matchType": "PARTIAL_REGEXP", - "value": ".*\\\\.(zip|doc)\\b.*", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "fullPageUrl", - "stringFilter": { - "matchType": "PARTIAL_REGEXP", - "value": ".*\\busps\\.com\\b.*", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "pageTitle", - "stringFilter": { - "value": "(other)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "pageTitle", - "stringFilter": { - "value": "null", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "pageTitle", - "stringFilter": { - "value": "", - "caseSensitive": false - } - } - } - }, { "notExpression": { "filter": { "fieldName": "pageTitle", - "stringFilter": { - "value": "(not set)", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -2268,30 +1792,14 @@ "notExpression": { "filter": { "fieldName": "unifiedScreenName", - "stringFilter": { - "value": "(other)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "unifiedScreenName", - "stringFilter": { - "value": "null", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "unifiedScreenName", - "stringFilter": { - "value": "", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -2345,30 +1853,14 @@ "notExpression": { "filter": { "fieldName": "hostname", - "stringFilter": { - "value": "(other)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "hostname", - "stringFilter": { - "value": "null", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "hostname", - "stringFilter": { - "value": "", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } } @@ -2471,30 +1963,14 @@ "notExpression": { "filter": { "fieldName": "videoTitle", - "stringFilter": { - "value": "(other)", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "videoTitle", - "stringFilter": { - "value": "null", - "caseSensitive": false - } - } - } - }, - { - "notExpression": { - "filter": { - "fieldName": "videoTitle", - "stringFilter": { - "value": "", + "inListFilter": { + "values": [ + "(other)", + "other", + "(not set)", + "null", + "" + ], "caseSensitive": false } }