Sql linting fixes (#2379)

* Structured Data fixes * CSS fixes * Fix CMS queries * Fix typo * Add Linting check on push to main * Revert "Fix CMS queries" This reverts commit 29b2e3d. * Fogot to hit save after fixing merge conflict
HTTPArchive · Oct 16, 2021 · 095a9ae · 095a9ae
1 parent 5ede879
commit 095a9ae
Show file tree

Hide file tree

Showing 23 changed files with 187 additions and 190 deletions.
diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml
@@ -8,8 +8,11 @@
 #
 name: Lint Code Base
 on:
-  - workflow_dispatch
-  - pull_request
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - main
 jobs:
   lint:
     name: Lint Code Base

diff --git a/sql/2021/css/box_sizing_border_box_selectors.sql b/sql/2021/css/box_sizing_border_box_selectors.sql
@@ -1,24 +1,24 @@
 #standardSQL
 # Top selectors used with box-sizing: border-box
 CREATE TEMP FUNCTION
-  getBorderBoxSelectors(css STRING)
-  RETURNS ARRAY<STRING>
-  LANGUAGE js AS '''
+getBorderBoxSelectors(css STRING)
+RETURNS ARRAY<STRING>
+LANGUAGE js AS '''
 try {
   var $ = JSON.parse(css);
   return $.stylesheet.rules.flatMap(rule => {
     if (!rule.selectors) {
       return [];
     }
-    
+
     const boxSizingPattern = /^(-(o|moz|webkit|ms)-)?box-sizing$/;
     const borderBoxPattern = /border-box/;
     if (!rule.declarations.find(d => {
       return boxSizingPattern.test(d.property) && borderBoxPattern.test(d.value);
     })) {
       return [];
     }
-    
+
     return rule.selectors;
   });
 } catch (e) {
@@ -62,5 +62,4 @@ FROM (
     selector)
 ORDER BY
   pct DESC
-LIMIT
-  1000
+LIMIT 1000
diff --git a/sql/2021/css/content_values.sql b/sql/2021/css/content_values.sql
@@ -26,8 +26,8 @@ FROM (
     client,
     IF(
       REGEXP_CONTAINS(content, r'[\'"]\\[ef][0-9a-f]{3}[\'"]'), '"\\f000"-like',
-    IF(
-      REGEXP_CONTAINS(content, r'[\'"]\\[a-f0-9]{4}[\'"]'), '"\\hex{4}"-like', content)) AS content,
+      IF(
+        REGEXP_CONTAINS(content, r'[\'"]\\[a-f0-9]{4}[\'"]'), '"\\hex{4}"-like', content)) AS content,
     COUNT(DISTINCT page) AS pages,
     total_pages,
     COUNT(DISTINCT page) / total_pages AS pct_pages,
@@ -59,5 +59,4 @@ WHERE
   pages >= 1000
 ORDER BY
   pct_pages DESC
-LIMIT
-  200
+LIMIT 200
diff --git a/sql/2021/css/flexbox_grid.sql b/sql/2021/css/flexbox_grid.sql
@@ -9,7 +9,7 @@ WITH totals AS (
     `httparchive.summary_pages.2021_07_01_*`
   GROUP BY
     client
-UNION ALL
+  UNION ALL
   SELECT
     CAST('2020-08-01' AS DATE) AS yyyymmdd,
     _TABLE_SUFFIX AS client,
@@ -18,7 +18,7 @@ UNION ALL
     `httparchive.summary_pages.2020_08_01_*`
   GROUP BY
     client
-UNION ALL
+  UNION ALL
   SELECT
     CAST('2019-07-01' AS DATE) AS yyyymmdd,
     _TABLE_SUFFIX AS client,

diff --git a/sql/2021/structured-data/classic_microformats_types.sql b/sql/2021/structured-data/classic_microformats_types.sql
@@ -1,18 +1,18 @@
 # standardSQL
-  # Count Classic Microformats types
-CREATE TEMP FUNCTION
-  getClassicMicroformatsTypes(rendered STRING)
-  RETURNS ARRAY<STRUCT<name STRING, count NUMERIC>>
-  LANGUAGE js AS r"""
+# Count Classic Microformats types
+CREATE TEMP FUNCTION getClassicMicroformatsTypes(rendered STRING)
+RETURNS ARRAY<STRUCT<name STRING, count NUMERIC>>
+LANGUAGE js AS """
   try {
     rendered = JSON.parse(rendered);
     return rendered.microformats_classic_types.map(microformats_classic_type => ({name: microformats_classic_type.name, count: microformats_classic_type.count}));
   } catch (e) {
     return [];
   }
 """;
+
 WITH
-  rendered_data AS (
+rendered_data AS (
   SELECT
     getClassicMicroformatsTypes(rendered) AS classic_microformats_types,
     client

diff --git a/sql/2021/structured-data/dublin_core_types.sql b/sql/2021/structured-data/dublin_core_types.sql
@@ -1,18 +1,18 @@
 # standardSQL
-  # Count Dublin Core types
-CREATE TEMP FUNCTION
-  getDublinCoreTypes(rendered STRING)
-  RETURNS ARRAY<STRING>
-  LANGUAGE js AS r"""
+# Count Dublin Core types
+CREATE TEMP FUNCTION getDublinCoreTypes(rendered STRING)
+RETURNS ARRAY<STRING>
+LANGUAGE js AS """
   try {
     rendered = JSON.parse(rendered);
     return rendered.dublin_core.map(dublin_core => dublin_core.name.toLowerCase());
   } catch (e) {
     return [];
   }
 """;
+
 WITH
-  rendered_data AS (
+rendered_data AS (
   SELECT
     getDublinCoreTypes(rendered) AS dublin_core_types,
     client

diff --git a/sql/2021/structured-data/facebook_types.sql b/sql/2021/structured-data/facebook_types.sql
@@ -1,18 +1,18 @@
 # standardSQL
-  # Count Facebook types
-CREATE TEMP FUNCTION
-  getFacebookTypes(rendered STRING)
-  RETURNS ARRAY<STRING>
-  LANGUAGE js AS r"""
+# Count Facebook types
+CREATE TEMP FUNCTION getFacebookTypes(rendered STRING)
+RETURNS ARRAY<STRING>
+LANGUAGE js AS """
   try {
     rendered = JSON.parse(rendered);
     return rendered.facebook.map(facebook => facebook.property.toLowerCase());
   } catch (e) {
     return [];
   }
 """;
+
 WITH
-  rendered_data AS (
+rendered_data AS (
   SELECT
     getFacebookTypes(rendered) AS facebook_types,
     client

diff --git a/sql/2021/structured-data/jsonld_contexts.sql b/sql/2021/structured-data/jsonld_contexts.sql
@@ -1,9 +1,8 @@
 # standardSQL
-  # Count JSON-LD contexts
-CREATE TEMP FUNCTION
-  getJSONLDContexts(rendered STRING)
-  RETURNS ARRAY<STRING>
-  LANGUAGE js AS r"""
+# Count JSON-LD contexts
+CREATE TEMP FUNCTION getJSONLDContexts(rendered STRING)
+RETURNS ARRAY<STRING>
+LANGUAGE js AS """
   try {
     const arrayify = (value) => Array.isArray(value) ? value : [value];
 
@@ -30,8 +29,9 @@ CREATE TEMP FUNCTION
     return [];
   }
 """;
+
 WITH
-  rendered_data AS (
+rendered_data AS (
   SELECT
     getJSONLDContexts(rendered) AS jsonld_contexts,
     client

diff --git a/sql/2021/structured-data/jsonld_entities_relationships.sql b/sql/2021/structured-data/jsonld_entities_relationships.sql
@@ -1,9 +1,8 @@
 # standardSQL
-  # Count JSON-LD entities and relationships
-CREATE TEMP FUNCTION
-  getJSONLDEntitiesRelationships(rendered STRING)
-  RETURNS ARRAY<STRUCT<_from STRING, relationship STRING, _to STRING, depth NUMERIC>>
-  LANGUAGE js AS r"""
+# Count JSON-LD entities and relationships
+CREATE TEMP FUNCTION getJSONLDEntitiesRelationships(rendered STRING)
+RETURNS ARRAY<STRUCT<_from STRING, relationship STRING, _to STRING, depth NUMERIC>>
+LANGUAGE js AS """
   try {
     const types = new Map();
 
@@ -41,8 +40,9 @@ CREATE TEMP FUNCTION
     return [];
   }
 """;
+
 WITH
-  rendered_data AS (
+rendered_data AS (
   SELECT
     getJSONLDEntitiesRelationships(rendered) AS jsonld_entities_relationships,
     client
@@ -74,5 +74,4 @@ GROUP BY
   client
 ORDER BY
   count DESC
-LIMIT
-  10000
+LIMIT 10000
diff --git a/sql/2021/structured-data/jsonld_max_depth.sql b/sql/2021/structured-data/jsonld_max_depth.sql
@@ -1,9 +1,8 @@
 # standardSQL
-  # Find the most nested entity in a JSON-LD document
-CREATE TEMP FUNCTION
-  getJSONLDEntitiesRelationships(rendered STRING)
-  RETURNS ARRAY<STRUCT<_from STRING, relationship STRING, _to STRING, depth NUMERIC>>
-  LANGUAGE js AS r"""
+# Find the most nested entity in a JSON-LD document
+CREATE TEMP FUNCTION getJSONLDEntitiesRelationships(rendered STRING)
+RETURNS ARRAY<STRUCT<_from STRING, relationship STRING, _to STRING, depth NUMERIC>>
+LANGUAGE js AS """
   try {
     const types = new Map();
 
@@ -41,8 +40,9 @@ CREATE TEMP FUNCTION
     return [];
   }
 """;
+
 WITH
-  rendered_data AS (
+rendered_data AS (
   SELECT
     getJSONLDEntitiesRelationships(rendered) AS jsonld_entities_relationships,
     client

diff --git a/sql/2021/structured-data/jsonld_relationship_values.sql b/sql/2021/structured-data/jsonld_relationship_values.sql
@@ -1,9 +1,8 @@
 # standardSQL
-  # Count JSON-LD relationships with their value entity types
-CREATE TEMP FUNCTION
-  getJSONLDEntitiesRelationships(rendered STRING)
-  RETURNS ARRAY<STRUCT<_from STRING, relationship STRING, _to STRING, depth NUMERIC>>
-  LANGUAGE js AS r"""
+# Count JSON-LD relationships with their value entity types
+CREATE TEMP FUNCTION getJSONLDEntitiesRelationships(rendered STRING)
+RETURNS ARRAY<STRUCT<_from STRING, relationship STRING, _to STRING, depth NUMERIC>>
+LANGUAGE js AS """
   try {
     const types = new Map();
 
@@ -41,8 +40,9 @@ CREATE TEMP FUNCTION
     return [];
   }
 """;
+
 WITH
-  rendered_data AS (
+rendered_data AS (
   SELECT
     getJSONLDEntitiesRelationships(rendered) AS jsonld_entities_relationships,
     client
@@ -70,5 +70,4 @@ GROUP BY
   client
 ORDER BY
   count DESC
-LIMIT
-  10000
+LIMIT 10000
diff --git a/sql/2021/structured-data/jsonld_relationships.sql b/sql/2021/structured-data/jsonld_relationships.sql
@@ -1,9 +1,8 @@
 # standardSQL
-  # Count JSON-LD relationships
-CREATE TEMP FUNCTION
-  getJSONLDRelationships(rendered STRING)
-  RETURNS ARRAY<STRING>
-  LANGUAGE js AS r"""
+# Count JSON-LD relationships
+CREATE TEMP FUNCTION getJSONLDRelationships(rendered STRING)
+RETURNS ARRAY<STRING>
+LANGUAGE js AS """
   try {
     const arrayify = (value) => Array.isArray(value) ? value : [value];
 
@@ -30,8 +29,9 @@ CREATE TEMP FUNCTION
     return [];
   }
 """;
+
 WITH
-  rendered_data AS (
+rendered_data AS (
   SELECT
     getJSONLDRelationships(rendered) AS jsonld_relationships,
     client
@@ -59,5 +59,4 @@ GROUP BY
   client
 ORDER BY
   count DESC
-LIMIT
-  10000
+LIMIT 10000
diff --git a/sql/2021/structured-data/jsonld_same_ases.sql b/sql/2021/structured-data/jsonld_same_ases.sql
@@ -1,9 +1,8 @@
 # standardSQL
-  # Count JSON-LD sameAs values
-CREATE TEMP FUNCTION
-  getJSONLDSameAses(rendered STRING)
-  RETURNS ARRAY<STRING>
-  LANGUAGE js AS r"""
+# Count JSON-LD sameAs values
+CREATE TEMP FUNCTION getJSONLDSameAses(rendered STRING)
+RETURNS ARRAY<STRING>
+LANGUAGE js AS """
   try {
     const arrayify = (value) => Array.isArray(value) ? value : [value];
 
@@ -30,8 +29,9 @@ CREATE TEMP FUNCTION
     return [];
   }
 """;
+
 WITH
-  rendered_data AS (
+rendered_data AS (
   SELECT
     getJSONLDSameAses(rendered) AS jsonld_sameases,
     client
@@ -59,5 +59,4 @@ GROUP BY
   client
 ORDER BY
   count DESC
-LIMIT
-  10000
+LIMIT 10000
diff --git a/sql/2021/structured-data/jsonld_types.sql b/sql/2021/structured-data/jsonld_types.sql
@@ -1,9 +1,8 @@
 # standardSQL
-  # Count JSON-LD types
-CREATE TEMP FUNCTION
-  getJSONLDTypes(rendered STRING)
-  RETURNS ARRAY<STRING>
-  LANGUAGE js AS r"""
+# Count JSON-LD types
+CREATE TEMP FUNCTION getJSONLDTypes(rendered STRING)
+RETURNS ARRAY<STRING>
+LANGUAGE js AS """
   try {
     const arrayify = (value) => Array.isArray(value) ? value : [value];
 
@@ -30,8 +29,9 @@ CREATE TEMP FUNCTION
     return [];
   }
 """;
+
 WITH
-  rendered_data AS (
+rendered_data AS (
   SELECT
     getJSONLDTypes(rendered) AS jsonld_types,
     client

diff --git a/sql/2021/structured-data/microdata_item_types.sql b/sql/2021/structured-data/microdata_item_types.sql
@@ -1,18 +1,18 @@
 # standardSQL
-  # Count Microdata item types
-CREATE TEMP FUNCTION
-  getMicrodataItemTypes(rendered STRING)
-  RETURNS ARRAY<STRING>
-  LANGUAGE js AS r"""
+# Count Microdata item types
+CREATE TEMP FUNCTION getMicrodataItemTypes(rendered STRING)
+RETURNS ARRAY<STRING>
+LANGUAGE js AS """
   try {
     rendered = JSON.parse(rendered);
     return rendered.microdata_itemtypes.map(itemType => itemType.toLowerCase());
   } catch (e) {
     return [];
   }
 """;
+
 WITH
-  rendered_data AS (
+rendered_data AS (
   SELECT
     getMicrodataItemTypes(rendered) AS microdata_item_types,
     client