diff --git a/README.md b/README.md index ae20ca36e..4059bee0f 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ A collection of AMP tools making it easier to publish and host AMP pages. The fo - **[amp-runtime-version](./packages/runtime-version):** a javascript library for querying the current AMP runtime version. - **[amp-update-cache](./packages/update-cache):** a javascript library for updating AMP documents in AMP Caches. - **[amp-update-linter](./packages/linter):** a javascript library for linting AMP documents (includes CLI mode). +- **[amp-validator-rules](./packages/validator-rules):** a javascript library for querying AMP validator rules. ## Development @@ -43,7 +44,7 @@ git clone https://github.com/your-fork/amp-toolbox.git # step into local repo cd amp-toolbox -# install dependencies +# install dependencies npm install # run tests diff --git a/package-lock.json b/package-lock.json index 7f4b487c7..6e24bb66e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4418,6 +4418,22 @@ } } }, + "@ampproject/toolbox-validator-rules": { + "version": "file:packages/validator-rules", + "requires": { + "@ampproject/toolbox-core": "^1.0.0-beta.3" + }, + "dependencies": { + "@ampproject/toolbox-core": { + "version": "1.0.0-beta.3", + "resolved": "https://registry.npmjs.org/@ampproject/toolbox-core/-/toolbox-core-1.0.0-beta.3.tgz", + "integrity": "sha512-IjZQRIRyAQDKA56PXsph2uyXvKQl+cVh+Xz1UX0Am0vIdbwXacHbx/oxYC7O0HD5Gbg6K508tHBeUTJFiUO0iA==", + "requires": { + "node-fetch": "2.6.0" + } + } + } + }, "@babel/code-frame": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.0.0.tgz", diff --git a/package.json b/package.json index 5c8eb39f5..bbc6e06f4 100644 --- a/package.json +++ b/package.json @@ -74,6 +74,7 @@ "@ampproject/toolbox-optimizer": "file:packages/optimizer", "@ampproject/toolbox-optimizer-express": "file:packages/optimizer-express", "@ampproject/toolbox-runtime-version": "file:packages/runtime-version", - "@ampproject/toolbox-update-cache": "file:packages/update-cache" + "@ampproject/toolbox-update-cache": "file:packages/update-cache", + "@ampproject/toolbox-validator-rules": "file:packages/validator-rules" } } diff --git a/packages/validator-rules/.gitignore b/packages/validator-rules/.gitignore new file mode 100644 index 000000000..6b37003e2 --- /dev/null +++ b/packages/validator-rules/.gitignore @@ -0,0 +1 @@ +validator.json diff --git a/packages/validator-rules/README.md b/packages/validator-rules/README.md new file mode 100644 index 000000000..caf4db5b5 --- /dev/null +++ b/packages/validator-rules/README.md @@ -0,0 +1,89 @@ +# AMP-Toolbox Validator Rules + +Queries published AMP Validator rules and extracts information about required +markup and attributes for all AMP formats. + +## Usage + +Install via: + +``` +$ npm install @ampproject/toolbox-validator-rules@canary +``` + +### Including the Module + +#### ES Module (Browser) + +```javascript +import validatorRules from '@ampproject/toolbox-validator-rules'; +``` + +#### CommonJs (Node) + +```javascript +const validatorRules = require('@ampproject/toolbox-validator-rules'); +``` + +### Using the module + +```javascript + // Loads the validator rules remotely with default options + const rules = await validatorRules.fetch(); + + + // The raw unprocessed rules + console.log(rules.raw); + + // All tags, combined with their respective attribute lists + console.log(rules.tags); + + // All extensions + console.log(rules.extensions); + + // Get all tag names used in AMP for Email + // The supported formats are AMP, AMP4EMAIL, AMP4ADS and ACTIONS + const tags = rules.getTagsForFormat('AMP4EMAIL'); + + // Display their names + console.log(tags.map(tag => tag.tagName)); + + // Get information about an extension + const ext = rules.getExtension('AMP4EMAIL', 'amp-carousel'); + + // Display supported versions + console.log(ext.versions); +``` + +### Format of rules + +The rules used closely follow the proto definitions from [validator.proto](https://github.com/ampproject/amphtml/blob/master/validator/validator.proto). + +Specifically: + +- The `raw` property is unprocessed [ValidatorRules](https://github.com/ampproject/amphtml/blob/master/validator/validator.proto#L643), the same format used by `https://cdn.ampproject.org/v0/validator.json` +- The result of `getTagsForFormat` and the `tags` property is a list of [TagSpec](https://github.com/ampproject/amphtml/blob/b892d81467594cab5473c803e071af5108f834a6/validator/validator.proto#L463) +- The result of `getExtension` is [ExtensionSpec](https://github.com/ampproject/amphtml/blob/b892d81467594cab5473c803e071af5108f834a6/validator/validator.proto#L388) with the `htmlFormat` field from `TagSpec` +- The `extensions` property a list of [ExtensionSpec](https://github.com/ampproject/amphtml/blob/b892d81467594cab5473c803e071af5108f834a6/validator/validator.proto#L388) with the `htmlFormat` field from `TagSpec` +- The `errors` property combines [ErrorFormat](https://github.com/ampproject/amphtml/blob/b892d81467594cab5473c803e071af5108f834a6/validator/validator.proto#L874) and [ErrorSpecificity](https://github.com/ampproject/amphtml/blob/b892d81467594cab5473c803e071af5108f834a6/validator/validator.proto#L869) + +### Options + +`fetch` optionally accepts an options object allowing you to customize its +behaviour. + +The following options are supported: + + * `noCache`: true to always fetch latest rules (by default, subsequent calls to `fetch` reuse the same result). + * `rules`: object to use locally specified rules instead of fetching them from the AMP CDN. + * `url`: override the URL where validator rules are fetched from. + * `source`: one of `'local'` (load rules from local file named "validator.json"), `'remote'` (fetch rules from CDN) or `'auto'` which is the default (tries looking for the local file first, then tries to fetch from CDN). + +Example: + +``` +validatorRules.fetch({ + noCache: true, + source: 'remote' +}); +``` diff --git a/packages/validator-rules/index.js b/packages/validator-rules/index.js new file mode 100644 index 000000000..0b7812b1d --- /dev/null +++ b/packages/validator-rules/index.js @@ -0,0 +1,39 @@ +/** + * Copyright 2019 The AMP HTML Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const loadRules = require('./lib/loadRules'); +const AmpValidatorRules = require('./lib/AmpValidatorRules'); + +let cached = null; + +async function fetch(opt = {}) { + if (!opt.noCache && cached) { + return cached; + } + + let rules = opt.rules; + + if (!rules) { + rules = await loadRules(opt); + } + + cached = new AmpValidatorRules(rules); + return cached; +} + +module.exports = {fetch}; diff --git a/packages/validator-rules/lib/AmpValidatorRules.js b/packages/validator-rules/lib/AmpValidatorRules.js new file mode 100644 index 000000000..998645418 --- /dev/null +++ b/packages/validator-rules/lib/AmpValidatorRules.js @@ -0,0 +1,201 @@ +/** + * Copyright 2019 The AMP HTML Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +class AmpValidatorRules { + /** + * Creates an instance of AmpValidatorRules. + * @param {Object} rules - rules imported from validator.json + */ + constructor(rules) { + /** + * Unprocessed validator rules. + * @type {Object} + */ + this.raw = rules; + /** + * List of all the tags processed from rules. + * @type {Array} + */ + this.tags = []; + /** + * List of all the extensions processed from rules. + * @type {Array} + */ + this.extensions = []; + /** + * Map of errors and their associated format and specificity. + * @type {Object} + */ + this.errors = {}; + + this.extensionCache_ = {}; + this.initRules_(rules); + } + + /** + * Returns the list of supported tags for the given format. + * + * @param {string} format - Format to return tags for + * @param {boolean} [transformed] - Use transformed version of the format + * @return {Array} List of tags supported by the given format + */ + getTagsForFormat(format, transformed = false) { + format = format.toLowerCase(); + return this.tags + .filter( + (tag) => + tag.htmlFormat.includes(format.toUpperCase()) && + this.checkEntityFormat_(tag, format) && + this.checkEntityTransformed_(tag, transformed) + ) + .map((tag) => { + tag = Object.assign({}, tag); + tag.attrs = tag.attrs.filter( + (attr) => + this.checkEntityFormat_(attr, format) && + this.checkEntityTransformed_(attr, transformed) + ); + return tag; + }); + } + + /** + * Returns the AMP extension spec for the given format and name. + * + * @param {string} format - Format to filter on + * @param {string} extension - Extension name + * @return {Object} Extension spec + */ + getExtension(format, extension) { + format = format.toLowerCase(); + extension = extension.toLowerCase(); + const key = `${format}|${extension}`; + return this.extensionCache_[key] || null; + } + + checkEntityTransformed_(entity, transformed) { + const isEnabled = this.isEnabled_(entity, 'transformed'); + const isDisabled = this.isDisabled_(entity, 'transformed'); + if (transformed) { + return isEnabled !== false && isDisabled !== true; + } + return isEnabled !== true && isDisabled !== false; + } + + checkEntityFormat_(entity, format) { + format = format.toLowerCase(); + const isEnabled = this.isEnabled_(entity, format); + const isDisabled = this.isDisabled_(entity, format); + return isEnabled !== false && isDisabled !== true; + } + + isEnabled_(entity, format) { + if (!entity.enabledBy) { + return null; + } + return entity.enabledBy.includes(format); + } + + isDisabled_(entity, format) { + if (!entity.disabledBy) { + return null; + } + return entity.disabledBy.includes(format); + } + + initRules_(rules) { + this.initErrors_(rules); + this.initAttrLists_(rules); + this.initTags_(rules); + this.initExtensions_(rules); + } + + initErrors_(rules) { + this.errors = {}; + for (const errorFormat of rules.errorFormats) { + const error = this.errors[errorFormat.code] || {}; + error.format = errorFormat.format; + this.errors[errorFormat.code] = error; + } + for (const errorSpecificity of rules.errorSpecificity) { + const error = this.errors[errorSpecificity.code] || {}; + error.specificity = errorSpecificity.specificity; + this.errors[errorSpecificity.code] = error; + } + } + + initAttrLists_(rules) { + this.attrLists_ = {}; + this.specialAttrLists_ = {}; + for (const {name, attrs} of rules.attrLists) { + if (name.startsWith('$')) { + this.specialAttrLists_[name] = attrs; + } else { + this.attrLists_[name] = attrs; + } + } + this.specialAttrLists_.$AMP_LAYOUT_ATTRS.forEach( + (attr) => (attr.layout = true) + ); + this.specialAttrLists_.$GLOBAL_ATTRS.forEach((attr) => (attr.global = true)); + } + + initTags_(rules) { + this.tags = rules.tags + .filter((tag) => !tag.extensionSpec) + .map((tag) => { + tag.attrs = tag.attrs || []; + + // `attrLists` contains list IDs that are looked up from the global + // attribute lists and merged into `attrs`. + if (tag.attrLists) { + for (const attrList of tag.attrLists) { + tag.attrs.push(...this.attrLists_[attrList]); + } + delete tag.attrLists; + } + + // $AMP_LAYOUT_ATTRS are present in all components with ampLayout + if (tag.ampLayout) { + tag.attrs.push(...this.specialAttrLists_.$AMP_LAYOUT_ATTRS); + } + + // $GLOBAL_ATTRS are present in all components + tag.attrs.push(...this.specialAttrLists_.$GLOBAL_ATTRS); + + return tag; + }); + } + + initExtensions_(rules) { + this.extensions = rules.tags + .filter((tag) => tag.extensionSpec) + .map((tag) => + Object.assign({}, tag.extensionSpec, {htmlFormat: tag.htmlFormat}) + ); + + for (const extension of this.extensions) { + const name = extension.name.toLowerCase(); + for (let format of extension.htmlFormat) { + format = format.toLowerCase(); + const key = `${format}|${name}`; + this.extensionCache_[key] = extension; + } + } + } +} + +module.exports = AmpValidatorRules; diff --git a/packages/validator-rules/lib/loadRules.js b/packages/validator-rules/lib/loadRules.js new file mode 100644 index 000000000..40f3bf917 --- /dev/null +++ b/packages/validator-rules/lib/loadRules.js @@ -0,0 +1,34 @@ +/** + * Copyright 2019 The AMP HTML Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This solution is temporary and will be replaced when +// https://github.com/ampproject/amp-toolbox/issues/378 is resolved. + +const {oneBehindFetch} = require('@ampproject/toolbox-core'); + +const VALIDATOR_RULES_URL = 'https://cdn.ampproject.org/v0/validator.json'; + +async function loadRemote(url) { + const req = await oneBehindFetch(url); + return req.json(); +} + +async function loadRules({url}) { + url = url || VALIDATOR_RULES_URL; + return loadRemote(url); +} + +module.exports = loadRules; diff --git a/packages/validator-rules/package.json b/packages/validator-rules/package.json new file mode 100644 index 000000000..5e9b756fe --- /dev/null +++ b/packages/validator-rules/package.json @@ -0,0 +1,30 @@ +{ + "name": "@ampproject/toolbox-validator-rules", + "version": "1.0.0-beta.3", + "description": "A library that helps query AMP Validator rules", + "main": "index.js", + "keywords": [ + "amp" + ], + "scripts": { + "fetchRules": "curl https://cdn.ampproject.org/v0/validator.json --output validator.json" + }, + "files": [ + "index.js", + "lib" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/ampproject/amp-toolbox.git" + }, + "author": "AMPHTML Team", + "license": "Apache-2.0", + "bugs": { + "url": "https://github.com/ampproject/amp-toolbox/issues" + }, + "dependencies": { + "@ampproject/toolbox-core": "^1.0.0-beta.3" + }, + "homepage": "https://github.com/ampproject/amp-toolbox/tree/master/packages/validator-rules", + "gitHead": "cb496f081c871a25dc73113ae1e1fccfb59b2732" +} diff --git a/packages/validator-rules/spec/AmpValidatorRulesSpec.js b/packages/validator-rules/spec/AmpValidatorRulesSpec.js new file mode 100644 index 000000000..820b098e9 --- /dev/null +++ b/packages/validator-rules/spec/AmpValidatorRulesSpec.js @@ -0,0 +1,170 @@ +/** + * Copyright 2019 The AMP HTML Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const AmpValidatorRules = require('../lib/AmpValidatorRules'); + +describe('AmpValidatorRules', () => { + it('Loads errors', () => { + const rules = makeRules({ + errorFormats: [ + { + code: 'TEST', + format: '%s error', + }, + ], + errorSpecificity: [ + { + code: 'TEST', + specificity: 1, + }, + ], + }); + expect(rules.errors).toEqual({ + TEST: { + format: '%s error', + specificity: 1, + }, + }); + }); + + it('Loads extensions', () => { + const rules = makeRules({ + tags: [ + { + extensionSpec: { + name: 'amp-some-component', + version: ['0.1', 'latest'], + }, + htmlFormat: ['AMP'], + tagName: 'SCRIPT', + }, + ], + }); + expect(rules.tags).toEqual([]); + expect(rules.extensions).toEqual([ + { + name: 'amp-some-component', + version: ['0.1', 'latest'], + htmlFormat: ['AMP'], + }, + ]); + expect(rules.getExtension('AMP', 'amp-some-component')).toEqual({ + name: 'amp-some-component', + version: ['0.1', 'latest'], + htmlFormat: ['AMP'], + }); + expect(rules.getExtension('AMP4EMAIL', 'amp-some-component')).toEqual(null); + }); + + it('Loads tags', () => { + const rules = makeRules({ + attrLists: [ + { + name: '$GLOBAL_ATTRS', + attrs: [{name: 'global'}], + }, + { + name: '$AMP_LAYOUT_ATTRS', + attrs: [{name: 'layoutattr'}], + }, + { + name: 'some-list', + attrs: [{name: 'test'}], + }, + ], + tags: [ + { + htmlFormat: ['AMP', 'AMP4EMAIL'], + attrs: [{name: 'align'}], + attrLists: ['some-list'], + tagName: 'DIV', + }, + { + htmlFormat: ['AMP', 'AMP4EMAIL'], + attrs: [{name: 'align'}], + disabledBy: ['transformed'], + ampLayout: { + supportedLayouts: ['FIXED', 'FIXED_HEIGHT'], + }, + tagName: 'AMP-IMG', + }, + ], + }); + + const tags = [ + { + htmlFormat: ['AMP', 'AMP4EMAIL'], + attrs: [ + {name: 'align'}, + {name: 'test'}, + { + name: 'global', + global: true, + }, + ], + tagName: 'DIV', + }, + { + htmlFormat: ['AMP', 'AMP4EMAIL'], + attrs: [ + {name: 'align'}, + { + name: 'layoutattr', + layout: true, + }, + { + name: 'global', + global: true, + }, + ], + disabledBy: ['transformed'], + ampLayout: { + supportedLayouts: ['FIXED', 'FIXED_HEIGHT'], + }, + tagName: 'AMP-IMG', + }, + ]; + expect(rules.tags).toEqual(tags); + expect(rules.extensions).toEqual([]); + expect(rules.getTagsForFormat('AMP4EMAIL')).toEqual(tags); + expect(rules.getTagsForFormat('AMP', true)).toEqual([tags[0]]); + }); +}); + +function makeRules(rules) { + return new AmpValidatorRules( + Object.assign( + { + errorFormats: [], + errorSpecificity: [], + attrLists: [ + { + name: '$AMP_LAYOUT_ATTRS', + attrs: [], + }, + { + name: '$GLOBAL_ATTRS', + attrs: [], + }, + ], + tags: [], + }, + rules + ) + ); +}