diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000000..be7798e81f --- /dev/null +++ b/.pylintrc @@ -0,0 +1,429 @@ +# This Pylint rcfile contains a best-effort configuration to uphold the +# best-practices and style described in the Google Python style guide: +# https://google.github.io/styleguide/pyguide.html +# +# Its canonical open-source location is: +# https://google.github.io/styleguide/pylintrc + +[MASTER] + +# Files or directories to be skipped. They should be base names, not paths. +ignore=third_party + +# Files or directories matching the regex patterns are skipped. The regex +# matches against base names, not paths. +ignore-patterns= + +# Pickle collected data for later comparisons. +persistent=no + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Use multiple processes to speed up Pylint. +jobs=4 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=abstract-method, + apply-builtin, + arguments-differ, + attribute-defined-outside-init, + backtick, + bad-option-value, + basestring-builtin, + buffer-builtin, + c-extension-no-member, + consider-using-enumerate, + cmp-builtin, + cmp-method, + coerce-builtin, + coerce-method, + delslice-method, + div-method, + duplicate-code, + eq-without-hash, + execfile-builtin, + file-builtin, + filter-builtin-not-iterating, + fixme, + getslice-method, + global-statement, + hex-method, + idiv-method, + implicit-str-concat, + import-error, + import-self, + import-star-module-level, + inconsistent-return-statements, + input-builtin, + intern-builtin, + invalid-str-codec, + locally-disabled, + long-builtin, + long-suffix, + map-builtin-not-iterating, + misplaced-comparison-constant, + missing-function-docstring, + metaclass-assignment, + next-method-called, + next-method-defined, + no-absolute-import, + no-else-break, + no-else-continue, + no-else-raise, + no-else-return, + no-init, # added + no-member, + no-name-in-module, + no-self-use, + nonzero-method, + oct-method, + old-division, + old-ne-operator, + old-octal-literal, + old-raise-syntax, + parameter-unpacking, + print-statement, + raising-string, + range-builtin-not-iterating, + raw_input-builtin, + rdiv-method, + reduce-builtin, + relative-import, + reload-builtin, + round-builtin, + setslice-method, + signature-differs, + standarderror-builtin, + suppressed-message, + sys-max-int, + too-few-public-methods, + too-many-ancestors, + too-many-arguments, + too-many-boolean-expressions, + too-many-branches, + too-many-instance-attributes, + too-many-locals, + too-many-nested-blocks, + too-many-public-methods, + too-many-return-statements, + too-many-statements, + trailing-newlines, + unichr-builtin, + unicode-builtin, + unnecessary-pass, + unpacking-in-except, + useless-else-on-loop, + useless-object-inheritance, + useless-suppression, + using-cmp-argument, + wrong-import-order, + xrange-builtin, + zip-builtin-not-iterating, + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages +reports=no + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + + +[BASIC] + +# Good variable names which should always be accepted, separated by a comma +good-names=main,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty,cached_property.cached_property,cached_property.threaded_cached_property,cached_property.cached_property_with_ttl,cached_property.threaded_cached_property_with_ttl + +# Regular expression matching correct function names +function-rgx=^(?:(?PsetUp|tearDown|setUpModule|tearDownModule)|(?P_?[A-Z][a-zA-Z0-9]*)|(?P_?[a-z][a-z0-9_]*))$ + +# Regular expression matching correct variable names +variable-rgx=^[a-z][a-z0-9_]*$ + +# Regular expression matching correct constant names +const-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ + +# Regular expression matching correct attribute names +attr-rgx=^_{0,2}[a-z][a-z0-9_]*$ + +# Regular expression matching correct argument names +argument-rgx=^[a-z][a-z0-9_]*$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=^[a-z][a-z0-9_]*$ + +# Regular expression matching correct class names +class-rgx=^_?[A-Z][a-zA-Z0-9]*$ + +# Regular expression matching correct module names +module-rgx=^(_?[a-z][a-z0-9_]*|__init__)$ + +# Regular expression matching correct method names +method-rgx=(?x)^(?:(?P_[a-z0-9_]+__|runTest|setUp|tearDown|setUpTestCase|tearDownTestCase|setupSelf|tearDownClass|setUpClass|(test|assert)_*[A-Z0-9][a-zA-Z0-9_]*|next)|(?P_{0,2}[A-Z][a-zA-Z0-9_]*)|(?P_{0,2}[a-z][a-z0-9_]*))$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=(__.*__|main|test.*|.*test|.*Test)$ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=10 + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager,contextlib2.contextmanager + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=80 + +# TODO(https://github.com/PyCQA/pylint/issues/3352): Direct pylint to exempt +# lines made too long by directives to pytype. + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=(?x)( + ^\s*(\#\ )??$| + ^\s*(from\s+\S+\s+)?import\s+.+$) + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=yes + +# Maximum number of lines in a module +max-module-lines=99999 + +# String used as indentation unit. The internal Google style guide mandates 2 +# spaces. Google's externaly-published style guide says 4, consistent with +# PEP 8. Here, we use 2 spaces, for conformity with many open-sourced Google +# projects (like TensorFlow). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=TODO + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=yes + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=^\*{0,2}(_$|unused_|dummy_) + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six,six.moves,past.builtins,future.builtins,functools + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging,absl.logging,tensorflow.io.logging + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub, + TERMIOS, + Bastion, + rexec, + sets + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant, absl + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls, + class_ + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=StandardError, + Exception, + BaseException diff --git a/bin/gcp_device_logs b/bin/gcp_device_logs index 9a118dd2df..0877fe7d74 100755 --- a/bin/gcp_device_logs +++ b/bin/gcp_device_logs @@ -14,6 +14,7 @@ import json import dateutil.parser import argparse +# pylint: disable-next=line-too-long SHELL_TEMPLATE = 'gcloud logging read "logName=projects/{}/logs/cloudiot.googleapis.com%2Fdevice_activity AND ({}) AND timestamp>=\\\"{}\\\"" --limit 1000 --format json --project {}' TIMESTAMP_FORMAT = '%Y-%m-%dT%H:%M:%SZ' #timestamp >= "2016-11-29T23:00:00Z" @@ -29,7 +30,8 @@ args = parse_command_line_args() target_devices = args.device_ids project_id = args.project_id -device_filter = ' OR '.join([f'labels.device_id={target}' for target in target_devices]) +device_filter = ' OR '.join([f'labels.device_id={target}' \ + for target in target_devices]) # sleep duration - balance of speed and accuracy for ordering as some entries # can be delayed by about 10 seconds @@ -42,8 +44,15 @@ seen = [] while True: try: - shell_command = SHELL_TEMPLATE.format(project_id, device_filter, search_timestamp.strftime(TIMESTAMP_FORMAT), project_id) - output = subprocess.run(shell_command, capture_output=True, shell=True, check=True) + shell_command = SHELL_TEMPLATE.format(project_id, + device_filter, + search_timestamp.strftime(TIMESTAMP_FORMAT), + project_id) + + output = subprocess.run(shell_command, + capture_output=True, + shell=True, + check=True) data = json.loads(output.stdout) @@ -56,16 +65,16 @@ while True: continue seen.append(insert_id) - - event_type = entry['jsonPayload']['eventType'] + log_data = entry['jsonPayload'] + event_type = log_data['eventType'] timestamp = entry['timestamp'] registry_id = entry['resource']['labels']['device_registry_id'] log_device_id = entry['labels']['device_id'] metadata = '' if event_type == 'PUBLISH': - metadata = entry['jsonPayload'].get('publishFromDeviceTopicType') - publishToDeviceTopicType = entry['jsonPayload'].get('publishToDeviceTopicType') + metadata = log_data.get('publishFromDeviceTopicType') + publishToDeviceTopicType = log_data.get('publishToDeviceTopicType') if publishToDeviceTopicType == 'CONFIG': event_type = 'CONFIG' metadata = '' @@ -73,21 +82,21 @@ while True: metadata = f'TO DEVICE {publishToDeviceTopicType}' if event_type == 'PUBACK': - metadata = entry['jsonPayload']['publishToDeviceTopicType'] + metadata = log_data['publishToDeviceTopicType'] if event_type == 'SUBSCRIBE': - metadata = entry['jsonPayload']['mqttTopic'] + metadata = log_data['mqttTopic'] if event_type == 'ATTACH_TO_GATEWAY': - metadata = entry['jsonPayload']['gateway']['id'] + metadata = log_data['gateway']['id'] if event_type == 'DISCONNECT': - metadata = entry['jsonPayload']['disconnectType'] + metadata = log_data['disconnectType'] - if entry['jsonPayload']['status'].get('code') != 0: - metadata = (f"{metadata}" - f"({entry['jsonPayload']['status'].get('description')}" - f"{entry['jsonPayload']['status'].get('message', '')})") + if log_data['status'].get('code') != 0: + metadata = (f'{metadata}' + f'({log_data["status"].get("description")}' + f'{log_data["status"].get("message", "")})') entries.append({'timestamp_obj': dateutil.parser.parse(timestamp), 'timestamp': timestamp, @@ -112,7 +121,5 @@ while True: print('Ensure gcloud is authenticated and account has permissions') print('to access cloud logging') sys.exit(1) - except Exception: - pass finally: time.sleep(dt) diff --git a/bin/gencode_categories b/bin/gencode_categories index 6534ce033d..6a65911574 100755 --- a/bin/gencode_categories +++ b/bin/gencode_categories @@ -1,22 +1,18 @@ #!/usr/bin/env python3 - -import hashlib -import json +""" Gencode generator for categories """ import re import os -import shutil -import sys GENCODE_MARKER = '@@ ' CATEGORY_MARKER = '* ' -WILDCARD_REGEX = ' *\\* _\?\?\?_: (.*)' -WILDCARD_SUFFIX = '.[a-z]+(_[a-z]+)*' +WILDCARD_REGEX = r' *\* _\?\?\?_: (.*)' +WILDCARD_SUFFIX = r'.[a-z]+(_[a-z]+)*' -CATEGORY_REGEX = ' *\\* _([a-z]+)_: (\(\*\*([A-Z]+)\*\*\) )?(.*)' +CATEGORY_REGEX = r' *\* _([a-z]+)_: (\(\*\*([A-Z]+)\*\*\) )?(.*)' JSON_FORMAT = '%s%s{ "pattern": "^%s$" }' DEVICE_PREFIX = 'device' -JAVA_DESCRIPTION = "\n%s// %s\n" +JAVA_DESCRIPTION = '\n%s// %s\n' JAVA_TARGET = '%spublic static final String %s = "%s";\n' JAVA_LEVEL = '%spublic static final Level %s_LEVEL = %s;\n' JAVA_MAP_ADD = '%sstatic { LEVEL.put(%s, %s); }\n' @@ -28,81 +24,88 @@ java_in = os.path.join('etc/Category.java') java_out = os.path.join('gencode/java/udmi/schema/Category.java') def read_categories(): - categories = [] - prefix = [] - previous = -1 - group = None - with open(doc_in) as doc: - while line := doc.readline(): - indent = line.find(CATEGORY_MARKER)//2 - wildcard = re.match(WILDCARD_REGEX, line) - if wildcard: - entry = (group + WILDCARD_SUFFIX, 'INFO', wildcard.group(1)) - categories.append(entry) - continue - match = re.match(CATEGORY_REGEX, line) - if indent < 0 or not match: - continue - if indent < previous: - for _ in range(indent, previous): - rem = prefix.pop(len(prefix) - 1) - elif indent > previous: - if group: - prefix.append(group) - previous = indent - group = match.group(1) - category = '.'.join(prefix + [group]) - level = match.group(3) - description = match.group(4) - if level: - entry = (category, level, description) - categories.append(entry) - return categories + categories = [] + prefix = [] + previous = -1 + group = None + with open(doc_in, 'r', encoding='utf-8') as doc: + while line := doc.readline(): + indent = line.find(CATEGORY_MARKER)//2 + wildcard = re.match(WILDCARD_REGEX, line) + if wildcard: + entry = (group + WILDCARD_SUFFIX, 'INFO', wildcard.group(1)) + categories.append(entry) + continue + match = re.match(CATEGORY_REGEX, line) + if indent < 0 or not match: + continue + if indent < previous: + for _ in range(indent, previous): + # pylint: disable-next=unused-variable + rem = prefix.pop(len(prefix) - 1) + elif indent > previous: + if group: + prefix.append(group) + previous = indent + group = match.group(1) + category = '.'.join(prefix + [group]) + level = match.group(3) + description = match.group(4) + if level: + entry = (category, level, description) + categories.append(entry) + return categories def write_schema_out(categories): - with open(schema_in) as inp: - with open(schema_out, 'w') as out: - while line := inp.readline(): - index = line.find(GENCODE_MARKER) - if index >= 0: - write_schema_categories(out, line[0:index], categories) - else: - out.write(line) + with open(schema_in, 'r', encoding='utf-8') as inp: + with open(schema_out, 'w', encoding='utf-8') as out: + while line := inp.readline(): + index = line.find(GENCODE_MARKER) + if index >= 0: + write_schema_categories(out, line[0:index], categories) + else: + out.write(line) def write_schema_categories(out, indent, categories): - prefix = '' - for category in categories: - target = category[0].replace('.', '\\\\.') - out.write(JSON_FORMAT % (prefix, indent, target)) - prefix = ',\n' - out.write('\n') + prefix = '' + for category in categories: + target = category[0].replace('.', '\\\\.') + out.write(JSON_FORMAT % (prefix, indent, target)) + prefix = ',\n' + out.write('\n') def write_java_out(categories): - os.makedirs(os.path.dirname(java_out), exist_ok=True) - with open(java_in) as inp: - with open(java_out, 'w') as out: - while line := inp.readline(): - index = line.find(GENCODE_MARKER) - if index >= 0: - indent = line[0:index] - write_java_categories(out, indent, categories) - else: - out.write(line) + os.makedirs(os.path.dirname(java_out), exist_ok=True) + with open(java_in, 'r', encoding='utf-8') as inp: + with open(java_out, 'w', encoding='utf-8') as out: + while line := inp.readline(): + index = line.find(GENCODE_MARKER) + if index >= 0: + indent = line[0:index] + write_java_categories(out, indent, categories) + else: + out.write(line) def write_java_categories(out, indent, categories): - for category in categories: - target = category[0] - if target.startswith(DEVICE_PREFIX): - continue - level = category[1] - desc = category[2] - const = target.replace('.', '_').upper() - out.write(JAVA_DESCRIPTION % (indent, desc)) - out.write(JAVA_TARGET % (indent, const, target)) - out.write(JAVA_LEVEL % (indent, const, level)) - out.write(JAVA_MAP_ADD % (indent, const, level)) + for category in categories: + target = category[0] + if target.startswith(DEVICE_PREFIX): + continue + level = category[1] + desc = category[2] + const = target.replace('.', '_').upper() + out.write(JAVA_DESCRIPTION % (indent, desc)) + out.write(JAVA_TARGET % (indent, const, target)) + out.write(JAVA_LEVEL % (indent, const, level)) + out.write(JAVA_MAP_ADD % (indent, const, level)) + + +def main(): + categories = read_categories() + write_schema_out(categories) + write_java_out(categories) + +if __name__ == '__main__': + main() -categories = read_categories() -write_schema_out(categories) -write_java_out(categories) diff --git a/bin/gencode_docs_checklinks b/bin/gencode_docs_checklinks index 2f1fa92dfa..64ce598db5 100755 --- a/bin/gencode_docs_checklinks +++ b/bin/gencode_docs_checklinks @@ -10,110 +10,118 @@ import os import sys HOSTED_GITHUB_PAGES = r'^https?:\/\/faucetsdn\.github\.io\/udmi\/' +# pylint: disable-next=line-too-long HOSTED_GITHUB = r'^https?:\/\/(?:www\.)?github\.com\/faucetsdn\/udmi\/(?:blob|tree)\/master\/' def split_link_fragment(link): - """ Splits a link into the path and the fragment. - e.g. file.md#header is split into `file.md` and `#header` + """ Splits a link into the path and the fragment. + e.g. file.md#header is split into `file.md` and `#header` - Arguments - Link + Arguments + Link - Returns tuple: - file_path - anchor None if not defined - """ - matches = re.search(r'^([^#]*)(#.*)?$', link) - if not matches: - raise ValueError - return matches.groups() + Returns tuple: + file_path + anchor None if not defined + """ + matches = re.search(r'^([^#]*)(#.*)?$', link) + if not matches: + raise ValueError + return matches.groups() def blank_regex_substitutions(string, *args): - """ - Applies multiple blank regex substitutions on a string and returns the result + """ + Applies multiple blank regex substitutions on a string and returns the + result - Arguments - string to substitute in - *args regex for substitution + Arguments + string to substitute in + *args regex for substitution - Returns - string with all regex substitutions applied - """ - for regex in args: - string = re.sub(regex, '', string) - return string + Returns + string with all regex substitutions applied + """ + for regex in args: + string = re.sub(regex, '', string) + return string def check_links(file_path): - """ - Checks if inline markdown links within the given file_path resolve to a - valid file or directory, or to a hosted (Github or Github Pages) copy of - the repository - - Arguments: - file_path: file to check links within - - Returns: - list of links which did not resolve - """ - - failing_links = [] - - with open(file_path, 'r') as f: - file_data = f.read() - if file_path.endswith('.md'): - file_lines = file_data.split('\n', 2) - # TODO: Make this more comprehensive and check actual path. - if not re.search('^\[\*\*UDMI.*\]\(\#\)$', file_lines[0]): - failing_links.append((file_path, 'header', file_lines[0])) - if file_lines[1]: - failing_links.append((file_path, 'missing blank', '')) - links = re.findall(r'\[(?:[^\]]*)\]\(([^\)]*)\)', file_data) - for link in links: - - if not re.search('^https?://', link): - link_path, link_anchor = split_link_fragment(link) - dir_name = os.path.dirname(os.path.realpath(file_path)) - - # Links are relative to the file they were found in - resolved_path = os.path.realpath(os.path.join(dir_name, link_path)) - - if not os.path.exists(resolved_path): - failing_links.append((file_path, 'link', resolved_path)) - else: - # Rewrite hosted links (e.g. github pages) to local by subtracting the web host - rewritten_link = blank_regex_substitutions(link, HOSTED_GITHUB, HOSTED_GITHUB_PAGES) - - if not re.search('^https?://', rewritten_link): - # The modified link now directs to a local path - resolved_path = os.path.realpath(rewritten_link) - # Append .md to any files without extensions (assumed from github pages links) - root, ext = os.path.splitext(resolved_path) - if not ext and not os.path.isdir(root): - resolved_path = f'{resolved_path}.md' - - if not os.path.exists(resolved_path): - failing_links.append((file_path, 'link', link)) - - return failing_links + """ + Checks if inline markdown links within the given file_path resolve to a + valid file or directory, or to a hosted (Github or Github Pages) copy of + the repository + + Arguments: + file_path: file to check links within + + Returns: + list of links which did not resolve + """ + + failing_links = [] + + with open(file_path, 'r', encoding='utf-8') as f: + file_data = f.read() + if file_path.endswith('.md'): + file_lines = file_data.split('\n', 2) + # TODO: Make this more comprehensive and check actual path. + if not re.search(r'^\[\*\*UDMI.*\]\(\#\)$', file_lines[0]): + failing_links.append((file_path, 'header', file_lines[0])) + if file_lines[1]: + failing_links.append((file_path, 'missing blank', '')) + links = re.findall(r'\[(?:[^\]]*)\]\(([^\)]*)\)', file_data) + for link in links: + + if not re.search('^https?://', link): + # function returns a tuple and link anchor not currently needed + # pylint: disable-next=unused-variable + link_path, link_anchor = split_link_fragment(link) + dir_name = os.path.dirname(os.path.realpath(file_path)) + + # Links are relative to the file they were found in + resolved_path = os.path.realpath(os.path.join(dir_name, link_path)) + + if not os.path.exists(resolved_path): + failing_links.append((file_path, 'link', resolved_path)) + else: + # Rewrite hosted links (e.g. github pages) to local by + # subtracting the web host + rewritten_link = blank_regex_substitutions(link, + HOSTED_GITHUB, + HOSTED_GITHUB_PAGES) + + if not re.search('^https?://', rewritten_link): + # The modified link now directs to a local path + resolved_path = os.path.realpath(rewritten_link) + # Append .md to any files without extensions + # (assumed from github pages links) + root, ext = os.path.splitext(resolved_path) + if not ext and not os.path.isdir(root): + resolved_path = f'{resolved_path}.md' + + if not os.path.exists(resolved_path): + failing_links.append((file_path, 'link', link)) + + return failing_links def main(): - check_paths = ['*.md', 'docs/**/*.md', 'schema/*.json', 'udmif/**/*.md'] - error = False - for check_path in check_paths: - file_paths = glob.glob(check_path, recursive=True) - for file_path in file_paths: - invalid_tuples = check_links(file_path) - if invalid_tuples: - error = True - for invalid_tuple in invalid_tuples: - print(f'** {invalid_tuple[0]} {invalid_tuple[1]} {invalid_tuple[2]}') - return error - -if __name__ == "__main__": - result = main() - if result: - sys.exit(1) \ No newline at end of file + check_paths = ['*.md', 'docs/**/*.md', 'schema/*.json', 'udmif/**/*.md'] + error = False + for check_path in check_paths: + file_paths = glob.glob(check_path, recursive=True) + for file_path in file_paths: + invalid_tuples = check_links(file_path) + if invalid_tuples: + error = True + for invalid_tuple in invalid_tuples: + print(f'** {invalid_tuple[0]} {invalid_tuple[1]} {invalid_tuple[2]}') + return error + +if __name__ == '__main__': + result = main() + if result: + sys.exit(1) diff --git a/bin/gencode_python b/bin/gencode_python index e1f271152c..70dc23cf47 100755 --- a/bin/gencode_python +++ b/bin/gencode_python @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# pylint: skip-file import hashlib import json @@ -6,12 +7,12 @@ import os import shutil import sys -FILE_PREFIX="file:" -FILE_SPLIT="#" -REF_KEY="$ref" -SOURCE_KEY="$source" -PREFIX="udmi/schema" -JSON_SUFFIX=".json" +FILE_PREFIX = 'file:' +FILE_SPLIT = '#' +REF_KEY = '$ref' +SOURCE_KEY = '$source' +PREFIX = 'udmi/schema' +JSON_SUFFIX = '.json' out_dir = os.path.join(sys.argv[1], PREFIX) in_files = sys.argv[2:] @@ -19,260 +20,260 @@ in_files = sys.argv[2:] processed = {} def eprint(*args, **kwargs): - print(*args, file=sys.stderr, **kwargs) + print(*args, file=sys.stderr, **kwargs) def get_title(contents, default=None): - title = contents.get('title') - if title: - return title - if default: - return default; - md5_hash = hashlib.md5(json.dumps(contents).encode()).hexdigest()[0:8] - return 'Object %s' % md5_hash.upper() + title = contents.get('title') + if title: + return title + if default: + return default + md5_hash = hashlib.md5(json.dumps(contents).encode()).hexdigest()[0:8] + return 'Object %s' % md5_hash.upper() def get_class_name(contents): - return get_title(contents).replace(' ', '') + return get_title(contents).replace(' ', '') def get_file_base(parent, contents): - source = contents[SOURCE_KEY] if SOURCE_KEY in contents else parent[SOURCE_KEY] - module = os.path.basename(source)[:-len(JSON_SUFFIX)] - return module + source = contents[SOURCE_KEY] if SOURCE_KEY in contents else parent[SOURCE_KEY] + module = os.path.basename(source)[:-len(JSON_SUFFIX)] + return module def process_file(source): - try: - name = os.path.basename(source) - if name in processed: - return processed[name] - eprint('Processing file', source) - with open(source) as fd: - contents = json.load(fd) - contents[SOURCE_KEY] = os.path.relpath(source) - generate_class(contents) - except Exception: - raise Exception('While processing file %s' % source) - processed[name] = contents - return contents + try: + name = os.path.basename(source) + if name in processed: + return processed[name] + eprint('Processing file', source) + with open(source) as fd: + contents = json.load(fd) + contents[SOURCE_KEY] = os.path.relpath(source) + generate_class(contents) + except Exception: + raise Exception('While processing file %s' % source) + processed[name] = contents + return contents def get_properties(contents): - return contents.get('properties', {}).keys() + return contents.get('properties', {}).keys() def is_array(contents): - return contents.get('type') == 'array' + return contents.get('type') == 'array' def is_map(contents): - return 'patternProperties' in contents + return 'patternProperties' in contents def write_class(writer, contents): - class_name = get_class_name(contents) - write_deps(writer, contents) - writer('') - writer('') - writer('class %s:' % class_name) - writer(' """Generated schema class"""') - write_init(writer, contents) - write_fromdict(writer, contents) - write_mapfrom(writer, contents) - write_todict(writer, contents) + class_name = get_class_name(contents) + write_deps(writer, contents) + writer('') + writer('') + writer('class %s:' % class_name) + writer(' """Generated schema class"""') + write_init(writer, contents) + write_fromdict(writer, contents) + write_mapfrom(writer, contents) + write_todict(writer, contents) def process_ref_path(contents, path): - if not path: - return contents - assert path.startswith('/'), 'expected path / start character: %s' % path - parts = path[1:].split('/') - source = contents[SOURCE_KEY] - for part in parts: - contents = contents[part] - contents[SOURCE_KEY] = source + if not path: return contents + assert path.startswith('/'), 'expected path / start character: %s' % path + parts = path[1:].split('/') + source = contents[SOURCE_KEY] + for part in parts: + contents = contents[part] + contents[SOURCE_KEY] = source + return contents def resolve_refs_file(parent, contents, ref): - index = ref.find(FILE_SPLIT) - none_index = index if index >= 0 else None - ref_file = ref[len(FILE_PREFIX):none_index] - postfix = ref[index + 1:] if index >= 0 else '' - source = parent[SOURCE_KEY] - full_path = os.path.join(os.path.dirname(source), ref_file) - contents = process_file(full_path) - subcontents = process_ref_path(contents, postfix) - return subcontents, True + index = ref.find(FILE_SPLIT) + none_index = index if index >= 0 else None + ref_file = ref[len(FILE_PREFIX):none_index] + postfix = ref[index + 1:] if index >= 0 else '' + source = parent[SOURCE_KEY] + full_path = os.path.join(os.path.dirname(source), ref_file) + contents = process_file(full_path) + subcontents = process_ref_path(contents, postfix) + return subcontents, True def resolve_refs_local(parent, contents, ref): - subcontents = process_ref_path(parent, ref[1:]) - return subcontents, True + subcontents = process_ref_path(parent, ref[1:]) + return subcontents, True def resolve_refs(parent, contents): - ref = contents.get(REF_KEY) - if not ref: - return contents, False - if ref.startswith(FILE_PREFIX): - return resolve_refs_file(parent, contents, ref) - if ref.startswith(FILE_SPLIT): - return resolve_refs_local(parent, contents, ref) - raise Exception('Unknown ref format %s' % ref) + ref = contents.get(REF_KEY) + if not ref: + return contents, False + if ref.startswith(FILE_PREFIX): + return resolve_refs_file(parent, contents, ref) + if ref.startswith(FILE_SPLIT): + return resolve_refs_local(parent, contents, ref) + raise Exception('Unknown ref format %s' % ref) def get_subclass(parent, contents): - resolved, _ = resolve_refs(parent, contents) - type = resolved.get('type') - if type != 'object': - return None - return get_class_name(resolved) + resolved, _ = resolve_refs(parent, contents) + type = resolved.get('type') + if type != 'object': + return None + return get_class_name(resolved) def write_mapfrom(writer, contents): - writer('') - writer(' @staticmethod') - writer(' def map_from(source):') - writer(' if not source:') - writer(' return None') - writer(' result = {}') - class_name = get_class_name(contents) - writer(' for key in source:') - writer(' result[key] = %s.from_dict(source[key])' % class_name) - writer(' return result') + writer('') + writer(' @staticmethod') + writer(' def map_from(source):') + writer(' if not source:') + writer(' return None') + writer(' result = {}') + class_name = get_class_name(contents) + writer(' for key in source:') + writer(' result[key] = %s.from_dict(source[key])' % class_name) + writer(' return result') def write_fromdict(writer, contents): - writer('') - writer(' @staticmethod') - writer(' def from_dict(source):') - writer(' if not source:') - writer(' return None') - class_name = get_class_name(contents) - properties = get_properties(contents) - writer(' result = %s()' % class_name) - for property in properties: - try: - subcontent, _ = resolve_refs(contents, contents['properties'][property]) - subclass = get_subclass(contents, subcontent) - is_submap = is_map(subcontent) - if is_submap or is_array(subcontent): - objcontent = subcontent['patternProperties'] if is_submap else subcontent['items'] - tmpcontent, _ = resolve_refs(contents, objcontent) - assert not is_submap or len(tmpcontent) == 1, 'multiple patternProperties not supported: %s' % tmpcontent - itemcontent = list(tmpcontent.values())[0] if is_submap else tmpcontent - itemclass = get_subclass(contents, itemcontent) - if not itemclass: - writer(" result.%s = source.get('%s')" % (property, property)) - elif is_submap: - writer(" result.%s = %s.map_from(source.get('%s'))" % (property, itemclass, property)) - else: - writer(" result.%s = %s.array_from(source.get('%s'))" % (property, itemclass, property)) - elif subclass: - assert subclass, 'sub class not defined: %s' % subcontent - writer(" result.%s = %s.from_dict(source.get('%s'))" % (property, subclass, property)) - else: - writer(" result.%s = source.get('%s')" % (property, property)) - except Exception: - raise Exception('While processing property %s' % property) - writer(' return result') + writer('') + writer(' @staticmethod') + writer(' def from_dict(source):') + writer(' if not source:') + writer(' return None') + class_name = get_class_name(contents) + properties = get_properties(contents) + writer(' result = %s()' % class_name) + for property in properties: + try: + subcontent, _ = resolve_refs(contents, contents['properties'][property]) + subclass = get_subclass(contents, subcontent) + is_submap = is_map(subcontent) + if is_submap or is_array(subcontent): + objcontent = subcontent['patternProperties'] if is_submap else subcontent['items'] + tmpcontent, _ = resolve_refs(contents, objcontent) + assert not is_submap or len(tmpcontent) == 1, 'multiple patternProperties not supported: %s' % tmpcontent + itemcontent = list(tmpcontent.values())[0] if is_submap else tmpcontent + itemclass = get_subclass(contents, itemcontent) + if not itemclass: + writer(" result.%s = source.get('%s')" % (property, property)) + elif is_submap: + writer(" result.%s = %s.map_from(source.get('%s'))" % (property, itemclass, property)) + else: + writer(" result.%s = %s.array_from(source.get('%s'))" % (property, itemclass, property)) + elif subclass: + assert subclass, 'sub class not defined: %s' % subcontent + writer(" result.%s = %s.from_dict(source.get('%s'))" % (property, subclass, property)) + else: + writer(" result.%s = source.get('%s')" % (property, property)) + except Exception: + raise Exception('While processing property %s' % property) + writer(' return result') def write_todict(writer, contents): - writer('') - writer(' @staticmethod') - writer(' def expand_dict(input):') - writer(' result = {}') - writer(' for property in input:') - writer(' result[property] = input[property].to_dict() if input[property] else {}') - writer(' return result') - writer('') - writer(' def to_dict(self):') - writer(' result = {}') - properties = get_properties(contents) - for property in properties: - try: - subcontent, _ = resolve_refs(contents, contents['properties'][property]) - subclass = get_subclass(contents, subcontent) - is_submap = is_map(subcontent) - writer(" if self.%s:" % property) - if is_submap or is_array(subcontent): - objcontent = subcontent['patternProperties'] if is_submap else subcontent['items'] - tmpcontent, _ = resolve_refs(contents, objcontent) - assert not is_submap or len(tmpcontent) == 1, 'multiple patternProperties not supported: %s' % tmpcontent - itemcontent = list(tmpcontent.values())[0] if is_submap else tmpcontent - itemclass = get_subclass(contents, itemcontent) - if not itemclass: - writer(" result['%s'] = self.%s # 1" % ((property,)*2)) - elif is_submap: - writer(" result['%s'] = %s.expand_dict(self.%s) # 2" % (property, itemclass, property)) - else: - writer(" result['%s'] = self.%s.to_dict() # 3" % ((property,)*2)) - elif subclass: - assert subclass, 'sub class not defined: %s' % subcontent - writer(" result['%s'] = self.%s.to_dict() # 4" % ((property,)*2)) - else: - writer(" result['%s'] = self.%s # 5" % ((property,)*2)) - except Exception: - raise Exception('While processing property %s' % property) - writer(' return result') + writer('') + writer(' @staticmethod') + writer(' def expand_dict(input):') + writer(' result = {}') + writer(' for property in input:') + writer(' result[property] = input[property].to_dict() if input[property] else {}') + writer(' return result') + writer('') + writer(' def to_dict(self):') + writer(' result = {}') + properties = get_properties(contents) + for property in properties: + try: + subcontent, _ = resolve_refs(contents, contents['properties'][property]) + subclass = get_subclass(contents, subcontent) + is_submap = is_map(subcontent) + writer(" if self.%s:" % property) + if is_submap or is_array(subcontent): + objcontent = subcontent['patternProperties'] if is_submap else subcontent['items'] + tmpcontent, _ = resolve_refs(contents, objcontent) + assert not is_submap or len(tmpcontent) == 1, 'multiple patternProperties not supported: %s' % tmpcontent + itemcontent = list(tmpcontent.values())[0] if is_submap else tmpcontent + itemclass = get_subclass(contents, itemcontent) + if not itemclass: + writer(" result['%s'] = self.%s # 1" % ((property,)*2)) + elif is_submap: + writer(" result['%s'] = %s.expand_dict(self.%s) # 2" % (property, itemclass, property)) + else: + writer(" result['%s'] = self.%s.to_dict() # 3" % ((property,)*2)) + elif subclass: + assert subclass, 'sub class not defined: %s' % subcontent + writer(" result['%s'] = self.%s.to_dict() # 4" % ((property,)*2)) + else: + writer(" result['%s'] = self.%s # 5" % ((property,)*2)) + except Exception: + raise Exception('While processing property %s' % property) + writer(' return result') def write_deps(writer, contents): - properties = get_properties(contents) - for property in properties: - try: - write_dep(writer, contents, contents['properties'][property]) - except Exception: - raise Exception('While processing property %s' % property) - definitions = contents.get('definitions', {}) - for definition in definitions: - try: - write_dep(writer, contents, contents['definitions'][definition]) - except Exception: - raise Exception('While processing definition %s' % definition) + properties = get_properties(contents) + for property in properties: + try: + write_dep(writer, contents, contents['properties'][property]) + except Exception: + raise Exception('While processing property %s' % property) + definitions = contents.get('definitions', {}) + for definition in definitions: + try: + write_dep(writer, contents, contents['definitions'][definition]) + except Exception: + raise Exception('While processing definition %s' % definition) def write_dep(writer, contents, depcontent): - subcontent, external = resolve_refs(contents, depcontent) - subclass = get_subclass(contents, subcontent) - if external and subclass: - writer('from .%s import %s' % (get_file_base(contents, subcontent), subclass)) - elif is_map(subcontent): - mapcontent, _ = resolve_refs(contents, subcontent['patternProperties']) - assert len(mapcontent) == 1, 'multiple patternProperties not supported' - write_dep(writer, contents, list(mapcontent.values())[0]) - elif subclass: - subcontent[SOURCE_KEY] = contents[SOURCE_KEY] - write_class(writer, subcontent) + subcontent, external = resolve_refs(contents, depcontent) + subclass = get_subclass(contents, subcontent) + if external and subclass: + writer('from .%s import %s' % (get_file_base(contents, subcontent), subclass)) + elif is_map(subcontent): + mapcontent, _ = resolve_refs(contents, subcontent['patternProperties']) + assert len(mapcontent) == 1, 'multiple patternProperties not supported' + write_dep(writer, contents, list(mapcontent.values())[0]) + elif subclass: + subcontent[SOURCE_KEY] = contents[SOURCE_KEY] + write_class(writer, subcontent) def write_definitions(writer, contents): - definitions = contents.get('definitions') - if not definitions: - return - writer('') - for definition in definitions: - if definitions[definition].get('title'): - class_name = get_class_name(definitions[definition]) - writer(' %s = %s' % (class_name, class_name)) - writer('') - + definitions = contents.get('definitions') + if not definitions: + return + writer('') + for definition in definitions: + if definitions[definition].get('title'): + class_name = get_class_name(definitions[definition]) + writer(' %s = %s' % (class_name, class_name)) + writer('') + def write_init(writer, contents): - writer('') - write_definitions(writer, contents) - writer(' def __init__(self):') - properties = get_properties(contents) - if not properties: - writer(' pass') - for property in properties: - writer(' self.%s = None' % property) + writer('') + write_definitions(writer, contents) + writer(' def __init__(self):') + properties = get_properties(contents) + if not properties: + writer(' pass') + for property in properties: + writer(' self.%s = None' % property) def generate_class(contents): - file_name = get_file_base(contents, contents) + '.py' - full_path = os.path.join(out_dir, file_name) - eprint('Generating %s...' % full_path) - with open(full_path, 'w') as fd: - writer = lambda line: fd.write(line + '\n') - writer('"""Generated class for %s"""' % os.path.basename(contents[SOURCE_KEY])) - write_class(writer, contents) + file_name = get_file_base(contents, contents) + '.py' + full_path = os.path.join(out_dir, file_name) + eprint('Generating %s...' % full_path) + with open(full_path, 'w') as fd: + writer = lambda line: fd.write(line + '\n') + writer('"""Generated class for %s"""' % os.path.basename(contents[SOURCE_KEY])) + write_class(writer, contents) def generate_module(in_files): - shutil.rmtree(out_dir) - os.makedirs(out_dir, exist_ok=True) - full_path = os.path.join(out_dir, '__init__.py') - with open(full_path, 'w') as fd: - writer = lambda line: fd.write(line + '\n') - for in_file in in_files: - try: - source = os.path.join(os.getcwd(), in_file) - contents = process_file(source) - writer('from .%s import %s' % (get_file_base(contents, contents), get_class_name(contents))) - except Exception: - raise Exception('While processing file %s' % in_file) + shutil.rmtree(out_dir) + os.makedirs(out_dir, exist_ok=True) + full_path = os.path.join(out_dir, '__init__.py') + with open(full_path, 'w') as fd: + writer = lambda line: fd.write(line + '\n') + for in_file in in_files: + try: + source = os.path.join(os.getcwd(), in_file) + contents = process_file(source) + writer('from .%s import %s' % (get_file_base(contents, contents), get_class_name(contents))) + except Exception: + raise Exception('While processing file %s' % in_file) generate_module(in_files) diff --git a/bin/gencode_root_schemas b/bin/gencode_root_schemas index f4b6b4b629..80ea225a8f 100755 --- a/bin/gencode_root_schemas +++ b/bin/gencode_root_schemas @@ -48,6 +48,8 @@ def nested_key_extract(key, obj): return values def strip_extension(file): + # returns a tuple and only first variable is needed + # pylint: disable-next=unused-variable root, ext = os.path.splitext(file) return root @@ -63,6 +65,7 @@ def file_from_json_ref(ref): try: m = re.match(r'^file:([A-Za-z_0-9/\\\.]*\.json)#?.*$', ref) return m.group(1) + # pylint: disable-next=broad-except except Exception: return None @@ -80,7 +83,7 @@ if __name__ == '__main__': root_schemas.append(schema_file) if schema_file in IGNORE_LIST: continue - with open(file_path) as f: + with open(file_path, 'r', encoding='utf-8') as f: schema_json = json.load(f) refs = nested_key_extract('$ref', schema_json) refs = [file_from_json_ref(ref) for ref in refs] @@ -90,6 +93,7 @@ if __name__ == '__main__': try: if single_ref not in ALWAYS_ROOT: root_schemas.remove(single_ref) + # pylint: disable-next=broad-except except Exception: pass diff --git a/bin/gencode_seq b/bin/gencode_seq index a26cc0ee9f..9a16bba5a9 100755 --- a/bin/gencode_seq +++ b/bin/gencode_seq @@ -8,38 +8,38 @@ SEQUENCE_MD=docs/specs/sequences/generated.md # Create doc of generated sequence steps prefix=sites/udmi_site_model/out/devices/AHU-1/tests if [[ -d $prefix ]]; then - echo Updating $SEQUENCE_MD from $prefix: + echo Updating $SEQUENCE_MD from $prefix: - # Clear out existing generated sequences - sed -i '/