Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Resolve pylint errors #4

Merged
merged 2 commits into from
Dec 22, 2022
Merged

Resolve pylint errors #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1456a1f
pylint
noursaidi Dec 21, 2022
86a7dfc
fix gencode categories
noursaidi Dec 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
429 changes: 429 additions & 0 deletions .pylintrc
View file
Open in desktop

Large diffs are not rendered by default.

41 changes: 24 additions & 17 deletions bin/gcp_device_logs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import json
import dateutil.parser
import argparse

# pylint: disable-next=line-too-long
SHELL_TEMPLATE = 'gcloud logging read "logName=projects/{}/logs/cloudiot.googleapis.com%2Fdevice_activity AND ({}) AND timestamp>=\\\"{}\\\"" --limit 1000 --format json --project {}'
TIMESTAMP_FORMAT = '%Y-%m-%dT%H:%M:%SZ' #timestamp >= "2016-11-29T23:00:00Z"

Expand All @@ -29,7 +30,8 @@ args = parse_command_line_args()
target_devices = args.device_ids
project_id = args.project_id

device_filter = ' OR '.join([f'labels.device_id={target}' for target in target_devices])
device_filter = ' OR '.join([f'labels.device_id={target}' \
for target in target_devices])

# sleep duration - balance of speed and accuracy for ordering as some entries
# can be delayed by about 10 seconds
Expand All @@ -42,8 +44,15 @@ seen = []

while True:
try:
shell_command = SHELL_TEMPLATE.format(project_id, device_filter, search_timestamp.strftime(TIMESTAMP_FORMAT), project_id)
output = subprocess.run(shell_command, capture_output=True, shell=True, check=True)
shell_command = SHELL_TEMPLATE.format(project_id,
device_filter,
search_timestamp.strftime(TIMESTAMP_FORMAT),
project_id)

output = subprocess.run(shell_command,
capture_output=True,
shell=True,
check=True)

data = json.loads(output.stdout)

Expand All @@ -56,38 +65,38 @@ while True:
continue

seen.append(insert_id)

event_type = entry['jsonPayload']['eventType']
log_data = entry['jsonPayload']
event_type = log_data['eventType']
timestamp = entry['timestamp']
registry_id = entry['resource']['labels']['device_registry_id']
log_device_id = entry['labels']['device_id']
metadata = ''

if event_type == 'PUBLISH':
metadata = entry['jsonPayload'].get('publishFromDeviceTopicType')
publishToDeviceTopicType = entry['jsonPayload'].get('publishToDeviceTopicType')
metadata = log_data.get('publishFromDeviceTopicType')
publishToDeviceTopicType = log_data.get('publishToDeviceTopicType')
if publishToDeviceTopicType == 'CONFIG':
event_type = 'CONFIG'
metadata = ''
elif not metadata and publishToDeviceTopicType:
metadata = f'TO DEVICE {publishToDeviceTopicType}'

if event_type == 'PUBACK':
metadata = entry['jsonPayload']['publishToDeviceTopicType']
metadata = log_data['publishToDeviceTopicType']

if event_type == 'SUBSCRIBE':
metadata = entry['jsonPayload']['mqttTopic']
metadata = log_data['mqttTopic']

if event_type == 'ATTACH_TO_GATEWAY':
metadata = entry['jsonPayload']['gateway']['id']
metadata = log_data['gateway']['id']

if event_type == 'DISCONNECT':
metadata = entry['jsonPayload']['disconnectType']
metadata = log_data['disconnectType']

if entry['jsonPayload']['status'].get('code') != 0:
metadata = (f"{metadata}"
f"({entry['jsonPayload']['status'].get('description')}"
f"{entry['jsonPayload']['status'].get('message', '')})")
if log_data['status'].get('code') != 0:
metadata = (f'{metadata}'
f'({log_data["status"].get("description")}'
f'{log_data["status"].get("message", "")})')

entries.append({'timestamp_obj': dateutil.parser.parse(timestamp),
'timestamp': timestamp,
Expand All @@ -112,7 +121,5 @@ while True:
print('Ensure gcloud is authenticated and account has permissions')
print('to access cloud logging')
sys.exit(1)
except Exception:
pass
finally:
time.sleep(dt)
157 changes: 80 additions & 77 deletions bin/gencode_categories
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,22 +1,18 @@
#!/usr/bin/env python3

import hashlib
import json
""" Gencode generator for categories """
import re
import os
import shutil
import sys

GENCODE_MARKER = '@@ '
CATEGORY_MARKER = '* '
WILDCARD_REGEX = ' *\\* _\?\?\?_: (.*)'
WILDCARD_SUFFIX = '.[a-z]+(_[a-z]+)*'
WILDCARD_REGEX = r' *\* _\?\?\?_: (.*)'
WILDCARD_SUFFIX = r'.[a-z]+(_[a-z]+)*'

CATEGORY_REGEX = ' *\\* _([a-z]+)_: (\(\*\*([A-Z]+)\*\*\) )?(.*)'
CATEGORY_REGEX = r' *\* _([a-z]+)_: (\(\*\*([A-Z]+)\*\*\) )?(.*)'
JSON_FORMAT = '%s%s{ "pattern": "^%s$" }'
DEVICE_PREFIX = 'device'

JAVA_DESCRIPTION = "\n%s// %s\n"
JAVA_DESCRIPTION = '\n%s// %s\n'
JAVA_TARGET = '%spublic static final String %s = "%s";\n'
JAVA_LEVEL = '%spublic static final Level %s_LEVEL = %s;\n'
JAVA_MAP_ADD = '%sstatic { LEVEL.put(%s, %s); }\n'
Expand All @@ -28,81 +24,88 @@ java_in = os.path.join('etc/Category.java')
java_out = os.path.join('gencode/java/udmi/schema/Category.java')

def read_categories():
categories = []
prefix = []
previous = -1
group = None
with open(doc_in) as doc:
while line := doc.readline():
indent = line.find(CATEGORY_MARKER)//2
wildcard = re.match(WILDCARD_REGEX, line)
if wildcard:
entry = (group + WILDCARD_SUFFIX, 'INFO', wildcard.group(1))
categories.append(entry)
continue
match = re.match(CATEGORY_REGEX, line)
if indent < 0 or not match:
continue
if indent < previous:
for _ in range(indent, previous):
rem = prefix.pop(len(prefix) - 1)
elif indent > previous:
if group:
prefix.append(group)
previous = indent
group = match.group(1)
category = '.'.join(prefix + [group])
level = match.group(3)
description = match.group(4)
if level:
entry = (category, level, description)
categories.append(entry)
return categories
categories = []
prefix = []
previous = -1
group = None
with open(doc_in, 'r', encoding='utf-8') as doc:
while line := doc.readline():
indent = line.find(CATEGORY_MARKER)//2
wildcard = re.match(WILDCARD_REGEX, line)
if wildcard:
entry = (group + WILDCARD_SUFFIX, 'INFO', wildcard.group(1))
categories.append(entry)
continue
match = re.match(CATEGORY_REGEX, line)
if indent < 0 or not match:
continue
if indent < previous:
for _ in range(indent, previous):
# pylint: disable-next=unused-variable
rem = prefix.pop(len(prefix) - 1)
elif indent > previous:
if group:
prefix.append(group)
previous = indent
group = match.group(1)
category = '.'.join(prefix + [group])
level = match.group(3)
description = match.group(4)
if level:
entry = (category, level, description)
categories.append(entry)
return categories

def write_schema_out(categories):
with open(schema_in) as inp:
with open(schema_out, 'w') as out:
while line := inp.readline():
index = line.find(GENCODE_MARKER)
if index >= 0:
write_schema_categories(out, line[0:index], categories)
else:
out.write(line)
with open(schema_in, 'r', encoding='utf-8') as inp:
with open(schema_out, 'w', encoding='utf-8') as out:
while line := inp.readline():
index = line.find(GENCODE_MARKER)
if index >= 0:
write_schema_categories(out, line[0:index], categories)
else:
out.write(line)

def write_schema_categories(out, indent, categories):
prefix = ''
for category in categories:
target = category[0].replace('.', '\\\\.')
out.write(JSON_FORMAT % (prefix, indent, target))
prefix = ',\n'
out.write('\n')
prefix = ''
for category in categories:
target = category[0].replace('.', '\\\\.')
out.write(JSON_FORMAT % (prefix, indent, target))
prefix = ',\n'
out.write('\n')

def write_java_out(categories):
os.makedirs(os.path.dirname(java_out), exist_ok=True)
with open(java_in) as inp:
with open(java_out, 'w') as out:
while line := inp.readline():
index = line.find(GENCODE_MARKER)
if index >= 0:
indent = line[0:index]
write_java_categories(out, indent, categories)
else:
out.write(line)
os.makedirs(os.path.dirname(java_out), exist_ok=True)
with open(java_in, 'r', encoding='utf-8') as inp:
with open(java_out, 'w', encoding='utf-8') as out:
while line := inp.readline():
index = line.find(GENCODE_MARKER)
if index >= 0:
indent = line[0:index]
write_java_categories(out, indent, categories)
else:
out.write(line)

def write_java_categories(out, indent, categories):
for category in categories:
target = category[0]
if target.startswith(DEVICE_PREFIX):
continue
level = category[1]
desc = category[2]
const = target.replace('.', '_').upper()
out.write(JAVA_DESCRIPTION % (indent, desc))
out.write(JAVA_TARGET % (indent, const, target))
out.write(JAVA_LEVEL % (indent, const, level))
out.write(JAVA_MAP_ADD % (indent, const, level))
for category in categories:
target = category[0]
if target.startswith(DEVICE_PREFIX):
continue
level = category[1]
desc = category[2]
const = target.replace('.', '_').upper()
out.write(JAVA_DESCRIPTION % (indent, desc))
out.write(JAVA_TARGET % (indent, const, target))
out.write(JAVA_LEVEL % (indent, const, level))
out.write(JAVA_MAP_ADD % (indent, const, level))


def main():
categories = read_categories()
write_schema_out(categories)
write_java_out(categories)

if __name__ == '__main__':
main()


categories = read_categories()
write_schema_out(categories)
write_java_out(categories)
Loading