u_gnss_cfg_val_key.py

#!/usr/bin/env python

'''Update the file u_gnss_cfg_val_key.h with key ID macros.'''

from multiprocessing import Process, freeze_support # Needed to make Windows behave
                                                    # when run under multiprocessing,
from signal import signal, SIGINT   # For CTRL-C handling
import os
import sys # For exit() and stdout
import argparse
import subprocess
import platform # Figure out current OS

# This script reads the file u_gnss_cfg_val_key.h
# and re-writes the end of that file with C compiler
# macros generated through reading the first part of
# the file.
#
# It works like this:
#
# 1. Finds the enum uGnssCfgValKeySize_t: this
#    contains the encoded values for the different sizes
#    in the key ID.
#
# 2. Finds the enum uGnssCfgValKeyGroupId_t: this
#    contains all of the group IDs for all of the
#    key IDs macros that will be generated.
#
# 3. For each entry in uGnssCfgValKeyGroupId_t it
#    finds the corresponding enum listing the items.  For
#    instance for the group ID U_GNSS_CFG_VAL_KEY_GROUP_ID_ANA
#    it will look for the enum uGnssCfgValKeyItemAna_t,
#    i.e. an enum where the bit after U_GNSS_CFG_VAL_KEY_GROUP_ID_
#    appears, following the prefix uGnssCfgValKeyItem and with
#    camel case applied instead of upper/snake case.
#
# 4. For each entry in the group enum it creates a key ID
#    macro, so for instance U_GNSS_CFG_VAL_KEY_ITEM_ANA_USE_ANA_L,
#    which it now knows is in group ID 0x23 (ANA),
#    item ID 0x01 (USE_ANA) and is of size 1 bit (L), would
#    become the following macro:
#
#    #define U_GNSS_CFG_VAL_KEY_ID_ANA_USE_ANA_L 0x10230001
#
# 5. It looks for two markers in the file:
#
#    // *** DO NOT MODIFY THIS LINE OR BELOW: AUTO-GENERATED BY u_gnss_cfg_val_key.py ***
#
#   ...and
#
#    // *** DO NOT MODIFY THIS LINE OR ABOVE: DO NOT MODIFY AREA ENDS ***
#
#    ...erases anything between them and and writes all of the
#    generated macros there instead.  A backup is made of the
#    current file, just in case.

# The file to be read/modified
TARGET_FILE_NAME = "u_gnss_cfg_val_key.h"

# The file extension to be used for the back-up of the file
BACKUP_EXTENSION = "_bak"

# The initial part of the prefix, to be expected on every entry in every enum
ENUM_ENTRY_PREFIX_ALL = "U_GNSS_CFG_VAL_KEY_"

# The key size enum name
ENUM_NAME_KEY_SIZE = "uGnssCfgValKeySize_t"

# The prefix to expect on every entry in the key size enum
ENUM_ENTRY_PREFIX_KEY_SIZE = ENUM_ENTRY_PREFIX_ALL + "SIZE_"

# The main group ID enum name
ENUM_NAME_GROUP_ID = "uGnssCfgValKeyGroupId_t"

# The prefix to expect on every entry in the main group ID enum
ENUM_ENTRY_PREFIX_GROUP_ID = ENUM_ENTRY_PREFIX_ALL + "GROUP_ID_"

# The prefix of the enum defining the items in a group
ENUM_NAME_PREFIX_ITEMS = "uGnssCfgValKeyItem"

# The post-fix to be expected on all enumerated types
ENUM_NAME_POSTFIX = "_t"

# The prefix to expect on every item
ENUM_ENTRY_PREFIX_ITEMS = ENUM_ENTRY_PREFIX_ALL + "ITEM_"

# The marker to look for, beyond which we can re-write the target
# file up to FILE_REWRITE_MARKER_END
FILE_REWRITE_MARKER_START = "// *** DO NOT MODIFY THIS LINE OR BELOW: AUTO-GENERATED BY u_gnss_cfg_val_key.py ***"

# The marker up to which the target file can be re-written
FILE_REWRITE_MARKER_END = "// *** DO NOT MODIFY THIS LINE OR ABOVE: DO NOT MODIFY AREA ENDS ***"

# The strings that represent each key size and the values
# in uGnssCfgValKeySize_t they correspond to
KEY_SIZE_STRING_LIST = [("L", "ONE_BIT"),
                        ("U1", "ONE_BYTE"),
                        ("U2", "TWO_BYTES"),
                        ("U4", "FOUR_BYTES"),
                        ("U8", "EIGHT_BYTES"),
                        ("E1", "ONE_BYTE"),
                        ("E2", "TWO_BYTES"),
                        ("E4", "FOUR_BYTES"),
                        ("X1", "ONE_BYTE"),
                        ("X2", "TWO_BYTES"),
                        ("X4", "FOUR_BYTES"),
                        ("X8", "EIGHT_BYTES"),
                        ("I1", "ONE_BYTE"),
                        ("I2", "TWO_BYTES"),
                        ("I4", "FOUR_BYTES"),
                        ("I8", "EIGHT_BYTES"),
                        ("R4", "FOUR_BYTES"),
                        ("R8", "EIGHT_BYTES")]

def signal_handler(sig, frame):
    '''CTRL-C Handler'''
    del sig
    del frame
    sys.stdout.write('\n')
    print("CTRL-C received, EXITING.")
    sys.exit(-1)

# subprocess arguments behaves a little differently on Linux and Windows
# depending if a shell is used or not, which can be read here:
# https://stackoverflow.com/a/15109975
# This function will compensate for these deviations
def subprocess_osify(cmd, shell=True):
    '''Expects an array of strings being [command, param, ...]'''
    if platform.system() != "Windows" and shell:
        line = ""
        for command in cmd:
            # Put everything in a single string and quote args containing spaces
            if " " in command:
                line += f"\"{command}\" "
            else:
                line += f"{command} "
        cmd = line
    return cmd

def read_enum(enum_name, enum_items_prefix, line_list):
    '''Find an enum in the set of lines and return a list of tuples containing its entries'''
    entry_list = []
    last_line_index = -1

    # First find the line with the enum name on it preceded by a '}'
    for idx, line in enumerate(line_list):
        if enum_name in line:
            bits = line.split()
            if bits[0][0] == '}':
                last_line_index = idx
                break
    if last_line_index >= 0:
        # Now work backwards to find the start of an enum declaration,
        # processing the lines into tuples as we go, ensuring that
        # each begins with the correct prefix
        for idx in reversed(range(0, last_line_index)):
            bits = line_list[idx].split()
            if len(bits) > 2 and bits[0][0].isalpha() and bits[1] == "=":
                # Got a line that could have an entry on it
                entry_tuple = ()
                if bits[0].startswith(enum_items_prefix):
                    # This line has an entry with the right prefix
                    entry_name = bits[0].split(enum_items_prefix)
                    if len(entry_name) == 2 and bits[1] == "=":
                        # Not only the prefix but something after it also,
                        # and then an equals sign; check for an integer
                        # coming afterwards, taking off the comma
                        more_bits = bits[2].split(",")
                        try:
                            entry_value = int(more_bits[0], 0)
                            # The bit after it is an integer: add the tuple
                            # to the list
                            entry_tuple = (entry_name[1], entry_value)
                        except ValueError:
                            print("In enum {}, line {}, found an entry \"{}\""  \
                                  " that has a malformed value \"{}\"".         \
                                  format(enum_name, idx, bits[0], bits[2]))
                            entry_list = []
                            break
                if entry_tuple:
                    entry_list.append(entry_tuple)
                else:
                    print("In enum {}, line {}, found an entry \"{}\""  \
                          "  that doesn't match the prefix \"{}\".".    \
                          format(enum_name, idx, bits[0], enum_items_prefix))
                    entry_list = []
                    break
            else:
                # Check whether we're now at the start of the enum
                if len(bits) == 3 and bits[0] == "typedef" and bits[1] == "enum" and bits[2] == "{":
                    break
    if entry_list:
        print("Found enum {}, entry prefix \"{}\", with {} member(s).".      \
              format(enum_name, enum_items_prefix, len(entry_list)))

    return entry_list

def convert_group_id_to_items(group_id_name):
    ''' Given a group ID (e.g. ANA) make the enum name (e.g. uGnssCfgValKeyItemAna_t) and entry prefix'''
    enum_name_items = ENUM_NAME_PREFIX_ITEMS
    enum_entry_prefix_items = ENUM_ENTRY_PREFIX_ITEMS
    bits = group_id_name.split("_")

    for idx, bit in enumerate(bits):
        enum_name_items += bit[0].upper() + bit[1:].lower()
        if idx > 0:
            enum_entry_prefix_items += "_"
        enum_entry_prefix_items += bit.upper()

    enum_name_items += ENUM_NAME_POSTFIX

    return enum_name_items, enum_entry_prefix_items + "_"

def create_key_id(item_tuple, group_id_value, key_size_list):
    ''' Create a key ID from an item tuple, the group ID value and the key size enum '''
    key_id = -1

    # Get the key size for this item by looking at the
    # _E1 or whatever bit on the end of the item name
    bits = item_tuple[0].split("_")
    for key_size_string in KEY_SIZE_STRING_LIST:
        size = -1
        if bits[len(bits) - 1] in key_size_string[0]:
            size_string = key_size_string[1]
            for key_size in key_size_list:
                if size_string in key_size[0]:
                    size = key_size[1]
                    break
            if size >= 0:
                break
    if size >= 0:
        # Got the size, put the key ID together
        # The key size is bits 28 to 30
        # the group ID is bits 16 to 24
        # the item ID in the group is bits 0 to 12
        key_id = ((size & 0x07) << 28) | (group_id_value << 16) | item_tuple[1]

    return key_id

def rewrite_line_list(key_id_list, input_line_list):
    '''Re-write the line_list with the macros'''
    column_offset = 0
    output_line_list = []
    start_marker_index = -1
    end_marker_index = -1
    output_line_list_one = []
    output_line_list_two = []
    output_line_list_three = []

    for key_id_tuple in key_id_list:
        # Run through the list to determine the column offset for the value field
        if len(key_id_tuple[0]) > column_offset:
            column_offset = len(key_id_tuple[0])

    for idx, line in enumerate(input_line_list):
        # Make a list of all lines up to and include the start marker
        output_line_list_one.append(line)
        if line.startswith(FILE_REWRITE_MARKER_START):
            start_marker_index = idx
            break

    if start_marker_index >= 0:
        # Found the start marker, create our list of stuff, aligning the columns nicely
        # and reversing the order of the last as things are "appended" the start
        for key_id_tuple in key_id_list:
            output_line = "#define " + key_id_tuple[0] + " "
            for _ in range(len(key_id_tuple[0]), column_offset):
                output_line += " "
            output_line_list_two.append(output_line + hex(key_id_tuple[1]) + "\n")
        # Make a list of all lines from [including] the end marker to the end of the list
        for idx, line in enumerate(input_line_list[start_marker_index:]):
            if end_marker_index < 0 and line.startswith(FILE_REWRITE_MARKER_END):
                end_marker_index = idx
            if end_marker_index >= 0:
                output_line_list_three.append(line)

    if start_marker_index < 0:
        print("Could not find the start marker \"{}\" in the file, stopping.".  \
              format(FILE_REWRITE_MARKER_START))
    else:
        if end_marker_index < 0:
            print("Could not find the end marker \"{}\" in the file, stopping.".  \
                  format(FILE_REWRITE_MARKER_END))
        else:
            # Combine the three lists
            output_line_list = output_line_list_one + ["\n"] + output_line_list_two + \
                               ["\n"] + output_line_list_three

    return output_line_list

def copy_file(source, destination):
    '''Copy a file from source to destination using OS commands'''
    success = False

    call_list = []
    if platform.system() == "Windows":
        call_list.append("copy")
        call_list.append("/Y")
    else:
        call_list.append("cp")
    call_list.append(source)
    call_list.append(destination)
    try:
        print(f"Copying {source} to {destination}...")
        subprocess.check_output(subprocess_osify(call_list), shell=True)
        success = True
    except subprocess.CalledProcessError as error:
        print(f"Error when copying {source} to {destination}," \
              f"{error.cmd} {error.returncode}: \"{ error.output}\"")
    return success

def main(target_file):
    '''Main as a function'''
    return_value = 1
    keep_going = True
    line_list = []
    key_size_list = []
    key_id_list = []

    signal(SIGINT, signal_handler)

    if os.path.isfile(target_file):
        with open(target_file, "r", encoding="utf8") as file:
            # Read the lot in
            print(f"Reading file {target_file}...")
            line_list = file.readlines()
        if line_list:
            print("Looking for the key size and group ID enums {} and {}...". \
                  format(ENUM_NAME_KEY_SIZE, ENUM_NAME_GROUP_ID))
            # Look for the uGnssCfgValKeySize_t enumeration
            key_size_list = read_enum(ENUM_NAME_KEY_SIZE, ENUM_ENTRY_PREFIX_KEY_SIZE, line_list)
        if key_size_list:
            print("Found the following key sizes and encoded values:")
            for key_size_tuple in key_size_list:
                key_size_string = ""
                for key_size_string_tuple in KEY_SIZE_STRING_LIST:
                    if key_size_string_tuple[1] == key_size_tuple[0]:
                        if key_size_string:
                            key_size_string += ", "
                        key_size_string += key_size_string_tuple[0]
                if key_size_string:
                    print("  {}: {} 0x{:02x}". \
                          format(key_size_string, ENUM_ENTRY_PREFIX_KEY_SIZE + key_size_tuple[0], key_size_tuple[1]))
            # Read the whole group ID enum from the file
            group_id_list = read_enum(ENUM_NAME_GROUP_ID, ENUM_ENTRY_PREFIX_GROUP_ID, line_list)
            if group_id_list:
                print(f"Looking for each item enum pointed-to by {ENUM_NAME_GROUP_ID}, " \
                      f" all of which should begin with \"{ENUM_NAME_PREFIX_ITEMS}\"...")
                # For each group ID, find the enum giving its items in the file;
                # do it reversed as an "append" actually adds to the front, this
                # we the IDs are in the right order.
                for group_id_tuple in reversed(group_id_list):
                    enum_name_items, enum_entry_prefix_items = convert_group_id_to_items(group_id_tuple[0])
                    if enum_name_items and enum_entry_prefix_items:
                        item_list = read_enum(enum_name_items, enum_entry_prefix_items, line_list)
                        if item_list:
                            for item_tuple in reversed(item_list):
                                # For a group we can now create all the key IDs
                                key_id = create_key_id(item_tuple, group_id_tuple[1], key_size_list)
                                if key_id >= 0:
                                    key_id_list.append((enum_entry_prefix_items.replace("ITEM", "ID") + \
                                                       item_tuple[0], key_id))
                                else: 
                                    print("Could not find key size for item \"{}\";"      \
                                          " does it have an _X on the end, where X"       \
                                          " is a size indicator from the list, e.g."      \
                                          " _{}? Stopping.".                              \
                                          format(enum_entry_prefix_items + item_tuple[0], \
                                                 KEY_SIZE_STRING_LIST[len(KEY_SIZE_STRING_LIST) - 1][0]))
                                    key_id_list = []
                                    break
                        else:
                            print("Could not find a valid {} enum (needed for group ID {})" \
                                  " in {}, stopping.".format(enum_name_items,               \
                                  ENUM_ENTRY_PREFIX_GROUP_ID + group_id_tuple[0], target_file))
                            key_id_list = []
                            break
                    else:
                        print("Could not convert \"{}\" into an enum name and enum" \
                              " item prefix, stopping.".format(group_id_tuple[0]))
                        key_id_list = []
                    if not key_id_list:
                        break
            else:
                print("Could not find group ID enum {} in {}, stopping.".  \
                      format(ENUM_NAME_GROUP_ID, target_file))
        else:
            print("Could not find key size enum {} in {}, stopping.". \
                   format(ENUM_NAME_KEY_SIZE, target_file))

        if key_id_list:
            print(f"{len(key_id_list)} key ID macros created, re-writing file...")
            # Have a list of key IDs, re-write the line-list using it
            line_list = rewrite_line_list(key_id_list, line_list)
            if line_list:
                # Done everything; make a back-up copy of the file
                if copy_file(target_file, target_file + BACKUP_EXTENSION):
                    #... and write line_list back to the file
                    with open(target_file, "w", encoding="utf8") as file:
                        file.writelines(line_list)
                        print("{} has been re-written.".format(target_file))
                        return_value = 0
    else:
        print(f"\"{target_file}\" is not a file.")

    return return_value

if __name__ == "__main__":
    PARSER = argparse.ArgumentParser(description="A script to"      \
                                     " update the key ID macros"    \
                                     " in " + TARGET_FILE_NAME + ".\n")
    PARSER.add_argument("-f", default=TARGET_FILE_NAME, help="the" \
                        " file name to update, default " + TARGET_FILE_NAME)
    ARGS = PARSER.parse_args()

    # Call main()
    RETURN_VALUE = main(ARGS.f)

    sys.exit(RETURN_VALUE)

# A main is required because Windows needs it in order to
# behave when this module is called during multiprocessing
# see https://docs.python.org/2/library/multiprocessing.html#windows
if __name__ == '__main__':
    freeze_support()
    PROCESS = Process(target=main)
    PROCESS.start()