diff --git a/README.md b/README.md index d3f58d6..d66834a 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ * [Configuration](#configuration) * [Sample cFS Workspace](#sample-cfs-workspace) * [Running the Sample cFS Workspace Scripts](#running-the-sample-cfs-workspace-scripts) + * [Troubleshooting and Support](#troubleshooting-and-support) * [Using CTF on a New cFS Project](#using-ctf-on-a-new-cfs-project) * [CCSDS Message Definitions](#ccsds-message-definitions) * [CCDD Auto Export](#ccdd-auto-export) @@ -143,9 +144,8 @@ Sample CTF output is shown below. [13:00:47.214] ccsds_plugin (35 ) *** WARNING: CFS Plugin not yet loaded... [13:00:47.345] plugin_manager (235) *** INFO: Found plugin class: plugins.cfs.cfs_plugin.CfsPlugin [13:00:47.346] plugin_manager (235) *** INFO: Found plugin class: plugins.example_plugin.example_plugin.ExamplePlugin -[13:00:47.348] plugin_manager (235) *** INFO: Found plugin class: plugins.sp0_plugin.sp0_plugin.sp0Plugin [13:00:47.348] plugin_manager (235) *** INFO: Found plugin class: plugins.ssh.ssh_plugin.SshPlugin -[13:00:47.349] plugin_manager (211) *** INFO: From package: plugins - Loaded the following plugins: ['CCSDS Plugin', 'CFS Plugin', 'ExamplePlugin', 'sp0_plugin', 'SshPlugin'] +[13:00:47.349] plugin_manager (211) *** INFO: From package: plugins - Loaded the following plugins: ['CCSDS Plugin', 'CFS Plugin', 'ExamplePlugin', 'SshPlugin'] [13:00:47.349] ctf (50 ) *** INFO: Loading Plugins - End [13:00:47.349] ctf (58 ) *** INFO: Status_Manager created [13:00:47.349] ctf (62 ) *** INFO: Script_Manager created @@ -156,7 +156,6 @@ Sample CTF output is shown below. [13:00:47.350] ccsds_plugin (38 ) *** INFO: Initialized CCSDS Plugin [13:00:47.350] cfs_time_manager (45 ) *** INFO: CfsTimeManager Initialized. Verification Poll Period = 0.05. [13:00:47.350] cfs_plugin (152) *** INFO: Initialized CfsPlugin -[13:00:47.350] sp0_plugin (64 ) *** INFO: Initialized SP0 plugin [13:00:47.351] test_script (92 ) *** INFO: Verification Test Name: CFS CI Functions Test [13:00:47.351] test_script (93 ) *** INFO: Verification Test Number: CFS-CI-Functions-Test [13:00:47.351] test_script (94 ) *** INFO: Test Conductor: ashehata @@ -203,6 +202,10 @@ xterm: cannot load font '-misc-fixed-medium-r-semicondensed--13-120-75-75-c-60-i [13:00:54.408] test (134) *** TEST_PASS: Verification Passed CheckTlmValue: {'name': '', 'mid': 'CI_HK_TLM_MID', 'args': [{'variable': 'usCmdCnt', 'value': [1], 'compare': '=='}, {'variable': 'usCmdErrCnt', 'value': [0], 'compare': '=='}]} ``` +### Troubleshooting and Support + +See the [Troubleshooting & FAQ Guide](docs/ctf/faq.md) for solutions to common issues. There is additional technical documentation for CTF and the editor in the `docs` directory. If your problem is not resolved, or you have other questions or suggestions for CTF, please open an issue on the public repository at https://github.com/nasa/CTF/issues + ## Using CTF on a New cFS Project ### CCSDS Message Definitions @@ -379,6 +382,23 @@ Upon completion, a summary will be printed to the console and the CTF unit test ## Release Notes +### v1.3.1 +10/19/2021 + +* CTF Plugins Changes + + * CFS shutdown on Linux logs an error, but does not fail, if no such process is found. + + * CFS Plugin macros now require the format `#MACRO#` instead of `#MACRO`. + + * Fix the 'disable' attribute in function definition. + +* CTF Documentation Changes + * Updated JSON Test Script Guide + + * Add Troubleshooting & FAQ Guide section. + + ### v1.3 08/26/2021 @@ -447,14 +467,12 @@ Upon completion, a summary will be printed to the console and the CTF unit test * Minor improvements and bug fixes. * CFS Plugin Changes - * Update SP0 Protocol and FTP Interface with changes from GSFC functional testing. + * Update FTP Interface with changes from GSFC functional testing. * Fix the use of CCDD macros in test scripts. * Improve argument type checking in `SendCfsCommand` instruction. - * Improve validation of the SP0 and Remote CFS configs. `RegisterCfs` will fail if any errors are found. - * Change `ValidateCcsdsData` parameter from `name` to `target` to match CFS instructions. * Minor improvements and bug fixes. @@ -497,8 +515,6 @@ Upon completion, a summary will be printed to the console and the CTF unit test * Resolve CheckTlmValue passing if one or more variables fails to be evaluated from the telemetry packet. - * Resolve connection and deployment issues with cFS targets running with the SP0 protocol (WIP) - * Minor improvements and bug fixes. @@ -667,11 +683,11 @@ Major backend updates to improve reliability/maintainability of CTF. * cFS Plugin Changes * Add new instruction "ArchiveCfsFiles", which accepts a FSW artifacts and moves any files created during the test run to the test results directory. - * Note: Currently, this feature is supported for Linux cFS targets only. SSH and SP0 support for archiving FSW artifacts is planned for a later version. + * Note: Currently, this feature is supported for Linux cFS targets only. SSH support for archiving FSW artifacts is planned for a later version. * Redesign multi-cfs architecture to be purely configuration based (remove script-specific overrides to allow scripts to be platform independent.) * Each cFS Target should have its own configuration section in the config INI file. * RegisterCfs receives a name, and attempts to load target configuration (including protocol) from the loaded configuration. - * A PASS/FAIL for that instruction will be set after validating the necessary config fields for cFS and the specific protocol (Linux, SSH, SP0) + * A PASS/FAIL for that instruction will be set after validating the necessary config fields for cFS and the specific protocol (Linux, SSH) * Rework macro replacement logic such that * Macro replacement occurs for command argument values, telemetry field values, and array indices. * All macros in a test script should start with the # prefix (example: "#SOME_MACRO_IN_MACRO_MAP"). @@ -717,12 +733,11 @@ Major backend updates to improve reliability/maintainability of CTF. * Add CCDD JSON Export Reader * Multi/Remote cFS Support * Allow the cFS Plugin to execute/manage multiple instances of cFS running on local or remote targets - * Remote Targets Support: SSH, SP0, Local (linux). More targets can be added as needed. + * Remote Targets Support: SSH, Local (linux). More targets can be added as needed. * Change command argument structure in test script JSON. Arguments are now encoded in the test script as a JSON dictionary. * Initial Plugin Implementations For * SSH Plugin - * SP0 Plugin * CCSDS Plugin * Re-haul CTF Editor to support generic CCSDS JSON Exports, and multi/remote cFS. diff --git a/configs/example_configs/example_config_sp0.ini b/configs/example_configs/example_config_sp0.ini deleted file mode 100644 index b4b889c..0000000 --- a/configs/example_configs/example_config_sp0.ini +++ /dev/null @@ -1,275 +0,0 @@ -################################# -# CTF Core Configuration -################################# - -[core] - -# Global verification timeout for any "verify_required" CTF instructions -# Unit: Seconds if using generic system time manager) -ctf_verification_timeout = 5.0 - -# Global polling period for any "verify_required" CTF insturction. -# Defines how often to run verification commands until verification either -# passes or timeout -# Unit: Seconds if using generic system time manager) -ctf_verification_poll_period = 0.5 - - -# Reset plugins between scripts? This is useful if -# scripts assume a fresh state of CFS/Trick_CFS -# If set to false, plugins will not shutdown/re-initialize -# between scripts -reset_plugins_between_scripts = true - -# How long to wait between test scripts -delay_between_scripts = 2 - -# End test on fail? -end_test_on_fail = false - -# Disabled plugins (directory name of plugin). Comma-seperated -disabled_plugins = - -################################# -# CTF logging -################################# - -[logging] - -# Output directory for CTF scripts -results_output_dir = ./CTF_Results - -# Filename for CTF Log -ctf_log_file = CTF_Log_File.log - -# Generate a json version of the regression_results_summary (true) -# in addition to the text file version -json_results = True - -# What level of logging? -# ERROR : only show error logs - very minimal output -# INFO : only show info, warning, error, and critical logs -# DEBUG: show all logs! -log_level = DEBUG - -################################# -# ccsds options -################################# -[ccsds] -# Header Info Included in CCSDS Exports? -CCSDS_header_info_included = false - -#### -# This python module contains the definitions of header types to be exposed in the CCSDS plugin. -# These classes will be used by the CFS plugin to construct command and telemetry packets with -# the appropriate header formats. -#### -CCSDS_header_path = ./plugins/ccsds_plugin/cfe/ccsds_v2/ccsds_v2.py - -################################# -# Base settings for cfs -################################# - -[cfs] -# CFS workspace directory. All cfs paths will be relative to workspace dir. -####### -# -# Note - The CFS Workspace Directory is the CFS project root directory -# which the remaining configuration fields will utilize. -# -####### - -workspace_dir = ~/sample_cfs_workspace - -# cfs protocol setting either: -# local (local host) -# ssh (ssh to host) -# sp0 (sp0 host) -cfs_protocol = local - -# Build the CFS project? -build_cfs = false - -# Build directory for the CFS project -cfs_build_dir = ${cfs:workspace_dir} - -# Build command to run -cfs_build_cmd = make; make install - -# Run directory for the CFS project -cfs_run_dir = ${cfs:workspace_dir}/build/exe/lx1 - -# Executable to run within the cfs_run_dir -cfs_exe = core-lx1 - -# Include CFS UDP port in arg (-p portNum)? -cfs_port_arg = False - -# Additional arguments -cfs_run_args = - -# File name where CFS output will be saved -# The target name will be prepended to ensure unique files -cfs_output_file = cfs_stdout.txt - -# Automatically remove continuous checks once they fail -# Not doing so may flood the output with error messages -remove_continuous_on_fail = True - -# CCSDS Data Directory -#### -# This directory contains the command and telemetry definitions in JSON format. -# These definitions are used for CTF to command/receive telemetry, as well as -# used by the editor for auto-suggestion features. -#### - -CCSDS_data_dir = ${cfs:workspace_dir}/ccdd/json - -# Log CCSDS Import Process (Logs all messages parsed from CCSDS Data Directory) -log_ccsds_imports = true - -# Name of the target in CCSDS data files -CCSDS_target = set1 - -# What endianess is the target machine -endianess_of_target = little - -# Output directory for CFS EVS -evs_log_file = evs_event_msgs.log - -# Run in debug mode using GDB? -cfs_debug = false - -# Run in a seperate terminal window as CTF? -cfs_run_in_xterm = True - -#ip address of the target system -cfs_target_ip = 127.0.0.1 - -#CI commanding port -cmd_udp_port = 5010 - -# Set tlm_udp_port to 0 if you want the os to -# choose the port. -# If you want to manually set tlm_udp_port than set it equal to -# the port you want to use -tlm_udp_port = 5011 - -# Do you want to use TO or DIAG. Needs to be the exact name of the class to be used -# tlm_app_choice = DiagApi -tlm_app_choice = ToApi - -# What CCSDS version -ccsds_ver = 2 - -# How long to look back in the evs messages to validate -# Note - Setting this value to 0 means CheckEvent packet must -# be received *while* polling that instruction. -# Setting this value to X will allow CTF to validate -# events received with the past X time-units, as well -# as new events packets -evs_messages_clear_after_time = 5 - -# Name of MID to collect cfe EVS long messages -evs_event_mid_name = CFE_EVS_LONG_EVENT_MSG_MID - -# Name of MID to collect cfe EVS short messages -evs_short_event_mid_name = CFE_EVS_SHORT_EVENT_MSG_MID - -################################# -# -################################# -[ssh] - -# Command timeout for the execution plugin -command_timeout = 60 - -# Print stdout while command is running? -print_stdout = False - -# Log stdout when command complete? -log_stdout = True - -################################# -# Example settings for SP01 -################################# -[cfs_SP01] -# cfs protocol setting either: -# local (local host) -# ssh (ssh to host) -# sp0 (sp0 host) -cfs_protocol = sp0 - -# Build the CFS project? -build_cfs = false - -# Include CFS UDP port in arg (-p portNum)? -cfs_port_arg = False - -# Additional arguments -cfs_run_args = - -# File name where CFS output will be saved -# The target name will be prepended to ensure unique files -cfs_output_file = cfs_stdout.txt - -# Do you want to use TO or DIAG. Needs to be the exact name of the class to be used -# tlm_app_choice = DiagApi -tlm_app_choice = ToApi - -# Build command to run -cfs_build_cmd = make; make install - -# Run directory for the CFS project -cfs_run_dir = ${cfs:workspace_dir}/build/exe/vw1 - -# Executable to run within the cfs_run_dir -cfs_exe = core-vw1.exe - -# Execution path on the target -cfs_exe_path = /ram0 - -# The main function to start cfs -cfs_entry_point = CFE_PSP_Main - -# What endianess is the target machine -endianess_of_target = big - -# Output directory for CFS EVS -evs_log_file = evs_event_msgs.log - -# How long to look back in the evs messages to validate -# Note - Setting this value to 0 means CheckEvent packet must -# be received *while* polling that instruction. -# Setting this value to X will allow CTF to validate -# events received with the past X time-units, as well -# as new events packets -evs_messages_clear_after_time = 5 - -cfs_target_ip = 10.5.5.238 -cmd_udp_port = 5010 - -# Set tlm_udp_port to 0 if you want the os to -# choose the port. -# If you want to manually set tlm_udp_port than set it equal to -# the port you want to use -tlm_udp_port = 5011 - -# What CCSDS version -ccsds_ver = 2 - -# Name of MID to collect cfe EVS long messages -evs_event_mid_name = CFE_EVS_LONG_EVENT_MSG_MID - -# Name of MID to collect cfe EVS short messages -evs_short_event_mid_name = CFE_EVS_SHORT_EVENT_MSG_MID - - -# The SP0 will be rebooted before starting the test. -reboot = True - -# The time to delay before trying to send commands to CFS. -cfs_startup_time = 20 - -# Log the stdout text for each command sent to the SP0. -log_stdout = True diff --git a/configs/example_configs/example_config_sp01_sp02.ini b/configs/example_configs/example_config_sp01_sp02.ini deleted file mode 100644 index dc24dd8..0000000 --- a/configs/example_configs/example_config_sp01_sp02.ini +++ /dev/null @@ -1,391 +0,0 @@ -# CTF Core Configuration -################################# - -[core] - -# Global verification timeout for any "verify_required" CTF instructions -# Unit: Seconds if using generic system time manager) -ctf_verification_timeout = 5.0 - -# Global polling period for any "verify_required" CTF insturction. -# Defines how often to run verification commands until verification either -# passes or timeout -# Unit: Seconds if using generic system time manager) -ctf_verification_poll_period = 0.5 - - -# Reset plugins between scripts? This is useful if -# scripts assume a fresh state of CFS/Trick_CFS -# If set to false, plugins will not shutdown/re-initialize -# between scripts -reset_plugins_between_scripts = true - -# How long to wait between test scripts -delay_between_scripts = 2 - -# End test on fail? -end_test_on_fail = false - -# Disabled plugins (directory name of plugin). Comma-seperated -disabled_plugins = - - -################################# -# CTF logging -################################# - -[logging] - -# Output directory for CTF scripts -results_output_dir = ./CTF_Results - -# Filename for CTF Log -ctf_log_file = CTF_Log_File.log - -# Generate a json version of the regression_results_summary (true) -# in addition to the text file version -json_results = True - -# What level of logging? -# ERROR : only show error logs - very minimal output -# INFO : only show info, warning, error, and critical logs -# DEBUG: show all logs! -log_level = DEBUG - - -################################# -# Setting for ccsds options -################################# -[ccsds] -# Header Info Included in CCSDS Exports? -CCSDS_header_info_included = false - -#### -# This python module contains the definitions of header types to be exposed in the CCSDS plugin. -# These classes will be used by the CFS plugin to construct command and telemetry packets with -# the appropriate header formats. -#### -CCSDS_header_path = ${cfs:workspace_dir}/tools/ctf/plugins/ccsds_plugin/gw/ccsds_gw.py - -################################# -# Base settings for cfs -################################# -[cfs] -# CFS workspace directory. All cfs paths will be relative to workspace dir. -####### -# -# Note - The CFS Workspace Directory is the CFS project root directory -# which the remaining configuration fields will utilize. -# -####### - -workspace_dir = ~/sample_cfs_workspace - -# cfs protocol setting either: -# local (local host) -# ssh (ssh to host) -# sp0 (sp0 host) -cfs_protocol = local - -# Build the CFS project? -build_cfs = false - -# Build directory for the CFS project -cfs_build_dir = ${cfs:workspace_dir} - -# Build command to run -cfs_build_cmd = make; make install - -# Run directory for the CFS project -cfs_run_dir = ${cfs:workspace_dir}/build/exe/lx1 - -# Executable to run within the cfs_run_dir -cfs_exe = core-lx1 - -# Include CFS UDP port in arg (-p portNum)? -cfs_port_arg = False - -# Additional arguments -cfs_run_args = - -# File name where CFS output will be saved -# The target name will be prepended to ensure unique files -cfs_output_file = cfs_stdout.txt - -# Automatically remove continuous checks once they fail -# Not doing so may flood the output with error messages -remove_continuous_on_fail = True - -# CCSDS Data Directory -#### -# This directory contains the command and telemetry definitions in JSON format. -# These definitions are used for CTF to command/receive telemetry, as well as -# used by the editor for auto-suggestion features. -#### - -CCSDS_data_dir = ${cfs:workspace_dir}/ccdd/json - -# Log CCSDS Import Process (Logs all messages parsed from CCSDS Data Directory) -log_ccsds_imports = true - -# Name of the target in CCSDS data files -CCSDS_target = set1 - -# What endianess is the target machine -endianess_of_target = little - -# Output directory for CFS EVS -evs_log_file = evs_event_msgs.log - -# Run in debug mode using GDB? -cfs_debug = false - -# Run in a seperate terminal window as CTF? -cfs_run_in_xterm = True - -#ip address of the target system -cfs_target_ip = 127.0.0.1 - -#CI commanding port -cmd_udp_port = 5010 - -# Set tlm_udp_port to 0 if you want the os to -# choose the port. -# If you want to manually set tlm_udp_port than set it equal to -# the port you want to use -tlm_udp_port = 5011 - -# Do you want to use TO or DIAG. Needs to be the exact name of the class to be used -# tlm_app_choice = DiagApi -tlm_app_choice = ToApi - -# What CCSDS version -ccsds_ver = 2 - -# How long to look back in the evs messages to validate -# Note - Setting this value to 0 means CheckEvent packet must -# be received *while* polling that instruction. -# Setting this value to X will allow CTF to validate -# events received with the past X time-units, as well -# as new events packets -evs_messages_clear_after_time = 5 - -# Name of MID to collect cfe EVS long messages -evs_event_mid_name = CFE_EVS_LONG_EVENT_MSG_MID - -# Name of MID to collect cfe EVS short messages -evs_short_event_mid_name = CFE_EVS_SHORT_EVENT_MSG_MID - -################################# -# Example settings for SP01 -################################# -[cfs_SP01] -# cfs protocol setting either: -# local (local host) -# ssh (ssh to host) -# sp0 (sp0 host) -cfs_protocol = sp0 - -# Build the CFS project? -build_cfs = false - -# Include CFS UDP port in arg (-p portNum)? -cfs_port_arg = False - -# Additional arguments -cfs_run_args = - -# File name where CFS output will be saved -# The target name will be prepended to ensure unique files -cfs_output_file = cfs_stdout.txt - -# Do you want to use TO or DIAG. Needs to be the exact name of the class to be used -# tlm_app_choice = DiagApi -tlm_app_choice = ToApi - -# CCSDS Data Directory -#### -# This directory contains the command and telemetry definitions in JSON format. -# These definitions are used for CTF to command/receive telemetry, as well as -# used by the editor for auto-suggestion features. -#### - -CCSDS_data_dir = ${cfs:workspace_dir}/ccdd/json - -# Log CCSDS Import Process (Logs all messages parsed from CCSDS Data Directory) -log_ccsds_imports = true - -# Build directory for the CFS project -cfs_build_dir = ${cfs:workspace_dir} - -# Build command to run -cfs_build_cmd = make; make install - -# Run directory for the CFS project -cfs_run_dir = ${cfs:workspace_dir}/build/exe/vw1 - -# Executable to run within the cfs_run_dir -cfs_exe = core-vw1.exe - -# Execution path on the target -cfs_exe_path = /ram0 - -# The main function to start cfs -cfs_entry_point = CFE_PSP_Main - -# What endianess is the target machine -endianess_of_target = big - -# Output directory for CFS EVS -evs_log_file = evs_event_msgs.log - -# How long to look back in the evs messages to validate -# Note - Setting this value to 0 means CheckEvent packet must -# be received *while* polling that instruction. -# Setting this value to X will allow CTF to validate -# events received with the past X time-units, as well -# as new events packets -evs_messages_clear_after_time = 5 - -cfs_target_ip = 10.5.5.238 -cmd_udp_port = 5010 - -# Set tlm_udp_port to 0 if you want the os to -# choose the port. -# If you want to manually set tlm_udp_port than set it equal to -# the port you want to use -tlm_udp_port = 5011 - -# What CCSDS version -ccsds_ver = 2 - -# Name of MID to collect cfe EVS long messages -evs_event_mid_name = CFE_EVS_LONG_EVENT_MSG_MID - -# Name of MID to collect cfe EVS short messages -evs_short_event_mid_name = CFE_EVS_SHORT_EVENT_MSG_MID - - -# The SP0 will be rebooted before starting the test. -reboot = True - -# The time to delay before trying to send commands to CFS. -cfs_startup_time = 20 - -# Log the stdout text for each command sent to the SP0. -log_stdout = True - -################################# -# Example setting for SP02 -################################# -[cfs_SP02] -# cfs protocol setting either: -# local (local host) -# ssh (ssh to host) -# sp0 (sp0 host) -cfs_protocol = sp0 - -# Build the CFS project? -build_cfs = false - -# Include CFS UDP port in arg (-p portNum)? -cfs_port_arg = False - -# Additional arguments -cfs_run_args = - -# File name where CFS output will be saved -# The target name will be prepended to ensure unique files -cfs_output_file = cfs_stdout.txt - -# Do you want to use TO or DIAG. Needs to be the exact name of the class to be used -# tlm_app_choice = DiagApi -tlm_app_choice = ToApi - -# CCSDS Data Directory -#### -# This directory contains the command and telemetry definitions in JSON format. -# These definitions are used for CTF to command/receive telemetry, as well as -# used by the editor for auto-suggestion features. -#### - -CCSDS_data_dir = ${cfs:workspace_dir}/ccdd/json - -# Log CCSDS Import Process (Logs all messages parsed from CCSDS Data Directory) -log_ccsds_imports = true - -# Build directory for the CFS project -cfs_build_dir = ${cfs:workspace_dir} - -# Build command to run -cfs_build_cmd = make; make install - -# Run directory for the CFS project -cfs_run_dir = ${cfs:workspace_dir}/build/exe/vw1 - -# Executable to run within the cfs_run_dir -cfs_exe = core-vw1.exe - -# Execution path on the target -cfs_exe_path = /ram0 - -# The main function to start cfs -cfs_entry_point = CFE_PSP_Main - -# What endianess is the target machine -endianess_of_target = big - -# Output directory for CFS EVS -evs_log_file = evs_event_msgs.log - -# How long to look back in the evs messages to validate -# Note - Setting this value to 0 means CheckEvent packet must -# be received *while* polling that instruction. -# Setting this value to X will allow CTF to validate -# events received with the past X time-units, as well -# as new events packets -evs_messages_clear_after_time = 5 - -# ip address of the target system -cfs_target_ip = 10.5.5.239 -cmd_udp_port = 5040 - -# Set tlm_udp_port to 0 if you want the os to -# choose the port. -# If you want to manually set tlm_udp_port than set it equal to -# the port you want to use -tlm_udp_port = 5041 - -# What CCSDS version -ccsds_ver = 2 - -# Name of MID to collect cfe EVS long messages -evs_event_mid_name = CFE_EVS_LONG_EVENT_MSG_MID - -# Name of MID to collect cfe EVS short messages -evs_short_event_mid_name = CFE_EVS_SHORT_EVENT_MSG_MID - - -# The SP0 will be rebooted before starting the test. -reboot = True - -# The time to delay before trying to send commands to CFS. -cfs_startup_time = 20 - -# Log the stdout text for each command sent to the SP0. -log_stdout = True - -################################# -# -################################# -[ssh] - -# Command timeout for the execution plugin -command_timeout = 60 - -# Print stdout while command is running? -print_stdout = False - -# Log stdout when command complete? -log_stdout = True - diff --git a/ctf b/ctf index 870c8c0..c7d340d 100755 --- a/ctf +++ b/ctf @@ -33,8 +33,8 @@ from lib.ctf_utility import expand_path from lib.logger import logger as log, set_logger_options_from_config, change_log_file from lib.exceptions import CtfTestError -CTF_VERSION = "v1.3" -CTF_RELEASE_DATE = "Aug 26 2021" +CTF_VERSION = "v1.3.1" +CTF_RELEASE_DATE = "Oct 19 2021" def main(): diff --git a/docs/ctf/ctf_json_test_script_guide.md b/docs/ctf/ctf_json_test_script_guide.md index 3eab6e4..022a967 100644 --- a/docs/ctf/ctf_json_test_script_guide.md +++ b/docs/ctf/ctf_json_test_script_guide.md @@ -5,7 +5,7 @@ Table of Contents * [Test Scripts](#test-scripts) * [Functions](#functions) * [Tests](#tests) - * [CTF Commands](#ctf-commands) + * [CTF Instructions](#ctf-instructions) * [Complete Example](#complete-example) @@ -15,7 +15,7 @@ you can find a great introduction here: https://www.w3schools.com/js/js_json_syn ## Test Scripts The JSON CTF input file, or **Test Script**, is comprised of a number of properties that describe the tests that will be executed. Each **Test Script** can have multiple tests, with each each testing multiple requirements. -Example test scripts can be found at `examples/aspect_project/scripts` in the repo. +Example test scripts can be found at `scripts` folder in the repo. The **Test Script** has the following properties: - **test_number**: Unique identifier for the test script. Format Should be a descriptive test ID that would allow the review to understand what is being tested. @@ -36,7 +36,7 @@ Example: }, "description": "Basic CTF Example Script Showing Simple Commands/Telemetry Verification", "owner": "CTF", - "test_setup": "Script will start ASPECT, execute a verification command test and close ASPECT", + "test_setup": "Script will start ASPECT, execute a verification instruction test and close ASPECT", "ctf_options": { "verify_timeout": 4 }, @@ -44,15 +44,15 @@ Example: "tests": [ { "case_number": "ASPECT-Plugin-Test-001", - "description": "Start CFS, Send TO NOOP command", - "commands": [ + "description": "Start CFS, Send TO NOOP instruction", + "instructions": [ { - "command": "StartCfs", + "instruction": "StartCfs", "wait": 1, "data": {} }, { - "command": "CheckTlmValue", + "instruction": "CheckTlmValue", "data": { "mid": "TO_HK_TLM_MID", "args": [ @@ -68,7 +68,7 @@ Example: "wait": 1 }, { - "command": "SendCfsCommand", + "instruction": "SendCfsCommand", "data": { "mid": "TO_CMD_MID", "cc": "TO_NOOP_CC", @@ -80,7 +80,7 @@ Example: "wait": 1 }, { - "command": "CheckTlmValue", + "instruction": "CheckTlmValue", "data": { "mid": "TO_HK_TLM_MID", "args": [ @@ -96,7 +96,7 @@ Example: "wait": 1 }, { - "command": "ShutdownCfs", + "instruction": "ShutdownCfs", "data": {}, "wait": 1 } @@ -107,7 +107,7 @@ Example: ``` ## Functions -The **Function** objects are collections of **CTF Commands** that can be executed by each test case. These functions can be defined within the same **Test Script** or in a separate file and be imported (see `examples/aspect_projects/script/utils/` for an example. The object has the following properties: +The **Function** objects are collections of **CTF Instructions** that can be executed by each test case. These functions can be defined within the same **Test Script** or in a separate file and be imported (see `scripts/cfe_6_7_tests/libs/`). The object has the following properties: - **description**: Describe purpose of the function including the steps the function with take and the results the function will provide. Intended use details should also be included. - **varlist**: List of arguments required by the function @@ -117,8 +117,8 @@ Example: "UserDefinedFunction":{ "description":"", "varlist":["_arg1", "_arg2"], - "commands":[ - //CTF Command objects go here + "instructions":[ + //CTF Instruction objects go here ] } } @@ -126,33 +126,33 @@ Example: ## Tests The **Test** objects each describe a single test, which can verify any number of requirements. -Each **Test** contains the requirements verified by the test, a description of the test, and a number of **Command** +Each **Test** contains the requirements verified by the test, a description of the test, and a number of **Instructions** objects that define the actions to take during the test. The object has the following properties: - **case_number**: Unique identifier of test case in the script. Use Test name with a unique number. -- **description**: Describes what actions this test is performing. Include Preconditions, the expected actions/commands, and the expected results that are going to be observed. -- **commands**: An array of **Command** objects that define the actions taken by the test (described below) +- **description**: Describes what actions this test is performing. Include Preconditions, the expected actions/instructions, and the expected results that are going to be observed. +- **instructions**: An array of **Instruction** objects that define the actions taken by the test (described below) Example: ```javascript { "case_number":"ExampleTest-1", "description":"", - "commands":[ - //Command Objects go here + "instructions":[ + //Instruction Objects go here ] } ``` -## CTF Commands -The **CTF Command** object defines what action CTF is supposed to take. Each command object has the following base structure: -- **command:**: The **CTF command** to execute. Must be one of the values listed above (string) -- **data:** The arguments for the **CTF Command**. Each **Command** is going to have a different data object, described below. -- **wait:** The amount of time to wait from the completion of the last command before executing the current command. -- **timeout:** (Optional) The amount of time for verification commands to wait for verification to succeed before timing out. +## CTF Instructions +The **CTF Instructions** object defines what action CTF is supposed to take. Each instruction object has the following base structure: +- **instruction:**: The **CTF instruction** to execute. Must be one of the values listed above (string) +- **data:** The arguments for the **CTF Instruction**. Each **Instruction** is going to have a different data object, described below. +- **wait:** The amount of time to wait from the completion of the last instruction before executing the current instruction. +- **timeout:** (Optional) The amount of time for verification instructions to wait for verification to succeed before timing out. -Commands are implemented in their respective CTF plugin. Refer to the specific plugin's README document for more -information on commands supported by each plugin. +Instructions are implemented in their respective CTF plugin. Refer to the specific plugin's README document for more +information on instructions supported by each plugin. ## Complete Example The following example shows the basic CTF test script found in the repo. @@ -166,7 +166,7 @@ The following example shows the basic CTF test script found in the repo. }, "description": "Basic CTF Example Script Showing Simple Commands/Telemetry Verification", "owner": "CTF", - "test_setup": "Script will start CFE-6-7, execute a verification command test and close CFE-6-7", + "test_setup": "Script will start CFE-6-7, execute a verification instruction test and close CFE-6-7", "ctf_options": { "verify_timeout": 4 }, @@ -174,17 +174,17 @@ The following example shows the basic CTF test script found in the repo. "tests": [ { "case_number": "CFE-6-7-Plugin-Test-001", - "description": "Start CFS, Send TO NOOP command", - "commands": [ + "description": "Start CFS, Send TO NOOP instruction", + "instructions": [ { - "command": "StartCfs", + "instruction": "StartCfs", "data": { "name": "" }, "wait": 1 }, { - "command": "EnableCfsOutput", + "instruction": "EnableCfsOutput", "data": { "name": "" }, @@ -193,7 +193,7 @@ The following example shows the basic CTF test script found in the repo. "comment": "Need this to enable the telemetry thread. Enable Output counts as 1 TO cmd" }, { - "command": "CheckEvent", + "instruction": "CheckEvent", "data": { "app": "TO", "id": "3", @@ -206,7 +206,7 @@ The following example shows the basic CTF test script found in the repo. "comment": "Need this to enable the telemetry thread. Enable Output counts as 1 TO cmd" }, { - "command": "SendCfsCommand", + "instruction": "SendCfsCommand", "data": { "name": "", "mid": "TO_CMD_MID", @@ -216,7 +216,7 @@ The following example shows the basic CTF test script found in the repo. "wait": 1 }, { - "command": "CheckTlmContinuous", + "instruction": "CheckTlmContinuous", "comment": "Change 'name' to 'cfs_name' for all instructions", "data": { "name": "", @@ -235,7 +235,7 @@ The following example shows the basic CTF test script found in the repo. "wait": 1 }, { - "command": "CheckTlmValue", + "instruction": "CheckTlmValue", "data": { "name": "", "mid": "TO_HK_TLM_MID", @@ -252,7 +252,7 @@ The following example shows the basic CTF test script found in the repo. "wait": 1 }, { - "command": "SendCfsCommand", + "instruction": "SendCfsCommand", "data": { "name": "", "mid": "TO_CMD_MID", @@ -262,7 +262,7 @@ The following example shows the basic CTF test script found in the repo. "wait": 1 }, { - "command": "CheckTlmValue", + "instruction": "CheckTlmValue", "data": { "name": "", "mid": "TO_HK_TLM_MID", @@ -279,7 +279,7 @@ The following example shows the basic CTF test script found in the repo. "wait": 1 }, { - "command": "RemoveCheckTlmContinuous", + "instruction": "RemoveCheckTlmContinuous", "data": { "name": "", "verification_id": "no_error" diff --git a/docs/ctf/ctf_plugin_guide.md b/docs/ctf/ctf_plugin_guide.md index ca7a505..6053f55 100644 --- a/docs/ctf/ctf_plugin_guide.md +++ b/docs/ctf/ctf_plugin_guide.md @@ -1,13 +1,13 @@ # Introduction -CTF Plugins allow developers to extend CTF with new test commands. Each new test command is mapped to its respective implementation in the plugin. +CTF Plugins allow developers to extend CTF with new test instructions. Each new test instruction is mapped to its respective implementation in the plugin. -# Types of test commands +# Types of test instructions -There are two types of supported test commands -* Non-verification Commands: Commands that do not require verification at a later time and can be validated right away. Examples include sending data such as sending a CFS Command. +There are two types of supported test instructions +* Non-verification Instructions: Instructions that do not require verification at a later time and can be validated right away. Examples include sending data such as sending a CFS Command. -* Verification Commands: Commands that require verification (via polling) until the verification is satisfied, or a timeout is reached. Examples including checking that a piece of telemetry changes at some point in the test. The implementation of those commands must be non-blocking in order to return control to CTF between the verification polls. +* Verification Instructions: Instructions that require verification (via polling) until the verification is satisfied, or a timeout is reached. Examples including checking that a piece of telemetry changes at some point in the test. The implementation of those instructions must be non-blocking in order to return control to CTF between the verification polls. # Plugin Creation To create a new plugin, create a directory with the `plugin-name` under `plugins/`. For example we can create `example_plugin` at the directory `ctf/plugins/example_plugin`. @@ -18,8 +18,8 @@ In the `example_plugin` directory, we create a new .py file `example_plugin.py`. * `name` - plugin name * `description` - plugin description -* `command_map` - python dict object mapping between the command string from the test script, to a python function for the implementation of that command. -* `verify_required_commands` - python list containing list of command strings that require verification. +* `command_map` - python dict object mapping between the instruction string from the test script, to a python function for the implementation of that instruction. +* `verify_required_commands` - python list containing list of instruction strings that require verification. Our `example_plugin.py` will look as follows @@ -47,7 +47,7 @@ class ExamplePlugin(Plugin): # Command implementation goes here... - # Return status of that command + # Return status of that instruction status = True return status @@ -72,7 +72,7 @@ class ExamplePlugin(Plugin): print("Optional shutdown/cleanup implementation for plugin") ``` -With the above plugin definition, the following snippet of a JSON test script shows how the commands are used. +With the above plugin definition, the following snippet of a JSON test script shows how the instructions are used. ```JSON { @@ -83,7 +83,7 @@ With the above plugin definition, the following snippet of a JSON test script sh }, "description": "Testing Example Plugin", "owner": "CTF", - "test_setup": "Script will execute a single command, and a single verification", + "test_setup": "Script will execute a single instruction, and a single verification", "ctf_options": { "verif_timeout": 2.0 }, @@ -91,9 +91,9 @@ With the above plugin definition, the following snippet of a JSON test script sh { "case_number": "Example Plugin Test", "description": "No description", - "commands": [ + "instructions": [ { - "command": "TestCommand", + "instructions": "TestCommand", "data": { "arg1": "foo", "arg2": 42 @@ -101,7 +101,7 @@ With the above plugin definition, the following snippet of a JSON test script sh "wait": 1 }, { - "command": "TestVerifyCommand", + "instructions": "TestVerifyCommand", "data": {}, "wait": 1 } @@ -112,7 +112,7 @@ With the above plugin definition, the following snippet of a JSON test script sh ``` -Executing the script produces the following output showing the test command successfully processed, and the test verification command executed 6 times before it the verification was satisfied. +Executing the script produces the following output showing the test instruction successfully processed, and the test verification instruction executed 6 times before it the verification was satisfied. ``` [14:13:33.882] test_script (85 ) *** INFO: Input file utilized : test_ctf_example_plugin.json diff --git a/docs/ctf/faq.md b/docs/ctf/faq.md new file mode 100644 index 0000000..27073db --- /dev/null +++ b/docs/ctf/faq.md @@ -0,0 +1,161 @@ +# Troubleshooting & FAQ + +[[_TOC_]] + +## What platforms does CTF support? + +CTF is developed and tested on CentOS 7 Linux. It may work with other Linux distributions or Docker containers, but is not officially supported. + +--- +## What version of CTF am I running? + +Check the first line in the terminal output, or of `CTF_Log_File.log` in your log output directory, for `*** INFO: cFS Test Framework (vX.X) Starting...` where X.X is the version number. For the latest version, see https://github.com/nasa/CTF/releases + +--- +## I see an error "pidof: command not found" in CTF output, and/or cFS does not shut down +The `pidof` command is used to stop cFS processes on Linux. You may need to install the `sysvinit-tools` package in your environment to enable `pidof`. + +--- +## I'm having trouble with a config (.ini) file. What properties should I look out for? +## How do I configure a cFS target? + +*TBD: complete this section* + +Configuration of the CFS Plugin is based around targets and registration. A target is a cFS instance represented by a corresponding section of the config file, and referred to by name in the test script. Registration reads a section, associates it with the name, and initializes a connection to the target. +Registration may be done explicitly via the `RegisterCfs` instruction with a target name; automatically if `BuildCfs` or `StartCfs` are used first, or if `RegisterCfs` is used without a target name. Automatic registration takes only those sections beginning with `cfs_`, or the `cfs` section only if no others are found. + +cFS targets are defined entirely within the config file by their respectively named sections. The test script only refers to targets by name, which must match a section in the config file. +The section `cfs` provides default values for other targets, and should be included even if you are not explicitly registering it. This section is only used to register a target if no other sections beginning with `cfs_` are found. Other target names do not necessarily have to begin with `cfs_`, but such targets must be registered explicitly using `RegisterCfs`. +`cfs` is not required to include all properties if it is not registered directly, but may provide values that are common to all targets. Other targets may then only define properties that differ. Any properties missing from a target section will be taken from the `cfs` section. If it is not defined in either section, an error will be reported and registration will fail. During automatic registration, + +There are several properties in `cfs` that you will likely need to customize for each target and/or project: + +`cfs_protocol` +Must be one of "local", "ssh", or "sp0". Determines the type of connection made, and which additional config properties are needed. SSH targets require `destination`. SP0 targets require `reboot`, `cfs_exe_path`, `cfs_entry_point`, `cfs_startup_time`, `log_stdout`, and `stop_command`. + +`workspace_dir` +Not read by CTF directly, but is provided in the example configs as a convenient way to map paths relative to the root working directory. You may use `${cfs:workspace_dir}` (or another key) to append relative paths such as `cfs_build_dir` or `CCSDS_data_dir`. If you're seeing errors such as file not found or unexpected paths on startup, check `workspace_dir`. + +`cfs_run_in_xterm` +Only applies to targets using the "local" protocol. If true, and xterm is found, each target's stdout will be displayed in a separate xterm window that opens when cFS starts and closes when it stops. Otherwise, all cFS output will be printed inline on the console. It is recommended to set this to false for tests run in CI, via a script, or otherwise headless. + +Other properties that may be commonly changed, or could cause unexpected behavior: + +`core:reset_plugins_between_scripts` + +`logging:log_level` and `cfs:log_ccsds_imports` + +`ccsds:CCSDS_header_path` +This is a path to the file containing the CCSDS header definitions to be used by CTF. This must be a Python file that conforms to the CCSDS packet interface found in `ctf/plugins/ccsds_plugin/ccsds_packet_interface.py` + + +--- +## I have unexpected or missing telemetry, or EnableCfsOutput is failing. + +An instance of flight software may still be running in the background and holding onto resources. Run `ps -aux | grep core` (or the name of your cFS exe) before and/or after a test run to check for any persisting processes. Rebooting the host computer may also help, if some other process is using a required port. + +Also, check the `cmd_udp_port` and `tlm_udp_port` in the `[cfs]` or target section of your config file. The command port will be determined by cFS and is where CTF sends commands; the telemetry port is determined by CTF and is where it informs cFS to send telemetry by the enable output command. + +--- +## CheckEvent or CheckTlmValue is failing, but I see the events in CTF logs. + +Your checks may be happening too early or too late. For event checks, CTF will only check events that arrive between the start of the check and the verification timeout. Compare the execution times in `cfs_tlm_msgs.log` with the timestamps of the checks in the CTF test output file. (Note that execution time is the offset in seconds from the start of the test.) We recommend using a "wait" of 0 for all checks, unless you know that the packet will be delayed: +``` + + { + "instruction": "EnableCfsOutput", + "data": {}, + "wait": 2, + }, + { + "instruction": "CheckEvent", + "data": { + "app": "TO", + "id": "3", + "msg": "TO - ENABLE_OUTPUT cmd succesful for routeMask:0x00000001" + }, + "wait": 0 + }, +``` + +--- +## My first telemetry check for a given packet passes, but others fail. + +Checking any value in a packet clears that packet from memory, to avoid checking stale data again later in the test. If you need to check multiple values from the same packet, combine the conditions in one instruction using a list: +``` +"args": [ + { + "compare": "==", + "variable": "cmdCount", + "value": 1 + }, + { + "compare": "==", + "variable": "errCount", + "value": 0 + }, +] +``` + +--- +## How do I run multiple test scripts at once? + +When executing CTF, you may provide any number of directory and/or file paths to be executed in order. For directories, all .json files will be executed in indeterminate order. + +Example: `./ctf --config_file=./configs/default_config.ini test_scripts script1.json script2.json script3.json` +This will execute all .json files in test_scripts, followed by script1, script2, and finally script3. + +If the tests are designed to be executed in sequence, you may wish to set `reset_plugins_between_scripts` to False in your config file to allow the plugin state, and potentially the flight software process(es), to be preserved across all test scripts - similarly to multiple test cases in one script. This allows you to avoid registering and starting cFS targets in each test script. Note that when doing so, cFS will not be stopped until the end of the final test, unless `ShutdownCfs` is used. + +--- +## When should I use RegisterCfs, or use a target name vs an empty string? +## Why do some of the example tests not specify a cFS target name? +## How do I write test scripts so that they can be reused with different configurations? + +Using `RegisterCfs` to declare your cFS target(s) is optional. The CFS Plugin will automatically register and use CFS targets as defined in the config file if no target name is explicitly provided. Automatic registration occurs at the first occurrence of `BuildCfs` or `StartCfs` if no target has yet been registered. During automatic registration, any section in the config file beginning with `cfs_` will be treated as a CFS target, such as `[cfs_lx1]`. The required section `[cfs]` will be used to register a default target if no others are provided. When explicitly registering a target with RegisterCfs, the name must match a section in the config file, but it is not required to begin with `cfs_`. In other words, `[cfs_lx1]` may be registered automatically or with `RegisterCfs` but `[lx1]` can only be registered with `RegisterCfs`. + +Likewise in your test script, you may or may not provide a target name for any CFS Plugin instruction. If a target name is provided, it must match a section name in the config file, whether it was registered explicitly or automatically. If no target is provided (an empty string `""`, or omitting `"target"` altogether), the instruction will apply to **any and all** registered targets sequentially. Thus omission of target names allows a test script to be trivially reused with different config files, but is most practical when configured with a single target. + +These features allow you to mix and match test scripts with different configurations by relying on automatic registration of targets. If the test script does not name or register any targets, its instructions will apply to any one or more targets as defined in the config file. Alternatively, you may use `RegisterCfs` to specify only those targets from your config file that you want to register for the test, even if there are others defined. + +--- +## I'm getting unusual Python errors relating to my instruction arguments. +##When do I use certain types and comparison operators in instruction arguments? + +Instructions that evaluate expressions, like `CheckTlmValue`, support different operators for comparing values. Standard logical operators like ==, !=, <=, etc are allowed for numeric comparisons. When comparing strings, use `streq` for equality or `strneq` for inequality. You cannot use string comparison for numbers or vice versa. + +*TBD: correct and incorrect example of comparison operators* + +Types used in JSON should match the corresponding data as closely as possible. Use whole numbers for integers, decimals for floats or doubles, and strings for strings or characters. Do not put a numeric value in quotes unless it is to be handled as a string. + +*TBD: correct and incorrect example of type matching* + +For complex types, such as "args" for `SendCfsCommand`, use nested JSON objects with key-value pairs to represent the structure. Within an object, you may use an ordered list for values that are arrays (ie, those having a nonzero `array_size` attribute in their data definition). Do not use lists to represent multiple fields in an object. + +*TBD: correct and incorrect example of nested args* + +To specify a single value in an array, you may use the indexed name of the element (eg `"myArray[3]" = 1`). You may also provide a single value to an array name to fill the array with that value (eg `"myArray" = 1` sets ALL elements in `myArray` to 1). Both methods can be used in combination. + +*TBD: example of default and indexed values for an array* + +For examples of correct usage of these and other instruction arguments, see the test scripts in `scripts\example_tests`. + +--- +## Can I check for a partial string match on an event? +## How do I use macros, regular expressions, or string arguments? + +*TBD: complete this section* + +*TBD: example of macro usage* +*TBD: example of event msg_args* +*TBD: example of event regex match* + +--- +## How do I get color-coded log output? + +Simply install the Python package `colorlog`: `(pip install colorlog)`. The package is not installed by default since many users run CTF via the editor or scripts and do not inspect the output directly at runtime. Colors only apply to the command line output. + +--- +## I am seeing some other odd behavior not described here. + +Make sure you are using the latest CTF and flight software. Pull from the git repo(s), check out the latest release branch/tag, reinstall your Anaconda environment, and rebuild everything. Try running CTF directly without the editor or intermediate scripts. If the problem persists, contact a developer or open an issue at https://github.com/nasa/CTF/issues diff --git a/lib/patchwork/.coveragerc b/lib/patchwork/.coveragerc new file mode 100644 index 0000000..6a3ed99 --- /dev/null +++ b/lib/patchwork/.coveragerc @@ -0,0 +1,3 @@ +[run] +branch = True +include = patchwork/* diff --git a/lib/patchwork/.gitignore b/lib/patchwork/.gitignore new file mode 100644 index 0000000..b7e1ca4 --- /dev/null +++ b/lib/patchwork/.gitignore @@ -0,0 +1,3 @@ +docs/_build +.cache +.coverage diff --git a/lib/patchwork/.travis.yml b/lib/patchwork/.travis.yml new file mode 100644 index 0000000..324732d --- /dev/null +++ b/lib/patchwork/.travis.yml @@ -0,0 +1,29 @@ +sudo: false +language: python +python: + - "2.7" + - "3.4" + - "3.5" + - "3.6" + - "pypy" + - "pypy3" +matrix: + # pypy3 (as of 2.4.0) has a wacky arity issue in its source loader. Allow it + # to fail until we can test on, and require, PyPy3.3+. See invoke#358. + allow_failures: + - python: pypy3 + # Disabled per https://github.com/travis-ci/travis-ci/issues/1696 + # fast_finish: true +install: + - pip install -r dev-requirements.txt +script: + # Run tests w/ coverage first, so it uses the local-installed copy. + # (If we do this after the below installation tests, coverage will think + # nothing got covered!) + - inv coverage --report=xml + # TODO: tighten up these install test tasks so they can be one-shotted + - inv travis.test-installation --package=patchwork --sanity="inv sanity" + - inv travis.test-packaging --package=patchwork --sanity="inv sanity" + - inv docs --nitpick + - flake8 +# TODO: after_success -> codecov, once coverage sucks less XD diff --git a/lib/readers/json_script_reader.py b/lib/readers/json_script_reader.py index 35f3edf..8935aff 100644 --- a/lib/readers/json_script_reader.py +++ b/lib/readers/json_script_reader.py @@ -209,14 +209,14 @@ def process_tests(self): else: function_call_delay = 1.0 - disabled = bool(command.get('disabled', False)) + function_disabled = bool(command.get('disabled', False)) for c_index, c_inline in enumerate(inline_commands): delay = 0 if "wait" in c_inline.keys(): delay = c_inline["wait"] if c_index == 0: delay += function_call_delay - disabled |= bool(c_inline.get('disabled', False)) + disabled = function_disabled or bool(c_inline.get('disabled', False)) instruction_list.append(Instruction(delay, c_inline, len(test_list), default_index, disabled)) else: delay = 0 diff --git a/lib/test.py b/lib/test.py index 42bb5f5..0afcf1d 100644 --- a/lib/test.py +++ b/lib/test.py @@ -223,7 +223,7 @@ def run_commands(self): details = "Instruction is disabled. Skipping..." self.status_manager.update_command_status(status, details, index=self.current_instruction_index) self.status_manager.end_command() - log.info("Skipping disabled test instruction {} ".format(instruction)) + log.info("Skipping disabled test instruction {} ".format(i.command)) self.num_skipped += 1 self.current_instruction_index += 1 continue diff --git a/plugins/ccsds_plugin/readers/ccdd_export_reader.py b/plugins/ccsds_plugin/readers/ccdd_export_reader.py index 8491b43..8362766 100644 --- a/plugins/ccsds_plugin/readers/ccdd_export_reader.py +++ b/plugins/ccsds_plugin/readers/ccdd_export_reader.py @@ -307,6 +307,13 @@ def _create_parameterized_type(self, type_dict, type_id=None, arg_id=None, subty data_type = None log.error("Failed to create type class {}!".format(data_type_name)) + primary_type_name = ('int8', 'int16', 'int32', 'int64','uint8', 'uint16', 'uint32', 'uint64','float', + 'double', 'char', 'string', 'bool', 'boolean', 'address', 'cpuaddr') + if data_type_name in primary_type_name and data_type_name in self.type_dict: + log.error("Override primary data type {}, it may be a potential issue".format(data_type_name)) + elif data_type_name in self.type_dict: + log.warning("Data type {} is already created, it will be overridden".format(data_type_name)) + self.type_dict[data_type_name] = data_type parameterized_type = data_type, type_enums return parameterized_type diff --git a/plugins/cfs/cfs_config.py b/plugins/cfs/cfs_config.py index dfffb55..bcfd124 100644 --- a/plugins/cfs/cfs_config.py +++ b/plugins/cfs/cfs_config.py @@ -289,6 +289,7 @@ def __init__(self, name): self.cfs_entry_point = None self.cfs_startup_time = None self.log_stdout = None + self.stop_command = None super().__init__(name) # Overrides @@ -314,3 +315,4 @@ def load_config_data(self, section_name): self.validation.validate_number) self.log_stdout = self.load_field(self.name, "log_stdout", Global.config.getboolean, self.validation.validate_boolean) + self.stop_command = self.load_field(self.name, "stop_command", Global.config.get) diff --git a/plugins/cfs/pycfs/cfs_controllers.py b/plugins/cfs/pycfs/cfs_controllers.py index 3002f7d..c002001 100644 --- a/plugins/cfs/pycfs/cfs_controllers.py +++ b/plugins/cfs/pycfs/cfs_controllers.py @@ -32,6 +32,7 @@ import re import shutil import time +from subprocess import run, PIPE, STDOUT import traceback from ast import literal_eval from pathlib import Path @@ -305,12 +306,12 @@ def resolve_macros(self, arg): search macro_map to convert arg to string. """ if isinstance(arg, str): - while MACRO_MARKER in arg: - macro = arg.split(MACRO_MARKER)[1].split(']')[0] + while arg.count(MACRO_MARKER) > 1: + macro = arg.split(MACRO_MARKER, 1)[1].split(MACRO_MARKER, 1)[0] if macro in self.macro_map: - arg = arg.replace("{}{}".format(MACRO_MARKER, macro), str(self.macro_map[macro])) + arg = arg.replace("{}{}{}".format(MACRO_MARKER, macro, MACRO_MARKER), str(self.macro_map[macro])) else: - raise CtfParameterError("Unknown macro {} in arg {}".format(macro, arg), arg) + raise CtfParameterError("Unknown macro '{}' in arg {}. Use format #MACRO#".format(macro, arg), arg) return arg # noinspection PyProtectedMember @@ -573,6 +574,15 @@ def shutdown_cfs(self): if self.cfs: self.cfs.stop_cfs() + # check whether cFS instance exists + pidof_cfs = "pidof {}".format(self.config.cfs_run_cmd) + pid = run(pidof_cfs, stdout=PIPE, stderr=STDOUT, shell=True, check=False).stdout.decode() + if pid == "": + log.error("CFS executable {} had already terminated!".format(self.config.cfs_run_cmd)) + self.cfs_process_list = [] + self.cfs_running = False + return True + kill_string = "kill -9 $(pidof {})".format(self.config.cfs_exe) status = os.system(kill_string) == 0 if not status: @@ -831,16 +841,15 @@ def shutdown_cfs(self): # if self.sp0_plugin.last_result[self.config.name]: # log.debug(self.sp0_plugin.last_result[self.config.name].stdout.strip()) - if self.cfs: - if self.cfs_running: - log.info("Sending SP0 Reboot Command...") - self.sp0_plugin.send_command("reboot()\n", timeout=2, name=self.config.name) - self.cfs.stop_cfs() - - # Wait 2 time units for shutdown to complete - Global.time_manager.wait_seconds(2) + if self.cfs and self.cfs_running: + if self.config.stop_command: + log.info("Sending Stop Command '{}'...".format(self.config.stop_command)) + self.sp0_plugin.send_command(self.config.stop_command + "\n", timeout=2, name=self.config.name) + self.cfs.stop_cfs() + self.cfs_running = False - self.cfs = None + # Wait 2 time units for shutdown to complete + Global.time_manager.wait_seconds(2) def shutdown(self): """ diff --git a/plugins/cfs/tests/pycfs/test_cfs_controllers.py b/plugins/cfs/tests/pycfs/test_cfs_controllers.py index 0ebf9f3..4b59c25 100644 --- a/plugins/cfs/tests/pycfs/test_cfs_controllers.py +++ b/plugins/cfs/tests/pycfs/test_cfs_controllers.py @@ -287,17 +287,17 @@ def test_cfs_controller_resolve_macros(cfs_controller_inited): Implementation of helper function resolve_macros, search macro_map to convert arg to string. """ # nominal case - assert cfs_controller_inited.resolve_macros('#CF_RESET_CC') == '1' + assert cfs_controller_inited.resolve_macros('#CF_RESET_CC#') == '1' # embedded macro - assert cfs_controller_inited.resolve_macros('1 == #CF_RESET_CC') == '1 == 1' + assert cfs_controller_inited.resolve_macros('1 == #CF_RESET_CC#') == '1 == 1' # no macro - assert cfs_controller_inited.resolve_macros('CF_RESET_CC') == 'CF_RESET_CC' + assert cfs_controller_inited.resolve_macros('#CF_RESET_CC') == '#CF_RESET_CC' # invalid macro with pytest.raises(CtfParameterError): - cfs_controller_inited.resolve_macros('#CF_RESET_CC_X') + cfs_controller_inited.resolve_macros('#CF_RESET_CC_X#') def test_cfs_controller_resolve_simple_type(cfs_controller_inited): @@ -482,7 +482,7 @@ def test_cfs_controller_convert_check_tlm_args(cfs_controller_inited): assert cfs_controller_inited.convert_check_tlm_args(args) == args # value is macro - args = [{'variable': 'Payload.CommandErrorCounter', 'value': '#FALSE', 'compare': '=='}] + args = [{'variable': 'Payload.CommandErrorCounter', 'value': '#FALSE#', 'compare': '=='}] assert cfs_controller_inited.convert_check_tlm_args(args)[0]['value'] == '0' @@ -598,8 +598,10 @@ def test_cfs_controller_shutdown_cfs_fail(cfs_controller_inited): (shutdown_cfs) is executed, it calls CfsController instance's shutdown_cfs function. """ cfs_controller_inited.cfs_process_list = ['pid-1', 'pid-2'] - with patch('os.system') as mock_system: + with patch('os.system') as mock_system, \ + patch('plugins.cfs.pycfs.cfs_controllers.run') as mock_run: mock_system.return_value = 255 + mock_run.stdout = "0" assert not cfs_controller_inited.shutdown_cfs() assert not cfs_controller_inited.cfs_process_list assert not cfs_controller_inited.cfs_running diff --git a/plugins/cfs/tests/test_cfs_config.py b/plugins/cfs/tests/test_cfs_config.py index 39c95dd..92e035e 100644 --- a/plugins/cfs/tests/test_cfs_config.py +++ b/plugins/cfs/tests/test_cfs_config.py @@ -263,3 +263,4 @@ def test_sp0_cfs_config_init(sp0_cfs_config): assert sp0_cfs_config.cfs_entry_point == "CFE_PSP_Main" assert sp0_cfs_config.cfs_startup_time == 20 assert sp0_cfs_config.log_stdout is True + assert sp0_cfs_config.stop_command == "reboot()" diff --git a/plugins/ssh/ssh_plugin.py b/plugins/ssh/ssh_plugin.py index ac72e76..1532931 100644 --- a/plugins/ssh/ssh_plugin.py +++ b/plugins/ssh/ssh_plugin.py @@ -1,6 +1,6 @@ """ @namespace plugins.ssh_plugin -The SSH Plug provides remote and local shell command execution capability for CTF. +The SSH Plugin provides remote and local shell command execution capability for CTF. The module defines SshPlugin class and SshConfig, SshController helper class. """ @@ -428,7 +428,7 @@ def shutdown(self): @note The shutdown function is called by the CTF plugin manager upon completion of a test run. @note The shutdown function can be exposed to test scripts by adding it to the command map. """ - log.debug("SshPlugin.shutdown") + log.info("SshPlugin.shutdown") for name in self.targets: log.debug("Shutting down Execution target {}".format(name)) self.targets[name].shutdown() diff --git a/scripts/example_tests/test_ctf_advanced_example.json b/scripts/example_tests/test_ctf_advanced_example.json index ed61d00..9e077a2 100644 --- a/scripts/example_tests/test_ctf_advanced_example.json +++ b/scripts/example_tests/test_ctf_advanced_example.json @@ -288,8 +288,8 @@ "mid": "SCH_CMD_MID", "cc": "SCH_ENABLE_CC", "args": { - "SlotNumber": "#SCH_TOTAL_SLOTS", - "EntryNumber": "#SCH_ENTRIES_PER_SLOT" + "SlotNumber": "#SCH_TOTAL_SLOTS#", + "EntryNumber": "#SCH_ENTRIES_PER_SLOT#" } }, "wait": 1 diff --git a/scripts/example_tests/test_ctf_function.json b/scripts/example_tests/test_ctf_function.json new file mode 100644 index 0000000..e5628ec --- /dev/null +++ b/scripts/example_tests/test_ctf_function.json @@ -0,0 +1,181 @@ +{ + "test_number": "CFE-6-7-CFS-Function-Test", + "test_name": "CFE-6-7 CFS Function Test", + "requirements": { + "MyRequirement": "N/A" + }, + "description": "Testing CFE-6-7 Functions", + "owner": "CTF", + "test_setup": "CTF Example Script Showing Commands/Telemetry Verification, Event Checking, and Function Usage", + "ctf_options": { + "verify_timeout": 4 + }, + "import": {}, + "functions": { + "Increment Event": { + "description": "SendCfsCommand TO_NOOP_CC, and CheckTlmValue usCmdCnt & usCmdErrCnt", + "varlist": [ + "expectedCmdCnt", + "expectedErrCnt" + ], + "instructions": [ + { + "instruction": "SendCfsCommand", + "data": { + "target": "", + "mid": "TO_CMD_MID", + "cc": "TO_NOOP_CC", + "args": {} + }, + "wait": 1 + }, + { + "instruction": "CheckTlmValue", + "data": { + "target": "", + "mid": "TO_HK_TLM_MID", + "args": [ + { + "variable": "usCmdCnt", + "value": [ + "expectedCmdCnt" + ], + "compare": "==" + } + ] + }, + "disabled": true, + "wait": 1 + }, + { + "instruction": "CheckTlmValue", + "data": { + "target": "", + "mid": "TO_HK_TLM_MID", + "args": [ + { + "variable": "usCmdErrCnt", + "value": [ + "expectedErrCnt" + ], + "compare": "==" + } + ] + }, + "wait": 1 + } + ] + } + }, + "tests": [ + { + "case_number": "CFE-6-7-Function-Test-001", + "description": "CTF Example Script Showing Commands/Telemetry Verification, Event Checking, and Function Usage", + "instructions": [ + { + "instruction": "BuildCfs", + "data": { + "target": "" + }, + "wait": 1 + }, + { + "instruction": "StartCfs", + "data": { + "target": "" + }, + "wait": 1 + }, + { + "instruction": "EnableCfsOutput", + "data": { + "target": "" + }, + "wait": 5 + }, + { + "description": "This will format the message with msg_args and then check for a regex match", + "instruction": "CheckEvent", + "data": { + "app": "TO", + "id": "3", + "msg": "%s - %s cmd succesful for routeMask:0x[0-9]+", + "is_regex": true, + "msg_args": "('TO', 'ENABLE_OUTPUT')", + "target": "" + }, + "wait": 0 + }, + { + "instruction": "CheckTlmValue", + "data": { + "target": "", + "mid": "TO_HK_TLM_MID", + "args": [ + { + "compare": "==", + "variable": "usCmdErrCnt", + "value": [ + 0.1 + ], + "tolerance_minus": 0.1 + } + ] + }, + "wait": 1 + }, + { + "function": "Increment Event", + "params": { + "expectedCmdCnt": 2, + "expectedErrCnt": 0 + }, + "disabled": true, + "wait": 1 + }, + { + "function": "Increment Event", + "params": { + "expectedCmdCnt": 3, + "expectedErrCnt": 0 + }, + "wait": 1 + }, + { + "instruction": "SendCfsCommand", + "data": { + "target": "", + "mid": "TO_CMD_MID", + "cc": "TO_RESET_CC", + "args": {} + }, + "wait": 1 + }, + { + "instruction": "CheckTlmValue", + "data": { + "target": "", + "mid": "TO_HK_TLM_MID", + "args": [ + { + "compare": "==", + "variable": "usCmdCnt", + "value": [ + 0 + ] + } + ] + }, + "wait": 1 + }, + { + "instruction": "ShutdownCfs", + "data": { + "target": "" + }, + "wait": 1 + } + ] + } + ] +}