diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f498244f..21e8503e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,6 +29,7 @@ repos: selfsigned.py| defaultconfig.c4m| configs/.*.c4m| + tests/.*.c4m| data/.* )$ diff --git a/chalk.nimble b/chalk.nimble index 82f73ff5..efa1d4cb 100644 --- a/chalk.nimble +++ b/chalk.nimble @@ -8,7 +8,7 @@ bin = @["chalk"] # Dependencies requires "nim >= 2.0.0" -requires "https://github.com/crashappsec/con4m#e04278bc953540fcdb80735418b2fb17e79dec9f" +requires "https://github.com/crashappsec/con4m#da5a430616ef2740da603438b35436d184c36938" requires "https://github.com/viega/zippy == 0.10.7" requires "https://github.com/aruZeta/QRgen == 3.0.0" diff --git a/configs/new/compliance_docker.c4m b/configs/new/compliance_docker.c4m new file mode 100644 index 00000000..855d1298 --- /dev/null +++ b/configs/new/compliance_docker.c4m @@ -0,0 +1,3 @@ +use impersonate_docker +use reporting_server +use wrap_entrypoints diff --git a/configs/new/impersonate_docker.c4m b/configs/new/impersonate_docker.c4m new file mode 100644 index 00000000..42f9bdfd --- /dev/null +++ b/configs/new/impersonate_docker.c4m @@ -0,0 +1,3 @@ +## Forces setting of the default command to 'docker'. + +default_command: "docker" diff --git a/configs/new/log_report.c4m b/configs/new/log_report.c4m new file mode 100644 index 00000000..b25fb7e8 --- /dev/null +++ b/configs/new/log_report.c4m @@ -0,0 +1,10 @@ +parameter var disable_default_report { + shortdoc: "Disable the log report" + doc: """ +Causes Chalk to turn off the default log report when enabled. +""" +} + +if disable_default_report { + unsubscribe("report", "default_out") +} diff --git a/configs/new/reporting_server.c4m b/configs/new/reporting_server.c4m new file mode 100644 index 00000000..54e13440 --- /dev/null +++ b/configs/new/reporting_server.c4m @@ -0,0 +1,36 @@ +# TODO: default + +func validate_url(url) { + result := "" + + if (not url.starts_with("http://")) and (not url.starts_with("https://")) { + return "Only http / https URLs are supported" + } +} + +func get_local_url() { + out, code := system("ifconfig -a | grep inet | grep broadcast | head -1 | " + + "awk '{ print $2 }'") + if code != 0 { + return "https://localhost:7890" + } + + return "https://" + out.strip() + ":7890" +} + +parameter sink_config.output_to_http.uri { + shortdoc: "URL for reporting server" + doc: """ +A config for sending reports to a custom implementation of the test +reporting server. +""" + validator: func validate_url(string) -> string + default: func get_local_url() -> string +} + +sink_config output_to_http { + enabled: true + sink: "post" + + # The URI should get filled in automatically. +} \ No newline at end of file diff --git a/configs/new/terminal_report.c4m b/configs/new/terminal_report.c4m new file mode 100644 index 00000000..91e05841 --- /dev/null +++ b/configs/new/terminal_report.c4m @@ -0,0 +1,17 @@ +# Todo: the default should switch to true when using a docker recipe + +parameter var disable_terminal_reports { + default: false + shortdoc: "Disable terminal summary reports" + doc: """ +Controls whether to force off the default summary reports that get +print to your terminal. If you set this to 'true', any conflicting +code that attempts to ensure this is on would fail when the +configuration loads. +""" +} + +if disable_terminal_reports { + custom_report.terminal_chalk_time.enabled: false + custom_report.terminal_other_op.enabled: false +} diff --git a/configs/new/wrap_entrypoints.c4m b/configs/new/wrap_entrypoints.c4m new file mode 100644 index 00000000..21d7cef5 --- /dev/null +++ b/configs/new/wrap_entrypoints.c4m @@ -0,0 +1,2 @@ +# Ensures entrypoint wrapping is enabled in the config" +docker.wrap_entrypoint: true diff --git a/docker-compose.yml b/docker-compose.yml index 03ac79dd..6e5289f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -127,8 +127,6 @@ services: condition: service_healthy imds: condition: service_healthy - static: - condition: service_healthy environment: GITHUB_ACTIONS: ${GITHUB_ACTIONS:-} diff --git a/src/autocomplete/default.bash b/src/autocomplete/default.bash index e07b30b6..1937a9b7 100644 --- a/src/autocomplete/default.bash +++ b/src/autocomplete/default.bash @@ -39,7 +39,7 @@ function _chalk_delete_completions { function _chalk_load_completions { if [ ${_CHALK_CUR_WORD::1} = "-" ] ; then - COMPREPLY=($(compgen -W "--color --no-color --help --log-level --config-file --enable-report --disable-report --report-cache-file --time --no-time --use-embedded-config --no-use-embedded-config --use-external-config --no-use-external-config --show-config --no-show-config --use-report-cache --no-use-report-cache --debug --no-debug --validation --no-validation --validation-warning --no-validation-warning" -- ${_CHALK_CUR_WORD})) + COMPREPLY=($(compgen -W "--color --no-color --help --log-level --config-file --enable-report --disable-report --report-cache-file --time --no-time --use-embedded-config --no-use-embedded-config --use-external-config --no-use-external-config --show-config --no-show-config --use-report-cache --no-use-report-cache --debug --no-debug --replace --no-replace --validation --no-validation --validation-warning --no-validation-warning" -- ${_CHALK_CUR_WORD})) fi if [[ $_CHALK_CUR_IX -le $COMP_CWORD ]] ; then @@ -175,4 +175,4 @@ function _chalk_completions { } complete -F _chalk_completions chalk -# { "MAGIC" : "dadfedabbadabbed", "CHALK_ID" : "64W64C-SN6N-GP6S-B26XHK", "CHALK_VERSION" : "0.1.0", "TIMESTAMP_WHEN_CHALKED" : 1695626987741, "DATETIME_WHEN_CHALKED" : "2023-09-25T03:29:47.563-04:00", "ARTIFACT_TYPE" : "bash", "ARTIFACT_VERSION" : "0.1.1", "CHALK_PTR" : "This mark determines when to update the script. If there is no mark, or the mark is invalid it will be replaced. To customize w/o Chalk disturbing it when it can update, add a valid mark with a version key higher than the current chalk verison, or use version 0.0.0 to prevent updates", "HASH" : "18b355aceb7c188c08718bbdf0904069009193d49b24ed93bc74b92e99294d0f", "INJECTOR_COMMIT_ID" : "ce4922ec7f7458ba441f8c74652c01f802ebd802", "ORIGIN_URI" : "https://github.com/crashappsec/chalk-internal.git", "METADATA_ID" : "DS8HPG-RDKW-1AH8-H7Q0E5" } +# { "MAGIC" : "dadfedabbadabbed", "CHALK_ID" : "6SJ64D-K36W-TK6R-B668W6", "CHALK_VERSION" : "0.1.2", "TIMESTAMP_WHEN_CHALKED" : 1697041887568, "DATETIME_WHEN_CHALKED" : "2023-10-11T12:31:27.407-04:00", "ARTIFACT_TYPE" : "bash", "ARTIFACT_VERSION" : "0.1.2", "CHALK_PTR" : "This mark determines when to update the script. If there is no mark, or the mark is invalid it will be replaced. To customize w/o Chalk disturbing it when it can update, add a valid mark with a version key higher than the current chalk verison, or use version 0.0.0 to prevent updates", "HASH" : "6db6c753af28acc7ab086e1a939307b5f00ceb181d553a2442eb4c683c67c760", "INJECTOR_COMMIT_ID" : "b1c06256a1dce6d8720cde642f95e0e2b07052a3", "ORIGIN_URI" : "git@github.com:crashappsec/chalk.git", "METADATA_ID" : "4S9MRF-9JE8-W5C1-BQHJC3" } diff --git a/src/autocomplete/mac.bash b/src/autocomplete/mac.bash index 948c45bb..2e1a8f8a 100644 --- a/src/autocomplete/mac.bash +++ b/src/autocomplete/mac.bash @@ -32,7 +32,7 @@ function _chalk_delete_completions { function _chalk_load_completions { if [ ${_CHALK_CUR_WORD::1} = "-" ] ; then - COMPREPLY=($(compgen -W "--color --no-color --help --log-level --config-file --enable-report --disable-report --report-cache-file --time --no-time --use-embedded-config --no-use-embedded-config --use-external-config --no-use-external-config --show-config --no-show-config --use-report-cache --no-use-report-cache --debug --no-debug --validation --no-validation --validation-warning --no-validation-warning" -- ${_CHALK_CUR_WORD})) + COMPREPLY=($(compgen -W "--color --no-color --help --log-level --config-file --enable-report --disable-report --report-cache-file --time --no-time --use-embedded-config --no-use-embedded-config --use-external-config --no-use-external-config --show-config --no-show-config --use-report-cache --no-use-report-cache --debug --no-debug --replace --no-replace --validation --no-validation --validation-warning --no-validation-warning" -- ${_CHALK_CUR_WORD})) fi if [[ $_CHALK_CUR_IX -le $COMP_CWORD ]] ; then @@ -153,4 +153,4 @@ function _chalk_completions { } complete -F _chalk_completions chalk -# { "MAGIC" : "dadfedabbadabbed", "CHALK_ID" : "CRTP2D-HM6N-H3GE-1J61J6", "CHALK_VERSION" : "0.1.0", "TIMESTAMP_WHEN_CHALKED" : 1695626987742, "DATETIME_WHEN_CHALKED" : "2023-09-25T03:29:47.563-04:00", "ARTIFACT_TYPE" : "bash", "ARTIFACT_VERSION" : "0.1.1", "CHALK_PTR" : "This mark determines when to update the script. If there is no mark, or the mark is invalid it will be replaced. To customize w/o Chalk disturbing it when it can update, add a valid mark with a version key higher than the current chalk verison, or use version 0.0.0 to prevent updates", "HASH" : "f5a645b8820db0947a849b4fc11019b875f0f4c801ffc5462f4fae4fd1456ceb", "INJECTOR_COMMIT_ID" : "ce4922ec7f7458ba441f8c74652c01f802ebd802", "ORIGIN_URI" : "https://github.com/crashappsec/chalk-internal.git", "METADATA_ID" : "2X6AP1-ED7P-MXQS-4DBW12" } +# { "MAGIC" : "dadfedabbadabbed", "CHALK_ID" : "CNK3CD-K36C-V68R-HJ74RK", "CHALK_VERSION" : "0.1.2", "TIMESTAMP_WHEN_CHALKED" : 1697041887569, "DATETIME_WHEN_CHALKED" : "2023-10-11T12:31:27.407-04:00", "ARTIFACT_TYPE" : "bash", "ARTIFACT_VERSION" : "0.1.2", "CHALK_PTR" : "This mark determines when to update the script. If there is no mark, or the mark is invalid it will be replaced. To customize w/o Chalk disturbing it when it can update, add a valid mark with a version key higher than the current chalk verison, or use version 0.0.0 to prevent updates", "HASH" : "ef66c36db2913d2f7d04e28b936ee05364efcc642370a58927d77f7ac9309141", "INJECTOR_COMMIT_ID" : "b1c06256a1dce6d8720cde642f95e0e2b07052a3", "ORIGIN_URI" : "git@github.com:crashappsec/chalk.git", "METADATA_ID" : "BMHQQ0-QTAE-P5AA-MD40QG" } diff --git a/src/chalk.nim b/src/chalk.nim index b3d50717..0139e7f7 100644 --- a/src/chalk.nim +++ b/src/chalk.nim @@ -13,7 +13,7 @@ import config, confload, commands, norecurse, sinks, docker_base, when isMainModule: setupSignalHandlers() # util.nim addDefaultSinks() # nimutils/sinks.nim - loadAllConfigs() # config.nim + loadAllConfigs() # confload.nim recursionCheck() # norecurse.nim otherSetupTasks() # util.nim # Wait for this warning until after configs load. diff --git a/src/commands/cmd_docker.nim b/src/commands/cmd_docker.nim index ed766cea..30af22d2 100644 --- a/src/commands/cmd_docker.nim +++ b/src/commands/cmd_docker.nim @@ -22,7 +22,7 @@ ## But when wrapping docker, this module does the bulk of the work and ## is responsible for all of the collection logic. -import posix, unicode, base64, ../config, ../collect, ../reporting, +import posix, unicode, ../config, ../collect, ../reporting, ../chalkjson, ../docker_cmdline, ../docker_base, ../subscan, ../dockerfile, ../util, ../attestation, ../commands/cmd_help, ../plugin_api diff --git a/src/commands/cmd_load.nim b/src/commands/cmd_load.nim index 351ed55f..8702ef01 100644 --- a/src/commands/cmd_load.nim +++ b/src/commands/cmd_load.nim @@ -15,9 +15,9 @@ proc runCmdConfLoad*() = var newCon4m: string - let filename = getArgs()[0] + let url = getArgs()[0] - if filename == "0cool": + if url == "0cool": var args = ["nc", "crashoverride.run", "23"] egg = allocCstringArray(args) @@ -26,6 +26,7 @@ proc runCmdConfLoad*() = egg[0] = "telnet" discard execvp("telnet", egg) stderr.writeLine("I guess it's not easter.") + quit(0) let selfChalk = getSelfExtraction().getOrElse(nil) setAllChalks(@[selfChalk]) @@ -33,7 +34,7 @@ proc runCmdConfLoad*() = if selfChalk == nil or not canSelfInject: cantLoad("Platform does not support self-injection.") - if filename == "default": + if url == "default": if selfChalk.isMarked() and "$CHALK_CONFIG" notin selfChalk.extract: cantLoad("Already using the default configuration.") else: @@ -41,18 +42,7 @@ proc runCmdConfLoad*() = selfChalk.collectedData.del("$CHALK_CONFIG") info("Installing the default configuration file.") else: - if filename.startswith("http://") or filename.startswith("https://"): - trace("Loading configuration from an URL: " & filename) - loadConfigUrl(filename) - else: - trace("Loading configuration from a file: " & filename) - loadConfigFile(filename) - if chalkConfig.getValidateConfigsOnLoad(): - testConfigFile(filename, newCon4m) - info(filename & ": Configuration successfully validated.") - else: - warn("Skipping configuration validation. This could break chalk.") + url.handleConfigLoad() selfChalk.writeSelfConfig() - info("Updated configuration for " & selfChalk.name) doReporting() diff --git a/src/configs/base_chalk_templates.c4m b/src/configs/base_chalk_templates.c4m index 420cce41..b4fce064 100644 --- a/src/configs/base_chalk_templates.c4m +++ b/src/configs/base_chalk_templates.c4m @@ -170,6 +170,8 @@ mark_template mark_large { key.$CHALK_PUBLIC_KEY.use = true key.$CHALK_ENCRYPTED_PRIVATE_KEY.use = true key.$CHALK_ATTESTATION_TOKEN.use = true + key.$CHALK_COMPONENT_CACHE.use = true + key.$CHALK_SAVED_COMPONENT_PARAMETERS.use = true } mark_template mark_default { @@ -223,6 +225,8 @@ use the mark template named `reproducable`. key.$CHALK_PUBLIC_KEY.use = true key.$CHALK_ENCRYPTED_PRIVATE_KEY.use = true key.$CHALK_ATTESTATION_TOKEN.use = true + key.$CHALK_COMPONENT_CACHE.use = true + key.$CHALK_SAVED_COMPONENT_PARAMETERS.use = true } # This is the same as the `default` template, except the first three @@ -268,6 +272,8 @@ the time and the nonce removed. key.$CHALK_PUBLIC_KEY.use = true key.$CHALK_ENCRYPTED_PRIVATE_KEY.use = true key.$CHALK_ATTESTATION_TOKEN.use = true + key.$CHALK_COMPONENT_CACHE.use = true + key.$CHALK_SAVED_COMPONENT_PARAMETERS.use = true } mark_template minimal { @@ -290,6 +296,8 @@ This is the default for `docker` chalk marks. key.$CHALK_PUBLIC_KEY.use = true key.$CHALK_ENCRYPTED_PRIVATE_KEY.use = true key.$CHALK_ATTESTATION_TOKEN.use = true + key.$CHALK_COMPONENT_CACHE.use = true + key.$CHALK_SAVED_COMPONENT_PARAMETERS.use = true } mark_template chalk_labels { diff --git a/src/configs/base_init.c4m b/src/configs/base_init.c4m index 938c037d..b8dfb24f 100644 --- a/src/configs/base_init.c4m +++ b/src/configs/base_init.c4m @@ -17,16 +17,15 @@ exec { # TODO Remove all sections below # Currently, these need to be here for singleton defaults to take hold. -extract { -} +extract { } -source_marks { -} +source_marks { } + +docker { } + +load { } -docker { -} aws { - ec2 { - } + ec2 { } } diff --git a/src/configs/base_keyspecs.c4m b/src/configs/base_keyspecs.c4m index b1f5182d..1e4c5e1b 100644 --- a/src/configs/base_keyspecs.c4m +++ b/src/configs/base_keyspecs.c4m @@ -4777,3 +4777,52 @@ keyspec $CHALK_SECRET_ENDPOINT_URI { ... """ } + +keyspec $CHALK_SAVED_COMPONENT_PARAMETERS { + required_in_self_mark: true + kind: ChalkTimeArtifact + + # Note that I'm not sure the 'proper' type would work yet (in fact, + # I am somewhat sure it will not). I haven't had time to test it; + # ideally the fact that `x is in a typespec parameter would mean it + # re-binds for every typecheck against the list, but I don't think + # this is actually the case yet. + # + # At the very least, as long as con4m never operates on the values, + # it will happily accept `x. + # + # type: list[tuple[bool, string, string, typespec[`x], `x]] + + type: `x + standard: true + system: true + since: "0.1.2" + doc: """ +This is where we save configuration parameters for components that +have been imported. + +The items in the list consist of five-tuples: + +1) A boolean indicating whether it's an attribute parameter (false + means it's a variable parameter) +2) The base URL reference for the component +3) The name of the variable or attribute. +4) The Con4m type of the parameter. +5) The stored value (which will be of the type provided) +""" + +} + +keyspec $CHALK_COMPONENT_CACHE { + required_in_self_mark: true + kind: ChalkTimeArtifact + type: dict[string, string] + standard: true + system: true + since: "0.1.2" + doc: """ +This consists of URLs (minus the file extension) mapped to source code +for components. +""" + +} \ No newline at end of file diff --git a/src/configs/base_report_templates.c4m b/src/configs/base_report_templates.c4m index ee9c035e..886ec24a 100644 --- a/src/configs/base_report_templates.c4m +++ b/src/configs/base_report_templates.c4m @@ -379,6 +379,8 @@ report and subtract from it. key.$CHALK_API_REFRESH_TOKEN.use = true key.$CHALK_ATTESTATION_TOKEN.use = true key.$CHALK_SECRET_ENDPOINT_URI.use = true + key.$CHALK_COMPONENT_CACHE.use = false + key.$CHALK_SAVED_COMPONENT_PARAMETERS.use = true } report_template report_large { @@ -768,6 +770,8 @@ doc: """ key.$CHALK_PUBLIC_KEY.use = true key.$CHALK_ENCRYPTED_PRIVATE_KEY.use = true key.$CHALK_ATTESTATION_TOKEN.use = true + key.$CHALK_COMPONENT_CACHE.use = false + key.$CHALK_SAVED_COMPONENT_PARAMETERS.use = true } report_template report_default { @@ -1157,7 +1161,8 @@ container. key.$CHALK_PUBLIC_KEY.use = true key.$CHALK_ENCRYPTED_PRIVATE_KEY.use = true key.$CHALK_ATTESTATION_TOKEN.use = true - + key.$CHALK_COMPONENT_CACHE.use = false + key.$CHALK_SAVED_COMPONENT_PARAMETERS.use = true } report_template insertion_default { shortdoc: "The default template for insertion operations" @@ -1549,8 +1554,8 @@ and keep the run-time key. key.$CHALK_PUBLIC_KEY.use = true key.$CHALK_ENCRYPTED_PRIVATE_KEY.use = true key.$CHALK_ATTESTATION_TOKEN.use = true - - + key.$CHALK_COMPONENT_CACHE.use = false + key.$CHALK_SAVED_COMPONENT_PARAMETERS.use = true } report_template unknown_docker { diff --git a/src/configs/chalk.c42spec b/src/configs/chalk.c42spec index 33168e80..f1cd7ec1 100644 --- a/src/configs/chalk.c42spec +++ b/src/configs/chalk.c42spec @@ -292,7 +292,7 @@ available operational metadata from any one run of Chalk. for your report to bubble up the fields actually chalked). Other operations report these keys only if they're extracted from artifacts. - + To use the above template, we'd just have to tell the system when to use this template, as described below. """ @@ -809,7 +809,7 @@ object sink { user_def_ok: true validator: func sink_object_check doc: """ - + This object type is needed to add new data sinks to chalk. If you're not a Chalk developer, this probably isn't going to be particularly useful; Instead, use `sink_config` to configure a sink, and then @@ -936,7 +936,7 @@ object outconf { doc: """ ## Changing reports for operations - + Each chalk operation that reports metadata will have one or more associated `outconf` sections in the configuration. These sections do only two things: @@ -987,7 +987,7 @@ Similarly, `chalk exec` can produce two reports: The report name is always specified in the `_OPERATION` metadata key. The `outconf` section requires the **report name**, not the command that causes the report to run. - + Out of the box, Chalk defines default templates that you can edit. If you edit the template(s) already in use by an outconf section, you don't need to do anything else. However, if you wish to switch to a @@ -1039,10 +1039,10 @@ object custom_report { gen_typename: "ReportSpec" gen_setters: false user_def_ok: false - + doc: """ ## Adding additional reports - + A `custom_report` section allows you to create secondary reports for whatever purpose. For instance, in the default Chalk configuration, the *primary* report logs to a file, but a secondary report gives @@ -1174,7 +1174,7 @@ report will NOT run. If not specified, reports apply to any command that reports. """ - } + } field doc { type: string @@ -1189,7 +1189,7 @@ singleton extract { gen_setters: false user_def_ok: false doc: """ - + These are configuration options specific to how container extraction works for containers (plenty of the global options apply to extraction). Currently, the only options involve how we handle looking @@ -1259,7 +1259,7 @@ will run it again without itself in the way. Such cases are the only times in the default configuration where error messages are logged to the console (when running `chalk docker`). """ - allow getopts { } + allow getopts field wrap_entrypoint { type: bool @@ -1415,6 +1415,59 @@ ENV ARTIFACT_IDENTIFIER="X6VRPZ-C828-KDNS-QDXRT0" } } +singleton load { + gen_fieldname: "loadConfig" + gen_typename: "LoadConfig" + gen_setters: false + user_def_ok: false + doc: """ +Options that control how the `chalk load` command works. +""" + + field replace_conf { + type: bool + default: false + shortdoc: "Replace on load" + doc: """ + +When this value is true, the entire stored configuration file will be +REPLACED with the specified configuration, as long as that +configuration loads successfully. + +Otherwise, the passed configuration is treated like a component: + +1. If you are not using the component in your embbeded configuration + already, it will be added to your config, and if it requires any + parameters, you will be prompted to configure them. + +2. If you are already using it, it will be updated, and you will be + prompted to reconfigure any items necessary for the component. + +This flag is ignored when running `chalk load default`, which will +_always_ reset the embedded configuration to the default. + +""" + } + + field validate_configs_on_load { + type: bool + default: true + hidden: true + doc: """ +Suppress validation of configuration files on loading. Please don't do this! +""" + } + + field validation_warning { + type: bool + default: true + shortdoc: "Show 'chalk load' validation warning" + doc: """ +Show the (admittedly verbose) warning you get when running 'chalk load'. +""" + } +} + singleton exec { gen_fieldname: "execConfig" gen_typename: "ExecConfig" @@ -1589,7 +1642,7 @@ singleton env_config { gen_setters: false user_def_ok: false doc: """ - + This section is for internal configuration information gathering runtime environment information when running with the 'env' command, which is similar to the exec command, but where the exec command @@ -1924,7 +1977,7 @@ Configuration information for AWS EC2 doc: """ The fields in this section probably will never need to be changed by end users. -""" +""" field sys_hypervisor_path { # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/identify_ec2_instances.html @@ -1959,7 +2012,7 @@ singleton aws { shortdoc: """ Configuration information for the AWS Cloud Provider """ - allow ec2 { } + allow ec2 } root { @@ -1974,24 +2027,25 @@ root { gen_setters: false user_def_ok: false - allow keyspec { } - allow plugin { } - allow sink { } - allow sink_config { } - allow mark_template { } - allow report_template { } - allow outconf { } - allow custom_report { } - allow tool { } - allow extract { } - allow docker { } - allow exec { } - allow env_config { } - allow source_marks { } - allow aws { } + allow keyspec + allow plugin + allow sink + allow sink_config + allow mark_template + allow report_template + allow outconf + allow custom_report + allow tool + allow extract + allow docker + allow exec + allow load + allow env_config + allow source_marks + allow aws shortdoc: "Chalk Configuration Options" - + doc: """ This guide details all of the configuration options available in Chalk. These are all configurable variables that you can add in your configuration file. In some cases, there will also be other ways to set these values: @@ -2017,7 +2071,7 @@ The config file types can be disabled with the command-line flags `--no-use-embe field config_filename { type: string - default: "chalk.conf" + default: "chalk.c4m" write_lock: true doc: "The file name to look for when searching for a file" shortdoc: "Configuration File Name" @@ -2140,7 +2194,7 @@ By default, virtual chalk marks will get appended to the file `./virtual-chalk.j hidden: true doc: "Extensions that we assume are in ZIP format for the zip codec" } - + field pyc_extensions { type: list[string] default: ["pyc", "pyo", "pyd"] @@ -2202,7 +2256,7 @@ Chalk can report the time from start until the time a report is produced by subscribing to the `_CHALK_RUN_TIME` host key. However, if you're running on the command line, and want the total time to be output to stderr as the very last thing, you can use this option (`--time` on the command line). -""" +""" } field audit_location { @@ -2280,7 +2334,7 @@ allows you to pick a place for temporary files to use IF no value for If neither is provided, you may very well end up with `/tmp` or `/var/tmp`, which should be great in most cases. -""" +""" } field always_try_to_sign { @@ -2644,24 +2698,6 @@ if env_exists("AWS_IAM_ROLE") { aws_iam_role = env("AWS_IAM_ROLE") } ``` -""" - } - - field validate_configs_on_load { - type: bool - default: true - hidden: true - doc: """ -Suppress validation of configuration files on loading. Please don't do this! -""" - } - - field validation_warning { - type: bool - default: true - shortdoc: "Show 'chalk load' validation warning" - doc: """ -Show the (admittedly verbose) warning you get when running 'chalk load'. """ } @@ -2770,7 +2806,7 @@ When using the help system, this controls whether documents are dumped directly to the terminal, or passed through your system's pager. To skip the pager on the command line, use the `--no-pager` flag. -""" +""" } field crashoverride_usage_reporting_url { @@ -2925,7 +2961,7 @@ func validate_mark_template(template_name) { if template_name == "" { return } - + sects := sections("mark_template") if not sects.contains(template_name) { @@ -2947,7 +2983,7 @@ func validate_mark_template(template_name) { full_value_path := key_attr_path + "." + template_keys[i] + ".use" if find(template_keys[i], "_") == 0 { - return ("Chalk mark template '" + template_name + + return ("Chalk mark template '" + template_name + "' contains a key: '" + template_keys[i] + "' which is not a chalk-time key. Chalk-time " + "keys are those that do NOT start with an underscore.") @@ -3006,7 +3042,7 @@ func outconf_mark_template_check(name, template_name) { func outconf_report_template_check(name, template_name) { result := validate_report_template(template_name) - + if result != "" { result := in_outconf(name) + result } @@ -3014,7 +3050,7 @@ func outconf_report_template_check(name, template_name) { func custom_report_template_check(name, report_template_name) { result := validate_report_template(report_template_name) - + if result != "" { result := in_report(name) + result } @@ -3308,7 +3344,7 @@ func sink_config_check(path) { func outconf_mark_templates_exist() { result := "" - + # If they've been explicitly set, then they will have been checked by # a value validator. So all we need to do here is make sure that the # insertion outconfs have non-null values. diff --git a/src/configs/getopts.c4m b/src/configs/getopts.c4m index 6f8d98d7..477e1612 100644 --- a/src/configs/getopts.c4m +++ b/src/configs/getopts.c4m @@ -482,15 +482,24 @@ You can use the 'dump' command to dump the output first. From the command line, See 'help config' for an overview of the configuration file format. """ + flag_yn replace { + field_to_set: "load.replace_conf" + doc: """ +When on, the entire stored configuration file will be REPLACED with the +provided argument. When off, it's used only as a component that's added +to the config. +""" + } + flag_yn validation { - field_to_set: "validate_configs_on_load" + field_to_set: "load.validate_configs_on_load" doc: """ When on, validate config files before loading them, by doing a trial run. """ } flag_yn validation_warning { - field_to_set: "validation_warning" + field_to_set: "load.validation_warning" doc: """ This verbose flag controls whether or not you get the verbose warning. It's much better turning this off in your embedded configuration :) """ diff --git a/src/confload.nim b/src/confload.nim index b19f3710..df60fd21 100644 --- a/src/confload.nim +++ b/src/confload.nim @@ -19,6 +19,8 @@ import config, selfextract, con4mfuncs, plugin_load import macros except error +const chalkDefaultconfigStore = "https://chalkdust.io/" + # Since these are system keys, we are the only one able to write them, # and it's easier to do it directly here than in the system plugin. proc stashFlags(winner: ArgResult) = @@ -32,7 +34,27 @@ proc stashFlags(winner: ArgResult) = hostInfo["_OP_CMD_FLAGS"] = pack(flagStrs) -# TODO: static code to validate loaded specs. +proc installComponentParams(params: seq[Box]) = + let runtime = getChalkRuntime() + + for item in params: + let + row = unpack[seq[Box]](item) + attr = unpack[bool](row[0]) + url = unpack[string](row[1]) + sym = unpack[string](row[2]) + c4mType = toCon4mType(unpack[string](row[3])) + value = row[4] + if attr: + runtime.setAttributeParamValue(url, sym, value, c4mType) + else: + runtime.setVariableParamValue(url, sym, value, c4mType) + +proc loadCachedComponents(cache: OrderedTableRef[string, string]) = + for url, src in cache: + let component = getChalkRuntime().getComponentReference(url) + component.cacheComponent(src) + trace("Loaded cached version of: " & url & ".c4m") proc getEmbeddedConfig(): string = result = defaultConfig @@ -40,15 +62,27 @@ proc getEmbeddedConfig(): string = if extraction.isSome(): let selfChalk = extraction.get() - if selfChalk.extract != nil and selfChalk.extract.contains("$CHALK_CONFIG"): - trace("Found embedded config file in self-chalk.") - return unpack[string](selfChalk.extract["$CHALK_CONFIG"]) - else: - if selfChalk.marked: - trace("Found an embedded chalk mark, but it did not contain a config.") + if selfChalk.extract != nil: + if selfChalk.extract.contains("$CHALK_CONFIG"): + trace("Found embedded config file in self-chalk.") + result = unpack[string](selfChalk.extract["$CHALK_CONFIG"]) + else: + if selfChalk.marked: + trace("Found a chalk mark, but it did not contain a config.") + else: + trace("No embedded chalk mark.") + trace("Using the default user config. See 'chalk dump' to view.") + if selfChalk.extract.contains("$CHALK_SAVED_COMPONENT_PARAMETERS"): + let params = selfChalk.extract["$CHALK_SAVED_COMPONENT_PARAMETERS"] + installComponentParams(unpack[seq[Box]](params)) else: - trace("No embedded chalk mark.") - trace("Using the default user config. See 'chalk dump' to view.") + trace("No saved component parameters; skipping install.") + if selfChalk.extract.contains("$CHALK_COMPONENT_CACHE"): + let + componentInfo = selfChalk.extract["$CHALK_COMPONENT_CACHE"] + unpackedInfo = unpack[OrderedTableRef[string, string]](componentInfo) + + loadCachedComponents(unpackedInfo) else: trace("Since this binary can't be marked, using the default config.") @@ -123,6 +157,8 @@ proc loadAllConfigs*() = res: ArgResult # Used across macros above. resFound: bool + setDefaultStoreUrl(chalkDefaultConfigStore) + let toStream = newStringStream stack = newConfigStack() diff --git a/src/selfextract.nim b/src/selfextract.nim index ad99669b..1658bd54 100644 --- a/src/selfextract.nim +++ b/src/selfextract.nim @@ -7,7 +7,7 @@ ## Code specific to reading and writing Chalk's own chalk mark. -import config, httpclient, plugin_api, posix, collect, con4mfuncs, chalkjson, util, uri, nimutils/sinks +import config, plugin_api, posix, collect, con4mfuncs, chalkjson, util proc handleSelfChalkWarnings*() = if not canSelfInject: @@ -18,6 +18,11 @@ proc handleSelfChalkWarnings*() = elif "CHALK_ID" notin selfChalk.extract: error("Self-chalk mark found, but is invalid.") +template cantLoad*(s: string) = + error(s) + quit(1) + + proc getSelfExtraction*(): Option[ChalkObj] = # If we call twice and we're on a platform where we don't # have a codec for this type of executable, avoid dupe errors. @@ -70,25 +75,33 @@ proc getSelfExtraction*(): Option[ChalkObj] = else: result = none(ChalkObj) -template selfChalkGetKey*(keyName: string): Option[Box] = +proc selfChalkGetKey*(keyName: string): Option[Box] = if selfChalk == nil or selfChalk.extract == nil or keyName notin selfChalk.extract: - none(Box) + return none(Box) else: - some(selfChalk.extract[keyName]) + return some(selfChalk.extract[keyName]) + +proc selfChalkSetKey*(keyName: string, val: Box) = + if selfChalk.extract != nil: + # Overwrite what we extracted, as it'll get "preserved" when + # writing out the chalk file. + selfChalk.extract[keyName] = val + selfChalk.collectedData[keyName] = val + +proc selfChalkDelKey*(keyName: string) = + if selfChalk.extract != nil and keyName in selfChalk.extract: + selfChalk.extract.del(keyName) + if keyName in selfChalk.collectedData: + selfChalk.collectedData.del(keyName) # The rest of this is specific to writing the self-config. -template cantLoad*(s: string) = - error(s) - quit(1) proc newConfFileError(err, tb: string): bool = if chalkConfig != nil and chalkConfig.getChalkDebug(): - error(err & "\n" & tb) + cantLoad(err & "\n" & tb) else: - error(err) - - quit(1) + cantLoad(err) proc makeExecutable(f: File) = ## Todo: this can move to nimutils actually. @@ -108,7 +121,7 @@ proc makeExecutable(f: File) = proc writeSelfConfig*(selfChalk: ChalkObj): bool {.cdecl, exportc, discardable.} = - selfChalk.persistInternalValues() + selfChalk.persistInternalValues() # Found in run_management.nim collectChalkTimeHostInfo() let lastCount = if "$CHALK_LOAD_COUNT" notin selfChalk.collectedData: @@ -155,47 +168,9 @@ proc writeSelfConfig*(selfChalk: ChalkObj): bool selfChalk.makeNewValuesAvailable() return true -template loadConfigFile*(filename: string) = - let f = newFileStream(resolvePath(filename)) - if f == nil: - cantLoad(filename & ": could not open configuration file") - loadConfigStream(filename, f) - -template loadConfigUrl*(url: string) = - let uri = parseUri(url) - var stream: Stream - try: - let - client = newHttpClient(timeout = 5000) # 5 seconds - response = client.safeRequest(uri) - stream = response.bodyStream - - except: - dumpExOnDebug() - cantLoad(url & ": could not request configuration") - - loadConfigStream(url, stream) - -template loadConfigStream*(name: string, stream: Stream) = - try: - newCon4m = stream.readAll() - if selfChalk.extract != nil: - # Overwrite what we extracted, as it'll get "preserved" when - # writing out the chalk file. - selfChalk.extract["$CHALK_CONFIG"] = pack(newCon4m) - else: - selfChalk.collectedData["$CHALK_CONFIG"] = pack(newCon4m) - - except: - dumpExOnDebug() - cantLoad(name & ": could not read configuration") - - finally: - stream.close() - -template testConfigFile*(filename: string, newCon4m: string) = - info(filename & ": Validating configuration.") - if chalkConfig.getValidationWarning(): +proc testConfigFile*(uri: string, newCon4m: string) = + info(uri & ": Validating configuration.") + if chalkConfig.loadConfig.getValidationWarning(): warn("Note: validation involves creating a new configuration context" & " and evaluating your code to make sure it at least evaluates " & "fine on a default path. subscribe() and unsubscribe() will " & @@ -214,17 +189,130 @@ template testConfigFile*(filename: string, newCon4m: string) = addSpecLoad(chalkSpecName, toStream(chalkC42Spec)). addConfLoad(baseConfName, toStream(baseConfig)). setErrorHandler(newConfFileError). - addConfLoad(ioConfName, toStream(ioConfig)) + addConfLoad(ioConfName, toStream(ioConfig)). + addConfLoad(attestConfName, toStream(attestConfig)). + addConfLoad(sbomConfName, toStream(sbomConfig)). + addConfLoad(sastConfName, toStream(sastConfig)) try: # Test Run will cause (un)subscribe() to ignore subscriptions, and # will suppress log messages, etc. stack.run() startTestRun() - stack.addConfLoad(filename, toStream(newCon4m)).run() + stack.addConfLoad(uri, toStream(newCon4m)).run() endTestRun() if stack.errored: quit(1) + info(uri & ": Configuration successfully validated.") except: dumpExOnDebug() - error(getCurrentExceptionMsg() & "\n") - quit(1) + cantLoad(getCurrentExceptionMsg() & "\n") + +proc paramsToBox(a: bool, b, c: string, d: Con4mType, e: Box): Box = + # Though you can pack / unpack con4m types, we don't have a JSON + # mapping for them, so it's best for now to just pack the string + # repr and re-parse it on the other end. + var arr = @[ pack(a), pack(b), pack(c), pack($(d)), e ] + return pack(arr) + +const nocache = ["configs/ioconfig.c4m", "configs/sastconfig.c4m", + "[embedded config]", "configs/base_*.c4m", + "configs/sbomconfig.c4m", "configs/attestation.c4m", + "configs/getopts.c4m"] + +proc handleConfigLoad*(inpath: string) = + assert selfChalk != nil + + var path: string + + if inpath.endswith(".c4m"): + path = inpath + else: + path = inpath & ".c4m" + + if fileExists(path): + path = inpath.resolvePath() + else: + path = inpath + let + runtime = getChalkRuntime() + alreadyCached = haveComponentFromUrl(runtime, path).isSome() + (uri, module, _) = path.fullUrlToParts() + curConfOpt = selfChalkGetKey("$CHALK_CONFIG") + + var + component: ComponentInfo + replace: bool + + try: + component = runtime.loadComponentFromUrl(path) + replace = chalkConfig.loadConfig.getReplaceConf() + + except: + dumpExOnDebug() + cantLoad(getCurrentExceptionMsg() & "\n") + + var + toConfigure = component.getUsedComponents(paramOnly = true) + newEmbedded: string + + if replace or curConfOpt.isNone(): + newEmbedded = "" + else: + newEmbedded = unpack[string](curConfOpt.get()) + + if not alreadyCached: + if not replace: + if not newEmbedded.endswith("\n"): + newEmbedded.add("\n") + + newEmbedded.add("use " & module & " from \"" & uri & "\"\n") + else: + newEmbedded = component.source + + if len(toConfigure) == 0: + info("Attempting to replace base configuration from: " & path) + else: + info("Attempting to load configuration module from: " & path) + runtime.basicConfigureParameters(component, toConfigure) + + if replace or alreadyCached == false: + # If we just reconfigured a component, then we don't bother testing. + if chalkConfig.loadConfig.getValidateConfigsOnLoad(): + testConfigFile(path, newEmbedded) + else: + warn("Skipping configuration validation. This could break chalk.") + + selfChalkSetKey("$CHALK_CONFIG", pack(newEmbedded)) + + # Now, load the code cache. + var cachedCode = OrderedTableRef[string, string]() + + for _, onecomp in runtime.components: + if onecomp.url in nocache: + continue + if replace and onecomp == component: + continue + if onecomp.source != "": + cachedCode[onecomp.url] = onecomp.source + + # Load any saved parameters. + var + allComponents = runtime.programRoot.getUsedComponents() + params: seq[Box] + + for item in toConfigure: + if item notin allComponents: + allComponents.add(item) + + for component in allComponents: + for _, v in component.varParams: + params.add(paramsToBox(false, component.url, v.name, v.defaultType, + v.value.get())) + + + for _, v in component.attrParams: + params.add(paramsToBox(true, component.url, v.name, v.defaultType, + v.value.get())) + + selfChalkSetKey("$CHALK_COMPONENT_CACHE", pack(cachedCode)) + selfChalkSetKey("$CHALK_SAVED_COMPONENT_PARAMETERS", pack(params)) diff --git a/tests/chalk/runner.py b/tests/chalk/runner.py index f8b29fe9..0763ea4e 100644 --- a/tests/chalk/runner.py +++ b/tests/chalk/runner.py @@ -208,6 +208,7 @@ def run( virtual: bool = False, debug: bool = False, heartbeat: bool = False, + replace: bool = False, log_level: Optional[ChalkLogLevel] = None, exec_command: Optional[str | Path] = None, as_parent: Optional[bool] = None, @@ -241,6 +242,8 @@ def run( cmd += ["--chalk-as-parent"] if heartbeat: cmd += ["--heartbeat"] + if replace: + cmd += ["--replace"] if debug: cmd += ["--debug"] if no_color: @@ -342,15 +345,18 @@ def load( self, config: Path | str, *, + replace: bool = True, use_embedded: bool = False, expected_success: bool = True, ignore_errors: bool = False, + log_level: ChalkLogLevel = "error", ) -> ChalkProgram: hash = sha256(self.binary) result = self.run( command="load", params=[str(config)], - log_level="error", + log_level=log_level, + replace=replace, use_embedded=use_embedded, expected_success=expected_success, ignore_errors=ignore_errors, diff --git a/tests/conf.py b/tests/conf.py index 7f30e11d..a68b804e 100644 --- a/tests/conf.py +++ b/tests/conf.py @@ -31,6 +31,7 @@ # insecure registry https://docs.docker.com/registry/insecure/ REGISTRY = "localhost:5044" +SERVER_CHALKDUST = "https://chalkdust.io" SERVER_IMDS = "http://169.254.169.254" SERVER_STATIC = "http://static:8000" SERVER_HTTP = "http://chalk.local:8585" diff --git a/tests/conftest.py b/tests/conftest.py index 656dc9c6..e2df3ed6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,6 +19,7 @@ from .conf import ( GDB_PATH, SERVER_CERT, + SERVER_CHALKDUST, SERVER_DB, SERVER_HTTP, SERVER_HTTPS, @@ -141,7 +142,7 @@ def chalk( chalk = Chalk(binary=tmp) # sanity check assert chalk.binary and chalk.binary.is_file() - chalk.load(Path(__file__).parent / "testing.conf", use_embedded=False) + chalk.load(Path(__file__).parent / "testing.c4m", use_embedded=False) yield chalk @@ -220,3 +221,8 @@ def server_static(): if not is_server_up(f"{SERVER_STATIC}/conftest.py"): pytest.skip(f"{SERVER_STATIC} is down. skipping test") return SERVER_STATIC + + +@pytest.fixture() +def server_chalkdust(): + return SERVER_CHALKDUST diff --git a/tests/data/configs/docker_heartbeat.conf b/tests/data/configs/docker_heartbeat.c4m similarity index 100% rename from tests/data/configs/docker_heartbeat.conf rename to tests/data/configs/docker_heartbeat.c4m diff --git a/tests/data/configs/docker_wrap.conf b/tests/data/configs/docker_wrap.c4m similarity index 100% rename from tests/data/configs/docker_wrap.conf rename to tests/data/configs/docker_wrap.c4m diff --git a/tests/data/configs/heartbeat.conf b/tests/data/configs/heartbeat.c4m similarity index 100% rename from tests/data/configs/heartbeat.conf rename to tests/data/configs/heartbeat.c4m diff --git a/tests/data/configs/imds.conf b/tests/data/configs/imds.c4m similarity index 100% rename from tests/data/configs/imds.conf rename to tests/data/configs/imds.c4m diff --git a/tests/data/configs/nosecretmanager.conf b/tests/data/configs/nosecretmanager.c4m similarity index 100% rename from tests/data/configs/nosecretmanager.conf rename to tests/data/configs/nosecretmanager.c4m diff --git a/tests/data/configs/profiles/default.conf b/tests/data/configs/profiles/default.c4m similarity index 100% rename from tests/data/configs/profiles/default.conf rename to tests/data/configs/profiles/default.c4m diff --git a/tests/data/configs/profiles/empty_profile.conf b/tests/data/configs/profiles/empty_profile.c4m similarity index 100% rename from tests/data/configs/profiles/empty_profile.conf rename to tests/data/configs/profiles/empty_profile.c4m diff --git a/tests/data/configs/profiles/large_profile.conf b/tests/data/configs/profiles/large_profile.c4m similarity index 100% rename from tests/data/configs/profiles/large_profile.conf rename to tests/data/configs/profiles/large_profile.c4m diff --git a/tests/data/configs/profiles/minimal_profile.conf b/tests/data/configs/profiles/minimal_profile.c4m similarity index 100% rename from tests/data/configs/profiles/minimal_profile.conf rename to tests/data/configs/profiles/minimal_profile.c4m diff --git a/tests/data/configs/validation/custom_report.conf b/tests/data/configs/validation/custom_report.c4m similarity index 100% rename from tests/data/configs/validation/custom_report.conf rename to tests/data/configs/validation/custom_report.c4m diff --git a/tests/data/configs/validation/default.conf b/tests/data/configs/validation/default.c4m similarity index 100% rename from tests/data/configs/validation/default.conf rename to tests/data/configs/validation/default.c4m diff --git a/tests/data/configs/validation/invalid_1.conf b/tests/data/configs/validation/invalid_1.c4m similarity index 100% rename from tests/data/configs/validation/invalid_1.conf rename to tests/data/configs/validation/invalid_1.c4m diff --git a/tests/data/configs/validation/invalid_2.conf b/tests/data/configs/validation/invalid_2.c4m similarity index 100% rename from tests/data/configs/validation/invalid_2.conf rename to tests/data/configs/validation/invalid_2.c4m diff --git a/tests/data/configs/validation/valid_1.conf b/tests/data/configs/validation/valid_1.c4m similarity index 100% rename from tests/data/configs/validation/valid_1.conf rename to tests/data/configs/validation/valid_1.c4m diff --git a/tests/data/sink_configs/file.conf b/tests/data/sink_configs/file.c4m similarity index 100% rename from tests/data/sink_configs/file.conf rename to tests/data/sink_configs/file.c4m diff --git a/tests/data/sink_configs/post.conf b/tests/data/sink_configs/post.c4m similarity index 100% rename from tests/data/sink_configs/post.conf rename to tests/data/sink_configs/post.c4m diff --git a/tests/data/sink_configs/post_http_local.conf b/tests/data/sink_configs/post_http_local.c4m similarity index 100% rename from tests/data/sink_configs/post_http_local.conf rename to tests/data/sink_configs/post_http_local.c4m diff --git a/tests/data/sink_configs/post_https_local.conf b/tests/data/sink_configs/post_https_local.c4m similarity index 100% rename from tests/data/sink_configs/post_https_local.conf rename to tests/data/sink_configs/post_https_local.c4m diff --git a/tests/data/sink_configs/rotating_log.conf b/tests/data/sink_configs/rotating_log.c4m similarity index 100% rename from tests/data/sink_configs/rotating_log.conf rename to tests/data/sink_configs/rotating_log.c4m diff --git a/tests/data/sink_configs/s3.conf b/tests/data/sink_configs/s3.c4m similarity index 100% rename from tests/data/sink_configs/s3.conf rename to tests/data/sink_configs/s3.c4m diff --git a/tests/test_command.py b/tests/test_command.py index 6aa90dfb..4e6806c4 100644 --- a/tests/test_command.py +++ b/tests/test_command.py @@ -186,7 +186,7 @@ def test_setup(chalk_copy: Chalk): command="setup", no_api_login=True, log_level="error", - config=CONFIGS / "nosecretmanager.conf", + config=CONFIGS / "nosecretmanager.c4m", ) report = result.report diff --git a/tests/test_config.py b/tests/test_config.py index b6527b37..8fb0468c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -29,7 +29,7 @@ @pytest.mark.parametrize("use_embedded", [True, False]) def test_dump_load(tmp_data_dir: Path, chalk_copy: Chalk, use_embedded: bool): # output for updated config - tmp_conf = tmp_data_dir / "testconf.conf" + tmp_conf = tmp_data_dir / "testconf.c4m" # dump config to file chalk_copy.dump(tmp_conf) @@ -41,17 +41,17 @@ def test_dump_load(tmp_data_dir: Path, chalk_copy: Chalk, use_embedded: bool): result = chalk_copy.load( tmp_conf, use_embedded=use_embedded, expected_success=False ) - assert VALIDATION_ERROR in result.logs + assert VALIDATION_ERROR in " ".join(i.strip() for i in result.logs.splitlines()) # sanity check that default config has not changed -# if it has then the thing to do is usually update "default.conf" with the new default config +# if it has then the thing to do is usually update "default.c4m" with the new default config # this test is mainly to catch any default changes that might impact other tests, or if the default config loaded to the binary is wrong -@pytest.mark.parametrize("test_config_file", ["validation/default.conf"]) +@pytest.mark.parametrize("test_config_file", ["validation/default.c4m"]) def test_default_config( tmp_data_dir: Path, chalk_default: Chalk, test_config_file: str ): - tmp_conf = tmp_data_dir / "testconf.conf" + tmp_conf = tmp_data_dir / "testconf.c4m" # dump config to file chalk_default.dump(tmp_conf) @@ -62,8 +62,8 @@ def test_default_config( @pytest.mark.parametrize( "test_config_file", [ - "validation/invalid_1.conf", - "validation/invalid_2.conf", + "validation/invalid_1.c4m", + "validation/invalid_2.c4m", ], ) @pytest.mark.parametrize( @@ -80,8 +80,6 @@ def test_invalid_load(chalk_copy: Chalk, test_config_file: str, use_embedded: bo use_embedded=use_embedded, expected_success=False, ) - assert all(x in load.logs for x in parse_error) - # chalk should still have default config embedded # and further calls should not fail and not have any errors extract = chalk_copy.extract(chalk_copy.binary) @@ -93,7 +91,7 @@ def test_invalid_load(chalk_copy: Chalk, test_config_file: str, use_embedded: bo @pytest.mark.parametrize( "test_config_file, expected_error", [ - ("validation/valid_1.conf", VALIDATION_ERROR), + ("validation/valid_1.c4m", VALIDATION_ERROR), ], ) @pytest.mark.parametrize( @@ -128,15 +126,18 @@ def test_valid_load( @pytest.mark.parametrize( "path, expected_success", [ - ("data/configs/validation/valid_1.conf", True), - ("data/configs/validation/invalid_1.conf", False), + ("demo-http.c4m", True), ("nonexisting", False), ], ) def test_load_url( - chalk_copy: Chalk, server_static: str, path: str, expected_success: bool + chalk_copy: Chalk, server_chalkdust: str, path: str, expected_success: bool ): - chalk_copy.load(f"{server_static}/{path}", expected_success=expected_success) + chalk_copy.load( + f"{server_chalkdust}/{path}", + log_level="trace", + expected_success=expected_success, + ) # tests for configs that are found in the chalk search path @@ -144,12 +145,12 @@ def test_load_url( # as these configs are global across the system, # test needs to be exclusive so nothing else executes in parallel @pytest.mark.exclusive -@pytest.mark.parametrize("tmp_file", [{"path": "/etc/chalk.conf"}], indirect=True) +@pytest.mark.parametrize("tmp_file", [{"path": "/etc/chalk.c4m"}], indirect=True) @pytest.mark.parametrize( "config_path, expected_success, expected_error", [ - ("validation/valid_1.conf", True, VALIDATION_ERROR), - ("validation/invalid_1.conf", False, ""), + ("validation/valid_1.c4m", True, VALIDATION_ERROR), + ("validation/invalid_1.c4m", False, ""), ], ) def test_external_configs( @@ -184,9 +185,7 @@ def test_external_configs( assert expected_error in result_external.logs -@pytest.mark.parametrize( - "test_config_file", [CONFIGS / "validation/custom_report.conf"] -) +@pytest.mark.parametrize("test_config_file", [CONFIGS / "validation/custom_report.c4m"]) @pytest.mark.parametrize("copy_files", [[LS_PATH]], indirect=True) def test_custom_report( chalk_copy: Chalk, @@ -414,10 +413,10 @@ def validate_report_keys(report: dict[str, Any], expected_keys: set[str]): @pytest.mark.parametrize( "test_config_file", [ - ("profiles/empty_profile.conf"), - ("profiles/default.conf"), - ("profiles/minimal_profile.conf"), - ("profiles/large_profile.conf"), + ("profiles/empty_profile.c4m"), + ("profiles/default.c4m"), + ("profiles/minimal_profile.c4m"), + ("profiles/large_profile.c4m"), ], ) @pytest.mark.parametrize( diff --git a/tests/test_docker.py b/tests/test_docker.py index 549a96e1..afbcf109 100644 --- a/tests/test_docker.py +++ b/tests/test_docker.py @@ -95,7 +95,7 @@ def test_build( cwd=cwd, tag=random_hex if tag else None, buildkit=buildkit, - config=CONFIGS / "docker_wrap.conf", + config=CONFIGS / "docker_wrap.c4m", ) assert image_id @@ -135,7 +135,7 @@ def test_composite_build( dockerfile=test, buildkit=buildkit, args={"BASE": random_hex}, - config=CONFIGS / "docker_wrap.conf", + config=CONFIGS / "docker_wrap.c4m", expecting_report=buildkit, ) assert second_image_id @@ -228,7 +228,7 @@ def test_nonvirtual_valid(chalk: Chalk, test_file: str, random_hex: str): image_hash, build = chalk.docker_build( dockerfile=DOCKERFILES / test_file / "Dockerfile", tag=tag, - config=CONFIGS / "docker_wrap.conf", + config=CONFIGS / "docker_wrap.c4m", ) # artifact is the docker image @@ -278,7 +278,7 @@ def test_nonvirtual_invalid(chalk: Chalk, test_file: str, random_hex: str): @pytest.mark.slow() def test_docker_heartbeat(chalk_copy: Chalk, random_hex: str): tag = f"test_image_{random_hex}" - chalk_copy.load(CONFIGS / "docker_heartbeat.conf", use_embedded=False) + chalk_copy.load(CONFIGS / "docker_heartbeat.c4m", use_embedded=False) # build dockerfile with chalk docker entrypoint wrapping chalk_copy.docker_build( @@ -310,7 +310,7 @@ def test_docker_labels(chalk: Chalk, random_hex: str): chalk.docker_build( dockerfile=DOCKERFILES / "valid" / "sample_1" / "Dockerfile", tag=tag, - config=CONFIGS / "docker_heartbeat.conf", + config=CONFIGS / "docker_heartbeat.c4m", ) inspected = Docker.inspect(tag) diff --git a/tests/test_exec.py b/tests/test_exec.py index b67e3926..f3604026 100644 --- a/tests/test_exec.py +++ b/tests/test_exec.py @@ -117,7 +117,7 @@ def test_exec_heartbeat( result = chalk.run( command="exec", heartbeat=True, - config=CONFIGS / "heartbeat.conf", + config=CONFIGS / "heartbeat.c4m", exec_command=bin_path, params=["5"], ) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index f58d596d..0f6ca8b2 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -140,7 +140,7 @@ def test_imds( with tmp_file as fid: fid.write(b"amazon") bin_path = copy_files[0] - insert = chalk.insert(bin_path, config=CONFIGS / "imds.conf") + insert = chalk.insert(bin_path, config=CONFIGS / "imds.c4m") assert insert.report.contains( { "_AWS_AMI_ID": "ami-0abcdef1234567890", diff --git a/tests/test_sink.py b/tests/test_sink.py index e37df295..abdacc28 100644 --- a/tests/test_sink.py +++ b/tests/test_sink.py @@ -56,7 +56,7 @@ def test_file_present(tmp_data_dir: Path, chalk: Chalk, copy_files: list[Path]): os.utime(file_output_path, None) assert file_output_path.is_file(), "file sink path must be a valid path" - config = SINK_CONFIGS / "file.conf" + config = SINK_CONFIGS / "file.c4m" chalk.insert(config=config, artifact=artifact) # check that file output is correct @@ -82,7 +82,7 @@ def test_rotating_log(tmp_data_dir: Path, copy_files: list[Path], chalk: Chalk): rotating_log_output_path = Path(os.environ["SINK_TEST_OUTPUT_ROTATING_LOG"]) rotating_log_output_path.unlink(missing_ok=True) - config = SINK_CONFIGS / "rotating_log.conf" + config = SINK_CONFIGS / "rotating_log.c4m" chalk.insert(config=config, artifact=artifact) # check that file output is correct @@ -113,7 +113,7 @@ def test_s3(tmp_data_dir: Path, copy_files: list[Path], chalk: Chalk): # basic validation of s3 env vars assert os.environ["AWS_S3_BUCKET_URI"], "s3 bucket uri must not be empty" - config = SINK_CONFIGS / "s3.conf" + config = SINK_CONFIGS / "s3.c4m" proc = chalk.insert(config=config, artifact=artifact, log_level="info") # expecting log line from chalk of form @@ -153,7 +153,7 @@ def test_post_http_fastapi( _test_server( artifact=copy_files[0], chalk=chalk, - conf="post_https_local.conf", + conf="post_https_local.c4m", url=server_http, server_sql=server_sql, verify=None, @@ -180,7 +180,7 @@ def test_post_https_fastapi( _test_server( artifact=copy_files[0], chalk=chalk, - conf="post_https_local.conf", + conf="post_https_local.c4m", url=server_https, server_sql=server_sql, verify=server_cert, diff --git a/tests/testing.c4m b/tests/testing.c4m new file mode 100644 index 00000000..1a7d2a0a --- /dev/null +++ b/tests/testing.c4m @@ -0,0 +1 @@ +subscribe("report", "json_console_out") diff --git a/tests/testing.conf b/tests/testing.conf deleted file mode 100644 index 59e03a62..00000000 --- a/tests/testing.conf +++ /dev/null @@ -1 +0,0 @@ -subscribe("report", "json_console_out") \ No newline at end of file