feat: falling back to mozilla CAs when system has no certs

this will allow chalk to communicate with external APIs even when system
has no certs such as in busybox containers

most of the change is in nimutils and then its bubbled up to con4m and
chalk

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## Main
 
+### New Features
+
+- Chalk falls back to bundled Mozilla CA Store when there
+  are no system TLS certs to use (e.g. busybox container).
+  [#196](https://github.com/crashappsec/chalk/pull/196)
+
 ### Fixes
 
 - Fixes possible exception when signing backup service

chalk.nimble

@@ -11,7 +11,7 @@ bin           = @["chalk"]
 
 # Dependencies
 requires "nim >= 2.0.0"
-requires "https://github.com/crashappsec/con4m#d2a081b93d72cad4a86bcd51bbf6fbe4934a845e"
+requires "https://github.com/crashappsec/con4m#ee3df100a930ba4f642ae314640123e742d8f529"
 requires "https://github.com/viega/zippy == 0.10.7" # MIT
 
 # this allows us to get version externally

src/attestation.nim

@@ -141,16 +141,14 @@ proc callTheSigningKeyBackupService(base:    string,
     authHeaders = auth.implementation.injectHeaders(auth, headers)
 
   # Call the API with authz header - rety twice with backoff
-  uri       = parseUri(url)
-  context   = newContext(verifyMode = CVerifyPeer)
-  client    = newHttpClient(sslContext = context, timeout = timeout)
   try:
-    response = client.safeRequest(url = uri,
-                                  httpMethod        = mth,
-                                  headers           = authHeaders,
-                                  body              = bodytxt,
-                                  retries           = 2,
-                                  firstRetryDelayMs = 100)
+    response = safeRequest(url = url,
+                           httpMethod        = mth,
+                           headers           = authHeaders,
+                           body              = bodytxt,
+                           timeout           = timeout,
+                           retries           = 2,
+                           firstRetryDelayMs = 100)
 
     trace("Signing Key Backup Service URL: " & $uri)
     trace("Signing Key Backup Service HTTP headers: " & $authHeaders)

src/plugins/awsEcs.nim

@@ -25,11 +25,9 @@ if ecsUrl == "":
 proc requestECSMetadata(path: string): Option[Box] =
   let url = ecsUrl & path
   var body = ""
-  info(url)
   try:
     var
-      client = newHttpClient()
-      resp   = client.safeRequest(url)
+      resp   = safeRequest(url)
     if resp.code != Http200:
       error("ecs: " & url & " returned " & resp.status)
       return none(Box)

src/plugins/cloudMetadata.nim

@@ -7,7 +7,7 @@
 
 ## Query common AWS metadata va IMDSv2
 
-import std/[httpclient, net, uri, strutils, json]
+import std/[httpclient, net, strutils, json]
 import ".."/[config, plugin_api, chalkjson]
 
 const
@@ -19,33 +19,36 @@ const
 
 proc getAwsToken(): Option[string] =
   let
-    uri      = parseURI(awsBaseUri & "api/token")
+    url      = awsBaseUri & "api/token"
     hdrs     = newHttpHeaders([("X-aws-ec2-metadata-token-ttl-seconds", "10")])
-    client   = newHttpClient(timeout = 250) # 1/4 of a second
-    response = client.safeRequest(url = uri, httpMethod = HttpPut, headers = hdrs)
+    response = safeRequest(url        = url,
+                           httpMethod = HttpPut,
+                           timeout    = 250, # 1/4 of a second
+                           headers    = hdrs)
 
   if not response.code.is2xx():
-    trace("Could not retrieve IMDSv2 token from: " & $uri)
+    trace("Could not retrieve IMDSv2 token from: " & url)
     return none(string)
 
   trace("Retrieved AWS metadata token")
   return some(response.bodyStream.readAll().strip())
 
 proc hitProviderEndpoint(path: string, hdrs: HttpHeaders): Option[string] =
   let
-    uri      = parseUri(path)
-    client   = newHttpClient(timeout = 250) # 1/4 of a second
-    response = client.safeRequest(url = uri, httpMethod = HttpGet, headers = hdrs)
+    response = safeRequest(url        = path,
+                           httpMethod = HttpGet,
+                           timeout    = 250, # 1/4 of a second
+                           headers    = hdrs)
 
   if not response.code.is2xx():
-    trace("Could not retrieve metadata from: " & $uri)
+    trace("Could not retrieve metadata from: " & path)
     return none(string)
 
-  trace("Retrieved metadata from: " & uri.path)
+  trace("Retrieved metadata from: " & path)
   result = some(response.bodyStream.readAll().strip())
   if not result.isSome():
     # log failing keys in trace mode only as some are expected to be absent
-    trace("Got empty metadata from: " & uri.path)
+    trace("Got empty metadata from: " & path)
 
 template oneItem(keyname: string, url: string) =
   if isSubscribedKey(keyname):

tests/test_config.py

@@ -4,12 +4,13 @@
 # (see https://crashoverride.com/docs/chalk)
 import itertools
 from pathlib import Path
-from typing import Any, IO, Iterator, Callable, Optional
+from typing import Any, Iterator, Callable, Optional
 
 import pytest
 
 from .chalk.runner import Chalk, ChalkMark, ChalkReport
 from .utils.log import get_logger
+from .utils.docker import Docker
 from .conf import (
     BASE_OUTCONF,
     LS_PATH,
@@ -484,3 +485,21 @@ def test_profiles(
     # delete
     delete = chalk_copy.delete(bin_path)
     validate_chalk_report_keys(delete.report, configs["delete"])
+
+
+def test_no_certs(chalk_default: Chalk, server_chalkdust: str):
+    """
+    chalk should be able to connect to chalkdust even when system has no system certs
+    by using bundled mozilla root CA store
+    """
+    assert Docker.run(
+        # busybox does not ship with any system certs vs for example alpine
+        image="busybox",
+        entrypoint="/bin/sh",
+        params=[
+            "-c",
+            f"cp /chalk /chalk.test && /chalk.test load {server_chalkdust}/debug.c4m",
+        ],
+        tty=False,
+        volumes={chalk_default.binary: "/chalk"},
+    )

tests/test_sink.py

@@ -5,7 +5,6 @@
 import json
 from pathlib import Path
 from typing import Any, Callable
-from unittest import mock
 
 import boto3
 import os