Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bundling mozilla root CA store for when there are no default certs #45

Merged
merged 3 commits into from
Feb 14, 2024
Merged

bundling mozilla root CA store for when there are no default certs #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4f084c4
bundling mozilla root CA store for when there are no default certs
miki725 Feb 8, 2024
8c89804
style: addressing review comments
miki725 Feb 12, 2024
5daa9b9
refactor: switching to CompileDate vs subprocess date
miki725 Feb 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions nimutils/awsclient.nim
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,9 @@ proc newAwsClient*(creds: AwsCredentials, region,
service: string): AwsClient =
let
# TODO - use some kind of template and compile-time variable to put the correct kernel used to build the sdk in the UA?
httpclient = newHttpClient("nimaws-sdk/0.3.3; "&defUserAgent.replace(" ",
"-").toLower&"; darwin/16.7.0")
httpclient = createHttpClient(
userAgent = "nimaws-sdk/0.3.3; "&defUserAgent.replace(" ", "-").toLower&"; darwin/16.7.0",
)
scope = AwsScope(date: getAmzDateString(), region: region, service: service)

return AwsClient(httpClient: httpclient, credentials: creds, scope: scope,
Expand Down
123 changes: 117 additions & 6 deletions nimutils/net.nim
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,46 @@
import std/[asyncfutures, net, httpclient, uri, math, os]
import std/[asyncfutures, net, httpclient, uri, math, os, streams, strutils]
import openssl
import ./managedtmp

proc getRootCAStoreContent(): string =
const
caWiki = "https://wiki.mozilla.org/CA/Included_Certificates"
# link is taken directly from wiki page above
# p.s. kind of odd its to salesforce vs one of mozilla-owned domains :shrug:
caURL = "https://ccadb.my.salesforce-sites.com/mozilla/IncludedRootsPEMTxt?TrustBitsInclude=Websites"
cache = "mozilla-root-store-" & CompileDate # cache certs by day
curlCmd = "curl -fsSL --retry 5 " & caURL
(contents, curlExitCode) = gorgeEx(curlCmd, cache=cache)
if curlExitCode != 0:
raise newException(
ValueError,
"Could not download CA root store: " & contents
)
const
opensslCmd = "openssl storeutl -noout -certs /dev/stdin"
(check, checkExitCode) = gorgeEx(opensslCmd, input=contents)
checkLines = check.splitLines()
if checkExitCode != 0:
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Non-blocking comment: this is a relatively costly split, since it allocates a string for each line (3500 ish of them) at compile time - it's calling the splitLines proc, not the iterator.

It probably doesn't have a significant impact on compile times, but it'd be better to get the index of the start of the last line, and slice from there to the end.

raise newException(
ValueError,
"Could not validate CA root store certificates. " &
"Maybe server didnt return valid PEM file? " &
check
)
echo("Embedding Mozilla Root CA store with certificates " & checkLines[^1].toLower())
echo("For more information see " & caWiki)
contents

var tmpCAStore = ""
proc getCAStorePath(): string =
const contents = getRootCAStoreContent()
if tmpCAStore != "":
return tmpCAStore
let (stream, tmp) = getNewTempFile("cabundle", ".pem")
stream.write(contents)
stream.close()
tmpCAStore = tmp
return tmp

{.emit: """
#include <stdlib.h>
Expand All @@ -9,7 +51,6 @@ import std/[asyncfutures, net, httpclient, uri, math, os]

#include <stdio.h>


// Cloudflare DNS
const char * dummy_dst = "1.1.1.1";
const int dummy_port = 53;
Expand Down Expand Up @@ -57,11 +98,10 @@ proc timeoutGuard(client: HttpClient | AsyncHttpClient, url: Uri | string) =
# TCP connection can be established before attempting to make
# HTTP request
if client.timeout > 0:
var uri: Uri
when url is string:
uri = parseUri(url)
let uri = when url is string:
parseUri(url)
else:
uri = url
url
let hostname = uri.hostname
# port is optional in the Uri so we use default ports
var port: Port
Expand Down Expand Up @@ -118,3 +158,74 @@ proc safeRequest*(client: HttpClient,
withRetry(retries, firstRetryDelayMs):
return client.request(url = url, httpMethod = httpMethod, body = body,
headers = headers, multipart = multipart)

# https://github.com/nim-lang/Nim/blob/a45f43da3407dbbf8ecd15ce8ecb361af677add7/lib/pure/httpclient.nim#L380-L386
# similar to stdlib but defaults to bundled CAs
proc getSSLContext(caFile: string = ""): SslContext =
if caFile != "":
# note when caFile is provided there is no try..except
# otherwise we would silently fail to bundled CA root store
# if caFile is invalid/does not exist
return newContext(verifyMode = CVerifyPeer, caFile = caFile)
else:
try:
return newContext(verifyMode = CVerifyPeer)
except:
return newContext(verifyMode = CVerifyPeer, caFile = getCAStorePath())

proc createHttpClient*(uri: Uri = parseUri(""),
maxRedirects: int = 3,
timeout: int = 1000, # in ms - 1 second
pinnedCert: string = "",
disallowHttp: bool = false,
userAgent: string = defUserAgent,
): HttpClient =
var context: SslContext

if uri.scheme in @["", "https"]:
context = getSSLContext(caFile = pinnedCert)
else:
if disallowHttp:
raise newException(ValueError, "http:// URLs not allowed (only https).")
elif pinnedCert != "":
raise newException(ValueError, "Pinned cert not allowed with http " &
"URL (only https).")

let client = newHttpClient(sslContext = context,
userAgent = userAgent,
timeout = timeout,
maxRedirects = maxRedirects)

if client == nil:
raise newException(ValueError, "Invalid HTTP configuration")

return client

proc safeRequest*(url: Uri | string,
httpMethod: HttpMethod | string = HttpGet,
body = "",
headers: HttpHeaders = nil,
multipart: MultipartData = nil,
retries: int = 0,
firstRetryDelayMs: int = 0,
timeout: int = 1000,
pinnedCert: string = "",
maxRedirects: int = 3,
disallowHttp: bool = false,
): Response =
let uri = when url is string:
parseUri(url)
else:
url
let client = createHttpClient(uri = uri,
maxRedirects = maxRedirects,
timeout = timeout,
pinnedCert = pinnedCert,
disallowHttp = disallowHttp)
return client.safeRequest(url = uri,
httpMethod = httpMethod,
body = body,
headers = headers,
multipart = multipart,
retries = retries,
firstRetryDelayMs = firstRetryDelayMs)
100 changes: 43 additions & 57 deletions nimutils/sinks.nim
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

import streams, tables, options, os, strutils, std/[net, uri, httpclient],
s3client, pubsub, misc, random, encodings, std/tempfiles,
parseutils, openssl, file, std/asyncfutures, net
parseutils, file, std/asyncfutures, net

const defaultLogSearchPath = @["/var/log/", "~/.log/", "."]

Expand Down Expand Up @@ -304,10 +304,7 @@ proc s3SinkOut(msg: string, cfg: SinkConfig, t: Topic, ignored: StringTable) =
else:
cfg.iolog(t, "Post to: " & newPath & "; response = " & response.status)

proc SSL_CTX_load_verify_file(ctx: SslCtx, CAfile: cstring):
cint {.cdecl, dynlib: DLLSSLName, importc.}

proc httpHeadersForSink(cfg: SinkConfig): HttpHeaders =
proc httpHeaders(cfg: SinkConfig): HttpHeaders =
var
tups: seq[(string, string)] = @[]
contentType: string = cfg.params["content_type"]
Expand Down Expand Up @@ -336,16 +333,20 @@ proc httpHeadersForSink(cfg: SinkConfig): HttpHeaders =

return headers

template httpUriForSink(cfg: SinkConfig): Uri =
parseURI(cfg.params["uri"])

proc httpClientForSink(cfg: SinkConfig, uri: Uri, maxRedirects: int = 5): HttpClient =
proc httpParams(cfg: SinkConfig): tuple[
uri: Uri,
headers: HttpHeaders,
timeout: int,
disallowHttp: bool,
pinnedCert: string,
] =
let
uri = parseURI(cfg.params["uri"])
headers = cfg.httpHeaders()
disallowHttp = "disallow_http" in cfg.params
var
client: HttpClient
timeout: int
pinnedCert: string = ""
context: SslContext

timeout = 1000 # in ms - 1 second
pinnedCert = ""
if "pinned_cert_file" in cfg.params:
pinnedCert = cfg.params["pinned_cert_file"]
if "timeout" in cfg.params:
Expand All @@ -355,38 +356,20 @@ proc httpClientForSink(cfg: SinkConfig, uri: Uri, maxRedirects: int = 5): HttpCl
"represented as an integer, or 0 for no timeout.")
elif timeout <= 0:
timeout = -1
else:
timeout = 1000 # in ms - 1 second

if uri.scheme == "https":
context = newContext(verifyMode = CVerifyPeer)
if pinnedCert != "":
discard context.context.SSL_CTX_load_verify_file(cstring(pinnedCert))
client = newHttpClient(sslContext=context, timeout=timeout, maxRedirects = maxRedirects)
else:
if "disallow_http" in cfg.params:
raise newException(ValueError, "http:// URLs not allowed (only https).")
elif pinnedCert != "":
raise newException(ValueError, "Pinned cert not allowed with http " &
"URL (only https).")
client = newHttpClient(sslContext=nil, timeout=timeout, maxRedirects = maxRedirects)

if client == nil:
raise newException(ValueError, "Invalid HTTP configuration")

return client
return (uri, headers, timeout, disallowHttp, pinnedCert)

proc postSinkOut(msg: string, cfg: SinkConfig, t: Topic, ignored: StringTable) =
let
headers = httpHeadersForSink(cfg)
uri = httpUriForSink(cfg)
client = httpClientForSink(cfg, uri)
response = client.safeRequest(url = uri,
httpMethod = HttpPost,
body = msg,
headers = headers,
retries = 2,
firstRetryDelayMs = 100)
params = cfg.httpParams()
response = safeRequest(url = params.uri,
timeout = params.timeout,
headers = params.headers,
disallowHttp = params.disallowHttp,
pinnedCert = params.pinnedCert,
httpMethod = HttpPost,
body = msg,
retries = 2,
firstRetryDelayMs = 100)

if not response.code.is2xx():
raise newException(ValueError, response.status & ": " & response.body())
Expand All @@ -395,21 +378,22 @@ proc postSinkOut(msg: string, cfg: SinkConfig, t: Topic, ignored: StringTable) =

proc presignSinkOut(msg: string, cfg: SinkConfig, t: Topic, ignored: StringTable) =
let
headers = httpHeadersForSink(cfg)
signUri = httpUriForSink(cfg)
params = cfg.httpParams()
# for the sign request, we do not want to send full request payload as:
# * we expect a redirect response
# * server might not accept large requests (hence presigning sink is used)
# * no need to waste bandwidth
# and will only send it to the returned signed URL
# which is why we disallow redirects here via maxRedirects
# NOTE this assumes that the endpoint immediately returns presigned URL
signClient = httpClientForSink(cfg, signUri, maxRedirects = 0)
signResponse = signClient.safeRequest(url = signUri,
httpMethod = HttpPut,
headers = headers,
retries = 2,
firstRetryDelayMs = 100)
signResponse = safeRequest(url = params.uri,
timeout = params.timeout,
headers = params.headers,
disallowHttp = params.disallowHttp,
pinnedCert = params.pinnedCert,
httpMethod = HttpPut,
retries = 2,
firstRetryDelayMs = 100)

if signResponse.code notin [Http302, Http307]:
raise newException(ValueError, "Presign requires 302/307 redirect but received: " & signResponse.status)
Expand All @@ -423,12 +407,14 @@ proc presignSinkOut(msg: string, cfg: SinkConfig, t: Topic, ignored: StringTable
raise newException(ValueError, "Presign edirect Location header needs to be absolute URL")

let
client = httpClientForSink(cfg, uri)
response = client.safeRequest(url = uri,
httpMethod = HttpPut,
body = msg,
retries = 2,
firstRetryDelayMs = 100)
response = safeRequest(url = uri,
timeout = params.timeout,
disallowHttp = params.disallowHttp,
pinnedCert = params.pinnedCert,
httpMethod = HttpPut,
body = msg,
retries = 2,
firstRetryDelayMs = 100)

if not response.code.is2xx():
raise newException(ValueError, response.status & ": " & response.body())
Expand Down
Loading