Merge pull request #757 from jakemas/update-pq-branch-01-17-23

Update pq branch 01 17 23

.gitignore

@@ -12,6 +12,8 @@ doc/*.html
 doc/doc.css
 bindings/rust/tmp/*
 bindings/rust/generate/target
+!bindings/rust/aws-lc-sys-template/build
+!bindings/rust/aws-lc-fips-sys-template/build
 
 util/bot/android_ndk
 util/bot/android_sdk/public
@@ -24,6 +26,7 @@ util/bot/libcxx
 util/bot/libcxxabi
 util/bot/llvm-build
 util/bot/nasm-win32.exe
+util/bot/ninja
 util/bot/perl-win32
 util/bot/perl-win32.zip
 util/bot/sde-linux64

CMakeLists.txt

@@ -193,11 +193,17 @@ macro(check_compiler file_to_test flag_to_set)
           COMPILE_DEFINITIONS "-Werror"
           OUTPUT_VARIABLE ERROR_MESSAGE)
   if(RESULT)
-    set(C_CXX_FLAGS "${C_CXX_FLAGS} -D${flag_to_set}")
-    message(STATUS "Test ${file_to_test} passed, enabling ${flag_to_set}")
+    set(COMPILER_CHECK_FLAGS "${COMPILER_CHECK_FLAGS} -D${flag_to_set}")
+    message(STATUS "${file_to_test} probe is positive, enabling ${flag_to_set}")
   else()
-    message(STATUS "Test ${file_to_test} failed, NOT enabling ${flag_to_set}:")
-    message(STATUS "    ${ERROR_MESSAGE}")
+    message(STATUS "${file_to_test} probe is negative, NOT enabling ${flag_to_set}:")
+    # Some build applications use regexes on build output to highlight build
+    # errors. Below, we modify a compiler error message to avoid that a negative
+    # probe is considered an error in such build applications.
+    string(REPLACE ": error:"
+      ": compiler_error:" ERROR_MESSAGE_PROCESSED
+      ${ERROR_MESSAGE})
+    message(STATUS "    ${ERROR_MESSAGE_PROCESSED}")
   endif()
 endmacro()
 
@@ -260,11 +266,6 @@ if (GCC)
 endif ()
 
 if(GCC OR CLANG)
-  check_compiler("stdalign_check.c" AWS_LC_STDALIGN_AVAILABLE)
-  check_compiler("builtin_swap_check.c" AWS_LC_BUILTIN_SWAP_SUPPORTED)
-  if(FIPS AND NOT APPLE)
-    check_compiler("linux_u32.c" AWS_LC_URANDOM_U32)
-  endif()
   # Note clang-cl is odd and sets both CLANG and MSVC. We base our configuration
   # primarily on our normal Clang one.
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
@@ -359,6 +360,15 @@ if(GCC OR CLANG)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-array-bounds")
   endif()
 
+  check_compiler("stdalign_check.c" AWS_LC_STDALIGN_AVAILABLE)
+  check_compiler("builtin_swap_check.c" AWS_LC_BUILTIN_SWAP_SUPPORTED)
+  if(FIPS AND NOT APPLE)
+    check_compiler("linux_u32.c" AWS_LC_URANDOM_U32)
+  endif()
+
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_CHECK_FLAGS}")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_CHECK_FLAGS}")
+
 elseif(MSVC)
   set(MSVC_DISABLED_WARNINGS_LIST
       "C4061" # enumerator 'identifier' in switch of enum 'enumeration' is not
@@ -692,20 +702,13 @@ else()
   set(ARCH "generic")
 endif()
 
-if(ANDROID AND NOT ANDROID_NDK_REVISION AND ARCH STREQUAL "arm")
-  # The third-party Android-NDK CMake files somehow fail to set the -march flag
-  # for assembly files. Without this flag, the compiler believes that it's
-  # building for ARMv5.
-  set(CMAKE_ASM_FLAGS "-march=${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_ASM_FLAGS}")
-endif()
-
 if(USE_CUSTOM_LIBCXX)
   if(NOT CLANG)
     message(FATAL_ERROR "USE_CUSTOM_LIBCXX only supported with Clang")
   endif()
 
-  # The docker images set an environement variable to the llvm project directory which the sandbox builds will use,
-  # you can also pass in the llvm project path as a CMake parameter which takes precedance over the environment variable
+  # The docker images set an environment variable to the llvm project directory which the sandbox builds will use,
+  # you can also pass in the llvm project path as a CMake parameter which takes precedence over the environment variable
   if(DEFINED ENV{LLVM_PROJECT_HOME} AND NOT LLVM_PROJECT_HOME)
     set(LLVM_PROJECT_HOME $ENV{LLVM_PROJECT_HOME})
   endif()
@@ -729,7 +732,7 @@ if(USE_CUSTOM_LIBCXX)
   # This is patterned after buildtools/third_party/libc++/BUILD.gn and
   # buildtools/third_party/libc++abi/BUILD.gn in Chromium.
 
-  file(GLOB LIBCXX_SOURCES "${LLVM_PROJECT_HOME}/libcxx/src/*.cpp")
+  file(GLOB LIBCXX_SOURCES "${LLVM_PROJECT_HOME}/libcxx/src/*.cpp" "${LLVM_PROJECT_HOME}/libcxx/src/ryu/*.cpp")
   file(GLOB LIBCXXABI_SOURCES "${LLVM_PROJECT_HOME}/libcxxabi/src/*.cpp")
 
   # This file is meant for exception-less builds.
@@ -748,9 +751,6 @@ if(USE_CUSTOM_LIBCXX)
     libcxxabi PRIVATE
     -D_LIBCPP_ENABLE_CXX17_REMOVED_UNEXPECTED_FUNCTIONS
   )
-  set_target_properties(libcxxabi PROPERTIES COMPILE_FLAGS "-Wno-missing-prototypes -Wno-implicit-fallthrough")
-  # libc++abi depends on libc++ internal headers.
-  set_property(TARGET libcxxabi APPEND PROPERTY INCLUDE_DIRECTORIES "${LLVM_PROJECT_HOME}/libcxx/src")
 
   add_library(libcxx ${LIBCXX_SOURCES})
   if(ASAN OR MSAN OR TSAN)
@@ -765,6 +765,15 @@ if(USE_CUSTOM_LIBCXX)
     -D_LIBCPP_BUILDING_LIBRARY
     -DLIBCXX_BUILDING_LIBCXXABI
   )
+  set_target_properties(
+    libcxx libcxxabi PROPERTIES
+    COMPILE_FLAGS "-Wno-missing-prototypes -Wno-implicit-fallthrough"
+    # libc++ and libc++abi must be built in C++20 mode.
+    CXX_STANDARD 20
+    CXX_STANDARD_REQUIRED TRUE
+  )
+  # libc++abi depends on libc++ internal headers.
+  set_property(TARGET libcxx libcxxabi APPEND PROPERTY INCLUDE_DIRECTORIES "${LLVM_PROJECT_HOME}/libcxx/src")
   target_link_libraries(libcxx libcxxabi)
 endif()
 

bindings/rust/aws-lc-fips-sys-template/Cargo.toml

@@ -8,7 +8,7 @@ repository = "https://github.com/awslabs/aws-lc"
 license = "Apache-2.0 OR ISC"
 rust-version = "1.57.0"
 include = [
-    "build.rs",
+    "build/**/*.rs",
     "Cargo.toml",
     "deps/aws-lc/**/*.c",
     "deps/aws-lc/**/*.cc",
@@ -19,13 +19,17 @@ include = [
     "deps/aws-lc/**/*.cmake",
     "deps/aws-lc/**/*.errordata",
     "deps/aws-lc/**/*.go",
+    "deps/aws-lc/**/*.mod",
+    "deps/aws-lc/**/*.sum",
     "src/**/*.rs",
     "tests/**/*.rs",
 ]
+build = "build/main.rs"
 
 [features]
 asan = []
 internal_generate = ["bindgen"] # Only for internal use, this line is removed prior to publishing.
+ssl = []
 
 [build-dependencies]
 cmake = "0.1.48"
@@ -36,3 +40,4 @@ cfg_aliases = "0.1.1"
 
 [dependencies]
 libc = "0.2"
+paste = "1.0.11"

bindings/rust/aws-lc-fips-sys-template/README.md

@@ -0,0 +1,39 @@
+# aws-lc-fips-sys
+
+[![crates.io](https://img.shields.io/crates/v/aws-lc-fips-sys.svg)](https://crates.io/crates/aws-lc-fips-sys)
+
+Low level AWS-LC FIPS bindings for the Rust programming language. We do not recommend directly relying on these bindings.
+
+[Documentation](https://github.com/awslabs/aws-lc).
+
+## Release Support
+
+This crate pulls in the source code of the latest AWS-LC FIPS branch to build with it. Bindings for platforms we officially support are pre-generated.
+The platforms which `aws-lc-fips-sys` builds on is limited to the platforms where the AWS-LC FIPS static build is supported.
+
+### Pregenerated Bindings Availability
+
+CPU|OS
+-------------|-------------
+x86-64|Linux
+arm-64|Linux
+
+### Tested AWS-LC FIPS Build Environments
+
+`aws-lc-fips-sys` currently relies on the AWS-LC FIPS static build, please see our CI documentation at [AWS-LC](https://github.com/awslabs/aws-lc/tree/main/tests/ci#unit-tests).
+
+## Build Prerequisites
+
+Since this crate builds AWS-LC as a native library, all build tools needed to build AWS-LC are applicable to `aws-lc-fips-sys` as well. This includes Go and Perl, which are hard dependencies for the AWS-LC FIPS build.
+
+[Building AWS-LC](https://github.com/awslabs/aws-lc/blob/main/BUILDING.md)
+
+If you use a different build combination for FIPS and would like us to support it, please open an issue to us at [AWS-LC](https://github.com/awslabs/aws-lc/issues/new?assignees=&labels=&template=build-issue.md&title=).
+
+## Contribution
+
+See contributing file at [AWS-LC](https://github.com/awslabs/aws-lc/blob/main/CONTRIBUTING.md)
+
+## Licensing
+
+See license at [AWS-LC](https://github.com/awslabs/aws-lc/blob/main/LICENSE)

bindings/rust/aws-lc-fips-sys-template/build_bindgen.rs → bindings/rust/aws-lc-fips-sys-template/build/bindgen.rs

@@ -29,7 +29,7 @@ impl ParseCallbacks for StripPrefixCallback {
     }
 }
 
-fn prepare_clang_args(manifest_dir: &Path, build_prefix: Option<&str>) -> Vec<String> {
+fn prepare_clang_args(manifest_dir: &Path, build_prefix: &Option<&str>) -> Vec<String> {
     let mut clang_args: Vec<String> = vec![
         "-I".to_string(),
         get_include_path(manifest_dir).display().to_string(),
@@ -51,8 +51,15 @@ const PRELUDE: &str = r#"
 #![allow(unused_imports, non_camel_case_types, non_snake_case, non_upper_case_globals, improper_ctypes)]
 "#;
 
-fn prepare_bindings_builder(manifest_dir: &Path, build_prefix: Option<&str>) -> bindgen::Builder {
-    let clang_args = prepare_clang_args(manifest_dir, build_prefix);
+#[derive(Default)]
+pub(crate) struct BindingOptions<'a> {
+    pub build_prefix: Option<&'a str>,
+    pub include_ssl: bool,
+    pub disable_prelude: bool,
+}
+
+fn prepare_bindings_builder(manifest_dir: &Path, options: BindingOptions<'_>) -> bindgen::Builder {
+    let clang_args = prepare_clang_args(manifest_dir, &options.build_prefix);
 
     let mut builder = bindgen::Builder::default()
         .derive_copy(true)
@@ -61,10 +68,7 @@ fn prepare_bindings_builder(manifest_dir: &Path, build_prefix: Option<&str>) ->
         .derive_eq(true)
         .allowlist_file(".*/openssl/[^/]+\\.h")
         .allowlist_file(".*/rust_wrapper\\.h")
-        .default_enum_style(bindgen::EnumVariation::NewType {
-            is_bitfield: false,
-            is_global: false,
-        })
+        .rustified_enum("point_conversion_form_t")
         .default_macro_constant_type(bindgen::MacroTypeVariation::Signed)
         .generate_comments(true)
         .fit_macro_constants(false)
@@ -74,15 +78,28 @@ fn prepare_bindings_builder(manifest_dir: &Path, build_prefix: Option<&str>) ->
         .rustfmt_bindings(true)
         .clang_args(clang_args)
         .raw_line(COPYRIGHT)
-        .raw_line(PRELUDE)
         .header(
             get_include_path(manifest_dir)
                 .join("rust_wrapper.h")
                 .display()
                 .to_string(),
         );
 
-    if let Some(ps) = build_prefix {
+    if !options.disable_prelude {
+        builder = builder.raw_line(PRELUDE);
+    }
+
+    if options.include_ssl {
+        builder = builder.header_contents(
+            "rust_ssl_wrapper.h",
+            "\
+#include <openssl/ssl.h>
+#include <openssl/ssl3.h>
+",
+        );
+    }
+
+    if let Some(ps) = &options.build_prefix {
         builder = builder.parse_callbacks(Box::new(StripPrefixCallback::new(ps)));
     }
 
@@ -91,13 +108,10 @@ fn prepare_bindings_builder(manifest_dir: &Path, build_prefix: Option<&str>) ->
 
 pub(crate) fn generate_bindings(
     manifest_dir: &Path,
-    build_prefix: Option<&str>,
-    output_name: &str,
-) -> Result<(), &'static str> {
-    let bindings_file = manifest_dir.join("src").join(&output_name);
-    let builder = prepare_bindings_builder(&manifest_dir, build_prefix);
-    let bindings = builder.generate().expect("Unable to generate bindings.");
-    Ok(bindings
-        .write_to_file(bindings_file)
-        .expect("Unable to write bindings to file."))
+    options: BindingOptions<'_>,
+) -> Result<bindgen::Bindings, &'static str> {
+    let bindings = prepare_bindings_builder(&manifest_dir, options)
+        .generate()
+        .expect("Unable to generate bindings.");
+    Ok(bindings)
 }