bindgen: use bindgen to provide Rust bindings to C - v7

.github/workflows/rust.yml

@@ -21,20 +21,18 @@ jobs:
     runs-on: ubuntu-latest
     container: almalinux:9
     steps:
-      - name: Cache rust
-        uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2
-        with:
-          path: ~/.cargo
-          key: check-rust
-
       - name: Install system packages
         run: |
-          dnf -y install dnf-plugins-core
+          dnf -y install dnf-plugins-core epel-release
           dnf config-manager --set-enabled crb
           dnf -y install \
                 autoconf \
                 automake \
-                cargo-vendor \
+                bindgen \
+                cargo \
+                cbindgen \
+                clang-devel \
+                clippy \
                 diffutils \
                 numactl-devel \
                 dpdk-devel \
@@ -61,21 +59,28 @@ jobs:
                 python3-devel \
                 python3-sphinx \
                 python3-yaml \
+                rust \
+                rustfmt \
                 sudo \
                 which \
                 zlib-devel
 
-      - name: Installing Rust
-        run: |
-          curl https://sh.rustup.rs -sSf | sh -s -- -y
-          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
-      - name: Install cbindgen
-        run: cargo install --debug cbindgen
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
       - run: git config --global --add safe.directory /__w/suricata/suricata
       - run: ./scripts/bundle.sh
       - run: ./autogen.sh
       - run: ./configure --enable-warnings
+      - name: Checking bindgen output
+        working-directory: rust
+        run: |
+          bindgen --version
+          make check-bindgen-bindings
+          diff=$(git diff sys)
+          if [ "${diff}" ]; then
+              echo "${diff}"
+              echo "::error ::Bindgen bindings appear to be out of date"
+              exit 1
+          fi
       - run: cargo clippy --all-features --fix --allow-no-vcs
         working-directory: rust
       - run: |

configure.ac

@@ -2269,6 +2269,9 @@ fi
        fi
     fi
 
+    AC_PATH_PROG([BINDGEN], [bindgen], [no])
+    AM_CONDITIONAL([HAVE_BINDGEN], [test "x$BINDGEN" != "xno"])
+
     AC_PATH_PROG(CBINDGEN, cbindgen, "no")
     if test "x$CBINDGEN" != "xno"; then
       cbindgen_version=$(cbindgen --version 2>&1 | cut -d' ' -f2-)
@@ -2516,6 +2519,7 @@ AC_SUBST(enable_non_bundled_htp)
 AM_CONDITIONAL([BUILD_SHARED_LIBRARY], [test "x$enable_shared" = "xyes"] && [test "x$can_build_shared_library" = "xyes"])
 
 AC_CONFIG_FILES(Makefile src/Makefile rust/Makefile rust/Cargo.lock rust/Cargo.toml rust/derive/Cargo.toml rust/.cargo/config.toml)
+AC_CONFIG_FILES(rust/sys/Makefile rust/sys/Cargo.toml)
 AC_CONFIG_FILES(qa/Makefile qa/coccinelle/Makefile)
 AC_CONFIG_FILES(rules/Makefile doc/Makefile doc/userguide/Makefile)
 AC_CONFIG_FILES(contrib/Makefile contrib/file_processor/Makefile contrib/file_processor/Action/Makefile contrib/file_processor/Processor/Makefile)

rust/Cargo.toml.in

@@ -7,7 +7,7 @@ edition = "2021"
 rust-version = "1.67.1"
 
 [workspace]
-members = [".", "./derive"]
+members = [".", "./derive", "sys"]
 
 [lib]
 crate-type = ["staticlib", "rlib"]
@@ -68,6 +68,7 @@ hex = "~0.4.3"
 time = "~0.3.36"
 
 suricata-derive = { path = "./derive", version = "@PACKAGE_VERSION@" }
+suricata-sys = { path = "./sys", version = "@PACKAGE_VERSION@" }
 
 suricata-lua-sys = { version = "0.1.0-alpha.5" }
 

rust/Makefile.am

@@ -1,10 +1,14 @@
+SUBDIRS =	sys
+
 EXTRA_DIST =	src derive \
 		.cargo/config.toml.in \
 		cbindgen.toml \
 		dist/rust-bindings.h \
 		vendor \
 		Cargo.toml Cargo.lock \
-		derive/Cargo.toml
+		derive/Cargo.toml \
+		sys \
+		sys/Cargo.toml
 
 if !DEBUG
 RELEASE = --release
@@ -38,12 +42,18 @@ if HAVE_CYGPATH
 CARGO_ENV =	@rustup_home@ \
 		CARGO_HOME="$(CARGO_HOME)" \
 		CARGO_TARGET_DIR="$(e_rustdir)/target" \
-		SURICATA_LUA_SYS_HEADER_DST="$(e_rustdir)/gen"
+		SURICATA_LUA_SYS_HEADER_DST="$(e_rustdir)/gen" \
+		TOP_BUILDDIR=$(abs_top_builddir) \
+		TOP_SRCDIR=$(abs_top_srcdir) \
+		RUST_GENDIR=$(e_rustdir)/gen
 else
 CARGO_ENV =	@rustup_home@ \
 		CARGO_HOME="$(CARGO_HOME)" \
 		CARGO_TARGET_DIR="$(abs_top_builddir)/rust/target" \
-		SURICATA_LUA_SYS_HEADER_DST="$(abs_top_builddir)/rust/gen"
+		SURICATA_LUA_SYS_HEADER_DST="$(abs_top_builddir)/rust/gen" \
+		TOP_BUILDDIR=$(abs_top_builddir) \
+		TOP_SRCDIR=$(abs_top_srcdir) \
+		RUST_GENDIR=$(abs_top_builddir)/rust/gen
 endif
 
 all-local: Cargo.toml
@@ -75,14 +85,51 @@ clean-local:
 distclean-local:
 	rm -rf vendor dist
 
+check-bindgen-bindings:
+if HAVE_BINDGEN
+	if test "$(top_srcdir)" = "$(top_builddir)"; then \
+		cp sys/src/sys.rs sys/src/sys.rs.orig; \
+		$(MAKE) update-bindings; \
+		if diff sys/src/sys.rs sys/src/sys.rs.orig > /dev/null 2>&1; then \
+			rm -f sys/src/sys.rs.orig; \
+		else \
+			echo "WARNING: bindgen bindings may be out of date"; \
+		fi \
+	else \
+		echo "Not checking bindings for out of tree build"; \
+	fi
+else
+	@echo "Unable to check bindgen bindings: bindgen not found"
+endif
+
 check:
 	cd $(abs_top_srcdir)/rust && \
 		$(CARGO_ENV) \
 		$(CARGO) test --all $(RELEASE) --features "$(RUST_FEATURES)"
+	$(MAKE) check-bindgen-bindings
 
 vendor:
 	$(CARGO_ENV) $(CARGO) vendor
 
+update-bindings:
+if HAVE_BINDGEN
+	$(BINDGEN) \
+		-o sys/src/sys.rs \
+		--rust-target 1.67.1 \
+		--disable-header-comment \
+		--default-enum-style rust \
+		--allowlist-type 'SC.*' \
+		--allowlist-type 'AppProto.*' \
+		--allowlist-type 'AppLayer.*' \
+		--allowlist-function 'jb_.*' \
+		$(abs_top_srcdir)/src/bindgen.h \
+		-- \
+		-DHAVE_CONFIG_H -I../src -I../rust/gen $(CPPFLAGS)
+else
+	@echo "error: bindgen not installed, can't update bindings"
+	exit 1
+endif
+
 if HAVE_CBINDGEN
 gen/rust-bindings.h: $(RUST_SURICATA_LIB)
 	cd $(abs_top_srcdir)/rust && \
@@ -93,7 +140,10 @@ gen/rust-bindings.h:
 endif
 
 doc:
-	CARGO_HOME=$(CARGO_HOME) $(CARGO) doc --all-features --no-deps
+	CARGO_HOME=$(CARGO_HOME) \
+		$(CARGO_ENV) \
+		SURICATA_LUA_SYS_HEADER_DST="" $(CARGO) doc \
+		--all-features --no-deps
 
 if HAVE_CBINDGEN
 dist/rust-bindings.h:

rust/derive/src/applayerevent.rs

@@ -89,15 +89,15 @@ pub fn derive_app_layer_event(input: TokenStream) -> TokenStream {
             unsafe extern "C" fn get_event_info(
                 event_name: *const std::os::raw::c_char,
                 event_id: *mut u8,
-                event_type: *mut #crate_id::core::AppLayerEventType,
+                event_type: *mut #crate_id::sys::SCAppLayerEventType,
             ) -> std::os::raw::c_int {
                 #crate_id::applayer::get_event_info::<#name>(event_name, event_id, event_type)
             }
 
             unsafe extern "C" fn get_event_info_by_id(
                 event_id: u8,
                 event_name: *mut *const std::os::raw::c_char,
-                event_type: *mut #crate_id::core::AppLayerEventType,
+                event_type: *mut #crate_id::sys::SCAppLayerEventType,
             ) -> std::os::raw::c_int {
                 #crate_id::applayer::get_event_info_by_id::<#name>(event_id, event_name, event_type)
             }

rust/src/applayer.rs

@@ -18,10 +18,11 @@
 //! Parser registration functions and common interface module.
 
 use std;
-use crate::core::{self,DetectEngineState,AppLayerEventType,AppProto};
+use crate::core::{self,DetectEngineState,AppProto};
 use crate::direction::Direction;
 use crate::filecontainer::FileContainer;
 use crate::flow::Flow;
+use crate::sys::SCAppLayerEventType;
 use std::os::raw::{c_void,c_char,c_int};
 use crate::core::SC;
 use std::ffi::CStr;
@@ -30,6 +31,7 @@ use crate::core::StreamingBufferConfig;
 // Make the AppLayerEvent derive macro available to users importing
 // AppLayerEvent from this module.
 pub use suricata_derive::AppLayerEvent;
+use suricata_sys::sys;
 
 #[repr(C)]
 pub struct StreamSlice {
@@ -131,7 +133,7 @@ pub struct AppLayerTxData {
     detect_flags_tc: u64,
 
     de_state: *mut DetectEngineState,
-    pub events: *mut core::AppLayerDecoderEvents,
+    pub events: *mut sys::AppLayerDecoderEvents,
 }
 
 impl Default for AppLayerTxData {
@@ -387,7 +389,7 @@ pub struct RustParser {
     /// Function to get an event id from a description
     pub get_eventinfo:      Option<GetEventInfoFn>,
     /// Function to get an event description from an event id
-    pub get_eventinfo_byid: Option<GetEventInfoByIdFn>,
+    pub get_eventinfo_byid: crate::sys::SCAppLayerStateGetEventInfoByIdFn,
 
     /// Function to allocate local storage
     pub localstorage_new:   Option<LocalStorageNewFn>,
@@ -457,8 +459,8 @@ pub type StateTxFreeFn  = unsafe extern "C" fn (*mut c_void, u64);
 pub type StateGetTxFn            = unsafe extern "C" fn (*mut c_void, u64) -> *mut c_void;
 pub type StateGetTxCntFn         = unsafe extern "C" fn (*mut c_void) -> u64;
 pub type StateGetProgressFn = unsafe extern "C" fn (*mut c_void, u8) -> c_int;
-pub type GetEventInfoFn     = unsafe extern "C" fn (*const c_char, event_id: *mut u8, *mut AppLayerEventType) -> c_int;
-pub type GetEventInfoByIdFn = unsafe extern "C" fn (event_id: u8, *mut *const c_char, *mut AppLayerEventType) -> c_int;
+pub type GetEventInfoFn     = unsafe extern "C" fn (*const c_char, event_id: *mut u8, *mut SCAppLayerEventType) -> c_int;
+pub type GetEventInfoByIdFn = unsafe extern "C" fn (event_id: u8, *mut *const c_char, *mut SCAppLayerEventType) -> c_int;
 pub type LocalStorageNewFn  = extern "C" fn () -> *mut c_void;
 pub type LocalStorageFreeFn = extern "C" fn (*mut c_void);
 pub type GetTxFilesFn       = unsafe extern "C" fn (*mut c_void, u8) -> AppLayerGetFileState;
@@ -596,13 +598,13 @@ pub trait AppLayerEvent {
     unsafe extern "C" fn get_event_info(
         event_name: *const std::os::raw::c_char,
         event_id: *mut u8,
-        event_type: *mut core::AppLayerEventType,
+        event_type: *mut SCAppLayerEventType,
     ) -> std::os::raw::c_int;
 
     unsafe extern "C" fn get_event_info_by_id(
         event_id: u8,
         event_name: *mut *const std::os::raw::c_char,
-        event_type: *mut core::AppLayerEventType,
+        event_type: *mut SCAppLayerEventType,
     ) -> std::os::raw::c_int;
 }
 
@@ -625,7 +627,7 @@ pub trait AppLayerEvent {
 pub unsafe fn get_event_info<T: AppLayerEvent>(
     event_name: *const std::os::raw::c_char,
     event_id: *mut u8,
-    event_type: *mut core::AppLayerEventType,
+    event_type: *mut SCAppLayerEventType,
 ) -> std::os::raw::c_int {
     if event_name.is_null() {
         return -1;
@@ -637,7 +639,7 @@ pub unsafe fn get_event_info<T: AppLayerEvent>(
             return -1;
         }
     };
-    *event_type = core::AppLayerEventType::APP_LAYER_EVENT_TYPE_TRANSACTION;
+    *event_type = SCAppLayerEventType::APP_LAYER_EVENT_TYPE_TRANSACTION;
     *event_id = event;
     return 0;
 }
@@ -648,11 +650,11 @@ pub unsafe fn get_event_info<T: AppLayerEvent>(
 pub unsafe fn get_event_info_by_id<T: AppLayerEvent>(
     event_id: u8,
     event_name: *mut *const std::os::raw::c_char,
-    event_type: *mut core::AppLayerEventType,
+    event_type: *mut SCAppLayerEventType,
 ) -> std::os::raw::c_int {
     if let Some(e) = T::from_id(event_id) {
         *event_name = e.to_cstring().as_ptr() as *const std::os::raw::c_char;
-        *event_type = core::AppLayerEventType::APP_LAYER_EVENT_TYPE_TRANSACTION;
+        *event_type = SCAppLayerEventType::APP_LAYER_EVENT_TYPE_TRANSACTION;
         return 0;
     }
     return -1;

rust/src/core.rs

@@ -18,20 +18,15 @@
 //! This module exposes items from the core "C" code to Rust.
 
 use std;
+use suricata_sys::sys;
+
 use crate::filecontainer::*;
 use crate::flow::Flow;
 
+pub use crate::sys::{AppProto, AppProtoEnum};
+
 /// Opaque C types.
 pub enum DetectEngineState {}
-pub enum AppLayerDecoderEvents {}
-
-#[repr(C)]
-#[derive(Debug, PartialEq, Eq, Clone, Copy)]
-#[allow(non_camel_case_types)]
-pub enum AppLayerEventType {
-    APP_LAYER_EVENT_TYPE_TRANSACTION = 1,
-    APP_LAYER_EVENT_TYPE_PACKET = 2,
-}
 
 pub const STREAM_START:    u8 = 0x01;
 pub const STREAM_EOF:      u8 = 0x02;
@@ -41,11 +36,8 @@ pub const STREAM_GAP:      u8 = 0x10;
 pub const STREAM_DEPTH:    u8 = 0x20;
 pub const STREAM_MIDSTREAM:u8 = 0x40;
 
-// Application layer protocol identifiers (app-layer-protos.h)
-pub type AppProto = u16;
-
-pub const ALPROTO_UNKNOWN : AppProto = 0;
-pub const ALPROTO_FAILED : AppProto = 1;
+pub const ALPROTO_UNKNOWN : AppProto = AppProtoEnum::ALPROTO_UNKNOWN as u16;
+pub const ALPROTO_FAILED : AppProto = AppProtoEnum::ALPROTO_FAILED as u16;
 
 pub const IPPROTO_TCP : u8 = 6;
 pub const IPPROTO_UDP : u8 = 17;
@@ -92,11 +84,11 @@ pub type DetectEngineStateFreeFunc =
 pub type AppLayerParserTriggerRawStreamReassemblyFunc =
     extern "C" fn (flow: *const Flow, direction: i32);
 pub type AppLayerDecoderEventsSetEventRawFunc =
-    extern "C" fn (events: *mut *mut AppLayerDecoderEvents,
+    extern "C" fn (events: *mut *mut sys::AppLayerDecoderEvents,
                    event: u8);
 
 pub type AppLayerDecoderEventsFreeEventsFunc =
-    extern "C" fn (events: *mut *mut AppLayerDecoderEvents);
+    extern "C" fn (events: *mut *mut sys::AppLayerDecoderEvents);
 
 pub enum StreamingBufferConfig {}
 
@@ -214,7 +206,7 @@ pub fn sc_app_layer_parser_trigger_raw_stream_reassembly(flow: *const Flow, dire
 
 /// AppLayerDecoderEventsSetEventRaw wrapper.
 pub fn sc_app_layer_decoder_events_set_event_raw(
-    events: *mut *mut AppLayerDecoderEvents, event: u8)
+    events: *mut *mut sys::AppLayerDecoderEvents, event: u8)
 {
     unsafe {
         if let Some(c) = SC {
@@ -225,7 +217,7 @@ pub fn sc_app_layer_decoder_events_set_event_raw(
 
 /// AppLayerDecoderEventsFreeEvents wrapper.
 pub fn sc_app_layer_decoder_events_free_events(
-    events: *mut *mut AppLayerDecoderEvents)
+    events: *mut *mut sys::AppLayerDecoderEvents)
 {
     unsafe {
         if let Some(c) = SC {