From dacb0e2419de1a68bd366fae24f945762d580898 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 14 Jan 2025 09:41:11 +0100 Subject: [PATCH] WIP detect: introduce explicit hooks Generic: :request_done and :response_done Per protocol, it uses the registered progress (state) values. E.g. tls:client_hello_done A rule ruleset could be: pass tls:client_hello_done any any -> any any (tls.sni; content:"www.google.com"; sid:21; alert;) drop tls:client_hello_done any any -> any any (sid:22;) The pass rule is evaluated when the client hello is parsed, and if it doesn't match the drop rule will be evaluated. Registers each generic lists as "::generic" (e.g. "tls:client_hello_done:generic"). Ticket: #7485. --- src/detect-engine-build.c | 6 ++ src/detect-engine-mpm.c | 4 +- src/detect-engine-register.c | 2 + src/detect-engine.c | 36 ++++++- src/detect-parse.c | 186 ++++++++++++++++++++++++++++++++++- src/detect-parse.h | 2 + src/detect.c | 10 ++ src/detect.h | 23 +++++ 8 files changed, 263 insertions(+), 6 deletions(-) diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c index 7c9b66013e35..af2ad13c38eb 100644 --- a/src/detect-engine-build.c +++ b/src/detect-engine-build.c @@ -1645,6 +1645,12 @@ void SignatureSetType(DetectEngineCtx *de_ctx, Signature *s) BUG_ON(s->type != SIG_TYPE_NOT_SET); int iponly = 0; + if (s->init_data->hook.type == SIGNATURE_HOOK_TYPE_APP) { + s->type = SIG_TYPE_APP_TX; + SCLogNotice("%u: set to app_tx due to hook type app", s->id); + SCReturn; + } + /* see if the sig is dp only */ if (SignatureIsPDOnly(de_ctx, s) == 1) { s->type = SIG_TYPE_PDONLY; diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c index 19f0c2872d7d..d46391773e67 100644 --- a/src/detect-engine-mpm.c +++ b/src/detect-engine-mpm.c @@ -2514,7 +2514,7 @@ void EngineAnalysisAddAllRulePatterns(DetectEngineCtx *de_ctx, const Signature * for (; app != NULL; app = app->next) { DEBUG_VALIDATE_BUG_ON(app->smd == NULL); SigMatchData *smd = app->smd; - do { + while (smd) { switch (smd->type) { case DETECT_CONTENT: { const DetectContentData *cd = (const DetectContentData *)smd->ctx; @@ -2542,7 +2542,7 @@ void EngineAnalysisAddAllRulePatterns(DetectEngineCtx *de_ctx, const Signature * if (smd->is_last) break; smd++; - } while (1); + } } const DetectEnginePktInspectionEngine *pkt = s->pkt_inspect; for (; pkt != NULL; pkt = pkt->next) { diff --git a/src/detect-engine-register.c b/src/detect-engine-register.c index 658c17150aec..01c0be6ec8b5 100644 --- a/src/detect-engine-register.c +++ b/src/detect-engine-register.c @@ -503,6 +503,8 @@ void SigTableInit(void) void SigTableSetup(void) { + DetectRegisterAppLayerHookLists(); + DetectSidRegister(); DetectPriorityRegister(); DetectPrefilterRegister(); diff --git a/src/detect-engine.c b/src/detect-engine.c index c8dfe4a132fd..eefc1215ba76 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -681,7 +681,7 @@ static void AppendAppInspectEngine(DetectEngineCtx *de_ctx, new_engine->sm_list = t->sm_list; new_engine->sm_list_base = t->sm_list_base; new_engine->smd = smd; - new_engine->match_on_null = DetectContentInspectionMatchOnAbsentBuffer(smd); + new_engine->match_on_null = smd ? DetectContentInspectionMatchOnAbsentBuffer(smd) : false; new_engine->progress = t->progress; new_engine->v2 = t->v2; SCLogDebug("sm_list %d new_engine->v2 %p/%p/%p", new_engine->sm_list, new_engine->v2.Callback, @@ -747,6 +747,7 @@ int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature const int files_id = DetectBufferTypeGetByName("files"); bool head_is_mpm = false; uint8_t last_id = DE_STATE_FLAG_BASE; + SCLogNotice("%u: setup app inspect engines. %u buffers", s->id, s->init_data->buffer_index); for (uint32_t x = 0; x < s->init_data->buffer_index; x++) { SigMatchData *smd = SigMatchList2DataArray(s->init_data->buffers[x].head); @@ -786,6 +787,39 @@ int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature } } + /* handle rules that have an app-layer hook w/o bringing their own app inspect engine, + * e.g. `alert dns:request_complete ... (sid:1;)` + * + * Here we use a minimal stub inspect engine in which we set: + * - alproto + * - progress + * - sm_list/sm_list_base to get the mapping to the hook name + * - dir based on sig direction + * + * The inspect engine has no callback and is thus considered a straight match. + */ + if (s->init_data->buffer_index == 0 && s->init_data->hook.type == SIGNATURE_HOOK_TYPE_APP) { + int dir = 0; + if ((s->flags & (SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT)) == + (SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT)) + abort(); + if ((s->flags & (SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT)) == 0) + abort(); + if (s->flags & SIG_FLAG_TOSERVER) + dir = 0; + else if (s->flags & SIG_FLAG_TOCLIENT) + dir = 1; + + DetectEngineAppInspectionEngine t = { + .alproto = s->init_data->hook.t.app.alproto, + .progress = (uint16_t)s->init_data->hook.t.app.app_progress, + .sm_list = (uint16_t)s->init_data->hook.sm_list, + .sm_list_base = (uint16_t)s->init_data->hook.sm_list, + .dir = dir, + }; + AppendAppInspectEngine(de_ctx, &t, s, NULL, mpm_list, files_id, &last_id, &head_is_mpm); + } + if ((s->init_data->init_flags & SIG_FLAG_INIT_STATE_MATCH) && s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) { diff --git a/src/detect-parse.c b/src/detect-parse.c index 031b4b315cf4..2b3374540372 100644 --- a/src/detect-parse.c +++ b/src/detect-parse.c @@ -1105,6 +1105,148 @@ static int SigParseAddress(DetectEngineCtx *de_ctx, return -1; } +static bool IsBuiltIn(const char *n) +{ + if (strcmp(n, "request_complete") == 0 || strcmp(n, "response_complete") == 0) { + return true; + } + return false; +} + +/** \brief register app hooks as generic lists + * + * Register each hook in each app protocol as: + * ::generic + * These lists can be used by lua scripts to hook into. + * + * \todo move elsewhere? maybe a detect-engine-hook.c? + */ +void DetectRegisterAppLayerHookLists(void) +{ + for (AppProto a = ALPROTO_FAILED + 1; a < g_alproto_max; a++) { + const char *alproto_name = AppProtoToString(a); + if (strcmp(alproto_name, "http") == 0) + alproto_name = "http1"; + SCLogDebug("alproto %u/%s", a, alproto_name); + + const int max_progress_ts = + AppLayerParserGetStateProgressCompletionStatus(a, STREAM_TOSERVER); + const int max_progress_tc = + AppLayerParserGetStateProgressCompletionStatus(a, STREAM_TOCLIENT); + + char ts_tx_complete[64]; + snprintf(ts_tx_complete, sizeof(ts_tx_complete), "%s:request_complete:generic", + alproto_name); + DetectAppLayerInspectEngineRegister(ts_tx_complete, a, SIG_FLAG_TOSERVER, max_progress_ts, + DetectEngineInspectGenericList, NULL); + SCLogDebug("- hook %s:%s list %s (%u)", alproto_name, "request_name", ts_tx_complete, + (uint32_t)strlen(ts_tx_complete)); + + char tc_tx_complete[64]; + snprintf(tc_tx_complete, sizeof(tc_tx_complete), "%s:response_complete:generic", + alproto_name); + DetectAppLayerInspectEngineRegister(tc_tx_complete, a, SIG_FLAG_TOCLIENT, max_progress_tc, + DetectEngineInspectGenericList, NULL); + SCLogDebug("- hook %s:%s list %s (%u)", alproto_name, "response_name", tc_tx_complete, + (uint32_t)strlen(tc_tx_complete)); + + for (int p = 0; p <= max_progress_ts; p++) { + const char *name = AppLayerParserGetStateNameById( + IPPROTO_TCP /* TODO no ipproto */, a, p, STREAM_TOSERVER); + if (name != NULL && !IsBuiltIn(name)) { + char list_name[64]; + snprintf(list_name, sizeof(list_name), "%s:%s:generic", alproto_name, name); + SCLogDebug("- hook %s:%s list %s (%u)", alproto_name, name, list_name, + (uint32_t)strlen(list_name)); + + DetectAppLayerInspectEngineRegister( + list_name, a, SIG_FLAG_TOSERVER, p, DetectEngineInspectGenericList, NULL); + } + } + for (int p = 0; p <= max_progress_tc; p++) { + const char *name = AppLayerParserGetStateNameById( + IPPROTO_TCP /* TODO no ipproto */, a, p, STREAM_TOCLIENT); + if (name != NULL && !IsBuiltIn(name)) { + char list_name[64]; + snprintf(list_name, sizeof(list_name), "%s:%s:generic", alproto_name, name); + SCLogDebug("- hook %s:%s list %s (%u)", alproto_name, name, list_name, + (uint32_t)strlen(list_name)); + + DetectAppLayerInspectEngineRegister( + list_name, a, SIG_FLAG_TOCLIENT, p, DetectEngineInspectGenericList, NULL); + } + } + } +} + +static const char *SignatureHookTypeToString(enum SignatureHookType t) +{ + switch (t) { + case SIGNATURE_HOOK_TYPE_NOT_SET: + return "not_set"; + case SIGNATURE_HOOK_TYPE_APP: + return "app"; + // case SIGNATURE_HOOK_TYPE_PKT: + // return "pkt"; + } + return "unknown"; +} + +static SignatureHook SetAppHook(const AppProto alproto, int progress) +{ + SignatureHook h = { + .type = SIGNATURE_HOOK_TYPE_APP, + .t.app.alproto = alproto, + .t.app.app_progress = progress, + }; + return h; +} + +/** + * \param proto_hook string of protocol and hook, e.g. dns:request_complete + */ +static int SigParseProtoHookApp(Signature *s, const char *proto_hook, const char *p, const char *h) +{ + if (strcmp(h, "request_complete") == 0) { + s->flags |= SIG_FLAG_TOSERVER; + s->init_data->hook = SetAppHook(s->alproto, + AppLayerParserGetStateProgressCompletionStatus(s->alproto, STREAM_TOSERVER)); + } else if (strcmp(h, "response_complete") == 0) { + s->flags |= SIG_FLAG_TOCLIENT; + s->init_data->hook = SetAppHook(s->alproto, + AppLayerParserGetStateProgressCompletionStatus(s->alproto, STREAM_TOCLIENT)); + } else { + const int progress_ts = AppLayerParserGetStateIdByName( + IPPROTO_TCP /* TODO */, s->alproto, h, STREAM_TOSERVER); + if (progress_ts >= 0) { + s->flags |= SIG_FLAG_TOSERVER; + s->init_data->hook = SetAppHook(s->alproto, progress_ts); + } else { + const int progress_tc = AppLayerParserGetStateIdByName( + IPPROTO_TCP /* TODO */, s->alproto, h, STREAM_TOCLIENT); + if (progress_tc < 0) { + return -1; + } + s->flags |= SIG_FLAG_TOCLIENT; + s->init_data->hook = SetAppHook(s->alproto, progress_tc); + } + } + + char generic_hook_name[64]; + snprintf(generic_hook_name, sizeof(generic_hook_name), "%s:generic", proto_hook); + int list = DetectBufferTypeGetByName(generic_hook_name); + if (list < 0) { + SCLogError("no list registered as %s for hook %s", generic_hook_name, proto_hook); + return -1; + } + s->init_data->hook.sm_list = list; + + SCLogNotice("protocol:%s hook:%s: type:%s alproto:%u hook:%d", p, h, + SignatureHookTypeToString(s->init_data->hook.type), s->init_data->hook.t.app.alproto, + s->init_data->hook.t.app.app_progress); + return 0; +} + /** * \brief Parses the protocol supplied by the Signature. * @@ -1120,15 +1262,37 @@ static int SigParseAddress(DetectEngineCtx *de_ctx, static int SigParseProto(Signature *s, const char *protostr) { SCEnter(); + if (strlen(protostr) > 32) + return -1; + + char proto[33]; + strlcpy(proto, protostr, 33); + const char *p = proto; + const char *h = NULL; - int r = DetectProtoParse(&s->proto, (char *)protostr); + bool has_hook = strchr(proto, ':') != NULL; + if (has_hook) { + char *xsaveptr = NULL; + p = strtok_r(proto, ":", &xsaveptr); + h = strtok_r(NULL, ":", &xsaveptr); + SCLogNotice("p: '%s' h: '%s'", p, h); + } + + int r = DetectProtoParse(&s->proto, p); if (r < 0) { - s->alproto = AppLayerGetProtoByName((char *)protostr); + s->alproto = AppLayerGetProtoByName(p); /* indicate that the signature is app-layer */ if (s->alproto != ALPROTO_UNKNOWN) { s->flags |= SIG_FLAG_APPLAYER; AppLayerProtoDetectSupportedIpprotos(s->alproto, s->proto.proto); + + if (h) { + if (SigParseProtoHookApp(s, protostr, p, h) < 0) { + SCLogError("protocol \"%s\" does not support hook \"%s\"", p, h); + SCReturnInt(-1); + } + } } else { SCLogError("protocol \"%s\" cannot be used " @@ -1136,7 +1300,7 @@ static int SigParseProto(Signature *s, const char *protostr) "is not yet supported OR detection has been disabled for " "protocol through the yaml option " "app-layer.protocols.%s.detection-enabled", - protostr, protostr); + p, p); SCReturnInt(-1); } } @@ -2058,6 +2222,22 @@ static int SigValidate(DetectEngineCtx *de_ctx, Signature *s) SCLogDebug("b->id %d nlists %d", b->id, nlists); bufdir[b->id].ts += (app->dir == 0); bufdir[b->id].tc += (app->dir == 1); + + /* only allow rules to use the hook for engines at that + * exact progress for now. */ + if (s->init_data->hook.type == SIGNATURE_HOOK_TYPE_APP) { + if ((s->flags & SIG_FLAG_TOSERVER) && (app->dir == 0) && + app->progress != s->init_data->hook.t.app.app_progress) { + SCLogError("engine progress value %d doesn't match hook %u", app->progress, + s->init_data->hook.t.app.app_progress); + SCReturnInt(0); + } + if ((s->flags & SIG_FLAG_TOCLIENT) && (app->dir == 1) && + app->progress != s->init_data->hook.t.app.app_progress) { + SCLogError("engine progress value doesn't match hook"); + SCReturnInt(0); + } + } } } diff --git a/src/detect-parse.h b/src/detect-parse.h index ec2c204c0f42..6d00ec37cf59 100644 --- a/src/detect-parse.h +++ b/src/detect-parse.h @@ -120,4 +120,6 @@ int SC_Pcre2SubstringCopy( int SC_Pcre2SubstringGet(pcre2_match_data *match_data, uint32_t number, PCRE2_UCHAR **bufferptr, PCRE2_SIZE *bufflen); +void DetectRegisterAppLayerHookLists(void); + #endif /* SURICATA_DETECT_PARSE_H */ diff --git a/src/detect.c b/src/detect.c index 1f169612443c..fba0fd2c5273 100644 --- a/src/detect.c +++ b/src/detect.c @@ -1208,6 +1208,16 @@ static bool DetectRunTxInspectRule(ThreadVars *tv, if (unlikely(engine->stream && can->stream_stored)) { match = can->stream_result; TRACE_SID_TXS(s->id, tx, "stream skipped, stored result %d used instead", match); + } else if (engine->v2.Callback == NULL) { + /* TODO is this the cleanest way to support a non-app sig on a app hook? */ + + /* we don't have to store a "hook" match, also don't want to keep any state to make + * sure the hook gets invoked again until tx progress progresses. */ + if (tx->tx_progress <= engine->progress) + return DETECT_ENGINE_INSPECT_SIG_MATCH; + + /* if progress > engine progress, track state to avoid additional matches */ + match = DETECT_ENGINE_INSPECT_SIG_MATCH; } else { KEYWORD_PROFILING_SET_LIST(det_ctx, engine->sm_list); DEBUG_VALIDATE_BUG_ON(engine->v2.Callback == NULL); diff --git a/src/detect.h b/src/detect.h index bc5745855ce0..a62c0c3b68cd 100644 --- a/src/detect.h +++ b/src/detect.h @@ -538,7 +538,30 @@ typedef struct SignatureInitDataBuffer_ { SigMatch *tail; } SignatureInitDataBuffer; +enum SignatureHookType { + SIGNATURE_HOOK_TYPE_NOT_SET, + // SIGNATURE_HOOK_TYPE_PKT, + SIGNATURE_HOOK_TYPE_APP, +}; + +// dns:request_complete should add DetectBufferTypeGetByName("dns:request_complete"); +// TODO to json +typedef struct SignatureHook_ { + enum SignatureHookType type; + int sm_list; /**< list id for the hook's generic list. e.g. for dns:request_complete:generic */ + union { + struct { + AppProto alproto; + /** progress value of the app-layer hook specified in the rule. Sets the app_proto + * specific progress value. */ + int app_progress; + } app; + } t; +} SignatureHook; + typedef struct SignatureInitData_ { + SignatureHook hook; + /** Number of sigmatches. Used for assigning SigMatch::idx */ uint16_t sm_cnt;