FB8-54, FB8-55, FB8-70, FB8-101: Expose more information to audit plu…

…gin (facebook#934) (facebook#934)

Summary:
JIRA: https://jira.percona.com/browse/FB8-54
JIRA: https://jira.percona.com/browse/FB8-55
JIRA: https://jira.percona.com/browse/FB8-70
JIRA: https://jira.percona.com/browse/FB8-101

This commit adds the following fields to the generic event in audit log:
* query_id
* database
* affected_rows
* connection_certificate

Reference Patch: facebook@1def6b7
Reference Patch: facebook@ce95a09
Reference Patch: facebook@588be34
Reference Patch: facebook@ba03c70
Reference Patch: facebook@be8c587
Reference Patch: facebook@22b2508

We need some extra info for the shadowing and security logging. This is a
simple first step of info that MariaDB actually also exposes.
Now we would have the `query_id` and the database name for general events.
Making as few changes as possible to accomplish it, so I'm just taking the
information from the TDH and exposing it through `mysql_event_general`
struct and as a argument to disconnect.

Forward the connection certificate to the audit plugin. The connection certificate can then be parsed by the audit plugin and handled appropriately. It made more sense for the certificate to live in the connection events, since they generally don't change between every general event, so the move was done.

This is done by caching a BUF_MEM struct on the THD object. Since it's not possible to change certificates on the same connection, this caching should be correct. The BUF_MEM is released on THD::release_resources.

If upstream bumps the MYSQL_AUDIT_INTERFACE_VERSION, we should bump ours to be greater or equal to it.

Expose the port current mysqld is running on for the audit plugin. If no port, 0 is used.
Pull Request resolved: facebook#934

Reviewed By: lloyd

Differential Revision: D13874133

Pulled By: lth

include/mysql/plugin_audit.h

@@ -37,7 +37,7 @@
 #include "my_sqlcommand.h"
 #include "plugin_audit_message_types.h"
 
-#define MYSQL_AUDIT_INTERFACE_VERSION 0x0401
+#define MYSQL_AUDIT_INTERFACE_VERSION 0x0402
 
 /**
  @enum mysql_event_class_t
@@ -141,6 +141,11 @@ struct mysql_event_general {
   MYSQL_LEX_CSTRING general_sql_command;
   MYSQL_LEX_CSTRING general_external_user;
   MYSQL_LEX_CSTRING general_ip;
+  /* Added in version 402 */
+  long long query_id;
+  MYSQL_LEX_CSTRING database;
+  long long affected_rows;
+  unsigned int port;
 };
 
 #define MYSQL_AUDIT_CONNECTION_ALL                                      \
@@ -182,6 +187,8 @@ struct mysql_event_connection {
         - 5 Shared memory
   */
   int connection_type;
+  MYSQL_LEX_CSTRING connection_certificate;
+  unsigned int port;
 };
 
 /**

include/mysql/plugin_audit.h.pp

@@ -409,6 +409,10 @@
   MYSQL_LEX_CSTRING general_sql_command;
   MYSQL_LEX_CSTRING general_external_user;
   MYSQL_LEX_CSTRING general_ip;
+  long long query_id;
+  MYSQL_LEX_CSTRING database;
+  long long affected_rows;
+  unsigned int port;  
 };
 struct mysql_event_connection {
   mysql_event_connection_subclass_t event_subclass;
@@ -422,6 +426,8 @@
   MYSQL_LEX_CSTRING ip;
   MYSQL_LEX_CSTRING database;
   int connection_type;
+  MYSQL_LEX_CSTRING connection_certificate;
+  unsigned int port;
 };
 typedef enum {
   MYSQL_AUDIT_PARSE_PREPARSE = 1 << 0,

mysql-test/suite/audit_null/r/event_params.result

@@ -0,0 +1,33 @@
+INSTALL PLUGIN null_audit SONAME 'adt_null.so';
+SET @@null_audit_extended_log = 1;
+CREATE TABLE foo (v INT);
+DROP TABLE foo;
+SHOW STATUS LIKE "Audit_null_generic_event_response";
+Variable_name	Value
+Audit_null_generic_event_response	database:test
+CREATE TABLE foo (v INT);
+SHOW STATUS LIKE "Audit_null_generic_event_response";
+Variable_name	Value
+Audit_null_generic_event_response	affected_rows:0
+INSERT INTO foo VALUES (1), (2);
+SHOW STATUS LIKE "Audit_null_generic_event_response";
+Variable_name	Value
+Audit_null_generic_event_response	affected_rows:2
+SELECT * FROM foo;
+v
+1
+2
+SHOW STATUS LIKE "Audit_null_generic_event_response";
+Variable_name	Value
+Audit_null_generic_event_response	affected_rows:-1
+DELETE FROM foo;
+SHOW STATUS LIKE "Audit_null_generic_event_response";
+Variable_name	Value
+Audit_null_generic_event_response	affected_rows:2
+DROP TABLE foo;
+SHOW STATUS LIKE "Audit_null_generic_event_response";
+Variable_name	Value
+Audit_null_generic_event_response	port:MYSQLD_PORT
+UNINSTALL PLUGIN null_audit;
+Warnings:
+Warning	1620	Plugin is busy and will be uninstalled on shutdown

mysql-test/suite/audit_null/r/event_params_cert.result

@@ -0,0 +1,16 @@
+INSTALL PLUGIN null_audit SONAME 'adt_null.so';
+CREATE USER cert_auth@localhost REQUIRE X509;
+GRANT SELECT ON test.* TO cert_auth@localhost;
+CREATE TABLE foo (i INT);
+FLUSH PRIVILEGES;
+SET @@null_audit_extended_log = 1;
+SELECT * FROM foo;
+i
+SHOW STATUS LIKE "Audit_null_connect_event_response";
+Variable_name	Value
+Audit_null_connect_event_response	connection_certificate:-----BEGIN CERTIFICATE-----\nMIIDyDCCArCgAwIBAgIJAOG0pVw936YVMA0GCSqGSIb3DQEBCwUAMGMxCzAJBgNV\nBAYTAlNFMRIwEAYDVQQIDAlTdG9ja2hvbG0xEjAQBgNVBAcMCVN0b2NraG9sbTEP\nMA0GA1UECgwGT3JhY2xlMQ4wDAYDVQQLDAVNeVNRTDELMAkGA1UEAwwCQ0EwHhcN\nMTQxMjA1MDQ0OTIzWhcNMjkxMjAx
+DROP USER cert_auth@localhost;
+DROP TABLE foo;
+UNINSTALL PLUGIN null_audit;
+Warnings:
+Warning	1620	Plugin is busy and will be uninstalled on shutdown

mysql-test/suite/audit_null/t/event_params-master.opt

@@ -0,0 +1 @@
+$AUDIT_NULL_OPT

mysql-test/suite/audit_null/t/event_params.test

@@ -0,0 +1,44 @@
+--source include/have_null_audit_plugin.inc
+--source include/count_sessions.inc
+--source include/have_debug.inc
+
+eval INSTALL PLUGIN null_audit SONAME '$AUDIT_NULL';
+
+SET @@null_audit_extended_log = 1;
+
+## database name in generic event
+CREATE TABLE foo (v INT);
+DROP TABLE foo;
+
+--replace_regex /.*(database:[^;]*).*/\1/
+SHOW STATUS LIKE "Audit_null_generic_event_response";
+
+## affected_rows in generic event
+CREATE TABLE foo (v INT);
+--replace_regex /.*(affected_rows:[^;]*).*/\1/
+SHOW STATUS LIKE "Audit_null_generic_event_response";
+
+INSERT INTO foo VALUES (1), (2);
+--replace_regex /.*(affected_rows:[^;]*).*/\1/
+SHOW STATUS LIKE "Audit_null_generic_event_response";
+
+SELECT * FROM foo;
+--replace_regex /.*(affected_rows:[^;]*).*/\1/
+SHOW STATUS LIKE "Audit_null_generic_event_response";
+
+DELETE FROM foo;
+--replace_regex /.*(affected_rows:[^;]*).*/\1/
+SHOW STATUS LIKE "Audit_null_generic_event_response";
+
+DROP TABLE foo;
+
+## port
+let $MYSQLD_PORT= `SELECT @@port`;
+--replace_result $MYSQLD_PORT MYSQLD_PORT
+--replace_regex /.*(port:[^;]*).*/\1/
+SHOW STATUS LIKE "Audit_null_generic_event_response";
+
+
+UNINSTALL PLUGIN null_audit;
+
+--source include/wait_until_count_sessions.inc

mysql-test/suite/audit_null/t/event_params_cert-client.opt

@@ -0,0 +1,4 @@
+--ssl-mode=VERIFY_CA
+--ssl-ca=$MYSQL_TEST_DIR/std_data/cacert.pem
+--ssl-cert=$MYSQL_TEST_DIR/std_data/client-cert.pem
+--ssl-key=$MYSQL_TEST_DIR/std_data/client-key.pem

mysql-test/suite/audit_null/t/event_params_cert-master.opt

@@ -0,0 +1 @@
+$AUDIT_NULL_OPT

mysql-test/suite/audit_null/t/event_params_cert.test

@@ -0,0 +1,29 @@
+--source include/count_sessions.inc
+--source include/have_debug.inc
+
+eval INSTALL PLUGIN null_audit SONAME '$AUDIT_NULL';
+
+CREATE USER cert_auth@localhost REQUIRE X509;
+GRANT SELECT ON test.* TO cert_auth@localhost;
+CREATE TABLE foo (i INT);
+FLUSH PRIVILEGES;
+connect(con1,localhost,cert_auth,,,,,SSL);
+
+SET @@null_audit_extended_log = 1;
+
+SELECT * FROM foo;
+
+# As "certificate:" part in the status var may be truncated because of the
+# max status var length limit, taking only first 255 chars here to make this
+# test stable
+--replace_regex /.*(connection_certificate:[^;]{255}).*/\1/
+SHOW STATUS LIKE "Audit_null_connect_event_response";
+
+disconnect con1;
+connection default;
+DROP USER cert_auth@localhost;
+DROP TABLE foo;
+
+UNINSTALL PLUGIN null_audit;
+
+--source include/wait_until_count_sessions.inc

mysql-test/suite/query_rewrite_plugins/r/ddl_rewriter.result

@@ -20,7 +20,7 @@ SELECT PLUGIN_NAME, PLUGIN_TYPE, PLUGIN_VERSION, PLUGIN_TYPE_VERSION
 FROM information_schema.plugins
 WHERE plugin_name LIKE 'ddl_rewriter';
 PLUGIN_NAME	PLUGIN_TYPE	PLUGIN_VERSION	PLUGIN_TYPE_VERSION
-ddl_rewriter	AUDIT	1.0	4.1
+ddl_rewriter	AUDIT	1.0	4.2
 #
 # Restore dump file.
 #

mysql-test/suite/query_rewrite_plugins/r/rpl_ddl_rewriter.result

@@ -11,7 +11,7 @@ SELECT PLUGIN_NAME, PLUGIN_TYPE, PLUGIN_VERSION, PLUGIN_TYPE_VERSION
 FROM information_schema.plugins
 WHERE plugin_name LIKE 'ddl_rewriter';
 PLUGIN_NAME	PLUGIN_TYPE	PLUGIN_VERSION	PLUGIN_TYPE_VERSION
-ddl_rewriter	AUDIT	1.0	4.1
+ddl_rewriter	AUDIT	1.0	4.2
 include/sync_slave_sql_with_master.inc
 SELECT PLUGIN_NAME, PLUGIN_TYPE, PLUGIN_VERSION, PLUGIN_TYPE_VERSION
 FROM information_schema.plugins

mysql-test/suite/query_rewrite_plugins/r/rpl_skip_rewrite.result

@@ -12,7 +12,7 @@ SELECT PLUGIN_NAME, PLUGIN_TYPE, PLUGIN_VERSION, PLUGIN_TYPE_VERSION
 FROM information_schema.plugins
 WHERE plugin_name LIKE 'Rewriter';
 PLUGIN_NAME	PLUGIN_TYPE	PLUGIN_VERSION	PLUGIN_TYPE_VERSION
-Rewriter	AUDIT	0.2	4.1
+Rewriter	AUDIT	0.2	4.2
 INSERT INTO query_rewrite.rewrite_rules ( pattern, replacement)
 VALUES ('INSERT INTO test.t1 (a,b) VALUES (?, ?)',
 'INSERT INTO test.t1 (b,a) VALUES (?, ?)');

plugin/audit_null/CMakeLists.txt

@@ -25,6 +25,9 @@ MYSQL_ADD_PLUGIN(audit_null
   MODULE_ONLY
   MODULE_OUTPUT_NAME "adt_null"
   )
+
+INCLUDE_DIRECTORIES(SYSTEM ${BOOST_PATCHES_DIR} ${BOOST_INCLUDE_DIR})
+
 MYSQL_ADD_PLUGIN(test_security_context
   test_security_context.cc
   MODULE_ONLY

plugin/audit_null/audit_null.cc

@@ -25,6 +25,8 @@
 #include <mysqld_error.h>
 #include <stdio.h>
 #include <sys/types.h>
+#include <boost/algorithm/string/replace.hpp>
+#include <sstream>
 
 #include "lex_string.h"
 #include "m_ctype.h"
@@ -137,6 +139,16 @@ static char *g_record_buffer;
 
 #undef AUDIT_NULL_VAR
 
+#ifndef NDEBUG
+static const constexpr size_t event_response_buffer_len = 1000;
+static char generic_event_response[event_response_buffer_len + 1] = {
+    0,
+};
+static char connect_event_response[event_response_buffer_len + 1] = {
+    0,
+};
+#endif
+
 /*
   Plugin status variables for SHOW STATUS
 */
@@ -155,6 +167,13 @@ static SHOW_VAR simple_status[] = {
 
 #undef AUDIT_NULL_VAR
 
+#ifndef NDEBUG
+    {"Audit_null_generic_event_response", (char *)generic_event_response,
+     SHOW_CHAR, SHOW_SCOPE_GLOBAL},
+    {"Audit_null_connect_event_response", (char *)connect_event_response,
+     SHOW_CHAR, SHOW_SCOPE_GLOBAL},
+#endif
+
     {nullptr, nullptr, SHOW_UNDEF, SHOW_SCOPE_GLOBAL}};
 
 /*
@@ -187,6 +206,12 @@ static MYSQL_THDVAR_INT(event_order_check_exact, PLUGIN_VAR_RQCMDARG,
                         "Plugin checks exact event order.", nullptr, nullptr, 1,
                         0, 1, 0);
 
+#ifndef NDEBUG
+static MYSQL_THDVAR_INT(extended_log, PLUGIN_VAR_RQCMDARG,
+                        "Provide extended debug information with audit_null.",
+                        NULL, NULL, 1, 0, 1, 0);
+#endif
+
 static MYSQL_THDVAR_STR(event_record_def,
                         PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC,
                         "Event recording definition", nullptr, nullptr,
@@ -423,6 +448,53 @@ static int process_command(MYSQL_THD thd, LEX_CSTRING event_command,
   return 0;
 }
 
+#ifndef NDEBUG
+/*
+ * Exposes a generic audit log event in a status variable
+ */
+static void log_event(const mysql_event_general *event) {
+#define EVENT_PARAM(name) event_str << #name ":" << event->name << ";";
+#define EVENT_PARAM_STR(name)                             \
+  {                                                       \
+    std::string tmp(event->name.str, event->name.length); \
+    boost::replace_all(tmp, "\n", "\\n");                 \
+    event_str << #name ":" << tmp << ";";                 \
+  }
+  std::stringstream event_str;
+  EVENT_PARAM(event_subclass);
+  EVENT_PARAM(general_error_code);
+  // skipping general_thread_id
+  EVENT_PARAM_STR(general_user);
+  EVENT_PARAM_STR(general_command);
+  EVENT_PARAM_STR(general_query);
+  // EVENT_PARAM_STR(general_charset);
+  EVENT_PARAM(general_time);
+  EVENT_PARAM(general_rows);
+  EVENT_PARAM_STR(general_host);
+  EVENT_PARAM_STR(general_sql_command);
+  EVENT_PARAM_STR(general_external_user);
+  EVENT_PARAM_STR(general_ip);
+  EVENT_PARAM(query_id);
+  EVENT_PARAM_STR(database);
+  EVENT_PARAM(affected_rows);
+  EVENT_PARAM(port);
+
+  const std::string str = event_str.str();
+  strncpy(generic_event_response, str.c_str(), event_response_buffer_len);
+}
+
+static void log_connect_event(const mysql_event_connection *event) {
+  std::stringstream event_str;
+
+  EVENT_PARAM_STR(connection_certificate);
+
+  const std::string str = event_str.str();
+  strncpy(connect_event_response, str.c_str(), event_response_buffer_len);
+}
+#undef EVENT_PARAM
+#undef EVENT_PARAM_STR
+#endif
+
 /**
   @brief Plugin function handler.
 
@@ -467,9 +539,16 @@ static int audit_null_notify(MYSQL_THD thd, mysql_event_class_t event_class,
       case MYSQL_AUDIT_GENERAL_RESULT:
         number_of_calls_general_result++;
         break;
-      case MYSQL_AUDIT_GENERAL_STATUS:
+      case MYSQL_AUDIT_GENERAL_STATUS: {
+#ifndef NDEBUG
+        const int extended_info = static_cast<int>(THDVAR(thd, extended_log));
+        if (extended_info != 0) {
+          log_event(static_cast<const mysql_event_general *>(event));
+        }
+#endif
         number_of_calls_general_status++;
         break;
+      }
       default:
         break;
     }
@@ -478,9 +557,16 @@ static int audit_null_notify(MYSQL_THD thd, mysql_event_class_t event_class,
         (const struct mysql_event_connection *)event;
 
     switch (event_connection->event_subclass) {
-      case MYSQL_AUDIT_CONNECTION_CONNECT:
+      case MYSQL_AUDIT_CONNECTION_CONNECT: {
+#ifndef NDEBUG
+        const int extended_info = static_cast<int>(THDVAR(thd, extended_log));
+        if (extended_info != 0) {
+          log_connect_event(event_connection);
+        }
+#endif
         number_of_calls_connection_connect++;
         break;
+      }
       case MYSQL_AUDIT_CONNECTION_DISCONNECT:
         number_of_calls_connection_disconnect++;
         break;
@@ -831,6 +917,9 @@ static SYS_VAR *system_variables[] = {
     MYSQL_SYSVAR(event_order_check_consume_ignore_count),
     MYSQL_SYSVAR(event_order_started),
     MYSQL_SYSVAR(event_order_check_exact),
+#ifndef NDEBUG
+    MYSQL_SYSVAR(extended_log),
+#endif
 
     MYSQL_SYSVAR(event_record_def),
     MYSQL_SYSVAR(event_record),