Merge pull request #4713 from sysown/v2.x-logging_mem

V2.x logging mem [WIP]
sysown · Jan 16, 2025 · 8aea001 · 8aea001
2 parents 70b6234 + 737f3a1
commit 8aea001
Show file tree

Hide file tree

Showing 12 changed files with 1,314 additions and 115 deletions.
diff --git a/include/MySQL_Logger.hpp b/include/MySQL_Logger.hpp
diff --git a/include/MySQL_Session.h b/include/MySQL_Session.h
@@ -197,8 +197,8 @@ class MySQL_Session
 	 * @param myds If not null, should point to a MySQL_Data_Stream (backend connection) which connection status
 	 *   should be updated, and previous query resources cleanup.
 	 */
-	void RequestEnd(MySQL_Data_Stream *);
-	void LogQuery(MySQL_Data_Stream *);
+	void RequestEnd(MySQL_Data_Stream * myds, const unsigned int myerrno = 0, const char * errmsg = nullptr);
+	void LogQuery(MySQL_Data_Stream * myds, const unsigned int myerrno = 0, const char * errmsg = nullptr);
 
 	void handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY___create_mirror_session();
 	int handler_again___status_PINGING_SERVER();

diff --git a/include/MySQL_Thread.h b/include/MySQL_Thread.h
@@ -535,6 +535,9 @@ class MySQL_Threads_Handler
 		int connpoll_reset_queue_length;
 		char *eventslog_filename;
 		int eventslog_filesize;
+		int eventslog_buffer_history_size;
+		int eventslog_table_memory_size;
+		int eventslog_buffer_max_query_length;
 		int eventslog_default_log;
 		int eventslog_format;
 		char *auditlog_filename;

diff --git a/include/ProxySQL_Statistics.hpp b/include/ProxySQL_Statistics.hpp
@@ -79,6 +79,7 @@
 
 #define STATSDB_SQLITE_TABLE_HISTORY_MYSQL_QUERY_DIGEST "CREATE TABLE history_mysql_query_digest (dump_time INT , hostgroup INT , schemaname VARCHAR NOT NULL , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , digest VARCHAR NOT NULL , digest_text VARCHAR NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , sum_time INTEGER NOT NULL , min_time INTEGER NOT NULL , max_time INTEGER NOT NULL , sum_rows_affected INTEGER NOT NULL , sum_rows_sent INTEGER NOT NULL)"
 
+#define STATSDB_SQLITE_TABLE_HISTORY_MYSQL_QUERY_EVENTS "CREATE TABLE history_mysql_query_events (id INTEGER PRIMARY KEY AUTOINCREMENT , thread_id INTEGER , username TEXT , schemaname TEXT , start_time INTEGER , end_time INTEGER , query_digest TEXT , query TEXT , server TEXT , client TEXT , event_type INTEGER , hid INTEGER , extra_info TEXT , affected_rows INTEGER , last_insert_id INTEGER , rows_sent INTEGER , client_stmt_id INTEGER , gtid TEXT , errno INT , error TEXT)"
 class ProxySQL_Statistics {
 	SQLite3DB *statsdb_mem; // internal statistics DB
 	std::vector<table_def_t *> *tables_defs_statsdb_mem;
@@ -90,6 +91,7 @@ class ProxySQL_Statistics {
 	unsigned long long next_timer_MySQL_Threads_Handler;
 	unsigned long long next_timer_mysql_query_digest_to_disk;
 	unsigned long long next_timer_system_cpu;
+	unsigned long long last_timer_mysql_dump_eventslog_to_disk = 0;
 #ifndef NOJEM
 	unsigned long long next_timer_system_memory;
 #endif
@@ -105,6 +107,7 @@ class ProxySQL_Statistics {
 		int stats_mysql_query_cache;
 		int stats_system_cpu;
 		int stats_mysql_query_digest_to_disk;
+		int stats_mysql_eventslog_sync_buffer_to_disk;
 #ifndef NOJEM
 		int stats_system_memory;
 #endif
@@ -117,6 +120,16 @@ class ProxySQL_Statistics {
 	bool MySQL_Threads_Handler_timetoget(unsigned long long);
 	bool mysql_query_digest_to_disk_timetoget(unsigned long long);
 	bool system_cpu_timetoget(unsigned long long);
+	/**
+	 * @brief Checks if it's time to dump the events log to disk based on the configured interval.
+	 * @param currentTimeMicros The current time in microseconds.
+	 * @return True if it's time to dump the events log, false otherwise.
+	 *
+	 * This function checks if the current time exceeds the last dump time plus the configured dump interval.
+	 * The dump interval is retrieved from the ProxySQL configuration.  If the dump interval is 0, no dumping is performed.
+	 */
+	bool MySQL_Logger_dump_eventslog_timetoget(unsigned long long currentTimeMicros);
+
 #ifndef NOJEM
 	bool system_memory_timetoget(unsigned long long);
 #endif

diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h
@@ -255,6 +255,7 @@ class ProxySQL_Admin {
 		int stats_mysql_connections;
 		int stats_mysql_query_cache;
 		int stats_mysql_query_digest_to_disk;
+		int stats_mysql_eventslog_sync_buffer_to_disk;
 		int stats_system_cpu;
 		int stats_system_memory;
 		int mysql_show_processlist_extended;

diff --git a/include/proxysql_structs.h b/include/proxysql_structs.h
@@ -893,6 +893,9 @@ __thread char * mysql_thread___ssl_p2s_crlpath;
 /* variables used by events log */
 __thread char * mysql_thread___eventslog_filename;
 __thread int mysql_thread___eventslog_filesize;
+__thread int mysql_thread___eventslog_buffer_history_size;
+__thread int mysql_thread___eventslog_table_memory_size;
+__thread int mysql_thread___eventslog_buffer_max_query_length;
 __thread int mysql_thread___eventslog_default_log;
 __thread int mysql_thread___eventslog_format;
 
@@ -1066,6 +1069,9 @@ extern __thread char * mysql_thread___ssl_p2s_crlpath;
 /* variables used by events log */
 extern __thread char * mysql_thread___eventslog_filename;
 extern __thread int mysql_thread___eventslog_filesize;
+extern __thread int mysql_thread___eventslog_buffer_history_size;
+extern __thread int mysql_thread___eventslog_table_memory_size;
+extern __thread int mysql_thread___eventslog_buffer_max_query_length;
 extern __thread int mysql_thread___eventslog_default_log;
 extern __thread int mysql_thread___eventslog_format;