We recently had an instance where mlogger lost its connection to the database resulting in 317 write errors on mysql_insert and (Good) did report connection errors to the Midas log but (Bad) didn't attempt to restore the connection. With some help from Gemini, I've drafted a potential solution to reheal. Our copy uses mysql_query_debug line 1903 in our copy that inserts rows, creates tables, etc:
/* execut sql query */
status = mysql_query(db, query);
if (status)
cm_msg(MERROR, "mysql_query_debug", "SQL error: %s", mysql_error(db));
return status;
to add a reconnection attempt like the following (untested) block:
/* execute sql query */
status = mysql_query(db, query);
if (status != 0) {
unsigned int err_num = mysql_errno(db);
cm_msg(MERROR, "mysql_query_debug", "SQL query execution failed (Error %u: %s).", err_num, mysql_error(db));
cm_msg(MTRCE, "mysql_query_debug", "Network drop suspected. Backing off 5 seconds to heal handle socket...");
#ifdef OS_WIN64
Sleep(5000);
#else
sleep(5);
#endif
cm_msg(MTRCE, "mysql_query_debug", "Initiating handle re-authentication to server...");
// Setting MYSQL_OPT_PROTOCOL to MYSQL_PROTOCOL_TCP ensures that even when passing NULL for host configuration
// fallbacks, the client library strictly forces a network TCP connection instead of a local Unix socket.
unsigned int protocol = MYSQL_PROTOCOL_TCP;
mysql_options(db, MYSQL_OPT_PROTOCOL, &protocol);
if (!mysql_real_connect(db, NULL, NULL, NULL, NULL, 0, NULL, 0)) {
cm_msg(MERROR, "mysql_query_debug", "Automatic handle reconciliation failed: %s. Logging halted.", mysql_error(db));
} else {
cm_msg(MINFO, "mysql_query_debug", "Database connection handle successfully restored.");
// Retry the blocked execution string
status = mysql_query(db, query);
if (status != 0) {
cm_msg(MERROR, "mysql_query_debug", "Query retry failed after reconnection (Error %u: %s).", mysql_errno(db), mysql_error(db));
} else {
cm_msg(MINFO, "mysql_query_debug", "SQL query successfully recovered and executed on retry.");
}
}
}
return status;
|