diff --git a/include/GTID_Server_Data.h b/include/GTID_Server_Data.h new file mode 100644 index 0000000000..9bc9219fda --- /dev/null +++ b/include/GTID_Server_Data.h @@ -0,0 +1,27 @@ +#ifndef CLASS_GTID_Server_Data_H +#define CLASS_GTID_Server_Data_H +class GTID_Server_Data { + public: + char *address; + uint16_t port; + uint16_t mysql_port; + char *data; + size_t len; + size_t size; + size_t pos; + struct ev_io *w; + char uuid_server[64]; + unsigned long long events_read; + gtid_set_t gtid_executed; + bool active; + GTID_Server_Data(struct ev_io *_w, char *_address, uint16_t _port, uint16_t _mysql_port); + void resize(size_t _s); + ~GTID_Server_Data(); + bool readall(); + bool writeout(); + bool read_next_gtid(); + bool gtid_exists(char *gtid_uuid, uint64_t gtid_trxid); + void read_all_gtids(); + void dump(); +}; +#endif // CLASS_GTID_Server_Data_H diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index f01a030ee2..eaa71a293c 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -130,6 +130,9 @@ class MyHGC; std::string gtid_executed_to_string(gtid_set_t& gtid_executed); void addGtid(const gtid_t& gtid, gtid_set_t& gtid_executed); +#include "GTID_Server_Data.h" + +/* class GTID_Server_Data { public: char *address; @@ -154,7 +157,7 @@ class GTID_Server_Data { void read_all_gtids(); void dump(); }; - +*/ class MySrvConnList { diff --git a/include/MySQL_Prepared_Stmt_info.h b/include/MySQL_Prepared_Stmt_info.h new file mode 100644 index 0000000000..ad221720a5 --- /dev/null +++ b/include/MySQL_Prepared_Stmt_info.h @@ -0,0 +1,25 @@ +#ifndef CLASS_MySQL_Prepared_Stmt_info_H +#define CLASS_MySQL_Prepared_Stmt_info_H +class MySQL_Prepared_Stmt_info { + public: + uint32_t statement_id; + uint16_t num_columns; + uint16_t num_params; + uint16_t warning_count; + uint16_t pending_num_columns; + uint16_t pending_num_params; + MySQL_Prepared_Stmt_info(unsigned char *pkt, unsigned int length) { + pkt += 5; + statement_id = CPY4(pkt); + pkt += sizeof(uint32_t); + num_columns = CPY2(pkt); + pkt += sizeof(uint16_t); + num_params = CPY2(pkt); + pkt += sizeof(uint16_t); + pkt++; // reserved_1 + warning_count = CPY2(pkt); + pending_num_columns=num_columns; + pending_num_params=num_params; + } +}; +#endif // CLASS_MySQL_Prepared_Stmt_info_H diff --git a/include/MySQL_Protocol.h b/include/MySQL_Protocol.h index 0d30b223bf..a95251f8bf 100644 --- a/include/MySQL_Protocol.h +++ b/include/MySQL_Protocol.h @@ -4,6 +4,7 @@ #include "proxysql.h" #include "cpp.h" #include "MySQL_Variables.h" +#include "MySQL_Prepared_Stmt_info.h" #define RESULTSET_BUFLEN 16300 @@ -70,16 +71,6 @@ class MySQL_ResultSet { unsigned long long current_size(); }; -class MySQL_Prepared_Stmt_info { - public: - uint32_t statement_id; - uint16_t num_columns; - uint16_t num_params; - uint16_t warning_count; - uint16_t pending_num_columns; - uint16_t pending_num_params; - MySQL_Prepared_Stmt_info(unsigned char *, unsigned int); -}; uint8_t mysql_decode_length(unsigned char *ptr, uint64_t *len); diff --git a/include/MySQL_encode.h b/include/MySQL_encode.h new file mode 100644 index 0000000000..27273d6127 --- /dev/null +++ b/include/MySQL_encode.h @@ -0,0 +1,22 @@ +#ifndef CLASS_MySQL_encode_H +#define CLASS_MySQL_encode_H +#ifdef DEBUG +void __dump_pkt(const char *func, unsigned char *_ptr, unsigned int len); +#endif // DEBUG +char *sha1_pass_hex(char *sha1_pass); +double proxy_my_rnd(struct rand_struct *rand_st); +void proxy_create_random_string(char *_to, uint length, struct rand_struct *rand_st); +int write_encoded_length(unsigned char *p, uint64_t val, uint8_t len, char prefix); +int write_encoded_length_and_string(unsigned char *p, uint64_t val, uint8_t len, char prefix, char *string); +void proxy_compute_sha1_hash_multi(uint8_t *digest, const char *buf1, int len1, const char *buf2, int len2); +void proxy_compute_sha1_hash(uint8_t *digest, const char *buf, int len); +void proxy_compute_two_stage_sha1_hash(const char *password, size_t pass_len, uint8_t *hash_stage1, uint8_t *hash_stage2); +void proxy_my_crypt(char *to, const uint8_t *s1, const uint8_t *s2, uint len); +unsigned char decode_char(char x); +void unhex_pass(uint8_t *out, const char *in); +void proxy_scramble(char *to, const char *message, const char *password); +bool proxy_scramble_sha1(char *pass_reply, const char *message, const char *sha1_sha1_pass, char *sha1_pass); +unsigned int CPY3(unsigned char *ptr); +uint64_t CPY8(unsigned char *ptr); +uint8_t mysql_encode_length(uint64_t len, char *hd); +#endif // CLASS_MySQL_encode_H diff --git a/include/QP_rule_text.h b/include/QP_rule_text.h new file mode 100644 index 0000000000..9cbd7e7368 --- /dev/null +++ b/include/QP_rule_text.h @@ -0,0 +1,22 @@ +#ifndef CLASS_QR_RULE_H +#define CLASS_QR_RULE_H + +#define QP_RE_MOD_CASELESS 1 +#define QP_RE_MOD_GLOBAL 2 + +class QP_rule_text_hitsonly { + public: + char **pta; + QP_rule_text_hitsonly(QP_rule_t *QPr); + ~QP_rule_text_hitsonly(); +}; + +class QP_rule_text { + public: + char **pta; + int num_fields; + QP_rule_text(QP_rule_t *QPr); + ~QP_rule_text(); +}; + +#endif // CLASS_QR_RULE_H diff --git a/include/c_tokenizer.h b/include/c_tokenizer.h index abe0fdb30d..4444ca2e6a 100644 --- a/include/c_tokenizer.h +++ b/include/c_tokenizer.h @@ -1,5 +1,3 @@ -/* c_tokenizer.h */ -// some code borrowed from http://www.cplusplus.com/faq/sequences/strings/split/ #pragma once #ifndef C_TOKENIZER_H @@ -34,7 +32,6 @@ const char* free_tokenizer( tokenizer_t* tokenizer ); const char* tokenize( tokenizer_t* tokenizer ); char * mysql_query_digest_first_stage(const char* const q, int q_len, char** const fst_cmnt, char* const buf); char * mysql_query_digest_second_stage(const char* const q, int q_len, char** const fst_cmnt, char* const buf); -char * mysql_query_digest_and_first_comment(char *s , int len , char **first_comment, char *buf); char * mysql_query_digest_and_first_comment_2(const char* const q, int q_len, char** const fst_cmnt, char* const buf); char * mysql_query_digest_and_first_comment_one_it(char *s , int len , char **first_comment, char *buf); char * mysql_query_strip_comments(char *s , int len); diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 6906091cc7..c56d68bbb6 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -15,6 +15,8 @@ #include "ProxySQL_RESTAPI_Server.hpp" +#include "proxysql_typedefs.h" + typedef struct { uint32_t hash; uint32_t key; } t_symstruct; class ProxySQL_Config; class ProxySQL_Restapi; diff --git a/include/proxysql_typedefs.h b/include/proxysql_typedefs.h new file mode 100644 index 0000000000..f88af0def8 --- /dev/null +++ b/include/proxysql_typedefs.h @@ -0,0 +1,5 @@ +#ifndef PROXYSQL_COMMON_TYPEDEF +#define PROXYSQL_COMMON_TYPEDEF +typedef std::unordered_map umap_query_digest; +typedef std::unordered_map umap_query_digest_text; +#endif // PROXYSQL_COMMON_TYPEDEF diff --git a/include/query_processor.h b/include/query_processor.h index c9dd826a3d..3d7d8d1561 100644 --- a/include/query_processor.h +++ b/include/query_processor.h @@ -16,9 +16,7 @@ #include "khash.h" KHASH_MAP_INIT_STR(khStrInt, int) -typedef std::unordered_map umap_query_digest; -typedef std::unordered_map umap_query_digest_text; - +#include "proxysql_typedefs.h" #define WUS_NOT_FOUND 0 // couldn't find any filter #define WUS_OFF 1 // allow the query diff --git a/lib/GTID_Server_Data.cpp b/lib/GTID_Server_Data.cpp new file mode 100644 index 0000000000..6dbf572354 --- /dev/null +++ b/lib/GTID_Server_Data.cpp @@ -0,0 +1,469 @@ +#include "MySQL_HostGroups_Manager.h" + +#include "ev.h" +#include + + +extern ProxySQL_Admin *GloAdmin; + +extern MySQL_Threads_Handler *GloMTH; + +extern MySQL_Monitor *GloMyMon; + +static pthread_mutex_t ev_loop_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void gtid_async_cb(struct ev_loop *loop, struct ev_async *watcher, int revents) { + if (glovars.shutdown) { + ev_break(loop); + } + pthread_mutex_lock(&ev_loop_mutex); + MyHGM->gtid_missing_nodes = false; + MyHGM->generate_mysql_gtid_executed_tables(); + pthread_mutex_unlock(&ev_loop_mutex); + return; +} + +static void gtid_timer_cb (struct ev_loop *loop, struct ev_timer *timer, int revents) { + if (GloMTH == nullptr) { return; } + ev_timer_stop(loop, timer); + ev_timer_set(timer, __sync_add_and_fetch(&GloMTH->variables.binlog_reader_connect_retry_msec,0)/1000, 0); + if (glovars.shutdown) { + ev_break(loop); + } + if (MyHGM->gtid_missing_nodes) { + pthread_mutex_lock(&ev_loop_mutex); + MyHGM->gtid_missing_nodes = false; + MyHGM->generate_mysql_gtid_executed_tables(); + pthread_mutex_unlock(&ev_loop_mutex); + } + ev_timer_start(loop, timer); + return; +} + +void reader_cb(struct ev_loop *loop, struct ev_io *w, int revents) { + pthread_mutex_lock(&ev_loop_mutex); + if (revents & EV_READ) { + GTID_Server_Data *sd = (GTID_Server_Data *)w->data; + bool rc = true; + rc = sd->readall(); + if (rc == false) { + //delete sd; + std::string s1 = sd->address; + s1.append(":"); + s1.append(std::to_string(sd->mysql_port)); + MyHGM->gtid_missing_nodes = true; + proxy_warning("GTID: failed to connect to ProxySQL binlog reader on port %d for server %s:%d\n", sd->port, sd->address, sd->mysql_port); + std::unordered_map ::iterator it2; + it2 = MyHGM->gtid_map.find(s1); + if (it2 != MyHGM->gtid_map.end()) { + //MyHGM->gtid_map.erase(it2); + it2->second = NULL; + delete sd; + } + ev_io_stop(MyHGM->gtid_ev_loop, w); + free(w); + } else { + sd->dump(); + } + } + pthread_mutex_unlock(&ev_loop_mutex); +} + +void connect_cb(EV_P_ ev_io *w, int revents) { + pthread_mutex_lock(&ev_loop_mutex); + struct ev_io * c = w; + if (revents & EV_WRITE) { + int optval = 0; + socklen_t optlen = sizeof(optval); + if ((getsockopt(w->fd, SOL_SOCKET, SO_ERROR, &optval, &optlen) == -1) || + (optval != 0)) { + /* Connection failed; try the next address in the list. */ + //int errnum = optval ? optval : errno; + ev_io_stop(MyHGM->gtid_ev_loop, w); + close(w->fd); + MyHGM->gtid_missing_nodes = true; + GTID_Server_Data * custom_data = (GTID_Server_Data *)w->data; + GTID_Server_Data *sd = custom_data; + std::string s1 = sd->address; + s1.append(":"); + s1.append(std::to_string(sd->mysql_port)); + proxy_warning("GTID: failed to connect to ProxySQL binlog reader on port %d for server %s:%d\n", sd->port, sd->address, sd->mysql_port); + std::unordered_map ::iterator it2; + it2 = MyHGM->gtid_map.find(s1); + if (it2 != MyHGM->gtid_map.end()) { + //MyHGM->gtid_map.erase(it2); + it2->second = NULL; + delete sd; + } + //delete custom_data; + free(c); + } else { + ev_io_stop(MyHGM->gtid_ev_loop, w); + int fd=w->fd; + struct ev_io * new_w = (struct ev_io*) malloc(sizeof(struct ev_io)); + new_w->data = w->data; + GTID_Server_Data * custom_data = (GTID_Server_Data *)new_w->data; + custom_data->w = new_w; + free(w); + ev_io_init(new_w, reader_cb, fd, EV_READ); + ev_io_start(MyHGM->gtid_ev_loop, new_w); + } + } + pthread_mutex_unlock(&ev_loop_mutex); +} + +struct ev_io * new_connector(char *address, uint16_t gtid_port, uint16_t mysql_port) { + //struct sockaddr_in a; + int s; + + if ((s = socket(AF_INET, SOCK_STREAM, 0)) == -1) { + perror("socket"); + close(s); + return NULL; + } +/* + memset(&a, 0, sizeof(a)); + a.sin_port = htons(gtid_port); + a.sin_family = AF_INET; + if (!inet_aton(address, (struct in_addr *) &a.sin_addr.s_addr)) { + perror("bad IP address format"); + close(s); + return NULL; + } +*/ + ioctl_FIONBIO(s,1); + + struct addrinfo hints; + struct addrinfo *res = NULL; + memset(&hints, 0, sizeof(hints)); + hints.ai_protocol= IPPROTO_TCP; + hints.ai_family= AF_UNSPEC; + hints.ai_socktype= SOCK_STREAM; + + char str_port[NI_MAXSERV+1]; + sprintf(str_port,"%d", gtid_port); + int gai_rc = getaddrinfo(address, str_port, &hints, &res); + if (gai_rc) { + freeaddrinfo(res); + //exit here + return NULL; + } + + //int status = connect(s, (struct sockaddr *) &a, sizeof(a)); + int status = connect(s, res->ai_addr, res->ai_addrlen); + if ((status == 0) || ((status == -1) && (errno == EINPROGRESS))) { + struct ev_io *c = (struct ev_io *)malloc(sizeof(struct ev_io)); + if (c) { + ev_io_init(c, connect_cb, s, EV_WRITE); + GTID_Server_Data * custom_data = new GTID_Server_Data(c, address, gtid_port, mysql_port); + c->data = (void *)custom_data; + return c; + } + /* else error */ + } + return NULL; +} + + + +GTID_Server_Data::GTID_Server_Data(struct ev_io *_w, char *_address, uint16_t _port, uint16_t _mysql_port) { + active = true; + w = _w; + size = 1024; // 1KB buffer + data = (char *)malloc(size); + memset(uuid_server, 0, sizeof(uuid_server)); + pos = 0; + len = 0; + address = strdup(_address); + port = _port; + mysql_port = _mysql_port; + events_read = 0; +} + +void GTID_Server_Data::resize(size_t _s) { + char *data_ = (char *)malloc(_s); + memcpy(data_, data, (_s > size ? size : _s)); + size = _s; + free(data); + data = data_; +} + +GTID_Server_Data::~GTID_Server_Data() { + free(address); + free(data); +} + +bool GTID_Server_Data::readall() { + bool ret = true; + if (size == len) { + // buffer is full, expand + resize(len*2); + } + int rc = 0; + rc = read(w->fd,data+len,size-len); + if (rc > 0) { + len += rc; + } else { + int myerr = errno; + proxy_error("Read returned %d bytes, error %d\n", rc, myerr); + if ( + (rc == 0) || + (rc==-1 && myerr != EINTR && myerr != EAGAIN) + ) { + ret = false; + } + } + return ret; +} + + +bool GTID_Server_Data::gtid_exists(char *gtid_uuid, uint64_t gtid_trxid) { + std::string s = gtid_uuid; + auto it = gtid_executed.find(s); +// fprintf(stderr,"Checking if server %s:%d has GTID %s:%lu ... ", address, port, gtid_uuid, gtid_trxid); + if (it == gtid_executed.end()) { +// fprintf(stderr,"NO\n"); + return false; + } + for (auto itr = it->second.begin(); itr != it->second.end(); ++itr) { + if ((int64_t)gtid_trxid >= itr->first && (int64_t)gtid_trxid <= itr->second) { +// fprintf(stderr,"YES\n"); + return true; + } + } +// fprintf(stderr,"NO\n"); + return false; +} + +void GTID_Server_Data::read_all_gtids() { + while (read_next_gtid()) { + } + } + +void GTID_Server_Data::dump() { + if (len==0) { + return; + } + read_all_gtids(); + //int rc = write(1,data+pos,len-pos); + fflush(stdout); + ///pos += rc; + if (pos >= len/2) { + memmove(data,data+pos,len-pos); + len = len-pos; + pos = 0; + } +} + +bool GTID_Server_Data::writeout() { + bool ret = true; + if (len==0) { + return ret; + } + int rc = 0; + rc = write(w->fd,data+pos,len-pos); + if (rc > 0) { + pos += rc; + if (pos >= len/2) { + memmove(data,data+pos,len-pos); + len = len-pos; + pos = 0; + } + } + return ret; +} + +bool GTID_Server_Data::read_next_gtid() { + if (len==0) { + return false; + } + void *nlp = NULL; + nlp = memchr(data+pos,'\n',len-pos); + if (nlp == NULL) { + return false; + } + int l = (char *)nlp - (data+pos); + char rec_msg[80]; + if (strncmp(data+pos,(char *)"ST=",3)==0) { + // we are reading the bootstrap + char *bs = (char *)malloc(l+1-3); // length + 1 (null byte) - 3 (header) + memcpy(bs, data+pos+3, l-3); + bs[l-3] = '\0'; + char *saveptr1=NULL; + char *saveptr2=NULL; + //char *saveptr3=NULL; + char *token = NULL; + char *subtoken = NULL; + //char *subtoken2 = NULL; + char *str1 = NULL; + char *str2 = NULL; + //char *str3 = NULL; + for (str1 = bs; ; str1 = NULL) { + token = strtok_r(str1, ",", &saveptr1); + if (token == NULL) { + break; + } + int j = 0; + for (str2 = token; ; str2 = NULL) { + subtoken = strtok_r(str2, ":", &saveptr2); + if (subtoken == NULL) { + break; + } + j++; + if (j%2 == 1) { // we are reading the uuid + char *p = uuid_server; + for (unsigned int k=0; kfirst; + s.insert(8,"-"); + s.insert(13,"-"); + s.insert(18,"-"); + s.insert(23,"-"); + s = s + ":"; + for (auto itr = it->second.begin(); itr != it->second.end(); ++itr) { + std::string s2 = s; + s2 = s2 + std::to_string(itr->first); + s2 = s2 + "-"; + s2 = s2 + std::to_string(itr->second); + s2 = s2 + ","; + gtid_set = gtid_set + s2; + } + } + // Extract latest comma only in case 'gtid_executed' isn't empty + if (gtid_set.empty() == false) { + gtid_set.pop_back(); + } + return gtid_set; +} + + + +void addGtid(const gtid_t& gtid, gtid_set_t& gtid_executed) { + auto it = gtid_executed.find(gtid.first); + if (it == gtid_executed.end()) + { + gtid_executed[gtid.first].emplace_back(gtid.second, gtid.second); + return; + } + + bool flag = true; + for (auto itr = it->second.begin(); itr != it->second.end(); ++itr) + { + if (gtid.second >= itr->first && gtid.second <= itr->second) + return; + if (gtid.second + 1 == itr->first) + { + --itr->first; + flag = false; + break; + } + else if (gtid.second == itr->second + 1) + { + ++itr->second; + flag = false; + break; + } + else if (gtid.second < itr->first) + { + it->second.emplace(itr, gtid.second, gtid.second); + return; + } + } + + if (flag) + it->second.emplace_back(gtid.second, gtid.second); + + for (auto itr = it->second.begin(); itr != it->second.end(); ++itr) + { + auto next_itr = std::next(itr); + if (next_itr != it->second.end() && itr->second + 1 == next_itr->first) + { + itr->second = next_itr->second; + it->second.erase(next_itr); + break; + } + } +} + +void * GTID_syncer_run() { + //struct ev_loop * gtid_ev_loop; + //gtid_ev_loop = NULL; + MyHGM->gtid_ev_loop = ev_loop_new (EVBACKEND_POLL | EVFLAG_NOENV); + if (MyHGM->gtid_ev_loop == NULL) { + proxy_error("could not initialise GTID sync loop\n"); + exit(EXIT_FAILURE); + } + //ev_async_init(gtid_ev_async, gtid_async_cb); + //ev_async_start(gtid_ev_loop, gtid_ev_async); + MyHGM->gtid_ev_timer = (struct ev_timer *)malloc(sizeof(struct ev_timer)); + ev_async_init(MyHGM->gtid_ev_async, gtid_async_cb); + ev_async_start(MyHGM->gtid_ev_loop, MyHGM->gtid_ev_async); + //ev_timer_init(MyHGM->gtid_ev_timer, gtid_timer_cb, __sync_add_and_fetch(&GloMTH->variables.binlog_reader_connect_retry_msec,0)/1000, 0); + ev_timer_init(MyHGM->gtid_ev_timer, gtid_timer_cb, 3, 0); + ev_timer_start(MyHGM->gtid_ev_loop, MyHGM->gtid_ev_timer); + //ev_ref(gtid_ev_loop); + ev_run(MyHGM->gtid_ev_loop, 0); + //sleep(1000); + return NULL; +} + diff --git a/lib/Makefile b/lib/Makefile index b94dd2fc65..58b17c29eb 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -127,6 +127,9 @@ default: libproxysql.a _OBJ_CXX := ProxySQL_GloVars.oo network.oo debug.oo configfile.oo Query_Cache.oo SpookyV2.oo MySQL_Authentication.oo gen_utils.oo sqlite3db.oo mysql_connection.oo MySQL_HostGroups_Manager.oo mysql_data_stream.oo MySQL_Thread.oo MySQL_Session.oo MySQL_Protocol.oo mysql_backend.oo Query_Processor.oo ProxySQL_Admin.oo ProxySQL_Config.oo ProxySQL_Restapi.oo MySQL_Monitor.oo MySQL_Logger.oo thread.oo MySQL_PreparedStatement.oo ProxySQL_Cluster.oo ClickHouse_Authentication.oo ClickHouse_Server.oo ProxySQL_Statistics.oo Chart_bundle_js.oo ProxySQL_HTTP_Server.oo ProxySQL_RESTAPI_Server.oo font-awesome.min.css.oo main-bundle.min.css.oo set_parser.oo MySQL_Variables.oo c_tokenizer.oo proxysql_utils.oo proxysql_coredump.oo proxysql_sslkeylog.oo \ sha256crypt.oo \ + QP_rule_text.oo QP_query_digest_stats.oo \ + GTID_Server_Data.oo MyHGC.oo MySrvConnList.oo MySrvList.oo MySrvC.oo \ + MySQL_encode.oo MySQL_ResultSet.oo \ proxysql_find_charset.oo ProxySQL_Poll.oo OBJ_CXX := $(patsubst %,$(ODIR)/%,$(_OBJ_CXX)) HEADERS := ../include/*.h ../include/*.hpp diff --git a/lib/MyHGC.cpp b/lib/MyHGC.cpp new file mode 100644 index 0000000000..6daa6be295 --- /dev/null +++ b/lib/MyHGC.cpp @@ -0,0 +1,384 @@ +#include "MySQL_HostGroups_Manager.h" + + +extern MySQL_Threads_Handler *GloMTH; + +MyHGC::MyHGC(int _hid) { + hid=_hid; + mysrvs=new MySrvList(this); + current_time_now = 0; + new_connections_now = 0; + attributes.initialized = false; + reset_attributes(); + // Uninitialized server defaults. Should later be initialized via 'mysql_hostgroup_attributes'. + servers_defaults.weight = -1; + servers_defaults.max_connections = -1; + servers_defaults.use_ssl = -1; +} + +void MyHGC::reset_attributes() { + if (attributes.initialized == false) { + attributes.init_connect = NULL; + attributes.comment = NULL; + attributes.ignore_session_variables_text = NULL; + } + attributes.initialized = true; + attributes.configured = false; + attributes.max_num_online_servers = 1000000; + attributes.throttle_connections_per_sec = 1000000; + attributes.autocommit = -1; + attributes.free_connections_pct = 10; + attributes.handle_warnings = -1; + attributes.multiplex = true; + attributes.connection_warming = false; + free(attributes.init_connect); + attributes.init_connect = NULL; + free(attributes.comment); + attributes.comment = NULL; + free(attributes.ignore_session_variables_text); + attributes.ignore_session_variables_text = NULL; + attributes.ignore_session_variables_json = json(); +} + +MyHGC::~MyHGC() { + reset_attributes(); // free all memory + delete mysrvs; +} + +MySrvC *MyHGC::get_random_MySrvC(char * gtid_uuid, uint64_t gtid_trxid, int max_lag_ms, MySQL_Session *sess) { + MySrvC *mysrvc=NULL; + unsigned int j; + unsigned int sum=0; + unsigned int TotalUsedConn=0; + unsigned int l=mysrvs->cnt(); + static time_t last_hg_log = 0; +#ifdef TEST_AURORA + unsigned long long a1 = array_mysrvc_total/10000; + array_mysrvc_total += l; + unsigned long long a2 = array_mysrvc_total/10000; + if (a2 > a1) { + fprintf(stderr, "Total: %llu, Candidates: %llu\n", array_mysrvc_total-l, array_mysrvc_cands); + } +#endif // TEST_AURORA + MySrvC *mysrvcCandidates_static[32]; + MySrvC **mysrvcCandidates = mysrvcCandidates_static; + unsigned int num_candidates = 0; + bool max_connections_reached = false; + if (l>32) { + mysrvcCandidates = (MySrvC **)malloc(sizeof(MySrvC *)*l); + } + if (l) { + //int j=0; + for (j=0; jidx(j); + if (mysrvc->status==MYSQL_SERVER_STATUS_ONLINE) { // consider this server only if ONLINE + if (mysrvc->ConnectionsUsed->conns_length() < mysrvc->max_connections) { // consider this server only if didn't reach max_connections + if ( mysrvc->current_latency_us < ( mysrvc->max_latency_us ? mysrvc->max_latency_us : mysql_thread___default_max_latency_ms*1000 ) ) { // consider the host only if not too far + if (gtid_trxid) { + if (MyHGM->gtid_exists(mysrvc, gtid_uuid, gtid_trxid)) { + sum+=mysrvc->weight; + TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); + mysrvcCandidates[num_candidates]=mysrvc; + num_candidates++; + } + } else { + if (max_lag_ms >= 0) { + if ((unsigned int)max_lag_ms >= mysrvc->aws_aurora_current_lag_us/1000) { + sum+=mysrvc->weight; + TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); + mysrvcCandidates[num_candidates]=mysrvc; + num_candidates++; + } else { + sess->thread->status_variables.stvar[st_var_aws_aurora_replicas_skipped_during_query]++; + } + } else { + sum+=mysrvc->weight; + TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); + mysrvcCandidates[num_candidates]=mysrvc; + num_candidates++; + } + } + } + } else { + max_connections_reached = true; + } + } else { + if (mysrvc->status==MYSQL_SERVER_STATUS_SHUNNED) { + // try to recover shunned servers + if (mysrvc->shunned_automatic && mysql_thread___shun_recovery_time_sec) { + time_t t; + t=time(NULL); + // we do all these changes without locking . We assume the server is not used from long + // even if the server is still in used and any of the follow command fails it is not critical + // because this is only an attempt to recover a server that is probably dead anyway + + // the next few lines of code try to solve issue #530 + int max_wait_sec = ( mysql_thread___shun_recovery_time_sec * 1000 >= mysql_thread___connect_timeout_server_max ? mysql_thread___connect_timeout_server_max/1000 - 1 : mysql_thread___shun_recovery_time_sec ); + if (max_wait_sec < 1) { // min wait time should be at least 1 second + max_wait_sec = 1; + } + if (t > mysrvc->time_last_detected_error && (t - mysrvc->time_last_detected_error) > max_wait_sec) { + if ( + (mysrvc->shunned_and_kill_all_connections==false) // it is safe to bring it back online + || + (mysrvc->shunned_and_kill_all_connections==true && mysrvc->ConnectionsUsed->conns_length()==0 && mysrvc->ConnectionsFree->conns_length()==0) // if shunned_and_kill_all_connections is set, ensure all connections are already dropped + ) { +#ifdef DEBUG + if (GloMTH->variables.hostgroup_manager_verbose >= 3) { + proxy_info("Unshunning server %s:%d.\n", mysrvc->address, mysrvc->port); + } +#endif + mysrvc->status=MYSQL_SERVER_STATUS_ONLINE; + mysrvc->shunned_automatic=false; + mysrvc->shunned_and_kill_all_connections=false; + mysrvc->connect_ERR_at_time_last_detected_error=0; + mysrvc->time_last_detected_error=0; + // note: the following function scans all the hostgroups. + // This is ok for now because we only have a global mutex. + // If one day we implement a mutex per hostgroup (unlikely, + // but possible), this must be taken into consideration + if (mysql_thread___unshun_algorithm == 1) { + MyHGM->unshun_server_all_hostgroups(mysrvc->address, mysrvc->port, t, max_wait_sec, &mysrvc->myhgc->hid); + } + // if a server is taken back online, consider it immediately + if ( mysrvc->current_latency_us < ( mysrvc->max_latency_us ? mysrvc->max_latency_us : mysql_thread___default_max_latency_ms*1000 ) ) { // consider the host only if not too far + if (gtid_trxid) { + if (MyHGM->gtid_exists(mysrvc, gtid_uuid, gtid_trxid)) { + sum+=mysrvc->weight; + TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); + mysrvcCandidates[num_candidates]=mysrvc; + num_candidates++; + } + } else { + if (max_lag_ms >= 0) { + if ((unsigned int)max_lag_ms >= mysrvc->aws_aurora_current_lag_us/1000) { + sum+=mysrvc->weight; + TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); + mysrvcCandidates[num_candidates]=mysrvc; + num_candidates++; + } + } else { + sum+=mysrvc->weight; + TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); + mysrvcCandidates[num_candidates]=mysrvc; + num_candidates++; + } + } + } + } + } + } + } + } + } + if (max_lag_ms > 0) { // we are using AWS Aurora, as this logic is implemented only here + unsigned int min_num_replicas = sess->thread->variables.aurora_max_lag_ms_only_read_from_replicas; + if (min_num_replicas) { + if (num_candidates >= min_num_replicas) { // there are at least N replicas + // we try to remove the writer + unsigned int total_aws_aurora_current_lag_us=0; + for (j=0; jaws_aurora_current_lag_us; + } + if (total_aws_aurora_current_lag_us) { // we are just double checking that we don't have all servers with aws_aurora_current_lag_us==0 + for (j=0; jaws_aurora_current_lag_us==0) { + sum-=mysrvc->weight; + TotalUsedConn-=mysrvc->ConnectionsUsed->conns_length(); + if (j < num_candidates-1) { + mysrvcCandidates[j]=mysrvcCandidates[num_candidates-1]; + } + num_candidates--; + } + } + } + } + } + } + if (sum==0) { + // per issue #531 , we try a desperate attempt to bring back online any shunned server + // we do this lowering the maximum wait time to 10% + // most of the follow code is copied from few lines above + time_t t; + t=time(NULL); + int max_wait_sec = ( mysql_thread___shun_recovery_time_sec * 1000 >= mysql_thread___connect_timeout_server_max ? mysql_thread___connect_timeout_server_max/10000 - 1 : mysql_thread___shun_recovery_time_sec/10 ); + if (max_wait_sec < 1) { // min wait time should be at least 1 second + max_wait_sec = 1; + } + if (t - last_hg_log > 1) { // log this at most once per second to avoid spamming the logs + last_hg_log = time(NULL); + + if (gtid_trxid) { + proxy_error("Hostgroup %u has no servers ready for GTID '%s:%ld'. Waiting for replication...\n", hid, gtid_uuid, gtid_trxid); + } else { + proxy_error("Hostgroup %u has no servers available%s! Checking servers shunned for more than %u second%s\n", hid, + (max_connections_reached ? " or max_connections reached for all servers" : ""), max_wait_sec, max_wait_sec == 1 ? "" : "s"); + } + } + for (j=0; jidx(j); + if (mysrvc->status==MYSQL_SERVER_STATUS_SHUNNED && mysrvc->shunned_automatic==true) { + if ((t - mysrvc->time_last_detected_error) > max_wait_sec) { + mysrvc->status=MYSQL_SERVER_STATUS_ONLINE; + mysrvc->shunned_automatic=false; + mysrvc->connect_ERR_at_time_last_detected_error=0; + mysrvc->time_last_detected_error=0; + // if a server is taken back online, consider it immediately + if ( mysrvc->current_latency_us < ( mysrvc->max_latency_us ? mysrvc->max_latency_us : mysql_thread___default_max_latency_ms*1000 ) ) { // consider the host only if not too far + if (gtid_trxid) { + if (MyHGM->gtid_exists(mysrvc, gtid_uuid, gtid_trxid)) { + sum+=mysrvc->weight; + TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); + mysrvcCandidates[num_candidates]=mysrvc; + num_candidates++; + } + } else { + if (max_lag_ms >= 0) { + if ((unsigned int)max_lag_ms >= mysrvc->aws_aurora_current_lag_us/1000) { + sum+=mysrvc->weight; + TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); + mysrvcCandidates[num_candidates]=mysrvc; + num_candidates++; + } + } else { + sum+=mysrvc->weight; + TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); + mysrvcCandidates[num_candidates]=mysrvc; + num_candidates++; + } + } + } + } + } + } + } + if (sum==0) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySrvC NULL because no backend ONLINE or with weight\n"); + if (l>32) { + free(mysrvcCandidates); + } +#ifdef TEST_AURORA + array_mysrvc_cands += num_candidates; +#endif // TEST_AURORA + return NULL; // if we reach here, we couldn't find any target + } + +/* + unsigned int New_sum=0; + unsigned int New_TotalUsedConn=0; + // we will now scan again to ignore overloaded servers + for (j=0; jConnectionsUsed->conns_length(); + if ((len * sum) <= (TotalUsedConn * mysrvc->weight * 1.5 + 1)) { + + New_sum+=mysrvc->weight; + New_TotalUsedConn+=len; + } else { + // remove the candidate + if (j+1 < num_candidates) { + mysrvcCandidates[j] = mysrvcCandidates[num_candidates-1]; + } + j--; + num_candidates--; + } + } +*/ + + unsigned int New_sum=sum; + + if (New_sum==0) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySrvC NULL because no backend ONLINE or with weight\n"); + if (l>32) { + free(mysrvcCandidates); + } +#ifdef TEST_AURORA + array_mysrvc_cands += num_candidates; +#endif // TEST_AURORA + return NULL; // if we reach here, we couldn't find any target + } + + // latency awareness algorithm is enabled only when compiled with USE_MYSRVC_ARRAY + if (sess && sess->thread->variables.min_num_servers_lantency_awareness) { + if ((int) num_candidates >= sess->thread->variables.min_num_servers_lantency_awareness) { + unsigned int servers_with_latency = 0; + unsigned int total_latency_us = 0; + // scan and verify that all servers have some latency + for (j=0; jcurrent_latency_us) { + servers_with_latency++; + total_latency_us += mysrvc->current_latency_us; + } + } + if (servers_with_latency == num_candidates) { + // all servers have some latency. + // That is good. If any server have no latency, something is wrong + // and we will skip this algorithm + sess->thread->status_variables.stvar[st_var_ConnPool_get_conn_latency_awareness]++; + unsigned int avg_latency_us = 0; + avg_latency_us = total_latency_us/num_candidates; + for (j=0; jcurrent_latency_us > avg_latency_us) { + // remove the candidate + if (j+1 < num_candidates) { + mysrvcCandidates[j] = mysrvcCandidates[num_candidates-1]; + } + j--; + num_candidates--; + } + } + // we scan again to adjust weight + New_sum = 0; + for (j=0; jweight; + } + } + } + } + + + unsigned int k; + if (New_sum > 32768) { + k=rand()%New_sum; + } else { + k=fastrand()%New_sum; + } + k++; + New_sum=0; + + for (j=0; jweight; + if (k<=New_sum) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySrvC %p, server %s:%d\n", mysrvc, mysrvc->address, mysrvc->port); + if (l>32) { + free(mysrvcCandidates); + } +#ifdef TEST_AURORA + array_mysrvc_cands += num_candidates; +#endif // TEST_AURORA + return mysrvc; + } + } + } else { + time_t t = time(NULL); + + if (t - last_hg_log > 1) { + last_hg_log = time(NULL); + proxy_error("Hostgroup %u has no servers available!\n", hid); + } + } + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySrvC NULL\n"); + if (l>32) { + free(mysrvcCandidates); + } +#ifdef TEST_AURORA + array_mysrvc_cands += num_candidates; +#endif // TEST_AURORA + return NULL; // if we reach here, we couldn't find any target +} diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index d1f6188ec4..90bc415367 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -57,39 +57,9 @@ class MySrvC; class MySrvList; class MyHGC; -//static struct ev_async * gtid_ev_async; +struct ev_io * new_connector(char *address, uint16_t gtid_port, uint16_t mysql_port); +void * GTID_syncer_run(); -static pthread_mutex_t ev_loop_mutex; - -//static std::unordered_map gtid_map; - -static void gtid_async_cb(struct ev_loop *loop, struct ev_async *watcher, int revents) { - if (glovars.shutdown) { - ev_break(loop); - } - pthread_mutex_lock(&ev_loop_mutex); - MyHGM->gtid_missing_nodes = false; - MyHGM->generate_mysql_gtid_executed_tables(); - pthread_mutex_unlock(&ev_loop_mutex); - return; -} - -static void gtid_timer_cb (struct ev_loop *loop, struct ev_timer *timer, int revents) { - if (GloMTH == nullptr) { return; } - ev_timer_stop(loop, timer); - ev_timer_set(timer, __sync_add_and_fetch(&GloMTH->variables.binlog_reader_connect_retry_msec,0)/1000, 0); - if (glovars.shutdown) { - ev_break(loop); - } - if (MyHGM->gtid_missing_nodes) { - pthread_mutex_lock(&ev_loop_mutex); - MyHGM->gtid_missing_nodes = false; - MyHGM->generate_mysql_gtid_executed_tables(); - pthread_mutex_unlock(&ev_loop_mutex); - } - ev_timer_start(loop, timer); - return; -} static int wait_for_mysql(MYSQL *mysql, int status) { struct pollfd pfd; @@ -115,432 +85,6 @@ static int wait_for_mysql(MYSQL *mysql, int status) { } } -void reader_cb(struct ev_loop *loop, struct ev_io *w, int revents) { - pthread_mutex_lock(&ev_loop_mutex); - if (revents & EV_READ) { - GTID_Server_Data *sd = (GTID_Server_Data *)w->data; - bool rc = true; - rc = sd->readall(); - if (rc == false) { - //delete sd; - std::string s1 = sd->address; - s1.append(":"); - s1.append(std::to_string(sd->mysql_port)); - MyHGM->gtid_missing_nodes = true; - proxy_warning("GTID: failed to connect to ProxySQL binlog reader on port %d for server %s:%d\n", sd->port, sd->address, sd->mysql_port); - std::unordered_map ::iterator it2; - it2 = MyHGM->gtid_map.find(s1); - if (it2 != MyHGM->gtid_map.end()) { - //MyHGM->gtid_map.erase(it2); - it2->second = NULL; - delete sd; - } - ev_io_stop(MyHGM->gtid_ev_loop, w); - free(w); - } else { - sd->dump(); - } - } - pthread_mutex_unlock(&ev_loop_mutex); -} - -void connect_cb(EV_P_ ev_io *w, int revents) { - pthread_mutex_lock(&ev_loop_mutex); - struct ev_io * c = w; - if (revents & EV_WRITE) { - int optval = 0; - socklen_t optlen = sizeof(optval); - if ((getsockopt(w->fd, SOL_SOCKET, SO_ERROR, &optval, &optlen) == -1) || - (optval != 0)) { - /* Connection failed; try the next address in the list. */ - //int errnum = optval ? optval : errno; - ev_io_stop(MyHGM->gtid_ev_loop, w); - close(w->fd); - MyHGM->gtid_missing_nodes = true; - GTID_Server_Data * custom_data = (GTID_Server_Data *)w->data; - GTID_Server_Data *sd = custom_data; - std::string s1 = sd->address; - s1.append(":"); - s1.append(std::to_string(sd->mysql_port)); - proxy_warning("GTID: failed to connect to ProxySQL binlog reader on port %d for server %s:%d\n", sd->port, sd->address, sd->mysql_port); - std::unordered_map ::iterator it2; - it2 = MyHGM->gtid_map.find(s1); - if (it2 != MyHGM->gtid_map.end()) { - //MyHGM->gtid_map.erase(it2); - it2->second = NULL; - delete sd; - } - //delete custom_data; - free(c); - } else { - ev_io_stop(MyHGM->gtid_ev_loop, w); - int fd=w->fd; - struct ev_io * new_w = (struct ev_io*) malloc(sizeof(struct ev_io)); - new_w->data = w->data; - GTID_Server_Data * custom_data = (GTID_Server_Data *)new_w->data; - custom_data->w = new_w; - free(w); - ev_io_init(new_w, reader_cb, fd, EV_READ); - ev_io_start(MyHGM->gtid_ev_loop, new_w); - } - } - pthread_mutex_unlock(&ev_loop_mutex); -} - -struct ev_io * new_connector(char *address, uint16_t gtid_port, uint16_t mysql_port) { - //struct sockaddr_in a; - int s; - - if ((s = socket(AF_INET, SOCK_STREAM, 0)) == -1) { - perror("socket"); - close(s); - return NULL; - } -/* - memset(&a, 0, sizeof(a)); - a.sin_port = htons(gtid_port); - a.sin_family = AF_INET; - if (!inet_aton(address, (struct in_addr *) &a.sin_addr.s_addr)) { - perror("bad IP address format"); - close(s); - return NULL; - } -*/ - ioctl_FIONBIO(s,1); - - struct addrinfo hints; - struct addrinfo *res = NULL; - memset(&hints, 0, sizeof(hints)); - hints.ai_protocol= IPPROTO_TCP; - hints.ai_family= AF_UNSPEC; - hints.ai_socktype= SOCK_STREAM; - - char str_port[NI_MAXSERV+1]; - sprintf(str_port,"%d", gtid_port); - int gai_rc = getaddrinfo(address, str_port, &hints, &res); - if (gai_rc) { - freeaddrinfo(res); - //exit here - return NULL; - } - - //int status = connect(s, (struct sockaddr *) &a, sizeof(a)); - int status = connect(s, res->ai_addr, res->ai_addrlen); - if ((status == 0) || ((status == -1) && (errno == EINPROGRESS))) { - struct ev_io *c = (struct ev_io *)malloc(sizeof(struct ev_io)); - if (c) { - ev_io_init(c, connect_cb, s, EV_WRITE); - GTID_Server_Data * custom_data = new GTID_Server_Data(c, address, gtid_port, mysql_port); - c->data = (void *)custom_data; - return c; - } - /* else error */ - } - return NULL; -} - - - -GTID_Server_Data::GTID_Server_Data(struct ev_io *_w, char *_address, uint16_t _port, uint16_t _mysql_port) { - active = true; - w = _w; - size = 1024; // 1KB buffer - data = (char *)malloc(size); - memset(uuid_server, 0, sizeof(uuid_server)); - pos = 0; - len = 0; - address = strdup(_address); - port = _port; - mysql_port = _mysql_port; - events_read = 0; -} - -void GTID_Server_Data::resize(size_t _s) { - char *data_ = (char *)malloc(_s); - memcpy(data_, data, (_s > size ? size : _s)); - size = _s; - free(data); - data = data_; -} - -GTID_Server_Data::~GTID_Server_Data() { - free(address); - free(data); -} - -bool GTID_Server_Data::readall() { - bool ret = true; - if (size == len) { - // buffer is full, expand - resize(len*2); - } - int rc = 0; - rc = read(w->fd,data+len,size-len); - if (rc > 0) { - len += rc; - } else { - int myerr = errno; - proxy_error("Read returned %d bytes, error %d\n", rc, myerr); - if ( - (rc == 0) || - (rc==-1 && myerr != EINTR && myerr != EAGAIN) - ) { - ret = false; - } - } - return ret; -} - - -bool GTID_Server_Data::gtid_exists(char *gtid_uuid, uint64_t gtid_trxid) { - std::string s = gtid_uuid; - auto it = gtid_executed.find(s); -// fprintf(stderr,"Checking if server %s:%d has GTID %s:%lu ... ", address, port, gtid_uuid, gtid_trxid); - if (it == gtid_executed.end()) { -// fprintf(stderr,"NO\n"); - return false; - } - for (auto itr = it->second.begin(); itr != it->second.end(); ++itr) { - if ((int64_t)gtid_trxid >= itr->first && (int64_t)gtid_trxid <= itr->second) { -// fprintf(stderr,"YES\n"); - return true; - } - } -// fprintf(stderr,"NO\n"); - return false; -} - -void GTID_Server_Data::read_all_gtids() { - while (read_next_gtid()) { - } - } - -void GTID_Server_Data::dump() { - if (len==0) { - return; - } - read_all_gtids(); - //int rc = write(1,data+pos,len-pos); - fflush(stdout); - ///pos += rc; - if (pos >= len/2) { - memmove(data,data+pos,len-pos); - len = len-pos; - pos = 0; - } -} - -bool GTID_Server_Data::writeout() { - bool ret = true; - if (len==0) { - return ret; - } - int rc = 0; - rc = write(w->fd,data+pos,len-pos); - if (rc > 0) { - pos += rc; - if (pos >= len/2) { - memmove(data,data+pos,len-pos); - len = len-pos; - pos = 0; - } - } - return ret; -} - -bool GTID_Server_Data::read_next_gtid() { - if (len==0) { - return false; - } - void *nlp = NULL; - nlp = memchr(data+pos,'\n',len-pos); - if (nlp == NULL) { - return false; - } - int l = (char *)nlp - (data+pos); - char rec_msg[80]; - if (strncmp(data+pos,(char *)"ST=",3)==0) { - // we are reading the bootstrap - char *bs = (char *)malloc(l+1-3); // length + 1 (null byte) - 3 (header) - memcpy(bs, data+pos+3, l-3); - bs[l-3] = '\0'; - char *saveptr1=NULL; - char *saveptr2=NULL; - //char *saveptr3=NULL; - char *token = NULL; - char *subtoken = NULL; - //char *subtoken2 = NULL; - char *str1 = NULL; - char *str2 = NULL; - //char *str3 = NULL; - for (str1 = bs; ; str1 = NULL) { - token = strtok_r(str1, ",", &saveptr1); - if (token == NULL) { - break; - } - int j = 0; - for (str2 = token; ; str2 = NULL) { - subtoken = strtok_r(str2, ":", &saveptr2); - if (subtoken == NULL) { - break; - } - j++; - if (j%2 == 1) { // we are reading the uuid - char *p = uuid_server; - for (unsigned int k=0; kfirst; - s.insert(8,"-"); - s.insert(13,"-"); - s.insert(18,"-"); - s.insert(23,"-"); - s = s + ":"; - for (auto itr = it->second.begin(); itr != it->second.end(); ++itr) { - std::string s2 = s; - s2 = s2 + std::to_string(itr->first); - s2 = s2 + "-"; - s2 = s2 + std::to_string(itr->second); - s2 = s2 + ","; - gtid_set = gtid_set + s2; - } - } - // Extract latest comma only in case 'gtid_executed' isn't empty - if (gtid_set.empty() == false) { - gtid_set.pop_back(); - } - return gtid_set; -} - - - -void addGtid(const gtid_t& gtid, gtid_set_t& gtid_executed) { - auto it = gtid_executed.find(gtid.first); - if (it == gtid_executed.end()) - { - gtid_executed[gtid.first].emplace_back(gtid.second, gtid.second); - return; - } - - bool flag = true; - for (auto itr = it->second.begin(); itr != it->second.end(); ++itr) - { - if (gtid.second >= itr->first && gtid.second <= itr->second) - return; - if (gtid.second + 1 == itr->first) - { - --itr->first; - flag = false; - break; - } - else if (gtid.second == itr->second + 1) - { - ++itr->second; - flag = false; - break; - } - else if (gtid.second < itr->first) - { - it->second.emplace(itr, gtid.second, gtid.second); - return; - } - } - - if (flag) - it->second.emplace_back(gtid.second, gtid.second); - - for (auto itr = it->second.begin(); itr != it->second.end(); ++itr) - { - auto next_itr = std::next(itr); - if (next_itr != it->second.end() && itr->second + 1 == next_itr->first) - { - itr->second = next_itr->second; - it->second.erase(next_itr); - break; - } - } -} - -static void * GTID_syncer_run() { - //struct ev_loop * gtid_ev_loop; - //gtid_ev_loop = NULL; - MyHGM->gtid_ev_loop = ev_loop_new (EVBACKEND_POLL | EVFLAG_NOENV); - if (MyHGM->gtid_ev_loop == NULL) { - proxy_error("could not initialise GTID sync loop\n"); - exit(EXIT_FAILURE); - } - //ev_async_init(gtid_ev_async, gtid_async_cb); - //ev_async_start(gtid_ev_loop, gtid_ev_async); - MyHGM->gtid_ev_timer = (struct ev_timer *)malloc(sizeof(struct ev_timer)); - ev_async_init(MyHGM->gtid_ev_async, gtid_async_cb); - ev_async_start(MyHGM->gtid_ev_loop, MyHGM->gtid_ev_async); - //ev_timer_init(MyHGM->gtid_ev_timer, gtid_timer_cb, __sync_add_and_fetch(&GloMTH->variables.binlog_reader_connect_retry_msec,0)/1000, 0); - ev_timer_init(MyHGM->gtid_ev_timer, gtid_timer_cb, 3, 0); - ev_timer_start(MyHGM->gtid_ev_loop, MyHGM->gtid_ev_timer); - //ev_ref(gtid_ev_loop); - ev_run(MyHGM->gtid_ev_loop, 0); - //sleep(1000); - return NULL; -} //static void * HGCU_thread_run() { static void * HGCU_thread_run() { @@ -661,266 +205,6 @@ static void * HGCU_thread_run() { } -MySQL_Connection *MySrvConnList::index(unsigned int _k) { - return (MySQL_Connection *)conns->index(_k); -} - -MySQL_Connection * MySrvConnList::remove(int _k) { - return (MySQL_Connection *)conns->remove_index_fast(_k); -} - -/* -unsigned int MySrvConnList::conns_length() { - return conns->len; -} -*/ - -MySrvConnList::MySrvConnList(MySrvC *_mysrvc) { - mysrvc=_mysrvc; - conns=new PtrArray(); -} - -void MySrvConnList::add(MySQL_Connection *c) { - conns->add(c); -} - -MySrvConnList::~MySrvConnList() { - mysrvc=NULL; - while (conns_length()) { - MySQL_Connection *conn=(MySQL_Connection *)conns->remove_index_fast(0); - delete conn; - } - delete conns; -} - -MySrvList::MySrvList(MyHGC *_myhgc) { - myhgc=_myhgc; - servers=new PtrArray(); -} - -void MySrvList::add(MySrvC *s) { - if (s->myhgc==NULL) { - s->myhgc=myhgc; - } - servers->add(s); -} - - -int MySrvList::find_idx(MySrvC *s) { - for (unsigned int i=0; ilen; i++) { - MySrvC *mysrv=(MySrvC *)servers->index(i); - if (mysrv==s) { - return (unsigned int)i; - } - } - return -1; -} - -void MySrvList::remove(MySrvC *s) { - int i=find_idx(s); - assert(i>=0); - servers->remove_index_fast((unsigned int)i); -} - -void MySrvConnList::drop_all_connections() { - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Dropping all connections (%u total) on MySrvConnList %p for server %s:%d , hostgroup=%d , status=%d\n", conns_length(), this, mysrvc->address, mysrvc->port, mysrvc->myhgc->hid, mysrvc->status); - while (conns_length()) { - MySQL_Connection *conn=(MySQL_Connection *)conns->remove_index_fast(0); - delete conn; - } -} - - -MySrvC::MySrvC( - char* add, uint16_t p, uint16_t gp, int64_t _weight, enum MySerStatus _status, unsigned int _compression, - int64_t _max_connections, unsigned int _max_replication_lag, int32_t _use_ssl, unsigned int _max_latency_ms, - char* _comment -) { - address=strdup(add); - port=p; - gtid_port=gp; - weight=_weight; - status=_status; - compression=_compression; - max_connections=_max_connections; - max_replication_lag=_max_replication_lag; - use_ssl=_use_ssl; - cur_replication_lag=0; - cur_replication_lag_count=0; - max_latency_us=_max_latency_ms*1000; - current_latency_us=0; - aws_aurora_current_lag_us = 0; - connect_OK=0; - connect_ERR=0; - queries_sent=0; - bytes_sent=0; - bytes_recv=0; - max_connections_used=0; - queries_gtid_sync=0; - time_last_detected_error=0; - connect_ERR_at_time_last_detected_error=0; - shunned_automatic=false; - shunned_and_kill_all_connections=false; // false to default - //charset=_charset; - myhgc=NULL; - comment=strdup(_comment); - ConnectionsUsed=new MySrvConnList(this); - ConnectionsFree=new MySrvConnList(this); -} - -void MySrvC::connect_error(int err_num, bool get_mutex) { - // NOTE: this function operates without any mutex - // although, it is not extremely important if any counter is lost - // as a single connection failure won't make a significant difference - proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Connect failed with code '%d'\n", err_num); - __sync_fetch_and_add(&connect_ERR,1); - __sync_fetch_and_add(&MyHGM->status.server_connections_aborted,1); - if (err_num >= 1048 && err_num <= 1052) - return; - if (err_num >= 1054 && err_num <= 1075) - return; - if (err_num >= 1099 && err_num <= 1104) - return; - if (err_num >= 1106 && err_num <= 1113) - return; - if (err_num >= 1116 && err_num <= 1118) - return; - if (err_num == 1136 || (err_num >= 1138 && err_num <= 1149)) - return; - switch (err_num) { - case 1007: // Can't create database - case 1008: // Can't drop database - case 1044: // access denied - case 1045: // access denied -/* - case 1048: // Column cannot be null - case 1049: // Unknown database - case 1050: // Table already exists - case 1051: // Unknown table - case 1052: // Column is ambiguous -*/ - case 1120: - case 1203: // User %s already has more than 'max_user_connections' active connections - case 1226: // User '%s' has exceeded the '%s' resource (current value: %ld) - case 3118: // Access denied for user '%s'. Account is locked.. - return; - break; - default: - break; - } - time_t t=time(NULL); - if (t > time_last_detected_error) { - time_last_detected_error=t; - connect_ERR_at_time_last_detected_error=1; - } else { - if (t < time_last_detected_error) { - // time_last_detected_error is in the future - // this means that monitor has a ping interval too big and tuned that in the future - return; - } - // same time - /** - * @brief The expected configured retries set by 'mysql-connect_retries_on_failure' + '2' extra expected - * connection errors. - * @details This two extra connections errors are expected: - * 1. An initial connection error generated by the datastream and the connection when being created, - * this is, right after the session has requested a connection to the connection pool. This error takes - * places directly in the state machine from 'MySQL_Connection'. Because of this, we consider this - * additional error to be a consequence of the two states machines, and it's not considered for - * 'connect_retries'. - * 2. A second connection connection error, which is the initial connection error generated by 'MySQL_Session' - * when already in the 'CONNECTING_SERVER' state. This error is an 'extra error' to always consider, since - * it's not part of the retries specified by 'mysql_thread___connect_retries_on_failure', thus, we set the - * 'connect_retries' to be 'mysql_thread___connect_retries_on_failure + 1'. - */ - int connect_retries = mysql_thread___connect_retries_on_failure + 1; - int max_failures = mysql_thread___shun_on_failures > connect_retries ? connect_retries : mysql_thread___shun_on_failures; - - if (__sync_add_and_fetch(&connect_ERR_at_time_last_detected_error,1) >= (unsigned int)max_failures) { - bool _shu=false; - if (get_mutex==true) - MyHGM->wrlock(); // to prevent race conditions, lock here. See #627 - if (status==MYSQL_SERVER_STATUS_ONLINE) { - status=MYSQL_SERVER_STATUS_SHUNNED; - shunned_automatic=true; - _shu=true; - } else { - _shu=false; - } - if (get_mutex==true) - MyHGM->wrunlock(); - if (_shu) { - proxy_error("Shunning server %s:%d with %u errors/sec. Shunning for %u seconds\n", address, port, connect_ERR_at_time_last_detected_error , mysql_thread___shun_recovery_time_sec); - } - } - } -} - -void MySrvC::shun_and_killall() { - status=MYSQL_SERVER_STATUS_SHUNNED; - shunned_automatic=true; - shunned_and_kill_all_connections=true; -} - -MySrvC::~MySrvC() { - if (address) free(address); - if (comment) free(comment); - delete ConnectionsUsed; - delete ConnectionsFree; -} - -MySrvList::~MySrvList() { - myhgc=NULL; - while (servers->len) { - MySrvC *mysrvc=(MySrvC *)servers->remove_index_fast(0); - delete mysrvc; - } - delete servers; -} - - -MyHGC::MyHGC(int _hid) { - hid=_hid; - mysrvs=new MySrvList(this); - current_time_now = 0; - new_connections_now = 0; - attributes.initialized = false; - reset_attributes(); - // Uninitialized server defaults. Should later be initialized via 'mysql_hostgroup_attributes'. - servers_defaults.weight = -1; - servers_defaults.max_connections = -1; - servers_defaults.use_ssl = -1; -} - -void MyHGC::reset_attributes() { - if (attributes.initialized == false) { - attributes.init_connect = NULL; - attributes.comment = NULL; - attributes.ignore_session_variables_text = NULL; - } - attributes.initialized = true; - attributes.configured = false; - attributes.max_num_online_servers = 1000000; - attributes.throttle_connections_per_sec = 1000000; - attributes.autocommit = -1; - attributes.free_connections_pct = 10; - attributes.handle_warnings = -1; - attributes.multiplex = true; - attributes.connection_warming = false; - free(attributes.init_connect); - attributes.init_connect = NULL; - free(attributes.comment); - attributes.comment = NULL; - free(attributes.ignore_session_variables_text); - attributes.ignore_session_variables_text = NULL; - attributes.ignore_session_variables_json = json(); -} - -MyHGC::~MyHGC() { - reset_attributes(); // free all memory - delete mysrvs; -} - using metric_name = std::string; using metric_help = std::string; using metric_tags = std::map; @@ -1296,7 +580,6 @@ hg_metrics_map = std::make_tuple( ); MySQL_HostGroups_Manager::MySQL_HostGroups_Manager() { - pthread_mutex_init(&ev_loop_mutex, NULL); status.client_connections=0; status.client_connections_aborted=0; status.client_connections_created=0; @@ -3137,557 +2420,6 @@ void MySQL_HostGroups_Manager::push_MyConn_to_pool_array(MySQL_Connection **ca, wrunlock(); } -MySrvC *MyHGC::get_random_MySrvC(char * gtid_uuid, uint64_t gtid_trxid, int max_lag_ms, MySQL_Session *sess) { - MySrvC *mysrvc=NULL; - unsigned int j; - unsigned int sum=0; - unsigned int TotalUsedConn=0; - unsigned int l=mysrvs->cnt(); - static time_t last_hg_log = 0; -#ifdef TEST_AURORA - unsigned long long a1 = array_mysrvc_total/10000; - array_mysrvc_total += l; - unsigned long long a2 = array_mysrvc_total/10000; - if (a2 > a1) { - fprintf(stderr, "Total: %llu, Candidates: %llu\n", array_mysrvc_total-l, array_mysrvc_cands); - } -#endif // TEST_AURORA - MySrvC *mysrvcCandidates_static[32]; - MySrvC **mysrvcCandidates = mysrvcCandidates_static; - unsigned int num_candidates = 0; - bool max_connections_reached = false; - if (l>32) { - mysrvcCandidates = (MySrvC **)malloc(sizeof(MySrvC *)*l); - } - if (l) { - //int j=0; - for (j=0; jidx(j); - if (mysrvc->status==MYSQL_SERVER_STATUS_ONLINE) { // consider this server only if ONLINE - if (mysrvc->ConnectionsUsed->conns_length() < mysrvc->max_connections) { // consider this server only if didn't reach max_connections - if ( mysrvc->current_latency_us < ( mysrvc->max_latency_us ? mysrvc->max_latency_us : mysql_thread___default_max_latency_ms*1000 ) ) { // consider the host only if not too far - if (gtid_trxid) { - if (MyHGM->gtid_exists(mysrvc, gtid_uuid, gtid_trxid)) { - sum+=mysrvc->weight; - TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); - mysrvcCandidates[num_candidates]=mysrvc; - num_candidates++; - } - } else { - if (max_lag_ms >= 0) { - if ((unsigned int)max_lag_ms >= mysrvc->aws_aurora_current_lag_us/1000) { - sum+=mysrvc->weight; - TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); - mysrvcCandidates[num_candidates]=mysrvc; - num_candidates++; - } else { - sess->thread->status_variables.stvar[st_var_aws_aurora_replicas_skipped_during_query]++; - } - } else { - sum+=mysrvc->weight; - TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); - mysrvcCandidates[num_candidates]=mysrvc; - num_candidates++; - } - } - } - } else { - max_connections_reached = true; - } - } else { - if (mysrvc->status==MYSQL_SERVER_STATUS_SHUNNED) { - // try to recover shunned servers - if (mysrvc->shunned_automatic && mysql_thread___shun_recovery_time_sec) { - time_t t; - t=time(NULL); - // we do all these changes without locking . We assume the server is not used from long - // even if the server is still in used and any of the follow command fails it is not critical - // because this is only an attempt to recover a server that is probably dead anyway - - // the next few lines of code try to solve issue #530 - int max_wait_sec = ( mysql_thread___shun_recovery_time_sec * 1000 >= mysql_thread___connect_timeout_server_max ? mysql_thread___connect_timeout_server_max/1000 - 1 : mysql_thread___shun_recovery_time_sec ); - if (max_wait_sec < 1) { // min wait time should be at least 1 second - max_wait_sec = 1; - } - if (t > mysrvc->time_last_detected_error && (t - mysrvc->time_last_detected_error) > max_wait_sec) { - if ( - (mysrvc->shunned_and_kill_all_connections==false) // it is safe to bring it back online - || - (mysrvc->shunned_and_kill_all_connections==true && mysrvc->ConnectionsUsed->conns_length()==0 && mysrvc->ConnectionsFree->conns_length()==0) // if shunned_and_kill_all_connections is set, ensure all connections are already dropped - ) { -#ifdef DEBUG - if (GloMTH->variables.hostgroup_manager_verbose >= 3) { - proxy_info("Unshunning server %s:%d.\n", mysrvc->address, mysrvc->port); - } -#endif - mysrvc->status=MYSQL_SERVER_STATUS_ONLINE; - mysrvc->shunned_automatic=false; - mysrvc->shunned_and_kill_all_connections=false; - mysrvc->connect_ERR_at_time_last_detected_error=0; - mysrvc->time_last_detected_error=0; - // note: the following function scans all the hostgroups. - // This is ok for now because we only have a global mutex. - // If one day we implement a mutex per hostgroup (unlikely, - // but possible), this must be taken into consideration - if (mysql_thread___unshun_algorithm == 1) { - MyHGM->unshun_server_all_hostgroups(mysrvc->address, mysrvc->port, t, max_wait_sec, &mysrvc->myhgc->hid); - } - // if a server is taken back online, consider it immediately - if ( mysrvc->current_latency_us < ( mysrvc->max_latency_us ? mysrvc->max_latency_us : mysql_thread___default_max_latency_ms*1000 ) ) { // consider the host only if not too far - if (gtid_trxid) { - if (MyHGM->gtid_exists(mysrvc, gtid_uuid, gtid_trxid)) { - sum+=mysrvc->weight; - TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); - mysrvcCandidates[num_candidates]=mysrvc; - num_candidates++; - } - } else { - if (max_lag_ms >= 0) { - if ((unsigned int)max_lag_ms >= mysrvc->aws_aurora_current_lag_us/1000) { - sum+=mysrvc->weight; - TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); - mysrvcCandidates[num_candidates]=mysrvc; - num_candidates++; - } - } else { - sum+=mysrvc->weight; - TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); - mysrvcCandidates[num_candidates]=mysrvc; - num_candidates++; - } - } - } - } - } - } - } - } - } - if (max_lag_ms > 0) { // we are using AWS Aurora, as this logic is implemented only here - unsigned int min_num_replicas = sess->thread->variables.aurora_max_lag_ms_only_read_from_replicas; - if (min_num_replicas) { - if (num_candidates >= min_num_replicas) { // there are at least N replicas - // we try to remove the writer - unsigned int total_aws_aurora_current_lag_us=0; - for (j=0; jaws_aurora_current_lag_us; - } - if (total_aws_aurora_current_lag_us) { // we are just double checking that we don't have all servers with aws_aurora_current_lag_us==0 - for (j=0; jaws_aurora_current_lag_us==0) { - sum-=mysrvc->weight; - TotalUsedConn-=mysrvc->ConnectionsUsed->conns_length(); - if (j < num_candidates-1) { - mysrvcCandidates[j]=mysrvcCandidates[num_candidates-1]; - } - num_candidates--; - } - } - } - } - } - } - if (sum==0) { - // per issue #531 , we try a desperate attempt to bring back online any shunned server - // we do this lowering the maximum wait time to 10% - // most of the follow code is copied from few lines above - time_t t; - t=time(NULL); - int max_wait_sec = ( mysql_thread___shun_recovery_time_sec * 1000 >= mysql_thread___connect_timeout_server_max ? mysql_thread___connect_timeout_server_max/10000 - 1 : mysql_thread___shun_recovery_time_sec/10 ); - if (max_wait_sec < 1) { // min wait time should be at least 1 second - max_wait_sec = 1; - } - if (t - last_hg_log > 1) { // log this at most once per second to avoid spamming the logs - last_hg_log = time(NULL); - - if (gtid_trxid) { - proxy_error("Hostgroup %u has no servers ready for GTID '%s:%ld'. Waiting for replication...\n", hid, gtid_uuid, gtid_trxid); - } else { - proxy_error("Hostgroup %u has no servers available%s! Checking servers shunned for more than %u second%s\n", hid, - (max_connections_reached ? " or max_connections reached for all servers" : ""), max_wait_sec, max_wait_sec == 1 ? "" : "s"); - } - } - for (j=0; jidx(j); - if (mysrvc->status==MYSQL_SERVER_STATUS_SHUNNED && mysrvc->shunned_automatic==true) { - if ((t - mysrvc->time_last_detected_error) > max_wait_sec) { - mysrvc->status=MYSQL_SERVER_STATUS_ONLINE; - mysrvc->shunned_automatic=false; - mysrvc->connect_ERR_at_time_last_detected_error=0; - mysrvc->time_last_detected_error=0; - // if a server is taken back online, consider it immediately - if ( mysrvc->current_latency_us < ( mysrvc->max_latency_us ? mysrvc->max_latency_us : mysql_thread___default_max_latency_ms*1000 ) ) { // consider the host only if not too far - if (gtid_trxid) { - if (MyHGM->gtid_exists(mysrvc, gtid_uuid, gtid_trxid)) { - sum+=mysrvc->weight; - TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); - mysrvcCandidates[num_candidates]=mysrvc; - num_candidates++; - } - } else { - if (max_lag_ms >= 0) { - if ((unsigned int)max_lag_ms >= mysrvc->aws_aurora_current_lag_us/1000) { - sum+=mysrvc->weight; - TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); - mysrvcCandidates[num_candidates]=mysrvc; - num_candidates++; - } - } else { - sum+=mysrvc->weight; - TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); - mysrvcCandidates[num_candidates]=mysrvc; - num_candidates++; - } - } - } - } - } - } - } - if (sum==0) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySrvC NULL because no backend ONLINE or with weight\n"); - if (l>32) { - free(mysrvcCandidates); - } -#ifdef TEST_AURORA - array_mysrvc_cands += num_candidates; -#endif // TEST_AURORA - return NULL; // if we reach here, we couldn't find any target - } - -/* - unsigned int New_sum=0; - unsigned int New_TotalUsedConn=0; - // we will now scan again to ignore overloaded servers - for (j=0; jConnectionsUsed->conns_length(); - if ((len * sum) <= (TotalUsedConn * mysrvc->weight * 1.5 + 1)) { - - New_sum+=mysrvc->weight; - New_TotalUsedConn+=len; - } else { - // remove the candidate - if (j+1 < num_candidates) { - mysrvcCandidates[j] = mysrvcCandidates[num_candidates-1]; - } - j--; - num_candidates--; - } - } -*/ - - unsigned int New_sum=sum; - - if (New_sum==0) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySrvC NULL because no backend ONLINE or with weight\n"); - if (l>32) { - free(mysrvcCandidates); - } -#ifdef TEST_AURORA - array_mysrvc_cands += num_candidates; -#endif // TEST_AURORA - return NULL; // if we reach here, we couldn't find any target - } - - // latency awareness algorithm is enabled only when compiled with USE_MYSRVC_ARRAY - if (sess && sess->thread->variables.min_num_servers_lantency_awareness) { - if ((int) num_candidates >= sess->thread->variables.min_num_servers_lantency_awareness) { - unsigned int servers_with_latency = 0; - unsigned int total_latency_us = 0; - // scan and verify that all servers have some latency - for (j=0; jcurrent_latency_us) { - servers_with_latency++; - total_latency_us += mysrvc->current_latency_us; - } - } - if (servers_with_latency == num_candidates) { - // all servers have some latency. - // That is good. If any server have no latency, something is wrong - // and we will skip this algorithm - sess->thread->status_variables.stvar[st_var_ConnPool_get_conn_latency_awareness]++; - unsigned int avg_latency_us = 0; - avg_latency_us = total_latency_us/num_candidates; - for (j=0; jcurrent_latency_us > avg_latency_us) { - // remove the candidate - if (j+1 < num_candidates) { - mysrvcCandidates[j] = mysrvcCandidates[num_candidates-1]; - } - j--; - num_candidates--; - } - } - // we scan again to adjust weight - New_sum = 0; - for (j=0; jweight; - } - } - } - } - - - unsigned int k; - if (New_sum > 32768) { - k=rand()%New_sum; - } else { - k=fastrand()%New_sum; - } - k++; - New_sum=0; - - for (j=0; jweight; - if (k<=New_sum) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySrvC %p, server %s:%d\n", mysrvc, mysrvc->address, mysrvc->port); - if (l>32) { - free(mysrvcCandidates); - } -#ifdef TEST_AURORA - array_mysrvc_cands += num_candidates; -#endif // TEST_AURORA - return mysrvc; - } - } - } else { - time_t t = time(NULL); - - if (t - last_hg_log > 1) { - last_hg_log = time(NULL); - proxy_error("Hostgroup %u has no servers available!\n", hid); - } - } - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySrvC NULL\n"); - if (l>32) { - free(mysrvcCandidates); - } -#ifdef TEST_AURORA - array_mysrvc_cands += num_candidates; -#endif // TEST_AURORA - return NULL; // if we reach here, we couldn't find any target -} - -//unsigned int MySrvList::cnt() { -// return servers->len; -//} - -//MySrvC * MySrvList::idx(unsigned int i) { return (MySrvC *)servers->index(i); } - -void MySrvConnList::get_random_MyConn_inner_search(unsigned int start, unsigned int end, unsigned int& conn_found_idx, unsigned int& connection_quality_level, unsigned int& number_of_matching_session_variables, const MySQL_Connection * client_conn) { - char *schema = client_conn->userinfo->schemaname; - MySQL_Connection * conn=NULL; - unsigned int k; - for (k = start; k < end; k++) { - conn = (MySQL_Connection *)conns->index(k); - if (conn->match_tracked_options(client_conn)) { - if (connection_quality_level == 0) { - // this is our best candidate so far - connection_quality_level = 1; - conn_found_idx = k; - } - if (conn->requires_CHANGE_USER(client_conn)==false) { - if (connection_quality_level == 1) { - // this is our best candidate so far - connection_quality_level = 2; - conn_found_idx = k; - } - unsigned int cnt_match = 0; // number of matching session variables - unsigned int not_match = 0; // number of not matching session variables - cnt_match = conn->number_of_matching_session_variables(client_conn, not_match); - if (strcmp(conn->userinfo->schemaname,schema)==0) { - cnt_match++; - } else { - not_match++; - } - if (not_match==0) { - // it seems we found the perfect connection - number_of_matching_session_variables = cnt_match; - connection_quality_level = 3; - conn_found_idx = k; - return; // exit immediately, we found the perfect connection - } else { - // we didn't find the perfect connection - // but maybe is better than what we have so far? - if (cnt_match > number_of_matching_session_variables) { - // this is our best candidate so far - number_of_matching_session_variables = cnt_match; - conn_found_idx = k; - } - } - } else { - if (connection_quality_level == 1) { - int rca = mysql_thread___reset_connection_algorithm; - if (rca==1) { - int ql = GloMTH->variables.connpoll_reset_queue_length; - if (ql==0) { - // if: - // mysql-reset_connection_algorithm=1 and - // mysql-connpoll_reset_queue_length=0 - // we will not return a connection with connection_quality_level == 1 - // because we want to run COM_CHANGE_USER - // This change was introduced to work around Galera bug - // https://github.com/codership/galera/issues/613 - connection_quality_level = 0; - } - } - } - } - } - } -} - - - -MySQL_Connection * MySrvConnList::get_random_MyConn(MySQL_Session *sess, bool ff) { - MySQL_Connection * conn=NULL; - unsigned int i; - unsigned int conn_found_idx; - unsigned int l=conns_length(); - unsigned int connection_quality_level = 0; - bool needs_warming = false; - // connection_quality_level: - // 0 : not found any good connection, tracked options are not OK - // 1 : tracked options are OK , but CHANGE USER is required - // 2 : tracked options are OK , CHANGE USER is not required, but some SET statement or INIT_DB needs to be executed - // 3 : tracked options are OK , CHANGE USER is not required, and it seems that SET statements or INIT_DB ARE not required - unsigned int number_of_matching_session_variables = 0; // this includes session variables AND schema - bool connection_warming = mysql_thread___connection_warming; - int free_connections_pct = mysql_thread___free_connections_pct; - if (mysrvc->myhgc->attributes.configured == true) { - // mysql_hostgroup_attributes takes priority - connection_warming = mysrvc->myhgc->attributes.connection_warming; - free_connections_pct = mysrvc->myhgc->attributes.free_connections_pct; - } - if (connection_warming == true) { - unsigned int total_connections = mysrvc->ConnectionsFree->conns_length()+mysrvc->ConnectionsUsed->conns_length(); - unsigned int expected_warm_connections = free_connections_pct*mysrvc->max_connections/100; - if (total_connections < expected_warm_connections) { - needs_warming = true; - } - } - if (l && ff==false && needs_warming==false) { - if (l>32768) { - i=rand()%l; - } else { - i=fastrand()%l; - } - if (sess && sess->client_myds && sess->client_myds->myconn && sess->client_myds->myconn->userinfo) { - MySQL_Connection * client_conn = sess->client_myds->myconn; - get_random_MyConn_inner_search(i, l, conn_found_idx, connection_quality_level, number_of_matching_session_variables, client_conn); - if (connection_quality_level !=3 ) { // we didn't find the perfect connection - get_random_MyConn_inner_search(0, i, conn_found_idx, connection_quality_level, number_of_matching_session_variables, client_conn); - } - // connection_quality_level: - // 1 : tracked options are OK , but CHANGE USER is required - // 2 : tracked options are OK , CHANGE USER is not required, but some SET statement or INIT_DB needs to be executed - switch (connection_quality_level) { - case 0: // not found any good connection, tracked options are not OK - // we must check if connections need to be freed before - // creating a new connection - { - unsigned int conns_free = mysrvc->ConnectionsFree->conns_length(); - unsigned int conns_used = mysrvc->ConnectionsUsed->conns_length(); - unsigned int pct_max_connections = (3 * mysrvc->max_connections) / 4; - unsigned int connections_to_free = 0; - - if (conns_free >= 1) { - // connection cleanup is triggered when connections exceed 3/4 of the total - // allowed max connections, this cleanup ensures that at least *one connection* - // will be freed. - if (pct_max_connections <= (conns_free + conns_used)) { - connections_to_free = (conns_free + conns_used) - pct_max_connections; - if (connections_to_free == 0) connections_to_free = 1; - } - - while (conns_free && connections_to_free) { - MySQL_Connection* conn = mysrvc->ConnectionsFree->remove(0); - delete conn; - - conns_free = mysrvc->ConnectionsFree->conns_length(); - connections_to_free -= 1; - } - } - - // we must create a new connection - conn = new MySQL_Connection(); - conn->parent=mysrvc; - // if attributes.multiplex == true , STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG is set to false. And vice-versa - conn->set_status(!conn->parent->myhgc->attributes.multiplex, STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG); - __sync_fetch_and_add(&MyHGM->status.server_connections_created, 1); - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySQL Connection %p, server %s:%d\n", conn, conn->parent->address, conn->parent->port); - } - break; - case 1: //tracked options are OK , but CHANGE USER is required - // we may consider creating a new connection - { - unsigned int conns_free = mysrvc->ConnectionsFree->conns_length(); - unsigned int conns_used = mysrvc->ConnectionsUsed->conns_length(); - if ((conns_used > conns_free) && (mysrvc->max_connections > (conns_free/2 + conns_used/2)) ) { - conn = new MySQL_Connection(); - conn->parent=mysrvc; - // if attributes.multiplex == true , STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG is set to false. And vice-versa - conn->set_status(!conn->parent->myhgc->attributes.multiplex, STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG); - __sync_fetch_and_add(&MyHGM->status.server_connections_created, 1); - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySQL Connection %p, server %s:%d\n", conn, conn->parent->address, conn->parent->port); - } else { - conn=(MySQL_Connection *)conns->remove_index_fast(conn_found_idx); - } - } - break; - case 2: // tracked options are OK , CHANGE USER is not required, but some SET statement or INIT_DB needs to be executed - case 3: // tracked options are OK , CHANGE USER is not required, and it seems that SET statements or INIT_DB ARE not required - // here we return the best connection we have, no matter if connection_quality_level is 2 or 3 - conn=(MySQL_Connection *)conns->remove_index_fast(conn_found_idx); - break; - default: // this should never happen - // LCOV_EXCL_START - assert(0); - break; - // LCOV_EXCL_STOP - } - } else { - conn=(MySQL_Connection *)conns->remove_index_fast(i); - } - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySQL Connection %p, server %s:%d\n", conn, conn->parent->address, conn->parent->port); - return conn; - } else { - unsigned long long curtime = monotonic_time(); - curtime = curtime / 1000 / 1000; // convert to second - MyHGC *_myhgc = mysrvc->myhgc; - if (curtime > _myhgc->current_time_now) { - _myhgc->current_time_now = curtime; - _myhgc->new_connections_now = 0; - } - _myhgc->new_connections_now++; - unsigned int throttle_connections_per_sec_to_hostgroup = (unsigned int) mysql_thread___throttle_connections_per_sec_to_hostgroup; - if (_myhgc->attributes.configured == true) { - // mysql_hostgroup_attributes takes priority - throttle_connections_per_sec_to_hostgroup = _myhgc->attributes.throttle_connections_per_sec; - } - if (_myhgc->new_connections_now > (unsigned int) throttle_connections_per_sec_to_hostgroup) { - __sync_fetch_and_add(&MyHGM->status.server_connections_delayed, 1); - return NULL; - } else { - conn = new MySQL_Connection(); - conn->parent=mysrvc; - // if attributes.multiplex == true , STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG is set to false. And vice-versa - conn->set_status(!conn->parent->myhgc->attributes.multiplex, STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG); - __sync_fetch_and_add(&MyHGM->status.server_connections_created, 1); - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySQL Connection %p, server %s:%d\n", conn, conn->parent->address, conn->parent->port); - return conn; - } - } - return NULL; // never reach here -} - void MySQL_HostGroups_Manager::unshun_server_all_hostgroups(const char * address, uint16_t port, time_t t, int max_wait_sec, unsigned int *skip_hid) { // we scan all hostgroups looking for a specific server to unshun // if skip_hid is not NULL , the specific hostgroup is skipped diff --git a/lib/MySQL_PreparedStatement.cpp b/lib/MySQL_PreparedStatement.cpp index 5a47e81119..1b27c6a6e9 100644 --- a/lib/MySQL_PreparedStatement.cpp +++ b/lib/MySQL_PreparedStatement.cpp @@ -738,53 +738,16 @@ MySQL_STMTs_local_v14::~MySQL_STMTs_local_v14() { GloMyStmt->ref_count_server(global_stmt_id, -1); } } -/* - for (std::map::iterator it = m.begin(); - it != m.end(); ++it) { - uint32_t stmt_id = it->first; - MYSQL_STMT *stmt = it->second; - if (stmt) { // is a server - if (stmt->mysql) { - stmt->mysql->stmts = - list_delete(stmt->mysql->stmts, &stmt->list); - } - // we do a hack here: we pretend there is no server associate - // the connection will be dropped anyway immediately after - stmt->mysql = NULL; - mysql_stmt_close(stmt); - GloMyStmt->ref_count(stmt_id, -1, true, false); - } else { // is a client - GloMyStmt->ref_count(stmt_id, -1, true, true); - } - } - m.erase(m.begin(), m.end()); -*/ } MySQL_STMT_Global_info *MySQL_STMT_Manager_v14::find_prepared_statement_by_hash( uint64_t hash) { - //uint64_t hash, bool lock) { // removed in 2.3 MySQL_STMT_Global_info *ret = NULL; // assume we do not find it -/* removed in 2.3 - if (lock) { - pthread_rwlock_wrlock(&rwlock_); - } -*/ auto s = map_stmt_hash_to_info.find(hash); if (s != map_stmt_hash_to_info.end()) { ret = s->second; - //__sync_fetch_and_add(&ret->ref_count_client,1); // increase reference - //count -// __sync_fetch_and_add(&find_prepared_statement_by_hash_calls, 1); -// __sync_fetch_and_add(&ret->ref_count_client, 1); } - -/* removed in 2.3 - if (lock) { - pthread_rwlock_unlock(&rwlock_); - } -*/ return ret; } @@ -808,15 +771,6 @@ MySQL_STMT_Global_info *MySQL_STMT_Manager_v14::find_prepared_statement_by_stmt_ uint32_t MySQL_STMTs_local_v14::generate_new_client_stmt_id(uint64_t global_statement_id) { uint32_t ret=0; -/* - //auto s2 = global_stmt_to_client_ids.find(global_statement_id); - std::pair::iterator, std::multimap::iterator> itret; - itret = global_stmt_to_client_ids.equal_range(global_statement_id); - for (std::multimap::iterator it=itret.first; it!=itret.second; ++it) { - ret = it->second; - return ret; - } -*/ if (free_client_ids.size()) { ret=free_client_ids.top(); free_client_ids.pop(); @@ -893,7 +847,6 @@ MySQL_STMT_Global_info *MySQL_STMT_Manager_v14::add_prepared_statement( } else { next_id = next_statement_id; next_statement_id++; - //__sync_fetch_and_add(&next_statement_id, 1); } //next_statement_id++; @@ -902,14 +855,7 @@ MySQL_STMT_Global_info *MySQL_STMT_Manager_v14::add_prepared_statement( // insert it in both maps map_stmt_id_to_info.insert(std::make_pair(a->statement_id, a)); map_stmt_hash_to_info.insert(std::make_pair(a->hash, a)); - // ret=a->statement_id; ret = a; - // next_statement_id++; // increment it - //__sync_fetch_and_add(&ret->ref_count_client,1); // increase reference - //count -// __sync_fetch_and_add(&ret->ref_count_client, -// 1); // increase reference count -// *is_new = true; __sync_add_and_fetch(&num_stmt_with_ref_client_count_zero,1); __sync_add_and_fetch(&num_stmt_with_ref_server_count_zero,1); } @@ -918,9 +864,6 @@ MySQL_STMT_Global_info *MySQL_STMT_Manager_v14::add_prepared_statement( } ret->ref_count_server++; statuses.s_total++; -// __sync_fetch_and_add(&add_prepared_statement_calls, 1); -// __sync_fetch_and_add(&ret->ref_count_server, -// 1); // increase reference count if (lock) { pthread_rwlock_unlock(&rwlock_); } @@ -1101,14 +1044,6 @@ SQLite3_result * MySQL_STMT_Manager_v14::get_prepared_statements_global_infos() pgs->free_row(pta); delete pgs; } -/* - for (std::unordered_map::iterator it=digest_umap.begin(); it!=digest_umap.end(); ++it) { - QP_query_digest_stats *qds=(QP_query_digest_stats *)it->second; - char **pta=qds->get_row(); - result->add_row(pta); - qds->free_row(pta); - } -*/ unlock(); return result; } diff --git a/lib/MySQL_Protocol.cpp b/lib/MySQL_Protocol.cpp index 527c87bc93..cba71c6a8a 100644 --- a/lib/MySQL_Protocol.cpp +++ b/lib/MySQL_Protocol.cpp @@ -26,8 +26,6 @@ extern ClickHouse_Authentication *GloClickHouseAuth; #undef max_allowed_packet #endif -//#define RESULTSET_BUFLEN 16300 - #ifndef CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA #define CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA 0x00200000 #endif @@ -43,240 +41,7 @@ static const char *plugins[3] = { "caching_sha2_password", }; -#ifdef DEBUG -static void __dump_pkt(const char *func, unsigned char *_ptr, unsigned int len) { - - if (GloVars.global.gdbg==0) return; - if (GloVars.global.gdbg_lvl[PROXY_DEBUG_MYSQL_PROTOCOL].verbosity < 8 ) return; - unsigned int i; - fprintf(stderr,"DUMP %d bytes FROM %s\n", len, func); - for(i = 0; i < len; i++) { - if(isprint(_ptr[i])) fprintf(stderr,"%c", _ptr[i]); else fprintf(stderr,"."); - if (i>0 && (i%16==15 || i==len-1)) { - unsigned int j; - if (i%16!=15) { - j=15-i%16; - while (j--) fprintf(stderr," "); - } - fprintf(stderr," --- "); - for (j=(i==len-1 ? ((int)(i/16))*16 : i-15 ) ; j<=i; j++) { - fprintf(stderr,"%02x ", _ptr[j]); - } - fprintf(stderr,"\n"); - } - } - fprintf(stderr,"\n\n"); - - -} -#endif - -char *sha1_pass_hex(char *sha1_pass) { - if (sha1_pass==NULL) return NULL; - char *buff=(char *)malloc(SHA_DIGEST_LENGTH*2+2); - buff[0]='*'; - buff[SHA_DIGEST_LENGTH*2+1]='\0'; - int i; - uint8_t a = 0; - for (i=0;iseed1= (rand_st->seed1*3+rand_st->seed2) % rand_st->max_value; - rand_st->seed2= (rand_st->seed1+rand_st->seed2+33) % rand_st->max_value; - return (((double) rand_st->seed1) / rand_st->max_value_dbl); -} - -void proxy_create_random_string(char *_to, uint length, struct rand_struct *rand_st) { - unsigned char * to = (unsigned char *)_to; - int rc = 0; - uint i; - rc = RAND_bytes((unsigned char *)to,length); -#ifdef DEBUG - if (rc==1) { - // For code coverage (to test the following code and other function) - // in DEBUG mode we pretend that RAND_bytes() fails 1% of the time - if(rand()%100==0) { - rc=0; - } - } -#endif // DEBUG - if (rc!=1) { - for (i=0; i 127) { - *to -= 128; - } - if (*to == 0) { - *to = 'a'; - } - to++; - } - } - *to= '\0'; -} - -static inline int write_encoded_length(unsigned char *p, uint64_t val, uint8_t len, char prefix) { - if (len==1) { - *p=(char)val; - return 1; - } - *p=prefix; - p++; - memcpy(p,&val,len-1); - return len; -} - -static inline int write_encoded_length_and_string(unsigned char *p, uint64_t val, uint8_t len, char prefix, char *string) { - int l=write_encoded_length(p,val,len,prefix); - if (val) { - memcpy(p+l,string,val); - } - return l+val; -} - -void proxy_compute_sha1_hash_multi(uint8_t *digest, const char *buf1, int len1, const char *buf2, int len2) { - PROXY_TRACE(); - const EVP_MD *evp_digest = EVP_get_digestbyname("sha1"); - assert(evp_digest != NULL); - EVP_MD_CTX *ctx = EVP_MD_CTX_new(); - EVP_MD_CTX_init(ctx); - EVP_DigestInit_ex(ctx, evp_digest, NULL); - EVP_DigestUpdate(ctx, buf1, len1); - EVP_DigestUpdate(ctx, buf2, len2); - unsigned int olen = 0; - EVP_DigestFinal(ctx, digest, &olen); - EVP_MD_CTX_free(ctx); -} - -void proxy_compute_sha1_hash(uint8_t *digest, const char *buf, int len) { - PROXY_TRACE(); - const EVP_MD *evp_digest = EVP_get_digestbyname("sha1"); - assert(evp_digest != NULL); - EVP_MD_CTX *ctx = EVP_MD_CTX_new(); - EVP_MD_CTX_init(ctx); - EVP_DigestInit_ex(ctx, evp_digest, NULL); - EVP_DigestUpdate(ctx, buf, len); - unsigned int olen = 0; - EVP_DigestFinal(ctx, digest, &olen); - EVP_MD_CTX_free(ctx); -} - -void proxy_compute_two_stage_sha1_hash(const char *password, size_t pass_len, uint8_t *hash_stage1, uint8_t *hash_stage2) { - proxy_compute_sha1_hash(hash_stage1, password, pass_len); - proxy_compute_sha1_hash(hash_stage2, (const char *) hash_stage1, SHA_DIGEST_LENGTH); -} - -void proxy_my_crypt(char *to, const uint8_t *s1, const uint8_t *s2, uint len) { - const uint8_t *s1_end= s1 + len; - while (s1 < s1_end) - *to++= *s1++ ^ *s2++; -} - -unsigned char decode_char(char x) { - if (x >= '0' && x <= '9') - return (x - 0x30); - else if (x >= 'A' && x <= 'F') - return(x - 0x37); - else if (x >= 'a' && x <= 'f') - return(x - 0x57); - else { - proxy_error("Invalid char"); - return 0; - } -} - -void unhex_pass(uint8_t *out, const char *in) { - int i=0; - for (i=0;ibuffer + myrs->buffer_used; myrs->buffer_used += size; } - memcpy(_ptr, &myhdr, sizeof(mysql_hdr)); - int l=sizeof(mysql_hdr); + memcpy(_ptr, &myhdr, sizeof(mysql_hdr)); + int l=sizeof(mysql_hdr); _ptr[l]=0xfe; l++; int16_t internal_status = status; if (sess) { @@ -2977,501 +2725,6 @@ bool MySQL_Protocol::generate_COM_QUERY_from_COM_FIELD_LIST(PtrSize_t *pkt) { return true; } -MySQL_ResultSet::MySQL_ResultSet() { - buffer = NULL; - //reset_pid = true; -} - -void MySQL_ResultSet::buffer_init(MySQL_Protocol* myproto) { - if (buffer==NULL) { - buffer=(unsigned char *)malloc(RESULTSET_BUFLEN); - } - - buffer_used=0; - myprot = myproto; -} - -void MySQL_ResultSet::init(MySQL_Protocol *_myprot, MYSQL_RES *_res, MYSQL *_my, MYSQL_STMT *_stmt) { - PROXY_TRACE2(); - transfer_started=false; - resultset_completed=false; - myprot=_myprot; - mysql=_my; - stmt=_stmt; - if (buffer==NULL) { - //if (_stmt==NULL) { // we allocate this buffer only for not prepared statements - // removing the previous assumption. We allocate this buffer also for prepared statements - buffer=(unsigned char *)malloc(RESULTSET_BUFLEN); - //} - } - buffer_used=0; - myds=NULL; - if (myprot) { // if myprot = NULL , this is a mirror - myds=myprot->get_myds(); - } - //if (reset_pid==true) { - sid=0; - //PSarrayOUT = NULL; - if (myprot) { // if myprot = NULL , this is a mirror - sid=myds->pkt_sid+1; - //PSarrayOUT = new PtrSizeArray(8); - } - //} - //reset_pid=true; - result=_res; - resultset_size=0; - num_rows=0; - num_fields=mysql_field_count(mysql); - PtrSize_t pkt; - // immediately generate the first set of packets - // columns count - if (myprot==NULL) { - return; // this is a mirror - } - MySQL_Data_Stream * c_myds = *(myprot->myds); - if (c_myds->com_field_list==false) { - myprot->generate_pkt_column_count(false,&pkt.ptr,&pkt.size,sid,num_fields,this); - sid++; - resultset_size+=pkt.size; - } - // columns description - for (unsigned int i=0; icom_field_list==false) { - // we are replacing generate_pkt_field() with a more efficient version - //myprot->generate_pkt_field(false,&pkt.ptr,&pkt.size,sid,field->db,field->table,field->org_table,field->name,field->org_name,field->charsetnr,field->length,field->type,field->flags,field->decimals,false,0,NULL,this); - myprot->generate_pkt_field2(&pkt.ptr,&pkt.size,sid,field,this); - resultset_size+=pkt.size; - sid++; - } else { - if (c_myds->com_field_wild==NULL || mywildcmp(c_myds->com_field_wild,field->name)) { - myprot->generate_pkt_field(false,&pkt.ptr,&pkt.size,sid,field->db,field->table,field->org_table,field->name,field->org_name,field->charsetnr,field->length,field->type,field->flags,field->decimals,true,4,(char *)"null",this); - resultset_size+=pkt.size; - sid++; - } - } - } - - deprecate_eof_active = c_myds->myconn && (c_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF); - - // first EOF - unsigned int nTrx=myds->sess->NumActiveTransactions(); - uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0 ); - if (myds->sess->autocommit) setStatus += SERVER_STATUS_AUTOCOMMIT; - setStatus |= ( mysql->server_status & ~SERVER_STATUS_AUTOCOMMIT ); // get flags from server_status but ignore autocommit - setStatus = setStatus & ~SERVER_STATUS_CURSOR_EXISTS; // Do not send cursor #1128 -// if (_stmt) { // binary protocol , we also assume we have ALL the resultset -// myprot->generate_pkt_EOF(false,&pkt.ptr,&pkt.size,sid,0,mysql->server_status|setStatus); -// sid++; -// PSarrayOUT.add(pkt.ptr,pkt.size); -// resultset_size+=pkt.size; - //} else { - if (RESULTSET_BUFLEN <= (buffer_used + 9)) { - buffer_to_PSarrayOut(); - } - if (!deprecate_eof_active && myds->com_field_list==false) { - // up to 2.2.0 we used to add an EOF here. - // due to bug #3547 we move the logic into add_eof() that can now handle also prepared statements - PROXY_TRACE2(); - // if the backend server has CLIENT_DEPRECATE_EOF enabled, and the client does not support - // CLIENT_DEPRECATE_EOF, warning_count will be excluded from the intermediate EOF packet - add_eof((mysql->server_capabilities & CLIENT_DEPRECATE_EOF)); - } -} - - -// due to bug #3547 , in case of an error we remove the EOF -// and replace it with an ERR -// note that EOF is added on a packet on its own, instead of using a buffer, -// so that can be removed using remove_last_eof() -void MySQL_ResultSet::remove_last_eof() { - PROXY_TRACE2(); - PtrSize_t pkt; - if (PSarrayOUT.len) { - unsigned int l = PSarrayOUT.len-1; - PtrSize_t * pktp = PSarrayOUT.index(l); - if (pktp->size == 9) { - PROXY_TRACE2(); - PSarrayOUT.remove_index(l,&pkt); - l_free(pkt.size, pkt.ptr); - sid--; - } - } -} - -void MySQL_ResultSet::init_with_stmt(MySQL_Connection *myconn) { - PROXY_TRACE2(); - assert(stmt); - MYSQL_STMT *_stmt = stmt; - MySQL_Data_Stream * c_myds = *(myprot->myds); - buffer_to_PSarrayOut(); - unsigned long long total_size=0; - MYSQL_ROWS *r=_stmt->result.data; - if (r) { - total_size+=r->length; - if (r->length > 0xFFFFFF) { - total_size+=(r->length / 0xFFFFFF) * sizeof(mysql_hdr); - } - total_size+=sizeof(mysql_hdr); - while(r->next) { - r=r->next; - total_size+=r->length; - if (r->length > 0xFFFFFF) { - total_size+=(r->length / 0xFFFFFF) * sizeof(mysql_hdr); - } - total_size+=sizeof(mysql_hdr); - } -#define MAXBUFFSTMT 12*1024*1024 // hardcoded to LESS *very important* than 16MB - if (total_size < MAXBUFFSTMT) { - PtrSize_t pkt; - pkt.size=total_size; - pkt.ptr=malloc(pkt.size); - total_size=0; - r=_stmt->result.data; - add_row2(r,(unsigned char *)pkt.ptr); - total_size+=r->length; - if (r->length > 0xFFFFFF) { - total_size+=(r->length / 0xFFFFFF) * sizeof(mysql_hdr); - } - total_size+=sizeof(mysql_hdr); - while(r->next) { - r=r->next; - add_row2(r,(unsigned char *)pkt.ptr+total_size); - total_size+=r->length; - if (r->length > 0xFFFFFF) { - total_size+=(r->length / 0xFFFFFF) * sizeof(mysql_hdr); - } - total_size+=sizeof(mysql_hdr); - } - PSarrayOUT.add(pkt.ptr,pkt.size); - if (resultset_size/0xFFFFFFF != ((resultset_size+pkt.size)/0xFFFFFFF)) { - // generate a heartbeat every 256MB - unsigned long long curtime=monotonic_time(); - c_myds->sess->thread->atomic_curtime=curtime; - } - resultset_size+=pkt.size; - } else { // this code fixes a bug: resultset larger than 4GB would cause a crash - unsigned long long tmp_pkt_size = 0; - r=_stmt->result.data; - MYSQL_ROWS * r2 = NULL; - while (r) { - if (r->length >= MAXBUFFSTMT) { - // we have a large row - // we will send just that - tmp_pkt_size = r->length; - if (r->length > 0xFFFFFF) { - tmp_pkt_size+=(r->length / 0xFFFFFF) * sizeof(mysql_hdr); - } - tmp_pkt_size += sizeof(mysql_hdr); - PtrSize_t pkt; - pkt.size=tmp_pkt_size; - pkt.ptr=malloc(pkt.size); - add_row2(r,(unsigned char *)pkt.ptr); - PSarrayOUT.add(pkt.ptr,pkt.size); - if (resultset_size/0xFFFFFFF != ((resultset_size+pkt.size)/0xFFFFFFF)) { - // generate a heartbeat every 256MB - unsigned long long curtime=monotonic_time(); - c_myds->sess->thread->atomic_curtime=curtime; - } - resultset_size+=pkt.size; - r=r->next; // next row - } else { // we have small row - r2 = r; - tmp_pkt_size = 0; - unsigned int a = 0; - while (r && (tmp_pkt_size + r->length) < MAXBUFFSTMT) { - a++; - tmp_pkt_size += r->length; - tmp_pkt_size += sizeof(mysql_hdr); - //if (r->next) { - r = r->next; - //} - } - r = r2; // we reset it back to the beginning - if (tmp_pkt_size) { // this should always be true - unsigned long long tmp2 = 0; - PtrSize_t pkt; - pkt.size=tmp_pkt_size; - pkt.ptr=malloc(pkt.size); - while (tmp2 < tmp_pkt_size) { - add_row2(r,(unsigned char *)pkt.ptr+tmp2); - tmp2 += r->length; - tmp2 += sizeof(mysql_hdr); - r = r->next; - } - PSarrayOUT.add(pkt.ptr,pkt.size); - if (resultset_size/0xFFFFFFF != ((resultset_size+pkt.size)/0xFFFFFFF)) { - // generate a heartbeat every 256MB - unsigned long long curtime=monotonic_time(); - c_myds->sess->thread->atomic_curtime=curtime; - } - resultset_size+=pkt.size; - } - } - } - } - } - // up to 2.2.0 we were always adding an EOF - // due to bug #3547 , in case of an error we remove the EOF - // and replace it with an ERR - // note that EOF is added on a packet on its own, instead of using a buffer, - // so that can be removed - // - // NOTE: After 2.4.5 previous behavior is modified in favor of the following: - // - // When CLIENT_DEPRECATE_EOF two EOF packets are two be expected in the response: - // 1. After the columns definitions (This is added directly by 'MySQL_ResultSet::init'). - // 2. After the rows values, this can either be and EOF packet or a ERR packet in case of error. - // - // First EOF packet isn't optional, and it's just the second the one that is optionaly either an EOF - // or an ERR packet. The following code adds either the final EOF or ERR packet. This is equally valid - // for when CLIENT_DEPRECATE_EOF is enabled or not. If CLIENT_DEPRECATE_EOF is: - // * DISABLED: The behavior is as described before. - // * ENABLED: Code is identical for this case. The initial EOF packet is conditionally added by - // 'MySQL_ResultSet::init', thus, this packet should not be present if not needed at this point. - // In case of error an ERR packet needs to be added, otherwise `add_eof` handles the generation of - // the equivalent OK packet replacing the final EOF packet. - int myerr = mysql_stmt_errno(_stmt); - if (myerr) { - PROXY_TRACE2(); - add_err(myconn->myds); - } else { - PROXY_TRACE2(); - add_eof(); - } -} - -MySQL_ResultSet::~MySQL_ResultSet() { - PtrSize_t pkt; - //if (PSarrayOUT) { - while (PSarrayOUT.len) { - PSarrayOUT.remove_index_fast(0,&pkt); - l_free(pkt.size, pkt.ptr); - } - //delete PSarrayOUT; - //} - if (buffer) { - free(buffer); - buffer=NULL; - } - //if (myds) myds->pkt_sid=sid-1; -} - -// this function is used for binary protocol -// maybe later on can be adapted for text protocol too -unsigned int MySQL_ResultSet::add_row(MYSQL_ROWS *rows) { - unsigned int pkt_length=0; - MYSQL_ROW row = rows->data; - unsigned long row_length = rows->length; - // we call generate_pkt_row3 passing row_length - sid=myprot->generate_pkt_row3(this, &pkt_length, sid, 0, NULL, row, row_length); - sid++; - resultset_size+=pkt_length; - num_rows++; - return pkt_length; -} - - -// this function is used for text protocol -unsigned int MySQL_ResultSet::add_row(MYSQL_ROW row) { - unsigned long *lengths=mysql_fetch_lengths(result); - unsigned int pkt_length=0; - if (myprot) { - // we call generate_pkt_row3 without passing row_length - sid=myprot->generate_pkt_row3(this, &pkt_length, sid, num_fields, lengths, row, 0); - } else { - unsigned int col=0; - for (col=0; collength; - num_rows++; - uint8_t pkt_sid=sid; - if (length < (0xFFFFFF+sizeof(mysql_hdr))) { - mysql_hdr myhdr; - myhdr.pkt_length=length; - myhdr.pkt_id=pkt_sid; - memcpy(offset, &myhdr, sizeof(mysql_hdr)); - memcpy(offset+sizeof(mysql_hdr), row->data, row->length); - pkt_sid++; - } else { - unsigned int left=length; - unsigned int copied=0; - while (left>=0xFFFFFF) { - mysql_hdr myhdr; - myhdr.pkt_length=0xFFFFFF; - myhdr.pkt_id=pkt_sid; - pkt_sid++; - memcpy(offset, &myhdr, sizeof(mysql_hdr)); - offset+=sizeof(mysql_hdr); - char *o = (char *) row->data; - o += copied; - memcpy(offset, o, myhdr.pkt_length); - offset+=0xFFFFFF; - // we are writing a large packet (over 16MB), we assume we are always outside the buffer - copied+=0xFFFFFF; - left-=0xFFFFFF; - } - mysql_hdr myhdr; - myhdr.pkt_length=left; - myhdr.pkt_id=pkt_sid; - pkt_sid++; - memcpy(offset, &myhdr, sizeof(mysql_hdr)); - offset+=sizeof(mysql_hdr); - char *o = (char *) row->data; - o += copied; - memcpy(offset, o, myhdr.pkt_length); - // we are writing a large packet (over 16MB), we assume we are always outside the buffer - } - sid=pkt_sid; - return length; -} - -void MySQL_ResultSet::add_eof(bool suppress_warning_count) { - if (myprot) { - unsigned int nTrx=myds->sess->NumActiveTransactions(); - uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0 ); - if (myds->sess->autocommit) setStatus += SERVER_STATUS_AUTOCOMMIT; - setStatus |= ( mysql->server_status & ~SERVER_STATUS_AUTOCOMMIT ); // get flags from server_status but ignore autocommit - setStatus = setStatus & ~SERVER_STATUS_CURSOR_EXISTS; // Do not send cursor #1128 - //myprot->generate_pkt_EOF(false,&pkt.ptr,&pkt.size,sid,0,mysql->server_status|setStatus); - //PSarrayOUT->add(pkt.ptr,pkt.size); - //sid++; - //resultset_size+=pkt.size; - - // Note: warnings count will only be sent to the client if mysql-query_digests is enabled - const MySQL_Backend* _mybe = myds->sess->mybe; - const MySQL_Data_Stream* _server_myds = (_mybe && _mybe->server_myds) ? _mybe->server_myds : nullptr; - const MySQL_Connection* _myconn = (_server_myds && _server_myds->myds_type == MYDS_BACKEND && _server_myds->myconn) ? - _server_myds->myconn : nullptr; - const unsigned int warning_count = (_myconn && suppress_warning_count == false) ? _myconn->warning_count : 0; - if (deprecate_eof_active) { - PtrSize_t pkt; - buffer_to_PSarrayOut(); - myprot->generate_pkt_OK(false, &pkt.ptr, &pkt.size, sid, 0, 0, setStatus, warning_count, NULL, true); - PSarrayOUT.add(pkt.ptr, pkt.size); - resultset_size += pkt.size; - } - else { - // due to bug #3547 , in case of an error we remove the EOF - // and replace it with an ERR - // note that EOF is added on a packet on its own, instead of using a buffer, - // so that can be removed using remove_last_eof() - buffer_to_PSarrayOut(); - myprot->generate_pkt_EOF(false, NULL, NULL, sid, warning_count, setStatus, this); - resultset_size += 9; - buffer_to_PSarrayOut(); - } - sid++; - } - resultset_completed=true; -} - -void MySQL_ResultSet::add_err(MySQL_Data_Stream *_myds) { - PtrSize_t pkt; - if (myprot) { - MYSQL *_mysql=_myds->myconn->mysql; - buffer_to_PSarrayOut(); - char sqlstate[10]; - sprintf(sqlstate,"%s",mysql_sqlstate(_mysql)); - if (_myds && _myds->killed_at) { // see case #750 - if (_myds->kill_type == 0) { - myprot->generate_pkt_ERR(false,&pkt.ptr,&pkt.size,sid,1907,sqlstate,(char *)"Query execution was interrupted, query_timeout exceeded"); - MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, _myds->myconn->parent->myhgc->hid, _myds->myconn->parent->address, _myds->myconn->parent->port, 1907); - } else { - myprot->generate_pkt_ERR(false,&pkt.ptr,&pkt.size,sid,1317,sqlstate,(char *)"Query execution was interrupted"); - MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, _myds->myconn->parent->myhgc->hid, _myds->myconn->parent->address, _myds->myconn->parent->port, 1317); - } - } else { - int myerr = 0; - // the error code is returned from: - // - mysql_stmt_errno() if using a prepared statement - // - mysql_errno() if not using a prepared statement - if (stmt) { - myerr = mysql_stmt_errno(stmt); - myprot->generate_pkt_ERR(false,&pkt.ptr,&pkt.size,sid,myerr,sqlstate,mysql_stmt_error(stmt)); - } else { - myerr = mysql_errno(_mysql); - myprot->generate_pkt_ERR(false,&pkt.ptr,&pkt.size,sid,myerr,sqlstate,mysql_error(_mysql)); - } - // TODO: Check this is a mysql error - MyHGM->p_update_mysql_error_counter(p_mysql_error_type::mysql, _myds->myconn->parent->myhgc->hid, _myds->myconn->parent->address, _myds->myconn->parent->port, myerr); - } - PSarrayOUT.add(pkt.ptr,pkt.size); - sid++; - resultset_size+=pkt.size; - } - resultset_completed=true; -} - -/* -bool MySQL_ResultSet::get_COM_FIELD_LIST_response(PtrSizeArray *PSarrayFinal) { - transfer_started=true; - if (myprot) { - } - return resultset_completed; -} -*/ - -bool MySQL_ResultSet::get_resultset(PtrSizeArray *PSarrayFinal) { - transfer_started=true; - if (myprot) { - PSarrayFinal->copy_add(&PSarrayOUT,0,PSarrayOUT.len); - while (PSarrayOUT.len) - PSarrayOUT.remove_index(PSarrayOUT.len-1,NULL); - } - return resultset_completed; -} - -void MySQL_ResultSet::buffer_to_PSarrayOut(bool _last) { - if (buffer_used==0) - return; // exit immediately if the buffer is empty - if (buffer_used < RESULTSET_BUFLEN/2) { - if (_last == false) { - buffer=(unsigned char *)realloc(buffer,buffer_used); - } - } - PSarrayOUT.add(buffer,buffer_used); - if (_last) { - buffer = NULL; - } else { - buffer=(unsigned char *)malloc(RESULTSET_BUFLEN); - } - buffer_used=0; -} - -unsigned long long MySQL_ResultSet::current_size() { - unsigned long long intsize=0; - intsize+=sizeof(MySQL_ResultSet); - intsize+=RESULTSET_BUFLEN; // size of buffer - if (PSarrayOUT.len==0) // see bug #699 - return intsize; - intsize+=sizeof(PtrSizeArray); - intsize+=(PSarrayOUT.size*sizeof(PtrSize_t *)); - unsigned int i; - for (i=0; isize>RESULTSET_BUFLEN) { - intsize+=pkt->size; - } else { - intsize+=RESULTSET_BUFLEN; - } - } - return intsize; -} - my_bool proxy_mysql_stmt_close(MYSQL_STMT* stmt) { // Clean internal structures for 'stmt->mysql->stmts'. if (stmt->mysql) { diff --git a/lib/MySQL_ResultSet.cpp b/lib/MySQL_ResultSet.cpp new file mode 100644 index 0000000000..87e713aa76 --- /dev/null +++ b/lib/MySQL_ResultSet.cpp @@ -0,0 +1,549 @@ +#include +#include "proxysql.h" +#include "cpp.h" +#include "re2/re2.h" +#include "re2/regexp.h" + +#include "MySQL_PreparedStatement.h" +#include "MySQL_Data_Stream.h" +#include "MySQL_Authentication.hpp" +#include "MySQL_LDAP_Authentication.hpp" +#include "MySQL_Variables.h" + +#include + +//#include + +extern MySQL_Authentication *GloMyAuth; +extern MySQL_LDAP_Authentication *GloMyLdapAuth; +extern MySQL_Threads_Handler *GloMTH; + +#ifdef PROXYSQLCLICKHOUSE +extern ClickHouse_Authentication *GloClickHouseAuth; +#endif /* PROXYSQLCLICKHOUSE */ + +#ifdef max_allowed_packet +#undef max_allowed_packet +#endif + +#ifndef CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA +#define CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA 0x00200000 +#endif + +#include "proxysql_find_charset.h" + + +extern "C" char * sha256_crypt_r (const char *key, const char *salt, char *buffer, int buflen); + + +uint8_t mysql_encode_length(uint64_t len, char *hd); + + +MySQL_ResultSet::MySQL_ResultSet() { + buffer = NULL; + //reset_pid = true; +} + +void MySQL_ResultSet::buffer_init(MySQL_Protocol* myproto) { + if (buffer==NULL) { + buffer=(unsigned char *)malloc(RESULTSET_BUFLEN); + } + + buffer_used=0; + myprot = myproto; +} + +void MySQL_ResultSet::init(MySQL_Protocol *_myprot, MYSQL_RES *_res, MYSQL *_my, MYSQL_STMT *_stmt) { + PROXY_TRACE2(); + transfer_started=false; + resultset_completed=false; + myprot=_myprot; + mysql=_my; + stmt=_stmt; + if (buffer==NULL) { + //if (_stmt==NULL) { // we allocate this buffer only for not prepared statements + // removing the previous assumption. We allocate this buffer also for prepared statements + buffer=(unsigned char *)malloc(RESULTSET_BUFLEN); + //} + } + buffer_used=0; + myds=NULL; + if (myprot) { // if myprot = NULL , this is a mirror + myds=myprot->get_myds(); + } + //if (reset_pid==true) { + sid=0; + //PSarrayOUT = NULL; + if (myprot) { // if myprot = NULL , this is a mirror + sid=myds->pkt_sid+1; + //PSarrayOUT = new PtrSizeArray(8); + } + //} + //reset_pid=true; + result=_res; + resultset_size=0; + num_rows=0; + num_fields=mysql_field_count(mysql); + PtrSize_t pkt; + // immediately generate the first set of packets + // columns count + if (myprot==NULL) { + return; // this is a mirror + } + MySQL_Data_Stream * c_myds = *(myprot->myds); + if (c_myds->com_field_list==false) { + myprot->generate_pkt_column_count(false,&pkt.ptr,&pkt.size,sid,num_fields,this); + sid++; + resultset_size+=pkt.size; + } + // columns description + for (unsigned int i=0; icom_field_list==false) { + // we are replacing generate_pkt_field() with a more efficient version + //myprot->generate_pkt_field(false,&pkt.ptr,&pkt.size,sid,field->db,field->table,field->org_table,field->name,field->org_name,field->charsetnr,field->length,field->type,field->flags,field->decimals,false,0,NULL,this); + myprot->generate_pkt_field2(&pkt.ptr,&pkt.size,sid,field,this); + resultset_size+=pkt.size; + sid++; + } else { + if (c_myds->com_field_wild==NULL || mywildcmp(c_myds->com_field_wild,field->name)) { + myprot->generate_pkt_field(false,&pkt.ptr,&pkt.size,sid,field->db,field->table,field->org_table,field->name,field->org_name,field->charsetnr,field->length,field->type,field->flags,field->decimals,true,4,(char *)"null",this); + resultset_size+=pkt.size; + sid++; + } + } + } + + deprecate_eof_active = c_myds->myconn && (c_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF); + + // first EOF + unsigned int nTrx=myds->sess->NumActiveTransactions(); + uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0 ); + if (myds->sess->autocommit) setStatus += SERVER_STATUS_AUTOCOMMIT; + setStatus |= ( mysql->server_status & ~SERVER_STATUS_AUTOCOMMIT ); // get flags from server_status but ignore autocommit + setStatus = setStatus & ~SERVER_STATUS_CURSOR_EXISTS; // Do not send cursor #1128 +// if (_stmt) { // binary protocol , we also assume we have ALL the resultset +// myprot->generate_pkt_EOF(false,&pkt.ptr,&pkt.size,sid,0,mysql->server_status|setStatus); +// sid++; +// PSarrayOUT.add(pkt.ptr,pkt.size); +// resultset_size+=pkt.size; + //} else { + if (RESULTSET_BUFLEN <= (buffer_used + 9)) { + buffer_to_PSarrayOut(); + } + if (!deprecate_eof_active && myds->com_field_list==false) { + // up to 2.2.0 we used to add an EOF here. + // due to bug #3547 we move the logic into add_eof() that can now handle also prepared statements + PROXY_TRACE2(); + // if the backend server has CLIENT_DEPRECATE_EOF enabled, and the client does not support + // CLIENT_DEPRECATE_EOF, warning_count will be excluded from the intermediate EOF packet + add_eof((mysql->server_capabilities & CLIENT_DEPRECATE_EOF)); + } +} + + +// due to bug #3547 , in case of an error we remove the EOF +// and replace it with an ERR +// note that EOF is added on a packet on its own, instead of using a buffer, +// so that can be removed using remove_last_eof() +void MySQL_ResultSet::remove_last_eof() { + PROXY_TRACE2(); + PtrSize_t pkt; + if (PSarrayOUT.len) { + unsigned int l = PSarrayOUT.len-1; + PtrSize_t * pktp = PSarrayOUT.index(l); + if (pktp->size == 9) { + PROXY_TRACE2(); + PSarrayOUT.remove_index(l,&pkt); + l_free(pkt.size, pkt.ptr); + sid--; + } + } +} + +void MySQL_ResultSet::init_with_stmt(MySQL_Connection *myconn) { + PROXY_TRACE2(); + assert(stmt); + MYSQL_STMT *_stmt = stmt; + MySQL_Data_Stream * c_myds = *(myprot->myds); + buffer_to_PSarrayOut(); + unsigned long long total_size=0; + MYSQL_ROWS *r=_stmt->result.data; + if (r) { + total_size+=r->length; + if (r->length > 0xFFFFFF) { + total_size+=(r->length / 0xFFFFFF) * sizeof(mysql_hdr); + } + total_size+=sizeof(mysql_hdr); + while(r->next) { + r=r->next; + total_size+=r->length; + if (r->length > 0xFFFFFF) { + total_size+=(r->length / 0xFFFFFF) * sizeof(mysql_hdr); + } + total_size+=sizeof(mysql_hdr); + } +#define MAXBUFFSTMT 12*1024*1024 // hardcoded to LESS *very important* than 16MB + if (total_size < MAXBUFFSTMT) { + PtrSize_t pkt; + pkt.size=total_size; + pkt.ptr=malloc(pkt.size); + total_size=0; + r=_stmt->result.data; + add_row2(r,(unsigned char *)pkt.ptr); + total_size+=r->length; + if (r->length > 0xFFFFFF) { + total_size+=(r->length / 0xFFFFFF) * sizeof(mysql_hdr); + } + total_size+=sizeof(mysql_hdr); + while(r->next) { + r=r->next; + add_row2(r,(unsigned char *)pkt.ptr+total_size); + total_size+=r->length; + if (r->length > 0xFFFFFF) { + total_size+=(r->length / 0xFFFFFF) * sizeof(mysql_hdr); + } + total_size+=sizeof(mysql_hdr); + } + PSarrayOUT.add(pkt.ptr,pkt.size); + if (resultset_size/0xFFFFFFF != ((resultset_size+pkt.size)/0xFFFFFFF)) { + // generate a heartbeat every 256MB + unsigned long long curtime=monotonic_time(); + c_myds->sess->thread->atomic_curtime=curtime; + } + resultset_size+=pkt.size; + } else { // this code fixes a bug: resultset larger than 4GB would cause a crash + unsigned long long tmp_pkt_size = 0; + r=_stmt->result.data; + MYSQL_ROWS * r2 = NULL; + while (r) { + if (r->length >= MAXBUFFSTMT) { + // we have a large row + // we will send just that + tmp_pkt_size = r->length; + if (r->length > 0xFFFFFF) { + tmp_pkt_size+=(r->length / 0xFFFFFF) * sizeof(mysql_hdr); + } + tmp_pkt_size += sizeof(mysql_hdr); + PtrSize_t pkt; + pkt.size=tmp_pkt_size; + pkt.ptr=malloc(pkt.size); + add_row2(r,(unsigned char *)pkt.ptr); + PSarrayOUT.add(pkt.ptr,pkt.size); + if (resultset_size/0xFFFFFFF != ((resultset_size+pkt.size)/0xFFFFFFF)) { + // generate a heartbeat every 256MB + unsigned long long curtime=monotonic_time(); + c_myds->sess->thread->atomic_curtime=curtime; + } + resultset_size+=pkt.size; + r=r->next; // next row + } else { // we have small row + r2 = r; + tmp_pkt_size = 0; + unsigned int a = 0; + while (r && (tmp_pkt_size + r->length) < MAXBUFFSTMT) { + a++; + tmp_pkt_size += r->length; + tmp_pkt_size += sizeof(mysql_hdr); + //if (r->next) { + r = r->next; + //} + } + r = r2; // we reset it back to the beginning + if (tmp_pkt_size) { // this should always be true + unsigned long long tmp2 = 0; + PtrSize_t pkt; + pkt.size=tmp_pkt_size; + pkt.ptr=malloc(pkt.size); + while (tmp2 < tmp_pkt_size) { + add_row2(r,(unsigned char *)pkt.ptr+tmp2); + tmp2 += r->length; + tmp2 += sizeof(mysql_hdr); + r = r->next; + } + PSarrayOUT.add(pkt.ptr,pkt.size); + if (resultset_size/0xFFFFFFF != ((resultset_size+pkt.size)/0xFFFFFFF)) { + // generate a heartbeat every 256MB + unsigned long long curtime=monotonic_time(); + c_myds->sess->thread->atomic_curtime=curtime; + } + resultset_size+=pkt.size; + } + } + } + } + } + // up to 2.2.0 we were always adding an EOF + // due to bug #3547 , in case of an error we remove the EOF + // and replace it with an ERR + // note that EOF is added on a packet on its own, instead of using a buffer, + // so that can be removed + // + // NOTE: After 2.4.5 previous behavior is modified in favor of the following: + // + // When CLIENT_DEPRECATE_EOF two EOF packets are two be expected in the response: + // 1. After the columns definitions (This is added directly by 'MySQL_ResultSet::init'). + // 2. After the rows values, this can either be and EOF packet or a ERR packet in case of error. + // + // First EOF packet isn't optional, and it's just the second the one that is optionaly either an EOF + // or an ERR packet. The following code adds either the final EOF or ERR packet. This is equally valid + // for when CLIENT_DEPRECATE_EOF is enabled or not. If CLIENT_DEPRECATE_EOF is: + // * DISABLED: The behavior is as described before. + // * ENABLED: Code is identical for this case. The initial EOF packet is conditionally added by + // 'MySQL_ResultSet::init', thus, this packet should not be present if not needed at this point. + // In case of error an ERR packet needs to be added, otherwise `add_eof` handles the generation of + // the equivalent OK packet replacing the final EOF packet. + int myerr = mysql_stmt_errno(_stmt); + if (myerr) { + PROXY_TRACE2(); + add_err(myconn->myds); + } else { + PROXY_TRACE2(); + add_eof(); + } +} + +MySQL_ResultSet::~MySQL_ResultSet() { + PtrSize_t pkt; + //if (PSarrayOUT) { + while (PSarrayOUT.len) { + PSarrayOUT.remove_index_fast(0,&pkt); + l_free(pkt.size, pkt.ptr); + } + //delete PSarrayOUT; + //} + if (buffer) { + free(buffer); + buffer=NULL; + } + //if (myds) myds->pkt_sid=sid-1; +} + +// this function is used for binary protocol +// maybe later on can be adapted for text protocol too +unsigned int MySQL_ResultSet::add_row(MYSQL_ROWS *rows) { + unsigned int pkt_length=0; + MYSQL_ROW row = rows->data; + unsigned long row_length = rows->length; + // we call generate_pkt_row3 passing row_length + sid=myprot->generate_pkt_row3(this, &pkt_length, sid, 0, NULL, row, row_length); + sid++; + resultset_size+=pkt_length; + num_rows++; + return pkt_length; +} + + +// this function is used for text protocol +unsigned int MySQL_ResultSet::add_row(MYSQL_ROW row) { + unsigned long *lengths=mysql_fetch_lengths(result); + unsigned int pkt_length=0; + if (myprot) { + // we call generate_pkt_row3 without passing row_length + sid=myprot->generate_pkt_row3(this, &pkt_length, sid, num_fields, lengths, row, 0); + } else { + unsigned int col=0; + for (col=0; collength; + num_rows++; + uint8_t pkt_sid=sid; + if (length < (0xFFFFFF+sizeof(mysql_hdr))) { + mysql_hdr myhdr; + myhdr.pkt_length=length; + myhdr.pkt_id=pkt_sid; + memcpy(offset, &myhdr, sizeof(mysql_hdr)); + memcpy(offset+sizeof(mysql_hdr), row->data, row->length); + pkt_sid++; + } else { + unsigned int left=length; + unsigned int copied=0; + while (left>=0xFFFFFF) { + mysql_hdr myhdr; + myhdr.pkt_length=0xFFFFFF; + myhdr.pkt_id=pkt_sid; + pkt_sid++; + memcpy(offset, &myhdr, sizeof(mysql_hdr)); + offset+=sizeof(mysql_hdr); + char *o = (char *) row->data; + o += copied; + memcpy(offset, o, myhdr.pkt_length); + offset+=0xFFFFFF; + // we are writing a large packet (over 16MB), we assume we are always outside the buffer + copied+=0xFFFFFF; + left-=0xFFFFFF; + } + mysql_hdr myhdr; + myhdr.pkt_length=left; + myhdr.pkt_id=pkt_sid; + pkt_sid++; + memcpy(offset, &myhdr, sizeof(mysql_hdr)); + offset+=sizeof(mysql_hdr); + char *o = (char *) row->data; + o += copied; + memcpy(offset, o, myhdr.pkt_length); + // we are writing a large packet (over 16MB), we assume we are always outside the buffer + } + sid=pkt_sid; + return length; +} + +void MySQL_ResultSet::add_eof(bool suppress_warning_count) { + if (myprot) { + unsigned int nTrx=myds->sess->NumActiveTransactions(); + uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0 ); + if (myds->sess->autocommit) setStatus += SERVER_STATUS_AUTOCOMMIT; + setStatus |= ( mysql->server_status & ~SERVER_STATUS_AUTOCOMMIT ); // get flags from server_status but ignore autocommit + setStatus = setStatus & ~SERVER_STATUS_CURSOR_EXISTS; // Do not send cursor #1128 + //myprot->generate_pkt_EOF(false,&pkt.ptr,&pkt.size,sid,0,mysql->server_status|setStatus); + //PSarrayOUT->add(pkt.ptr,pkt.size); + //sid++; + //resultset_size+=pkt.size; + + // Note: warnings count will only be sent to the client if mysql-query_digests is enabled + const MySQL_Backend* _mybe = myds->sess->mybe; + const MySQL_Data_Stream* _server_myds = (_mybe && _mybe->server_myds) ? _mybe->server_myds : nullptr; + const MySQL_Connection* _myconn = (_server_myds && _server_myds->myds_type == MYDS_BACKEND && _server_myds->myconn) ? + _server_myds->myconn : nullptr; + const unsigned int warning_count = (_myconn && suppress_warning_count == false) ? _myconn->warning_count : 0; + if (deprecate_eof_active) { + PtrSize_t pkt; + buffer_to_PSarrayOut(); + myprot->generate_pkt_OK(false, &pkt.ptr, &pkt.size, sid, 0, 0, setStatus, warning_count, NULL, true); + PSarrayOUT.add(pkt.ptr, pkt.size); + resultset_size += pkt.size; + } + else { + // due to bug #3547 , in case of an error we remove the EOF + // and replace it with an ERR + // note that EOF is added on a packet on its own, instead of using a buffer, + // so that can be removed using remove_last_eof() + buffer_to_PSarrayOut(); + myprot->generate_pkt_EOF(false, NULL, NULL, sid, warning_count, setStatus, this); + resultset_size += 9; + buffer_to_PSarrayOut(); + } + sid++; + } + resultset_completed=true; +} + +void MySQL_ResultSet::add_err(MySQL_Data_Stream *_myds) { + PtrSize_t pkt; + if (myprot) { + MYSQL *_mysql=_myds->myconn->mysql; + buffer_to_PSarrayOut(); + char sqlstate[10]; + sprintf(sqlstate,"%s",mysql_sqlstate(_mysql)); + if (_myds && _myds->killed_at) { // see case #750 + if (_myds->kill_type == 0) { + myprot->generate_pkt_ERR(false,&pkt.ptr,&pkt.size,sid,1907,sqlstate,(char *)"Query execution was interrupted, query_timeout exceeded"); + MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, _myds->myconn->parent->myhgc->hid, _myds->myconn->parent->address, _myds->myconn->parent->port, 1907); + } else { + myprot->generate_pkt_ERR(false,&pkt.ptr,&pkt.size,sid,1317,sqlstate,(char *)"Query execution was interrupted"); + MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, _myds->myconn->parent->myhgc->hid, _myds->myconn->parent->address, _myds->myconn->parent->port, 1317); + } + } else { + int myerr = 0; + // the error code is returned from: + // - mysql_stmt_errno() if using a prepared statement + // - mysql_errno() if not using a prepared statement + if (stmt) { + myerr = mysql_stmt_errno(stmt); + myprot->generate_pkt_ERR(false,&pkt.ptr,&pkt.size,sid,myerr,sqlstate,mysql_stmt_error(stmt)); + } else { + myerr = mysql_errno(_mysql); + myprot->generate_pkt_ERR(false,&pkt.ptr,&pkt.size,sid,myerr,sqlstate,mysql_error(_mysql)); + } + // TODO: Check this is a mysql error + MyHGM->p_update_mysql_error_counter(p_mysql_error_type::mysql, _myds->myconn->parent->myhgc->hid, _myds->myconn->parent->address, _myds->myconn->parent->port, myerr); + } + PSarrayOUT.add(pkt.ptr,pkt.size); + sid++; + resultset_size+=pkt.size; + } + resultset_completed=true; +} + +/* +bool MySQL_ResultSet::get_COM_FIELD_LIST_response(PtrSizeArray *PSarrayFinal) { + transfer_started=true; + if (myprot) { + } + return resultset_completed; +} +*/ + +bool MySQL_ResultSet::get_resultset(PtrSizeArray *PSarrayFinal) { + transfer_started=true; + if (myprot) { + PSarrayFinal->copy_add(&PSarrayOUT,0,PSarrayOUT.len); + while (PSarrayOUT.len) + PSarrayOUT.remove_index(PSarrayOUT.len-1,NULL); + } + return resultset_completed; +} + +void MySQL_ResultSet::buffer_to_PSarrayOut(bool _last) { + if (buffer_used==0) + return; // exit immediately if the buffer is empty + if (buffer_used < RESULTSET_BUFLEN/2) { + if (_last == false) { + buffer=(unsigned char *)realloc(buffer,buffer_used); + } + } + PSarrayOUT.add(buffer,buffer_used); + if (_last) { + buffer = NULL; + } else { + buffer=(unsigned char *)malloc(RESULTSET_BUFLEN); + } + buffer_used=0; +} + +unsigned long long MySQL_ResultSet::current_size() { + unsigned long long intsize=0; + intsize+=sizeof(MySQL_ResultSet); + intsize+=RESULTSET_BUFLEN; // size of buffer + if (PSarrayOUT.len==0) // see bug #699 + return intsize; + intsize+=sizeof(PtrSizeArray); + intsize+=(PSarrayOUT.size*sizeof(PtrSize_t *)); + unsigned int i; + for (i=0; isize>RESULTSET_BUFLEN) { + intsize+=pkt->size; + } else { + intsize+=RESULTSET_BUFLEN; + } + } + return intsize; +} + +/* +my_bool proxy_mysql_stmt_close(MYSQL_STMT* stmt) { + // Clean internal structures for 'stmt->mysql->stmts'. + if (stmt->mysql) { + stmt->mysql->stmts = + list_delete(stmt->mysql->stmts, &stmt->list); + } + // Nullify 'mysql' field to avoid sending a blocking command to the server. + stmt->mysql = NULL; + // Perform the regular close operation. + return mysql_stmt_close(stmt); +} +*/ diff --git a/lib/MySQL_encode.cpp b/lib/MySQL_encode.cpp new file mode 100644 index 0000000000..13a4f0461b --- /dev/null +++ b/lib/MySQL_encode.cpp @@ -0,0 +1,239 @@ +#include +#include "proxysql.h" +#include "cpp.h" + +#ifdef DEBUG +void __dump_pkt(const char *func, unsigned char *_ptr, unsigned int len) { + + if (GloVars.global.gdbg==0) return; + if (GloVars.global.gdbg_lvl[PROXY_DEBUG_MYSQL_PROTOCOL].verbosity < 8 ) return; + unsigned int i; + fprintf(stderr,"DUMP %d bytes FROM %s\n", len, func); + for(i = 0; i < len; i++) { + if(isprint(_ptr[i])) fprintf(stderr,"%c", _ptr[i]); else fprintf(stderr,"."); + if (i>0 && (i%16==15 || i==len-1)) { + unsigned int j; + if (i%16!=15) { + j=15-i%16; + while (j--) fprintf(stderr," "); + } + fprintf(stderr," --- "); + for (j=(i==len-1 ? ((int)(i/16))*16 : i-15 ) ; j<=i; j++) { + fprintf(stderr,"%02x ", _ptr[j]); + } + fprintf(stderr,"\n"); + } + } + fprintf(stderr,"\n\n"); + + +} +#endif + +char *sha1_pass_hex(char *sha1_pass) { + if (sha1_pass==NULL) return NULL; + char *buff=(char *)malloc(SHA_DIGEST_LENGTH*2+2); + buff[0]='*'; + buff[SHA_DIGEST_LENGTH*2+1]='\0'; + int i; + uint8_t a = 0; + for (i=0;iseed1= (rand_st->seed1*3+rand_st->seed2) % rand_st->max_value; + rand_st->seed2= (rand_st->seed1+rand_st->seed2+33) % rand_st->max_value; + return (((double) rand_st->seed1) / rand_st->max_value_dbl); +} + +void proxy_create_random_string(char *_to, uint length, struct rand_struct *rand_st) { + unsigned char * to = (unsigned char *)_to; + int rc = 0; + uint i; + rc = RAND_bytes((unsigned char *)to,length); +#ifdef DEBUG + if (rc==1) { + // For code coverage (to test the following code and other function) + // in DEBUG mode we pretend that RAND_bytes() fails 1% of the time + if(rand()%100==0) { + rc=0; + } + } +#endif // DEBUG + if (rc!=1) { + for (i=0; i 127) { + *to -= 128; + } + if (*to == 0) { + *to = 'a'; + } + to++; + } + } + *to= '\0'; +} + +int write_encoded_length(unsigned char *p, uint64_t val, uint8_t len, char prefix) { + if (len==1) { + *p=(char)val; + return 1; + } + *p=prefix; + p++; + memcpy(p,&val,len-1); + return len; +} + +int write_encoded_length_and_string(unsigned char *p, uint64_t val, uint8_t len, char prefix, char *string) { + int l=write_encoded_length(p,val,len,prefix); + if (val) { + memcpy(p+l,string,val); + } + return l+val; +} + +void proxy_compute_sha1_hash_multi(uint8_t *digest, const char *buf1, int len1, const char *buf2, int len2) { + PROXY_TRACE(); + const EVP_MD *evp_digest = EVP_get_digestbyname("sha1"); + assert(evp_digest != NULL); + EVP_MD_CTX *ctx = EVP_MD_CTX_new(); + EVP_MD_CTX_init(ctx); + EVP_DigestInit_ex(ctx, evp_digest, NULL); + EVP_DigestUpdate(ctx, buf1, len1); + EVP_DigestUpdate(ctx, buf2, len2); + unsigned int olen = 0; + EVP_DigestFinal(ctx, digest, &olen); + EVP_MD_CTX_free(ctx); +} + +void proxy_compute_sha1_hash(uint8_t *digest, const char *buf, int len) { + PROXY_TRACE(); + const EVP_MD *evp_digest = EVP_get_digestbyname("sha1"); + assert(evp_digest != NULL); + EVP_MD_CTX *ctx = EVP_MD_CTX_new(); + EVP_MD_CTX_init(ctx); + EVP_DigestInit_ex(ctx, evp_digest, NULL); + EVP_DigestUpdate(ctx, buf, len); + unsigned int olen = 0; + EVP_DigestFinal(ctx, digest, &olen); + EVP_MD_CTX_free(ctx); +} + +void proxy_compute_two_stage_sha1_hash(const char *password, size_t pass_len, uint8_t *hash_stage1, uint8_t *hash_stage2) { + proxy_compute_sha1_hash(hash_stage1, password, pass_len); + proxy_compute_sha1_hash(hash_stage2, (const char *) hash_stage1, SHA_DIGEST_LENGTH); +} + +void proxy_my_crypt(char *to, const uint8_t *s1, const uint8_t *s2, uint len) { + const uint8_t *s1_end= s1 + len; + while (s1 < s1_end) + *to++= *s1++ ^ *s2++; +} + +unsigned char decode_char(char x) { + if (x >= '0' && x <= '9') + return (x - 0x30); + else if (x >= 'A' && x <= 'F') + return(x - 0x37); + else if (x >= 'a' && x <= 'f') + return(x - 0x57); + else { + proxy_error("Invalid char"); + return 0; + } +} + +void unhex_pass(uint8_t *out, const char *in) { + int i=0; + for (i=0;i +#include +#include + +#include +#include +#include +#include + +#include "prometheus_helpers.h" +#include "proxysql_utils.h" + +#define char_malloc (char *)malloc +#define itostr(__s, __i) { __s=char_malloc(32); sprintf(__s, "%lld", __i); } + +#include "thread.h" +#include "wqueue.h" + +#include "ev.h" + +#include +#include +#include + +using std::function; + +#ifdef TEST_AURORA +static unsigned long long array_mysrvc_total = 0; +static unsigned long long array_mysrvc_cands = 0; +#endif // TEST_AURORA + +#define SAFE_SQLITE3_STEP(_stmt) do {\ + do {\ + rc=(*proxy_sqlite3_step)(_stmt);\ + if (rc!=SQLITE_DONE) {\ + assert(rc==SQLITE_LOCKED);\ + usleep(100);\ + }\ + } while (rc!=SQLITE_DONE);\ +} while (0) + +extern ProxySQL_Admin *GloAdmin; + +extern MySQL_Threads_Handler *GloMTH; + +extern MySQL_Monitor *GloMyMon; +*/ + +class MySrvConnList; +class MySrvC; +class MySrvList; +class MyHGC; + +MySrvC::MySrvC( + char* add, uint16_t p, uint16_t gp, int64_t _weight, enum MySerStatus _status, unsigned int _compression, + int64_t _max_connections, unsigned int _max_replication_lag, int32_t _use_ssl, unsigned int _max_latency_ms, + char* _comment +) { + address=strdup(add); + port=p; + gtid_port=gp; + weight=_weight; + status=_status; + compression=_compression; + max_connections=_max_connections; + max_replication_lag=_max_replication_lag; + use_ssl=_use_ssl; + cur_replication_lag=0; + cur_replication_lag_count=0; + max_latency_us=_max_latency_ms*1000; + current_latency_us=0; + aws_aurora_current_lag_us = 0; + connect_OK=0; + connect_ERR=0; + queries_sent=0; + bytes_sent=0; + bytes_recv=0; + max_connections_used=0; + queries_gtid_sync=0; + time_last_detected_error=0; + connect_ERR_at_time_last_detected_error=0; + shunned_automatic=false; + shunned_and_kill_all_connections=false; // false to default + //charset=_charset; + myhgc=NULL; + comment=strdup(_comment); + ConnectionsUsed=new MySrvConnList(this); + ConnectionsFree=new MySrvConnList(this); +} + +void MySrvC::connect_error(int err_num, bool get_mutex) { + // NOTE: this function operates without any mutex + // although, it is not extremely important if any counter is lost + // as a single connection failure won't make a significant difference + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Connect failed with code '%d'\n", err_num); + __sync_fetch_and_add(&connect_ERR,1); + __sync_fetch_and_add(&MyHGM->status.server_connections_aborted,1); + if (err_num >= 1048 && err_num <= 1052) + return; + if (err_num >= 1054 && err_num <= 1075) + return; + if (err_num >= 1099 && err_num <= 1104) + return; + if (err_num >= 1106 && err_num <= 1113) + return; + if (err_num >= 1116 && err_num <= 1118) + return; + if (err_num == 1136 || (err_num >= 1138 && err_num <= 1149)) + return; + switch (err_num) { + case 1007: // Can't create database + case 1008: // Can't drop database + case 1044: // access denied + case 1045: // access denied +/* + case 1048: // Column cannot be null + case 1049: // Unknown database + case 1050: // Table already exists + case 1051: // Unknown table + case 1052: // Column is ambiguous +*/ + case 1120: + case 1203: // User %s already has more than 'max_user_connections' active connections + case 1226: // User '%s' has exceeded the '%s' resource (current value: %ld) + case 3118: // Access denied for user '%s'. Account is locked.. + return; + break; + default: + break; + } + time_t t=time(NULL); + if (t > time_last_detected_error) { + time_last_detected_error=t; + connect_ERR_at_time_last_detected_error=1; + } else { + if (t < time_last_detected_error) { + // time_last_detected_error is in the future + // this means that monitor has a ping interval too big and tuned that in the future + return; + } + // same time + /** + * @brief The expected configured retries set by 'mysql-connect_retries_on_failure' + '2' extra expected + * connection errors. + * @details This two extra connections errors are expected: + * 1. An initial connection error generated by the datastream and the connection when being created, + * this is, right after the session has requested a connection to the connection pool. This error takes + * places directly in the state machine from 'MySQL_Connection'. Because of this, we consider this + * additional error to be a consequence of the two states machines, and it's not considered for + * 'connect_retries'. + * 2. A second connection connection error, which is the initial connection error generated by 'MySQL_Session' + * when already in the 'CONNECTING_SERVER' state. This error is an 'extra error' to always consider, since + * it's not part of the retries specified by 'mysql_thread___connect_retries_on_failure', thus, we set the + * 'connect_retries' to be 'mysql_thread___connect_retries_on_failure + 1'. + */ + int connect_retries = mysql_thread___connect_retries_on_failure + 1; + int max_failures = mysql_thread___shun_on_failures > connect_retries ? connect_retries : mysql_thread___shun_on_failures; + + if (__sync_add_and_fetch(&connect_ERR_at_time_last_detected_error,1) >= (unsigned int)max_failures) { + bool _shu=false; + if (get_mutex==true) + MyHGM->wrlock(); // to prevent race conditions, lock here. See #627 + if (status==MYSQL_SERVER_STATUS_ONLINE) { + status=MYSQL_SERVER_STATUS_SHUNNED; + shunned_automatic=true; + _shu=true; + } else { + _shu=false; + } + if (get_mutex==true) + MyHGM->wrunlock(); + if (_shu) { + proxy_error("Shunning server %s:%d with %u errors/sec. Shunning for %u seconds\n", address, port, connect_ERR_at_time_last_detected_error , mysql_thread___shun_recovery_time_sec); + } + } + } +} + +void MySrvC::shun_and_killall() { + status=MYSQL_SERVER_STATUS_SHUNNED; + shunned_automatic=true; + shunned_and_kill_all_connections=true; +} + +MySrvC::~MySrvC() { + if (address) free(address); + if (comment) free(comment); + delete ConnectionsUsed; + delete ConnectionsFree; +} diff --git a/lib/MySrvConnList.cpp b/lib/MySrvConnList.cpp new file mode 100644 index 0000000000..abe0c44ee7 --- /dev/null +++ b/lib/MySrvConnList.cpp @@ -0,0 +1,256 @@ +#include "MySQL_HostGroups_Manager.h" + +#include "MySQL_Data_Stream.h" + +extern ProxySQL_Admin *GloAdmin; + +extern MySQL_Threads_Handler *GloMTH; + +extern MySQL_Monitor *GloMyMon; + +class MySrvConnList; +class MySrvC; +class MySrvList; +class MyHGC; + +MySQL_Connection *MySrvConnList::index(unsigned int _k) { + return (MySQL_Connection *)conns->index(_k); +} + +MySQL_Connection * MySrvConnList::remove(int _k) { + return (MySQL_Connection *)conns->remove_index_fast(_k); +} + +MySrvConnList::MySrvConnList(MySrvC *_mysrvc) { + mysrvc=_mysrvc; + conns=new PtrArray(); +} + +void MySrvConnList::add(MySQL_Connection *c) { + conns->add(c); +} + +MySrvConnList::~MySrvConnList() { + mysrvc=NULL; + while (conns_length()) { + MySQL_Connection *conn=(MySQL_Connection *)conns->remove_index_fast(0); + delete conn; + } + delete conns; +} + +void MySrvConnList::drop_all_connections() { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Dropping all connections (%u total) on MySrvConnList %p for server %s:%d , hostgroup=%d , status=%d\n", conns_length(), this, mysrvc->address, mysrvc->port, mysrvc->myhgc->hid, mysrvc->status); + while (conns_length()) { + MySQL_Connection *conn=(MySQL_Connection *)conns->remove_index_fast(0); + delete conn; + } +} + +void MySrvConnList::get_random_MyConn_inner_search(unsigned int start, unsigned int end, unsigned int& conn_found_idx, unsigned int& connection_quality_level, unsigned int& number_of_matching_session_variables, const MySQL_Connection * client_conn) { + char *schema = client_conn->userinfo->schemaname; + MySQL_Connection * conn=NULL; + unsigned int k; + for (k = start; k < end; k++) { + conn = (MySQL_Connection *)conns->index(k); + if (conn->match_tracked_options(client_conn)) { + if (connection_quality_level == 0) { + // this is our best candidate so far + connection_quality_level = 1; + conn_found_idx = k; + } + if (conn->requires_CHANGE_USER(client_conn)==false) { + if (connection_quality_level == 1) { + // this is our best candidate so far + connection_quality_level = 2; + conn_found_idx = k; + } + unsigned int cnt_match = 0; // number of matching session variables + unsigned int not_match = 0; // number of not matching session variables + cnt_match = conn->number_of_matching_session_variables(client_conn, not_match); + if (strcmp(conn->userinfo->schemaname,schema)==0) { + cnt_match++; + } else { + not_match++; + } + if (not_match==0) { + // it seems we found the perfect connection + number_of_matching_session_variables = cnt_match; + connection_quality_level = 3; + conn_found_idx = k; + return; // exit immediately, we found the perfect connection + } else { + // we didn't find the perfect connection + // but maybe is better than what we have so far? + if (cnt_match > number_of_matching_session_variables) { + // this is our best candidate so far + number_of_matching_session_variables = cnt_match; + conn_found_idx = k; + } + } + } else { + if (connection_quality_level == 1) { + int rca = mysql_thread___reset_connection_algorithm; + if (rca==1) { + int ql = GloMTH->variables.connpoll_reset_queue_length; + if (ql==0) { + // if: + // mysql-reset_connection_algorithm=1 and + // mysql-connpoll_reset_queue_length=0 + // we will not return a connection with connection_quality_level == 1 + // because we want to run COM_CHANGE_USER + // This change was introduced to work around Galera bug + // https://github.com/codership/galera/issues/613 + connection_quality_level = 0; + } + } + } + } + } + } +} + + + +MySQL_Connection * MySrvConnList::get_random_MyConn(MySQL_Session *sess, bool ff) { + MySQL_Connection * conn=NULL; + unsigned int i; + unsigned int conn_found_idx; + unsigned int l=conns_length(); + unsigned int connection_quality_level = 0; + bool needs_warming = false; + // connection_quality_level: + // 0 : not found any good connection, tracked options are not OK + // 1 : tracked options are OK , but CHANGE USER is required + // 2 : tracked options are OK , CHANGE USER is not required, but some SET statement or INIT_DB needs to be executed + // 3 : tracked options are OK , CHANGE USER is not required, and it seems that SET statements or INIT_DB ARE not required + unsigned int number_of_matching_session_variables = 0; // this includes session variables AND schema + bool connection_warming = mysql_thread___connection_warming; + int free_connections_pct = mysql_thread___free_connections_pct; + if (mysrvc->myhgc->attributes.configured == true) { + // mysql_hostgroup_attributes takes priority + connection_warming = mysrvc->myhgc->attributes.connection_warming; + free_connections_pct = mysrvc->myhgc->attributes.free_connections_pct; + } + if (connection_warming == true) { + unsigned int total_connections = mysrvc->ConnectionsFree->conns_length()+mysrvc->ConnectionsUsed->conns_length(); + unsigned int expected_warm_connections = free_connections_pct*mysrvc->max_connections/100; + if (total_connections < expected_warm_connections) { + needs_warming = true; + } + } + if (l && ff==false && needs_warming==false) { + if (l>32768) { + i=rand()%l; + } else { + i=fastrand()%l; + } + if (sess && sess->client_myds && sess->client_myds->myconn && sess->client_myds->myconn->userinfo) { + MySQL_Connection * client_conn = sess->client_myds->myconn; + get_random_MyConn_inner_search(i, l, conn_found_idx, connection_quality_level, number_of_matching_session_variables, client_conn); + if (connection_quality_level !=3 ) { // we didn't find the perfect connection + get_random_MyConn_inner_search(0, i, conn_found_idx, connection_quality_level, number_of_matching_session_variables, client_conn); + } + // connection_quality_level: + // 1 : tracked options are OK , but CHANGE USER is required + // 2 : tracked options are OK , CHANGE USER is not required, but some SET statement or INIT_DB needs to be executed + switch (connection_quality_level) { + case 0: // not found any good connection, tracked options are not OK + // we must check if connections need to be freed before + // creating a new connection + { + unsigned int conns_free = mysrvc->ConnectionsFree->conns_length(); + unsigned int conns_used = mysrvc->ConnectionsUsed->conns_length(); + unsigned int pct_max_connections = (3 * mysrvc->max_connections) / 4; + unsigned int connections_to_free = 0; + + if (conns_free >= 1) { + // connection cleanup is triggered when connections exceed 3/4 of the total + // allowed max connections, this cleanup ensures that at least *one connection* + // will be freed. + if (pct_max_connections <= (conns_free + conns_used)) { + connections_to_free = (conns_free + conns_used) - pct_max_connections; + if (connections_to_free == 0) connections_to_free = 1; + } + + while (conns_free && connections_to_free) { + MySQL_Connection* conn = mysrvc->ConnectionsFree->remove(0); + delete conn; + + conns_free = mysrvc->ConnectionsFree->conns_length(); + connections_to_free -= 1; + } + } + + // we must create a new connection + conn = new MySQL_Connection(); + conn->parent=mysrvc; + // if attributes.multiplex == true , STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG is set to false. And vice-versa + conn->set_status(!conn->parent->myhgc->attributes.multiplex, STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG); + __sync_fetch_and_add(&MyHGM->status.server_connections_created, 1); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySQL Connection %p, server %s:%d\n", conn, conn->parent->address, conn->parent->port); + } + break; + case 1: //tracked options are OK , but CHANGE USER is required + // we may consider creating a new connection + { + unsigned int conns_free = mysrvc->ConnectionsFree->conns_length(); + unsigned int conns_used = mysrvc->ConnectionsUsed->conns_length(); + if ((conns_used > conns_free) && (mysrvc->max_connections > (conns_free/2 + conns_used/2)) ) { + conn = new MySQL_Connection(); + conn->parent=mysrvc; + // if attributes.multiplex == true , STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG is set to false. And vice-versa + conn->set_status(!conn->parent->myhgc->attributes.multiplex, STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG); + __sync_fetch_and_add(&MyHGM->status.server_connections_created, 1); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySQL Connection %p, server %s:%d\n", conn, conn->parent->address, conn->parent->port); + } else { + conn=(MySQL_Connection *)conns->remove_index_fast(conn_found_idx); + } + } + break; + case 2: // tracked options are OK , CHANGE USER is not required, but some SET statement or INIT_DB needs to be executed + case 3: // tracked options are OK , CHANGE USER is not required, and it seems that SET statements or INIT_DB ARE not required + // here we return the best connection we have, no matter if connection_quality_level is 2 or 3 + conn=(MySQL_Connection *)conns->remove_index_fast(conn_found_idx); + break; + default: // this should never happen + // LCOV_EXCL_START + assert(0); + break; + // LCOV_EXCL_STOP + } + } else { + conn=(MySQL_Connection *)conns->remove_index_fast(i); + } + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySQL Connection %p, server %s:%d\n", conn, conn->parent->address, conn->parent->port); + return conn; + } else { + unsigned long long curtime = monotonic_time(); + curtime = curtime / 1000 / 1000; // convert to second + MyHGC *_myhgc = mysrvc->myhgc; + if (curtime > _myhgc->current_time_now) { + _myhgc->current_time_now = curtime; + _myhgc->new_connections_now = 0; + } + _myhgc->new_connections_now++; + unsigned int throttle_connections_per_sec_to_hostgroup = (unsigned int) mysql_thread___throttle_connections_per_sec_to_hostgroup; + if (_myhgc->attributes.configured == true) { + // mysql_hostgroup_attributes takes priority + throttle_connections_per_sec_to_hostgroup = _myhgc->attributes.throttle_connections_per_sec; + } + if (_myhgc->new_connections_now > (unsigned int) throttle_connections_per_sec_to_hostgroup) { + __sync_fetch_and_add(&MyHGM->status.server_connections_delayed, 1); + return NULL; + } else { + conn = new MySQL_Connection(); + conn->parent=mysrvc; + // if attributes.multiplex == true , STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG is set to false. And vice-versa + conn->set_status(!conn->parent->myhgc->attributes.multiplex, STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG); + __sync_fetch_and_add(&MyHGM->status.server_connections_created, 1); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySQL Connection %p, server %s:%d\n", conn, conn->parent->address, conn->parent->port); + return conn; + } + } + return NULL; // never reach here +} + diff --git a/lib/MySrvList.cpp b/lib/MySrvList.cpp new file mode 100644 index 0000000000..22ed9b1427 --- /dev/null +++ b/lib/MySrvList.cpp @@ -0,0 +1,44 @@ +#include "MySQL_HostGroups_Manager.h" + +class MySrvConnList; +class MySrvC; +class MySrvList; +class MyHGC; + +MySrvList::MySrvList(MyHGC *_myhgc) { + myhgc=_myhgc; + servers=new PtrArray(); +} + +void MySrvList::add(MySrvC *s) { + if (s->myhgc==NULL) { + s->myhgc=myhgc; + } + servers->add(s); +} + + +int MySrvList::find_idx(MySrvC *s) { + for (unsigned int i=0; ilen; i++) { + MySrvC *mysrv=(MySrvC *)servers->index(i); + if (mysrv==s) { + return (unsigned int)i; + } + } + return -1; +} + +void MySrvList::remove(MySrvC *s) { + int i=find_idx(s); + assert(i>=0); + servers->remove_index_fast((unsigned int)i); +} + +MySrvList::~MySrvList() { + myhgc=NULL; + while (servers->len) { + MySrvC *mysrvc=(MySrvC *)servers->remove_index_fast(0); + delete mysrvc; + } + delete servers; +} diff --git a/lib/QP_query_digest_stats.cpp b/lib/QP_query_digest_stats.cpp new file mode 100644 index 0000000000..509951b506 --- /dev/null +++ b/lib/QP_query_digest_stats.cpp @@ -0,0 +1,189 @@ +#include "query_processor.h" + +// reverse: reverse string s in place +static void reverse(char s[]) { + int i, j; + char c; + int l = strlen(s); + for (i = 0, j = l-1; i 0); /* delete it */ + s[i] = '\0'; + reverse(s); +} + + +QP_query_digest_stats::QP_query_digest_stats(char *u, char *s, uint64_t d, char *dt, int h, char *ca) { + digest=d; + digest_text=NULL; + if (dt) { + digest_text=strndup(dt, mysql_thread___query_digests_max_digest_length); + } + if (strlen(u) < sizeof(username_buf)) { + strcpy(username_buf,u); + username = username_buf; + } else { + username=strdup(u); + } + if (strlen(s) < sizeof(schemaname_buf)) { + strcpy(schemaname_buf,s); + schemaname = schemaname_buf; + } else { + schemaname=strdup(s); + } + if (strlen(ca) < sizeof(client_address_buf)) { + strcpy(client_address_buf,ca); + client_address = client_address_buf; + } else { + client_address=strdup(ca); + } + count_star=0; + first_seen=0; + last_seen=0; + sum_time=0; + min_time=0; + max_time=0; + rows_affected=0; + rows_sent=0; + hid=h; +} +void QP_query_digest_stats::add_time( + unsigned long long t, unsigned long long n, unsigned long long ra, unsigned long long rs, + unsigned long long cnt +) { + count_star += cnt; + sum_time+=t; + rows_affected+=ra; + rows_sent+=rs; + if (t < min_time || min_time==0) { + if (t) min_time = t; + } + if (t > max_time) { + max_time = t; + } + if (first_seen==0) { + first_seen=n; + } + last_seen=n; +} +QP_query_digest_stats::~QP_query_digest_stats() { + if (digest_text) { + free(digest_text); + digest_text=NULL; + } + if (username) { + if (username == username_buf) { + } else { + free(username); + } + username=NULL; + } + if (schemaname) { + if (schemaname == schemaname_buf) { + } else { + free(schemaname); + } + schemaname=NULL; + } + if (client_address) { + if (client_address == client_address_buf) { + } else { + free(client_address); + } + client_address=NULL; + } +} + +// Funtion to get the digest text associated to a QP_query_digest_stats. +// QP_query_digest_stats member type "char *digest_text" may by NULL, so we +// have to get the digest text from "digest_text_umap". +char *QP_query_digest_stats::get_digest_text(const umap_query_digest_text *digest_text_umap) { + char *digest_text_str = NULL; + + if (digest_text) { + digest_text_str = digest_text; + } else { + std::unordered_map::const_iterator it; + it = digest_text_umap->find(digest); + if (it != digest_text_umap->end()) { + digest_text_str = it->second; + } else { + // LCOV_EXCL_START + assert(0); + // LCOV_EXCL_STOP + } + } + + return digest_text_str; +} + +char **QP_query_digest_stats::get_row(umap_query_digest_text *digest_text_umap, query_digest_stats_pointers_t *qdsp) { + char **pta=qdsp->pta; + + assert(schemaname); + pta[0]=schemaname; + assert(username); + pta[1]=username; + assert(client_address); + pta[2]=client_address; + + assert(qdsp != NULL); + assert(qdsp->digest); + sprintf(qdsp->digest,"0x%016llX", (long long unsigned int)digest); + pta[3]=qdsp->digest; + + pta[4] = get_digest_text(digest_text_umap); + + //sprintf(qdsp->count_star,"%u",count_star); + my_itoa(qdsp->count_star, count_star); + pta[5]=qdsp->count_star; + + time_t __now; + time(&__now); + unsigned long long curtime=monotonic_time(); + time_t seen_time; + seen_time= __now - curtime/1000000 + first_seen/1000000; + //sprintf(qdsp->first_seen,"%ld", seen_time); + my_itoa(qdsp->first_seen, seen_time); + pta[6]=qdsp->first_seen; + + seen_time= __now - curtime/1000000 + last_seen/1000000; + //sprintf(qdsp->last_seen,"%ld", seen_time); + my_itoa(qdsp->last_seen, seen_time); + pta[7]=qdsp->last_seen; + //sprintf(qdsp->sum_time,"%llu",sum_time); + my_itoa(qdsp->sum_time,sum_time); + pta[8]=qdsp->sum_time; + //sprintf(qdsp->min_time,"%llu",min_time); + my_itoa(qdsp->min_time,min_time); + pta[9]=qdsp->min_time; + //sprintf(qdsp->max_time,"%llu",max_time); + my_itoa(qdsp->max_time,max_time); + pta[10]=qdsp->max_time; + // we are reverting this back to the use of sprintf instead of my_itoa + // because with my_itoa we are losing the sign + // see issue #2285 + sprintf(qdsp->hid,"%d",hid); + //my_itoa(qdsp->hid,hid); + pta[11]=qdsp->hid; + //sprintf(qdsp->rows_affected,"%llu",rows_affected); + my_itoa(qdsp->rows_affected,rows_affected); + pta[12]=qdsp->rows_affected; + //sprintf(qdsp->rows_sent,"%llu",rows_sent); + my_itoa(qdsp->rows_sent,rows_sent); + pta[13]=qdsp->rows_sent; + return pta; +} + diff --git a/lib/QP_rule_text.cpp b/lib/QP_rule_text.cpp new file mode 100644 index 0000000000..cd1a340425 --- /dev/null +++ b/lib/QP_rule_text.cpp @@ -0,0 +1,99 @@ +/* +#include // std::cout +#include // std::sort +#include // std::vector +#include "re2/re2.h" +#include "re2/regexp.h" +#include "proxysql.h" +#include "cpp.h" + +#include "MySQL_PreparedStatement.h" +#include "MySQL_Data_Stream.h" +*/ +#include "query_processor.h" +#include +#include "proxysql_macros.h" + +#include "QP_rule_text.h" + + +QP_rule_text_hitsonly::QP_rule_text_hitsonly(QP_rule_t *QPr) { + pta=NULL; + pta=(char **)malloc(sizeof(char *)*2); + itostr(pta[0], (long long)QPr->rule_id); + itostr(pta[1], (long long)QPr->hits); +} + +QP_rule_text_hitsonly::~QP_rule_text_hitsonly() { + for(int i=0; i<2; i++) { + free_null(pta[i]); + } + free(pta); +} + +QP_rule_text::QP_rule_text(QP_rule_t *QPr) { + num_fields=36; // this count the number of fields + pta=NULL; + pta=(char **)malloc(sizeof(char *)*num_fields); + itostr(pta[0], (long long)QPr->rule_id); + itostr(pta[1], (long long)QPr->active); + pta[2]=strdup_null(QPr->username); + pta[3]=strdup_null(QPr->schemaname); + itostr(pta[4], (long long)QPr->flagIN); + + pta[5]=strdup_null(QPr->client_addr); + pta[6]=strdup_null(QPr->proxy_addr); + itostr(pta[7], (long long)QPr->proxy_port); + + char buf[20]; + if (QPr->digest) { + sprintf(buf,"0x%016llX", (long long unsigned int)QPr->digest); + pta[8]=strdup(buf); + } else { + pta[8]=NULL; + } + + pta[9]=strdup_null(QPr->match_digest); + pta[10]=strdup_null(QPr->match_pattern); + itostr(pta[11], (long long)QPr->negate_match_pattern); + std::string re_mod; + re_mod=""; + if ((QPr->re_modifiers & QP_RE_MOD_CASELESS) == QP_RE_MOD_CASELESS) re_mod = "CASELESS"; + if ((QPr->re_modifiers & QP_RE_MOD_GLOBAL) == QP_RE_MOD_GLOBAL) { + if (re_mod.length()) { + re_mod = re_mod + ","; + } + re_mod = re_mod + "GLOBAL"; + } + pta[12]=strdup_null((char *)re_mod.c_str()); // re_modifiers + itostr(pta[13], (long long)QPr->flagOUT); + pta[14]=strdup_null(QPr->replace_pattern); + itostr(pta[15], (long long)QPr->destination_hostgroup); + itostr(pta[16], (long long)QPr->cache_ttl); + itostr(pta[17], (long long)QPr->cache_empty_result); + itostr(pta[18], (long long)QPr->cache_timeout); + itostr(pta[19], (long long)QPr->reconnect); + itostr(pta[20], (long long)QPr->timeout); + itostr(pta[21], (long long)QPr->retries); + itostr(pta[22], (long long)QPr->delay); + itostr(pta[23], (long long)QPr->next_query_flagIN); + itostr(pta[24], (long long)QPr->mirror_flagOUT); + itostr(pta[25], (long long)QPr->mirror_hostgroup); + pta[26]=strdup_null(QPr->error_msg); + pta[27]=strdup_null(QPr->OK_msg); + itostr(pta[28], (long long)QPr->sticky_conn); + itostr(pta[29], (long long)QPr->multiplex); + itostr(pta[30], (long long)QPr->gtid_from_hostgroup); + itostr(pta[31], (long long)QPr->log); + itostr(pta[32], (long long)QPr->apply); + pta[33]=strdup_null(QPr->attributes); + pta[34]=strdup_null(QPr->comment); // issue #643 + itostr(pta[35], (long long)QPr->hits); +} + +QP_rule_text::~QP_rule_text() { + for(int i=0; i #include + +#include "QP_rule_text.h" + extern MySQL_Threads_Handler *GloMTH; extern ProxySQL_Admin *GloAdmin; @@ -41,279 +42,6 @@ static int int_cmp(const void *a, const void *b) { return 0; } -class QP_rule_text_hitsonly { - public: - char **pta; - QP_rule_text_hitsonly(QP_rule_t *QPr) { - pta=NULL; - pta=(char **)malloc(sizeof(char *)*2); - itostr(pta[0], (long long)QPr->rule_id); - itostr(pta[1], (long long)QPr->hits); - } - ~QP_rule_text_hitsonly() { - for(int i=0; i<2; i++) { - free_null(pta[i]); - } - free(pta); - } -}; - -class QP_rule_text { - public: - char **pta; - int num_fields; - QP_rule_text(QP_rule_t *QPr) { - num_fields=36; // this count the number of fields - pta=NULL; - pta=(char **)malloc(sizeof(char *)*num_fields); - itostr(pta[0], (long long)QPr->rule_id); - itostr(pta[1], (long long)QPr->active); - pta[2]=strdup_null(QPr->username); - pta[3]=strdup_null(QPr->schemaname); - itostr(pta[4], (long long)QPr->flagIN); - - pta[5]=strdup_null(QPr->client_addr); - pta[6]=strdup_null(QPr->proxy_addr); - itostr(pta[7], (long long)QPr->proxy_port); - - char buf[20]; - if (QPr->digest) { - sprintf(buf,"0x%016llX", (long long unsigned int)QPr->digest); - pta[8]=strdup(buf); - } else { - pta[8]=NULL; - } - - pta[9]=strdup_null(QPr->match_digest); - pta[10]=strdup_null(QPr->match_pattern); - itostr(pta[11], (long long)QPr->negate_match_pattern); - std::string re_mod; - re_mod=""; - if ((QPr->re_modifiers & QP_RE_MOD_CASELESS) == QP_RE_MOD_CASELESS) re_mod = "CASELESS"; - if ((QPr->re_modifiers & QP_RE_MOD_GLOBAL) == QP_RE_MOD_GLOBAL) { - if (re_mod.length()) { - re_mod = re_mod + ","; - } - re_mod = re_mod + "GLOBAL"; - } - pta[12]=strdup_null((char *)re_mod.c_str()); // re_modifiers - itostr(pta[13], (long long)QPr->flagOUT); - pta[14]=strdup_null(QPr->replace_pattern); - itostr(pta[15], (long long)QPr->destination_hostgroup); - itostr(pta[16], (long long)QPr->cache_ttl); - itostr(pta[17], (long long)QPr->cache_empty_result); - itostr(pta[18], (long long)QPr->cache_timeout); - itostr(pta[19], (long long)QPr->reconnect); - itostr(pta[20], (long long)QPr->timeout); - itostr(pta[21], (long long)QPr->retries); - itostr(pta[22], (long long)QPr->delay); - itostr(pta[23], (long long)QPr->next_query_flagIN); - itostr(pta[24], (long long)QPr->mirror_flagOUT); - itostr(pta[25], (long long)QPr->mirror_hostgroup); - pta[26]=strdup_null(QPr->error_msg); - pta[27]=strdup_null(QPr->OK_msg); - itostr(pta[28], (long long)QPr->sticky_conn); - itostr(pta[29], (long long)QPr->multiplex); - itostr(pta[30], (long long)QPr->gtid_from_hostgroup); - itostr(pta[31], (long long)QPr->log); - itostr(pta[32], (long long)QPr->apply); - pta[33]=strdup_null(QPr->attributes); - pta[34]=strdup_null(QPr->comment); // issue #643 - itostr(pta[35], (long long)QPr->hits); - } - ~QP_rule_text() { - for(int i=0; i 0); /* delete it */ - s[i] = '\0'; - reverse(s); -} - -QP_query_digest_stats::QP_query_digest_stats(char *u, char *s, uint64_t d, char *dt, int h, char *ca) { - digest=d; - digest_text=NULL; - if (dt) { - digest_text=strndup(dt, mysql_thread___query_digests_max_digest_length); - } - if (strlen(u) < sizeof(username_buf)) { - strcpy(username_buf,u); - username = username_buf; - } else { - username=strdup(u); - } - if (strlen(s) < sizeof(schemaname_buf)) { - strcpy(schemaname_buf,s); - schemaname = schemaname_buf; - } else { - schemaname=strdup(s); - } - if (strlen(ca) < sizeof(client_address_buf)) { - strcpy(client_address_buf,ca); - client_address = client_address_buf; - } else { - client_address=strdup(ca); - } - count_star=0; - first_seen=0; - last_seen=0; - sum_time=0; - min_time=0; - max_time=0; - rows_affected=0; - rows_sent=0; - hid=h; -} -void QP_query_digest_stats::add_time( - unsigned long long t, unsigned long long n, unsigned long long ra, unsigned long long rs, - unsigned long long cnt -) { - count_star += cnt; - sum_time+=t; - rows_affected+=ra; - rows_sent+=rs; - if (t < min_time || min_time==0) { - if (t) min_time = t; - } - if (t > max_time) { - max_time = t; - } - if (first_seen==0) { - first_seen=n; - } - last_seen=n; -} -QP_query_digest_stats::~QP_query_digest_stats() { - if (digest_text) { - free(digest_text); - digest_text=NULL; - } - if (username) { - if (username == username_buf) { - } else { - free(username); - } - username=NULL; - } - if (schemaname) { - if (schemaname == schemaname_buf) { - } else { - free(schemaname); - } - schemaname=NULL; - } - if (client_address) { - if (client_address == client_address_buf) { - } else { - free(client_address); - } - client_address=NULL; - } -} - -// Funtion to get the digest text associated to a QP_query_digest_stats. -// QP_query_digest_stats member type "char *digest_text" may by NULL, so we -// have to get the digest text from "digest_text_umap". -char *QP_query_digest_stats::get_digest_text(const umap_query_digest_text *digest_text_umap) { - char *digest_text_str = NULL; - - if (digest_text) { - digest_text_str = digest_text; - } else { - std::unordered_map::const_iterator it; - it = digest_text_umap->find(digest); - if (it != digest_text_umap->end()) { - digest_text_str = it->second; - } else { - // LCOV_EXCL_START - assert(0); - // LCOV_EXCL_STOP - } - } - - return digest_text_str; -} - -char **QP_query_digest_stats::get_row(umap_query_digest_text *digest_text_umap, query_digest_stats_pointers_t *qdsp) { - char **pta=qdsp->pta; - - assert(schemaname); - pta[0]=schemaname; - assert(username); - pta[1]=username; - assert(client_address); - pta[2]=client_address; - - assert(qdsp != NULL); - assert(qdsp->digest); - sprintf(qdsp->digest,"0x%016llX", (long long unsigned int)digest); - pta[3]=qdsp->digest; - - pta[4] = get_digest_text(digest_text_umap); - - //sprintf(qdsp->count_star,"%u",count_star); - my_itoa(qdsp->count_star, count_star); - pta[5]=qdsp->count_star; - - time_t __now; - time(&__now); - unsigned long long curtime=monotonic_time(); - time_t seen_time; - seen_time= __now - curtime/1000000 + first_seen/1000000; - //sprintf(qdsp->first_seen,"%ld", seen_time); - my_itoa(qdsp->first_seen, seen_time); - pta[6]=qdsp->first_seen; - - seen_time= __now - curtime/1000000 + last_seen/1000000; - //sprintf(qdsp->last_seen,"%ld", seen_time); - my_itoa(qdsp->last_seen, seen_time); - pta[7]=qdsp->last_seen; - //sprintf(qdsp->sum_time,"%llu",sum_time); - my_itoa(qdsp->sum_time,sum_time); - pta[8]=qdsp->sum_time; - //sprintf(qdsp->min_time,"%llu",min_time); - my_itoa(qdsp->min_time,min_time); - pta[9]=qdsp->min_time; - //sprintf(qdsp->max_time,"%llu",max_time); - my_itoa(qdsp->max_time,max_time); - pta[10]=qdsp->max_time; - // we are reverting this back to the use of sprintf instead of my_itoa - // because with my_itoa we are losing the sign - // see issue #2285 - sprintf(qdsp->hid,"%d",hid); - //my_itoa(qdsp->hid,hid); - pta[11]=qdsp->hid; - //sprintf(qdsp->rows_affected,"%llu",rows_affected); - my_itoa(qdsp->rows_affected,rows_affected); - pta[12]=qdsp->rows_affected; - //sprintf(qdsp->rows_sent,"%llu",rows_sent); - my_itoa(qdsp->rows_sent,rows_sent); - pta[13]=qdsp->rows_sent; - return pta; -} struct __RE2_objects_t { pcrecpp::RE_Options *opt1; diff --git a/lib/c_tokenizer.cpp b/lib/c_tokenizer.cpp index c468e0b185..5b35ef0fb3 100644 --- a/lib/c_tokenizer.cpp +++ b/lib/c_tokenizer.cpp @@ -1,6 +1,3 @@ -/* c_tokenizer.c */ -// Borrowed from http://www.cplusplus.com/faq/sequences/strings/split/ - #include #include #include @@ -234,591 +231,6 @@ static inline void replace_with_q_mark( } } -char *mysql_query_digest_and_first_comment(char *s, int _len, char **first_comment, char *buf){ - int i = 0; - - char cur_comment[FIRST_COMMENT_MAX_LENGTH]; - cur_comment[0]=0; - int ccl=0; - int cmd=0; - - int len = _len; - if (_len > mysql_thread___query_digests_max_query_length) { - len = mysql_thread___query_digests_max_query_length; - } - char *r = buf; - if (r==NULL) { - r = (char *) malloc(len + SIZECHAR); - } - char *p_r = r; - char *p_r_t = r; - - char prev_char = 0; - char qutr_char = 0; - - char flag = 0; - char fc=0; - int fc_len=0; - - char fns=0; - - bool lowercase=0; - bool replace_null=0; - bool replace_number=0; - - char grouping_digest=0; - char grouping_limit_exceeded=0; - int grouping_count=0; - int grouping_lim = mysql_thread___query_digests_grouping_limit; - - lowercase=mysql_thread___query_digests_lowercase; - replace_null = mysql_thread___query_digests_replace_null; - replace_number = mysql_thread___query_digests_no_digits; - - while(i < len) - { - // Handy for debugging purposes - // ============================ - // printf( - // "state-1: { flag: `%d`, prev_char: `%c`, s: `%s`, p_r: `%s`, r: `%s`}\n", - // flag, prev_char, s, p_r, r - // ); - // ============================ - - // ================================================= - // START - read token char and set flag what's going on. - // ================================================= - if(flag == 0) - { - // store current position - p_r_t = p_r; - - // comment type 1 - start with '/*' - if(prev_char == '/' && *s == '*') - { - ccl=0; - flag = 1; - if (i != (len-1) && *(s+1)=='!') - cmd=1; - } - - // comment type 2 - start with '#' - else if(*s == '#') - { - flag = 2; - } - - // comment type 3 - start with '--' - - // NOTE: Looks like the general rule for parsing comments of this type could simply be: - // - // - `.*--.*` which could be translated into `(*s == '-' && *(s+1) == '-')`. - // - // But this can not hold, since the first '-' could have been consumed previously, for example - // during the parsing of a digit: - // - // - `select 1.1-- final_comment\n` - // - // For this reason 'prev_char' needs to be checked too when searching for the `--` pattern. - else if(i != (len-1) && prev_char == '-' && *s == '-' && ((*(s+1)==' ') || (*(s+1)=='\n') || (*(s+1)=='\r') || (*(s+1)=='\t') )) - { - flag = 3; - } - - // Previous character can be a consumed ' ' instead of '-' as in the previous case, for this - // reason, we need to look ahead for '--'. - // - // NOTE: There is no reason for not checking for the subsequent space char that should follow - // the '-- ', otherwise we would consider valid queries as `SELECT --1` like comments. - else if (i != (len-1) && *s == '-' && (*(s+1)=='-')) { - if (prev_char != '-') { - flag = 3; - } - else if (i==0) { - flag = 3; - } - } - - // string - start with ' - else if(*s == '\'' || *s == '"') - { - flag = 4; - qutr_char = *s; - } - - // may be digit - start with digit - else if(is_token_char(prev_char) && is_digit_char(*s)) - { - flag = 5; - if(len == i+1) - continue; - } - - // not above case - remove duplicated space char - else - { - flag = 0; - if (fns==0 && is_space_char(*s)) { - s++; - i++; - continue; - } - if (fns==0) fns=1; - if(is_space_char(prev_char) && is_space_char(*s)){ - prev_char = ' '; - *p_r = ' '; - s++; - i++; - continue; - } - if (replace_number) { - if (!is_digit_char(prev_char) && is_digit_char(*s)) { - *p_r++ = '?'; - while(*s != '\0' && is_digit_char(*s)) { - s++; - i++; - } - } - } - { - char* p = p_r - 2; - // suppress spaces before arithmetic operators - if (p >= r && is_space_char(prev_char) && is_arithmetic_op(*s)) { - if (*p == '?') { - prev_char = *s; - --p_r; - *p_r++ = *s; - s++; - i++; - continue; - } - } - // suppress spaces before and after commas - if (p >= r && is_space_char(prev_char) && ((*s == ',') || (*p == ','))) { - if (*s == ',') { - --p_r; - // only copy the comma if we are not grouping a query - if (!grouping_limit_exceeded) { - *p_r++ = *s; - } - prev_char = ','; - s++; - i++; - } else { - prev_char = ','; - --p_r; - } - continue; - } - // suppress spaces before closing brackets when grouping or mark is present - if (p >= r && (*p == '.' || *p == '?') && is_space_char(prev_char) && (*s == ')')) { - prev_char = *s; - --p_r; - *p_r++ = *s; - s++; - i++; - continue; - } - } - if (replace_null) { - if (*s == 'n' || *s == 'N') { // we search for NULL , #2171 - if (i && is_token_char(prev_char)) { - if (len>=4) { - if (i=2) fc_len-=2; - char *c=*first_comment+fc_len; - *c=0; - //*first_comment[fc_len]=0; - fc=2; - } - } - } - if( - // comment type 1 - /* .. */ - (flag == 1 && prev_char == '*' && *s == '/') || - - // comment type 2 - # ... \n - (flag == 2 && (*s == '\n' || *s == '\r' || (i == len - 1) )) - || - // comment type 3 - -- ... \n - (flag == 3 && (*s == '\n' || *s == '\r' || (i == len -1) )) - ) - { - p_r = p_r_t; - if (flag == 1 || (i == len -1)) { - p_r -= SIZECHAR; - } - if (cmd) { - cur_comment[ccl]=0; - if (ccl>=2) { - ccl-=2; - cur_comment[ccl]=0; - char el=0; - int fcc=0; - while (el==0 && fcc= r && ( *(_p+2) == '-' || *(_p+2) == '+') ) { - if ( - ( *(_p+1) == ',' ) || ( *(_p+1) == '(' ) || - ( ( *(_p+1) == ' ' ) && ( *_p == ',' || *_p == '(' ) ) - ) { - p_r--; - } - } - - replace_with_q_mark( - grouping_digest, grouping_lim, &grouping_count, &p_r, &grouping_limit_exceeded - ); - - flag = 0; - break; - } - - // need to be ignored case - if(p_r > p_r_t + SIZECHAR) - { - if( - (prev_char == '\\' && *s == '\\') || // to process '\\\\', '\\' - (prev_char == '\\' && *s == qutr_char) || // to process '\'' - (prev_char == qutr_char && *s == qutr_char) // to process '''' - ) - { - prev_char = 'X'; - s++; - i++; - continue; - } - } - - // satisfied closing string - swap string to ? - if(*s == qutr_char && (len == i+1 || *(s + SIZECHAR) != qutr_char)) - { - char *_p = p_r_t; - _p-=3; - p_r = p_r_t; - if ( _p >= r && ( *(_p+2) == '-' || *(_p+2) == '+') ) { - if ( - ( *(_p+1) == ',' ) || ( *(_p+1) == '(' ) || - ( ( *(_p+1) == ' ' ) && ( *_p == ',' || *_p == '(' ) ) - ) { - p_r--; - } - } - - // Remove spaces before each literal found - if ( _p >= r && is_space_char(*(_p + 2)) && !is_normal_char(*(_p + 1))) { - if ( _p >= r && ( *(_p+3) == '\''|| *(_p+3) == '"' )) { - p_r--; - } - } - - replace_with_q_mark( - grouping_digest, grouping_lim, &grouping_count, &p_r, &grouping_limit_exceeded - ); - - prev_char = qutr_char; - qutr_char = 0; - flag = 0; - if(i < len) - s++; - i++; - continue; - } - } - - // -------- - // digit - // -------- - else if(flag == 5) - { - // last single char - if(p_r_t == p_r) - { - char *_p = p_r_t; - _p-=3; - if ( _p >= r && ( *(_p+2) == '-' || *(_p+2) == '+') ) { - if ( - ( *(_p+1) == ',' ) || ( *(_p+1) == '(' ) || - ( ( *(_p+1) == ' ' ) && ( *_p == ',' || *_p == '(' ) ) - ) { - p_r--; - } - } - if ( _p >= r && is_space_char(*(_p + 2))) { - if ( _p >= r && ( *(_p+1) == '-' || *(_p+1) == '+' || *(_p+1) == '*' || *(_p+1) == '/' || *(_p+1) == '%' || *(_p+1) == ',')) { - p_r--; - } - } - *p_r++ = '?'; - i++; - continue; - } - - // is float - if (*s == '.' || *s == 'e' || ((*s == '+' || *s == '-') && prev_char == 'e')) { - prev_char = *s; - i++; - s++; - continue; - } - - // token char or last char - if(is_token_char(*s) || len == i+1) - { - if(is_digit_string(p_r_t, p_r)) - { - char *_p = p_r_t; - _p-=3; - p_r = p_r_t; - // remove symbol and keep parenthesis or comma - if ( _p >= r && ( *(_p+2) == '-' || *(_p+2) == '+') ) { - if ( - ( *(_p+1) == ',' ) || ( *(_p+1) == '(' ) || - ( ( *(_p+1) == ' ' ) && ( *_p == ',' || *_p == '(' ) ) - ) { - p_r--; - } - } - - // Remove spaces before number counting with possible '.' presence - if (_p >= r && *_p == '.' && (*(_p + 1) == ' ' || *(_p + 1) == '.') && (*(_p+2) == '-' || *(_p+2) == '+') ) { - if (*(_p + 1) == ' ') { - p_r--; - } - p_r--; - } - - // Remove spaces after a opening bracket when followed by a number - if (_p >= r && *(_p+1) == '(' && *(_p+2) == ' ') { - p_r--; - } - - // Remove spaces before number - if ( _p >= r && is_space_char(*(_p + 2))) { - // A point can be found prior to a number in case of query grouping - if ( _p >= r && ( *(_p+1) == '-' || *(_p+1) == '+' || *(_p+1) == '*' || *(_p+1) == '/' || *(_p+1) == '%' || *(_p+1) == ',' || *(_p+1) == '.')) { - p_r--; - } - } - - replace_with_q_mark( - grouping_digest, grouping_lim, &grouping_count, &p_r, &grouping_limit_exceeded - ); - - if(len == i+1) - { - if(is_token_char(*s)) - *p_r++ = *s; - i++; - continue; - } - } else { - // collapse any digits found in the string - if (replace_number) { - int str_len = p_r - p_r_t + 1; - int collapsed = 0; - - for (int j = 0; j < str_len; j++) { - char* const c_p_r_t = ((char*)p_r_t + j); - char* const n_p_r_t = ((char*)p_r_t + j + 1); - - if (is_digit_char(*c_p_r_t) && is_digit_char(*n_p_r_t)) { - memmove(c_p_r_t, c_p_r_t + 1, str_len - j); - collapsed += 1; - } - } - - p_r -= collapsed; - - int new_str_len = p_r - p_r_t + 1; - for (int j = 0; j < new_str_len; j++) { - char* const c_p_r_t = ((char*)p_r_t + j); - if (is_digit_char(*c_p_r_t)) { - *c_p_r_t = '?'; - } - } - } - } - - flag = 0; - } - } - } - - // ================================================= - // COPY CHAR - // ================================================= - // convert every space char to ' ' - if (*s == ')') { - if (grouping_digest > 0) { - grouping_digest -= 1; - }; - grouping_count = 0; - grouping_limit_exceeded = 0; - } - - if (lowercase==0) { - *p_r++ = !is_space_char(*s) ? *s : ' '; - } else { - *p_r++ = !is_space_char(*s) ? (tolower(*s)) : ' '; - } - - if (*s == '(') { - grouping_digest += 1; - grouping_count = 0; - grouping_limit_exceeded = 0; - } - - prev_char = *s++; - - i++; - } - - // remove a trailing space - if (p_r>r) { - char *e=p_r; - e--; - if (*e==' ') { - *e=0; - // maybe 2 trailing spaces . It happens with comments - e--; - if (*e==' ') { - *e=0; - } - } - } - - *p_r = 0; - - // process query stats - return r; -} - /** * @brief Struct for holding all the configuration options used for query digests generation. */ diff --git a/lib/c_tokenizer_legacy.cpp b/lib/c_tokenizer_legacy.cpp new file mode 100644 index 0000000000..ae93027afe --- /dev/null +++ b/lib/c_tokenizer_legacy.cpp @@ -0,0 +1,588 @@ +/* + this file is here only for reference. + It includes the old mysql_query_digest_and_first_comment() , outdated since ProxySQL 2.4.0 +*/ +char *mysql_query_digest_and_first_comment(char *s, int _len, char **first_comment, char *buf){ + int i = 0; + + char cur_comment[FIRST_COMMENT_MAX_LENGTH]; + cur_comment[0]=0; + int ccl=0; + int cmd=0; + + int len = _len; + if (_len > mysql_thread___query_digests_max_query_length) { + len = mysql_thread___query_digests_max_query_length; + } + char *r = buf; + if (r==NULL) { + r = (char *) malloc(len + SIZECHAR); + } + char *p_r = r; + char *p_r_t = r; + + char prev_char = 0; + char qutr_char = 0; + + char flag = 0; + char fc=0; + int fc_len=0; + + char fns=0; + + bool lowercase=0; + bool replace_null=0; + bool replace_number=0; + + char grouping_digest=0; + char grouping_limit_exceeded=0; + int grouping_count=0; + int grouping_lim = mysql_thread___query_digests_grouping_limit; + + lowercase=mysql_thread___query_digests_lowercase; + replace_null = mysql_thread___query_digests_replace_null; + replace_number = mysql_thread___query_digests_no_digits; + + while(i < len) + { + // Handy for debugging purposes + // ============================ + // printf( + // "state-1: { flag: `%d`, prev_char: `%c`, s: `%s`, p_r: `%s`, r: `%s`}\n", + // flag, prev_char, s, p_r, r + // ); + // ============================ + + // ================================================= + // START - read token char and set flag what's going on. + // ================================================= + if(flag == 0) + { + // store current position + p_r_t = p_r; + + // comment type 1 - start with '/*' + if(prev_char == '/' && *s == '*') + { + ccl=0; + flag = 1; + if (i != (len-1) && *(s+1)=='!') + cmd=1; + } + + // comment type 2 - start with '#' + else if(*s == '#') + { + flag = 2; + } + + // comment type 3 - start with '--' + + // NOTE: Looks like the general rule for parsing comments of this type could simply be: + // + // - `.*--.*` which could be translated into `(*s == '-' && *(s+1) == '-')`. + // + // But this can not hold, since the first '-' could have been consumed previously, for example + // during the parsing of a digit: + // + // - `select 1.1-- final_comment\n` + // + // For this reason 'prev_char' needs to be checked too when searching for the `--` pattern. + else if(i != (len-1) && prev_char == '-' && *s == '-' && ((*(s+1)==' ') || (*(s+1)=='\n') || (*(s+1)=='\r') || (*(s+1)=='\t') )) + { + flag = 3; + } + + // Previous character can be a consumed ' ' instead of '-' as in the previous case, for this + // reason, we need to look ahead for '--'. + // + // NOTE: There is no reason for not checking for the subsequent space char that should follow + // the '-- ', otherwise we would consider valid queries as `SELECT --1` like comments. + else if (i != (len-1) && *s == '-' && (*(s+1)=='-')) { + if (prev_char != '-') { + flag = 3; + } + else if (i==0) { + flag = 3; + } + } + + // string - start with ' + else if(*s == '\'' || *s == '"') + { + flag = 4; + qutr_char = *s; + } + + // may be digit - start with digit + else if(is_token_char(prev_char) && is_digit_char(*s)) + { + flag = 5; + if(len == i+1) + continue; + } + + // not above case - remove duplicated space char + else + { + flag = 0; + if (fns==0 && is_space_char(*s)) { + s++; + i++; + continue; + } + if (fns==0) fns=1; + if(is_space_char(prev_char) && is_space_char(*s)){ + prev_char = ' '; + *p_r = ' '; + s++; + i++; + continue; + } + if (replace_number) { + if (!is_digit_char(prev_char) && is_digit_char(*s)) { + *p_r++ = '?'; + while(*s != '\0' && is_digit_char(*s)) { + s++; + i++; + } + } + } + { + char* p = p_r - 2; + // suppress spaces before arithmetic operators + if (p >= r && is_space_char(prev_char) && is_arithmetic_op(*s)) { + if (*p == '?') { + prev_char = *s; + --p_r; + *p_r++ = *s; + s++; + i++; + continue; + } + } + // suppress spaces before and after commas + if (p >= r && is_space_char(prev_char) && ((*s == ',') || (*p == ','))) { + if (*s == ',') { + --p_r; + // only copy the comma if we are not grouping a query + if (!grouping_limit_exceeded) { + *p_r++ = *s; + } + prev_char = ','; + s++; + i++; + } else { + prev_char = ','; + --p_r; + } + continue; + } + // suppress spaces before closing brackets when grouping or mark is present + if (p >= r && (*p == '.' || *p == '?') && is_space_char(prev_char) && (*s == ')')) { + prev_char = *s; + --p_r; + *p_r++ = *s; + s++; + i++; + continue; + } + } + if (replace_null) { + if (*s == 'n' || *s == 'N') { // we search for NULL , #2171 + if (i && is_token_char(prev_char)) { + if (len>=4) { + if (i=2) fc_len-=2; + char *c=*first_comment+fc_len; + *c=0; + //*first_comment[fc_len]=0; + fc=2; + } + } + } + if( + // comment type 1 - /* .. */ + (flag == 1 && prev_char == '*' && *s == '/') || + + // comment type 2 - # ... \n + (flag == 2 && (*s == '\n' || *s == '\r' || (i == len - 1) )) + || + // comment type 3 - -- ... \n + (flag == 3 && (*s == '\n' || *s == '\r' || (i == len -1) )) + ) + { + p_r = p_r_t; + if (flag == 1 || (i == len -1)) { + p_r -= SIZECHAR; + } + if (cmd) { + cur_comment[ccl]=0; + if (ccl>=2) { + ccl-=2; + cur_comment[ccl]=0; + char el=0; + int fcc=0; + while (el==0 && fcc= r && ( *(_p+2) == '-' || *(_p+2) == '+') ) { + if ( + ( *(_p+1) == ',' ) || ( *(_p+1) == '(' ) || + ( ( *(_p+1) == ' ' ) && ( *_p == ',' || *_p == '(' ) ) + ) { + p_r--; + } + } + + replace_with_q_mark( + grouping_digest, grouping_lim, &grouping_count, &p_r, &grouping_limit_exceeded + ); + + flag = 0; + break; + } + + // need to be ignored case + if(p_r > p_r_t + SIZECHAR) + { + if( + (prev_char == '\\' && *s == '\\') || // to process '\\\\', '\\' + (prev_char == '\\' && *s == qutr_char) || // to process '\'' + (prev_char == qutr_char && *s == qutr_char) // to process '''' + ) + { + prev_char = 'X'; + s++; + i++; + continue; + } + } + + // satisfied closing string - swap string to ? + if(*s == qutr_char && (len == i+1 || *(s + SIZECHAR) != qutr_char)) + { + char *_p = p_r_t; + _p-=3; + p_r = p_r_t; + if ( _p >= r && ( *(_p+2) == '-' || *(_p+2) == '+') ) { + if ( + ( *(_p+1) == ',' ) || ( *(_p+1) == '(' ) || + ( ( *(_p+1) == ' ' ) && ( *_p == ',' || *_p == '(' ) ) + ) { + p_r--; + } + } + + // Remove spaces before each literal found + if ( _p >= r && is_space_char(*(_p + 2)) && !is_normal_char(*(_p + 1))) { + if ( _p >= r && ( *(_p+3) == '\''|| *(_p+3) == '"' )) { + p_r--; + } + } + + replace_with_q_mark( + grouping_digest, grouping_lim, &grouping_count, &p_r, &grouping_limit_exceeded + ); + + prev_char = qutr_char; + qutr_char = 0; + flag = 0; + if(i < len) + s++; + i++; + continue; + } + } + + // -------- + // digit + // -------- + else if(flag == 5) + { + // last single char + if(p_r_t == p_r) + { + char *_p = p_r_t; + _p-=3; + if ( _p >= r && ( *(_p+2) == '-' || *(_p+2) == '+') ) { + if ( + ( *(_p+1) == ',' ) || ( *(_p+1) == '(' ) || + ( ( *(_p+1) == ' ' ) && ( *_p == ',' || *_p == '(' ) ) + ) { + p_r--; + } + } + if ( _p >= r && is_space_char(*(_p + 2))) { + if ( _p >= r && ( *(_p+1) == '-' || *(_p+1) == '+' || *(_p+1) == '*' || *(_p+1) == '/' || *(_p+1) == '%' || *(_p+1) == ',')) { + p_r--; + } + } + *p_r++ = '?'; + i++; + continue; + } + + // is float + if (*s == '.' || *s == 'e' || ((*s == '+' || *s == '-') && prev_char == 'e')) { + prev_char = *s; + i++; + s++; + continue; + } + + // token char or last char + if(is_token_char(*s) || len == i+1) + { + if(is_digit_string(p_r_t, p_r)) + { + char *_p = p_r_t; + _p-=3; + p_r = p_r_t; + // remove symbol and keep parenthesis or comma + if ( _p >= r && ( *(_p+2) == '-' || *(_p+2) == '+') ) { + if ( + ( *(_p+1) == ',' ) || ( *(_p+1) == '(' ) || + ( ( *(_p+1) == ' ' ) && ( *_p == ',' || *_p == '(' ) ) + ) { + p_r--; + } + } + + // Remove spaces before number counting with possible '.' presence + if (_p >= r && *_p == '.' && (*(_p + 1) == ' ' || *(_p + 1) == '.') && (*(_p+2) == '-' || *(_p+2) == '+') ) { + if (*(_p + 1) == ' ') { + p_r--; + } + p_r--; + } + + // Remove spaces after a opening bracket when followed by a number + if (_p >= r && *(_p+1) == '(' && *(_p+2) == ' ') { + p_r--; + } + + // Remove spaces before number + if ( _p >= r && is_space_char(*(_p + 2))) { + // A point can be found prior to a number in case of query grouping + if ( _p >= r && ( *(_p+1) == '-' || *(_p+1) == '+' || *(_p+1) == '*' || *(_p+1) == '/' || *(_p+1) == '%' || *(_p+1) == ',' || *(_p+1) == '.')) { + p_r--; + } + } + + replace_with_q_mark( + grouping_digest, grouping_lim, &grouping_count, &p_r, &grouping_limit_exceeded + ); + + if(len == i+1) + { + if(is_token_char(*s)) + *p_r++ = *s; + i++; + continue; + } + } else { + // collapse any digits found in the string + if (replace_number) { + int str_len = p_r - p_r_t + 1; + int collapsed = 0; + + for (int j = 0; j < str_len; j++) { + char* const c_p_r_t = ((char*)p_r_t + j); + char* const n_p_r_t = ((char*)p_r_t + j + 1); + + if (is_digit_char(*c_p_r_t) && is_digit_char(*n_p_r_t)) { + memmove(c_p_r_t, c_p_r_t + 1, str_len - j); + collapsed += 1; + } + } + + p_r -= collapsed; + + int new_str_len = p_r - p_r_t + 1; + for (int j = 0; j < new_str_len; j++) { + char* const c_p_r_t = ((char*)p_r_t + j); + if (is_digit_char(*c_p_r_t)) { + *c_p_r_t = '?'; + } + } + } + } + + flag = 0; + } + } + } + + // ================================================= + // COPY CHAR + // ================================================= + // convert every space char to ' ' + if (*s == ')') { + if (grouping_digest > 0) { + grouping_digest -= 1; + }; + grouping_count = 0; + grouping_limit_exceeded = 0; + } + + if (lowercase==0) { + *p_r++ = !is_space_char(*s) ? *s : ' '; + } else { + *p_r++ = !is_space_char(*s) ? (tolower(*s)) : ' '; + } + + if (*s == '(') { + grouping_digest += 1; + grouping_count = 0; + grouping_limit_exceeded = 0; + } + + prev_char = *s++; + + i++; + } + + // remove a trailing space + if (p_r>r) { + char *e=p_r; + e--; + if (*e==' ') { + *e=0; + // maybe 2 trailing spaces . It happens with comments + e--; + if (*e==' ') { + *e=0; + } + } + } + + *p_r = 0; + + // process query stats + return r; +} diff --git a/test/tap/tap/tap.cpp b/test/tap/tap/tap.cpp index 8afb7939cd..b067432a2d 100644 --- a/test/tap/tap/tap.cpp +++ b/test/tap/tap/tap.cpp @@ -415,6 +415,10 @@ int tests_failed() { return g_test.failed; } +int tests_last() { + return g_test.last; +} + /** @mainpage Testing C and C++ using MyTAP diff --git a/test/tap/tap/tap.h b/test/tap/tap/tap.h index 4b7e76354a..7678c1a390 100644 --- a/test/tap/tap/tap.h +++ b/test/tap/tap/tap.h @@ -311,6 +311,12 @@ void todo_end(); int tests_failed(); +/** + * @brief Return the number of tests that have passed. + */ + +int tests_last(); + /** @} */ #ifdef __cplusplus diff --git a/test/tap/tests/test_digest_umap_aux-t.cpp b/test/tap/tests/test_digest_umap_aux-t.cpp index 227f5af08d..22391e05b8 100644 --- a/test/tap/tests/test_digest_umap_aux-t.cpp +++ b/test/tap/tests/test_digest_umap_aux-t.cpp @@ -178,7 +178,8 @@ int main(int argc, char** argv) { return EXIT_FAILURE; } - plan(1 + DUMMY_QUERIES.size() * 5); // always specify the number of tests that are going to be performed + int nplan = (1 + DUMMY_QUERIES.size() * 5); // always specify the number of tests that are going to be performed + plan(nplan); MYSQL *proxy_admin = mysql_init(NULL); if (!mysql_real_connect(proxy_admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) { @@ -314,6 +315,12 @@ int main(int argc, char** argv) { ); } + if (tests_last() == nplan && tests_failed == 0) { + string q = "TRUNCATE TABLE stats.stats_mysql_query_digest"; + diag("Running %s", q.c_str()); + MYSQL_QUERY(proxy_admin, q.c_str()); + } + mysql_close(proxy_admin); return exit_status(); diff --git a/test/tap/tests/test_mysql_query_digests_stages-t.cpp b/test/tap/tests/test_mysql_query_digests_stages-t.cpp index c4c79ddf0d..1533faf02e 100644 --- a/test/tap/tests/test_mysql_query_digests_stages-t.cpp +++ b/test/tap/tests/test_mysql_query_digests_stages-t.cpp @@ -89,11 +89,15 @@ uint64_t benchmark_parsing(const vector& queries, int mode, uint32_t ite char* c_res = NULL; if (mode == 0) { + diag("Invalid test. mysql_query_digest_and_first_comment() was deprecated in 2.4.0"); + exit(EXIT_FAILURE); +/* c_res = mysql_query_digest_and_first_comment( const_cast(query.c_str()), query.length(), &first_comment, ((query.size() < QUERY_DIGEST_BUF) ? buf : NULL) ); +*/ } else if (mode == 1) { c_res = mysql_query_digest_and_first_comment_one_it( diff --git a/test/tap/tests/test_tokenizer-t.cpp b/test/tap/tests/test_tokenizer-t.cpp deleted file mode 100644 index a409a55ddb..0000000000 --- a/test/tap/tests/test_tokenizer-t.cpp +++ /dev/null @@ -1,474 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include "proxysql.h" - -#include "tap.h" -#include "command_line.h" - -#include "utils.h" - -#include - -__thread int mysql_thread___query_digests_max_query_length = 65000; -__thread bool mysql_thread___query_digests_lowercase = false; -__thread bool mysql_thread___query_digests_replace_null = false; -__thread bool mysql_thread___query_digests_no_digits = false; -__thread bool mysql_thread___query_digests_keep_comment = false; -__thread int mysql_thread___query_digests_grouping_limit = 3; -__thread int mysql_thread___query_digests_groups_grouping_limit = 1; - -using std::vector; -using std::pair; -using std::string; -using std::tuple; - -const vector> query_digest_pairs { - // TODO: KnownIssue - 10 - // { - // "select /* COMMENT */ 1", - // "select ?" - // // Actual: "select ?# final_comment" - // }, - { - "select/* COMMENT */ 1", - "select ?" - }, - { - "select/* COMMENT */1", - "select ?" - }, - // initial '#' comments - { - "# random_comment \n select 1.1", - "select ?" - }, - { - "#random_comment \nselect 1.1", - "select ?" - }, - { - "#random_comment\nselect 1.1", - "select ?" - }, - // initial '--' comments - { - "-- random_comment \n select 1.1", - "select ?" - }, - { - "--random_comment \nselect 1.1", - "select ?" - }, - { - "--random_comment\nselect 1.1", - "select ?" - }, - // final/initial '#|--' comments - { - "# random_comment\n select 1.1 #final_comment\n ", - "select ?" - }, - // TODO: KnownIssue - 1 - // { - // "# random_comment\n select 1.1# final_comment \n", - // "select ?" - // // Actual: "select ?# final_comment" - // }, - { - "# random_comment\n select 1.1 #final_comment \n ", - "select ?" - }, - { - "-- random_comment\n select 1.1 --final_comment\n ", - "select ?" - }, - { - "-- random_comment\n select 1.1-- final_comment \n", - "select ?" - }, - // NOTE: Comments with '--' should always be followed by an space. - // { - // "-- random_comment\n select 1.1--final_comment \n", - // "select ?" - // }, - { - "-- random_comment\n select 1.1 --final_comment \n ", - "select ?" - }, - // floats - { "select 1.1", "select ?" }, - { "select 99.1929", "select ?" }, - // exponentials - { "select 1.1e9", "select ?" }, - { "select 1.1e+9", "select ?" }, - { "select 1.1e-9", "select ?" }, - // TODO: KnownIssue - 2: Exponentials are case sensitive - // { "select 1.1E9", "select ?" }, - // { "select 1.1E+9", "select ?" }, - // { "select 1.1E-9", "select ?" }, - // operators - { "select 1 +1", "select ?+?" }, - { "select 1+ 1", "select ?+?" }, - { "select 1- 1", "select ?-?" }, - { "select 1 -1", "select ?-?" }, - { "select 1* 1", "select ?*?" }, - { "select 1 *1", "select ?*?" }, - { "select 1/ 1", "select ?/?" }, - { "select 1 /1", "select ?/?" }, - { "select 1% 1", "select ?%?" }, - { "select 1 %1", "select ?%?" }, - // operators and commas - { - "select 1+ 1, 1 -1, 1 * 1 , 1/1 , 100 % 3", - "select ?+?,?-?,?*?,?/?,?%?", - }, - { - "SELECT * FROM t t1, t t2 ,t t3,t t4 LIMIT 0", - "SELECT * FROM t t1,t t2,t t3,t t4 LIMIT ?" - }, - // mixing operators, commas and literals - { - "select 1+ 1,'1 -1', 1 * 1 , '1/1 ',100 % 3", - "select ?+?,?,?*?,?,?%?" - }, - { - "select 1+ 1 ,'1 -1' ,1 * 1 , '1 ' , 100 % 3", - "select ?+?,?,?*?,?,?%?" - }, - { - "select 1 + 1 , '1 - 1' , 1 * 1 , '1 ' , 100 % 3 ", - "select ?+?,?,?*?,?,?%?" - }, - // TODO: KnownIssue - 8: Operators not removed when extra space precedes the value - { - "select + 1", - "select +?" - }, - // strings - simple - { - "select * from t where t = \"foo\"", - "select * from t where t = ?", - }, - { - "select \"1+ 1, 1 -1, 1 * 1 , 1/1 , 100 % 3\"", - "select ?", - }, - // string - preceded by signs - outside parenthesis, not preceded by commas - { "select -\"1\"", "select -?", }, - { "select +\"1\",'foo'", "select +?,?", }, - // string - preceded by signs - inside parenthesis, or preceded by commas - { "select (-'89')", "select (?)", }, - { "select 10,-'89'", "select ?,?", }, - { "select 10, -'89' ", "select ?,?", }, - // string - leading strings get it's extra spaces removed, but not firsts - { "select '10', -'89' ", "select ?,?", }, - { "select 10, -'89 ',+'5'", "select ?,?,?", }, - // TODO: KnownIssue - 7: Spaces not removed after parenthesis when literal strings are preceded by '+|-' - { "select CONCAT( -'89'+'10')", "select CONCAT( ?+?)", }, - // ^ preserved space - { "select CONCAT( -'89'+'10')", "select CONCAT( ?+?)", }, - { "select CONCAT( -'89' + '10' )", "select CONCAT( ?+?)", }, - // TODO: KnownIssue - 8: Operators not removed when extra space precedes the literal (value) - { "select CONCAT(- '89')", "select CONCAT(-?)", }, - // ^ preserved operator - - // not modified - { "select * fromt t", "select * fromt t" }, - // test query_digest reduction - { - "SELECT * FROM tablename WHERE id IN (1,2,3,4,5,6,7,8,9,10)", - "SELECT * FROM tablename WHERE id IN (?,?,?,...)" - }, - { - "SELECT * FROM tablename WHERE id IN (1,2,3,4)", - "SELECT * FROM tablename WHERE id IN (?,?,?,...)" - }, - // invalid request grouping - { - "SELECT * tablename where id IN (1,2,3,4,5,6,7,8, AND j in (1,2,3,4,5,6 and k=1", - "SELECT * tablename where id IN (?,?,?,...,AND j in (?,?,?,... and k=?" - }, - // random insert queries - { - "INSERT INTO db.table(col1) VALUES ('val')", - "INSERT INTO db.table(col1) VALUES (?)" - }, - { - "INSERT INTO db.table (col1) VALUES ('val')", - "INSERT INTO db.table (col1) VALUES (?)" - }, - { - "INSERT INTO db.table( col1) VALUES ( 'val' )", - "INSERT INTO db.table( col1) VALUES (?)" - }, - { - "INSERT INTO db.table( col1) VALUES ( 'val' )", - "INSERT INTO db.table( col1) VALUES (?)" - }, - // TODO: KnownIssue - 6: When 'no-digits' is enabled, space before parenthesis closing brakcet is - // collapsed when an identifier name finish with a number. - // { - // "INSERT INTO db.table ( col1 ) VALUES ( 'val' )", - // "INSERT INTO db.table ( col1 ) VALUES (?)" - // }, - { - "INSERT INTO db.table (col1, col2,col3,col4) VALUES ('val',2,3,'foo')", - "INSERT INTO db.table (col1,col2,col3,col4) VALUES (?,?,?,...)" - }, - // TODO: KnownIssue - 6: When 'no-digits' is enabled, space before parenthesis closing brakcet is - // collapsed when an identifier name finish with a number. - // { - // "INSERT INTO db.table ( col1, col2,col3,col4 ) VALUES ('val',2,3,'foo')", - // "INSERT INTO db.table ( col1,col2,col3,col4 ) VALUES (?,?,?,...)" - // }, - { - "INSERT INTO db.table_25 (col1, col2,col3,col4) VALUES ('val',2,3,'foo')", - "INSERT INTO db.table_25 (col1,col2,col3,col4) VALUES (?,?,?,...)" - }, - { - "INSERT INTO db.table1_25 ( col_121,col2121 ,col12_3, col41203_ ) VALUES (?,?,?,...)", - "INSERT INTO db.table1_25 ( col_121,col2121,col12_3,col41203_ ) VALUES (?,?,?,...)" - }, - // TODO: KnownIssue - 5: Arithmetics operators breaks grouping - // { - // "INSERT INTO db.table ( col1, col2,col3,col4, col5 ) VALUES ('val',2,3,'foo', 5 + 10, 6 - 9)", - // "INSERT INTO db.table ( col1,col2,col3,col4,col5 ) VALUES (?,?,?,...)" - // // Actual: "INSERT INTO db.table ( col1,col2,col3,col4,col5 ) VALUES (?,?,?,... + -)" - // }, -}; - -const vector> queries_digests_grouping { - // test query_digest reduction - { - "SELECT * FROM tablename WHERE id IN (1,2, 3,4 ,5 ,6,7,8,9,10)", - "SELECT * FROM tablename WHERE id IN (?,...)" - }, - // invalid request grouping - { - "SELECT * tablename where id IN (1,2,3,4,5 , 6,7,8, AND j in (1, 2,3, 4 ,5,6,7,8,9 and k=1", - "SELECT * tablename where id IN (?,...,AND j in (?,... and k=?" - }, - // more random grouping - { - "SELECT (1.1, 1, 2, 13, 4.81, 12) FROM db.table", - "SELECT (?,...) FROM db.table" - }, - { - "SELECT (1.1, 1.12934 , 21.32 , 91, 91, 12.93 ) FROM db.table2", - "SELECT (?,...) FROM db.table2" - }, - { - "SELECT (1.1, 1.12934 , 21.32 , 91.2 , 91, 12 ) FROM db.table7", - "SELECT (?,...) FROM db.table7" - }, - { - "SELECT (1.1, 1.12934, 21.32, 391,2381,28.493,1283 ) FROM db.table2", - "SELECT (?,...) FROM db.table2" - }, - { - "SELECT (1.1, 1.12934, 21.32 , 91, 91, 12.1 ) FROM db.table3", - "SELECT (?,...) FROM db.table3" - } -}; - -const vector> null_queries_digests { - { - "select Null , '1*2/2',NULL,null , '1 ' , 100 % 3 ", - "select Null,?,NULL,null,?,?%?", - "select ?,?,?,?,?,?%?" - }, - // TODO: KnownIssue - 3: Grouping count isn't reset by NULL. - // { - // "INSERT INTO db.table VALUES ( Null , NULL, '',NULL, 'a', 'b', 'z',nuLL)", - // "INSERT INTO db.table VALUES (Null,NULL,?,NULL,?,?,?,nuLL)", - // "INSERT INTO db.table VALUES (?,?,?,...)" - // // Act: INSERT INTO db.table VALUES ( Null,NULL,?,NULL,?,?,...,nuLL) - // }, - { - "INSERT INTO db.table VALUES ( NULL, 'a', 'b', 'z', -4, nuLL)", - // TODO: KnownIssue - 4: Spaces preceding NULL values are not properly removed - "INSERT INTO db.table VALUES ( NULL,?,?,?,...,nuLL)", - "INSERT INTO db.table VALUES (?,?,?,...)" - }, -}; - -std::string replace_str(const std::string& str, const std::string& match, const std::string& repl) { - if(match.empty()) { - return str; - } - - std::string result = str; - size_t start_pos = 0; - - while((start_pos = result.find(match, start_pos)) != std::string::npos) { - result.replace(start_pos, match.length(), repl); - start_pos += repl.length(); - } - - return result; -} - -std::string increase_mark_num(const std::string query, uint32_t num) { - std::string result = query; - std::string marks = ""; - - for (uint32_t i = 0; i < num - 1; i++) { - marks += "?,"; - } - marks += "?,..."; - - result = replace_str(result, "?,...", marks); - - return result; -} - -char is_digit_char(char c) { - if(c >= '0' && c <= '9') { - return 1; - } - return 0; -} - -vector extract_numbers(const string query) { - vector numbers {}; - string number {}; - - for (const char c : query) { - if (is_digit_char(c)) { - number += c; - } else { - if (!number.empty()) { - numbers.push_back(number); - number.clear(); - } - } - } - - return numbers; -} - -string replace_numbers(const string query, const char mark) { - vector numbers { extract_numbers(query) }; - std::sort( - numbers.begin(), numbers.end(), - [](const string& s1, const string& s2) -> bool { return s1.size() > s2.size(); } - ); - - string query_res { query }; - - for (const string& num : numbers) { - query_res = replace_str(query_res, num, string { mark }); - } - - return query_res; -} - -int main(int argc, char** argv) { - if (query_digest_pairs.size() != query_digest_pairs.size()) { - ok(0, "queries and exp_results sizes doesn't match"); - return exit_status(); - } - - plan( - query_digest_pairs.size()*2 + queries_digests_grouping.size()*5 + null_queries_digests.size()*2 - ); - - char buf[QUERY_DIGEST_BUF]; - - const auto test_query_digests = [&](bool replace_digits) -> void { - mysql_thread___query_digests_no_digits=replace_digits; - - for (size_t i = 0; i < query_digest_pairs.size(); i++) { - const auto& query = query_digest_pairs[i].first; - const auto& query_str_rep = replace_str(query_digest_pairs[i].first, "\n", "\\n"); - char* first_comment = NULL; - std::string exp_res {}; - - if (replace_digits == false) { - exp_res = query_digest_pairs[i].second; - } else { - exp_res = replace_numbers(query_digest_pairs[i].second, '?'); - } - - char* c_res = mysql_query_digest_and_first_comment(const_cast(query.c_str()), query.length(), &first_comment, buf); - const std::string result(c_res); - std::string ok_msg {}; - - if (replace_digits == false) { - ok_msg = "Digest should be equal to exp result:\n * Query: `%s`,\n * Act: `%s`,\n * Exp: `%s`"; - } else { - ok_msg = "No-Digits digest should be equal to exp result:\n * Query: `%s`,\n * Act: `%s`,\n * Exp: `%s`"; - } - - ok( - result == exp_res, ok_msg.c_str(), - query_str_rep.c_str(), result.c_str(), exp_res.c_str() - ); - } - - mysql_thread___query_digests_no_digits=0; - }; - - /* Test queries without replacing digits */ - test_query_digests(false); - /* Test queries replacing digits */ - test_query_digests(true); - - const auto test_null_replacting = [&](bool replace_nulls) -> void { - mysql_thread___query_digests_replace_null=replace_nulls; - - for (size_t i = 0; i < null_queries_digests.size(); i++) { - const auto& query = std::get<0>(null_queries_digests[i]); - std::string exp_res {}; - - if (replace_nulls) { - exp_res = std::get<2>(null_queries_digests[i]); - } else { - exp_res = std::get<1>(null_queries_digests[i]); - } - - char* c_res = mysql_query_digest_and_first_comment(const_cast(query.c_str()), query.length(), NULL, buf); - std::string result(c_res); - - ok( - result == exp_res, - "Replaced NULL values digest should be equal to exp result:" - "\n * Query: `%s`,\n * Act: `%s`,\n * Exp: `%s`", - query.c_str(), result.c_str(), exp_res.c_str() - ); - } - - mysql_thread___query_digests_replace_null=0; - }; - - /* Test queries containing 'NULL', NOT replacing the 'NULL' values */ - test_null_replacting(false); - /* Test queries containing 'NULL', replacing the 'NULL' values */ - test_null_replacting(true); - - for (size_t i = 0; i < queries_digests_grouping.size(); i++) { - for (int j = 1; j <= 5; j++) { - mysql_thread___query_digests_grouping_limit = j; - - const auto& query = queries_digests_grouping[i].first; - const auto& exp_res = increase_mark_num(queries_digests_grouping[i].second, j); - - char* c_res = mysql_query_digest_and_first_comment(const_cast(query.c_str()), query.length(), NULL, buf); - std::string result(c_res); - - ok( - result == exp_res, - "Grouping digest should be equal to exp result:" - "\n * Query: `%s`,\n * Act: `%s`,\n * Exp: `%s`", - query.c_str(), result.c_str(), exp_res.c_str() - ); - } - } - - return exit_status(); -}