From 7b8c0603def20501fd5f8f7e2a9466c514f5570e Mon Sep 17 00:00:00 2001 From: Charles JUDITH Date: Mon, 21 May 2018 21:34:50 +0200 Subject: [PATCH 01/24] Improve slave_running check The current code only consider condition Slave_SQL_Running is yes and then send slave_running equal 1. We should consider both of condition are Slave_IO_Running and Slave_SQL_Running to set the value of slave_running metric. --- mysql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql.py b/mysql.py index 95f9bb1..b4135e8 100644 --- a/mysql.py +++ b/mysql.py @@ -398,8 +398,8 @@ def fetch_mysql_slave_stats(conn): if 'delay' in row and row['delay'] != None: status['slave_lag'] = row['delay'] - status['slave_running'] = 1 if slave_row['Slave_SQL_Running'] == 'Yes' else 0 - status['slave_stopped'] = 1 if slave_row['Slave_SQL_Running'] != 'Yes' else 0 + status['slave_running'] = 1 if slave_row['Slave_SQL_Running'] == 'Yes' and slave_row['Slave_IO_Running'] == 'Yes' else 0 + status['slave_stopped'] = 1 if slave_row['Slave_SQL_Running'] != 'Yes' or slave_row['Slave_IO_Running'] != 'Yes' else 0 return status def fetch_mysql_process_states(conn): From 581b03d72132abafe1cf57114af7f0bf9a051d87 Mon Sep 17 00:00:00 2001 From: Charles JUDITH Date: Tue, 23 Oct 2018 13:49:09 +0400 Subject: [PATCH 02/24] Add database size metric This change will add the database size (db_size) metric. --- README.md | 8 ++++++++ mysql.py | 12 ++++++++++++ 2 files changed, 20 insertions(+) diff --git a/README.md b/README.md index df02a08..402e13b 100644 --- a/README.md +++ b/README.md @@ -296,6 +296,14 @@ TBD: hash_index_cells_used additional_pool_alloc +### Database size + +Collected by parsing the output of `SELECT table_schema 'db_name', Round(Sum(data_length + index_length) / 1024 / 1024, 0) 'db_size_mb' FROM information_schema.tables WHERE table_schema not in ('mysql', 'information_schema', 'performance_schema', 'heartbeat') GROUP BY table_schema;`: + + db_size/databasename=14907 + +The Database size is in MB. + ### Master/Slave Status diff --git a/mysql.py b/mysql.py index b4135e8..11627fa 100644 --- a/mysql.py +++ b/mysql.py @@ -402,6 +402,14 @@ def fetch_mysql_slave_stats(conn): status['slave_stopped'] = 1 if slave_row['Slave_SQL_Running'] != 'Yes' or slave_row['Slave_IO_Running'] != 'Yes' else 0 return status +def fetch_mysql_db_size(conn): + result = mysql_query(conn, "SELECT table_schema 'db_name', Round(Sum(data_length + index_length) / 1024 / 1024, 0) 'db_size_mb' FROM information_schema.tables WHERE table_schema not in ('mysql', 'information_schema', 'performance_schema', 'heartbeat') GROUP BY table_schema;") + + stats = {} + for row in result.fetchall(): + stats[row['db_name']] = row['db_size_mb'] + return stats + def fetch_mysql_process_states(conn): global MYSQL_PROCESS_STATES result = mysql_query(conn, 'SHOW PROCESSLIST') @@ -571,6 +579,10 @@ def read_callback(): for key in response_times: dispatch_value('response_time_total', str(key), response_times[key]['total'], 'counter') dispatch_value('response_time_count', str(key), response_times[key]['count'], 'counter') + + mysql_db_size = fetch_mysql_db_size(conn) + for key in mysql_db_size: + dispatch_value('db_size', key, mysql_db_size[key], 'counter') innodb_status = fetch_innodb_stats(conn) for key in MYSQL_INNODB_STATUS_VARS: From fff2b2ee4b6c8e6ecbdaec380ee09e99f1a57837 Mon Sep 17 00:00:00 2001 From: Charles JUDITH Date: Wed, 31 Oct 2018 18:50:21 +0100 Subject: [PATCH 03/24] Modify metric type for db_size metric Use gauge instead of counter for db_size metric. --- mysql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql.py b/mysql.py index 11627fa..05fac08 100644 --- a/mysql.py +++ b/mysql.py @@ -582,7 +582,7 @@ def read_callback(): mysql_db_size = fetch_mysql_db_size(conn) for key in mysql_db_size: - dispatch_value('db_size', key, mysql_db_size[key], 'counter') + dispatch_value('db_size', key, mysql_db_size[key], 'gauge') innodb_status = fetch_innodb_stats(conn) for key in MYSQL_INNODB_STATUS_VARS: From 6abeec1745ea478d26118663f4bfb016f11d2e07 Mon Sep 17 00:00:00 2001 From: Pierre DUMONT Date: Mon, 21 Jan 2019 17:47:44 +0100 Subject: [PATCH 04/24] Changed 'Innodb_row_lock_waits' from 'counter' to 'gauge' --- mysql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql.py b/mysql.py index 05fac08..54b8982 100644 --- a/mysql.py +++ b/mysql.py @@ -96,7 +96,7 @@ 'Innodb_row_lock_time': 'counter', 'Innodb_row_lock_time_avg': 'gauge', 'Innodb_row_lock_time_max': 'gauge', - 'Innodb_row_lock_waits': 'counter', + 'Innodb_row_lock_waits': 'gauge', 'Innodb_rows_deleted': 'counter', 'Innodb_rows_inserted': 'counter', 'Innodb_rows_read': 'counter', From 5d8cfd0f270f52868ff9b2f8566f4d7a6534b2c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kl=C3=83=C2=A9bert=20Hodin?= Date: Mon, 29 Apr 2019 15:42:50 +0200 Subject: [PATCH 05/24] Revert "Changed 'Innodb_row_lock_waits' from 'counter' to 'gauge'" This reverts commit 6abeec1745ea478d26118663f4bfb016f11d2e07. --- mysql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql.py b/mysql.py index 54b8982..05fac08 100644 --- a/mysql.py +++ b/mysql.py @@ -96,7 +96,7 @@ 'Innodb_row_lock_time': 'counter', 'Innodb_row_lock_time_avg': 'gauge', 'Innodb_row_lock_time_max': 'gauge', - 'Innodb_row_lock_waits': 'gauge', + 'Innodb_row_lock_waits': 'counter', 'Innodb_rows_deleted': 'counter', 'Innodb_rows_inserted': 'counter', 'Innodb_rows_read': 'counter', From 2f6fa415f33e0921c676f51c38766b5817a62b0b Mon Sep 17 00:00:00 2001 From: Charles JUDITH Date: Wed, 21 Aug 2019 17:18:37 +0200 Subject: [PATCH 06/24] Collect os_log_bytes_written from information schema. This is done to collect the os_log_bytes_written metric from infromation_schema.metrics table. --- README.md | 9 +++++++++ mysql.py | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/README.md b/README.md index 402e13b..8d2a49b 100644 --- a/README.md +++ b/README.md @@ -296,6 +296,15 @@ TBD: hash_index_cells_used additional_pool_alloc +### InnoDB metrics from information_schema + +**Warning: This is only available for MariaDB 10.x** + +Collected by parsing the output of `SELECT COUNT from INFORMATION_SCHEMA.INNODB_METRICS where name ='os_log_bytes_written';`: + + mysql plugin: Sending value: innodb/os_log_bytes_written=263167952896 + + ### Database size Collected by parsing the output of `SELECT table_schema 'db_name', Round(Sum(data_length + index_length) / 1024 / 1024, 0) 'db_size_mb' FROM information_schema.tables WHERE table_schema not in ('mysql', 'information_schema', 'performance_schema', 'heartbeat') GROUP BY table_schema;`: diff --git a/mysql.py b/mysql.py index 05fac08..055eeed 100644 --- a/mysql.py +++ b/mysql.py @@ -410,6 +410,11 @@ def fetch_mysql_db_size(conn): stats[row['db_name']] = row['db_size_mb'] return stats +def fetch_innodb_os_log_bytes_written(conn): + result = mysql_query(conn, "SELECT COUNT from INFORMATION_SCHEMA.INNODB_METRICS where name ='os_log_bytes_written';") + stats = result.fetchone() + return stats + def fetch_mysql_process_states(conn): global MYSQL_PROCESS_STATES result = mysql_query(conn, 'SHOW PROCESSLIST') @@ -584,6 +589,9 @@ def read_callback(): for key in mysql_db_size: dispatch_value('db_size', key, mysql_db_size[key], 'gauge') + innodb_log_bytes_written = fetch_innodb_os_log_bytes_written(conn) + dispatch_value('innodb', 'os_log_bytes_written', innodb_log_bytes_written['COUNT'], 'counter') + innodb_status = fetch_innodb_stats(conn) for key in MYSQL_INNODB_STATUS_VARS: if key not in innodb_status: continue From 18828b8a0ee07aa810929a9ae96b69fbae9af864 Mon Sep 17 00:00:00 2001 From: Charles JUDITH Date: Thu, 22 Aug 2019 12:15:30 +0200 Subject: [PATCH 07/24] Improve os_log_bytes_written implementation The innodb_metrics table is only available since MySQL 5.6 and MariaDB 10.x. This change will prevent this part of the code to raise an error for the MySQL < 5.6 and MariaDB < 10.x. --- README.md | 2 +- mysql.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8d2a49b..26edc73 100644 --- a/README.md +++ b/README.md @@ -298,7 +298,7 @@ TBD: ### InnoDB metrics from information_schema -**Warning: This is only available for MariaDB 10.x** +**Warning: This is only available in MariaDB >= 10.x and MySQL >= 5.6.** Collected by parsing the output of `SELECT COUNT from INFORMATION_SCHEMA.INNODB_METRICS where name ='os_log_bytes_written';`: diff --git a/mysql.py b/mysql.py index 055eeed..c8da7bd 100644 --- a/mysql.py +++ b/mysql.py @@ -411,8 +411,12 @@ def fetch_mysql_db_size(conn): return stats def fetch_innodb_os_log_bytes_written(conn): - result = mysql_query(conn, "SELECT COUNT from INFORMATION_SCHEMA.INNODB_METRICS where name ='os_log_bytes_written';") - stats = result.fetchone() + # This feature is only available for mariaDB >= 10.x and MySQL > 5.5. + try: + result = mysql_query(conn, "SELECT COUNT from INFORMATION_SCHEMA.INNODB_METRICS where name ='os_log_bytes_written';") + stats = result.fetchone() + except MySQLdb.OperationalError: + stats = {'COUNT': 0} return stats def fetch_mysql_process_states(conn): @@ -602,8 +606,8 @@ def read_callback(): collectd.register_config(configure_callback) if __name__ == "__main__" and not COLLECTD_ENABLED: - print "Running in test mode, invoke with" - print sys.argv[0] + " Host Port User Password " + print("Running in test mode, invoke with") + print(sys.argv[0] + " Host Port User Password ") MYSQL_CONFIG['Host'] = sys.argv[1] MYSQL_CONFIG['Port'] = int(sys.argv[2]) MYSQL_CONFIG['User'] = sys.argv[3] From c48050ed0d13fd3b6aafb41126eea4672b4f171a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20LEQUOY?= Date: Fri, 14 Feb 2020 19:11:25 +0100 Subject: [PATCH 08/24] Add number proccess locked by an other one --- mysql.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/mysql.py b/mysql.py index c8da7bd..bacf185 100644 --- a/mysql.py +++ b/mysql.py @@ -419,6 +419,15 @@ def fetch_innodb_os_log_bytes_written(conn): stats = {'COUNT': 0} return stats +def fetch_mariadb_lock(conn): + # This feature is only available for mariaDB with plugin METADATA_LOCK_INFO installed + try: + result = mysql_query(conn, "SELECT count(1) as `nb_lock` FROM INFORMATION_SCHEMA.PROCESSLIST P, INFORMATION_SCHEMA.METADATA_LOCK_INFO M WHERE LOCATE(lcase(M.LOCK_TYPE), lcase(P.STATE))>0;") + stats = result.fetchone() + except MySQLdb.OperationalError: + stats = {'nb_lock': 0} + return stats + def fetch_mysql_process_states(conn): global MYSQL_PROCESS_STATES result = mysql_query(conn, 'SHOW PROCESSLIST') @@ -430,7 +439,6 @@ def fetch_mysql_process_states(conn): state = state.lower().replace(" ", "_") if state not in states: state = 'other' states[state] += 1 - return states def fetch_mysql_variables(conn): @@ -596,6 +604,11 @@ def read_callback(): innodb_log_bytes_written = fetch_innodb_os_log_bytes_written(conn) dispatch_value('innodb', 'os_log_bytes_written', innodb_log_bytes_written['COUNT'], 'counter') + + meta_data_lock = fetch_mariadb_lock(conn) + dispatch_value('mariadb', 'lock', meta_data_lock['nb_lock'], 'gauge') + + innodb_status = fetch_innodb_stats(conn) for key in MYSQL_INNODB_STATUS_VARS: if key not in innodb_status: continue From c32f189cdaedd2d4f7b36967749cc296b11cd545 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20LEQUOY?= Date: Wed, 19 Feb 2020 11:10:49 +0100 Subject: [PATCH 09/24] Changed index from lock to innodb for nb_lock (METADATA_LOCK_INFO) --- mysql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql.py b/mysql.py index bacf185..6da0d92 100644 --- a/mysql.py +++ b/mysql.py @@ -606,7 +606,7 @@ def read_callback(): meta_data_lock = fetch_mariadb_lock(conn) - dispatch_value('mariadb', 'lock', meta_data_lock['nb_lock'], 'gauge') + dispatch_value('innodb', 'lock', meta_data_lock['nb_lock'], 'gauge') innodb_status = fetch_innodb_stats(conn) From dedeca149549561bc062ec426e6da6c46a020de9 Mon Sep 17 00:00:00 2001 From: Charles JUDITH Date: Fri, 21 Feb 2020 16:03:46 +0100 Subject: [PATCH 10/24] Update README.md to handle the metadata lock Update README.md to add more details about the new lock metrics. --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 26edc73..da6d306 100644 --- a/README.md +++ b/README.md @@ -304,6 +304,13 @@ Collected by parsing the output of `SELECT COUNT from INFORMATION_SCHEMA.INNODB_ mysql plugin: Sending value: innodb/os_log_bytes_written=263167952896 +To get the metadata lock information: + + mysql plugin: Sending value: innodb/lock=0 + +- MariaDB: https://mariadb.com/kb/en/metadata-lock-info-plugin/ +- MySQL: https://dev.mysql.com/doc/refman/5.7/en/metadata-locks-table.html + ### Database size From 270796f11a7b42272469b39cc2d9a237329846b5 Mon Sep 17 00:00:00 2001 From: Charles JUDITH Date: Tue, 14 Apr 2020 16:48:36 +0200 Subject: [PATCH 11/24] Add read_only metric Add read_only metric to detect is a server is in "read-only" mode. --- mysql.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql.py b/mysql.py index 6da0d92..886a1e9 100644 --- a/mysql.py +++ b/mysql.py @@ -212,6 +212,7 @@ 'query_cache_size', 'query_cache_size', 'read_buffer_size', + 'read_only', 'table_cache', 'table_definition_cache', 'table_open_cache', From abf1cad2838ed4b34aedf4c5baa98dd2a4943b9c Mon Sep 17 00:00:00 2001 From: Charles JUDITH Date: Wed, 15 Apr 2020 15:03:38 +0200 Subject: [PATCH 12/24] Add metric type to make it work with the previous change (#10) * Add read_only metric Add read_only metric to detect is a server is in "read-only" mode. * Add metric type to make it work with the previous change Co-authored-by: Charles JUDITH --- mysql.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mysql.py b/mysql.py index 886a1e9..4146eb6 100644 --- a/mysql.py +++ b/mysql.py @@ -448,7 +448,13 @@ def fetch_mysql_variables(conn): variables = {} for row in result.fetchall(): if row['Variable_name'] in MYSQL_VARS: - variables[row['Variable_name']] = row['Value'] + if row['Variable_name'] == 'read_only': + if row['Value'] == 'OFF': + variables[row['Variable_name']] = 0 + else: + variables[row['Variable_name']] = 1 + else: + variables[row['Variable_name']] = row['Value'] return variables From ff673a8ea9664af0fe1d9b8cbb9893ecad33b785 Mon Sep 17 00:00:00 2001 From: Charles JUDITH Date: Thu, 16 Apr 2020 15:50:12 +0200 Subject: [PATCH 13/24] Revamp indentation (replace 8 by for for tab space) (#11) This will make a code with a 4 spaces tab space. Co-authored-by: Charles JUDITH --- mysql.py | 1078 +++++++++++++++++++++++++++--------------------------- 1 file changed, 538 insertions(+), 540 deletions(-) diff --git a/mysql.py b/mysql.py index 4146eb6..fae9568 100644 --- a/mysql.py +++ b/mysql.py @@ -27,615 +27,613 @@ COLLECTD_ENABLED=True try: - import collectd + import collectd except ImportError: - # We're not running in CollectD, set this to False so we can make some changes - # accordingly for testing/development. - COLLECTD_ENABLED=False + # We're not running in CollectD, set this to False so we can make some changes + # accordingly for testing/development. + COLLECTD_ENABLED=False import re import MySQLdb MYSQL_CONFIG = { - 'Host': 'localhost', - 'Port': 3306, - 'User': 'root', - 'Password': '', - 'HeartbeatTable': '', - 'Verbose': False, + 'Host': 'localhost', + 'Port': 3306, + 'User': 'root', + 'Password': '', + 'HeartbeatTable': '', + 'Verbose': False, } MYSQL_STATUS_VARS = { - 'Aborted_clients': 'counter', - 'Aborted_connects': 'counter', - 'Binlog_cache_disk_use': 'counter', - 'Binlog_cache_use': 'counter', - 'Bytes_received': 'counter', - 'Bytes_sent': 'counter', - 'Connections': 'counter', - 'Created_tmp_disk_tables': 'counter', - 'Created_tmp_files': 'counter', - 'Created_tmp_tables': 'counter', - 'Innodb_buffer_pool_pages_data': 'gauge', - 'Innodb_buffer_pool_pages_dirty': 'gauge', - 'Innodb_buffer_pool_pages_free': 'gauge', - 'Innodb_buffer_pool_pages_total': 'gauge', - 'Innodb_buffer_pool_read_requests': 'counter', - 'Innodb_buffer_pool_reads': 'counter', - 'Innodb_checkpoint_age': 'gauge', - 'Innodb_checkpoint_max_age': 'gauge', - 'Innodb_data_fsyncs': 'counter', - 'Innodb_data_pending_fsyncs': 'gauge', - 'Innodb_data_pending_reads': 'gauge', - 'Innodb_data_pending_writes': 'gauge', - 'Innodb_data_read': 'counter', - 'Innodb_data_reads': 'counter', - 'Innodb_data_writes': 'counter', - 'Innodb_data_written': 'counter', - 'Innodb_deadlocks': 'counter', - 'Innodb_history_list_length': 'gauge', - 'Innodb_ibuf_free_list': 'gauge', - 'Innodb_ibuf_merged_delete_marks': 'counter', - 'Innodb_ibuf_merged_deletes': 'counter', - 'Innodb_ibuf_merged_inserts': 'counter', - 'Innodb_ibuf_merges': 'counter', - 'Innodb_ibuf_segment_size': 'gauge', - 'Innodb_ibuf_size': 'gauge', - 'Innodb_lsn_current': 'counter', - 'Innodb_lsn_flushed': 'counter', - 'Innodb_max_trx_id': 'counter', - 'Innodb_mem_adaptive_hash': 'gauge', - 'Innodb_mem_dictionary': 'gauge', - 'Innodb_mem_total': 'gauge', - 'Innodb_mutex_os_waits': 'counter', - 'Innodb_mutex_spin_rounds': 'counter', - 'Innodb_mutex_spin_waits': 'counter', - 'Innodb_os_log_pending_fsyncs': 'gauge', - 'Innodb_pages_created': 'counter', - 'Innodb_pages_read': 'counter', - 'Innodb_pages_written': 'counter', - 'Innodb_row_lock_time': 'counter', - 'Innodb_row_lock_time_avg': 'gauge', - 'Innodb_row_lock_time_max': 'gauge', - 'Innodb_row_lock_waits': 'counter', - 'Innodb_rows_deleted': 'counter', - 'Innodb_rows_inserted': 'counter', - 'Innodb_rows_read': 'counter', - 'Innodb_rows_updated': 'counter', - 'Innodb_s_lock_os_waits': 'counter', - 'Innodb_s_lock_spin_rounds': 'counter', - 'Innodb_s_lock_spin_waits': 'counter', - 'Innodb_uncheckpointed_bytes': 'gauge', - 'Innodb_unflushed_log': 'gauge', - 'Innodb_unpurged_txns': 'gauge', - 'Innodb_x_lock_os_waits': 'counter', - 'Innodb_x_lock_spin_rounds': 'counter', - 'Innodb_x_lock_spin_waits': 'counter', - 'Key_blocks_not_flushed': 'gauge', - 'Key_blocks_unused': 'gauge', - 'Key_blocks_used': 'gauge', - 'Key_read_requests': 'counter', - 'Key_reads': 'counter', - 'Key_write_requests': 'counter', - 'Key_writes': 'counter', - 'Max_used_connections': 'gauge', - 'Open_files': 'gauge', - 'Open_table_definitions': 'gauge', - 'Open_tables': 'gauge', - 'Opened_files': 'counter', - 'Opened_table_definitions': 'counter', - 'Opened_tables': 'counter', - 'Qcache_free_blocks': 'gauge', - 'Qcache_free_memory': 'gauge', - 'Qcache_hits': 'counter', - 'Qcache_inserts': 'counter', - 'Qcache_lowmem_prunes': 'counter', - 'Qcache_not_cached': 'counter', - 'Qcache_queries_in_cache': 'counter', - 'Qcache_total_blocks': 'counter', - 'Questions': 'counter', - 'Select_full_join': 'counter', - 'Select_full_range_join': 'counter', - 'Select_range': 'counter', - 'Select_range_check': 'counter', - 'Select_scan': 'counter', - 'Slave_open_temp_tables': 'gauge', - 'Slave_retried_transactions': 'counter', - 'Slow_launch_threads': 'counter', - 'Slow_queries': 'counter', - 'Sort_merge_passes': 'counter', - 'Sort_range': 'counter', - 'Sort_rows': 'counter', - 'Sort_scan': 'counter', - 'Table_locks_immediate': 'counter', - 'Table_locks_waited': 'counter', - 'Table_open_cache_hits': 'counter', - 'Table_open_cache_misses': 'counter', - 'Table_open_cache_overflows': 'counter', - 'Threadpool_idle_threads': 'gauge', - 'Threadpool_threads': 'gauge', - 'Threads_cached': 'gauge', - 'Threads_connected': 'gauge', - 'Threads_created': 'counter', - 'Threads_running': 'gauge', - 'Uptime': 'gauge', - 'wsrep_apply_oooe': 'gauge', - 'wsrep_apply_oool': 'gauge', - 'wsrep_apply_window': 'gauge', - 'wsrep_causal_reads': 'gauge', - 'wsrep_cert_deps_distance': 'gauge', - 'wsrep_cert_index_size': 'gauge', - 'wsrep_cert_interval': 'gauge', - 'wsrep_cluster_size': 'gauge', - 'wsrep_commit_oooe': 'gauge', - 'wsrep_commit_oool': 'gauge', - 'wsrep_commit_window': 'gauge', - 'wsrep_flow_control_paused': 'gauge', - 'wsrep_flow_control_paused_ns': 'counter', - 'wsrep_flow_control_recv': 'counter', - 'wsrep_flow_control_sent': 'counter', - 'wsrep_local_bf_aborts': 'counter', - 'wsrep_local_cert_failures': 'counter', - 'wsrep_local_commits': 'counter', - 'wsrep_local_recv_queue': 'gauge', - 'wsrep_local_recv_queue_avg': 'gauge', - 'wsrep_local_recv_queue_max': 'gauge', - 'wsrep_local_recv_queue_min': 'gauge', - 'wsrep_local_replays': 'gauge', - 'wsrep_local_send_queue': 'gauge', - 'wsrep_local_send_queue_avg': 'gauge', - 'wsrep_local_send_queue_max': 'gauge', - 'wsrep_local_send_queue_min': 'gauge', - 'wsrep_received': 'counter', - 'wsrep_received_bytes': 'counter', - 'wsrep_repl_data_bytes': 'counter', - 'wsrep_repl_keys': 'counter', - 'wsrep_repl_keys_bytes': 'counter', - 'wsrep_repl_other_bytes': 'counter', - 'wsrep_replicated': 'counter', - 'wsrep_replicated_bytes': 'counter', + 'Aborted_clients': 'counter', + 'Aborted_connects': 'counter', + 'Binlog_cache_disk_use': 'counter', + 'Binlog_cache_use': 'counter', + 'Bytes_received': 'counter', + 'Bytes_sent': 'counter', + 'Connections': 'counter', + 'Created_tmp_disk_tables': 'counter', + 'Created_tmp_files': 'counter', + 'Created_tmp_tables': 'counter', + 'Innodb_buffer_pool_pages_data': 'gauge', + 'Innodb_buffer_pool_pages_dirty': 'gauge', + 'Innodb_buffer_pool_pages_free': 'gauge', + 'Innodb_buffer_pool_pages_total': 'gauge', + 'Innodb_buffer_pool_read_requests': 'counter', + 'Innodb_buffer_pool_reads': 'counter', + 'Innodb_checkpoint_age': 'gauge', + 'Innodb_checkpoint_max_age': 'gauge', + 'Innodb_data_fsyncs': 'counter', + 'Innodb_data_pending_fsyncs': 'gauge', + 'Innodb_data_pending_reads': 'gauge', + 'Innodb_data_pending_writes': 'gauge', + 'Innodb_data_read': 'counter', + 'Innodb_data_reads': 'counter', + 'Innodb_data_writes': 'counter', + 'Innodb_data_written': 'counter', + 'Innodb_deadlocks': 'counter', + 'Innodb_history_list_length': 'gauge', + 'Innodb_ibuf_free_list': 'gauge', + 'Innodb_ibuf_merged_delete_marks': 'counter', + 'Innodb_ibuf_merged_deletes': 'counter', + 'Innodb_ibuf_merged_inserts': 'counter', + 'Innodb_ibuf_merges': 'counter', + 'Innodb_ibuf_segment_size': 'gauge', + 'Innodb_ibuf_size': 'gauge', + 'Innodb_lsn_current': 'counter', + 'Innodb_lsn_flushed': 'counter', + 'Innodb_max_trx_id': 'counter', + 'Innodb_mem_adaptive_hash': 'gauge', + 'Innodb_mem_dictionary': 'gauge', + 'Innodb_mem_total': 'gauge', + 'Innodb_mutex_os_waits': 'counter', + 'Innodb_mutex_spin_rounds': 'counter', + 'Innodb_mutex_spin_waits': 'counter', + 'Innodb_os_log_pending_fsyncs': 'gauge', + 'Innodb_pages_created': 'counter', + 'Innodb_pages_read': 'counter', + 'Innodb_pages_written': 'counter', + 'Innodb_row_lock_time': 'counter', + 'Innodb_row_lock_time_avg': 'gauge', + 'Innodb_row_lock_time_max': 'gauge', + 'Innodb_row_lock_waits': 'counter', + 'Innodb_rows_deleted': 'counter', + 'Innodb_rows_inserted': 'counter', + 'Innodb_rows_read': 'counter', + 'Innodb_rows_updated': 'counter', + 'Innodb_s_lock_os_waits': 'counter', + 'Innodb_s_lock_spin_rounds': 'counter', + 'Innodb_s_lock_spin_waits': 'counter', + 'Innodb_uncheckpointed_bytes': 'gauge', + 'Innodb_unflushed_log': 'gauge', + 'Innodb_unpurged_txns': 'gauge', + 'Innodb_x_lock_os_waits': 'counter', + 'Innodb_x_lock_spin_rounds': 'counter', + 'Innodb_x_lock_spin_waits': 'counter', + 'Key_blocks_not_flushed': 'gauge', + 'Key_blocks_unused': 'gauge', + 'Key_blocks_used': 'gauge', + 'Key_read_requests': 'counter', + 'Key_reads': 'counter', + 'Key_write_requests': 'counter', + 'Key_writes': 'counter', + 'Max_used_connections': 'gauge', + 'Open_files': 'gauge', + 'Open_table_definitions': 'gauge', + 'Open_tables': 'gauge', + 'Opened_files': 'counter', + 'Opened_table_definitions': 'counter', + 'Opened_tables': 'counter', + 'Qcache_free_blocks': 'gauge', + 'Qcache_free_memory': 'gauge', + 'Qcache_hits': 'counter', + 'Qcache_inserts': 'counter', + 'Qcache_lowmem_prunes': 'counter', + 'Qcache_not_cached': 'counter', + 'Qcache_queries_in_cache': 'counter', + 'Qcache_total_blocks': 'counter', + 'Questions': 'counter', + 'Select_full_join': 'counter', + 'Select_full_range_join': 'counter', + 'Select_range': 'counter', + 'Select_range_check': 'counter', + 'Select_scan': 'counter', + 'Slave_open_temp_tables': 'gauge', + 'Slave_retried_transactions': 'counter', + 'Slow_launch_threads': 'counter', + 'Slow_queries': 'counter', + 'Sort_merge_passes': 'counter', + 'Sort_range': 'counter', + 'Sort_rows': 'counter', + 'Sort_scan': 'counter', + 'Table_locks_immediate': 'counter', + 'Table_locks_waited': 'counter', + 'Table_open_cache_hits': 'counter', + 'Table_open_cache_misses': 'counter', + 'Table_open_cache_overflows': 'counter', + 'Threadpool_idle_threads': 'gauge', + 'Threadpool_threads': 'gauge', + 'Threads_cached': 'gauge', + 'Threads_connected': 'gauge', + 'Threads_created': 'counter', + 'Threads_running': 'gauge', + 'Uptime': 'gauge', + 'wsrep_apply_oooe': 'gauge', + 'wsrep_apply_oool': 'gauge', + 'wsrep_apply_window': 'gauge', + 'wsrep_causal_reads': 'gauge', + 'wsrep_cert_deps_distance': 'gauge', + 'wsrep_cert_index_size': 'gauge', + 'wsrep_cert_interval': 'gauge', + 'wsrep_cluster_size': 'gauge', + 'wsrep_commit_oooe': 'gauge', + 'wsrep_commit_oool': 'gauge', + 'wsrep_commit_window': 'gauge', + 'wsrep_flow_control_paused': 'gauge', + 'wsrep_flow_control_paused_ns': 'counter', + 'wsrep_flow_control_recv': 'counter', + 'wsrep_flow_control_sent': 'counter', + 'wsrep_local_bf_aborts': 'counter', + 'wsrep_local_cert_failures': 'counter', + 'wsrep_local_commits': 'counter', + 'wsrep_local_recv_queue': 'gauge', + 'wsrep_local_recv_queue_avg': 'gauge', + 'wsrep_local_recv_queue_max': 'gauge', + 'wsrep_local_recv_queue_min': 'gauge', + 'wsrep_local_replays': 'gauge', + 'wsrep_local_send_queue': 'gauge', + 'wsrep_local_send_queue_avg': 'gauge', + 'wsrep_local_send_queue_max': 'gauge', + 'wsrep_local_send_queue_min': 'gauge', + 'wsrep_received': 'counter', + 'wsrep_received_bytes': 'counter', + 'wsrep_repl_data_bytes': 'counter', + 'wsrep_repl_keys': 'counter', + 'wsrep_repl_keys_bytes': 'counter', + 'wsrep_repl_other_bytes': 'counter', + 'wsrep_replicated': 'counter', + 'wsrep_replicated_bytes': 'counter', } MYSQL_VARS = [ - 'binlog_stmt_cache_size', - 'innodb_additional_mem_pool_size', - 'innodb_buffer_pool_size', - 'innodb_concurrency_tickets', - 'innodb_io_capacity', - 'innodb_log_buffer_size', - 'innodb_log_file_size', - 'innodb_open_files', - 'innodb_open_files', - 'join_buffer_size', - 'max_connections', - 'open_files_limit', - 'query_cache_limit', - 'query_cache_size', - 'query_cache_size', - 'read_buffer_size', - 'read_only', - 'table_cache', - 'table_definition_cache', - 'table_open_cache', - 'thread_cache_size', - 'thread_cache_size', - 'thread_concurrency', - 'tmp_table_size', + 'binlog_stmt_cache_size', + 'innodb_additional_mem_pool_size', + 'innodb_buffer_pool_size', + 'innodb_concurrency_tickets', + 'innodb_io_capacity', + 'innodb_log_buffer_size', + 'innodb_log_file_size', + 'innodb_open_files', + 'innodb_open_files', + 'join_buffer_size', + 'max_connections', + 'open_files_limit', + 'query_cache_limit', + 'query_cache_size', + 'query_cache_size', + 'read_buffer_size', + 'read_only', + 'table_cache', + 'table_definition_cache', + 'table_open_cache', + 'thread_cache_size', + 'thread_cache_size', + 'thread_concurrency', + 'tmp_table_size', ] MYSQL_PROCESS_STATES = { - 'closing_tables': 0, - 'copying_to_tmp_table': 0, - 'end': 0, - 'freeing_items': 0, - 'init': 0, - 'locked': 0, - 'login': 0, - 'none': 0, - 'other': 0, - 'preparing': 0, - 'reading_from_net': 0, - 'sending_data': 0, - 'sorting_result': 0, - 'statistics': 0, - 'updating': 0, - 'writing_to_net': 0, - 'creating_table': 0, - 'opening_tables': 0, + 'closing_tables': 0, + 'copying_to_tmp_table': 0, + 'end': 0, + 'freeing_items': 0, + 'init': 0, + 'locked': 0, + 'login': 0, + 'none': 0, + 'other': 0, + 'preparing': 0, + 'reading_from_net': 0, + 'sending_data': 0, + 'sorting_result': 0, + 'statistics': 0, + 'updating': 0, + 'writing_to_net': 0, + 'creating_table': 0, + 'opening_tables': 0, } MYSQL_INNODB_STATUS_VARS = { - 'active_transactions': 'gauge', - 'current_transactions': 'gauge', - 'file_reads': 'counter', - 'file_system_memory': 'gauge', - 'file_writes': 'counter', - 'innodb_lock_structs': 'gauge', - 'innodb_lock_wait_secs': 'gauge', - 'innodb_locked_tables': 'gauge', - 'innodb_sem_wait_time_ms': 'gauge', - 'innodb_sem_waits': 'gauge', - 'innodb_tables_in_use': 'gauge', - 'lock_system_memory': 'gauge', - 'locked_transactions': 'gauge', - 'log_writes': 'counter', - 'page_hash_memory': 'gauge', - 'pending_aio_log_ios': 'gauge', - 'pending_buf_pool_flushes': 'gauge', - 'pending_chkp_writes': 'gauge', - 'pending_ibuf_aio_reads': 'gauge', - 'pending_log_writes':'gauge', - 'queries_inside': 'gauge', - 'queries_queued': 'gauge', - 'read_views': 'gauge', + 'active_transactions': 'gauge', + 'current_transactions': 'gauge', + 'file_reads': 'counter', + 'file_system_memory': 'gauge', + 'file_writes': 'counter', + 'innodb_lock_structs': 'gauge', + 'innodb_lock_wait_secs': 'gauge', + 'innodb_locked_tables': 'gauge', + 'innodb_sem_wait_time_ms': 'gauge', + 'innodb_sem_waits': 'gauge', + 'innodb_tables_in_use': 'gauge', + 'lock_system_memory': 'gauge', + 'locked_transactions': 'gauge', + 'log_writes': 'counter', + 'page_hash_memory': 'gauge', + 'pending_aio_log_ios': 'gauge', + 'pending_buf_pool_flushes': 'gauge', + 'pending_chkp_writes': 'gauge', + 'pending_ibuf_aio_reads': 'gauge', + 'pending_log_writes':'gauge', + 'queries_inside': 'gauge', + 'queries_queued': 'gauge', + 'read_views': 'gauge', } MYSQL_INNODB_STATUS_MATCHES = { - # 0 read views open inside InnoDB - 'read views open inside InnoDB': { - 'read_views': 0, - }, - # 5635328 OS file reads, 27018072 OS file writes, 20170883 OS fsyncs - ' OS file reads, ': { - 'file_reads': 0, - 'file_writes': 4, - }, - # ibuf aio reads: 0, log i/o's: 0, sync i/o's: 0 - 'ibuf aio reads': { - 'pending_ibuf_aio_reads': 3, - 'pending_aio_log_ios': 6, - 'pending_aio_sync_ios': 9, - }, - # Pending flushes (fsync) log: 0; buffer pool: 0 - 'Pending flushes (fsync)': { - 'pending_buf_pool_flushes': 7, - }, - # 16086708 log i/o's done, 106.07 log i/o's/second - " log i/o's done, ": { - 'log_writes': 0, - }, - # 0 pending log writes, 0 pending chkp writes - ' pending log writes, ': { - 'pending_log_writes': 0, - 'pending_chkp_writes': 4, - }, - # Page hash 2302856 (buffer pool 0 only) - 'Page hash ': { - 'page_hash_memory': 2, - }, - # File system 657820264 (812272 + 657007992) - 'File system ': { - 'file_system_memory': 2, - }, - # Lock system 143820296 (143819576 + 720) - 'Lock system ': { - 'lock_system_memory': 2, - }, - # 0 queries inside InnoDB, 0 queries in queue - 'queries inside InnoDB, ': { - 'queries_inside': 0, - 'queries_queued': 4, - }, - # --Thread 139954487744256 has waited at dict0dict.cc line 472 for 0.0000 seconds the semaphore: - 'seconds the semaphore': { - 'innodb_sem_waits': lambda row, stats: stats['innodb_sem_waits'] + 1, - 'innodb_sem_wait_time_ms': lambda row, stats: int(float(row[9]) * 1000), - }, - # mysql tables in use 1, locked 1 - 'mysql tables in use': { - 'innodb_tables_in_use': lambda row, stats: stats['innodb_tables_in_use'] + int(row[4]), - 'innodb_locked_tables': lambda row, stats: stats['innodb_locked_tables'] + int(row[6]), - }, - "------- TRX HAS BEEN": { - "innodb_lock_wait_secs": lambda row, stats: stats['innodb_lock_wait_secs'] + int(row[5]), - }, + # 0 read views open inside InnoDB + 'read views open inside InnoDB': { + 'read_views': 0, + }, + # 5635328 OS file reads, 27018072 OS file writes, 20170883 OS fsyncs + ' OS file reads, ': { + 'file_reads': 0, + 'file_writes': 4, + }, + # ibuf aio reads: 0, log i/o's: 0, sync i/o's: 0 + 'ibuf aio reads': { + 'pending_ibuf_aio_reads': 3, + 'pending_aio_log_ios': 6, + 'pending_aio_sync_ios': 9, + }, + # Pending flushes (fsync) log: 0; buffer pool: 0 + 'Pending flushes (fsync)': { + 'pending_buf_pool_flushes': 7, + }, + # 16086708 log i/o's done, 106.07 log i/o's/second + " log i/o's done, ": { + 'log_writes': 0, + }, + # 0 pending log writes, 0 pending chkp writes + ' pending log writes, ': { + 'pending_log_writes': 0, + 'pending_chkp_writes': 4, + }, + # Page hash 2302856 (buffer pool 0 only) + 'Page hash ': { + 'page_hash_memory': 2, + }, + # File system 657820264 (812272 + 657007992) + 'File system ': { + 'file_system_memory': 2, + }, + # Lock system 143820296 (143819576 + 720) + 'Lock system ': { + 'lock_system_memory': 2, + }, + # 0 queries inside InnoDB, 0 queries in queue + 'queries inside InnoDB, ': { + 'queries_inside': 0, + 'queries_queued': 4, + }, + # --Thread 139954487744256 has waited at dict0dict.cc line 472 for 0.0000 seconds the semaphore: + 'seconds the semaphore': { + 'innodb_sem_waits': lambda row, stats: stats['innodb_sem_waits'] + 1, + 'innodb_sem_wait_time_ms': lambda row, stats: int(float(row[9]) * 1000), + }, + # mysql tables in use 1, locked 1 + 'mysql tables in use': { + 'innodb_tables_in_use': lambda row, stats: stats['innodb_tables_in_use'] + int(row[4]), + 'innodb_locked_tables': lambda row, stats: stats['innodb_locked_tables'] + int(row[6]), + }, + "------- TRX HAS BEEN": { + "innodb_lock_wait_secs": lambda row, stats: stats['innodb_lock_wait_secs'] + int(row[5]), + }, } def get_mysql_conn(): - return MySQLdb.connect( - host=MYSQL_CONFIG['Host'], - port=MYSQL_CONFIG['Port'], - user=MYSQL_CONFIG['User'], - passwd=MYSQL_CONFIG['Password'] - ) + return MySQLdb.connect( + host=MYSQL_CONFIG['Host'], + port=MYSQL_CONFIG['Port'], + user=MYSQL_CONFIG['User'], + passwd=MYSQL_CONFIG['Password'] + ) def mysql_query(conn, query): - cur = conn.cursor(MySQLdb.cursors.DictCursor) - cur.execute(query) - return cur + cur = conn.cursor(MySQLdb.cursors.DictCursor) + cur.execute(query) + return cur def fetch_mysql_status(conn): - result = mysql_query(conn, 'SHOW GLOBAL STATUS') - status = {} - for row in result.fetchall(): - status[row['Variable_name']] = row['Value'] + result = mysql_query(conn, 'SHOW GLOBAL STATUS') + status = {} + for row in result.fetchall(): + status[row['Variable_name']] = row['Value'] - # calculate the number of unpurged txns from existing variables - if 'Innodb_max_trx_id' in status: - status['Innodb_unpurged_txns'] = int(status['Innodb_max_trx_id']) - int(status['Innodb_purge_trx_id']) + # calculate the number of unpurged txns from existing variables + if 'Innodb_max_trx_id' in status: + status['Innodb_unpurged_txns'] = int(status['Innodb_max_trx_id']) - int(status['Innodb_purge_trx_id']) - if 'Innodb_lsn_last_checkpoint' in status: - status['Innodb_uncheckpointed_bytes'] = int(status['Innodb_lsn_current'])- int(status['Innodb_lsn_last_checkpoint']) + if 'Innodb_lsn_last_checkpoint' in status: + status['Innodb_uncheckpointed_bytes'] = int(status['Innodb_lsn_current'])- int(status['Innodb_lsn_last_checkpoint']) - if 'Innodb_lsn_flushed' in status: - status['Innodb_unflushed_log'] = int(status['Innodb_lsn_current']) - int(status['Innodb_lsn_flushed']) + if 'Innodb_lsn_flushed' in status: + status['Innodb_unflushed_log'] = int(status['Innodb_lsn_current']) - int(status['Innodb_lsn_flushed']) - return status + return status def fetch_mysql_master_stats(conn): - try: - result = mysql_query(conn, 'SHOW BINARY LOGS') - except MySQLdb.OperationalError: - return {} + try: + result = mysql_query(conn, 'SHOW BINARY LOGS') + except MySQLdb.OperationalError: + return {} - stats = { - 'binary_log_space': 0, - } + stats = { + 'binary_log_space': 0, + } - for row in result.fetchall(): - if 'File_size' in row and row['File_size'] > 0: - stats['binary_log_space'] += int(row['File_size']) + for row in result.fetchall(): + if 'File_size' in row and row['File_size'] > 0: + stats['binary_log_space'] += int(row['File_size']) - return stats + return stats def fetch_mysql_slave_stats(conn): - result = mysql_query(conn, 'SHOW SLAVE STATUS') - slave_row = result.fetchone() - if slave_row is None: - return {} - - status = { - 'relay_log_space': slave_row['Relay_Log_Space'], - 'slave_lag': slave_row['Seconds_Behind_Master'] if slave_row['Seconds_Behind_Master'] != None else 0, - } - - if MYSQL_CONFIG['HeartbeatTable']: - query = """ - SELECT MAX(UNIX_TIMESTAMP() - UNIX_TIMESTAMP(ts)) AS delay - FROM %s - WHERE server_id = %s - """ % (MYSQL_CONFIG['HeartbeatTable'], slave_row['Master_Server_Id']) - result = mysql_query(conn, query) - row = result.fetchone() - if 'delay' in row and row['delay'] != None: - status['slave_lag'] = row['delay'] - - status['slave_running'] = 1 if slave_row['Slave_SQL_Running'] == 'Yes' and slave_row['Slave_IO_Running'] == 'Yes' else 0 - status['slave_stopped'] = 1 if slave_row['Slave_SQL_Running'] != 'Yes' or slave_row['Slave_IO_Running'] != 'Yes' else 0 - return status + result = mysql_query(conn, 'SHOW SLAVE STATUS') + slave_row = result.fetchone() + if slave_row is None: + return {} + + status = { + 'relay_log_space': slave_row['Relay_Log_Space'], + 'slave_lag': slave_row['Seconds_Behind_Master'] if slave_row['Seconds_Behind_Master'] != None else 0, + } + + if MYSQL_CONFIG['HeartbeatTable']: + query = """ + SELECT MAX(UNIX_TIMESTAMP() - UNIX_TIMESTAMP(ts)) AS delay + FROM %s + WHERE server_id = %s + """ % (MYSQL_CONFIG['HeartbeatTable'], slave_row['Master_Server_Id']) + result = mysql_query(conn, query) + row = result.fetchone() + if 'delay' in row and row['delay'] != None: + status['slave_lag'] = row['delay'] + + status['slave_running'] = 1 if slave_row['Slave_SQL_Running'] == 'Yes' and slave_row['Slave_IO_Running'] == 'Yes' else 0 + status['slave_stopped'] = 1 if slave_row['Slave_SQL_Running'] != 'Yes' or slave_row['Slave_IO_Running'] != 'Yes' else 0 + return status def fetch_mysql_db_size(conn): - result = mysql_query(conn, "SELECT table_schema 'db_name', Round(Sum(data_length + index_length) / 1024 / 1024, 0) 'db_size_mb' FROM information_schema.tables WHERE table_schema not in ('mysql', 'information_schema', 'performance_schema', 'heartbeat') GROUP BY table_schema;") + result = mysql_query(conn, "SELECT table_schema 'db_name', Round(Sum(data_length + index_length) / 1024 / 1024, 0) 'db_size_mb' FROM information_schema.tables WHERE table_schema not in ('mysql', 'information_schema', 'performance_schema', 'heartbeat') GROUP BY table_schema;") - stats = {} - for row in result.fetchall(): - stats[row['db_name']] = row['db_size_mb'] - return stats + stats = {} + for row in result.fetchall(): + stats[row['db_name']] = row['db_size_mb'] + return stats def fetch_innodb_os_log_bytes_written(conn): - # This feature is only available for mariaDB >= 10.x and MySQL > 5.5. - try: - result = mysql_query(conn, "SELECT COUNT from INFORMATION_SCHEMA.INNODB_METRICS where name ='os_log_bytes_written';") - stats = result.fetchone() - except MySQLdb.OperationalError: - stats = {'COUNT': 0} - return stats + # This feature is only available for mariaDB >= 10.x and MySQL > 5.5. + try: + result = mysql_query(conn, "SELECT COUNT from INFORMATION_SCHEMA.INNODB_METRICS where name ='os_log_bytes_written';") + stats = result.fetchone() + except MySQLdb.OperationalError: + stats = {'COUNT': 0} + return stats def fetch_mariadb_lock(conn): - # This feature is only available for mariaDB with plugin METADATA_LOCK_INFO installed - try: - result = mysql_query(conn, "SELECT count(1) as `nb_lock` FROM INFORMATION_SCHEMA.PROCESSLIST P, INFORMATION_SCHEMA.METADATA_LOCK_INFO M WHERE LOCATE(lcase(M.LOCK_TYPE), lcase(P.STATE))>0;") - stats = result.fetchone() - except MySQLdb.OperationalError: - stats = {'nb_lock': 0} - return stats + # This feature is only available for mariaDB with plugin METADATA_LOCK_INFO installed + try: + result = mysql_query(conn, "SELECT count(1) as `nb_lock` FROM INFORMATION_SCHEMA.PROCESSLIST P, INFORMATION_SCHEMA.METADATA_LOCK_INFO M WHERE LOCATE(lcase(M.LOCK_TYPE), lcase(P.STATE))>0;") + stats = result.fetchone() + except MySQLdb.OperationalError: + stats = {'nb_lock': 0} + return stats def fetch_mysql_process_states(conn): - global MYSQL_PROCESS_STATES - result = mysql_query(conn, 'SHOW PROCESSLIST') - states = MYSQL_PROCESS_STATES.copy() - for row in result.fetchall(): - state = row['State'] - if state == '' or state == None: state = 'none' - state = re.sub(r'^(Table lock|Waiting for .*lock)$', "Locked", state) - state = state.lower().replace(" ", "_") - if state not in states: state = 'other' - states[state] += 1 - return states + global MYSQL_PROCESS_STATES + result = mysql_query(conn, 'SHOW PROCESSLIST') + states = MYSQL_PROCESS_STATES.copy() + for row in result.fetchall(): + state = row['State'] + if state == '' or state == None: state = 'none' + state = re.sub(r'^(Table lock|Waiting for .*lock)$', "Locked", state) + state = state.lower().replace(" ", "_") + if state not in states: state = 'other' + states[state] += 1 + return states def fetch_mysql_variables(conn): - global MYSQL_VARS - result = mysql_query(conn, 'SHOW GLOBAL VARIABLES') - variables = {} - for row in result.fetchall(): - if row['Variable_name'] in MYSQL_VARS: - if row['Variable_name'] == 'read_only': - if row['Value'] == 'OFF': - variables[row['Variable_name']] = 0 - else: - variables[row['Variable_name']] = 1 - else: - variables[row['Variable_name']] = row['Value'] - - return variables + global MYSQL_VARS + result = mysql_query(conn, 'SHOW GLOBAL VARIABLES') + variables = {} + for row in result.fetchall(): + if row['Variable_name'] in MYSQL_VARS: + if row['Variable_name'] == 'read_only': + if row['Value'] == 'OFF': + variables[row['Variable_name']] = 0 + else: + variables[row['Variable_name']] = 1 + else: + variables[row['Variable_name']] = row['Value'] + + return variables def fetch_mysql_response_times(conn): - response_times = {} - try: - result = mysql_query(conn, """ - SELECT * - FROM INFORMATION_SCHEMA.QUERY_RESPONSE_TIME - WHERE `time` != 'TOO LONG' - ORDER BY `time` - """) - except MySQLdb.OperationalError: - return {} - - for i in range(1, 14): - row = result.fetchone() - - # fill in missing rows with zeros - if not row: - row = { 'count': 0, 'total': 0 } - - row = {key.lower(): val for key, val in row.items()} - - response_times[i] = { - 'time': float(row['time']), - 'count': int(row['count']), - 'total': round(float(row['total']) * 1000000, 0), - } - - return response_times + response_times = {} + try: + result = mysql_query(conn, """ + SELECT * + FROM INFORMATION_SCHEMA.QUERY_RESPONSE_TIME + WHERE `time` != 'TOO LONG' + ORDER BY `time` + """) + except MySQLdb.OperationalError: + return {} + + for i in range(1, 14): + row = result.fetchone() + + # fill in missing rows with zeros + if not row: + row = { 'count': 0, 'total': 0 } + + row = {key.lower(): val for key, val in row.items()} + + response_times[i] = { + 'time': float(row['time']), + 'count': int(row['count']), + 'total': round(float(row['total']) * 1000000, 0), + } + + return response_times def fetch_innodb_stats(conn): - global MYSQL_INNODB_STATUS_MATCHES, MYSQL_INNODB_STATUS_VARS - result = mysql_query(conn, 'SHOW ENGINE INNODB STATUS') - row = result.fetchone() - status = row['Status'] - stats = dict.fromkeys(MYSQL_INNODB_STATUS_VARS.keys(), 0) - - for line in status.split("\n"): - line = line.strip() - row = re.split(r' +', re.sub(r'[,;] ', ' ', line)) - if line == '': continue - - # ---TRANSACTION 124324402462, not started - # ---TRANSACTION 124324402468, ACTIVE 0 sec committing - if line.find("---TRANSACTION") != -1: - stats['current_transactions'] += 1 - if line.find("ACTIVE") != -1: - stats['active_transactions'] += 1 - # LOCK WAIT 228 lock struct(s), heap size 46632, 65 row lock(s), undo log entries 1 - # 205 lock struct(s), heap size 30248, 37 row lock(s), undo log entries 1 - elif line.find("lock struct(s)") != -1: - if line.find("LOCK WAIT") != -1: - stats['innodb_lock_structs'] += int(row[2]) - stats['locked_transactions'] += 1 - else: - stats['innodb_lock_structs'] += int(row[0]) - else: - for match in MYSQL_INNODB_STATUS_MATCHES: - if line.find(match) == -1: continue - for key in MYSQL_INNODB_STATUS_MATCHES[match]: - value = MYSQL_INNODB_STATUS_MATCHES[match][key] - if type(value) is int: - if value < len(row) and row[value].isdigit(): - stats[key] = int(row[value]) - else: - stats[key] = value(row, stats) - break - - return stats + global MYSQL_INNODB_STATUS_MATCHES, MYSQL_INNODB_STATUS_VARS + result = mysql_query(conn, 'SHOW ENGINE INNODB STATUS') + row = result.fetchone() + status = row['Status'] + stats = dict.fromkeys(MYSQL_INNODB_STATUS_VARS.keys(), 0) + + for line in status.split("\n"): + line = line.strip() + row = re.split(r' +', re.sub(r'[,;] ', ' ', line)) + if line == '': continue + + # ---TRANSACTION 124324402462, not started + # ---TRANSACTION 124324402468, ACTIVE 0 sec committing + if line.find("---TRANSACTION") != -1: + stats['current_transactions'] += 1 + if line.find("ACTIVE") != -1: + stats['active_transactions'] += 1 + # LOCK WAIT 228 lock struct(s), heap size 46632, 65 row lock(s), undo log entries 1 + # 205 lock struct(s), heap size 30248, 37 row lock(s), undo log entries 1 + elif line.find("lock struct(s)") != -1: + if line.find("LOCK WAIT") != -1: + stats['innodb_lock_structs'] += int(row[2]) + stats['locked_transactions'] += 1 + else: + stats['innodb_lock_structs'] += int(row[0]) + else: + for match in MYSQL_INNODB_STATUS_MATCHES: + if line.find(match) == -1: continue + for key in MYSQL_INNODB_STATUS_MATCHES[match]: + value = MYSQL_INNODB_STATUS_MATCHES[match][key] + if type(value) is int: + if value < len(row) and row[value].isdigit(): + stats[key] = int(row[value]) + else: + stats[key] = value(row, stats) + break + + return stats def log_verbose(msg): - if not MYSQL_CONFIG['Verbose']: - return - if COLLECTD_ENABLED: - collectd.info('mysql plugin: %s' % msg) - else: - print('mysql plugin: %s' % msg) + if not MYSQL_CONFIG['Verbose']: + return + if COLLECTD_ENABLED: + collectd.info('mysql plugin: %s' % msg) + else: + print('mysql plugin: %s' % msg) def dispatch_value(prefix, key, value, type, type_instance=None): - if not type_instance: - type_instance = key - - log_verbose('Sending value: %s/%s=%s' % (prefix, type_instance, value)) - if value is None: - return - try: - value = int(value) - except ValueError: - value = float(value) - - if COLLECTD_ENABLED: - val = collectd.Values(plugin='mysql', plugin_instance=prefix) - val.type = type - val.type_instance = type_instance - val.values = [value] - val.dispatch() + if not type_instance: + type_instance = key + + log_verbose('Sending value: %s/%s=%s' % (prefix, type_instance, value)) + if value is None: + return + try: + value = int(value) + except ValueError: + value = float(value) + + if COLLECTD_ENABLED: + val = collectd.Values(plugin='mysql', plugin_instance=prefix) + val.type = type + val.type_instance = type_instance + val.values = [value] + val.dispatch() def configure_callback(conf): - global MYSQL_CONFIG - for node in conf.children: - if node.key in MYSQL_CONFIG: - MYSQL_CONFIG[node.key] = node.values[0] + global MYSQL_CONFIG + for node in conf.children: + if node.key in MYSQL_CONFIG: + MYSQL_CONFIG[node.key] = node.values[0] - MYSQL_CONFIG['Port'] = int(MYSQL_CONFIG['Port']) - MYSQL_CONFIG['Verbose'] = bool(MYSQL_CONFIG['Verbose']) + MYSQL_CONFIG['Port'] = int(MYSQL_CONFIG['Port']) + MYSQL_CONFIG['Verbose'] = bool(MYSQL_CONFIG['Verbose']) def read_callback(): - global MYSQL_STATUS_VARS - conn = get_mysql_conn() - - mysql_status = fetch_mysql_status(conn) - for key in mysql_status: - if mysql_status[key] == '': mysql_status[key] = 0 - - # collect anything beginning with Com_/Handler_ as these change - # regularly between mysql versions and this is easier than a fixed - # list - if key.split('_', 2)[0] in ['Com', 'Handler']: - ds_type = 'counter' - elif key in MYSQL_STATUS_VARS: - ds_type = MYSQL_STATUS_VARS[key] - else: - continue - - dispatch_value('status', key, mysql_status[key], ds_type) - - mysql_variables = fetch_mysql_variables(conn) - for key in mysql_variables: - dispatch_value('variables', key, mysql_variables[key], 'gauge') - - mysql_master_status = fetch_mysql_master_stats(conn) - for key in mysql_master_status: - dispatch_value('master', key, mysql_master_status[key], 'gauge') - - mysql_states = fetch_mysql_process_states(conn) - for key in mysql_states: - dispatch_value('state', key, mysql_states[key], 'gauge') - - slave_status = fetch_mysql_slave_stats(conn) - for key in slave_status: - dispatch_value('slave', key, slave_status[key], 'gauge') - - response_times = fetch_mysql_response_times(conn) - for key in response_times: - dispatch_value('response_time_total', str(key), response_times[key]['total'], 'counter') - dispatch_value('response_time_count', str(key), response_times[key]['count'], 'counter') + global MYSQL_STATUS_VARS + conn = get_mysql_conn() + + mysql_status = fetch_mysql_status(conn) + for key in mysql_status: + if mysql_status[key] == '': mysql_status[key] = 0 + # collect anything beginning with Com_/Handler_ as these change + # regularly between mysql versions and this is easier than a fixed + # list + if key.split('_', 2)[0] in ['Com', 'Handler']: + ds_type = 'counter' + elif key in MYSQL_STATUS_VARS: + ds_type = MYSQL_STATUS_VARS[key] + else: + continue + + dispatch_value('status', key, mysql_status[key], ds_type) + + mysql_variables = fetch_mysql_variables(conn) + for key in mysql_variables: + dispatch_value('variables', key, mysql_variables[key], 'gauge') + + mysql_master_status = fetch_mysql_master_stats(conn) + for key in mysql_master_status: + dispatch_value('master', key, mysql_master_status[key], 'gauge') + + mysql_states = fetch_mysql_process_states(conn) + for key in mysql_states: + dispatch_value('state', key, mysql_states[key], 'gauge') + + slave_status = fetch_mysql_slave_stats(conn) + for key in slave_status: + dispatch_value('slave', key, slave_status[key], 'gauge') + + response_times = fetch_mysql_response_times(conn) + for key in response_times: + dispatch_value('response_time_total', str(key), response_times[key]['total'], 'counter') + dispatch_value('response_time_count', str(key), response_times[key]['count'], 'counter') - mysql_db_size = fetch_mysql_db_size(conn) - for key in mysql_db_size: - dispatch_value('db_size', key, mysql_db_size[key], 'gauge') + mysql_db_size = fetch_mysql_db_size(conn) + for key in mysql_db_size: + dispatch_value('db_size', key, mysql_db_size[key], 'gauge') - innodb_log_bytes_written = fetch_innodb_os_log_bytes_written(conn) - dispatch_value('innodb', 'os_log_bytes_written', innodb_log_bytes_written['COUNT'], 'counter') + innodb_log_bytes_written = fetch_innodb_os_log_bytes_written(conn) + dispatch_value('innodb', 'os_log_bytes_written', innodb_log_bytes_written['COUNT'], 'counter') - meta_data_lock = fetch_mariadb_lock(conn) - dispatch_value('innodb', 'lock', meta_data_lock['nb_lock'], 'gauge') + meta_data_lock = fetch_mariadb_lock(conn) + dispatch_value('innodb', 'lock', meta_data_lock['nb_lock'], 'gauge') - innodb_status = fetch_innodb_stats(conn) - for key in MYSQL_INNODB_STATUS_VARS: - if key not in innodb_status: continue - dispatch_value('innodb', key, innodb_status[key], MYSQL_INNODB_STATUS_VARS[key]) + innodb_status = fetch_innodb_stats(conn) + for key in MYSQL_INNODB_STATUS_VARS: + if key not in innodb_status: continue + dispatch_value('innodb', key, innodb_status[key], MYSQL_INNODB_STATUS_VARS[key]) if COLLECTD_ENABLED: - collectd.register_read(read_callback) - collectd.register_config(configure_callback) + collectd.register_read(read_callback) + collectd.register_config(configure_callback) if __name__ == "__main__" and not COLLECTD_ENABLED: - print("Running in test mode, invoke with") - print(sys.argv[0] + " Host Port User Password ") - MYSQL_CONFIG['Host'] = sys.argv[1] - MYSQL_CONFIG['Port'] = int(sys.argv[2]) - MYSQL_CONFIG['User'] = sys.argv[3] - MYSQL_CONFIG['Password'] = sys.argv[4] - MYSQL_CONFIG['Verbose'] = True - from pprint import pprint as pp - pp(MYSQL_CONFIG) - read_callback() - - -# vim:noexpandtab ts=8 sw=8 sts=8 + print("Running in test mode, invoke with") + print(sys.argv[0] + " Host Port User Password ") + MYSQL_CONFIG['Host'] = sys.argv[1] + MYSQL_CONFIG['Port'] = int(sys.argv[2]) + MYSQL_CONFIG['User'] = sys.argv[3] + MYSQL_CONFIG['Password'] = sys.argv[4] + MYSQL_CONFIG['Verbose'] = True + from pprint import pprint as pp + pp(MYSQL_CONFIG) + read_callback() + +# vim:noexpandtab ts=4 sw=4 sts=4 From 755a991b0653b940ce61af7d8061390f71a9b2fc Mon Sep 17 00:00:00 2001 From: Bastien Vallet Date: Thu, 16 Apr 2020 21:23:24 +0200 Subject: [PATCH 14/24] Fix tab/space mix --- mysql.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mysql.py b/mysql.py index fae9568..d1f4a58 100644 --- a/mysql.py +++ b/mysql.py @@ -475,7 +475,7 @@ def fetch_mysql_response_times(conn): # fill in missing rows with zeros if not row: - row = { 'count': 0, 'total': 0 } + row = { 'count': 0, 'total': 0 } row = {key.lower(): val for key, val in row.items()} @@ -548,11 +548,11 @@ def dispatch_value(prefix, key, value, type, type_instance=None): value = float(value) if COLLECTD_ENABLED: - val = collectd.Values(plugin='mysql', plugin_instance=prefix) - val.type = type - val.type_instance = type_instance - val.values = [value] - val.dispatch() + val = collectd.Values(plugin='mysql', plugin_instance=prefix) + val.type = type + val.type_instance = type_instance + val.values = [value] + val.dispatch() def configure_callback(conf): global MYSQL_CONFIG @@ -602,7 +602,7 @@ def read_callback(): for key in response_times: dispatch_value('response_time_total', str(key), response_times[key]['total'], 'counter') dispatch_value('response_time_count', str(key), response_times[key]['count'], 'counter') - + mysql_db_size = fetch_mysql_db_size(conn) for key in mysql_db_size: dispatch_value('db_size', key, mysql_db_size[key], 'gauge') From 277896b244a93f85b878ad9404fc9f10b6698867 Mon Sep 17 00:00:00 2001 From: Bastien Vallet Date: Thu, 16 Apr 2020 21:45:45 +0200 Subject: [PATCH 15/24] [lint] Fix coding style - Line too long - Invalid whitespace - Bad import order --- mysql.py | 77 +++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 20 deletions(-) diff --git a/mysql.py b/mysql.py index d1f4a58..69e6cbe 100644 --- a/mysql.py +++ b/mysql.py @@ -24,16 +24,17 @@ # import sys +import re + +import MySQLdb -COLLECTD_ENABLED=True +COLLECTD_ENABLED = True try: import collectd except ImportError: # We're not running in CollectD, set this to False so we can make some changes # accordingly for testing/development. - COLLECTD_ENABLED=False -import re -import MySQLdb + COLLECTD_ENABLED = False MYSQL_CONFIG = { 'Host': 'localhost', @@ -263,7 +264,7 @@ 'pending_buf_pool_flushes': 'gauge', 'pending_chkp_writes': 'gauge', 'pending_ibuf_aio_reads': 'gauge', - 'pending_log_writes':'gauge', + 'pending_log_writes': 'gauge', 'queries_inside': 'gauge', 'queries_queued': 'gauge', 'read_views': 'gauge', @@ -330,6 +331,7 @@ }, } + def get_mysql_conn(): return MySQLdb.connect( host=MYSQL_CONFIG['Host'], @@ -338,11 +340,13 @@ def get_mysql_conn(): passwd=MYSQL_CONFIG['Password'] ) + def mysql_query(conn, query): cur = conn.cursor(MySQLdb.cursors.DictCursor) cur.execute(query) return cur + def fetch_mysql_status(conn): result = mysql_query(conn, 'SHOW GLOBAL STATUS') status = {} @@ -354,13 +358,14 @@ def fetch_mysql_status(conn): status['Innodb_unpurged_txns'] = int(status['Innodb_max_trx_id']) - int(status['Innodb_purge_trx_id']) if 'Innodb_lsn_last_checkpoint' in status: - status['Innodb_uncheckpointed_bytes'] = int(status['Innodb_lsn_current'])- int(status['Innodb_lsn_last_checkpoint']) + status['Innodb_uncheckpointed_bytes'] = int(status['Innodb_lsn_current']) - int(status['Innodb_lsn_last_checkpoint']) if 'Innodb_lsn_flushed' in status: status['Innodb_unflushed_log'] = int(status['Innodb_lsn_current']) - int(status['Innodb_lsn_flushed']) return status + def fetch_mysql_master_stats(conn): try: result = mysql_query(conn, 'SHOW BINARY LOGS') @@ -377,6 +382,7 @@ def fetch_mysql_master_stats(conn): return stats + def fetch_mysql_slave_stats(conn): result = mysql_query(conn, 'SHOW SLAVE STATUS') slave_row = result.fetchone() @@ -385,7 +391,7 @@ def fetch_mysql_slave_stats(conn): status = { 'relay_log_space': slave_row['Relay_Log_Space'], - 'slave_lag': slave_row['Seconds_Behind_Master'] if slave_row['Seconds_Behind_Master'] != None else 0, + 'slave_lag': slave_row['Seconds_Behind_Master'] if slave_row['Seconds_Behind_Master'] is not None else 0, } if MYSQL_CONFIG['HeartbeatTable']: @@ -396,52 +402,74 @@ def fetch_mysql_slave_stats(conn): """ % (MYSQL_CONFIG['HeartbeatTable'], slave_row['Master_Server_Id']) result = mysql_query(conn, query) row = result.fetchone() - if 'delay' in row and row['delay'] != None: + if 'delay' in row and row['delay'] is not None: status['slave_lag'] = row['delay'] status['slave_running'] = 1 if slave_row['Slave_SQL_Running'] == 'Yes' and slave_row['Slave_IO_Running'] == 'Yes' else 0 status['slave_stopped'] = 1 if slave_row['Slave_SQL_Running'] != 'Yes' or slave_row['Slave_IO_Running'] != 'Yes' else 0 return status + def fetch_mysql_db_size(conn): - result = mysql_query(conn, "SELECT table_schema 'db_name', Round(Sum(data_length + index_length) / 1024 / 1024, 0) 'db_size_mb' FROM information_schema.tables WHERE table_schema not in ('mysql', 'information_schema', 'performance_schema', 'heartbeat') GROUP BY table_schema;") + result = mysql_query( + conn, + "SELECT table_schema 'db_name', Round(Sum(data_length + index_length) / 1024 / 1024, 0) 'db_size_mb' " + "FROM information_schema.tables " + "WHERE table_schema not in ('mysql', 'information_schema', 'performance_schema', 'heartbeat') " + "GROUP BY table_schema;" + ) stats = {} for row in result.fetchall(): stats[row['db_name']] = row['db_size_mb'] return stats + def fetch_innodb_os_log_bytes_written(conn): # This feature is only available for mariaDB >= 10.x and MySQL > 5.5. try: - result = mysql_query(conn, "SELECT COUNT from INFORMATION_SCHEMA.INNODB_METRICS where name ='os_log_bytes_written';") + result = mysql_query( + conn, + "SELECT COUNT from INFORMATION_SCHEMA.INNODB_METRICS " + "WHERE name ='os_log_bytes_written';" + ) stats = result.fetchone() except MySQLdb.OperationalError: stats = {'COUNT': 0} return stats + def fetch_mariadb_lock(conn): # This feature is only available for mariaDB with plugin METADATA_LOCK_INFO installed try: - result = mysql_query(conn, "SELECT count(1) as `nb_lock` FROM INFORMATION_SCHEMA.PROCESSLIST P, INFORMATION_SCHEMA.METADATA_LOCK_INFO M WHERE LOCATE(lcase(M.LOCK_TYPE), lcase(P.STATE))>0;") + result = mysql_query( + conn, + "SELECT count(1) as `nb_lock` " + "FROM INFORMATION_SCHEMA.PROCESSLIST P, INFORMATION_SCHEMA.METADATA_LOCK_INFO M " + "WHERE LOCATE(lcase(M.LOCK_TYPE), lcase(P.STATE))>0;" + ) stats = result.fetchone() except MySQLdb.OperationalError: stats = {'nb_lock': 0} return stats + def fetch_mysql_process_states(conn): global MYSQL_PROCESS_STATES result = mysql_query(conn, 'SHOW PROCESSLIST') states = MYSQL_PROCESS_STATES.copy() for row in result.fetchall(): state = row['State'] - if state == '' or state == None: state = 'none' + if state == '' or state is None: + state = 'none' state = re.sub(r'^(Table lock|Waiting for .*lock)$', "Locked", state) state = state.lower().replace(" ", "_") - if state not in states: state = 'other' + if state not in states: + state = 'other' states[state] += 1 return states + def fetch_mysql_variables(conn): global MYSQL_VARS result = mysql_query(conn, 'SHOW GLOBAL VARIABLES') @@ -458,6 +486,7 @@ def fetch_mysql_variables(conn): return variables + def fetch_mysql_response_times(conn): response_times = {} try: @@ -475,7 +504,7 @@ def fetch_mysql_response_times(conn): # fill in missing rows with zeros if not row: - row = { 'count': 0, 'total': 0 } + row = {'count': 0, 'total': 0} row = {key.lower(): val for key, val in row.items()} @@ -487,6 +516,7 @@ def fetch_mysql_response_times(conn): return response_times + def fetch_innodb_stats(conn): global MYSQL_INNODB_STATUS_MATCHES, MYSQL_INNODB_STATUS_VARS result = mysql_query(conn, 'SHOW ENGINE INNODB STATUS') @@ -497,7 +527,8 @@ def fetch_innodb_stats(conn): for line in status.split("\n"): line = line.strip() row = re.split(r' +', re.sub(r'[,;] ', ' ', line)) - if line == '': continue + if line == '': + continue # ---TRANSACTION 124324402462, not started # ---TRANSACTION 124324402468, ACTIVE 0 sec committing @@ -515,7 +546,8 @@ def fetch_innodb_stats(conn): stats['innodb_lock_structs'] += int(row[0]) else: for match in MYSQL_INNODB_STATUS_MATCHES: - if line.find(match) == -1: continue + if line.find(match) == -1: + continue for key in MYSQL_INNODB_STATUS_MATCHES[match]: value = MYSQL_INNODB_STATUS_MATCHES[match][key] if type(value) is int: @@ -527,6 +559,7 @@ def fetch_innodb_stats(conn): return stats + def log_verbose(msg): if not MYSQL_CONFIG['Verbose']: return @@ -535,6 +568,7 @@ def log_verbose(msg): else: print('mysql plugin: %s' % msg) + def dispatch_value(prefix, key, value, type, type_instance=None): if not type_instance: type_instance = key @@ -554,6 +588,7 @@ def dispatch_value(prefix, key, value, type, type_instance=None): val.values = [value] val.dispatch() + def configure_callback(conf): global MYSQL_CONFIG for node in conf.children: @@ -563,13 +598,15 @@ def configure_callback(conf): MYSQL_CONFIG['Port'] = int(MYSQL_CONFIG['Port']) MYSQL_CONFIG['Verbose'] = bool(MYSQL_CONFIG['Verbose']) + def read_callback(): global MYSQL_STATUS_VARS conn = get_mysql_conn() mysql_status = fetch_mysql_status(conn) for key in mysql_status: - if mysql_status[key] == '': mysql_status[key] = 0 + if mysql_status[key] == '': + mysql_status[key] = 0 # collect anything beginning with Com_/Handler_ as these change # regularly between mysql versions and this is easier than a fixed # list @@ -610,16 +647,16 @@ def read_callback(): innodb_log_bytes_written = fetch_innodb_os_log_bytes_written(conn) dispatch_value('innodb', 'os_log_bytes_written', innodb_log_bytes_written['COUNT'], 'counter') - meta_data_lock = fetch_mariadb_lock(conn) dispatch_value('innodb', 'lock', meta_data_lock['nb_lock'], 'gauge') - innodb_status = fetch_innodb_stats(conn) for key in MYSQL_INNODB_STATUS_VARS: - if key not in innodb_status: continue + if key not in innodb_status: + continue dispatch_value('innodb', key, innodb_status[key], MYSQL_INNODB_STATUS_VARS[key]) + if COLLECTD_ENABLED: collectd.register_read(read_callback) collectd.register_config(configure_callback) From 466d1957c0f2c76169c617f688d4277813ead0dd Mon Sep 17 00:00:00 2001 From: Bastien Vallet Date: Fri, 17 Apr 2020 10:37:03 +0200 Subject: [PATCH 16/24] [lint] Cleanup whitespaces --- mysql.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mysql.py b/mysql.py index 69e6cbe..b5ca945 100644 --- a/mysql.py +++ b/mysql.py @@ -37,12 +37,12 @@ COLLECTD_ENABLED = False MYSQL_CONFIG = { - 'Host': 'localhost', - 'Port': 3306, - 'User': 'root', - 'Password': '', + 'Host': 'localhost', + 'Port': 3306, + 'User': 'root', + 'Password': '', 'HeartbeatTable': '', - 'Verbose': False, + 'Verbose': False, } MYSQL_STATUS_VARS = { From 5e260ac25bb4aea2eb85d3234025aee0fd3f0669 Mon Sep 17 00:00:00 2001 From: Charles JUDITH Date: Fri, 17 Apr 2020 15:34:19 +0200 Subject: [PATCH 17/24] Replace MySQLdb by pymysql (#13) Replace MySQLdb by pymysql. Co-authored-by: Charles JUDITH --- README.md | 3 ++- mysql.py | 39 ++++++++++++++++++++++++++------------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index da6d306..ea6942b 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Pulls most of the same metrics as the Percona Monitoring Plugins for Cacti. Coll Most MySQL monitoring plugins fetch a lot of information by parsing the output of `SHOW ENGINE INNODB STATUS`. This plugin prefers InnoDB statistics from `SHOW GLOBAL STATUS`. Percona Server and MariaDB provide most of these InnoDB metrics on `SHOW GLOBAL STATUS`. -Requires the Python MySQLdb package. (`python-mysqldb` on Debian) +Requires the Python `pymysql` package. ## Installation 1. Place mysql.py in your CollectD python plugins directory @@ -187,6 +187,7 @@ Collected from `SHOW VARIABLES`: variables.thread_cache_size variables.thread_concurrency variables.tmp_table_size + variables.read_only ### MySQL Processes diff --git a/mysql.py b/mysql.py index b5ca945..abb47e1 100644 --- a/mysql.py +++ b/mysql.py @@ -17,7 +17,7 @@ # Verbose true (optional, to enable debugging) # # -# Requires "MySQLdb" for Python +# Requires "pymysql" for Python # # Author: Chris Boulton # License: MIT (http://www.opensource.org/licenses/mit-license.php) @@ -26,7 +26,7 @@ import sys import re -import MySQLdb +import pymysql COLLECTD_ENABLED = True try: @@ -333,7 +333,7 @@ def get_mysql_conn(): - return MySQLdb.connect( + return pymysql.connect( host=MYSQL_CONFIG['Host'], port=MYSQL_CONFIG['Port'], user=MYSQL_CONFIG['User'], @@ -342,7 +342,7 @@ def get_mysql_conn(): def mysql_query(conn, query): - cur = conn.cursor(MySQLdb.cursors.DictCursor) + cur = conn.cursor(pymysql.cursors.DictCursor) cur.execute(query) return cur @@ -369,7 +369,7 @@ def fetch_mysql_status(conn): def fetch_mysql_master_stats(conn): try: result = mysql_query(conn, 'SHOW BINARY LOGS') - except MySQLdb.OperationalError: + except pymysql.OperationalError: return {} stats = { @@ -434,7 +434,7 @@ def fetch_innodb_os_log_bytes_written(conn): "WHERE name ='os_log_bytes_written';" ) stats = result.fetchone() - except MySQLdb.OperationalError: + except pymysql.OperationalError: stats = {'COUNT': 0} return stats @@ -449,7 +449,7 @@ def fetch_mariadb_lock(conn): "WHERE LOCATE(lcase(M.LOCK_TYPE), lcase(P.STATE))>0;" ) stats = result.fetchone() - except MySQLdb.OperationalError: + except pymysql.OperationalError: stats = {'nb_lock': 0} return stats @@ -496,7 +496,7 @@ def fetch_mysql_response_times(conn): WHERE `time` != 'TOO LONG' ORDER BY `time` """) - except MySQLdb.OperationalError: + except pymysql.OperationalError: return {} for i in range(1, 14): @@ -559,6 +559,13 @@ def fetch_innodb_stats(conn): return stats +def get_mysql_version(conn): + try: + result = mysql_query(conn, "SELECT VERSION()") + mysql_version = result.fetchone() + except pymysql.OperationalError: + return {} + return mysql_version['VERSION()'] def log_verbose(msg): if not MYSQL_CONFIG['Verbose']: @@ -635,11 +642,17 @@ def read_callback(): for key in slave_status: dispatch_value('slave', key, slave_status[key], 'gauge') - response_times = fetch_mysql_response_times(conn) - for key in response_times: - dispatch_value('response_time_total', str(key), response_times[key]['total'], 'counter') - dispatch_value('response_time_count', str(key), response_times[key]['count'], 'counter') - + # This is only available in Percona Server and some MySQL versions but not in MariaDB + # https://www.percona.com/blog/2010/07/11/query-response-time-histogram-new-feature-in-percona-server/ + version = get_mysql_version(conn) + if version.startswith('10.'): + pass + else: + response_times = fetch_mysql_response_times(conn) + for key in response_times: + dispatch_value('response_time_total', str(key), response_times[key]['total'], 'counter') + dispatch_value('response_time_count', str(key), response_times[key]['count'], 'counter') + mysql_db_size = fetch_mysql_db_size(conn) for key in mysql_db_size: dispatch_value('db_size', key, mysql_db_size[key], 'gauge') From 80a4ea79323ee361be24ce442377a1f836f0a030 Mon Sep 17 00:00:00 2001 From: Charles JUDITH Date: Fri, 17 Apr 2020 17:18:58 +0200 Subject: [PATCH 18/24] Improve management of the on/off values in mysql vars (#14) Improve the management of the ON/OFF values in fetch_mysql_variables. This is currently applicable for MYSQL variables only. For example: read_only=OFF will be read_only=0 The previous review on that was only for the read_only variable, this will add it for all other variables. Co-authored-by: Charles JUDITH --- mysql.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/mysql.py b/mysql.py index abb47e1..838f65a 100644 --- a/mysql.py +++ b/mysql.py @@ -476,12 +476,6 @@ def fetch_mysql_variables(conn): variables = {} for row in result.fetchall(): if row['Variable_name'] in MYSQL_VARS: - if row['Variable_name'] == 'read_only': - if row['Value'] == 'OFF': - variables[row['Variable_name']] = 0 - else: - variables[row['Variable_name']] = 1 - else: variables[row['Variable_name']] = row['Value'] return variables @@ -628,6 +622,11 @@ def read_callback(): mysql_variables = fetch_mysql_variables(conn) for key in mysql_variables: + if mysql_variables[key] == 'ON': + mysql_variables[key] = 1 + elif mysql_variables[key] == 'OFF': + mysql_variables[key] = 0 + dispatch_value('variables', key, mysql_variables[key], 'gauge') mysql_master_status = fetch_mysql_master_stats(conn) From 33b44437bb3c52fb4464d2a5e9eef4ead37e71d7 Mon Sep 17 00:00:00 2001 From: Charles JUDITH Date: Wed, 22 Apr 2020 15:07:44 +0200 Subject: [PATCH 19/24] Add number of current connections per user (#15) This change will add: - The number of current connections per user - Apply linter Co-authored-by: Charles JUDITH --- README.md | 7 ++++ mysql.py | 115 +++++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 90 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index ea6942b..622fff4 100644 --- a/README.md +++ b/README.md @@ -335,6 +335,13 @@ From `SHOW SLAVE STATUS`: slave.slave_stopped - 1 when the slave is stopped, 0 when it's running slave.slave_running - 1 when the slave is running, 0 when it's stopped +### Current connections for users + +This is using performance_schema. If performance_schema is not enabled, it won't work. + + connection_per_user.current_connections_root + connection_per_user.current_connections_repl + ### Query Response Times For versions of MySQL with support for it and where enabled, `INFORMATION_SCHEMA.QUERY_RESPONSE_TIME` will be queried for metrics to generate a histogram of query response times. diff --git a/mysql.py b/mysql.py index 838f65a..3b1a3ba 100644 --- a/mysql.py +++ b/mysql.py @@ -1,27 +1,28 @@ -#!/usr/bin/env python -# CollectD MySQL plugin, designed for MySQL 5.5+ (specifically Percona Server) -# -# Pulls most of the same metrics as the Percona Monitoring Plugins for Cacti, -# however is designed to be used on newer versions of MySQL and as such drops -# all of the legacy compatibility, and favors metrics from SHOW GLOBAL STATUS -# as opposed to SHOW ENGINE INNODB STATUS where possible. -# -# Configuration: -# Import mysql -# -# Host localhost -# Port 3306 (optional) -# User root -# Password xxxx -# HeartbeatTable percona.heartbeat (optional, if using pt-heartbeat) -# Verbose true (optional, to enable debugging) -# -# -# Requires "pymysql" for Python -# -# Author: Chris Boulton -# License: MIT (http://www.opensource.org/licenses/mit-license.php) -# +#!/usr/bin/env python3 +""" +CollectD MySQL plugin, designed for MySQL 5.5+ (specifically Percona Server) + +Pulls most of the same metrics as the Percona Monitoring Plugins for Cacti, +however is designed to be used on newer versions of MySQL and as such drops +all of the legacy compatibility, and favors metrics from SHOW GLOBAL STATUS +as opposed to SHOW ENGINE INNODB STATUS where possible. + +Configuration: +Import mysql + + Host localhost + Port 3306 (optional) + User root + Password xxxx + HeartbeatTable percona.heartbeat (optional, if using pt-heartbeat) + Verbose true (optional, to enable debugging) + + +Requires "pymysql" for Python + +Author: Chris Boulton +License: MIT (http://www.opensource.org/licenses/mit-license.php) +""" import sys import re @@ -305,7 +306,7 @@ }, # File system 657820264 (812272 + 657007992) 'File system ': { - 'file_system_memory': 2, + 'file_system_memory': 2, }, # Lock system 143820296 (143819576 + 720) 'Lock system ': { @@ -333,6 +334,7 @@ def get_mysql_conn(): + """ Get connection parameters """ return pymysql.connect( host=MYSQL_CONFIG['Host'], port=MYSQL_CONFIG['Port'], @@ -342,12 +344,14 @@ def get_mysql_conn(): def mysql_query(conn, query): + """ Function to run MySQL queries """ cur = conn.cursor(pymysql.cursors.DictCursor) cur.execute(query) return cur def fetch_mysql_status(conn): + """ Fetch global status variables """ result = mysql_query(conn, 'SHOW GLOBAL STATUS') status = {} for row in result.fetchall(): @@ -367,6 +371,7 @@ def fetch_mysql_status(conn): def fetch_mysql_master_stats(conn): + """ Fetch master information """ try: result = mysql_query(conn, 'SHOW BINARY LOGS') except pymysql.OperationalError: @@ -384,6 +389,7 @@ def fetch_mysql_master_stats(conn): def fetch_mysql_slave_stats(conn): + """ Fetch slave status """ result = mysql_query(conn, 'SHOW SLAVE STATUS') slave_row = result.fetchone() if slave_row is None: @@ -411,6 +417,7 @@ def fetch_mysql_slave_stats(conn): def fetch_mysql_db_size(conn): + """ Fetch database size """ result = mysql_query( conn, "SELECT table_schema 'db_name', Round(Sum(data_length + index_length) / 1024 / 1024, 0) 'db_size_mb' " @@ -426,6 +433,7 @@ def fetch_mysql_db_size(conn): def fetch_innodb_os_log_bytes_written(conn): + """ Fetch innodb metric os_log_bytes_written """ # This feature is only available for mariaDB >= 10.x and MySQL > 5.5. try: result = mysql_query( @@ -439,7 +447,8 @@ def fetch_innodb_os_log_bytes_written(conn): return stats -def fetch_mariadb_lock(conn): +def fetch_mysql_lock(conn): + """ Fetch MySQL lock """ # This feature is only available for mariaDB with plugin METADATA_LOCK_INFO installed try: result = mysql_query( @@ -455,6 +464,7 @@ def fetch_mariadb_lock(conn): def fetch_mysql_process_states(conn): + """ Fetch process states """ global MYSQL_PROCESS_STATES result = mysql_query(conn, 'SHOW PROCESSLIST') states = MYSQL_PROCESS_STATES.copy() @@ -471,17 +481,46 @@ def fetch_mysql_process_states(conn): def fetch_mysql_variables(conn): + """ Fetch defined MySQL variables """ global MYSQL_VARS result = mysql_query(conn, 'SHOW GLOBAL VARIABLES') variables = {} for row in result.fetchall(): if row['Variable_name'] in MYSQL_VARS: - variables[row['Variable_name']] = row['Value'] + variables[row['Variable_name']] = row['Value'] return variables +def is_ps_enabled(conn): + """ Check is performance_schema is enabled """ + result = mysql_query(conn, 'SHOW GLOBAL VARIABLES LIKE "performance_schema"') + row = result.fetchone() + return bool(row['Value'] == 'ON') + + +def fetch_connections_per_account(conn): + """ Fetch number of connections per account """ + queries = {} + try: + result = mysql_query(conn, """ + SELECT user, sum(current_connections) as `current_connections` + FROM performance_schema.accounts + WHERE user is not null + GROUP BY user; + """) + for row in result.fetchall(): + user = str(row['user']) + queries["current_connections_"+user] = row['current_connections'] + + except pymysql.OperationalError: + return {} + + return queries + + def fetch_mysql_response_times(conn): + """ Fetch mysql response time from percona plugin """ response_times = {} try: result = mysql_query(conn, """ @@ -512,6 +551,7 @@ def fetch_mysql_response_times(conn): def fetch_innodb_stats(conn): + """ Fetch innodb statistics """ global MYSQL_INNODB_STATUS_MATCHES, MYSQL_INNODB_STATUS_VARS result = mysql_query(conn, 'SHOW ENGINE INNODB STATUS') row = result.fetchone() @@ -544,7 +584,7 @@ def fetch_innodb_stats(conn): continue for key in MYSQL_INNODB_STATUS_MATCHES[match]: value = MYSQL_INNODB_STATUS_MATCHES[match][key] - if type(value) is int: + if isinstance(value, int): if value < len(row) and row[value].isdigit(): stats[key] = int(row[value]) else: @@ -554,6 +594,7 @@ def fetch_innodb_stats(conn): return stats def get_mysql_version(conn): + """ Get MySQL version """ try: result = mysql_query(conn, "SELECT VERSION()") mysql_version = result.fetchone() @@ -561,7 +602,9 @@ def get_mysql_version(conn): return {} return mysql_version['VERSION()'] + def log_verbose(msg): + """ To enable verbose mode """ if not MYSQL_CONFIG['Verbose']: return if COLLECTD_ENABLED: @@ -571,6 +614,7 @@ def log_verbose(msg): def dispatch_value(prefix, key, value, type, type_instance=None): + """ Dispatch metrics """ if not type_instance: type_instance = key @@ -591,6 +635,7 @@ def dispatch_value(prefix, key, value, type, type_instance=None): def configure_callback(conf): + """ Config callback """ global MYSQL_CONFIG for node in conf.children: if node.key in MYSQL_CONFIG: @@ -601,6 +646,7 @@ def configure_callback(conf): def read_callback(): + """ Everything is happenning here """ global MYSQL_STATUS_VARS conn = get_mysql_conn() @@ -641,6 +687,11 @@ def read_callback(): for key in slave_status: dispatch_value('slave', key, slave_status[key], 'gauge') + if is_ps_enabled(conn) is True: + user_connections = fetch_connections_per_account(conn) + for key in user_connections: + dispatch_value('connection_per_user', key, user_connections[key], 'gauge') + # This is only available in Percona Server and some MySQL versions but not in MariaDB # https://www.percona.com/blog/2010/07/11/query-response-time-histogram-new-feature-in-percona-server/ version = get_mysql_version(conn) @@ -651,7 +702,7 @@ def read_callback(): for key in response_times: dispatch_value('response_time_total', str(key), response_times[key]['total'], 'counter') dispatch_value('response_time_count', str(key), response_times[key]['count'], 'counter') - + mysql_db_size = fetch_mysql_db_size(conn) for key in mysql_db_size: dispatch_value('db_size', key, mysql_db_size[key], 'gauge') @@ -659,7 +710,7 @@ def read_callback(): innodb_log_bytes_written = fetch_innodb_os_log_bytes_written(conn) dispatch_value('innodb', 'os_log_bytes_written', innodb_log_bytes_written['COUNT'], 'counter') - meta_data_lock = fetch_mariadb_lock(conn) + meta_data_lock = fetch_mysql_lock(conn) dispatch_value('innodb', 'lock', meta_data_lock['nb_lock'], 'gauge') innodb_status = fetch_innodb_stats(conn) @@ -674,8 +725,8 @@ def read_callback(): collectd.register_config(configure_callback) if __name__ == "__main__" and not COLLECTD_ENABLED: - print("Running in test mode, invoke with") - print(sys.argv[0] + " Host Port User Password ") + print('Running in test mode, invoke with') + print(sys.argv[0] + ' Host Port User Password ') MYSQL_CONFIG['Host'] = sys.argv[1] MYSQL_CONFIG['Port'] = int(sys.argv[2]) MYSQL_CONFIG['User'] = sys.argv[3] From 0edcad84dcc7c578f5f2105de9255b06ee5218fc Mon Sep 17 00:00:00 2001 From: Charles JUDITH Date: Mon, 27 Apr 2020 13:51:18 +0200 Subject: [PATCH 20/24] Small changes to prepare the code for the tests Prepare the code to be tested with flake8. --- .gitignore | 1 + mysql.py | 29 +++++++++++++++-------------- 2 files changed, 16 insertions(+), 14 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..598ae79 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/.vscode diff --git a/mysql.py b/mysql.py index 3b1a3ba..51107eb 100644 --- a/mysql.py +++ b/mysql.py @@ -390,7 +390,7 @@ def fetch_mysql_master_stats(conn): def fetch_mysql_slave_stats(conn): """ Fetch slave status """ - result = mysql_query(conn, 'SHOW SLAVE STATUS') + result = mysql_query(conn, 'SHOW SLAVE STATUS') slave_row = result.fetchone() if slave_row is None: return {} @@ -407,7 +407,7 @@ def fetch_mysql_slave_stats(conn): WHERE server_id = %s """ % (MYSQL_CONFIG['HeartbeatTable'], slave_row['Master_Server_Id']) result = mysql_query(conn, query) - row = result.fetchone() + row = result.fetchone() if 'delay' in row and row['delay'] is not None: status['slave_lag'] = row['delay'] @@ -504,11 +504,11 @@ def fetch_connections_per_account(conn): queries = {} try: result = mysql_query(conn, """ - SELECT user, sum(current_connections) as `current_connections` - FROM performance_schema.accounts - WHERE user is not null - GROUP BY user; - """) + SELECT user, sum(current_connections) as `current_connections` + FROM performance_schema.accounts + WHERE user is not null + GROUP BY user; + """) for row in result.fetchall(): user = str(row['user']) queries["current_connections_"+user] = row['current_connections'] @@ -554,13 +554,13 @@ def fetch_innodb_stats(conn): """ Fetch innodb statistics """ global MYSQL_INNODB_STATUS_MATCHES, MYSQL_INNODB_STATUS_VARS result = mysql_query(conn, 'SHOW ENGINE INNODB STATUS') - row = result.fetchone() + row = result.fetchone() status = row['Status'] - stats = dict.fromkeys(MYSQL_INNODB_STATUS_VARS.keys(), 0) + stats = dict.fromkeys(MYSQL_INNODB_STATUS_VARS.keys(), 0) for line in status.split("\n"): line = line.strip() - row = re.split(r' +', re.sub(r'[,;] ', ' ', line)) + row = re.split(r' +', re.sub(r'[,;] ', ' ', line)) if line == '': continue @@ -593,6 +593,7 @@ def fetch_innodb_stats(conn): return stats + def get_mysql_version(conn): """ Get MySQL version """ try: @@ -627,10 +628,10 @@ def dispatch_value(prefix, key, value, type, type_instance=None): value = float(value) if COLLECTD_ENABLED: - val = collectd.Values(plugin='mysql', plugin_instance=prefix) - val.type = type + val = collectd.Values(plugin='mysql', plugin_instance=prefix) + val.type = type val.type_instance = type_instance - val.values = [value] + val.values = [value] val.dispatch() @@ -641,7 +642,7 @@ def configure_callback(conf): if node.key in MYSQL_CONFIG: MYSQL_CONFIG[node.key] = node.values[0] - MYSQL_CONFIG['Port'] = int(MYSQL_CONFIG['Port']) + MYSQL_CONFIG['Port'] = int(MYSQL_CONFIG['Port']) MYSQL_CONFIG['Verbose'] = bool(MYSQL_CONFIG['Verbose']) From 7045b1e052991355e2c77e2f857025335ef7de90 Mon Sep 17 00:00:00 2001 From: Charles JUDITH Date: Mon, 27 Apr 2020 15:56:49 +0200 Subject: [PATCH 21/24] Add python tests with Github Actions Add python tests with Github Actions --- .github/workflows/pythonapp.yml | 37 +++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/pythonapp.yml diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml new file mode 100644 index 0000000..e1cafe7 --- /dev/null +++ b/.github/workflows/pythonapp.yml @@ -0,0 +1,37 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Python application + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.4, 3.5, 3.6] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + # TODO: Add requirements.txt instead + pip install PyMySQL + - name: Lint with flake8 + run: | + pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=19 --max-line-length=127 --statistics From dbed6816a0318265b2f6fd2abc2d65840dd882dd Mon Sep 17 00:00:00 2001 From: Charles JUDITH Date: Mon, 27 Apr 2020 16:01:32 +0200 Subject: [PATCH 22/24] Remove unexisting python 3.4 Remove unexisting python 3.4 --- .github/workflows/pythonapp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index e1cafe7..39bbcb2 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.4, 3.5, 3.6] + python-version: [3.5, 3.6] steps: - uses: actions/checkout@v2 From 76635e00cbe89e8f7fe187f0485753357dd8ffa7 Mon Sep 17 00:00:00 2001 From: Charles JUDITH Date: Fri, 14 Aug 2020 10:26:12 +0200 Subject: [PATCH 23/24] Change metric name for user connections - Change metric name connections_per_user by user_connections - Reduce metric name - Reduce username field --- README.md | 4 ++-- mysql.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 622fff4..e07bc7c 100644 --- a/README.md +++ b/README.md @@ -339,8 +339,8 @@ From `SHOW SLAVE STATUS`: This is using performance_schema. If performance_schema is not enabled, it won't work. - connection_per_user.current_connections_root - connection_per_user.current_connections_repl + user_connection.root + user_connection.user-repl ### Query Response Times diff --git a/mysql.py b/mysql.py index 51107eb..a75aadc 100644 --- a/mysql.py +++ b/mysql.py @@ -504,14 +504,14 @@ def fetch_connections_per_account(conn): queries = {} try: result = mysql_query(conn, """ - SELECT user, sum(current_connections) as `current_connections` + SELECT user, sum(current_connections) as `cur_conn` FROM performance_schema.accounts WHERE user is not null GROUP BY user; """) for row in result.fetchall(): user = str(row['user']) - queries["current_connections_"+user] = row['current_connections'] + queries[user] = row['cur_conn'] except pymysql.OperationalError: return {} @@ -691,7 +691,7 @@ def read_callback(): if is_ps_enabled(conn) is True: user_connections = fetch_connections_per_account(conn) for key in user_connections: - dispatch_value('connection_per_user', key, user_connections[key], 'gauge') + dispatch_value('user_connection', key, user_connections[key], 'gauge') # This is only available in Percona Server and some MySQL versions but not in MariaDB # https://www.percona.com/blog/2010/07/11/query-response-time-histogram-new-feature-in-percona-server/ From 0450e50d15d6101305c574f2c8a5630d2798d3a4 Mon Sep 17 00:00:00 2001 From: Pierre Dumont Date: Wed, 25 Aug 2021 17:38:11 +0200 Subject: [PATCH 24/24] Fix for MariaDB 10.5 The Innodb_max_trx_id status variable was re-introduced in 10.5 but Innodb_purge_trx_id was not, causing fetch_mysql_status() to crash. --- mysql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql.py b/mysql.py index a75aadc..a0d2c67 100644 --- a/mysql.py +++ b/mysql.py @@ -358,7 +358,7 @@ def fetch_mysql_status(conn): status[row['Variable_name']] = row['Value'] # calculate the number of unpurged txns from existing variables - if 'Innodb_max_trx_id' in status: + if 'Innodb_max_trx_id' in status and 'Innodb_purge_trx_id' in status: status['Innodb_unpurged_txns'] = int(status['Innodb_max_trx_id']) - int(status['Innodb_purge_trx_id']) if 'Innodb_lsn_last_checkpoint' in status: