From b1b39311385d5bb9692abf5d7f7b51073a3e5e11 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sat, 16 Sep 2023 13:18:11 -0600 Subject: [PATCH] update check_matrix_synapse --- check_dns_tls.py | 10 +- check_matrix_synapse.py | 17 +++- checker/synapse_grafana.py | 200 ++++++++++++++++--------------------- 3 files changed, 106 insertions(+), 121 deletions(-) diff --git a/check_dns_tls.py b/check_dns_tls.py index 5bd997b..1be0efe 100755 --- a/check_dns_tls.py +++ b/check_dns_tls.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 - import argparse +import socket import sys import time -import dns.query + import dns.message -import socket +import dns.query + def resolve_dns_server_ip(server): try: @@ -14,6 +15,7 @@ def resolve_dns_server_ip(server): raise ValueError("Failed to resolve DNS server domain") return ip_address + def main(): parser = argparse.ArgumentParser(description='Check DNS-over-TLS connectivity and response time') parser.add_argument('-c', '--critical', type=float, default=500, help='Critical threshold for response time in milliseconds (default: 500)') @@ -45,7 +47,6 @@ def main(): print(f'CRITICAL - DNS-over-TLS query failed: {error_message}') sys.exit(2) - response_time = time.time() - start_time response_time_ms = int(response_time * 1000) @@ -66,5 +67,6 @@ def main(): print(f'OK - DNS-over-TLS response time: {response_time:.2f} sec. {perfdata}') sys.exit(0) + if __name__ == '__main__': main() diff --git a/check_matrix_synapse.py b/check_matrix_synapse.py index 0fc5541..0c69cb0 100755 --- a/check_matrix_synapse.py +++ b/check_matrix_synapse.py @@ -11,7 +11,7 @@ from checker import nagios from checker.synapse_grafana import get_avg_python_gc_time, get_event_send_time, get_outgoing_http_request_rate, get_waiting_for_db parser = argparse.ArgumentParser(description='Process some integers.') -parser.add_argument('--grafana-server', required=True, help='Grafana server.') +parser.add_argument('--grafana-server', required=True, help='Grafana server. Example: http://10.0.0.43:3000') parser.add_argument('--synapse-server', required=True, help='Matrix Synapse server.') parser.add_argument('--grafana-api-key', required=True) parser.add_argument('--interval', default=15, type=int, help='Data interval in seconds.') @@ -29,7 +29,12 @@ def main(): # in seconds python_gc_time_sum_MAX = 0.002 if not args.crit else args.crit try: - python_gc_time_sum = np.round(np.average(get_avg_python_gc_time(args.grafana_api_key, args.interval, args.range, args.grafana_server)), 5) + raw_gc_time = get_avg_python_gc_time(args.grafana_api_key, args.interval, args.range, args.grafana_server) + if not len(raw_gc_time): + print(f"UNKNOWN: no data for average GC time") + sys.exit(nagios.UNKNOWN) + + python_gc_time_sum = np.round(np.average(), 5) if python_gc_time_sum >= python_gc_time_sum_MAX: print(f"CRITICAL: average GC time per collection is {python_gc_time_sum} sec. |'garbage-collection'={python_gc_time_sum}s;;;") sys.exit(nagios.CRITICAL) @@ -72,6 +77,10 @@ def main(): outgoing_http_request_rate_MAX = 10 if not args.crit else args.crit try: outgoing_http_request_rate = get_outgoing_http_request_rate(args.grafana_api_key, args.interval, args.range, args.grafana_server) + if not len(outgoing_http_request_rate.keys()): + print(f"UNKNOWN: no data for outgoing HTTP request rate") + sys.exit(nagios.UNKNOWN) + failed = {} perf_data = '|' for k, v in outgoing_http_request_rate.items(): @@ -93,6 +102,10 @@ def main(): event_send_time_MAX = 1 if not args.crit else args.crit try: event_send_time = get_event_send_time(args.grafana_api_key, args.interval, args.range, args.grafana_server) + if not event_send_time: + print(f"UNKNOWN: no data for event send time") + sys.exit(nagios.UNKNOWN) + if event_send_time > event_send_time_MAX: print(f"CRITICAL: average message send time is {event_send_time} sec. |'avg-send-time'={event_send_time}s;;;") sys.exit(nagios.CRITICAL) diff --git a/checker/synapse_grafana.py b/checker/synapse_grafana.py index 1ce844f..d88626e 100644 --- a/checker/synapse_grafana.py +++ b/checker/synapse_grafana.py @@ -1,3 +1,5 @@ +import time + import numpy as np import requests from urllib3.exceptions import InsecureRequestWarning @@ -5,36 +7,44 @@ from urllib3.exceptions import InsecureRequestWarning requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) +def timestamp_minutes_ago(minutes): + current_time = time.time() + minutes_in_seconds = minutes * 60 + time_in_past = current_time - minutes_in_seconds + timestamp_in_ms = int(time_in_past * 1000) + return timestamp_in_ms + + def get_avg_python_gc_time(api_key, interval, data_range, endpoint): json_data = { 'queries': [ { 'datasource': { 'type': 'prometheus', - 'uid': 'AbuT5CJ4z', + 'uid': 'DAMPdbiIz', }, - 'expr': 'rate(python_gc_time_sum{instance="10.0.0.34:9000",job=~"(federation-receiver|federation-sender|initialsync|synapse|synchrotron)",index=~".*"}[30s])/rate(python_gc_time_count[30s])', + 'expr': 'rate(python_gc_time_sum{instance="172.0.2.118:9000",job=~"synapse",index=~".*"}[2m])/rate(python_gc_time_count[2m])', 'format': 'time_series', 'intervalFactor': 2, 'refId': 'A', 'step': 20, 'target': '', 'interval': '', - # 'key': 'Q-7edaea76-89bd-4b29-8412-a68bf4646712-0', 'queryType': 'timeSeriesQuery', 'exemplar': False, - # 'requestId': 'Q-7edaea76-89bd-4b29-8412-a68bf4646712-0A', 'utcOffsetSec': -25200, 'legendFormat': '', - 'datasourceId': 1, + 'datasourceId': 8, 'intervalMs': interval * 1000, - # 'maxDataPoints': 1383, }, ], 'from': f'now-{data_range}m', 'to': 'now', } response = requests.post(f'{endpoint}/api/ds/query', headers={'Authorization': f'Bearer {api_key}'}, json=json_data, verify=False).json() + if not response['results'].get('A', {}).get('frames'): + return [] + good = [] for i in response['results']['A']['frames']: # This one can sometimes be null @@ -53,63 +63,48 @@ def get_avg_python_gc_time(api_key, interval, data_range, endpoint): def get_outgoing_http_request_rate(api_key, interval, data_range, endpoint): json_data = { - 'queries': [ - { - 'datasource': { - 'type': 'prometheus', - 'uid': 'AbuT5CJ4z', - }, - 'editorMode': 'code', - 'expr': 'rate(synapse_http_client_requests_total{job=~"(federation-receiver|federation-sender|initialsync|synapse|synchrotron)",index=~".*",instance="10.0.0.34:9000"}[2m])', - 'range': True, - 'refId': 'A', - 'interval': '', - # 'key': 'Q-8b3dabd7-358e-45ed-a9ba-7be3f5fcf274-0', - 'queryType': 'timeSeriesQuery', - 'exemplar': False, - # 'requestId': 'Q-8b3dabd7-358e-45ed-a9ba-7be3f5fcf274-0Q-c5c08c6b-7591-424c-8eac-53837fa51e89-1A', - 'utcOffsetSec': -25200, - 'legendFormat': '', - 'datasourceId': 1, - 'intervalMs': interval * 1000, - # 'maxDataPoints': 10, + 'queries': [{ + 'datasource': { + 'type': 'prometheus', + 'uid': 'DAMPdbiIz' }, - { - 'datasource': { - 'type': 'prometheus', - 'uid': 'AbuT5CJ4z', - }, - 'editorMode': 'code', - 'expr': 'rate(synapse_http_matrixfederationclient_requests_total{job=~"(federation-receiver|federation-sender|initialsync|synapse|synchrotron)",index=~".*",instance="10.0.0.34:9000"}[2m])', - 'range': True, - 'refId': 'B', - 'interval': '', - # 'key': 'Q-c5c08c6b-7591-424c-8eac-53837fa51e89-1', - 'queryType': 'timeSeriesQuery', - 'exemplar': False, - # 'requestId': 'Q-8b3dabd7-358e-45ed-a9ba-7be3f5fcf274-0Q-c5c08c6b-7591-424c-8eac-53837fa51e89-1B', - 'utcOffsetSec': -25200, - 'legendFormat': '', - 'datasourceId': 1, - 'intervalMs': interval * 1000, - # 'maxDataPoints': 10, + 'editorMode': 'code', + 'expr': 'rate(synapse_http_client_requests_total{job=~"synapse",index=~".*",instance="172.0.2.118:9000"}[2m])', + 'range': True, + 'refId': 'A', + 'interval': '', + 'exemplar': False, + 'utcOffsetSec': 0, + 'legendFormat': '', + 'datasourceId': 8, + 'intervalMs': interval * 1000, + }, { + 'datasource': { + 'type': 'prometheus', + 'uid': 'DAMPdbiIz' }, - ], + 'editorMode': 'code', + 'expr': 'rate(synapse_http_matrixfederationclient_requests_total{job=~"synapse",index=~".*",instance="172.0.2.118:9000"}[2m])', + 'range': True, + 'refId': 'B', + 'interval': '', + 'exemplar': False, + 'utcOffsetSec': 0, + 'legendFormat': '', + 'datasourceId': 8, + 'intervalMs': interval * 1000, + }], 'from': f'now-{data_range}m', - 'to': 'now', + 'to': 'now' } response = requests.post(f'{endpoint}/api/ds/query', headers={'Authorization': f'Bearer {api_key}'}, json=json_data, verify=False).json() + print(response) output = {} for letter, result in response['results'].items(): - name = result['frames'][0]['schema']['name'].split('=')[-1].strip('}').strip('"') - output[name] = np.round(np.average(result['frames'][0]['data']['values'][1]), 2) + if len(result['frames']): + name = result['frames'][0]['schema']['name'].split('=')[-1].strip('}').strip('"') + output[name] = np.round(np.average(result['frames'][0]['data']['values'][1]), 2) return output - # return { - # 'GET': np.round(np.average(response['results']['A']['frames'][0]['data']['values'][1]), 2), - # 'POST': np.round(np.average(response['results']['A']['frames'][1]['data']['values'][1]), 2), - # 'PUT': np.round(np.average(response['results']['A']['frames'][2]['data']['values'][1]), 2), - # 'fedr_GET': np.round(np.average(response['results']['B']['frames'][0]['data']['values'][1]), 2) - # } def get_event_send_time(api_key, interval, data_range, endpoint): @@ -118,166 +113,144 @@ def get_event_send_time(api_key, interval, data_range, endpoint): { 'datasource': { 'type': 'prometheus', - 'uid': 'AbuT5CJ4z', + 'uid': 'DAMPdbiIz', }, - 'expr': 'histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="10.0.0.34:9000",code=~"2.."}[2m])) by (le))', + 'expr': 'histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m])) by (le))', 'format': 'time_series', 'intervalFactor': 1, 'refId': 'D', 'interval': '', - # 'key': 'Q-d8eb3572-9aea-4a73-92f2-e08b33c21ecb-0', 'editorMode': 'builder', 'range': True, 'instant': True, 'queryType': 'timeSeriesQuery', 'exemplar': False, - # 'requestId': 'Q-d8eb3572-9aea-4a73-92f2-e08b33c21ecb-0Q-a9222e59-18ff-4b3b-80ae-27bea8f149a9-1Q-0378a458-1ade-410e-a4b3-ae4aaa91d709-2Q-da4c00b6-61c1-49f5-8a0a-9f19990acfb7-3Q-21254889-3cf6-4d97-8dc5-ddf68360847e-4Q-502b8ed5-4050-461c-befc-76f6796dce68-5Q-364dc896-c399-4e58-8930-cba2e3d1d579-6Q-9072e904-da8d-4b00-b454-dac45b7c38f0-7D', 'utcOffsetSec': -25200, 'legendFormat': '', - 'datasourceId': 1, + 'datasourceId': 8, 'intervalMs': interval * 1000, - # 'maxDataPoints': 1383, }, { 'datasource': { 'type': 'prometheus', - 'uid': 'AbuT5CJ4z', + 'uid': 'DAMPdbiIz', }, - 'expr': 'histogram_quantile(0.9, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="10.0.0.34:9000",code=~"2.."}[2m])) by (le))', + 'expr': 'histogram_quantile(0.9, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m])) by (le))', 'format': 'time_series', 'interval': '', 'intervalFactor': 1, 'refId': 'A', - # 'key': 'Q-a9222e59-18ff-4b3b-80ae-27bea8f149a9-1', 'queryType': 'timeSeriesQuery', 'exemplar': False, - # 'requestId': 'Q-d8eb3572-9aea-4a73-92f2-e08b33c21ecb-0Q-a9222e59-18ff-4b3b-80ae-27bea8f149a9-1Q-0378a458-1ade-410e-a4b3-ae4aaa91d709-2Q-da4c00b6-61c1-49f5-8a0a-9f19990acfb7-3Q-21254889-3cf6-4d97-8dc5-ddf68360847e-4Q-502b8ed5-4050-461c-befc-76f6796dce68-5Q-364dc896-c399-4e58-8930-cba2e3d1d579-6Q-9072e904-da8d-4b00-b454-dac45b7c38f0-7A', 'utcOffsetSec': -25200, 'legendFormat': '', - 'datasourceId': 1, + 'datasourceId': 8, 'intervalMs': interval * 1000, - # 'maxDataPoints': 1383, }, { 'datasource': { 'type': 'prometheus', - 'uid': 'AbuT5CJ4z', + 'uid': 'DAMPdbiIz', }, - 'expr': 'histogram_quantile(0.75, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="10.0.0.34:9000",code=~"2.."}[2m])) by (le))', + 'expr': 'histogram_quantile(0.75, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m])) by (le))', 'format': 'time_series', 'intervalFactor': 1, 'refId': 'C', 'interval': '', - # 'key': 'Q-0378a458-1ade-410e-a4b3-ae4aaa91d709-2', 'queryType': 'timeSeriesQuery', 'exemplar': False, - # 'requestId': 'Q-d8eb3572-9aea-4a73-92f2-e08b33c21ecb-0Q-a9222e59-18ff-4b3b-80ae-27bea8f149a9-1Q-0378a458-1ade-410e-a4b3-ae4aaa91d709-2Q-da4c00b6-61c1-49f5-8a0a-9f19990acfb7-3Q-21254889-3cf6-4d97-8dc5-ddf68360847e-4Q-502b8ed5-4050-461c-befc-76f6796dce68-5Q-364dc896-c399-4e58-8930-cba2e3d1d579-6Q-9072e904-da8d-4b00-b454-dac45b7c38f0-7C', 'utcOffsetSec': -25200, 'legendFormat': '', - 'datasourceId': 1, + 'datasourceId': 8, 'intervalMs': interval * 1000, - # 'maxDataPoints': 1383, }, { 'datasource': { 'type': 'prometheus', - 'uid': 'AbuT5CJ4z', + 'uid': 'DAMPdbiIz', }, - 'expr': 'histogram_quantile(0.5, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="10.0.0.34:9000",code=~"2.."}[2m])) by (le))', + 'expr': 'histogram_quantile(0.5, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m])) by (le))', 'format': 'time_series', 'intervalFactor': 1, 'refId': 'B', 'interval': '', - # 'key': 'Q-da4c00b6-61c1-49f5-8a0a-9f19990acfb7-3', 'queryType': 'timeSeriesQuery', 'exemplar': False, - # 'requestId': 'Q-d8eb3572-9aea-4a73-92f2-e08b33c21ecb-0Q-a9222e59-18ff-4b3b-80ae-27bea8f149a9-1Q-0378a458-1ade-410e-a4b3-ae4aaa91d709-2Q-da4c00b6-61c1-49f5-8a0a-9f19990acfb7-3Q-21254889-3cf6-4d97-8dc5-ddf68360847e-4Q-502b8ed5-4050-461c-befc-76f6796dce68-5Q-364dc896-c399-4e58-8930-cba2e3d1d579-6Q-9072e904-da8d-4b00-b454-dac45b7c38f0-7B', 'utcOffsetSec': -25200, 'legendFormat': '', - 'datasourceId': 1, + 'datasourceId': 8, 'intervalMs': interval * 1000, - # 'maxDataPoints': 1383, }, { 'datasource': { 'type': 'prometheus', - 'uid': 'AbuT5CJ4z', + 'uid': 'DAMPdbiIz', }, - 'expr': 'histogram_quantile(0.25, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="10.0.0.34:9000",code=~"2.."}[2m])) by (le))', + 'expr': 'histogram_quantile(0.25, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m])) by (le))', 'refId': 'F', 'interval': '', - # 'key': 'Q-21254889-3cf6-4d97-8dc5-ddf68360847e-4', 'queryType': 'timeSeriesQuery', 'exemplar': False, - # 'requestId': 'Q-d8eb3572-9aea-4a73-92f2-e08b33c21ecb-0Q-a9222e59-18ff-4b3b-80ae-27bea8f149a9-1Q-0378a458-1ade-410e-a4b3-ae4aaa91d709-2Q-da4c00b6-61c1-49f5-8a0a-9f19990acfb7-3Q-21254889-3cf6-4d97-8dc5-ddf68360847e-4Q-502b8ed5-4050-461c-befc-76f6796dce68-5Q-364dc896-c399-4e58-8930-cba2e3d1d579-6Q-9072e904-da8d-4b00-b454-dac45b7c38f0-7F', 'utcOffsetSec': -25200, 'legendFormat': '', - 'datasourceId': 1, + 'datasourceId': 8, 'intervalMs': interval * 1000, - # 'maxDataPoints': 1383, }, { 'datasource': { 'type': 'prometheus', - 'uid': 'AbuT5CJ4z', + 'uid': 'DAMPdbiIz', }, - 'expr': 'histogram_quantile(0.05, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="10.0.0.34:9000",code=~"2.."}[2m])) by (le))', + 'expr': 'histogram_quantile(0.05, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m])) by (le))', 'refId': 'G', 'interval': '', - # 'key': 'Q-502b8ed5-4050-461c-befc-76f6796dce68-5', 'queryType': 'timeSeriesQuery', 'exemplar': False, - # 'requestId': 'Q-d8eb3572-9aea-4a73-92f2-e08b33c21ecb-0Q-a9222e59-18ff-4b3b-80ae-27bea8f149a9-1Q-0378a458-1ade-410e-a4b3-ae4aaa91d709-2Q-da4c00b6-61c1-49f5-8a0a-9f19990acfb7-3Q-21254889-3cf6-4d97-8dc5-ddf68360847e-4Q-502b8ed5-4050-461c-befc-76f6796dce68-5Q-364dc896-c399-4e58-8930-cba2e3d1d579-6Q-9072e904-da8d-4b00-b454-dac45b7c38f0-7G', 'utcOffsetSec': -25200, 'legendFormat': '', - 'datasourceId': 1, + 'datasourceId': 8, 'intervalMs': interval * 1000, - # 'maxDataPoints': 1383, }, { 'datasource': { 'type': 'prometheus', - 'uid': 'AbuT5CJ4z', + 'uid': 'DAMPdbiIz', }, - 'expr': 'sum(rate(synapse_http_server_response_time_seconds_sum{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="10.0.0.34:9000",code=~"2.."}[2m])) / sum(rate(synapse_http_server_response_time_seconds_count{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="10.0.0.34:9000",code=~"2.."}[2m]))', + 'expr': 'sum(rate(synapse_http_server_response_time_seconds_sum{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m])) / sum(rate(synapse_http_server_response_time_seconds_count{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m]))', 'refId': 'H', 'interval': '', - # 'key': 'Q-364dc896-c399-4e58-8930-cba2e3d1d579-6', 'queryType': 'timeSeriesQuery', 'exemplar': False, - # 'requestId': 'Q-d8eb3572-9aea-4a73-92f2-e08b33c21ecb-0Q-a9222e59-18ff-4b3b-80ae-27bea8f149a9-1Q-0378a458-1ade-410e-a4b3-ae4aaa91d709-2Q-da4c00b6-61c1-49f5-8a0a-9f19990acfb7-3Q-21254889-3cf6-4d97-8dc5-ddf68360847e-4Q-502b8ed5-4050-461c-befc-76f6796dce68-5Q-364dc896-c399-4e58-8930-cba2e3d1d579-6Q-9072e904-da8d-4b00-b454-dac45b7c38f0-7H', 'utcOffsetSec': -25200, 'legendFormat': '', - 'datasourceId': 1, + 'datasourceId': 8, 'intervalMs': interval * 1000, - # 'maxDataPoints': 1383, }, { 'datasource': { 'type': 'prometheus', - 'uid': 'AbuT5CJ4z', + 'uid': 'DAMPdbiIz', }, - 'expr': 'sum(rate(synapse_storage_events_persisted_events_total{instance="10.0.0.34:9000"}[2m]))', + 'expr': 'sum(rate(synapse_storage_events_persisted_events_total{instance="172.0.2.118:9000"}[2m]))', 'hide': False, 'instant': False, 'refId': 'E', 'interval': '', - # 'key': 'Q-9072e904-da8d-4b00-b454-dac45b7c38f0-7', 'editorMode': 'code', 'queryType': 'timeSeriesQuery', 'exemplar': False, - # 'requestId': 'Q-d8eb3572-9aea-4a73-92f2-e08b33c21ecb-0Q-a9222e59-18ff-4b3b-80ae-27bea8f149a9-1Q-0378a458-1ade-410e-a4b3-ae4aaa91d709-2Q-da4c00b6-61c1-49f5-8a0a-9f19990acfb7-3Q-21254889-3cf6-4d97-8dc5-ddf68360847e-4Q-502b8ed5-4050-461c-befc-76f6796dce68-5Q-364dc896-c399-4e58-8930-cba2e3d1d579-6Q-9072e904-da8d-4b00-b454-dac45b7c38f0-7E', 'utcOffsetSec': -25200, 'legendFormat': '', - 'datasourceId': 1, + 'datasourceId': 8, 'intervalMs': interval * 1000, - # 'maxDataPoints': 1383, }, ], 'from': f'now-{data_range}m', 'to': 'now', } response = requests.post(f'{endpoint}/api/ds/query', headers={'Authorization': f'Bearer {api_key}'}, json=json_data, verify=False).json() + if not response['results'].get('E', {}).get('frames'): + return None return np.round(np.average(response['results']['E']['frames'][0]['data']['values'][1]), 2) @@ -287,23 +260,20 @@ def get_waiting_for_db(api_key, interval, data_range, endpoint): { 'datasource': { 'type': 'prometheus', - 'uid': 'AbuT5CJ4z', + 'uid': 'DAMPdbiIz', }, - 'expr': 'rate(synapse_storage_schedule_time_sum{instance="10.0.0.34:9000",job=~"(federation-receiver|federation-sender|initialsync|synapse|synchrotron)",index=~".*"}[30s])/rate(synapse_storage_schedule_time_count[30s])', + 'expr': 'rate(synapse_storage_schedule_time_sum{instance="172.0.2.118:9000",job=~"synapse",index=~".*"}[30s])/rate(synapse_storage_schedule_time_count[30s])', 'format': 'time_series', 'intervalFactor': 2, 'refId': 'A', 'step': 20, 'interval': '', - # 'key': 'Q-459af7f4-0427-4832-9353-46086b3f5c27-0', 'queryType': 'timeSeriesQuery', 'exemplar': False, - # 'requestId': 'Q-459af7f4-0427-4832-9353-46086b3f5c27-0A', 'utcOffsetSec': -25200, 'legendFormat': '', - 'datasourceId': 1, + 'datasourceId': 8, 'intervalMs': interval * 1000, - # 'maxDataPoints': 1383, }, ], 'from': f'now-{data_range}m', @@ -331,10 +301,10 @@ def get_stateres_worst_case(api_key, interval, data_range, endpoint): { 'datasource': { 'type': 'prometheus', - 'uid': 'AbuT5CJ4z', + 'uid': 'DAMPdbiIz', }, 'exemplar': False, - 'expr': 'sum(rate(synapse_state_res_db_for_biggest_room_seconds_total{instance="10.0.0.34:9000"}[1m]))', + 'expr': 'sum(rate(synapse_state_res_db_for_biggest_room_seconds_total{instance="172.0.2.118:9000"}[1m]))', 'format': 'time_series', 'hide': False, 'instant': False, @@ -343,17 +313,17 @@ def get_stateres_worst_case(api_key, interval, data_range, endpoint): 'queryType': 'timeSeriesQuery', 'utcOffsetSec': -25200, 'legendFormat': '', - 'datasourceId': 1, + 'datasourceId': 8, 'intervalMs': 15000, 'maxDataPoints': 1863, }, { 'datasource': { 'type': 'prometheus', - 'uid': 'AbuT5CJ4z', + 'uid': 'DAMPdbiIz', }, 'exemplar': False, - 'expr': 'sum(rate(synapse_state_res_cpu_for_biggest_room_seconds_total{instance="10.0.0.34:9000"}[1m]))', + 'expr': 'sum(rate(synapse_state_res_cpu_for_biggest_room_seconds_total{instance="172.0.2.118:9000"}[1m]))', 'format': 'time_series', 'hide': False, 'instant': False, @@ -362,7 +332,7 @@ def get_stateres_worst_case(api_key, interval, data_range, endpoint): 'queryType': 'timeSeriesQuery', 'utcOffsetSec': -25200, 'legendFormat': '', - 'datasourceId': 1, + 'datasourceId': 8, 'intervalMs': 15000, 'maxDataPoints': 1863, },