cyberes
/
icinga2-checks
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update check_matrix_synapse

This commit is contained in:
Cyberes 2023-09-16 13:18:11 -06:00
parent 6def1921c0
commit b1b3931138
3 changed files with 106 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
check_dns_tls.py
View File

@ -1,11 +1,12 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import argparse import argparse
import socket
import sys import sys
import time import time
import dns.query
import dns.message import dns.message
import socket import dns.query
def resolve_dns_server_ip(server): def resolve_dns_server_ip(server):
try: try:
@ -14,6 +15,7 @@ def resolve_dns_server_ip(server):
raise ValueError("Failed to resolve DNS server domain") raise ValueError("Failed to resolve DNS server domain")
return ip_address return ip_address
def main(): def main():
parser = argparse.ArgumentParser(description='Check DNS-over-TLS connectivity and response time') parser = argparse.ArgumentParser(description='Check DNS-over-TLS connectivity and response time')
parser.add_argument('-c', '--critical', type=float, default=500, help='Critical threshold for response time in milliseconds (default: 500)') parser.add_argument('-c', '--critical', type=float, default=500, help='Critical threshold for response time in milliseconds (default: 500)')
@ -45,7 +47,6 @@ def main():
print(f'CRITICAL - DNS-over-TLS query failed: {error_message}') print(f'CRITICAL - DNS-over-TLS query failed: {error_message}')
sys.exit(2) sys.exit(2)
response_time = time.time() - start_time response_time = time.time() - start_time
response_time_ms = int(response_time * 1000) response_time_ms = int(response_time * 1000)
@ -66,5 +67,6 @@ def main():
print(f'OK - DNS-over-TLS response time: {response_time:.2f} sec. {perfdata}') print(f'OK - DNS-over-TLS response time: {response_time:.2f} sec. {perfdata}')
sys.exit(0) sys.exit(0)
if __name__ == '__main__': if __name__ == '__main__':
main() main()

17
check_matrix_synapse.py
View File

@ -11,7 +11,7 @@ from checker import nagios
from checker.synapse_grafana import get_avg_python_gc_time, get_event_send_time, get_outgoing_http_request_rate, get_waiting_for_db from checker.synapse_grafana import get_avg_python_gc_time, get_event_send_time, get_outgoing_http_request_rate, get_waiting_for_db
parser = argparse.ArgumentParser(description='Process some integers.') parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument('--grafana-server', required=True, help='Grafana server.') parser.add_argument('--grafana-server', required=True, help='Grafana server. Example: http://10.0.0.43:3000')
parser.add_argument('--synapse-server', required=True, help='Matrix Synapse server.') parser.add_argument('--synapse-server', required=True, help='Matrix Synapse server.')
parser.add_argument('--grafana-api-key', required=True) parser.add_argument('--grafana-api-key', required=True)
parser.add_argument('--interval', default=15, type=int, help='Data interval in seconds.') parser.add_argument('--interval', default=15, type=int, help='Data interval in seconds.')
@ -29,7 +29,12 @@ def main():
# in seconds # in seconds
python_gc_time_sum_MAX = 0.002 if not args.crit else args.crit python_gc_time_sum_MAX = 0.002 if not args.crit else args.crit
try: try:
python_gc_time_sum = np.round(np.average(get_avg_python_gc_time(args.grafana_api_key, args.interval, args.range, args.grafana_server)), 5) raw_gc_time = get_avg_python_gc_time(args.grafana_api_key, args.interval, args.range, args.grafana_server)
if not len(raw_gc_time):
print(f"UNKNOWN: no data for average GC time")
sys.exit(nagios.UNKNOWN)
python_gc_time_sum = np.round(np.average(), 5)
if python_gc_time_sum >= python_gc_time_sum_MAX: if python_gc_time_sum >= python_gc_time_sum_MAX:
print(f"CRITICAL: average GC time per collection is {python_gc_time_sum} sec. |'garbage-collection'={python_gc_time_sum}s;;;") print(f"CRITICAL: average GC time per collection is {python_gc_time_sum} sec. |'garbage-collection'={python_gc_time_sum}s;;;")
sys.exit(nagios.CRITICAL) sys.exit(nagios.CRITICAL)
@ -72,6 +77,10 @@ def main():
outgoing_http_request_rate_MAX = 10 if not args.crit else args.crit outgoing_http_request_rate_MAX = 10 if not args.crit else args.crit
try: try:
outgoing_http_request_rate = get_outgoing_http_request_rate(args.grafana_api_key, args.interval, args.range, args.grafana_server) outgoing_http_request_rate = get_outgoing_http_request_rate(args.grafana_api_key, args.interval, args.range, args.grafana_server)
if not len(outgoing_http_request_rate.keys()):
print(f"UNKNOWN: no data for outgoing HTTP request rate")
sys.exit(nagios.UNKNOWN)
failed = {} failed = {}
perf_data = '|' perf_data = '|'
for k, v in outgoing_http_request_rate.items(): for k, v in outgoing_http_request_rate.items():
@ -93,6 +102,10 @@ def main():
event_send_time_MAX = 1 if not args.crit else args.crit event_send_time_MAX = 1 if not args.crit else args.crit
try: try:
event_send_time = get_event_send_time(args.grafana_api_key, args.interval, args.range, args.grafana_server) event_send_time = get_event_send_time(args.grafana_api_key, args.interval, args.range, args.grafana_server)
if not event_send_time:
print(f"UNKNOWN: no data for event send time")
sys.exit(nagios.UNKNOWN)
if event_send_time > event_send_time_MAX: if event_send_time > event_send_time_MAX:
print(f"CRITICAL: average message send time is {event_send_time} sec. |'avg-send-time'={event_send_time}s;;;") print(f"CRITICAL: average message send time is {event_send_time} sec. |'avg-send-time'={event_send_time}s;;;")
sys.exit(nagios.CRITICAL) sys.exit(nagios.CRITICAL)

200
checker/synapse_grafana.py
View File

@ -1,3 +1,5 @@
import time
import numpy as np import numpy as np
import requests import requests
from urllib3.exceptions import InsecureRequestWarning from urllib3.exceptions import InsecureRequestWarning
@ -5,36 +7,44 @@ from urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
def timestamp_minutes_ago(minutes):
current_time = time.time()
minutes_in_seconds = minutes * 60
time_in_past = current_time - minutes_in_seconds
timestamp_in_ms = int(time_in_past * 1000)
return timestamp_in_ms
def get_avg_python_gc_time(api_key, interval, data_range, endpoint): def get_avg_python_gc_time(api_key, interval, data_range, endpoint):
json_data = { json_data = {
'queries': [ 'queries': [
{ {
'datasource': { 'datasource': {
'type': 'prometheus', 'type': 'prometheus',
'uid': 'AbuT5CJ4z', 'uid': 'DAMPdbiIz',
}, },
'expr': 'rate(python_gc_time_sum{instance="10.0.0.34:9000",job=~"(federation-receiver|federation-sender|initialsync|synapse|synchrotron)",index=~".*"}[30s])/rate(python_gc_time_count[30s])', 'expr': 'rate(python_gc_time_sum{instance="172.0.2.118:9000",job=~"synapse",index=~".*"}[2m])/rate(python_gc_time_count[2m])',
'format': 'time_series', 'format': 'time_series',
'intervalFactor': 2, 'intervalFactor': 2,
'refId': 'A', 'refId': 'A',
'step': 20, 'step': 20,
'target': '', 'target': '',
'interval': '', 'interval': '',
# 'key': 'Q-7edaea76-89bd-4b29-8412-a68bf4646712-0',
'queryType': 'timeSeriesQuery', 'queryType': 'timeSeriesQuery',
'exemplar': False, 'exemplar': False,
# 'requestId': 'Q-7edaea76-89bd-4b29-8412-a68bf4646712-0A',
'utcOffsetSec': -25200, 'utcOffsetSec': -25200,
'legendFormat': '', 'legendFormat': '',
'datasourceId': 1, 'datasourceId': 8,
'intervalMs': interval * 1000, 'intervalMs': interval * 1000,
# 'maxDataPoints': 1383,
}, },
], ],
'from': f'now-{data_range}m', 'from': f'now-{data_range}m',
'to': 'now', 'to': 'now',
} }
response = requests.post(f'{endpoint}/api/ds/query', headers={'Authorization': f'Bearer {api_key}'}, json=json_data, verify=False).json() response = requests.post(f'{endpoint}/api/ds/query', headers={'Authorization': f'Bearer {api_key}'}, json=json_data, verify=False).json()
if not response['results'].get('A', {}).get('frames'):
return []
good = [] good = []
for i in response['results']['A']['frames']: for i in response['results']['A']['frames']:
# This one can sometimes be null # This one can sometimes be null
@ -53,63 +63,48 @@ def get_avg_python_gc_time(api_key, interval, data_range, endpoint):
def get_outgoing_http_request_rate(api_key, interval, data_range, endpoint): def get_outgoing_http_request_rate(api_key, interval, data_range, endpoint):
json_data = { json_data = {
'queries': [ 'queries': [{
{ 'datasource': {
'datasource': { 'type': 'prometheus',
'type': 'prometheus', 'uid': 'DAMPdbiIz'
'uid': 'AbuT5CJ4z',
},
'editorMode': 'code',
'expr': 'rate(synapse_http_client_requests_total{job=~"(federation-receiver|federation-sender|initialsync|synapse|synchrotron)",index=~".*",instance="10.0.0.34:9000"}[2m])',
'range': True,
'refId': 'A',
'interval': '',
# 'key': 'Q-8b3dabd7-358e-45ed-a9ba-7be3f5fcf274-0',
'queryType': 'timeSeriesQuery',
'exemplar': False,
# 'requestId': 'Q-8b3dabd7-358e-45ed-a9ba-7be3f5fcf274-0Q-c5c08c6b-7591-424c-8eac-53837fa51e89-1A',
'utcOffsetSec': -25200,
'legendFormat': '',
'datasourceId': 1,
'intervalMs': interval * 1000,
# 'maxDataPoints': 10,
}, },
{ 'editorMode': 'code',
'datasource': { 'expr': 'rate(synapse_http_client_requests_total{job=~"synapse",index=~".*",instance="172.0.2.118:9000"}[2m])',
'type': 'prometheus', 'range': True,
'uid': 'AbuT5CJ4z', 'refId': 'A',
}, 'interval': '',
'editorMode': 'code', 'exemplar': False,
'expr': 'rate(synapse_http_matrixfederationclient_requests_total{job=~"(federation-receiver|federation-sender|initialsync|synapse|synchrotron)",index=~".*",instance="10.0.0.34:9000"}[2m])', 'utcOffsetSec': 0,
'range': True, 'legendFormat': '',
'refId': 'B', 'datasourceId': 8,
'interval': '', 'intervalMs': interval * 1000,
# 'key': 'Q-c5c08c6b-7591-424c-8eac-53837fa51e89-1', }, {
'queryType': 'timeSeriesQuery', 'datasource': {
'exemplar': False, 'type': 'prometheus',
# 'requestId': 'Q-8b3dabd7-358e-45ed-a9ba-7be3f5fcf274-0Q-c5c08c6b-7591-424c-8eac-53837fa51e89-1B', 'uid': 'DAMPdbiIz'
'utcOffsetSec': -25200,
'legendFormat': '',
'datasourceId': 1,
'intervalMs': interval * 1000,
# 'maxDataPoints': 10,
}, },
], 'editorMode': 'code',
'expr': 'rate(synapse_http_matrixfederationclient_requests_total{job=~"synapse",index=~".*",instance="172.0.2.118:9000"}[2m])',
'range': True,
'refId': 'B',
'interval': '',
'exemplar': False,
'utcOffsetSec': 0,
'legendFormat': '',
'datasourceId': 8,
'intervalMs': interval * 1000,
}],
'from': f'now-{data_range}m', 'from': f'now-{data_range}m',
'to': 'now', 'to': 'now'
} }
response = requests.post(f'{endpoint}/api/ds/query', headers={'Authorization': f'Bearer {api_key}'}, json=json_data, verify=False).json() response = requests.post(f'{endpoint}/api/ds/query', headers={'Authorization': f'Bearer {api_key}'}, json=json_data, verify=False).json()
print(response)
output = {} output = {}
for letter, result in response['results'].items(): for letter, result in response['results'].items():
name = result['frames'][0]['schema']['name'].split('=')[-1].strip('}').strip('"') if len(result['frames']):
output[name] = np.round(np.average(result['frames'][0]['data']['values'][1]), 2) name = result['frames'][0]['schema']['name'].split('=')[-1].strip('}').strip('"')
output[name] = np.round(np.average(result['frames'][0]['data']['values'][1]), 2)
return output return output
# return {
# 'GET': np.round(np.average(response['results']['A']['frames'][0]['data']['values'][1]), 2),
# 'POST': np.round(np.average(response['results']['A']['frames'][1]['data']['values'][1]), 2),
# 'PUT': np.round(np.average(response['results']['A']['frames'][2]['data']['values'][1]), 2),
# 'fedr_GET': np.round(np.average(response['results']['B']['frames'][0]['data']['values'][1]), 2)
# }
def get_event_send_time(api_key, interval, data_range, endpoint): def get_event_send_time(api_key, interval, data_range, endpoint):
@ -118,166 +113,144 @@ def get_event_send_time(api_key, interval, data_range, endpoint):
{ {
'datasource': { 'datasource': {
'type': 'prometheus', 'type': 'prometheus',
'uid': 'AbuT5CJ4z', 'uid': 'DAMPdbiIz',
}, },
'expr': 'histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="10.0.0.34:9000",code=~"2.."}[2m])) by (le))', 'expr': 'histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m])) by (le))',
'format': 'time_series', 'format': 'time_series',
'intervalFactor': 1, 'intervalFactor': 1,
'refId': 'D', 'refId': 'D',
'interval': '', 'interval': '',
# 'key': 'Q-d8eb3572-9aea-4a73-92f2-e08b33c21ecb-0',
'editorMode': 'builder', 'editorMode': 'builder',
'range': True, 'range': True,
'instant': True, 'instant': True,
'queryType': 'timeSeriesQuery', 'queryType': 'timeSeriesQuery',
'exemplar': False, 'exemplar': False,
# 'requestId': 'Q-d8eb3572-9aea-4a73-92f2-e08b33c21ecb-0Q-a9222e59-18ff-4b3b-80ae-27bea8f149a9-1Q-0378a458-1ade-410e-a4b3-ae4aaa91d709-2Q-da4c00b6-61c1-49f5-8a0a-9f19990acfb7-3Q-21254889-3cf6-4d97-8dc5-ddf68360847e-4Q-502b8ed5-4050-461c-befc-76f6796dce68-5Q-364dc896-c399-4e58-8930-cba2e3d1d579-6Q-9072e904-da8d-4b00-b454-dac45b7c38f0-7D',
'utcOffsetSec': -25200, 'utcOffsetSec': -25200,
'legendFormat': '', 'legendFormat': '',
'datasourceId': 1, 'datasourceId': 8,
'intervalMs': interval * 1000, 'intervalMs': interval * 1000,
# 'maxDataPoints': 1383,
}, },
{ {
'datasource': { 'datasource': {
'type': 'prometheus', 'type': 'prometheus',
'uid': 'AbuT5CJ4z', 'uid': 'DAMPdbiIz',
}, },
'expr': 'histogram_quantile(0.9, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="10.0.0.34:9000",code=~"2.."}[2m])) by (le))', 'expr': 'histogram_quantile(0.9, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m])) by (le))',
'format': 'time_series', 'format': 'time_series',
'interval': '', 'interval': '',
'intervalFactor': 1, 'intervalFactor': 1,
'refId': 'A', 'refId': 'A',
# 'key': 'Q-a9222e59-18ff-4b3b-80ae-27bea8f149a9-1',
'queryType': 'timeSeriesQuery', 'queryType': 'timeSeriesQuery',
'exemplar': False, 'exemplar': False,
# 'requestId': 'Q-d8eb3572-9aea-4a73-92f2-e08b33c21ecb-0Q-a9222e59-18ff-4b3b-80ae-27bea8f149a9-1Q-0378a458-1ade-410e-a4b3-ae4aaa91d709-2Q-da4c00b6-61c1-49f5-8a0a-9f19990acfb7-3Q-21254889-3cf6-4d97-8dc5-ddf68360847e-4Q-502b8ed5-4050-461c-befc-76f6796dce68-5Q-364dc896-c399-4e58-8930-cba2e3d1d579-6Q-9072e904-da8d-4b00-b454-dac45b7c38f0-7A',
'utcOffsetSec': -25200, 'utcOffsetSec': -25200,
'legendFormat': '', 'legendFormat': '',
'datasourceId': 1, 'datasourceId': 8,
'intervalMs': interval * 1000, 'intervalMs': interval * 1000,
# 'maxDataPoints': 1383,
}, },
{ {
'datasource': { 'datasource': {
'type': 'prometheus', 'type': 'prometheus',
'uid': 'AbuT5CJ4z', 'uid': 'DAMPdbiIz',
}, },
'expr': 'histogram_quantile(0.75, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="10.0.0.34:9000",code=~"2.."}[2m])) by (le))', 'expr': 'histogram_quantile(0.75, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m])) by (le))',
'format': 'time_series', 'format': 'time_series',
'intervalFactor': 1, 'intervalFactor': 1,
'refId': 'C', 'refId': 'C',
'interval': '', 'interval': '',
# 'key': 'Q-0378a458-1ade-410e-a4b3-ae4aaa91d709-2',
'queryType': 'timeSeriesQuery', 'queryType': 'timeSeriesQuery',
'exemplar': False, 'exemplar': False,
# 'requestId': 'Q-d8eb3572-9aea-4a73-92f2-e08b33c21ecb-0Q-a9222e59-18ff-4b3b-80ae-27bea8f149a9-1Q-0378a458-1ade-410e-a4b3-ae4aaa91d709-2Q-da4c00b6-61c1-49f5-8a0a-9f19990acfb7-3Q-21254889-3cf6-4d97-8dc5-ddf68360847e-4Q-502b8ed5-4050-461c-befc-76f6796dce68-5Q-364dc896-c399-4e58-8930-cba2e3d1d579-6Q-9072e904-da8d-4b00-b454-dac45b7c38f0-7C',
'utcOffsetSec': -25200, 'utcOffsetSec': -25200,
'legendFormat': '', 'legendFormat': '',
'datasourceId': 1, 'datasourceId': 8,
'intervalMs': interval * 1000, 'intervalMs': interval * 1000,
# 'maxDataPoints': 1383,
}, },
{ {
'datasource': { 'datasource': {
'type': 'prometheus', 'type': 'prometheus',
'uid': 'AbuT5CJ4z', 'uid': 'DAMPdbiIz',
}, },
'expr': 'histogram_quantile(0.5, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="10.0.0.34:9000",code=~"2.."}[2m])) by (le))', 'expr': 'histogram_quantile(0.5, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m])) by (le))',
'format': 'time_series', 'format': 'time_series',
'intervalFactor': 1, 'intervalFactor': 1,
'refId': 'B', 'refId': 'B',
'interval': '', 'interval': '',
# 'key': 'Q-da4c00b6-61c1-49f5-8a0a-9f19990acfb7-3',
'queryType': 'timeSeriesQuery', 'queryType': 'timeSeriesQuery',
'exemplar': False, 'exemplar': False,
# 'requestId': 'Q-d8eb3572-9aea-4a73-92f2-e08b33c21ecb-0Q-a9222e59-18ff-4b3b-80ae-27bea8f149a9-1Q-0378a458-1ade-410e-a4b3-ae4aaa91d709-2Q-da4c00b6-61c1-49f5-8a0a-9f19990acfb7-3Q-21254889-3cf6-4d97-8dc5-ddf68360847e-4Q-502b8ed5-4050-461c-befc-76f6796dce68-5Q-364dc896-c399-4e58-8930-cba2e3d1d579-6Q-9072e904-da8d-4b00-b454-dac45b7c38f0-7B',
'utcOffsetSec': -25200, 'utcOffsetSec': -25200,
'legendFormat': '', 'legendFormat': '',
'datasourceId': 1, 'datasourceId': 8,
'intervalMs': interval * 1000, 'intervalMs': interval * 1000,
# 'maxDataPoints': 1383,
}, },
{ {
'datasource': { 'datasource': {
'type': 'prometheus', 'type': 'prometheus',
'uid': 'AbuT5CJ4z', 'uid': 'DAMPdbiIz',
}, },
'expr': 'histogram_quantile(0.25, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="10.0.0.34:9000",code=~"2.."}[2m])) by (le))', 'expr': 'histogram_quantile(0.25, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m])) by (le))',
'refId': 'F', 'refId': 'F',
'interval': '', 'interval': '',
# 'key': 'Q-21254889-3cf6-4d97-8dc5-ddf68360847e-4',
'queryType': 'timeSeriesQuery', 'queryType': 'timeSeriesQuery',
'exemplar': False, 'exemplar': False,
# 'requestId': 'Q-d8eb3572-9aea-4a73-92f2-e08b33c21ecb-0Q-a9222e59-18ff-4b3b-80ae-27bea8f149a9-1Q-0378a458-1ade-410e-a4b3-ae4aaa91d709-2Q-da4c00b6-61c1-49f5-8a0a-9f19990acfb7-3Q-21254889-3cf6-4d97-8dc5-ddf68360847e-4Q-502b8ed5-4050-461c-befc-76f6796dce68-5Q-364dc896-c399-4e58-8930-cba2e3d1d579-6Q-9072e904-da8d-4b00-b454-dac45b7c38f0-7F',
'utcOffsetSec': -25200, 'utcOffsetSec': -25200,
'legendFormat': '', 'legendFormat': '',
'datasourceId': 1, 'datasourceId': 8,
'intervalMs': interval * 1000, 'intervalMs': interval * 1000,
# 'maxDataPoints': 1383,
}, },
{ {
'datasource': { 'datasource': {
'type': 'prometheus', 'type': 'prometheus',
'uid': 'AbuT5CJ4z', 'uid': 'DAMPdbiIz',
}, },
'expr': 'histogram_quantile(0.05, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="10.0.0.34:9000",code=~"2.."}[2m])) by (le))', 'expr': 'histogram_quantile(0.05, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m])) by (le))',
'refId': 'G', 'refId': 'G',
'interval': '', 'interval': '',
# 'key': 'Q-502b8ed5-4050-461c-befc-76f6796dce68-5',
'queryType': 'timeSeriesQuery', 'queryType': 'timeSeriesQuery',
'exemplar': False, 'exemplar': False,
# 'requestId': 'Q-d8eb3572-9aea-4a73-92f2-e08b33c21ecb-0Q-a9222e59-18ff-4b3b-80ae-27bea8f149a9-1Q-0378a458-1ade-410e-a4b3-ae4aaa91d709-2Q-da4c00b6-61c1-49f5-8a0a-9f19990acfb7-3Q-21254889-3cf6-4d97-8dc5-ddf68360847e-4Q-502b8ed5-4050-461c-befc-76f6796dce68-5Q-364dc896-c399-4e58-8930-cba2e3d1d579-6Q-9072e904-da8d-4b00-b454-dac45b7c38f0-7G',
'utcOffsetSec': -25200, 'utcOffsetSec': -25200,
'legendFormat': '', 'legendFormat': '',
'datasourceId': 1, 'datasourceId': 8,
'intervalMs': interval * 1000, 'intervalMs': interval * 1000,
# 'maxDataPoints': 1383,
}, },
{ {
'datasource': { 'datasource': {
'type': 'prometheus', 'type': 'prometheus',
'uid': 'AbuT5CJ4z', 'uid': 'DAMPdbiIz',
}, },
'expr': 'sum(rate(synapse_http_server_response_time_seconds_sum{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="10.0.0.34:9000",code=~"2.."}[2m])) / sum(rate(synapse_http_server_response_time_seconds_count{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="10.0.0.34:9000",code=~"2.."}[2m]))', 'expr': 'sum(rate(synapse_http_server_response_time_seconds_sum{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m])) / sum(rate(synapse_http_server_response_time_seconds_count{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m]))',
'refId': 'H', 'refId': 'H',
'interval': '', 'interval': '',
# 'key': 'Q-364dc896-c399-4e58-8930-cba2e3d1d579-6',
'queryType': 'timeSeriesQuery', 'queryType': 'timeSeriesQuery',
'exemplar': False, 'exemplar': False,
# 'requestId': 'Q-d8eb3572-9aea-4a73-92f2-e08b33c21ecb-0Q-a9222e59-18ff-4b3b-80ae-27bea8f149a9-1Q-0378a458-1ade-410e-a4b3-ae4aaa91d709-2Q-da4c00b6-61c1-49f5-8a0a-9f19990acfb7-3Q-21254889-3cf6-4d97-8dc5-ddf68360847e-4Q-502b8ed5-4050-461c-befc-76f6796dce68-5Q-364dc896-c399-4e58-8930-cba2e3d1d579-6Q-9072e904-da8d-4b00-b454-dac45b7c38f0-7H',
'utcOffsetSec': -25200, 'utcOffsetSec': -25200,
'legendFormat': '', 'legendFormat': '',
'datasourceId': 1, 'datasourceId': 8,
'intervalMs': interval * 1000, 'intervalMs': interval * 1000,
# 'maxDataPoints': 1383,
}, },
{ {
'datasource': { 'datasource': {
'type': 'prometheus', 'type': 'prometheus',
'uid': 'AbuT5CJ4z', 'uid': 'DAMPdbiIz',
}, },
'expr': 'sum(rate(synapse_storage_events_persisted_events_total{instance="10.0.0.34:9000"}[2m]))', 'expr': 'sum(rate(synapse_storage_events_persisted_events_total{instance="172.0.2.118:9000"}[2m]))',
'hide': False, 'hide': False,
'instant': False, 'instant': False,
'refId': 'E', 'refId': 'E',
'interval': '', 'interval': '',
# 'key': 'Q-9072e904-da8d-4b00-b454-dac45b7c38f0-7',
'editorMode': 'code', 'editorMode': 'code',
'queryType': 'timeSeriesQuery', 'queryType': 'timeSeriesQuery',
'exemplar': False, 'exemplar': False,
# 'requestId': 'Q-d8eb3572-9aea-4a73-92f2-e08b33c21ecb-0Q-a9222e59-18ff-4b3b-80ae-27bea8f149a9-1Q-0378a458-1ade-410e-a4b3-ae4aaa91d709-2Q-da4c00b6-61c1-49f5-8a0a-9f19990acfb7-3Q-21254889-3cf6-4d97-8dc5-ddf68360847e-4Q-502b8ed5-4050-461c-befc-76f6796dce68-5Q-364dc896-c399-4e58-8930-cba2e3d1d579-6Q-9072e904-da8d-4b00-b454-dac45b7c38f0-7E',
'utcOffsetSec': -25200, 'utcOffsetSec': -25200,
'legendFormat': '', 'legendFormat': '',
'datasourceId': 1, 'datasourceId': 8,
'intervalMs': interval * 1000, 'intervalMs': interval * 1000,
# 'maxDataPoints': 1383,
}, },
], ],
'from': f'now-{data_range}m', 'from': f'now-{data_range}m',
'to': 'now', 'to': 'now',
} }
response = requests.post(f'{endpoint}/api/ds/query', headers={'Authorization': f'Bearer {api_key}'}, json=json_data, verify=False).json() response = requests.post(f'{endpoint}/api/ds/query', headers={'Authorization': f'Bearer {api_key}'}, json=json_data, verify=False).json()
if not response['results'].get('E', {}).get('frames'):
return None
return np.round(np.average(response['results']['E']['frames'][0]['data']['values'][1]), 2) return np.round(np.average(response['results']['E']['frames'][0]['data']['values'][1]), 2)
@ -287,23 +260,20 @@ def get_waiting_for_db(api_key, interval, data_range, endpoint):
{ {
'datasource': { 'datasource': {
'type': 'prometheus', 'type': 'prometheus',
'uid': 'AbuT5CJ4z', 'uid': 'DAMPdbiIz',
}, },
'expr': 'rate(synapse_storage_schedule_time_sum{instance="10.0.0.34:9000",job=~"(federation-receiver|federation-sender|initialsync|synapse|synchrotron)",index=~".*"}[30s])/rate(synapse_storage_schedule_time_count[30s])', 'expr': 'rate(synapse_storage_schedule_time_sum{instance="172.0.2.118:9000",job=~"synapse",index=~".*"}[30s])/rate(synapse_storage_schedule_time_count[30s])',
'format': 'time_series', 'format': 'time_series',
'intervalFactor': 2, 'intervalFactor': 2,
'refId': 'A', 'refId': 'A',
'step': 20, 'step': 20,
'interval': '', 'interval': '',
# 'key': 'Q-459af7f4-0427-4832-9353-46086b3f5c27-0',
'queryType': 'timeSeriesQuery', 'queryType': 'timeSeriesQuery',
'exemplar': False, 'exemplar': False,
# 'requestId': 'Q-459af7f4-0427-4832-9353-46086b3f5c27-0A',
'utcOffsetSec': -25200, 'utcOffsetSec': -25200,
'legendFormat': '', 'legendFormat': '',
'datasourceId': 1, 'datasourceId': 8,
'intervalMs': interval * 1000, 'intervalMs': interval * 1000,
# 'maxDataPoints': 1383,
}, },
], ],
'from': f'now-{data_range}m', 'from': f'now-{data_range}m',
@ -331,10 +301,10 @@ def get_stateres_worst_case(api_key, interval, data_range, endpoint):
{ {
'datasource': { 'datasource': {
'type': 'prometheus', 'type': 'prometheus',
'uid': 'AbuT5CJ4z', 'uid': 'DAMPdbiIz',
}, },
'exemplar': False, 'exemplar': False,
'expr': 'sum(rate(synapse_state_res_db_for_biggest_room_seconds_total{instance="10.0.0.34:9000"}[1m]))', 'expr': 'sum(rate(synapse_state_res_db_for_biggest_room_seconds_total{instance="172.0.2.118:9000"}[1m]))',
'format': 'time_series', 'format': 'time_series',
'hide': False, 'hide': False,
'instant': False, 'instant': False,
@ -343,17 +313,17 @@ def get_stateres_worst_case(api_key, interval, data_range, endpoint):
'queryType': 'timeSeriesQuery', 'queryType': 'timeSeriesQuery',
'utcOffsetSec': -25200, 'utcOffsetSec': -25200,
'legendFormat': '', 'legendFormat': '',
'datasourceId': 1, 'datasourceId': 8,
'intervalMs': 15000, 'intervalMs': 15000,
'maxDataPoints': 1863, 'maxDataPoints': 1863,
}, },
{ {
'datasource': { 'datasource': {
'type': 'prometheus', 'type': 'prometheus',
'uid': 'AbuT5CJ4z', 'uid': 'DAMPdbiIz',
}, },
'exemplar': False, 'exemplar': False,
'expr': 'sum(rate(synapse_state_res_cpu_for_biggest_room_seconds_total{instance="10.0.0.34:9000"}[1m]))', 'expr': 'sum(rate(synapse_state_res_cpu_for_biggest_room_seconds_total{instance="172.0.2.118:9000"}[1m]))',
'format': 'time_series', 'format': 'time_series',
'hide': False, 'hide': False,
'instant': False, 'instant': False,
@ -362,7 +332,7 @@ def get_stateres_worst_case(api_key, interval, data_range, endpoint):
'queryType': 'timeSeriesQuery', 'queryType': 'timeSeriesQuery',
'utcOffsetSec': -25200, 'utcOffsetSec': -25200,
'legendFormat': '', 'legendFormat': '',
'datasourceId': 1, 'datasourceId': 8,
'intervalMs': 15000, 'intervalMs': 15000,
'maxDataPoints': 1863, 'maxDataPoints': 1863,
}, },