icinga2-checks/checker/synapse_grafana.py

import time

import numpy as np
import requests
from urllib3.exceptions import InsecureRequestWarning

requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)


def timestamp_minutes_ago(minutes):
    current_time = time.time()
    minutes_in_seconds = minutes * 60
    time_in_past = current_time - minutes_in_seconds
    timestamp_in_ms = int(time_in_past * 1000)
    return timestamp_in_ms


def get_avg_python_gc_time(api_key, interval, data_range, endpoint):
    json_data = {
        'queries': [
            {
                'datasource': {
                    'type': 'prometheus',
                    'uid': 'DAMPdbiIz',
                },
                'expr': 'rate(python_gc_time_sum{instance="matrix.synapse",job=~".*",index=~".*"}[2m])/rate(python_gc_time_count[2m])',
                'format': 'time_series',
                'intervalFactor': 2,
                'refId': 'A',
                'step': 20,
                'target': '',
                'interval': '',
                'queryType': 'timeSeriesQuery',
                'exemplar': False,
                'utcOffsetSec': -25200,
                'legendFormat': '',
                'datasourceId': 8,
                'intervalMs': interval * 1000,
            },
        ],
        'from': f'now-{data_range}m',
        'to': 'now',
    }
    response = requests.post(f'{endpoint}/api/ds/query', headers={'Authorization': f'Bearer {api_key}'}, json=json_data, verify=False).json()
    if not response['results'].get('A', {}).get('frames'):
        return []

    good = []
    for i in response['results']['A']['frames']:
        # This one can sometimes be null
        new = []
        for x in range(len(i['data']['values'][1])):
            if i['data']['values'][1][x] is not None:
                new.append(i['data']['values'][1][x])
        good.append(new)
    # Remove empty arrays
    results = []
    for x in good:
        if len(x) > 0:
            results.append(x)
    return [np.round(np.average(i), 5) for i in results]


def get_outgoing_http_request_rate(api_key, interval, data_range, endpoint):
    json_data = {
        'queries': [{
            'datasource': {
                'type': 'prometheus',
                'uid': 'DAMPdbiIz'
            },
            'editorMode': 'code',
            'expr': 'rate(synapse_http_client_requests_total{job=~".*",index=~".*",instance="matrix.synapse"}[2m])',
            'range': True,
            'refId': 'A',
            'interval': '',
            'exemplar': False,
            'utcOffsetSec': 0,
            'legendFormat': '',
            'datasourceId': 8,
            'intervalMs': interval * 1000,
        }, {
            'datasource': {
                'type': 'prometheus',
                'uid': 'DAMPdbiIz'
            },
            'editorMode': 'code',
            'expr': 'rate(synapse_http_matrixfederationclient_requests_total{job=~".*",index=~".*",instance="matrix.synapse"}[2m])',
            'range': True,
            'refId': 'B',
            'interval': '',
            'exemplar': False,
            'utcOffsetSec': 0,
            'legendFormat': '',
            'datasourceId': 8,
            'intervalMs': interval * 1000,
        }],
        'from': f'now-{data_range}m',
        'to': 'now'
    }
    response = requests.post(f'{endpoint}/api/ds/query', headers={'Authorization': f'Bearer {api_key}'}, json=json_data, verify=False).json()
    output = {}
    for letter, result in response['results'].items():
        if len(result['frames']):
            name = result['frames'][0]['schema']['name'].split('=')[-1].strip('}').strip('"')
            output[name] = np.round(np.average(result['frames'][0]['data']['values'][1]), 2)
    return output


def get_event_send_time(api_key, interval, data_range, endpoint):
    json_data = {
        'queries': [
            {
                'datasource': {
                    'type': 'prometheus',
                    'uid': 'DAMPdbiIz',
                },
                'expr': 'histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="matrix.synapse",code=~"2.."}[2m])) by (le))',
                'format': 'time_series',
                'intervalFactor': 1,
                'refId': 'D',
                'interval': '',
                'editorMode': 'builder',
                'range': True,
                'instant': True,
                'queryType': 'timeSeriesQuery',
                'exemplar': False,
                'utcOffsetSec': -25200,
                'legendFormat': '',
                'datasourceId': 8,
                'intervalMs': interval * 1000,
            },
            {
                'datasource': {
                    'type': 'prometheus',
                    'uid': 'DAMPdbiIz',
                },
                'expr': 'histogram_quantile(0.9, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="matrix.synapse",code=~"2.."}[2m])) by (le))',
                'format': 'time_series',
                'interval': '',
                'intervalFactor': 1,
                'refId': 'A',
                'queryType': 'timeSeriesQuery',
                'exemplar': False,
                'utcOffsetSec': -25200,
                'legendFormat': '',
                'datasourceId': 8,
                'intervalMs': interval * 1000,
            },
            {
                'datasource': {
                    'type': 'prometheus',
                    'uid': 'DAMPdbiIz',
                },
                'expr': 'histogram_quantile(0.75, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="matrix.synapse",code=~"2.."}[2m])) by (le))',
                'format': 'time_series',
                'intervalFactor': 1,
                'refId': 'C',
                'interval': '',
                'queryType': 'timeSeriesQuery',
                'exemplar': False,
                'utcOffsetSec': -25200,
                'legendFormat': '',
                'datasourceId': 8,
                'intervalMs': interval * 1000,
            },
            {
                'datasource': {
                    'type': 'prometheus',
                    'uid': 'DAMPdbiIz',
                },
                'expr': 'histogram_quantile(0.5, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="matrix.synapse",code=~"2.."}[2m])) by (le))',
                'format': 'time_series',
                'intervalFactor': 1,
                'refId': 'B',
                'interval': '',
                'queryType': 'timeSeriesQuery',
                'exemplar': False,
                'utcOffsetSec': -25200,
                'legendFormat': '',
                'datasourceId': 8,
                'intervalMs': interval * 1000,
            },
            {
                'datasource': {
                    'type': 'prometheus',
                    'uid': 'DAMPdbiIz',
                },
                'expr': 'histogram_quantile(0.25, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="matrix.synapse",code=~"2.."}[2m])) by (le))',
                'refId': 'F',
                'interval': '',
                'queryType': 'timeSeriesQuery',
                'exemplar': False,
                'utcOffsetSec': -25200,
                'legendFormat': '',
                'datasourceId': 8,
                'intervalMs': interval * 1000,
            },
            {
                'datasource': {
                    'type': 'prometheus',
                    'uid': 'DAMPdbiIz',
                },
                'expr': 'histogram_quantile(0.05, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="matrix.synapse",code=~"2.."}[2m])) by (le))',
                'refId': 'G',
                'interval': '',
                'queryType': 'timeSeriesQuery',
                'exemplar': False,
                'utcOffsetSec': -25200,
                'legendFormat': '',
                'datasourceId': 8,
                'intervalMs': interval * 1000,
            },
            {
                'datasource': {
                    'type': 'prometheus',
                    'uid': 'DAMPdbiIz',
                },
                'expr': 'sum(rate(synapse_http_server_response_time_seconds_sum{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="matrix.synapse",code=~"2.."}[2m])) / sum(rate(synapse_http_server_response_time_seconds_count{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="matrix.synapse",code=~"2.."}[2m]))',
                'refId': 'H',
                'interval': '',
                'queryType': 'timeSeriesQuery',
                'exemplar': False,
                'utcOffsetSec': -25200,
                'legendFormat': '',
                'datasourceId': 8,
                'intervalMs': interval * 1000,
            },
            {
                'datasource': {
                    'type': 'prometheus',
                    'uid': 'DAMPdbiIz',
                },
                'expr': 'sum(rate(synapse_storage_events_persisted_events_total{instance="matrix.synapse"}[2m]))',
                'hide': False,
                'instant': False,
                'refId': 'E',
                'interval': '',
                'editorMode': 'code',
                'queryType': 'timeSeriesQuery',
                'exemplar': False,
                'utcOffsetSec': -25200,
                'legendFormat': '',
                'datasourceId': 8,
                'intervalMs': interval * 1000,
            },
        ],
        'from': f'now-{data_range}m',
        'to': 'now',
    }
    response = requests.post(f'{endpoint}/api/ds/query', headers={'Authorization': f'Bearer {api_key}'}, json=json_data, verify=False).json()
    if not response['results'].get('E', {}).get('frames'):
        return None
    return np.round(np.average(response['results']['E']['frames'][0]['data']['values'][1]), 2)


def get_waiting_for_db(api_key, interval, data_range, endpoint):
    json_data = {
        'queries': [
            {
                'datasource': {
                    'type': 'prometheus',
                    'uid': 'DAMPdbiIz',
                },
                'expr': 'rate(synapse_storage_schedule_time_sum{instance="matrix.synapse",job=~".*",index=~".*"}[30s])/rate(synapse_storage_schedule_time_count[30s])',
                'format': 'time_series',
                'intervalFactor': 2,
                'refId': 'A',
                'step': 20,
                'interval': '',
                'queryType': 'timeSeriesQuery',
                'exemplar': False,
                'utcOffsetSec': -25200,
                'legendFormat': '',
                'datasourceId': 8,
                'intervalMs': interval * 1000,
            },
        ],
        'from': f'now-{data_range}m',
        'to': 'now',
    }
    response = requests.post(f'{endpoint}/api/ds/query', headers={'Authorization': f'Bearer {api_key}'}, json=json_data, verify=False).json()
    if not len(response['results']['A']['frames']):
        return None, None, None
    raw_data = response['results']['A']['frames'][0]['data']['values'][1]
    data = []
    null_present = False
    for i in range(len(raw_data)):
        if raw_data[i] is not None:
            data.append(raw_data[i])
        else:
            null_present = True
    return np.round(np.average(data), 5), null_present, raw_data


def get_stateres_worst_case(api_key, interval, data_range, endpoint):
    """
    CPU and DB time spent on most expensive state resolution in a room, summed over all workers.
    This is a very rough proxy for "how fast is state res", but it doesn't accurately represent the system load (e.g. it completely ignores cheap state resolutions).
    """
    json_data = {
        'queries': [
            {
                'datasource': {
                    'type': 'prometheus',
                    'uid': 'DAMPdbiIz',
                },
                'exemplar': False,
                'expr': 'sum(rate(synapse_state_res_db_for_biggest_room_seconds_total{instance="matrix.synapse"}[1m]))',
                'format': 'time_series',
                'hide': False,
                'instant': False,
                'interval': '',
                'refId': 'B',
                'queryType': 'timeSeriesQuery',
                'utcOffsetSec': -25200,
                'legendFormat': '',
                'datasourceId': 8,
                'intervalMs': 15000,
                'maxDataPoints': 1863,
            },
            {
                'datasource': {
                    'type': 'prometheus',
                    'uid': 'DAMPdbiIz',
                },
                'exemplar': False,
                'expr': 'sum(rate(synapse_state_res_cpu_for_biggest_room_seconds_total{instance="matrix.synapse"}[1m]))',
                'format': 'time_series',
                'hide': False,
                'instant': False,
                'interval': '',
                'refId': 'C',
                'queryType': 'timeSeriesQuery',
                'utcOffsetSec': -25200,
                'legendFormat': '',
                'datasourceId': 8,
                'intervalMs': 15000,
                'maxDataPoints': 1863,
            },
        ],
        'range': {
            'from': '2023-02-23T04:36:12.870Z',
            'to': '2023-02-23T07:36:12.870Z',
            'raw': {
                'from': 'now-3h',
                'to': 'now',
            },
        },
        'from': f'now-{data_range}m',
        'to': 'now',
    }
    response = requests.post(f'{endpoint}/api/ds/query', headers={'Authorization': f'Bearer {api_key}'}, json=json_data, verify=False).json()

# AVerage CPU time per block