cyberes
/
icinga2-checks
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

check_iperf3: add retry if iperf3 server is busy\ 

check_matrix_synapse: fix
This commit is contained in:
Cyberes 2023-09-16 13:33:45 -06:00
parent 9bd9ba93a7
commit 7f95ce919e
2 changed files with 40 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
check_iperf3.sh
View File

@ -34,7 +34,6 @@
# Warnng and critical levels are based on your specific network speed. # Warnng and critical levels are based on your specific network speed.
# Default values # Default values
SERVER="" SERVER=""
WARNING_LEVEL="" WARNING_LEVEL=""
@ -42,8 +41,8 @@ CRITICAL_LEVEL=""
RSA_PUBLIC_KEY="" RSA_PUBLIC_KEY=""
USERNAME="" USERNAME=""
PASSWORD="" PASSWORD=""
RETRY=3
# Parse named arguments
while [[ $# -gt 0 ]]; do while [[ $# -gt 0 ]]; do
key="$1" key="$1"
@ -78,6 +77,11 @@ while [[ $# -gt 0 ]]; do
shift shift
shift shift
;; ;;
--retry)
RETRY="$2"
shift
shift
;;
*) *)
shift shift
;; ;;
@ -99,17 +103,27 @@ fi
export IPERF3_PASSWORD="$PASSWORD" export IPERF3_PASSWORD="$PASSWORD"
# Run iperf3 command with optional arguments # Run iperf3 command with optional arguments
if [[ -n "$RSA_PUBLIC_KEY" ]] && [[ -n "$USERNAME" ]]; then for ((i = 1; i <= RETRY; i++)); do
OUTPUT=$(iperf3 -c "$SERVER" -i 1 -t 10 -f m --rsa-public-key-path "$RSA_PUBLIC_KEY" --username "$USERNAME" 2>&1) if [[ -n "$RSA_PUBLIC_KEY" ]] && [[ -n "$USERNAME" ]]; then
else OUTPUT=$(iperf3 -c "$SERVER" -i 1 -t 30 -f m --rsa-public-key-path "$RSA_PUBLIC_KEY" --username "$USERNAME" 2>&1)
OUTPUT=$(iperf3 -c "$SERVER" -i 1 -t 10 -f m 2>&1) else
fi OUTPUT=$(iperf3 -c "$SERVER" -i 1 -t 10 -f m 2>&1)
fi
# Check if iperf3 command failed # Check if iperf3 command failed
if [[ $? -ne 0 ]]; then if [[ $? -ne 0 ]]; then
echo -e "UNKNOWN - iperf3 command failed: $OUTPUT\n" if [[ $OUTPUT == *"the server is busy running a test. try again later"* ]]; then
exit -1 if [[ $i -lt $RETRY ]]; then
fi sleep 30
continue
fi
fi
echo -e "UNKNOWN - iperf3 command failed: $OUTPUT\n"
exit -1
else
break
fi
done
# Extract the receiver bitrate # Extract the receiver bitrate
RECEIVER_BITRATE=$(echo "$OUTPUT" | grep -Eo '[0-9]+(\.[0-9]+)? Mbits/sec' | tail -1 | awk '{print $1}') RECEIVER_BITRATE=$(echo "$OUTPUT" | grep -Eo '[0-9]+(\.[0-9]+)? Mbits/sec' | tail -1 | awk '{print $1}')

28
checker/synapse_grafana.py
View File

@ -23,7 +23,7 @@ def get_avg_python_gc_time(api_key, interval, data_range, endpoint):
'type': 'prometheus', 'type': 'prometheus',
'uid': 'DAMPdbiIz', 'uid': 'DAMPdbiIz',
}, },
'expr': 'rate(python_gc_time_sum{instance="172.0.2.118:9000",job=~"synapse",index=~".*"}[2m])/rate(python_gc_time_count[2m])', 'expr': 'rate(python_gc_time_sum{instance="matrix.synapse",job=~".*",index=~".*"}[2m])/rate(python_gc_time_count[2m])',
'format': 'time_series', 'format': 'time_series',
'intervalFactor': 2, 'intervalFactor': 2,
'refId': 'A', 'refId': 'A',
@ -69,7 +69,7 @@ def get_outgoing_http_request_rate(api_key, interval, data_range, endpoint):
'uid': 'DAMPdbiIz' 'uid': 'DAMPdbiIz'
}, },
'editorMode': 'code', 'editorMode': 'code',
'expr': 'rate(synapse_http_client_requests_total{job=~"synapse",index=~".*",instance="172.0.2.118:9000"}[2m])', 'expr': 'rate(synapse_http_client_requests_total{job=~".*",index=~".*",instance="matrix.synapse"}[2m])',
'range': True, 'range': True,
'refId': 'A', 'refId': 'A',
'interval': '', 'interval': '',
@ -84,7 +84,7 @@ def get_outgoing_http_request_rate(api_key, interval, data_range, endpoint):
'uid': 'DAMPdbiIz' 'uid': 'DAMPdbiIz'
}, },
'editorMode': 'code', 'editorMode': 'code',
'expr': 'rate(synapse_http_matrixfederationclient_requests_total{job=~"synapse",index=~".*",instance="172.0.2.118:9000"}[2m])', 'expr': 'rate(synapse_http_matrixfederationclient_requests_total{job=~".*",index=~".*",instance="matrix.synapse"}[2m])',
'range': True, 'range': True,
'refId': 'B', 'refId': 'B',
'interval': '', 'interval': '',
@ -114,7 +114,7 @@ def get_event_send_time(api_key, interval, data_range, endpoint):
'type': 'prometheus', 'type': 'prometheus',
'uid': 'DAMPdbiIz', 'uid': 'DAMPdbiIz',
}, },
'expr': 'histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m])) by (le))', 'expr': 'histogram_quantile(0.99, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="matrix.synapse",code=~"2.."}[2m])) by (le))',
'format': 'time_series', 'format': 'time_series',
'intervalFactor': 1, 'intervalFactor': 1,
'refId': 'D', 'refId': 'D',
@ -134,7 +134,7 @@ def get_event_send_time(api_key, interval, data_range, endpoint):
'type': 'prometheus', 'type': 'prometheus',
'uid': 'DAMPdbiIz', 'uid': 'DAMPdbiIz',
}, },
'expr': 'histogram_quantile(0.9, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m])) by (le))', 'expr': 'histogram_quantile(0.9, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="matrix.synapse",code=~"2.."}[2m])) by (le))',
'format': 'time_series', 'format': 'time_series',
'interval': '', 'interval': '',
'intervalFactor': 1, 'intervalFactor': 1,
@ -151,7 +151,7 @@ def get_event_send_time(api_key, interval, data_range, endpoint):
'type': 'prometheus', 'type': 'prometheus',
'uid': 'DAMPdbiIz', 'uid': 'DAMPdbiIz',
}, },
'expr': 'histogram_quantile(0.75, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m])) by (le))', 'expr': 'histogram_quantile(0.75, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="matrix.synapse",code=~"2.."}[2m])) by (le))',
'format': 'time_series', 'format': 'time_series',
'intervalFactor': 1, 'intervalFactor': 1,
'refId': 'C', 'refId': 'C',
@ -168,7 +168,7 @@ def get_event_send_time(api_key, interval, data_range, endpoint):
'type': 'prometheus', 'type': 'prometheus',
'uid': 'DAMPdbiIz', 'uid': 'DAMPdbiIz',
}, },
'expr': 'histogram_quantile(0.5, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m])) by (le))', 'expr': 'histogram_quantile(0.5, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="matrix.synapse",code=~"2.."}[2m])) by (le))',
'format': 'time_series', 'format': 'time_series',
'intervalFactor': 1, 'intervalFactor': 1,
'refId': 'B', 'refId': 'B',
@ -185,7 +185,7 @@ def get_event_send_time(api_key, interval, data_range, endpoint):
'type': 'prometheus', 'type': 'prometheus',
'uid': 'DAMPdbiIz', 'uid': 'DAMPdbiIz',
}, },
'expr': 'histogram_quantile(0.25, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m])) by (le))', 'expr': 'histogram_quantile(0.25, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="matrix.synapse",code=~"2.."}[2m])) by (le))',
'refId': 'F', 'refId': 'F',
'interval': '', 'interval': '',
'queryType': 'timeSeriesQuery', 'queryType': 'timeSeriesQuery',
@ -200,7 +200,7 @@ def get_event_send_time(api_key, interval, data_range, endpoint):
'type': 'prometheus', 'type': 'prometheus',
'uid': 'DAMPdbiIz', 'uid': 'DAMPdbiIz',
}, },
'expr': 'histogram_quantile(0.05, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m])) by (le))', 'expr': 'histogram_quantile(0.05, sum(rate(synapse_http_server_response_time_seconds_bucket{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="matrix.synapse",code=~"2.."}[2m])) by (le))',
'refId': 'G', 'refId': 'G',
'interval': '', 'interval': '',
'queryType': 'timeSeriesQuery', 'queryType': 'timeSeriesQuery',
@ -215,7 +215,7 @@ def get_event_send_time(api_key, interval, data_range, endpoint):
'type': 'prometheus', 'type': 'prometheus',
'uid': 'DAMPdbiIz', 'uid': 'DAMPdbiIz',
}, },
'expr': 'sum(rate(synapse_http_server_response_time_seconds_sum{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m])) / sum(rate(synapse_http_server_response_time_seconds_count{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="172.0.2.118:9000",code=~"2.."}[2m]))', 'expr': 'sum(rate(synapse_http_server_response_time_seconds_sum{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="matrix.synapse",code=~"2.."}[2m])) / sum(rate(synapse_http_server_response_time_seconds_count{servlet=\'RoomSendEventRestServlet\',index=~".*",instance="matrix.synapse",code=~"2.."}[2m]))',
'refId': 'H', 'refId': 'H',
'interval': '', 'interval': '',
'queryType': 'timeSeriesQuery', 'queryType': 'timeSeriesQuery',
@ -230,7 +230,7 @@ def get_event_send_time(api_key, interval, data_range, endpoint):
'type': 'prometheus', 'type': 'prometheus',
'uid': 'DAMPdbiIz', 'uid': 'DAMPdbiIz',
}, },
'expr': 'sum(rate(synapse_storage_events_persisted_events_total{instance="172.0.2.118:9000"}[2m]))', 'expr': 'sum(rate(synapse_storage_events_persisted_events_total{instance="matrix.synapse"}[2m]))',
'hide': False, 'hide': False,
'instant': False, 'instant': False,
'refId': 'E', 'refId': 'E',
@ -261,7 +261,7 @@ def get_waiting_for_db(api_key, interval, data_range, endpoint):
'type': 'prometheus', 'type': 'prometheus',
'uid': 'DAMPdbiIz', 'uid': 'DAMPdbiIz',
}, },
'expr': 'rate(synapse_storage_schedule_time_sum{instance="172.0.2.118:9000",job=~"synapse",index=~".*"}[30s])/rate(synapse_storage_schedule_time_count[30s])', 'expr': 'rate(synapse_storage_schedule_time_sum{instance="matrix.synapse",job=~".*",index=~".*"}[30s])/rate(synapse_storage_schedule_time_count[30s])',
'format': 'time_series', 'format': 'time_series',
'intervalFactor': 2, 'intervalFactor': 2,
'refId': 'A', 'refId': 'A',
@ -305,7 +305,7 @@ def get_stateres_worst_case(api_key, interval, data_range, endpoint):
'uid': 'DAMPdbiIz', 'uid': 'DAMPdbiIz',
}, },
'exemplar': False, 'exemplar': False,
'expr': 'sum(rate(synapse_state_res_db_for_biggest_room_seconds_total{instance="172.0.2.118:9000"}[1m]))', 'expr': 'sum(rate(synapse_state_res_db_for_biggest_room_seconds_total{instance="matrix.synapse"}[1m]))',
'format': 'time_series', 'format': 'time_series',
'hide': False, 'hide': False,
'instant': False, 'instant': False,
@ -324,7 +324,7 @@ def get_stateres_worst_case(api_key, interval, data_range, endpoint):
'uid': 'DAMPdbiIz', 'uid': 'DAMPdbiIz',
}, },
'exemplar': False, 'exemplar': False,
'expr': 'sum(rate(synapse_state_res_cpu_for_biggest_room_seconds_total{instance="172.0.2.118:9000"}[1m]))', 'expr': 'sum(rate(synapse_state_res_cpu_for_biggest_room_seconds_total{instance="matrix.synapse"}[1m]))',
'format': 'time_series', 'format': 'time_series',
'hide': False, 'hide': False,
'instant': False, 'instant': False,