merge dev

This commit is contained in:
Cyberes 2023-04-21 23:54:20 -06:00
parent e62d7111cb
commit ba347d60db
11 changed files with 309 additions and 270 deletions

32
Other/auto-acknowledge-apt.py Executable file
View File

@ -0,0 +1,32 @@
import argparse
import json
import requests
from urllib3.exceptions import InsecureRequestWarning
parser = argparse.ArgumentParser(description='Check OPNsense network traffic for a host.')
parser.add_argument('--api', required=True, help='Full URL to your Icinga2 API.')
parser.add_argument('--user', required=True, help='API username.')
parser.add_argument('--password', required=True, help='API password.')
args = parser.parse_args()
# Retrieve a list of unacknowledged alerts
requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
payload = {
"type": "Service",
"filter": "service.name == \"apt\" && service.acknowledgement == 0",
"author": "Auto-Acknowledgement Script",
"comment": "This alert has been automatically acknowledged.",
"notify": True,
"pretty": True
}
# Send the API request
response = requests.post(f'{args.api}/v1/actions/acknowledge-problem', headers={"Accept": "application/json"}, auth=(args.user, args.password), data=json.dumps(payload), verify=False)
# Check the response status code
if response.status_code == 200:
print("All pending alerts have been acknowledged.")
else:
print("Failed to acknowledge the alerts. Status code:", response.status_code)
print(response.text)

View File

@ -0,0 +1,13 @@
#!/bin/bash
apt-get install snmpd sudo perl libconfig-general-perl libdata-dumper-simple-perl libsort-naturally-perl libexception-class-perl libencode-perl make
TMP=$(mktemp -d)
git clone https://github.com/Tontonitch/interfacetable_v3t.git "$TMP"
cd "$TMP"
chmod a+x ./configure
./configure --prefix=/usr/local/interfacetable_v3t --with-nagios-user=nagios --with-nagios-group=nagios
make install
rm -rf "$TMP"

View File

@ -1,240 +0,0 @@
#!/bin/bash
# startup checks
if [ -z "$BASH" ]; then
echo "Please use BASH."
exit 3
fi
if [ ! -e "/usr/bin/which" ]; then
echo "/usr/bin/which is missing."
exit 3
fi
curl=$(which curl)
if [ $? -ne 0 ]; then
echo "Please install curl."
exit 3
fi
# Default Values
proxy=""
method="GET"
body=""
contains=""
lacks=""
insecure=0
debug=0
warning=700
encodeurl=0
critical=2000
url=""
follow=0
header=""
name="default"
cookies=0
# Usage Info
usage() {
echo '''Usage: check_curl [OPTIONS]
[OPTIONS]:
-U URL Target URL
-M METHOD HTTP Method (default: GET)
-N NAME Display Name of scanned object (default: default)
-B BODY Request Body to be sent (default: not sent)
-E ENCODEURL Send body defined with url encoding (curl --data-urlencode) (default: off)
-I INSECURE Sets the curl flag --insecure
-C CONTAINS If not contained in response body, CRITICAL will be returned
-L LACKS If contained in response body, CRITICAL will be returned (-C has priority when both are set)
-w WARNING Warning threshold in milliseconds (default: 700)
-c CRITICAL Critical threshold in milliseconds (default: 2000)
-H HEADER Send Header (i.E. "AUTHORIZATION: Bearer 8*.UdUYwrl!nK")
-F FOLLOW Follow redirects (default: OFF)
-D DEBUG Only prints the curl command (default: OFF)
-P PROXY Set Proxy Address (default: No Proxy)
-K COOKIES Enables/Disabled cookie handling in a temporary cookie jar'''
}
# Check which threshold was reached
checkTime() {
if [ $1 -gt $critical ]; then
echo -n "CRITICAL: Slow "
elif [ $1 -gt $warning ]; then
echo -n "WARNING: Slow "
else
echo -n "OK"
fi
}
# Return code value
getStatus() {
if [ $1 -gt $critical ]; then
return 2
elif [ $1 -gt $warning ]; then
return 1
else
return 0
fi
}
#main
#get options
while getopts "P:M:B:C:w:c:U:H:IFN:O:EL:D:K" opt; do
case $opt in
K)
cookies=1
;;
P)
proxy=$OPTARG
;;
M)
method=$OPTARG
;;
B)
body=$OPTARG
;;
C)
contains=$OPTARG
;;
w)
warning=$OPTARG
;;
c)
critical=$OPTARG
;;
U)
url=$OPTARG
;;
L)
lacks=$OPTARG
;;
I)
insecure=1
;;
N)
name=$( echo $OPTARG | sed -e 's/[^A-Za-z0-9._-]/_/g' )
;;
E)
encodeurl=1
;;
H)
header=$OPTARG
;;
F)
follow=1
;;
D)
debug=1
;;
*)
usage
exit 3
;;
esac
done
#hostname is required
if [ -z "$url" ] || [ $# -eq 0 ]; then
echo "Error: URL is required"
usage
exit 3
fi
proxyarg=""
if [ ! -z $proxy ] ; then
proxyarg=" -x "$proxy" "
fi
headerarg=""
if [ ! -z "$header" ] ; then
headerarg=' -H "'$header'" '
fi
followarg=""
if [ $follow -eq 1 ] ; then
followarg=" -L "
fi
insecurearg=""
if [ $insecure -eq 1 ] ; then
insecurearg=" --insecure "
fi
cookiesarg=""
if [ $cookies -eq 1 ] ; then
COOKIE_JAR_TEMP_PATH=$(mktemp /tmp/check_curl_cookiejar.XXXXXX)
cookiesarg=" -c ${COOKIE_JAR_TEMP_PATH} -b ${COOKIE_JAR_TEMP_PATH}"
fi
bodyarg=""
if [ ! -z $body ]; then
body=$(echo $body| sed "s/\"/\\\\\"/g")
bodyarg=" --data \""$body"\""
if [ $encodeurl -eq 1 ]; then
bodyarg=" --data-urlencode \""$body"\""
fi
fi
if [ $debug -eq 1 ]; then
echo $curl --no-keepalive -s $insecurearg $proxyarg $followarg $bodyarg $headerarg -X $method $cookiesarg "$url"
exit 0
else
start=$(echo $(($(date +%s%N)/1000000)))
body=$(eval $curl --no-keepalive -s $insecurearg $proxyarg $followarg $bodyarg $headerarg -X $method $cookiesarg "$url")
status=$?
fi
if [ $cookies -eq 1 ] ; then
rm -f ${COOKIE_JAR_TEMP_PATH}
fi
end=$(echo $(($(date +%s%N)/1000000)))
#decide output by return code
if [ $status -eq 0 ] ; then
if [ -n "$contains" ]; then
if [[ ! $body =~ $contains ]]; then
echo "CRITICAL: body does not contain '${contains}'. Body: '$(echo $body | sed 's/\(.\{50\}\).*/\1.../')' |time=$((end - start))ms;${warning};${critical};0;"$critical"ms"
exit 2
fi
fi
if [ -n "$lacks" ]; then
if [[ $body == *$lacks* ]]; then
echo "CRITICAL: body contains '${lacks}'|time=$((end - start))ms;${warning};${critical};0;"$critical"ms"
exit 2
fi
fi
echo "$(checkTime $((end - start))) $((end - start))ms - ${url}|time=$((end - start))ms;${warning};${critical};0;"$critical"ms"
getStatus $((end - start))
exit $?
else
case $status in
1)
echo "CRITICAL: Unsupported protocol"
;;
3)
echo "CRITICAL: Malformed URL"
;;
5)
echo "CRITICAL: Couldn't resolve proxy $proxy"
;;
6)
echo "CRITICAL: Couldn't resolve host"
;;
7)
echo "CRITICAL: Couldn't connect to proxy $proxy"
;;
22)
echo "CRITICAL: Server returned http code >= 400"
;;
52)
echo "CRITICAL: Server returned empty response (52)"
;;
56)
echo "CRITICAL: Failure recieving network data (56)"
;;
60)
echo "CRITICAL: SSL/TLS connection problem (60)"
;;
*)
echo "UNKNOWN: $status - ${url}"
exit 3
;;
esac
exit 2
fi

View File

@ -7,19 +7,11 @@ import traceback
import hurry
import numpy as np
import requests
from hurry.filesize import size
from urllib3.exceptions import InsecureRequestWarning
import checker.nagios as nagios
from checker.markdown import list_to_markdown_table
def filesize(bytes: int, spaces: bool = True):
x = size(bytes, system=hurry.filesize.alternative)
if spaces:
return x
else:
return x.replace(' ', '')
from checker.units import filesize
def main():
@ -28,7 +20,7 @@ def main():
parser.add_argument('--key', required=True, help='OPNsense API key.')
parser.add_argument('--secret', required=True, help='OPNsense API secret.')
parser.add_argument('--interface', required=True, help='Interface to check (e.g., lan). Can be something like "lan,wan"')
parser.add_argument('--host', required=True, help='IP of the host to check.')
parser.add_argument('--host', required=True, help='Address of the host to check.')
parser.add_argument('--duration', default=10, type=int, help='How many seconds to gather statistics.')
args = parser.parse_args()
@ -39,7 +31,7 @@ def main():
# Map interface names to their internal names
interfaces_mapping = requests.get(f'https://{args.opnsense}/api/diagnostics/traffic/interface',
headers={'Accept': 'application/json'}, auth=(args.key, args.secret), verify=False)
headers={'Accept': 'application/json'}, auth=(args.key, args.secret), verify=False, timeout=10)
if interfaces_mapping.status_code != 200:
print(f'UNKNOWN: unable to query OPNsense API for interface mappings: {interfaces_mapping.status_code}\n{interfaces_mapping.text}')
sys.exit(nagios.UNKNOWN)
@ -60,7 +52,7 @@ def main():
traffic_data = []
for _ in range(args.duration):
response = requests.get(f'https://{args.opnsense}/api/diagnostics/traffic/top/{interface}',
headers={'Accept': 'application/json'}, auth=(args.key, args.secret), verify=False)
headers={'Accept': 'application/json'}, auth=(args.key, args.secret), verify=False, timeout=10)
if response.status_code != 200:
print(f'UNKNOWN: unable to query OPNsense API for {interface}: {response.status_code}\n{response.text}')
sys.exit(nagios.UNKNOWN)
@ -70,7 +62,7 @@ def main():
time.sleep(1)
if not len(traffic_data):
print(f'UNKNOWN: Interface {args.interface} not found in OPNsense API response.')
print('UNKNOWN: Interface or host not found in OPNsense API response.')
sys.exit(nagios.UNKNOWN)
check_result[name] = {
@ -92,7 +84,6 @@ def main():
ok = []
perf_data = []
output_table = [('Interface', 'Rate In', 'Rate Out', 'Cumulative In', 'Cumulative Out', 'Connections', 'Status')]
for name, data in check_result.items():
# TODO: figure out status
if -1 >= crit_value:
@ -107,14 +98,16 @@ def main():
ok.append(name)
status = '[OK]'
perf_data.append(f'{name}-rate_in={filesize(data["rate_in"], spaces=False)};{warn_value};{crit_value};')
perf_data.append(f'{name}-rate_out={filesize(data["rate_out"], spaces=False)};{warn_value};{crit_value};')
perf_data.append(f'{name}-cumulative_in={filesize(data["cumulative_in"], spaces=False)};{warn_value};{crit_value};')
perf_data.append(f'{name}-cumulative_out={filesize(data["cumulative_out"], spaces=False)};{warn_value};{crit_value};')
perf_data.append(f'{name}-connections={data["connections"]};{warn_value};{crit_value};')
perf_data.append(f'{name}-rate_in={filesize(data["rate_in"], spaces=False)};{warn_value};{crit_value};0')
perf_data.append(f'{name}-rate_out={filesize(data["rate_out"], spaces=False)};{warn_value};{crit_value};0')
perf_data.append(f'{name}-cumulative_in={filesize(data["cumulative_in"], spaces=False)};{warn_value};{crit_value};0')
perf_data.append(f'{name}-cumulative_out={filesize(data["cumulative_out"], spaces=False)};{warn_value};{crit_value};0')
perf_data.append(f'{name}-connections={data["connections"]};{warn_value};{crit_value};0')
output_table.append((name, filesize(data['rate_in']), filesize(data['rate_out']), filesize(data['cumulative_in']), filesize(data['cumulative_out']),
data['connections'], status))
output_table = [
('Host', 'Interface', 'Rate In', 'Rate Out', 'Cumulative In', 'Cumulative Out', 'Connections', 'Status'),
(args.host, name, filesize(data['rate_in']), filesize(data['rate_out']), filesize(data['cumulative_in']), filesize(data['cumulative_out']), data['connections'], status)
]
print(list_to_markdown_table(output_table, align='left', seperator='!', borders=False))
print(f' |{" ".join(perf_data)}')

169
check_pve_guest_metrics.py Executable file
View File

@ -0,0 +1,169 @@
#!/usr/bin/env python3
import argparse
import json
import os
import sys
import traceback
from pathlib import Path
import certifi
import numpy as np
import requests
import checker.nagios as nagios
from checker.markdown import list_to_markdown_table
from checker.units import filesize
parser = argparse.ArgumentParser(description='Check the Proxmox API for network traffic for a host.')
parser.add_argument('--node', required=True, help='The name and address of Proxmox node in valid JSON in this format: ["bigserver", "192.168.1.222"]. This allows you to use datalists in Director.')
parser.add_argument('--user', required=True, help='The Proxmox user. Something like "monitoring@pve!icinga2"')
parser.add_argument('--password', required=True, help='Password.')
parser.add_argument('--host', required=True, help='The ID of the host to check.')
parser.add_argument('--type', required=True, choices=['qemu', 'lxc'], help='Type of host. "qemu" or "lxc"')
parser.add_argument('--metrics', required=True, help='What stats to check. Can list multiple seperated by commas. For example, "netin,netout"')
parser.add_argument('--levels', required=True, help='Warning levels. In JSON format: {"netin":{"warn":50, "crit":100, "type": "filesize"}, "netout":{"warn":50, "crit":100, "type": "filesize"}}')
parser.add_argument('--timeframe', default=5, help='Timeframe to average the data to in minutes. Default: 5 minutes')
parser.add_argument('--verify', default=True, help="What to verify the SSL connection with. Can be a file path, or false to disable verification. If you're having issues with CA certs, try setting it to your system's CA bundle (/etc/ssl/certs/ca-certificates.crt).")
parser.add_argument('--verify-force', action='store_true', help="Delete the certifi cert and replace it with whatever you specify in --verify")
parser.add_argument('--table', action='store_true', help='Print the results in a table.')
args = parser.parse_args()
# def where():
# return args.verify
def main():
if args.verify_force:
if not args.verify:
print('UNKNOWN: must supply --verify when using --verify-force')
sys.exit(nagios.UNKNOWN)
if Path(certifi.where()).exists():
os.remove(certifi.where())
os.symlink(args.verify, certifi.where())
print(f'Pointed {certifi.where()} to {args.verify}')
if Path(requests.certs.where()).exists():
os.remove(requests.certs.where())
os.symlink(args.verify, requests.certs.where())
print(f'Pointed {requests.certs.where()} to {args.verify}')
try:
metrics_levels = json.loads(args.levels)
except Exception as e:
print('UNKNOWN: Failed to parse --levels JSON:', e)
sys.exit(nagios.UNKNOWN)
try:
args.node = json.loads(args.node)
pve_node = args.node[0]
pve_node_address = args.node[1]
except Exception as e:
print('UNKNOWN: Failed to parse --node JSON:', e)
sys.exit(nagios.UNKNOWN)
# requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
try:
pve_auth_ticket = requests.post(f'https://{pve_node_address}:8006/api2/json/access/ticket', data={"username": args.user, "password": args.password}).json()['data']['ticket']
response = requests.get(f'https://{pve_node_address}:8006/api2/json/nodes/{pve_node}/{args.type}/{args.host}/rrddata?timeframe=hour',
# headers={"Authorization": f'PVEAPIToken={args.user}={args.token}'},
cookies={'PVEAuthCookie': pve_auth_ticket},
verify=args.verify)
except requests.exceptions.SSLError as e:
print('UNSKNOWN: SSL error ', e)
print('Using cert:', args.verify)
print('certifi using cert:', certifi.where())
print('requests using cert:', requests.certs.where())
sys.exit(nagios.UNKNOWN)
try:
api_data = json.loads(response.text)['data']
except Exception as e:
print(f'UNKNOWN: Failed to parse JSON {e}')
print(response.text)
sys.exit(nagios.UNKNOWN)
# Load the data
metrics_data = {}
for item in args.metrics.split(','):
if item not in metrics_levels.keys():
print(f'UNKNOWN: missing metric "{item}" in --levels')
sys.exit(nagios.UNKNOWN)
if 'warn' not in metrics_levels[item].keys():
print(f'UNKNOWN: missing key "warn" for metric "{item}" in --levels')
sys.exit(nagios.UNKNOWN)
if 'crit' not in metrics_levels[item].keys():
print(f'UNKNOWN: missing key "crit" for metric "{item}" in --levels')
sys.exit(nagios.UNKNOWN)
if 'type' not in metrics_levels[item].keys():
print(f'UNKNOWN: missing key "type" for metric "{item}" in --levels')
sys.exit(nagios.UNKNOWN)
metrics_data[item] = []
for m in api_data:
for k, v in m.items():
if k == item:
if isinstance(v, float):
v = np.round(v, 2)
metrics_data[item].append(v)
check_data = {}
exit_code = nagios.OK
for metric, value in metrics_data.items():
check_data[metric] = {}
# Average the data. Expects the interval to be 1 minute
avg = np.round(np.average(value[-5:-1]), 2)
check_data[metric]['value'] = avg
if metrics_levels[metric]['type'] == 'filesize':
check_data[metric]['value_str'] = filesize(avg)
else:
check_data[metric]['value_str'] = str(avg)
if avg >= metrics_levels[metric]['crit']:
check_data[metric]['status'] = nagios.CRITICAL
check_data[metric]['status_str'] = '[CRITICAL]'
elif avg >= metrics_levels[metric]['warn']:
check_data[metric]['status'] = nagios.WARN
check_data[metric]['status_str'] = '[WARNING]'
else:
check_data[metric]['status'] = nagios.OK
check_data[metric]['status_str'] = '[OK]'
if exit_code < check_data[metric]['status']:
exit_code = check_data[metric]['status']
if exit_code == nagios.OK:
output_str = 'OK: '
elif exit_code == nagios.WARNING:
output_str = 'WARNING: '
elif exit_code == nagios.CRITICAL:
output_str = 'CRITICAL: '
perf_data = []
for metric, data in check_data.items():
output_str = output_str + f"{metric} {data['value_str']}, "
perf_data.append(f"{metric}={data['value_str'].replace(' ', '')};{metrics_levels[metric]['warn']};{metrics_levels[metric]['crit']};;")
print(output_str.strip(', ').strip(), end=('\n' if args.table else ''))
perf_data_str = f'|{" ".join(perf_data)}'
if args.table:
output_table = [('Metric', 'Value', 'Status')]
for metric, data in check_data.items():
output_table.append((metric, data['value_str'], data['status_str']))
print(list_to_markdown_table(output_table, align='left', seperator='!', borders=False))
# else:
# perf_data_str = ' ' + perf_data_str
print(perf_data_str)
sys.exit(exit_code)
if __name__ == "__main__":
try:
main()
except Exception as e:
print(f'UNKNOWN: exception "{e}"')
print(traceback.format_exc())
sys.exit(nagios.UNKNOWN)

16
checker/units.py Normal file
View File

@ -0,0 +1,16 @@
from hurry.filesize import size
def filesize(bytes: int, spaces: bool = True):
system = [
(1024 ** 5, ' PB'),
(1024 ** 4, ' TB'),
(1024 ** 3, ' GB'),
(1024 ** 2, ' MB'),
(1024 ** 1, ' KB'),
(1024 ** 0, ' B'),
]
x = size(bytes, system=system)
if spaces:
return x
else:
return x.replace(' ', '')

View File

@ -1,20 +1,75 @@
#!/bin/bash
# apt update && apt install -y git && cd /opt && git -C /opt/icinga2-checks pull || git clone https://git.dp15.us/dpanzer/icinga2-checks.git /opt/icinga2-checks && bash /opt/icinga2-checks/install.sh
# 3 4 * * * /bin/bash /opt/icinga2-checks/install.sh
ln_existing() {
src=$1
dest=$2
if [[ -e "$dest" ]] || [[ -L "$dest" ]]; then
rm "$dest"
fi
ln -s "$src" "$dest"
}
apt update
apt install -y python3-venv git sysstat bc
pip install -U pip wheel setuptools
pip install psutil
mkdir -p /usr/lib64/nagios/plugins/
# Install global pip packages
#pip install psutil
git -C /opt/icinga2-checks pull || git clone https://git.dp15.us/dpanzer/icinga2-checks.git /opt/icinga2-checks
rm /usr/lib64/nagios/plugins/check_iowait
ln -s /opt/icinga2-checks/check_iowait.sh /usr/lib64/nagios/plugins/check_iowait
mkdir -p /opt/venvs
rm /usr/lib64/nagios/plugins/check_bandwidth
ln -s /opt/icinga2-checks/check_bandwidth /usr/lib64/nagios/plugins/check_bandwidth
if [[ ! -d /opt/venvs/icinga2_checks ]]; then
echo "Creating venv in /opt/venvs/icinga2_checks"
python3 -m venv /opt/venvs/icinga2_checks
fi
if [[ ! -d /opt/venvs/check_pve ]]; then
echo "Creating venv in /opt/venvs/check_pve"
python3 -m venv /opt/venvs/check_pve
fi
rm /usr/lib64/nagios/plugins/check_curl
ln -s /opt/icinga2-checks/check_curl /usr/lib64/nagios/plugins/check_curl
/opt/venvs/icinga2_checks/bin/pip install -U pip wheel setuptools
/opt/venvs/icinga2_checks/bin/pip install -r /opt/icinga2-checks/requirements.txt
/opt/venvs/check_pve/bin/pip install -U pip wheel setuptools
/opt/venvs/check_pve/bin/pip install -r /opt/icinga2-checks/check_pve/requirements.txt
ln_existing /opt/icinga2-checks/check_iowait.sh /usr/lib64/nagios/plugins/check_iowait
ln_existing /opt/icinga2-checks/check_bandwidth.py /usr/lib64/nagios/plugins/check_bandwidth
ln_existing /opt/icinga2-checks/check_curl.sh /usr/lib64/nagios/plugins/check_curl
ln_existing /etc/ssl/certs/ca-certificates.crt /opt/venvs/icinga2_checks/lib/python3.10/site-packages/certifi/cacert.pem
# ==============================================================================
# Install the Linuxfabrik checks
if [[ ! -d /opt/linuxfabrik-monitoring-plugins ]]; then
git clone https://github.com/Linuxfabrik/monitoring-plugins.git /opt/linuxfabrik-monitoring-plugins
if [[ ! -d /opt/venvs/linuxfabrik ]]; then
echo "Creating venv in /opt/venvs/linuxfabrik"
python3 -m venv /opt/venvs/linuxfabrik
fi
/opt/venvs/linuxfabrik/bin/pip install psutil
cd /opt/linuxfabrik-monitoring-plugins
rm -rf lib
git clone https://github.com/Linuxfabrik/lib.git
/opt/venvs/linuxfabrik/bin/pip install bs4 psutil smbprotocol vici pymysql
fi
ln_existing /opt/linuxfabrik-monitoring-plugins/check-plugins/disk-io/disk-io3 /usr/lib64/nagios/plugins/disk-io
ln_existing /opt/linuxfabrik-monitoring-plugins/check-plugins/cpu-usage/cpu-usage3 /usr/lib64/nagios/plugins/cpu-usage
ln_existing /opt/linuxfabrik-monitoring-plugins/check-plugins/disk-usage/disk-usage3 /usr/lib64/nagios/plugins/disk-usage
ln_existing /opt/linuxfabrik-monitoring-plugins/check-plugins/memory-usage/memory-usage3 /usr/lib64/nagios/plugins/memory-usage
ln_existing /opt/linuxfabrik-monitoring-plugins/check-plugins/ping/ping3 /usr/lib64/nagios/plugins/ping
ln_existing /opt/linuxfabrik-monitoring-plugins/check-plugins/disk-smart/disk-smart3 /usr/lib64/nagios/plugins/disk-smart
setcap cap_net_raw+ep /usr/lib/nagios/plugins/check_icmp
setcap cap_net_raw+ep /usr/lib/nagios/plugins/check_ping

View File

@ -12,4 +12,5 @@ urllib3~=1.26.14
aiofiles~=0.6.0
markdown~=3.4.1
psutil~=5.9.4
hurry.filesize
hurry.filesize
certifi