import argparse import json import logging import sys import time import requests from urllib3.exceptions import InsecureRequestWarning requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) def main(args): # Icinga2 API URL url = f"{args.api}/v1" logging.basicConfig(level=logging.INFO) while True: current_time = time.time() # Get all checks objects = ["hosts", "services"] for obj in objects: response = None for i in range(3): try: response = requests.get(url + "/objects/" + obj, auth=(args.username, args.password), verify=not args.insecure) response.raise_for_status() break except requests.exceptions.ConnectionError as e: logging.error(f'Failed to reach Icinga2: {e}') logging.error('Sleeping 60 seconds') time.sleep(60) if not response: logging.error('Gave up, no response.') sys.exit(1) # systemd should restart us checks = response.json()["results"] # Loop through all checks for check in checks: if not check['attrs']['active']: continue last_check_time = check["attrs"]["last_check"] check_interval = check["attrs"]["check_interval"] check_timeout = check["attrs"]["check_timeout"] or args.default_timeout # If the check is overdue. if current_time - last_check_time > check_interval + check_timeout + 60: headers = {'Accept': 'application/json', 'Content-Type': 'application/json'} next_check_time = last_check_time + check_interval + check_timeout if obj == 'services': check_filter = f'host.name=="{check["attrs"]["host_name"]}" && service.name=="{check["attrs"]["name"]}"' elif obj == 'hosts': check_filter = f'host.name=="{check["name"]}"' else: raise Exception # Set the check to unknown data = { "type": check['type'], "filter": check_filter, "exit_status": 3 if check['type'] == 'Service' else 1, "plugin_output": f"", } # Trigger a full failure. for _ in range(4): response = requests.post(url + "/actions/process-check-result", data=json.dumps(data), headers=headers, auth=(args.username, args.password), verify=not args.insecure) response.raise_for_status() time.sleep(3) # Rerun the check data = { "type": check['type'], "filter": check_filter, 'force': True } response = requests.post(url + "/actions/reschedule-check", data=json.dumps(data), headers=headers, auth=(args.username, args.password), verify=not args.insecure) response.raise_for_status() logging.info(f'Failed {check["type"].lower()} {check["name"]} - {int(current_time - next_check_time)} seconds overdue.') time.sleep(args.interval) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--api', default='https://localhost:5665', help='Base Icinga2 API.') parser.add_argument('--insecure', action='store_true', help='Disable SSL verification.') parser.add_argument('--username', default='icingaweb2', help='API username.') parser.add_argument('--password', required=True, help='API password.') parser.add_argument('--default-timeout', default=600, type=int, help='If a check does not have a timeout set, use this many seconds as the default. Default: 600 (10 minutes).') parser.add_argument('--interval', default=900, type=int, help='Interval between service scans. Default: 900 (15 minutes).') args = parser.parse_args() main(args)