From 979381f365558b8580f70d57c09e47654ae2478d Mon Sep 17 00:00:00 2001 From: Cyberes Date: Tue, 3 Sep 2024 16:37:39 -0600 Subject: [PATCH] add http server to serve generated maps, reorganize structure --- feeder/README.md | 4 +- feeder/cache.py | 49 ++++++++++++ feeder/global-image.py | 46 +++++++++++ feeder/lib/tecmap.py | 13 ++-- feeder/{main.py => mqtt.py} | 78 +++++++------------ feeder/requirements.txt | 6 +- feeder/server.py | 22 ++++++ feeder/systemd/space-weather-cache.service | 15 ++++ .../space-weather-global-image.service | 15 ++++ .../space-weather-mqtt.service} | 5 +- feeder/systemd/space-weather-server.service | 15 ++++ 11 files changed, 204 insertions(+), 64 deletions(-) create mode 100644 feeder/cache.py create mode 100644 feeder/global-image.py rename feeder/{main.py => mqtt.py} (55%) create mode 100644 feeder/server.py create mode 100644 feeder/systemd/space-weather-cache.service create mode 100644 feeder/systemd/space-weather-global-image.service rename feeder/{space-weather-feeder.service => systemd/space-weather-mqtt.service} (53%) create mode 100644 feeder/systemd/space-weather-server.service diff --git a/feeder/README.md b/feeder/README.md index 39fb822..15e799b 100644 --- a/feeder/README.md +++ b/feeder/README.md @@ -26,10 +26,10 @@ LON_RANGE_MIN= \ LON_RANGE_MAX= \ CDDIS_USERNAME= CDDIS_PASSWORD= \ MQTT_BROKER_HOST="" MQTT_BROKER_PORT=1883 MQTT_USERNAME="user" MQTT_PASSWORD="" \ -python3 main.py +python3 mqtt.py ``` -An example systemd service file is provided. +Example systemd service files are provided. ### Home Assistant MQTT Config diff --git a/feeder/cache.py b/feeder/cache.py new file mode 100644 index 0000000..e3d1110 --- /dev/null +++ b/feeder/cache.py @@ -0,0 +1,49 @@ +import logging +import os +import pickle +import signal +import sys +import threading +import time +import traceback +from datetime import datetime + +from redis import Redis + +from lib.cddis_fetch import fetch_latest_ionex +from lib.tecmap import get_tecmaps, parse_ionex_datetime + +logging.basicConfig(level=logging.INFO) + +CDDIS_USERNAME = os.getenv('CDDIS_USERNAME') +CDDIS_PASSWORD = os.getenv('CDDIS_PASSWORD') +if not CDDIS_USERNAME or not CDDIS_PASSWORD: + logging.critical('Must set CDDIS_USERNAME and CDDIS_PASSWORD environment variables') + sys.exit(1) + + +def update_cache(): + try: + redis = Redis(host='localhost', port=6379, db=0) + redis.flushall() + while True: + utc_hr = datetime.utcnow().hour + logging.info('Fetching latest IONEX data') + logging.info(f'Using hour {utc_hr}') + ionex_data = fetch_latest_ionex(CDDIS_USERNAME, CDDIS_PASSWORD) + parsed_data = [] + for tecmap, epoch in get_tecmaps(ionex_data): + parsed_dt = parse_ionex_datetime(epoch) + parsed_data.append((tecmap, parsed_dt)) + redis.set('tecmap_data', pickle.dumps(parsed_data)) + logging.info('Scrape complete') + time.sleep(1800) # 30 minutes + except: + logging.critical(traceback.format_exc()) + os.kill(os.getpid(), signal.SIGKILL) + + +if __name__ == '__main__': + threading.Thread(target=update_cache).start() + while True: + time.sleep(3600) diff --git a/feeder/global-image.py b/feeder/global-image.py new file mode 100644 index 0000000..6cd3c59 --- /dev/null +++ b/feeder/global-image.py @@ -0,0 +1,46 @@ +import io +import logging +import pickle +import sys +from datetime import datetime +from typing import List + +from PIL import Image +from redis import Redis + +from lib.tecmap import plot_tec_map + +logging.basicConfig(level=logging.INFO) + +LAT_RANGE_MIN = -90 +LAT_RANGE_MAX = 90 +LON_RANGE_MIN = -180 +LON_RANGE_MAX = 180 + +redis = Redis(host='localhost', port=6379, db=0) + +utc_hr = datetime.utcnow().hour +logging.info(f'Generating plot for hour {utc_hr}') + +ionex_data: List = pickle.loads(redis.get('tecmap_data')) +if ionex_data is None: + logging.critical('Redis has not been populated yet. Is cache.py running?') + sys.exit(1) + +for tecmap, epoch in ionex_data: + if epoch.hour == utc_hr: + plt = plot_tec_map(tecmap, [float(LON_RANGE_MIN), float(LON_RANGE_MAX)], [float(LAT_RANGE_MIN), float(LAT_RANGE_MAX)])[1] + + buf = io.BytesIO() + plt.savefig(buf, format='png', bbox_inches='tight', pad_inches=0, dpi=110) + plt.close() + del plt + + buf.seek(0) + img = Image.open(buf) + # img = img.resize((img.size[0], 500), Image.LANCZOS) + buf = io.BytesIO() + img.save(buf, format='PNG') + + redis.set('global_map', buf.getvalue()) + buf.close() diff --git a/feeder/lib/tecmap.py b/feeder/lib/tecmap.py index 9487ee5..67c595a 100644 --- a/feeder/lib/tecmap.py +++ b/feeder/lib/tecmap.py @@ -39,11 +39,11 @@ def plot_tec_map(tecmap, lon_range: list, lat_range: list): # Create arrays of latitudes and longitudes to match the geographical grid of the TEC map data. # This is hard coded and should never change. lat = np.arange(-87.5, 87.5, 2.5) - lon = np.arange(-180, 180, 2.5) + lon = np.arange(-180, 180, 5.0) # Create a mask for the data in the lat/lon range - lon_mask = (lon >= lon_range[0]) & (lon <= lon_range[1]) - lat_mask = (lat >= lat_range[0]) & (lat <= lat_range[1]) + lon_mask = (lon >= lon_range[0]) & (lon < lon_range[1]) + lat_mask = (lat >= lat_range[0]) & (lat < lat_range[1]) mask = np.ix_(lat_mask, lon_mask) # Select only the data in the lat/lon range @@ -54,16 +54,13 @@ def plot_tec_map(tecmap, lon_range: list, lat_range: list): # Make graph pretty ax.coastlines() - plt.title('VTEC map') + plt.title(datetime.now().strftime('%H:%M %d-%m-%Y'), fontsize=12, y=1.04) + plt.suptitle('Vertical Total Electron Count', fontsize=16, y=0.87) divider = make_axes_locatable(ax) ax_cb = divider.new_horizontal(size='5%', pad=0.1, axes_class=plt.Axes) f.add_axes(ax_cb) cb = plt.colorbar(h, cax=ax_cb) plt.rc('text', usetex=True) cb.set_label('TECU ($10^{16} \\mathrm{el}/\\mathrm{m}^2$)') - plt.show() - - # Deallocate - plt.close() return tecmap_ranged, plt diff --git a/feeder/main.py b/feeder/mqtt.py similarity index 55% rename from feeder/main.py rename to feeder/mqtt.py index 20d73bf..c8428d5 100644 --- a/feeder/main.py +++ b/feeder/mqtt.py @@ -1,15 +1,17 @@ import logging import os +import pickle import sys -import threading import time +import traceback from datetime import datetime +from typing import List import numpy as np import paho.mqtt.client as mqtt +from redis import Redis -from lib.cddis_fetch import fetch_latest_ionex -from lib.tecmap import get_tecmaps, plot_tec_map, parse_ionex_datetime +from lib.tecmap import plot_tec_map, parse_ionex_datetime logging.basicConfig(level=logging.INFO) @@ -26,6 +28,7 @@ LON_RANGE_MIN = os.getenv('LON_RANGE_MIN') LON_RANGE_MAX = os.getenv('LON_RANGE_MAX') if not LAT_RANGE_MIN or not LAT_RANGE_MAX or not LON_RANGE_MIN or not LON_RANGE_MAX: logging.critical('Must set LAT_RANGE_MIN, LAT_RANGE_MAX, LON_RANGE_MIN, and LON_RANGE_MAX environment variables') + print(LAT_RANGE_MIN, LAT_RANGE_MAX, LON_RANGE_MIN, LON_RANGE_MAX) sys.exit(1) CDDIS_USERNAME = os.getenv('CDDIS_USERNAME') @@ -57,56 +60,29 @@ def publish(topic: str, msg): logging.error(f'Failed to send message to topic {topic_expanded}.') -class DataCache: - def __init__(self): - self.value = None - self.lock = threading.Lock() +def main(): + try: + redis = Redis(host='localhost', port=6379, db=0) - def update(self, new_value): - with self.lock: - self.value = new_value - - def get(self): - with self.lock: - return self.value - - -cached_data = DataCache() - - -def update_cache(): - while True: - utc_hr = datetime.utcnow().hour - logging.info('Fetching latest IONEX data') - logging.info(f'Using hour {utc_hr}') - ionex_data = fetch_latest_ionex(CDDIS_USERNAME, CDDIS_PASSWORD) - avg_tec = None - for tecmap, epoch in get_tecmaps(ionex_data): - parsed_dt = parse_ionex_datetime(epoch) - if parsed_dt.hour == utc_hr: - avg_tec = np.mean(plot_tec_map(tecmap, [float(LON_RANGE_MIN), float(LON_RANGE_MAX)], [float(LAT_RANGE_MIN), float(LAT_RANGE_MAX)])[0]) - logging.info(f'Data timestamp: {parsed_dt.isoformat()}') - break - latest = round(avg_tec, 1) - cached_data.update(latest) - logging.info(f'Latest value: {latest}') - time.sleep(1800) # 30 minutes - - -def publish_cache(): - """ - A seperate thread that will send the current value to HA every minute. This - seems to help avoid HA reporting "unknown" for the VTEC value. - """ - while True: - latest = cached_data.get() - if latest is not None: + while True: + utc_hr = datetime.utcnow().hour + logging.info('Fetching latest IONEX data') + logging.info(f'Using hour {utc_hr}') + ionex_data: List = pickle.loads(redis.get('tecmap_data')) + avg_tec = None + for tecmap, epoch in ionex_data: + parsed_dt = parse_ionex_datetime(epoch) + if parsed_dt.hour == utc_hr: + avg_tec = np.mean(plot_tec_map(tecmap, [float(LON_RANGE_MIN), float(LON_RANGE_MAX)], [float(LAT_RANGE_MIN), float(LAT_RANGE_MAX)])[0]) + logging.info(f'Data timestamp: {parsed_dt.isoformat()}') + break + latest = round(avg_tec, 1) publish('vtec', latest) - time.sleep(60) + time.sleep(60) + except: + logging.critical(traceback.format_exc()) + sys.exit(1) if __name__ == '__main__': - threading.Thread(target=update_cache).start() - threading.Thread(target=publish_cache).start() - while True: - time.sleep(3600) + main() diff --git a/feeder/requirements.txt b/feeder/requirements.txt index ef24e5b..6717783 100644 --- a/feeder/requirements.txt +++ b/feeder/requirements.txt @@ -4,4 +4,8 @@ selenium==4.23.1 requests==2.32.3 matplotlib==3.9.2 cartopy==0.23.0 -numpy==2.0.1 \ No newline at end of file +numpy==2.0.1 +redis==5.0.8 +async-timeout==4.0.3 +Pillow +flask==3.0.3 \ No newline at end of file diff --git a/feeder/server.py b/feeder/server.py new file mode 100644 index 0000000..cbe197a --- /dev/null +++ b/feeder/server.py @@ -0,0 +1,22 @@ +import io + +import redis +from flask import Flask, send_file + +app = Flask(__name__) +redis_client = redis.Redis(host='localhost', port=6379) + + +@app.route('/global') +def serve_global_map(): + global_map_data = redis_client.get('global_map') + if global_map_data is None: + return "No global map available", 404 + + buf = io.BytesIO(global_map_data) + buf.seek(0) + return send_file(buf, mimetype='image/png') + + +if __name__ == '__main__': + app.run() diff --git a/feeder/systemd/space-weather-cache.service b/feeder/systemd/space-weather-cache.service new file mode 100644 index 0000000..44642d3 --- /dev/null +++ b/feeder/systemd/space-weather-cache.service @@ -0,0 +1,15 @@ +[Unit] +Description=Space Weather Cache +After=network.target + +[Service] +Type=simple +User=homeassistant +EnvironmentFile=/etc/secrets/space-weather +ExecStart=/srv/ha-noaa-space-weather/venv/bin/python /srv/ha-noaa-space-weather/feeder/cache.py +SyslogIdentifier=space-weather-cache +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=multi-user.target diff --git a/feeder/systemd/space-weather-global-image.service b/feeder/systemd/space-weather-global-image.service new file mode 100644 index 0000000..1bd365c --- /dev/null +++ b/feeder/systemd/space-weather-global-image.service @@ -0,0 +1,15 @@ +[Unit] +Description=Space Weather Global Image Generator +After=network.target + +[Service] +Type=simple +User=homeassistant +EnvironmentFile=/etc/secrets/space-weather +ExecStart=/srv/ha-noaa-space-weather/venv/bin/python /srv/ha-noaa-space-weather/feeder/global-image.py +SyslogIdentifier=space-weather-global-image +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=multi-user.target diff --git a/feeder/space-weather-feeder.service b/feeder/systemd/space-weather-mqtt.service similarity index 53% rename from feeder/space-weather-feeder.service rename to feeder/systemd/space-weather-mqtt.service index cb2b14c..7e03340 100644 --- a/feeder/space-weather-feeder.service +++ b/feeder/systemd/space-weather-mqtt.service @@ -1,12 +1,13 @@ [Unit] -Description=Space Weather VTEC +Description=Space Weather MQTT After=network.target [Service] Type=simple User=homeassistant EnvironmentFile=/etc/secrets/space-weather -ExecStart=/srv/space-weather/ha-noaa-space-weather/venv/bin/python /srv/space-weather/ha-noaa-space-weather/feeder/main.py +ExecStart=/srv/ha-noaa-space-weather/venv/bin/python /srv/ha-noaa-space-weather/feeder/mqtt.py +SyslogIdentifier=space-weather-mqtt Restart=on-failure RestartSec=5s diff --git a/feeder/systemd/space-weather-server.service b/feeder/systemd/space-weather-server.service new file mode 100644 index 0000000..4a19583 --- /dev/null +++ b/feeder/systemd/space-weather-server.service @@ -0,0 +1,15 @@ +[Unit] +Description=Space Weather Server +After=network.target + +[Service] +Type=simple +User=homeassistant +EnvironmentFile=/etc/secrets/space-weather +ExecStart=/srv/ha-noaa-space-weather/venv/bin/python /srv/ha-noaa-space-weather/feeder/server.py +SyslogIdentifier=space-weather-server +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=multi-user.target