add http server to serve generated maps, reorganize structure

feeder/README.md

@@ -26,10 +26,10 @@ LON_RANGE_MIN=<lower range for lon bounding box> \
 LON_RANGE_MAX=<upper range for lon bounding box> \
 CDDIS_USERNAME=<username> CDDIS_PASSWORD=<password> \
 MQTT_BROKER_HOST="<Home Assistant IP>" MQTT_BROKER_PORT=1883 MQTT_USERNAME="user" MQTT_PASSWORD="<password>" \
-python3 main.py
+python3 mqtt.py
 ```
 
-An example systemd service file is provided.
+Example systemd service files are provided.
 
 ### Home Assistant MQTT Config
 

feeder/cache.py

@@ -0,0 +1,49 @@
+import logging
+import os
+import pickle
+import signal
+import sys
+import threading
+import time
+import traceback
+from datetime import datetime
+
+from redis import Redis
+
+from lib.cddis_fetch import fetch_latest_ionex
+from lib.tecmap import get_tecmaps, parse_ionex_datetime
+
+logging.basicConfig(level=logging.INFO)
+
+CDDIS_USERNAME = os.getenv('CDDIS_USERNAME')
+CDDIS_PASSWORD = os.getenv('CDDIS_PASSWORD')
+if not CDDIS_USERNAME or not CDDIS_PASSWORD:
+    logging.critical('Must set CDDIS_USERNAME and CDDIS_PASSWORD environment variables')
+    sys.exit(1)
+
+
+def update_cache():
+    try:
+        redis = Redis(host='localhost', port=6379, db=0)
+        redis.flushall()
+        while True:
+            utc_hr = datetime.utcnow().hour
+            logging.info('Fetching latest IONEX data')
+            logging.info(f'Using hour {utc_hr}')
+            ionex_data = fetch_latest_ionex(CDDIS_USERNAME, CDDIS_PASSWORD)
+            parsed_data = []
+            for tecmap, epoch in get_tecmaps(ionex_data):
+                parsed_dt = parse_ionex_datetime(epoch)
+                parsed_data.append((tecmap, parsed_dt))
+            redis.set('tecmap_data', pickle.dumps(parsed_data))
+            logging.info('Scrape complete')
+            time.sleep(1800)  # 30 minutes
+    except:
+        logging.critical(traceback.format_exc())
+        os.kill(os.getpid(), signal.SIGKILL)
+
+
+if __name__ == '__main__':
+    threading.Thread(target=update_cache).start()
+    while True:
+        time.sleep(3600)

feeder/global-image.py

@@ -0,0 +1,46 @@
+import io
+import logging
+import pickle
+import sys
+from datetime import datetime
+from typing import List
+
+from PIL import Image
+from redis import Redis
+
+from lib.tecmap import plot_tec_map
+
+logging.basicConfig(level=logging.INFO)
+
+LAT_RANGE_MIN = -90
+LAT_RANGE_MAX = 90
+LON_RANGE_MIN = -180
+LON_RANGE_MAX = 180
+
+redis = Redis(host='localhost', port=6379, db=0)
+
+utc_hr = datetime.utcnow().hour
+logging.info(f'Generating plot for hour {utc_hr}')
+
+ionex_data: List = pickle.loads(redis.get('tecmap_data'))
+if ionex_data is None:
+    logging.critical('Redis has not been populated yet. Is cache.py running?')
+    sys.exit(1)
+
+for tecmap, epoch in ionex_data:
+    if epoch.hour == utc_hr:
+        plt = plot_tec_map(tecmap, [float(LON_RANGE_MIN), float(LON_RANGE_MAX)], [float(LAT_RANGE_MIN), float(LAT_RANGE_MAX)])[1]
+
+        buf = io.BytesIO()
+        plt.savefig(buf, format='png', bbox_inches='tight', pad_inches=0, dpi=110)
+        plt.close()
+        del plt
+
+        buf.seek(0)
+        img = Image.open(buf)
+        # img = img.resize((img.size[0], 500), Image.LANCZOS)
+        buf = io.BytesIO()
+        img.save(buf, format='PNG')
+
+        redis.set('global_map', buf.getvalue())
+        buf.close()

feeder/lib/tecmap.py

@@ -39,11 +39,11 @@ def plot_tec_map(tecmap, lon_range: list, lat_range: list):
     # Create arrays of latitudes and longitudes to match the geographical grid of the TEC map data.
     # This is hard coded and should never change.
     lat = np.arange(-87.5, 87.5, 2.5)
-    lon = np.arange(-180, 180, 2.5)
+    lon = np.arange(-180, 180, 5.0)
 
     # Create a mask for the data in the lat/lon range
-    lon_mask = (lon >= lon_range[0]) & (lon <= lon_range[1])
-    lat_mask = (lat >= lat_range[0]) & (lat <= lat_range[1])
+    lon_mask = (lon >= lon_range[0]) & (lon < lon_range[1])
+    lat_mask = (lat >= lat_range[0]) & (lat < lat_range[1])
     mask = np.ix_(lat_mask, lon_mask)
 
     # Select only the data in the lat/lon range
@@ -54,16 +54,13 @@ def plot_tec_map(tecmap, lon_range: list, lat_range: list):
 
     # Make graph pretty
     ax.coastlines()
-    plt.title('VTEC map')
+    plt.title(datetime.now().strftime('%H:%M %d-%m-%Y'), fontsize=12, y=1.04)
+    plt.suptitle('Vertical Total Electron Count', fontsize=16, y=0.87)
     divider = make_axes_locatable(ax)
     ax_cb = divider.new_horizontal(size='5%', pad=0.1, axes_class=plt.Axes)
     f.add_axes(ax_cb)
     cb = plt.colorbar(h, cax=ax_cb)
     plt.rc('text', usetex=True)
     cb.set_label('TECU ($10^{16} \\mathrm{el}/\\mathrm{m}^2$)')
-    plt.show()
-
-    # Deallocate
-    plt.close()
 
     return tecmap_ranged, plt

feeder/mqtt.py

@@ -1,15 +1,17 @@
 import logging
 import os
+import pickle
 import sys
-import threading
 import time
+import traceback
 from datetime import datetime
+from typing import List
 
 import numpy as np
 import paho.mqtt.client as mqtt
+from redis import Redis
 
-from lib.cddis_fetch import fetch_latest_ionex
-from lib.tecmap import get_tecmaps, plot_tec_map, parse_ionex_datetime
+from lib.tecmap import plot_tec_map, parse_ionex_datetime
 
 logging.basicConfig(level=logging.INFO)
 
@@ -26,6 +28,7 @@ LON_RANGE_MIN = os.getenv('LON_RANGE_MIN')
 LON_RANGE_MAX = os.getenv('LON_RANGE_MAX')
 if not LAT_RANGE_MIN or not LAT_RANGE_MAX or not LON_RANGE_MIN or not LON_RANGE_MAX:
     logging.critical('Must set LAT_RANGE_MIN, LAT_RANGE_MAX, LON_RANGE_MIN, and LON_RANGE_MAX environment variables')
+    print(LAT_RANGE_MIN, LAT_RANGE_MAX, LON_RANGE_MIN, LON_RANGE_MAX)
     sys.exit(1)
 
 CDDIS_USERNAME = os.getenv('CDDIS_USERNAME')
@@ -57,56 +60,29 @@ def publish(topic: str, msg):
     logging.error(f'Failed to send message to topic {topic_expanded}.')
 
 
-class DataCache:
-    def __init__(self):
-        self.value = None
-        self.lock = threading.Lock()
+def main():
+    try:
+        redis = Redis(host='localhost', port=6379, db=0)
 
-    def update(self, new_value):
-        with self.lock:
-            self.value = new_value
-
-    def get(self):
-        with self.lock:
-            return self.value
-
-
-cached_data = DataCache()
-
-
-def update_cache():
-    while True:
-        utc_hr = datetime.utcnow().hour
-        logging.info('Fetching latest IONEX data')
-        logging.info(f'Using hour {utc_hr}')
-        ionex_data = fetch_latest_ionex(CDDIS_USERNAME, CDDIS_PASSWORD)
-        avg_tec = None
-        for tecmap, epoch in get_tecmaps(ionex_data):
-            parsed_dt = parse_ionex_datetime(epoch)
-            if parsed_dt.hour == utc_hr:
-                avg_tec = np.mean(plot_tec_map(tecmap, [float(LON_RANGE_MIN), float(LON_RANGE_MAX)], [float(LAT_RANGE_MIN), float(LAT_RANGE_MAX)])[0])
-                logging.info(f'Data timestamp: {parsed_dt.isoformat()}')
-                break
-        latest = round(avg_tec, 1)
-        cached_data.update(latest)
-        logging.info(f'Latest value: {latest}')
-        time.sleep(1800)  # 30 minutes
-
-
-def publish_cache():
-    """
-    A seperate thread that will send the current value to HA every minute. This
-    seems to help avoid HA reporting "unknown" for the VTEC value.
-    """
-    while True:
-        latest = cached_data.get()
-        if latest is not None:
+        while True:
+            utc_hr = datetime.utcnow().hour
+            logging.info('Fetching latest IONEX data')
+            logging.info(f'Using hour {utc_hr}')
+            ionex_data: List = pickle.loads(redis.get('tecmap_data'))
+            avg_tec = None
+            for tecmap, epoch in ionex_data:
+                parsed_dt = parse_ionex_datetime(epoch)
+                if parsed_dt.hour == utc_hr:
+                    avg_tec = np.mean(plot_tec_map(tecmap, [float(LON_RANGE_MIN), float(LON_RANGE_MAX)], [float(LAT_RANGE_MIN), float(LAT_RANGE_MAX)])[0])
+                    logging.info(f'Data timestamp: {parsed_dt.isoformat()}')
+                    break
+            latest = round(avg_tec, 1)
             publish('vtec', latest)
-        time.sleep(60)
+            time.sleep(60)
+    except:
+        logging.critical(traceback.format_exc())
+        sys.exit(1)
 
 
 if __name__ == '__main__':
-    threading.Thread(target=update_cache).start()
-    threading.Thread(target=publish_cache).start()
-    while True:
-        time.sleep(3600)
+    main()

feeder/requirements.txt

@@ -4,4 +4,8 @@ selenium==4.23.1
 requests==2.32.3
 matplotlib==3.9.2
 cartopy==0.23.0
-numpy==2.0.1
+numpy==2.0.1
+redis==5.0.8
+async-timeout==4.0.3
+Pillow
+flask==3.0.3

feeder/server.py

@@ -0,0 +1,22 @@
+import io
+
+import redis
+from flask import Flask, send_file
+
+app = Flask(__name__)
+redis_client = redis.Redis(host='localhost', port=6379)
+
+
+@app.route('/global')
+def serve_global_map():
+    global_map_data = redis_client.get('global_map')
+    if global_map_data is None:
+        return "No global map available", 404
+
+    buf = io.BytesIO(global_map_data)
+    buf.seek(0)
+    return send_file(buf, mimetype='image/png')
+
+
+if __name__ == '__main__':
+    app.run()

feeder/systemd/space-weather-cache.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Space Weather Cache
+After=network.target
+
+[Service]
+Type=simple
+User=homeassistant
+EnvironmentFile=/etc/secrets/space-weather
+ExecStart=/srv/ha-noaa-space-weather/venv/bin/python /srv/ha-noaa-space-weather/feeder/cache.py
+SyslogIdentifier=space-weather-cache
+Restart=on-failure
+RestartSec=5s
+
+[Install]
+WantedBy=multi-user.target

feeder/systemd/space-weather-global-image.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Space Weather Global Image Generator
+After=network.target
+
+[Service]
+Type=simple
+User=homeassistant
+EnvironmentFile=/etc/secrets/space-weather
+ExecStart=/srv/ha-noaa-space-weather/venv/bin/python /srv/ha-noaa-space-weather/feeder/global-image.py
+SyslogIdentifier=space-weather-global-image
+Restart=on-failure
+RestartSec=5s
+
+[Install]
+WantedBy=multi-user.target

feeder/systemd/space-weather-mqtt.service

@@ -1,12 +1,13 @@
 [Unit]
-Description=Space Weather VTEC
+Description=Space Weather MQTT
 After=network.target
 
 [Service]
 Type=simple
 User=homeassistant
 EnvironmentFile=/etc/secrets/space-weather
-ExecStart=/srv/space-weather/ha-noaa-space-weather/venv/bin/python /srv/space-weather/ha-noaa-space-weather/feeder/main.py
+ExecStart=/srv/ha-noaa-space-weather/venv/bin/python /srv/ha-noaa-space-weather/feeder/mqtt.py
+SyslogIdentifier=space-weather-mqtt
 Restart=on-failure
 RestartSec=5s
 

feeder/systemd/space-weather-server.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Space Weather Server
+After=network.target
+
+[Service]
+Type=simple
+User=homeassistant
+EnvironmentFile=/etc/secrets/space-weather
+ExecStart=/srv/ha-noaa-space-weather/venv/bin/python /srv/ha-noaa-space-weather/feeder/server.py
+SyslogIdentifier=space-weather-server
+Restart=on-failure
+RestartSec=5s
+
+[Install]
+WantedBy=multi-user.target