implement cellular connection

.gitignore

@@ -2,6 +2,7 @@ ESP32_GENERIC-20240602-v1.23.0.bin
 config.py
 .idea
 test.py
+mpy-cross
 
 # ---> Python
 # Byte-compiled / optimized / DLL files

Doc/Building.md

@@ -0,0 +1,12 @@
+Compiling the MicroPython sourcecode to `.mpy` turns it into "frozen bytecode" which results in lower memory consumption
+during execution.
+
+```shell
+git clone https://github.com/micropython/micropython.git
+cd micropython/mpy-cross
+make
+cp build/mpy-cross <project root>
+```
+
+The `upload.sh` script will manage building this MicroPython project. If you get an error on the `Erase` step, that's
+fine and due to the files not existing to be erased.

src/config.sample.py

@@ -7,11 +7,22 @@ TRACCAR_HOST = 'https://traccar.example.com'
 # WiFi
 WIFI_SSID = 'homenetwork'
 WIFI_PASSWORD = 'password123'
-WIFI_CONNECT_TIMEOUT = 10
+
+# Cellular
+CELLULAR_APN = 'hologram'
+CELLULAR_STARTUP_TIMEOUT = 60  # restart the modem if it doesn't initalize
+
+# GNSS
+GNSS_LOCATION_RETRY = 5
+GNSS_OFFLINE_BUFFER_SIZE = 200
 
 # Intervals
 # All in seconds.
 INTERVAL_WIFI_SCAN = 10
-INTERVAL_NTP_SYNC = 86400
+INTERVAL_CLOCK_SYNC = 86400
 INTERVAL_ACTIVE_POSITION_SEND = 120  # only when active and moving
 INTERVAL_CLOCK_DRIFT_COMP = 1800
+
+# Timeouts
+TIMEOUT_WIFI_CONNECT = 10
+TIMEOUT_GNSS_LOCATION = 300

src/lib/gps/read.py

@@ -18,39 +18,29 @@ GPS_BAUDRATE = 115200
 
 uart = UART(1, baudrate=GPS_BAUDRATE, rx=PIN_GPS_UART_RXD, tx=PIN_GPS_UART_TXD)
 gps_power_pin = Pin(PIN_GPS_POWER, Pin.OUT)
-gps_power_pin.value(0)  # Turn off GPS power initially
+gps_power_pin.value(1)  # Turn on GPS power initially
 
 
 async def read_gps_uart(want: str):
     m = MicropyGPS(location_formatting='dd')
-    buffer = b''
-    decoded = ''
     sreader = asyncio.StreamReader(uart)
 
     while True:
-        if uart.any():
+        try:
-            c = await sreader.read(1)
+            decoded = (await sreader.readline()).decode()
-            if c == b'\r':
+        except UnicodeError:
-                continue
+            # We see sporadic decode errors for some reason, probably on strings we don't care about.
-            if c == b'\n':
+            continue
-                try:
-                    decoded = buffer.decode()
-                except UnicodeError:
-                    # We see sporadic decode errors for some reason, probably on strings we don't care about.
-                    continue
-                buffer = b''
-            else:
-                buffer += c
         if decoded.startswith(want):
             for x in decoded:
                 m.update(x)
         if m.latitude[0] > 0 and m.longitude[0] > 0:
             return m
-        decoded = ''
         await asyncio.sleep(0.05)
 
 
 async def get_position():
+    # TODO: fix altitude always being 0
     gps_power_pin.value(1)  # Always make sure the GPS is on.
 
     @timeout(TIMEOUT_GNSS_LOCATION)
@@ -73,7 +63,7 @@ async def get_position():
             gps_power_pin.value(1)
 
     # Set the clock if it's time.
-    if interval_tracker.check('ntp_sync'):
+    if interval_tracker.check('clock_sync'):
         initialize_rtc(p)
         logger('Updated time', source='GPS')
 

src/lib/interval_tracker.py

@@ -18,7 +18,7 @@ class IntervalTracker:
 
 interval_tracker = IntervalTracker(
     wifi_scan=INTERVAL_WIFI_SCAN,
-    ntp_sync=INTERVAL_NTP_SYNC,
+    clock_sync=INTERVAL_CLOCK_SYNC,
     active_position_send=INTERVAL_ACTIVE_POSITION_SEND,
     clock_drift_comp=INTERVAL_CLOCK_DRIFT_COMP
 )

src/lib/networking/cell_modem.py

@@ -0,0 +1,272 @@
+import asyncio
+import time
+
+import machine
+from machine import Pin
+
+from config import CELLULAR_APN, CELLULAR_STARTUP_TIMEOUT
+from lib.logging import logger, LogLevel
+from lib.runtime import timeout, uTimeoutError
+
+PIN_BEE_POWER = 27
+PIN_BEE_UART_RXD = 35
+PIN_BEE_UART_TXD = 2
+
+bee_power_pin = Pin(PIN_BEE_POWER, Pin.OUT)
+bee_power_pin.value(0)
+time.sleep(0.5)
+bee_power_pin.value(1)
+
+
+class ModemErrorResponse(Exception):
+    def __init__(self, message="Modem returned an error", code: tuple = None):
+        self.command = code[0]
+        self.lines = code[1]
+        super().__init__(f'{message}: {code}')
+
+
+class CellularModem:
+    _lock = asyncio.Lock()
+
+    def __init__(self):
+        self.uart = machine.UART(2, baudrate=115200, timeout=4, rx=PIN_BEE_UART_RXD, tx=PIN_BEE_UART_TXD)
+        self.swriter = asyncio.StreamWriter(self.uart, {})
+        self.sreader = asyncio.StreamReader(self.uart)
+        asyncio.create_task(self.keep_alive())
+
+    async def keep_alive(self):
+        while True:
+            await asyncio.sleep(40)
+            async with self._lock:
+                res = await self.send_command("AT")
+                if not res:
+                    raise OSError("Modem not responding")
+
+    async def send_command(self, command, require_ok: bool = False) -> tuple:
+        lines = []
+        async with self._lock:
+            while self.uart.any():
+                # Clear any initial garbage
+                self.uart.read(1)
+            await self.swriter.awrite(f'{command}\r\n')
+            for i in range(10):
+                raw_line = await self.sreader.readline()
+                if len(raw_line.decode().strip('\r\n')) > 0 and raw_line != f'{command}\r\r\n'.encode():
+                    lines.append(raw_line)
+                if raw_line == b'ERROR\r\n':
+                    raise ModemErrorResponse(code=(command, lines))
+                if len(lines) and lines[-1] == b'OK\r\n':
+                    break
+        response = tuple(x for x in [x.decode().strip('\r\n') for x in lines] if len(x))
+        if require_ok:
+            if response[-1] != 'OK':
+                raise ModemErrorResponse(code=(command, lines))
+        return response
+
+
+cellular = CellularModem()
+
+
+def _parse_csq(csq_value: int):
+    if csq_value == 99:
+        return 'disconnected'
+    elif csq_value == 31:
+        return '51 dBm or greater'
+    elif csq_value == 0:
+        return '113 dBm or less'
+    elif csq_value == 1:
+        return '111 dBm'
+    elif 2 <= csq_value <= 30:
+        return f'{109 - (csq_value - 2) * 2} dBm'
+    else:
+        return 'Invalid CSQ value'
+
+
+async def cellular_wake_modem():
+    i = 5
+    while i > 0:
+        resp = await cellular.send_command('AT')
+        if len(resp) == 1 and resp[0] == 'OK':
+            i -= 1
+        else:
+            logger('\n'.join(resp), source='CELL')
+            i += 1
+        await asyncio.sleep(0.5)
+
+
+async def cellular_wait_cpsi():
+    while True:
+        cpsi_resp = await cellular.send_command('AT+CPSI?')
+        parts = cpsi_resp[0].lstrip('+CPSI: ').split(',')
+        if parts[1] == 'Online':
+            return True, parts
+        elif parts[1] == 'Low Power Mode':
+            return False, parts
+        await asyncio.sleep(0.5)
+
+
+async def cellular_wait_creg():
+    async def check():
+        resp = await cellular.send_command('AT+CREG?')
+        if len(resp) == 2 and resp[0].startswith('+CREG: '):
+            return resp[0] == '+CREG: 0,1' or resp[0] == '+CREG: 0,5', resp
+        else:
+            raise Exception(resp)
+
+    i = 0
+    while True:
+        ready, code = await check()
+        if ready:
+            return ready, code
+        i += 1
+        await asyncio.sleep(0.5)
+
+
+async def cellular_signal_strength():
+    @timeout(10)
+    async def inner():
+        resp = await cellular.send_command('AT+CSQ')
+        if len(resp) == 2 and resp[0].startswith('+CSQ: '):
+            s = int(resp[0].lstrip('+CSQ: ').split(',')[0])
+            return _parse_csq(s)
+        else:
+            raise Exception(resp)
+
+    try:
+        signal_strength = await inner()
+    except uTimeoutError:
+        signal_strength = -1
+    return signal_strength
+
+
+async def cellular_ip():
+    @timeout(3)
+    async def inner():
+        try:
+            await cellular.send_command('AT+IPADDR')
+        except ModemErrorResponse as e:
+            if 'Network not opened' in e.lines[0]:
+                return '0.0.0.0'
+
+    try:
+        ip = await inner()
+    except uTimeoutError:
+        ip = '0.0.0.0'
+    return ip
+
+
+class RunWithtimeoutResult:
+    def __init__(self, failure: bool, result: any):
+        self.failure = failure
+        self.result = result
+
+
+async def run_with_timeout(timeout_sec: int, func, *args, **kwargs) -> RunWithtimeoutResult:
+    @timeout(timeout_sec)
+    async def inner():
+        return await func(*args, **kwargs)
+
+    try:
+        result = await inner()
+        return RunWithtimeoutResult(False, result)
+    except uTimeoutError:
+        return RunWithtimeoutResult(True, None)
+
+
+async def cellular_check_service_ready():
+    resp = await cellular.send_command('AT+CGATT?')
+    if len(resp) == 2 and resp[0].startswith('+CGATT: '):
+        return resp[0] == '+CGATT: 1', resp[0]
+    else:
+        raise Exception(resp)
+
+
+async def restart_modem():
+    bee_power_pin.value(0)
+    time.sleep(0.5)
+    bee_power_pin.value(1)
+    await asyncio.sleep(0.5)
+
+
+async def cellular_check_connected():
+    cpsi = await run_with_timeout(func=cellular_wait_cpsi, timeout_sec=1)
+    if cpsi.failure:
+        return False
+    if cpsi.result[1][0] != 'NO SERVICE':
+        return False
+    if (await cellular_ip()) == '0.0.0.0':
+        return False
+    if (await cellular_signal_strength()) == 'disconnected':
+        return False
+    return True
+
+
+@timeout(CELLULAR_STARTUP_TIMEOUT)
+async def start_modem():
+    while True:
+        if (await run_with_timeout(func=cellular_wake_modem, timeout_sec=30)).failure:
+            logger('Modem wake-up timed out', source='CELL', level=LogLevel.error)
+            await restart_modem()
+            continue
+
+        sim_resp = await cellular.send_command('AT+CPIN?')
+        if not sim_resp[0].endswith(' READY'):
+            msg = '\n'.join(sim_resp)
+            logger(f'SIM card not ready: {msg}', source='CELL', level=LogLevel.error)
+            await restart_modem()
+            continue
+
+        cmds = ['ATE0', 'ATI']
+        for cmd in cmds:
+            while True:
+                init_resp = await cellular.send_command(cmd)
+                logger('\n'.join((x for x in init_resp if x != 'OK')), source='CELL', level=LogLevel.error)
+                if init_resp[-1] != 'OK':
+                    await asyncio.sleep(0.1)
+                else:
+                    break
+
+        simcom_model_resp = await cellular.send_command('AT+SIMCOMATI')
+        model = simcom_model_resp[1]
+        if not model.startswith('Model:'):
+            raise Exception
+        if 'SIM7600' not in model:
+            raise Exception('Not implemented')
+
+        cpsi = await run_with_timeout(func=cellular_wait_cpsi, timeout_sec=1)
+        if cpsi.failure:
+            logger('AT+CPSI timeout', source='CELL', level=LogLevel.error)
+            await restart_modem()
+            continue
+        if not cpsi.result[0]:
+            logger('Bad AT+CPSI status', source='CELL', level=LogLevel.error)
+            await restart_modem()
+            continue
+
+        creg = await run_with_timeout(func=cellular_wait_creg, timeout_sec=1)
+        if creg.failure:
+            logger('AT+CREG timeout, network offline?', source='CELL', level=LogLevel.warning)
+        elif not creg.result[0]:
+            logger(f'Bad AT+CREG response: {creg.result[1]}', source='CELL', level=LogLevel.error)
+            await restart_modem()
+            continue
+
+        await cellular.send_command(f'AT+CGSOCKCONT=1,"IP","{CELLULAR_APN}"', require_ok=True)
+        await cellular.send_command('AT+CSOCKSETPN=1', require_ok=True)
+        await cellular.send_command('AT+CIPMODE=0', require_ok=True)
+        await cellular.send_command('AT+NETOPEN', require_ok=True)
+        break
+
+    logger(f'IP: {await cellular_ip()}', source='CELL')
+    logger(f'Signal strength: {await cellular_signal_strength()}', source='CELL')
+
+
+async def start_modem_task():
+    logger('Initalizing modem', source='CELL')
+    while True:
+        try:
+            await start_modem()
+            break
+        except uTimeoutError:
+            await asyncio.sleep(10)
+    # TODO: loop to reconnect modem

src/lib/networking/select.py

@@ -1,3 +1,4 @@
+from lib.networking.cell_modem import cellular_check_connected
 from lib.networking.wifi import wifi
 
 
@@ -7,9 +8,11 @@ class ConnectionTypes:
     offline = 'Offline'
 
 
-def select_connection_type():
+async def select_connection_type():
     if wifi.is_connected():
         return ConnectionTypes.wifi
-    # if cellular.is_connected():
+    elif await cellular_check_connected():
+        print('CELLULAR DATA NOT IMPLEMENTED')
+        return ConnectionTypes.offline
     else:
         return ConnectionTypes.offline

src/lib/traccar/send.py

@@ -13,12 +13,12 @@ from lib.networking.wifi import wifi
 from lib.traccar.request import TraccarGetRequest, WifiInfo
 
 
-def send_to_traccar(position: Position, wifi_info: WifiInfo, cell_info):
+async def send_to_traccar(position: Position, wifi_info: WifiInfo, cell_info):
     led_on()
     gc.collect()
     event = None
     params = 'None'
-    if select_connection_type() == ConnectionTypes.wifi:
+    if (await select_connection_type()) == ConnectionTypes.wifi:
         event = TraccarGetRequest(
             timestamp=position.timestamp,
             lat=position.latitude,
@@ -42,7 +42,7 @@ def send_to_traccar(position: Position, wifi_info: WifiInfo, cell_info):
     else:
         position_buffer.push(event)
         logger(f'Offline buffer: {position_buffer.size()}/{GNSS_OFFLINE_BUFFER_SIZE}', source='STORE')
-    if select_connection_type() != ConnectionTypes.offline:
+    if (await select_connection_type()) != ConnectionTypes.offline:
         logger(params, source='NET')
     led_off()
     gc.collect()

src/main.py

@@ -1,38 +1,3 @@
-import asyncio
+from run import entrypoint
-import gc
 
-from lib.interval_tracker import interval_tracker
+entrypoint()
-from lib.networking.wifi import wifi
-from lib.ntp import Ntp
-from lib.runtime import reboot
-from lib.traccar.send import send_to_traccar, assemble_position_message
-from startup import startup
-
-
-async def main():
-    await startup()
-    # TODO: background thread to sync items in the position buffer when we go back online
-
-    while True:
-        gc.collect()
-        if interval_tracker.check('wifi_scan'):
-            if not wifi.is_connected():
-                wifi.connect(ignore_missing=True)
-        if interval_tracker.check('active_position_send'):
-            send_to_traccar(*(await assemble_position_message()))
-        if interval_tracker.check('clock_drift_comp'):
-            Ntp.drift_compensate(Ntp.drift_us())
-        gc.collect()
-        await asyncio.sleep(1)
-
-
-try:
-    asyncio.run(main())
-except Exception as e:
-    import sys
-    import time
-
-    sys.print_exception(e)
-    print('\n-- REBOOTING --\n\n')
-    time.sleep(1)
-    reboot()

src/run.py

@@ -0,0 +1,41 @@
+import asyncio
+import gc
+
+from lib.interval_tracker import interval_tracker
+from lib.networking.cell_modem import start_modem_task
+from lib.networking.wifi import wifi
+from lib.ntp import Ntp
+from lib.runtime import reboot
+from lib.traccar.send import send_to_traccar, assemble_position_message
+from startup import startup
+
+
+async def main():
+    asyncio.create_task(start_modem_task())
+    await startup()
+    # TODO: background thread to sync items in the position buffer when we go back online
+
+    while True:
+        gc.collect()
+        if interval_tracker.check('wifi_scan'):
+            if not wifi.is_connected():
+                wifi.connect(ignore_missing=True)
+        if interval_tracker.check('active_position_send'):
+            await send_to_traccar(*(await assemble_position_message()))
+        if interval_tracker.check('clock_drift_comp'):
+            Ntp.drift_compensate(Ntp.drift_us())
+        gc.collect()
+        await asyncio.sleep(1)
+
+
+def entrypoint():
+    try:
+        asyncio.run(main())
+    except Exception as e:
+        import sys
+        import time
+
+        sys.print_exception(e)
+        print('\n-- REBOOTING --\n\n')
+        time.sleep(1)
+        reboot()

src/startup.py

@@ -56,7 +56,7 @@ async def startup():
     print('====================')
 
     # Send the first message
-    send_to_traccar(*(await assemble_position_message(position)))
+    await send_to_traccar(*(await assemble_position_message(position)))
 
 
 def validate_config():

upload.sh

@@ -1,15 +1,30 @@
 #!/bin/bash
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
+echo "Building..."
+SRC_DIR="$SCRIPT_DIR/src"
+TMP_DIR=$(mktemp -d)
+find "$SRC_DIR" -name '*.py' | while read -r FILE
+do
+    REL_PATH="${FILE#"$SRC_DIR"/}"
+    OUT_DIR="$TMP_DIR/$(dirname "$REL_PATH")"
+    BASE_NAME=$(basename "$FILE")
+    mkdir -p "$OUT_DIR"
+    "$SCRIPT_DIR/mpy-cross" "$FILE" -o "$OUT_DIR/${BASE_NAME%.py}.mpy"
+done
+rm "$TMP_DIR/main.mpy"
+cp "$SRC_DIR/main.py" "$TMP_DIR"
+
 echo "Erasing files..."
 "$SCRIPT_DIR"/venv/bin/ampy --port /dev/ttyUSB0 rmdir /lib
 
 echo "Uploading..."
-"$SCRIPT_DIR"/venv/bin/ampy --port /dev/ttyUSB0 put "$SCRIPT_DIR/src/" / || exit
+"$SCRIPT_DIR"/venv/bin/ampy --port /dev/ttyUSB0 put "$TMP_DIR" /
-"$SCRIPT_DIR"/venv/bin/ampy --port /dev/ttyUSB0 rm /config.sample.py
+"$SCRIPT_DIR"/venv/bin/ampy --port /dev/ttyUSB0 rm /config.sample.mpy
+rm -rf "$TMP_DIR"
 
 echo "Resetting..."
-"$SCRIPT_DIR"/venv/bin/ampy --port /dev/ttyUSB0 run --no-output "$SCRIPT_DIR/src/reset.py" || exit
+"$SCRIPT_DIR"/venv/bin/ampy --port /dev/ttyUSB0 run --no-output "$SCRIPT_DIR/src/reset.py"
 
 # Uploading can take a while so make a sound when it's done
 # https://unix.stackexchange.com/questions/1974/how-do-i-make-my-pc-speaker-beep/163716#163716