From 1a2f046be9c8c9a133bae9facd64f38850743bd5 Mon Sep 17 00:00:00 2001 From: Marcus Puckett <13648427+mpuckett159@users.noreply.github.com> Date: Mon, 15 Nov 2021 21:14:34 -0800 Subject: [PATCH] Implement backoff decorator for check_radio_calls function (#82) --- .env.example | 1 + requirements.txt | 1 + signal_scanner_bot/env.py | 3 ++ signal_scanner_bot/radio_monitor_alert.py | 50 ++++++++++++++++------- signal_scanner_bot/transport.py | 10 ++--- 5 files changed, 43 insertions(+), 22 deletions(-) diff --git a/.env.example b/.env.example index 6c67dba..2eca591 100644 --- a/.env.example +++ b/.env.example @@ -21,4 +21,5 @@ RADIO_MONITOR_UNITS=CSV of units to be looking for on radio IDs, case insensitiv RADIO_MONITOR_CONTACT=signal group ID to send the message to RADIO_MONITOR_LOOKBACK=Basically the check interval for looking for new calls from openmhz. Time value is in seconds and must be at least 45 or greater. If not set or less than 45 the interval will be set for 45 seconds. RADIO_AUDIO_CHUNK_SIZE=How many bytes to read when downloading an audio file from OpenMhz. Defaults to 10 bytes. Probably shouldn't be changed. +RADIO_CHASER_BACKOFF=The amount of time in seconds to backoff on the OpenMHz site for DEFAULT_TZ=TZ database formatted name for a timezone. Defaults to US/Pacific diff --git a/requirements.txt b/requirements.txt index 57976d0..ce700d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ aiofiles>=0.7.0 aiohttp>=3.7.4 +backoff>=1.11.1 Click==7.1.2 peony-twitter==2.0.2 pre-commit==2.9.0 diff --git a/signal_scanner_bot/env.py b/signal_scanner_bot/env.py index 3b249f9..721aa03 100644 --- a/signal_scanner_bot/env.py +++ b/signal_scanner_bot/env.py @@ -203,6 +203,9 @@ def _format_hashtags(to_cast: str) -> List[str]: RADIO_AUDIO_CHUNK_SIZE = _env( "RADIO_AUDIO_CHUNK_SIZE", convert=_cast_to_int, fail=False, default=10 ) +RADIO_CHASER_BACKOFF = _env( + "RRADIO_CHASER_BACKOFF", convert=_cast_to_int, fail=False, default=10800 +) # Check to make sure the lookback interval is greater than or equal to 45 seconds if RADIO_MONITOR_LOOKBACK < 45: diff --git a/signal_scanner_bot/radio_monitor_alert.py b/signal_scanner_bot/radio_monitor_alert.py index 5037987..f67ad9c 100644 --- a/signal_scanner_bot/radio_monitor_alert.py +++ b/signal_scanner_bot/radio_monitor_alert.py @@ -2,8 +2,9 @@ from datetime import datetime, timedelta from typing import Dict, List, Optional, Tuple +import aiohttp +import backoff import pytz -import requests from . import env @@ -32,24 +33,38 @@ def _calculate_lookback_time() -> str: return time_stamp_array[0] + time_stamp_array[1][:3] -def get_openmhz_calls() -> Dict: +async def get_openmhz_calls() -> Dict: lookback_time = _calculate_lookback_time() log.debug(f"Lookback is currently set to: {lookback_time}") - response = requests.get(env.OPENMHZ_URL, params={"time": lookback_time}) - return response.json()["calls"] + async with aiohttp.ClientSession(raise_for_status=True) as session: + async with session.get( + env.OPENMHZ_URL, params={"time": lookback_time} + ) as response: + return (await response.json())["calls"] -def get_pigs(calls: Dict) -> List[Tuple[Dict, str, str]]: +async def get_pigs(calls: Dict) -> List[Tuple[Dict, str, str]]: interesting_pigs = [] for call in calls: time = call["time"] - radios = radios = {f"7{radio['src']:0>5}" for radio in call["srcList"]} - if not len(radios): + # Due to requirements from aiohttp library it is required that the radios object be + # of the forms: + # * {'key1': 'value1', 'key2': 'value2'} + # * {"key": ["value1", "value2"]} + # * [("key", "value1"), ("key", "value2")] + # + # more pointedly, it can not be + # * {"key": {"value1", "value2"}} + # + # because aiohttp will choke on it. See the following link for more details + # https://docs.aiohttp.org/en/stable/client_quickstart.html#passing-parameters-in-urls + radios = {"radio": list({f"7{radio['src']:0>5}" for radio in call["srcList"]})} + if not len(radios["radio"]): continue - cops = requests.get(env.RADIO_CHASER_URL, params={"radio": radios}) - log.debug(f"URL requested: {cops.url}") - log.debug(f"List of cops returned by radio-chaser:\n{cops.json()}") - for cop in cops.json().values(): + async with aiohttp.ClientSession(raise_for_status=True) as session: + async with session.get(env.RADIO_CHASER_URL, params=radios) as response: + cops = await response.json() + for cop in cops.values(): if all( unit.lower() not in cop["unit_description"].lower() for unit in env.RADIO_MONITOR_UNITS @@ -75,10 +90,15 @@ def format_pigs(pigs: List[Tuple[Dict, str, str]]) -> List[Tuple[str, str]]: return formatted_pigs -def check_radio_calls() -> Optional[List[Tuple[str, str]]]: - calls = get_openmhz_calls() - log.debug(f"Calls from OpenMHz:\n{calls}") - pigs = get_pigs(calls) +@backoff.on_exception( + backoff.expo, + aiohttp.ClientError, + logger=log, + max_time=env.RADIO_CHASER_BACKOFF, +) +async def check_radio_calls() -> Optional[List[Tuple[str, str]]]: + calls = await get_openmhz_calls() + pigs = await get_pigs(calls) if not pigs: return None log.debug(f"Interesting pigs found\n{pigs}") diff --git a/signal_scanner_bot/transport.py b/signal_scanner_bot/transport.py index 487373e..83c4753 100644 --- a/signal_scanner_bot/transport.py +++ b/signal_scanner_bot/transport.py @@ -1,6 +1,5 @@ """Main module.""" import asyncio -import json import logging import subprocess from datetime import date, datetime, timedelta @@ -146,7 +145,9 @@ async def radio_monitor_alert_transport() -> None: while True: try: log.debug("Checking for monitored units' radio activity.") - if radio_monitor_alert_messages := radio_monitor_alert.check_radio_calls(): + if ( + radio_monitor_alert_messages := await radio_monitor_alert.check_radio_calls() + ): log.info( "Radio activity found for monitored units sending alert to group." ) @@ -159,11 +160,6 @@ async def radio_monitor_alert_transport() -> None: f"Sleeping for {env.RADIO_MONITOR_LOOKBACK}s before checking for monitored unit alerts again." ) await asyncio.sleep(env.RADIO_MONITOR_LOOKBACK) - except json.decoder.JSONDecodeError: - # Sometimes OpenMHz doesn't return properly and the Python json library throws this error. - # It is transient and so doesn't really need to panic or throw an exception, so we will - # ignore it. - pass except Exception as err: log.exception(err) signal.panic(err)