v0.5: Exponential backoff for retrys

This commit is contained in:
s3lph 2022-02-19 03:29:27 +01:00
parent 28c2b1d425
commit 90b80ae529
3 changed files with 30 additions and 14 deletions

View file

@ -1,6 +1,18 @@
# iCalendar Timeseries Server Changelog # iCalendar Timeseries Server Changelog
<!-- BEGIN RELEASE v0.5 -->
## Version 0.5
### Changes
<!-- BEGIN CHANGES 0.5 -->
- Retry calendar scraping with exponential backoff.
<!-- END CHANGES 0.5 -->
<!-- END RELEASE v0.5 -->
<!-- BEGIN RELEASE v0.4.1 --> <!-- BEGIN RELEASE v0.4.1 -->
## Version 0.4.1 ## Version 0.4.1

View file

@ -1,2 +1,2 @@
__version__ = '0.4.1' __version__ = '0.5'

View file

@ -60,12 +60,8 @@ def _scrape_calendar(name: str, config: CalendarConfig, start: datetime, end: da
todos = [] todos = []
opener: urllib.request.OpenerDirector = config.get_url_opener() opener: urllib.request.OpenerDirector = config.get_url_opener()
try:
with opener.open(config.url) as response: with opener.open(config.url) as response:
data = response.read().decode('utf-8') data = response.read().decode('utf-8')
except BaseException:
logging.exception(f'An error occurred while scraping the calendar endpoint "{name}" ({config.url})')
return
calendar = cal.Calendar.from_ical(data) calendar = cal.Calendar.from_ical(data)
for element in calendar.walk(): for element in calendar.walk():
@ -105,26 +101,34 @@ def _scrape_calendar(name: str, config: CalendarConfig, start: datetime, end: da
_TODO_SCRAPE_CACHE[name] = todos _TODO_SCRAPE_CACHE[name] = todos
def scrape_calendar(name: str, config: CalendarConfig): def scrape_calendar(name: str, config: CalendarConfig, retry: int):
# Get current time in configured timezone # Get current time in configured timezone
tz = get_config().tz tz = get_config().tz
now: datetime = datetime.now(tz) now: datetime = datetime.now(tz)
# Reschedule calendar scraping # Only scrape at most once a minute
cron = Timer(config.interval.totimedelta(start=now).total_seconds(), interval = max(int(config.interval.totimedelta(start=now).total_seconds()), 60)
lambda: scrape_calendar(name, config))
cron.start()
# Compute interval for which to return events # Compute interval for which to return events
start_delta: Duration = get_config().start_delta start_delta: Duration = get_config().start_delta
end_delta: Duration = get_config().end_delta end_delta: Duration = get_config().end_delta
start: datetime = now + start_delta start: datetime = now + start_delta
end: datetime = now + end_delta end: datetime = now + end_delta
# Scrape and parse the calendar # Scrape and parse the calendar
try:
_scrape_calendar(name, config, start, end) _scrape_calendar(name, config, start, end)
# Reschedule calendar scraping
cron = Timer(interval, lambda: scrape_calendar(name, config, 0))
except BaseException:
# reschedule with exponential backoff, but no more than the regular scrape interval
backoff_seconds = min(60 * 2**retry, interval)
logging.exception(f'An error occurred while scraping the calendar endpoint "{name}" '
f'({config.url}), retrying in {backoff_seconds}s.')
cron = Timer(backoff_seconds, lambda: scrape_calendar(name, config, retry+1))
cron.start()
def start_scrape_calendar(name: str, config: CalendarConfig): def start_scrape_calendar(name: str, config: CalendarConfig):
# Schedule first calendar scraping # Schedule first calendar scraping
cron = Timer(0, lambda: scrape_calendar(name, config)) cron = Timer(0, lambda: scrape_calendar(name, config, retry=0))
cron.start() cron.start()