jessikitty
f7c20de5ac
LIM configured no logging at all, so the instruction fetcher had no trail to follow. Adds a rotating file handler on the root logger plus two owner-only admin pages. - Admin > Logs: filter by level, source and text; tracebacks kept with their error; auto-refresh, raw download, rotated-file browsing and rotate-now - Admin > Logs > Brickset API calls: every outbound call recorded with method, params, caller, duration, result count and error, plus daily quota accounting (only getSets counts) and a comparison against getKeyUsageStats - New owner concept (services/ownership.py): a single account, distinct from is_admin, since these pages expose other admins activity. Defaults to the first admin; flask owner show/set to change - Services now log through module loggers; retention setting and pruning for call records - New CLI: flask owner show/set, flask logs path/prune - New table brickset_api_calls, created automatically at boot; LOG_* env vars, /data/logs in Docker
114 lines
3.6 KiB
Python
114 lines
3.6 KiB
Python
"""
|
|
The background ticker behind the instruction fetcher.
|
|
|
|
A daemon thread wakes every few seconds, asks the fetcher whether the
|
|
interval has elapsed, and if so processes exactly one set. All of the actual
|
|
pacing lives in instruction_fetcher (and in the database), so pausing the
|
|
worker or changing the interval from the admin page takes effect on the next
|
|
tick without a restart.
|
|
|
|
Deliberately a plain thread rather than a scheduler dependency: LIM runs
|
|
single-process under waitress, and Python 3.14 has been quite enough
|
|
dependency excitement for one project.
|
|
|
|
Run one process. If you ever scale LIM horizontally, turn the worker off with
|
|
INSTRUCTION_FETCH_WORKER=0 everywhere and drive it from cron/Task Scheduler
|
|
with `flask instructions run` instead.
|
|
|
|
Created by: Jess Rogerson (yelling commands at Claude.AI)
|
|
"""
|
|
|
|
import logging
|
|
import os
|
|
import sys
|
|
import threading
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_thread = None
|
|
_stop = threading.Event()
|
|
_lock = threading.Lock()
|
|
|
|
|
|
def _is_cli_command():
|
|
"""True when the app was loaded by `flask <cmd>` rather than to serve."""
|
|
if os.environ.get('FLASK_RUN_FROM_CLI') != 'true':
|
|
return False
|
|
args = [arg for arg in sys.argv[1:] if not arg.startswith('-')]
|
|
return bool(args) and args[0] != 'run'
|
|
|
|
|
|
def start(app):
|
|
"""Start the ticker unless it's disabled or already running."""
|
|
if not app.config.get('INSTRUCTION_FETCH_WORKER', True):
|
|
logger.info('Instruction fetch worker disabled by config')
|
|
return False
|
|
|
|
# `flask instructions run`, `flask backup restore` and friends shouldn't
|
|
# quietly spin up a fetcher of their own.
|
|
if _is_cli_command():
|
|
return False
|
|
|
|
# Flask's reloader imports the app twice in debug; only the child should
|
|
# own the thread, or two of them race for the same job.
|
|
if app.debug and os.environ.get('WERKZEUG_RUN_MAIN') != 'true':
|
|
return False
|
|
|
|
global _thread
|
|
with _lock:
|
|
if _thread is not None and _thread.is_alive():
|
|
return False
|
|
_stop.clear()
|
|
_thread = threading.Thread(target=_loop, args=(app,),
|
|
name='lim-instruction-fetcher',
|
|
daemon=True)
|
|
_thread.start()
|
|
|
|
logger.info('Instruction fetch worker started (tick %ss)',
|
|
app.config.get('INSTRUCTION_FETCH_TICK_SECONDS', 20))
|
|
return True
|
|
|
|
|
|
def stop(timeout=5):
|
|
"""Ask the ticker to finish (used by tests and the CLI)."""
|
|
_stop.set()
|
|
thread = _thread
|
|
if thread is not None and thread.is_alive():
|
|
thread.join(timeout=timeout)
|
|
|
|
|
|
def is_running():
|
|
return _thread is not None and _thread.is_alive()
|
|
|
|
|
|
def _loop(app):
|
|
tick = app.config.get('INSTRUCTION_FETCH_TICK_SECONDS', 20)
|
|
from app.services import instruction_fetcher as fetcher
|
|
|
|
while not _stop.wait(tick):
|
|
try:
|
|
with app.app_context():
|
|
fetcher.run_one()
|
|
except Exception:
|
|
logger.exception('Instruction fetch worker tick failed')
|
|
|
|
|
|
def run_once_async(app, force=False):
|
|
"""
|
|
Fire off a single job in the background.
|
|
|
|
Used by the admin page's "Run one now" button so the request returns
|
|
straight away instead of holding the browser open for a 40 MB PDF.
|
|
"""
|
|
def _work():
|
|
try:
|
|
with app.app_context():
|
|
from app.services import instruction_fetcher as fetcher
|
|
fetcher.run_one(force=force)
|
|
except Exception:
|
|
logger.exception('Manual instruction fetch failed')
|
|
|
|
thread = threading.Thread(target=_work, name='lim-fetch-once', daemon=True)
|
|
thread.start()
|
|
return thread
|