jessikitty
89f65e2d74
Downloads official instruction PDFs from LEGO, located via the Brickset API, one set per interval (10 minutes by default) so neither API gets hammered. Official sets only; MOCs are excluded throughout. - New admin page at /admin/instruction-fetch: queue builder, pacing controls, live countdown and per-set job history - Database-backed queue (instruction_fetch_jobs) and last-run timestamp, so pacing survives restarts - Background worker thread, plus 'flask instructions queue/run/status' for Task Scheduler or cron - SHA-256 dedupe (instructions.sha256), PDF header check, size cap and streaming downloads via temp files - brickset_api: getInstructions2 by set number and getInstructions by setID (the old code sent a set number to the setID method); getSets params now proper JSON; getKeyUsageStats added - Schema top-ups applied automatically at boot: new table plus instructions.sha256/source - Docker: fetcher env vars passed through compose, no worker thread in the entrypoint bootstrap
110 lines
3.5 KiB
Python
110 lines
3.5 KiB
Python
"""
|
|
The background ticker behind the instruction fetcher.
|
|
|
|
A daemon thread wakes every few seconds, asks the fetcher whether the
|
|
interval has elapsed, and if so processes exactly one set. All of the actual
|
|
pacing lives in instruction_fetcher (and in the database), so pausing the
|
|
worker or changing the interval from the admin page takes effect on the next
|
|
tick without a restart.
|
|
|
|
Deliberately a plain thread rather than a scheduler dependency: LIM runs
|
|
single-process under waitress, and Python 3.14 has been quite enough
|
|
dependency excitement for one project.
|
|
|
|
Run one process. If you ever scale LIM horizontally, turn the worker off with
|
|
INSTRUCTION_FETCH_WORKER=0 everywhere and drive it from cron/Task Scheduler
|
|
with `flask instructions run` instead.
|
|
|
|
Created by: Jess Rogerson (yelling commands at Claude.AI)
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import threading
|
|
|
|
_thread = None
|
|
_stop = threading.Event()
|
|
_lock = threading.Lock()
|
|
|
|
|
|
def _is_cli_command():
|
|
"""True when the app was loaded by `flask <cmd>` rather than to serve."""
|
|
if os.environ.get('FLASK_RUN_FROM_CLI') != 'true':
|
|
return False
|
|
args = [arg for arg in sys.argv[1:] if not arg.startswith('-')]
|
|
return bool(args) and args[0] != 'run'
|
|
|
|
|
|
def start(app):
|
|
"""Start the ticker unless it's disabled or already running."""
|
|
if not app.config.get('INSTRUCTION_FETCH_WORKER', True):
|
|
app.logger.info('Instruction fetch worker disabled by config')
|
|
return False
|
|
|
|
# `flask instructions run`, `flask backup restore` and friends shouldn't
|
|
# quietly spin up a fetcher of their own.
|
|
if _is_cli_command():
|
|
return False
|
|
|
|
# Flask's reloader imports the app twice in debug; only the child should
|
|
# own the thread, or two of them race for the same job.
|
|
if app.debug and os.environ.get('WERKZEUG_RUN_MAIN') != 'true':
|
|
return False
|
|
|
|
global _thread
|
|
with _lock:
|
|
if _thread is not None and _thread.is_alive():
|
|
return False
|
|
_stop.clear()
|
|
_thread = threading.Thread(target=_loop, args=(app,),
|
|
name='lim-instruction-fetcher',
|
|
daemon=True)
|
|
_thread.start()
|
|
|
|
app.logger.info('Instruction fetch worker started')
|
|
return True
|
|
|
|
|
|
def stop(timeout=5):
|
|
"""Ask the ticker to finish (used by tests and the CLI)."""
|
|
_stop.set()
|
|
thread = _thread
|
|
if thread is not None and thread.is_alive():
|
|
thread.join(timeout=timeout)
|
|
|
|
|
|
def is_running():
|
|
return _thread is not None and _thread.is_alive()
|
|
|
|
|
|
def _loop(app):
|
|
tick = app.config.get('INSTRUCTION_FETCH_TICK_SECONDS', 20)
|
|
from app.services import instruction_fetcher as fetcher
|
|
|
|
while not _stop.wait(tick):
|
|
try:
|
|
with app.app_context():
|
|
fetcher.run_one()
|
|
except Exception:
|
|
app.logger.exception('Instruction fetch worker tick failed')
|
|
|
|
|
|
def run_once_async(app, force=False):
|
|
"""
|
|
Fire off a single job in the background.
|
|
|
|
Used by the admin page's "Run one now" button so the request returns
|
|
straight away instead of holding the browser open for a 40 MB PDF.
|
|
"""
|
|
def _work():
|
|
try:
|
|
with app.app_context():
|
|
from app.services import instruction_fetcher as fetcher
|
|
fetcher.run_one(force=force)
|
|
except Exception:
|
|
app.logger.exception('Manual instruction fetch failed')
|
|
|
|
thread = threading.Thread(target=_work, name='lim-fetch-once', daemon=True)
|
|
thread.start()
|
|
return thread
|