Add automatic instruction fetcher for official sets
Downloads official instruction PDFs from LEGO, located via the Brickset API, one set per interval (10 minutes by default) so neither API gets hammered. Official sets only; MOCs are excluded throughout. - New admin page at /admin/instruction-fetch: queue builder, pacing controls, live countdown and per-set job history - Database-backed queue (instruction_fetch_jobs) and last-run timestamp, so pacing survives restarts - Background worker thread, plus 'flask instructions queue/run/status' for Task Scheduler or cron - SHA-256 dedupe (instructions.sha256), PDF header check, size cap and streaming downloads via temp files - brickset_api: getInstructions2 by set number and getInstructions by setID (the old code sent a set number to the setID method); getSets params now proper JSON; getKeyUsageStats added - Schema top-ups applied automatically at boot: new table plus instructions.sha256/source - Docker: fetcher env vars passed through compose, no worker thread in the entrypoint bootstrap
This commit is contained in:
1 parent
aac2b60517
commit
89f65e2d74
23 files changed
+2073
-76
No files matched your search
+87
@@ -8,6 +8,10 @@ Flask CLI commands for LIM backup and restore.
|
||||
flask backup restore ARCHIVE [--mode fresh|merge] [--force]
|
||||
[--overwrite-files] [--skip-files]
|
||||
|
||||
flask instructions queue [--missing/--all] [--limit N]
|
||||
flask instructions run [--count N] [--force]
|
||||
flask instructions status
|
||||
|
||||
`restore` is the path to use on a fresh Docker instance: no login required,
|
||||
and it works before any admin user exists.
|
||||
|
||||
@@ -181,5 +185,88 @@ def restore_command(archive, mode, force, overwrite_files, skip_files, yes):
|
||||
click.secho('Restore complete.', fg='green')
|
||||
|
||||
|
||||
instructions_cli = AppGroup(
|
||||
'instructions', help='Queue and run automatic instruction downloads.')
|
||||
|
||||
|
||||
@instructions_cli.command('queue')
|
||||
@click.option('--missing/--all', 'missing_only', default=True,
|
||||
help='Only official sets with no PDF yet (default), or every '
|
||||
'official set.')
|
||||
@click.option('--limit', default=None, type=int,
|
||||
help='Queue at most this many sets.')
|
||||
def instructions_queue(missing_only, limit):
|
||||
"""Add official sets to the fetch queue."""
|
||||
from app.services import instruction_fetcher as fetcher
|
||||
|
||||
if missing_only:
|
||||
added = fetcher.queue_missing(limit=limit)
|
||||
else:
|
||||
added = fetcher.queue_all_official(limit=limit)
|
||||
|
||||
minutes = added * fetcher.interval_minutes()
|
||||
click.secho(f'Queued {added} set(s).', fg='green')
|
||||
if added:
|
||||
click.echo(f' At one every {fetcher.interval_minutes()} minutes that '
|
||||
f'is roughly {round(minutes / 60, 1)} hours of work.')
|
||||
|
||||
|
||||
@instructions_cli.command('run')
|
||||
@click.option('--count', default=1, type=int,
|
||||
help='How many sets to process before exiting.')
|
||||
@click.option('--force', is_flag=True,
|
||||
help='Ignore the interval timer and the enabled switch.')
|
||||
def instructions_run(count, force):
|
||||
"""
|
||||
Process queued sets, respecting the configured interval.
|
||||
|
||||
Handy from Task Scheduler: `flask instructions run --count 1 --force`
|
||||
every 10 minutes gives the same pacing without the worker thread.
|
||||
"""
|
||||
from app.services import instruction_fetcher as fetcher
|
||||
|
||||
done = 0
|
||||
for _ in range(max(1, count)):
|
||||
result = fetcher.run_one(force=force)
|
||||
if result is None:
|
||||
if not fetcher.is_enabled() and not force:
|
||||
click.secho('Fetcher is switched off (use --force to '
|
||||
'override).', fg='yellow')
|
||||
elif not fetcher.is_due() and not force:
|
||||
wait = fetcher.seconds_until_due()
|
||||
click.secho(f'Not due for another {wait // 60}m '
|
||||
f'{wait % 60}s.', fg='yellow')
|
||||
else:
|
||||
click.echo('Queue is empty.')
|
||||
break
|
||||
|
||||
done += 1
|
||||
click.echo(f" {result['set_number']}: {result['status']} "
|
||||
f"({result['files_added']} file(s)) - {result['message']}")
|
||||
|
||||
click.secho(f'Processed {done} set(s).', fg='green')
|
||||
|
||||
|
||||
@instructions_cli.command('status')
|
||||
def instructions_status():
|
||||
"""Show queue counts and when the next fetch is allowed."""
|
||||
from app.services import instruction_fetcher as fetcher
|
||||
|
||||
summary = fetcher.status_summary()
|
||||
counts = summary['counts']
|
||||
wait = summary['next_run_in_seconds']
|
||||
|
||||
click.echo(' Enabled : ' + ('yes' if summary['enabled'] else 'no'))
|
||||
click.echo(f" Interval : {summary['interval_minutes']} minutes")
|
||||
click.echo(f" Next run : {'now' if wait == 0 else f'in {wait // 60}m {wait % 60}s'}")
|
||||
click.echo(f" Brickset : {'configured' if summary['brickset_configured'] else 'NOT configured'}")
|
||||
click.echo(' Queue : ' + ', '.join(
|
||||
f'{counts[key]} {key}' for key in
|
||||
('queued', 'running', 'done', 'empty', 'failed', 'cancelled')))
|
||||
click.echo(f" Library : {summary['official_sets']} official set(s), "
|
||||
f"{summary['sets_missing_pdfs']} without a PDF")
|
||||
|
||||
|
||||
def register_cli(app):
|
||||
app.cli.add_command(backup_cli)
|
||||
app.cli.add_command(instructions_cli)
|
||||
Reference in new issue
Block a user