jessikitty
89f65e2d74
Downloads official instruction PDFs from LEGO, located via the Brickset API, one set per interval (10 minutes by default) so neither API gets hammered. Official sets only; MOCs are excluded throughout. - New admin page at /admin/instruction-fetch: queue builder, pacing controls, live countdown and per-set job history - Database-backed queue (instruction_fetch_jobs) and last-run timestamp, so pacing survives restarts - Background worker thread, plus 'flask instructions queue/run/status' for Task Scheduler or cron - SHA-256 dedupe (instructions.sha256), PDF header check, size cap and streaming downloads via temp files - brickset_api: getInstructions2 by set number and getInstructions by setID (the old code sent a set number to the setID method); getSets params now proper JSON; getKeyUsageStats added - Schema top-ups applied automatically at boot: new table plus instructions.sha256/source - Docker: fetcher env vars passed through compose, no worker thread in the entrypoint bootstrap
273 lines
11 KiB
Python
273 lines
11 KiB
Python
"""
|
|
Flask CLI commands for LIM backup and restore.
|
|
|
|
flask backup create [--label nightly] [--keep 5] [--no-sqlite-file]
|
|
flask backup list
|
|
flask backup verify ARCHIVE
|
|
flask backup inspect ARCHIVE
|
|
flask backup restore ARCHIVE [--mode fresh|merge] [--force]
|
|
[--overwrite-files] [--skip-files]
|
|
|
|
flask instructions queue [--missing/--all] [--limit N]
|
|
flask instructions run [--count N] [--force]
|
|
flask instructions status
|
|
|
|
`restore` is the path to use on a fresh Docker instance: no login required,
|
|
and it works before any admin user exists.
|
|
|
|
Created by: Jess Rogerson (yelling commands at Claude.AI)
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
|
|
import click
|
|
from flask.cli import AppGroup
|
|
|
|
from app.services import backup_service as bsvc
|
|
|
|
backup_cli = AppGroup('backup', help='Backup, verify and restore LIM data.')
|
|
|
|
|
|
def _resolve(archive):
|
|
"""Accept a bare filename in the backup folder or any path."""
|
|
if os.path.isfile(archive):
|
|
return os.path.abspath(archive)
|
|
candidate = os.path.join(bsvc.backup_folder(), os.path.basename(archive))
|
|
if os.path.isfile(candidate):
|
|
return candidate
|
|
raise click.ClickException(f'Archive not found: {archive}')
|
|
|
|
|
|
@backup_cli.command('create')
|
|
@click.option('--label', default=None, help='Short suffix for the filename.')
|
|
@click.option('--keep', default=None, type=int,
|
|
help='Keep only this many archives afterwards.')
|
|
@click.option('--no-sqlite-file', is_flag=True,
|
|
help='Omit the raw SQLite snapshot (JSON dump only).')
|
|
def create_command(label, keep, no_sqlite_file):
|
|
"""Create a backup archive."""
|
|
bar_state = {'last': -1}
|
|
|
|
def on_progress(stage, done, total, note):
|
|
if total and done != bar_state['last']:
|
|
bar_state['last'] = done
|
|
pct = int(done / total * 100)
|
|
click.echo(f'\r [{pct:3d}%] {stage}: {note[:60]:<60}', nl=False)
|
|
|
|
click.echo('Creating backup...')
|
|
result = bsvc.create_backup(label=label, created_by='cli',
|
|
progress=on_progress,
|
|
include_sqlite_file=not no_sqlite_file)
|
|
click.echo('')
|
|
|
|
manifest = result['manifest']
|
|
click.echo(f" Archive : {result['filename']} ({result['size_human']})")
|
|
click.echo(f" Path : {result['path']}")
|
|
click.echo(f" Rows : " + ', '.join(
|
|
f'{v} {k}' for k, v in manifest['counts'].items()))
|
|
click.echo(f" Uploads : {manifest['uploads']['file_count']} files, "
|
|
f"{manifest['uploads']['total_human']}")
|
|
|
|
integrity = manifest['integrity']
|
|
if integrity['missing_count']:
|
|
click.secho(f" WARNING : {integrity['missing_count']} file(s) are "
|
|
f"referenced in the database but missing on disk.",
|
|
fg='yellow')
|
|
for name in integrity['missing_files'][:10]:
|
|
click.echo(f' - {name}')
|
|
if integrity['orphaned_count']:
|
|
click.echo(f" Note : {integrity['orphaned_count']} file(s) on disk "
|
|
f"are not referenced by any record (backed up anyway).")
|
|
|
|
if keep:
|
|
pruned = bsvc.prune_backups(keep)
|
|
if pruned:
|
|
click.echo(f" Pruned : {', '.join(pruned)}")
|
|
|
|
click.secho('Backup complete.', fg='green')
|
|
|
|
|
|
@backup_cli.command('list')
|
|
def list_command():
|
|
"""List archives in the backup folder."""
|
|
backups = bsvc.list_backups()
|
|
if not backups:
|
|
click.echo(f'No backups in {bsvc.backup_folder()}')
|
|
return
|
|
click.echo(f'Backups in {bsvc.backup_folder()}:')
|
|
for entry in backups:
|
|
manifest = entry['manifest'] or {}
|
|
counts = manifest.get('counts', {})
|
|
detail = ', '.join(f'{v} {k}' for k, v in counts.items()) or 'unreadable'
|
|
flag = '' if entry['valid'] else ' [NOT A LIM BACKUP]'
|
|
click.echo(f" {entry['modified']:%Y-%m-%d %H:%M} "
|
|
f"{entry['size_human']:>10} {entry['filename']}{flag}")
|
|
click.echo(f" {detail}")
|
|
|
|
|
|
@backup_cli.command('inspect')
|
|
@click.argument('archive')
|
|
def inspect_command(archive):
|
|
"""Print an archive's manifest."""
|
|
path = _resolve(archive)
|
|
click.echo(json.dumps(bsvc.read_manifest(path), indent=2))
|
|
|
|
|
|
@backup_cli.command('verify')
|
|
@click.argument('archive')
|
|
def verify_command(archive):
|
|
"""Recompute every checksum in an archive."""
|
|
path = _resolve(archive)
|
|
click.echo(f'Verifying {os.path.basename(path)}...')
|
|
report = bsvc.verify_backup(path)
|
|
click.echo(f" Entries checked : {report['checked']}")
|
|
for name in report['failed']:
|
|
click.secho(f' CHECKSUM FAIL : {name}', fg='red')
|
|
for name in report['missing']:
|
|
click.secho(f' MISSING ENTRY : {name}', fg='red')
|
|
for error in report['errors']:
|
|
click.secho(f' ERROR : {error}', fg='red')
|
|
if report['ok']:
|
|
click.secho('Archive is intact.', fg='green')
|
|
else:
|
|
raise click.ClickException('Archive failed verification.')
|
|
|
|
|
|
@backup_cli.command('restore')
|
|
@click.argument('archive')
|
|
@click.option('--mode', type=click.Choice(['fresh', 'merge']), default='fresh',
|
|
help='fresh preserves IDs into an empty DB; merge keeps existing data.')
|
|
@click.option('--force', is_flag=True,
|
|
help='In fresh mode, wipe existing rows first.')
|
|
@click.option('--overwrite-files', is_flag=True,
|
|
help='Overwrite upload files that already exist.')
|
|
@click.option('--skip-files', is_flag=True,
|
|
help='Restore database rows only.')
|
|
@click.option('--yes', is_flag=True, help='Skip the confirmation prompt.')
|
|
def restore_command(archive, mode, force, overwrite_files, skip_files, yes):
|
|
"""Restore an archive into this instance."""
|
|
path = _resolve(archive)
|
|
manifest = bsvc.read_manifest(path)
|
|
|
|
click.echo(f"Archive : {os.path.basename(path)}")
|
|
click.echo(f"Created : {manifest.get('created_at')} on "
|
|
f"{manifest.get('source', {}).get('hostname')}")
|
|
click.echo(f"Rows : " + ', '.join(
|
|
f'{v} {k}' for k, v in manifest.get('counts', {}).items()))
|
|
click.echo(f"Uploads : {manifest.get('uploads', {}).get('file_count')} files, "
|
|
f"{manifest.get('uploads', {}).get('total_human')}")
|
|
click.echo(f"Mode : {mode}"
|
|
+ (' (WIPES EXISTING ROWS)' if mode == 'fresh' and force else ''))
|
|
|
|
if not yes and not click.confirm('Proceed with restore?', default=False):
|
|
click.echo('Cancelled.')
|
|
return
|
|
|
|
def on_progress(stage, done, total, note):
|
|
if total:
|
|
click.echo(f'\r {stage}: {done}/{total} {note[:50]:<50}', nl=False)
|
|
|
|
summary = bsvc.restore_backup(path, mode=mode, force=force,
|
|
overwrite_files=overwrite_files,
|
|
skip_files=skip_files, progress=on_progress)
|
|
click.echo('')
|
|
click.echo(' Inserted : ' + ', '.join(
|
|
f'{v} {k}' for k, v in summary['inserted'].items()))
|
|
if any(summary['skipped'].values()):
|
|
click.echo(' Skipped : ' + ', '.join(
|
|
f'{v} {k}' for k, v in summary['skipped'].items() if v))
|
|
click.echo(f" Files : {summary['files_written']} written "
|
|
f"({summary['files_bytes_human']}), "
|
|
f"{summary['files_skipped']} already present")
|
|
for warning in summary['warnings']:
|
|
click.secho(f' Note : {warning}', fg='yellow')
|
|
click.secho('Restore complete.', fg='green')
|
|
|
|
|
|
instructions_cli = AppGroup(
|
|
'instructions', help='Queue and run automatic instruction downloads.')
|
|
|
|
|
|
@instructions_cli.command('queue')
|
|
@click.option('--missing/--all', 'missing_only', default=True,
|
|
help='Only official sets with no PDF yet (default), or every '
|
|
'official set.')
|
|
@click.option('--limit', default=None, type=int,
|
|
help='Queue at most this many sets.')
|
|
def instructions_queue(missing_only, limit):
|
|
"""Add official sets to the fetch queue."""
|
|
from app.services import instruction_fetcher as fetcher
|
|
|
|
if missing_only:
|
|
added = fetcher.queue_missing(limit=limit)
|
|
else:
|
|
added = fetcher.queue_all_official(limit=limit)
|
|
|
|
minutes = added * fetcher.interval_minutes()
|
|
click.secho(f'Queued {added} set(s).', fg='green')
|
|
if added:
|
|
click.echo(f' At one every {fetcher.interval_minutes()} minutes that '
|
|
f'is roughly {round(minutes / 60, 1)} hours of work.')
|
|
|
|
|
|
@instructions_cli.command('run')
|
|
@click.option('--count', default=1, type=int,
|
|
help='How many sets to process before exiting.')
|
|
@click.option('--force', is_flag=True,
|
|
help='Ignore the interval timer and the enabled switch.')
|
|
def instructions_run(count, force):
|
|
"""
|
|
Process queued sets, respecting the configured interval.
|
|
|
|
Handy from Task Scheduler: `flask instructions run --count 1 --force`
|
|
every 10 minutes gives the same pacing without the worker thread.
|
|
"""
|
|
from app.services import instruction_fetcher as fetcher
|
|
|
|
done = 0
|
|
for _ in range(max(1, count)):
|
|
result = fetcher.run_one(force=force)
|
|
if result is None:
|
|
if not fetcher.is_enabled() and not force:
|
|
click.secho('Fetcher is switched off (use --force to '
|
|
'override).', fg='yellow')
|
|
elif not fetcher.is_due() and not force:
|
|
wait = fetcher.seconds_until_due()
|
|
click.secho(f'Not due for another {wait // 60}m '
|
|
f'{wait % 60}s.', fg='yellow')
|
|
else:
|
|
click.echo('Queue is empty.')
|
|
break
|
|
|
|
done += 1
|
|
click.echo(f" {result['set_number']}: {result['status']} "
|
|
f"({result['files_added']} file(s)) - {result['message']}")
|
|
|
|
click.secho(f'Processed {done} set(s).', fg='green')
|
|
|
|
|
|
@instructions_cli.command('status')
|
|
def instructions_status():
|
|
"""Show queue counts and when the next fetch is allowed."""
|
|
from app.services import instruction_fetcher as fetcher
|
|
|
|
summary = fetcher.status_summary()
|
|
counts = summary['counts']
|
|
wait = summary['next_run_in_seconds']
|
|
|
|
click.echo(' Enabled : ' + ('yes' if summary['enabled'] else 'no'))
|
|
click.echo(f" Interval : {summary['interval_minutes']} minutes")
|
|
click.echo(f" Next run : {'now' if wait == 0 else f'in {wait // 60}m {wait % 60}s'}")
|
|
click.echo(f" Brickset : {'configured' if summary['brickset_configured'] else 'NOT configured'}")
|
|
click.echo(' Queue : ' + ', '.join(
|
|
f'{counts[key]} {key}' for key in
|
|
('queued', 'running', 'done', 'empty', 'failed', 'cancelled')))
|
|
click.echo(f" Library : {summary['official_sets']} official set(s), "
|
|
f"{summary['sets_missing_pdfs']} without a PDF")
|
|
|
|
|
|
def register_cli(app):
|
|
app.cli.add_command(backup_cli)
|
|
app.cli.add_command(instructions_cli)
|