jessikitty
f7c20de5ac
LIM configured no logging at all, so the instruction fetcher had no trail to follow. Adds a rotating file handler on the root logger plus two owner-only admin pages. - Admin > Logs: filter by level, source and text; tracebacks kept with their error; auto-refresh, raw download, rotated-file browsing and rotate-now - Admin > Logs > Brickset API calls: every outbound call recorded with method, params, caller, duration, result count and error, plus daily quota accounting (only getSets counts) and a comparison against getKeyUsageStats - New owner concept (services/ownership.py): a single account, distinct from is_admin, since these pages expose other admins activity. Defaults to the first admin; flask owner show/set to change - Services now log through module loggers; retention setting and pruning for call records - New CLI: flask owner show/set, flask logs path/prune - New table brickset_api_calls, created automatically at boot; LOG_* env vars, /data/logs in Docker
360 lines
13 KiB
Python
360 lines
13 KiB
Python
"""
|
|
Flask CLI commands for LIM backup and restore.
|
|
|
|
flask backup create [--label nightly] [--keep 5] [--no-sqlite-file]
|
|
flask backup list
|
|
flask backup verify ARCHIVE
|
|
flask backup inspect ARCHIVE
|
|
flask backup restore ARCHIVE [--mode fresh|merge] [--force]
|
|
[--overwrite-files] [--skip-files]
|
|
|
|
flask instructions queue [--missing/--all] [--limit N]
|
|
flask instructions run [--count N] [--force]
|
|
flask instructions status
|
|
|
|
flask owner show
|
|
flask owner set USERNAME
|
|
flask logs path [--tail N]
|
|
flask logs prune [--days N]
|
|
|
|
`restore` is the path to use on a fresh Docker instance: no login required,
|
|
and it works before any admin user exists.
|
|
|
|
Created by: Jess Rogerson (yelling commands at Claude.AI)
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
|
|
import click
|
|
from flask.cli import AppGroup
|
|
|
|
from app.services import backup_service as bsvc
|
|
|
|
backup_cli = AppGroup('backup', help='Backup, verify and restore LIM data.')
|
|
|
|
|
|
def _resolve(archive):
|
|
"""Accept a bare filename in the backup folder or any path."""
|
|
if os.path.isfile(archive):
|
|
return os.path.abspath(archive)
|
|
candidate = os.path.join(bsvc.backup_folder(), os.path.basename(archive))
|
|
if os.path.isfile(candidate):
|
|
return candidate
|
|
raise click.ClickException(f'Archive not found: {archive}')
|
|
|
|
|
|
@backup_cli.command('create')
|
|
@click.option('--label', default=None, help='Short suffix for the filename.')
|
|
@click.option('--keep', default=None, type=int,
|
|
help='Keep only this many archives afterwards.')
|
|
@click.option('--no-sqlite-file', is_flag=True,
|
|
help='Omit the raw SQLite snapshot (JSON dump only).')
|
|
def create_command(label, keep, no_sqlite_file):
|
|
"""Create a backup archive."""
|
|
bar_state = {'last': -1}
|
|
|
|
def on_progress(stage, done, total, note):
|
|
if total and done != bar_state['last']:
|
|
bar_state['last'] = done
|
|
pct = int(done / total * 100)
|
|
click.echo(f'\r [{pct:3d}%] {stage}: {note[:60]:<60}', nl=False)
|
|
|
|
click.echo('Creating backup...')
|
|
result = bsvc.create_backup(label=label, created_by='cli',
|
|
progress=on_progress,
|
|
include_sqlite_file=not no_sqlite_file)
|
|
click.echo('')
|
|
|
|
manifest = result['manifest']
|
|
click.echo(f" Archive : {result['filename']} ({result['size_human']})")
|
|
click.echo(f" Path : {result['path']}")
|
|
click.echo(f" Rows : " + ', '.join(
|
|
f'{v} {k}' for k, v in manifest['counts'].items()))
|
|
click.echo(f" Uploads : {manifest['uploads']['file_count']} files, "
|
|
f"{manifest['uploads']['total_human']}")
|
|
|
|
integrity = manifest['integrity']
|
|
if integrity['missing_count']:
|
|
click.secho(f" WARNING : {integrity['missing_count']} file(s) are "
|
|
f"referenced in the database but missing on disk.",
|
|
fg='yellow')
|
|
for name in integrity['missing_files'][:10]:
|
|
click.echo(f' - {name}')
|
|
if integrity['orphaned_count']:
|
|
click.echo(f" Note : {integrity['orphaned_count']} file(s) on disk "
|
|
f"are not referenced by any record (backed up anyway).")
|
|
|
|
if keep:
|
|
pruned = bsvc.prune_backups(keep)
|
|
if pruned:
|
|
click.echo(f" Pruned : {', '.join(pruned)}")
|
|
|
|
click.secho('Backup complete.', fg='green')
|
|
|
|
|
|
@backup_cli.command('list')
|
|
def list_command():
|
|
"""List archives in the backup folder."""
|
|
backups = bsvc.list_backups()
|
|
if not backups:
|
|
click.echo(f'No backups in {bsvc.backup_folder()}')
|
|
return
|
|
click.echo(f'Backups in {bsvc.backup_folder()}:')
|
|
for entry in backups:
|
|
manifest = entry['manifest'] or {}
|
|
counts = manifest.get('counts', {})
|
|
detail = ', '.join(f'{v} {k}' for k, v in counts.items()) or 'unreadable'
|
|
flag = '' if entry['valid'] else ' [NOT A LIM BACKUP]'
|
|
click.echo(f" {entry['modified']:%Y-%m-%d %H:%M} "
|
|
f"{entry['size_human']:>10} {entry['filename']}{flag}")
|
|
click.echo(f" {detail}")
|
|
|
|
|
|
@backup_cli.command('inspect')
|
|
@click.argument('archive')
|
|
def inspect_command(archive):
|
|
"""Print an archive's manifest."""
|
|
path = _resolve(archive)
|
|
click.echo(json.dumps(bsvc.read_manifest(path), indent=2))
|
|
|
|
|
|
@backup_cli.command('verify')
|
|
@click.argument('archive')
|
|
def verify_command(archive):
|
|
"""Recompute every checksum in an archive."""
|
|
path = _resolve(archive)
|
|
click.echo(f'Verifying {os.path.basename(path)}...')
|
|
report = bsvc.verify_backup(path)
|
|
click.echo(f" Entries checked : {report['checked']}")
|
|
for name in report['failed']:
|
|
click.secho(f' CHECKSUM FAIL : {name}', fg='red')
|
|
for name in report['missing']:
|
|
click.secho(f' MISSING ENTRY : {name}', fg='red')
|
|
for error in report['errors']:
|
|
click.secho(f' ERROR : {error}', fg='red')
|
|
if report['ok']:
|
|
click.secho('Archive is intact.', fg='green')
|
|
else:
|
|
raise click.ClickException('Archive failed verification.')
|
|
|
|
|
|
@backup_cli.command('restore')
|
|
@click.argument('archive')
|
|
@click.option('--mode', type=click.Choice(['fresh', 'merge']), default='fresh',
|
|
help='fresh preserves IDs into an empty DB; merge keeps existing data.')
|
|
@click.option('--force', is_flag=True,
|
|
help='In fresh mode, wipe existing rows first.')
|
|
@click.option('--overwrite-files', is_flag=True,
|
|
help='Overwrite upload files that already exist.')
|
|
@click.option('--skip-files', is_flag=True,
|
|
help='Restore database rows only.')
|
|
@click.option('--yes', is_flag=True, help='Skip the confirmation prompt.')
|
|
def restore_command(archive, mode, force, overwrite_files, skip_files, yes):
|
|
"""Restore an archive into this instance."""
|
|
path = _resolve(archive)
|
|
manifest = bsvc.read_manifest(path)
|
|
|
|
click.echo(f"Archive : {os.path.basename(path)}")
|
|
click.echo(f"Created : {manifest.get('created_at')} on "
|
|
f"{manifest.get('source', {}).get('hostname')}")
|
|
click.echo(f"Rows : " + ', '.join(
|
|
f'{v} {k}' for k, v in manifest.get('counts', {}).items()))
|
|
click.echo(f"Uploads : {manifest.get('uploads', {}).get('file_count')} files, "
|
|
f"{manifest.get('uploads', {}).get('total_human')}")
|
|
click.echo(f"Mode : {mode}"
|
|
+ (' (WIPES EXISTING ROWS)' if mode == 'fresh' and force else ''))
|
|
|
|
if not yes and not click.confirm('Proceed with restore?', default=False):
|
|
click.echo('Cancelled.')
|
|
return
|
|
|
|
def on_progress(stage, done, total, note):
|
|
if total:
|
|
click.echo(f'\r {stage}: {done}/{total} {note[:50]:<50}', nl=False)
|
|
|
|
summary = bsvc.restore_backup(path, mode=mode, force=force,
|
|
overwrite_files=overwrite_files,
|
|
skip_files=skip_files, progress=on_progress)
|
|
click.echo('')
|
|
click.echo(' Inserted : ' + ', '.join(
|
|
f'{v} {k}' for k, v in summary['inserted'].items()))
|
|
if any(summary['skipped'].values()):
|
|
click.echo(' Skipped : ' + ', '.join(
|
|
f'{v} {k}' for k, v in summary['skipped'].items() if v))
|
|
click.echo(f" Files : {summary['files_written']} written "
|
|
f"({summary['files_bytes_human']}), "
|
|
f"{summary['files_skipped']} already present")
|
|
for warning in summary['warnings']:
|
|
click.secho(f' Note : {warning}', fg='yellow')
|
|
click.secho('Restore complete.', fg='green')
|
|
|
|
|
|
instructions_cli = AppGroup(
|
|
'instructions', help='Queue and run automatic instruction downloads.')
|
|
|
|
|
|
@instructions_cli.command('queue')
|
|
@click.option('--missing/--all', 'missing_only', default=True,
|
|
help='Only official sets with no PDF yet (default), or every '
|
|
'official set.')
|
|
@click.option('--limit', default=None, type=int,
|
|
help='Queue at most this many sets.')
|
|
def instructions_queue(missing_only, limit):
|
|
"""Add official sets to the fetch queue."""
|
|
from app.services import instruction_fetcher as fetcher
|
|
|
|
if missing_only:
|
|
added = fetcher.queue_missing(limit=limit)
|
|
else:
|
|
added = fetcher.queue_all_official(limit=limit)
|
|
|
|
minutes = added * fetcher.interval_minutes()
|
|
click.secho(f'Queued {added} set(s).', fg='green')
|
|
if added:
|
|
click.echo(f' At one every {fetcher.interval_minutes()} minutes that '
|
|
f'is roughly {round(minutes / 60, 1)} hours of work.')
|
|
|
|
|
|
@instructions_cli.command('run')
|
|
@click.option('--count', default=1, type=int,
|
|
help='How many sets to process before exiting.')
|
|
@click.option('--force', is_flag=True,
|
|
help='Ignore the interval timer and the enabled switch.')
|
|
def instructions_run(count, force):
|
|
"""
|
|
Process queued sets, respecting the configured interval.
|
|
|
|
Handy from Task Scheduler: `flask instructions run --count 1 --force`
|
|
every 10 minutes gives the same pacing without the worker thread.
|
|
"""
|
|
from app.services import instruction_fetcher as fetcher
|
|
|
|
done = 0
|
|
for _ in range(max(1, count)):
|
|
result = fetcher.run_one(force=force)
|
|
if result is None:
|
|
if not fetcher.is_enabled() and not force:
|
|
click.secho('Fetcher is switched off (use --force to '
|
|
'override).', fg='yellow')
|
|
elif not fetcher.is_due() and not force:
|
|
wait = fetcher.seconds_until_due()
|
|
click.secho(f'Not due for another {wait // 60}m '
|
|
f'{wait % 60}s.', fg='yellow')
|
|
else:
|
|
click.echo('Queue is empty.')
|
|
break
|
|
|
|
done += 1
|
|
click.echo(f" {result['set_number']}: {result['status']} "
|
|
f"({result['files_added']} file(s)) - {result['message']}")
|
|
|
|
click.secho(f'Processed {done} set(s).', fg='green')
|
|
|
|
|
|
@instructions_cli.command('status')
|
|
def instructions_status():
|
|
"""Show queue counts and when the next fetch is allowed."""
|
|
from app.services import instruction_fetcher as fetcher
|
|
|
|
summary = fetcher.status_summary()
|
|
counts = summary['counts']
|
|
wait = summary['next_run_in_seconds']
|
|
|
|
click.echo(' Enabled : ' + ('yes' if summary['enabled'] else 'no'))
|
|
click.echo(f" Interval : {summary['interval_minutes']} minutes")
|
|
click.echo(f" Next run : {'now' if wait == 0 else f'in {wait // 60}m {wait % 60}s'}")
|
|
click.echo(f" Brickset : {'configured' if summary['brickset_configured'] else 'NOT configured'}")
|
|
click.echo(' Queue : ' + ', '.join(
|
|
f'{counts[key]} {key}' for key in
|
|
('queued', 'running', 'done', 'empty', 'failed', 'cancelled')))
|
|
click.echo(f" Library : {summary['official_sets']} official set(s), "
|
|
f"{summary['sets_missing_pdfs']} without a PDF")
|
|
|
|
|
|
owner_cli = AppGroup('owner', help='Who owns this instance.')
|
|
|
|
|
|
@owner_cli.command('show')
|
|
def owner_show():
|
|
"""Show the owner account (the only one who can read the logs)."""
|
|
from app.services.ownership import get_owner
|
|
|
|
owner = get_owner()
|
|
if owner is None:
|
|
click.secho('No users exist yet.', fg='yellow')
|
|
return
|
|
click.secho(f'Owner: {owner.username} <{owner.email}> (id {owner.id})',
|
|
fg='green')
|
|
|
|
|
|
@owner_cli.command('set')
|
|
@click.argument('username')
|
|
def owner_set(username):
|
|
"""Hand ownership to USERNAME (promotes them to admin if needed)."""
|
|
from app.models.user import User
|
|
from app.services.ownership import set_owner
|
|
|
|
user = User.query.filter_by(username=username).first()
|
|
if user is None:
|
|
click.secho(f'No user called {username!r}.', fg='red')
|
|
raise SystemExit(1)
|
|
|
|
set_owner(user)
|
|
click.secho(f'{user.username} is now the owner.', fg='green')
|
|
|
|
|
|
logs_cli = AppGroup('logs', help='Application log file.')
|
|
|
|
|
|
@logs_cli.command('path')
|
|
@click.option('--tail', default=0, type=int,
|
|
help='Also print the last N lines.')
|
|
def logs_path(tail):
|
|
"""Where the log lives, and optionally its tail."""
|
|
import os
|
|
|
|
from flask import current_app
|
|
|
|
from app.services import logging_setup
|
|
|
|
path = logging_setup.log_path(current_app)
|
|
exists = os.path.isfile(path)
|
|
click.echo(f' {path}')
|
|
click.echo(f' exists: {exists}'
|
|
+ (f' size: {os.path.getsize(path)} bytes' if exists else ''))
|
|
click.echo(f' handler active: {logging_setup.get_handler() is not None}')
|
|
|
|
if tail and exists:
|
|
entries, total = logging_setup.read_entries(path, max_entries=tail)
|
|
click.echo(f'\n last {len(entries)} of {total} entries:\n')
|
|
for entry in entries:
|
|
click.echo(f" {entry['time']} {entry['level']:<8} "
|
|
f"{entry['logger']} | {entry['message']}")
|
|
|
|
|
|
@logs_cli.command('prune')
|
|
@click.option('--days', default=None, type=int,
|
|
help='Override the configured retention.')
|
|
def logs_prune(days):
|
|
"""Delete old Brickset API call records."""
|
|
from app.models.api_call import BricksetApiCall
|
|
from app.models.site_setting import SiteSetting
|
|
|
|
if days is None:
|
|
try:
|
|
days = int(SiteSetting.get('api_log_retention_days', '90'))
|
|
except (TypeError, ValueError):
|
|
days = 90
|
|
|
|
removed = BricksetApiCall.prune(days)
|
|
click.secho(f'Removed {removed} record(s) older than {days} days.',
|
|
fg='green')
|
|
|
|
|
|
def register_cli(app):
|
|
app.cli.add_command(backup_cli)
|
|
app.cli.add_command(instructions_cli)
|
|
app.cli.add_command(owner_cli)
|
|
app.cli.add_command(logs_cli)
|