diff --git a/.env.docker.example b/.env.docker.example index 28eebf3..9e18170 100644 --- a/.env.docker.example +++ b/.env.docker.example @@ -24,3 +24,13 @@ SETS_PER_PAGE=20 BRICKSET_API_KEY= BRICKSET_USERNAME= BRICKSET_PASSWORD= + +# --- instruction fetcher (optional) ----------------------------------------- +# Pacing (interval, delays, on/off) is set in the admin UI, not here. +# Leave the worker on unless you scale the app container past one replica or +# drive the queue yourself with `docker compose exec app flask instructions run`. +INSTRUCTION_FETCH_WORKER=1 +INSTRUCTION_FETCH_TICK_SECONDS=20 +# Largest instruction PDF to accept, in megabytes. +FETCH_MAX_FILE_MB=150 +FETCH_TIMEOUT_SECONDS=60 diff --git a/.env.example b/.env.example index 20059d5..2a41740 100644 --- a/.env.example +++ b/.env.example @@ -26,3 +26,15 @@ BACKUP_KEEP=5 # Application Configuration SETS_PER_PAGE=20 + +# --- Automatic instruction fetcher ----------------------------------------- +# Pacing lives in the admin UI (Admin > Instruction Fetcher). These are the +# process-level knobs. +# Set to 0 to disable the background worker thread and drive the queue from +# Task Scheduler / cron with `flask instructions run` instead. +INSTRUCTION_FETCH_WORKER=1 +# How often the worker checks whether the interval has elapsed, in seconds. +INSTRUCTION_FETCH_TICK_SECONDS=20 +# Largest instruction PDF to accept, in megabytes. +FETCH_MAX_FILE_MB=150 +FETCH_TIMEOUT_SECONDS=60 diff --git a/README.md b/README.md index 3dd992c..1a65f06 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ A Python web application for organizing and managing your LEGO instruction manua - 🖼️ **Image Gallery**: View instruction images in an organized gallery - 📄 **PDF Support**: View and download PDF instructions directly - 💾 **Backup & Restore**: One-click archive of the database plus every uploaded file, ready to restore anywhere +- ⬇️ **Automatic Instruction Fetcher**: Downloads official instruction PDFs from LEGO (found via Brickset) for official sets, one every 10 minutes so nobody's API gets hammered — see [docs/INSTRUCTION_FETCHER.md](docs/INSTRUCTION_FETCHER.md) ## 🚀 Quick Start @@ -185,6 +186,11 @@ flask --app run.py shell ### Instruction - id, set_id, file_type, file_path, file_name, file_size, thumbnail_path - page_number, user_id, uploaded_at +- sha256, source (set by the automatic fetcher; null for manual uploads) + +### InstructionFetchJob +- id, set_id, status, attempts, files_added, source, message +- queued_at, not_before, started_at, finished_at, queued_by_id ### ExtraFile - id, set_id, file_name, original_filename, file_path, file_type, file_size diff --git a/app/__init__.py b/app/__init__.py index 1c85277..1e8098f 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -47,6 +47,7 @@ def create_app(config_name='default'): from app.routes.extra_files import extra_files_bp from app.routes.moc import bp as moc_bp from app.routes.backup import backup_bp + from app.routes.instruction_fetch import instruction_fetch_bp app.register_blueprint(auth_bp) app.register_blueprint(main_bp) @@ -56,6 +57,7 @@ def create_app(config_name='default'): app.register_blueprint(extra_files_bp) app.register_blueprint(moc_bp) app.register_blueprint(backup_bp) + app.register_blueprint(instruction_fetch_bp) # Import models to ensure they're registered with SQLAlchemy from app.models.user import User @@ -64,6 +66,7 @@ def create_app(config_name='default'): from app.models.extra_file import ExtraFile from app.models.set_share import SetShare from app.models.site_setting import SiteSetting + from app.models.instruction_fetch import InstructionFetchJob # Add any columns missing from an older database (additive only). if app.config.get('AUTO_SCHEMA_UPGRADE', True): @@ -96,9 +99,14 @@ def create_app(config_name='default'): def uploaded_file(filename): return send_from_directory(app.config['UPLOAD_FOLDER'], filename) - # Register CLI commands (flask backup create/list/verify/restore) + # Register CLI commands (flask backup ... / flask instructions ...) from app.cli import register_cli register_cli(app) + + # Background ticker for the automatic instruction fetcher. It does + # nothing at all until an admin enables it on the fetcher page. + from app.services import fetch_worker + fetch_worker.start(app) # Context processor for global template variables @app.context_processor diff --git a/app/cli.py b/app/cli.py index e2310ab..760d237 100644 --- a/app/cli.py +++ b/app/cli.py @@ -8,6 +8,10 @@ Flask CLI commands for LIM backup and restore. flask backup restore ARCHIVE [--mode fresh|merge] [--force] [--overwrite-files] [--skip-files] + flask instructions queue [--missing/--all] [--limit N] + flask instructions run [--count N] [--force] + flask instructions status + `restore` is the path to use on a fresh Docker instance: no login required, and it works before any admin user exists. @@ -181,5 +185,88 @@ def restore_command(archive, mode, force, overwrite_files, skip_files, yes): click.secho('Restore complete.', fg='green') +instructions_cli = AppGroup( + 'instructions', help='Queue and run automatic instruction downloads.') + + +@instructions_cli.command('queue') +@click.option('--missing/--all', 'missing_only', default=True, + help='Only official sets with no PDF yet (default), or every ' + 'official set.') +@click.option('--limit', default=None, type=int, + help='Queue at most this many sets.') +def instructions_queue(missing_only, limit): + """Add official sets to the fetch queue.""" + from app.services import instruction_fetcher as fetcher + + if missing_only: + added = fetcher.queue_missing(limit=limit) + else: + added = fetcher.queue_all_official(limit=limit) + + minutes = added * fetcher.interval_minutes() + click.secho(f'Queued {added} set(s).', fg='green') + if added: + click.echo(f' At one every {fetcher.interval_minutes()} minutes that ' + f'is roughly {round(minutes / 60, 1)} hours of work.') + + +@instructions_cli.command('run') +@click.option('--count', default=1, type=int, + help='How many sets to process before exiting.') +@click.option('--force', is_flag=True, + help='Ignore the interval timer and the enabled switch.') +def instructions_run(count, force): + """ + Process queued sets, respecting the configured interval. + + Handy from Task Scheduler: `flask instructions run --count 1 --force` + every 10 minutes gives the same pacing without the worker thread. + """ + from app.services import instruction_fetcher as fetcher + + done = 0 + for _ in range(max(1, count)): + result = fetcher.run_one(force=force) + if result is None: + if not fetcher.is_enabled() and not force: + click.secho('Fetcher is switched off (use --force to ' + 'override).', fg='yellow') + elif not fetcher.is_due() and not force: + wait = fetcher.seconds_until_due() + click.secho(f'Not due for another {wait // 60}m ' + f'{wait % 60}s.', fg='yellow') + else: + click.echo('Queue is empty.') + break + + done += 1 + click.echo(f" {result['set_number']}: {result['status']} " + f"({result['files_added']} file(s)) - {result['message']}") + + click.secho(f'Processed {done} set(s).', fg='green') + + +@instructions_cli.command('status') +def instructions_status(): + """Show queue counts and when the next fetch is allowed.""" + from app.services import instruction_fetcher as fetcher + + summary = fetcher.status_summary() + counts = summary['counts'] + wait = summary['next_run_in_seconds'] + + click.echo(' Enabled : ' + ('yes' if summary['enabled'] else 'no')) + click.echo(f" Interval : {summary['interval_minutes']} minutes") + click.echo(f" Next run : {'now' if wait == 0 else f'in {wait // 60}m {wait % 60}s'}") + click.echo(f" Brickset : {'configured' if summary['brickset_configured'] else 'NOT configured'}") + click.echo(' Queue : ' + ', '.join( + f'{counts[key]} {key}' for key in + ('queued', 'running', 'done', 'empty', 'failed', 'cancelled'))) + click.echo(f" Library : {summary['official_sets']} official set(s), " + f"{summary['sets_missing_pdfs']} without a PDF") + + def register_cli(app): app.cli.add_command(backup_cli) + app.cli.add_command(instructions_cli) diff --git a/app/config.py b/app/config.py index a3f3af7..d2668ef 100644 --- a/app/config.py +++ b/app/config.py @@ -59,6 +59,23 @@ class Config: # stream straight to disk and never go through Flask's form parser. BACKUP_MAX_UPLOAD = int(os.environ.get('BACKUP_MAX_UPLOAD', 0)) + # Automatic instruction fetcher + # Pacing (interval, delays, on/off) lives in site_settings so it can be + # changed from the admin page. These are the hard limits and the switch + # for the in-process worker thread. + # + # Turn the worker off (0) if you drive the queue from Task Scheduler or + # cron with `flask instructions run` instead, or if you ever run more + # than one LIM process against the same database. + INSTRUCTION_FETCH_WORKER = os.environ.get('INSTRUCTION_FETCH_WORKER', '1') != '0' + # How often the worker wakes to check whether the interval has elapsed. + INSTRUCTION_FETCH_TICK_SECONDS = int( + os.environ.get('INSTRUCTION_FETCH_TICK_SECONDS', 20)) + # Refuse anything larger than this, in megabytes. Some Technic manuals + # are genuinely enormous, hence the generous default. + FETCH_MAX_FILE_MB = int(os.environ.get('FETCH_MAX_FILE_MB', 150)) + FETCH_TIMEOUT_SECONDS = int(os.environ.get('FETCH_TIMEOUT_SECONDS', 60)) + # Add missing columns automatically at startup. Turn off if you'd # rather manage schema changes with Flask-Migrate yourself. AUTO_SCHEMA_UPGRADE = os.environ.get('AUTO_SCHEMA_UPGRADE', '1') != '0' diff --git a/app/models/__init__.py b/app/models/__init__.py index eba1c70..5c8676f 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -4,5 +4,7 @@ from app.models.instruction import Instruction from app.models.extra_file import ExtraFile from app.models.set_share import SetShare from app.models.site_setting import SiteSetting +from app.models.instruction_fetch import InstructionFetchJob -__all__ = ['User', 'Set', 'Instruction', 'ExtraFile', 'SetShare', 'SiteSetting'] +__all__ = ['User', 'Set', 'Instruction', 'ExtraFile', 'SetShare', + 'SiteSetting', 'InstructionFetchJob'] diff --git a/app/models/instruction.py b/app/models/instruction.py index 7cb9b9d..d9f3443 100644 --- a/app/models/instruction.py +++ b/app/models/instruction.py @@ -16,6 +16,13 @@ class Instruction(db.Model): file_name = db.Column(db.String(200), nullable=False) file_size = db.Column(db.Integer) # Size in bytes thumbnail_path = db.Column(db.String(500)) # Thumbnail preview (especially for PDFs) + + # SHA-256 of the stored file. Set by the automatic fetcher so the same + # PDF is never downloaded twice; null for older manual uploads. + sha256 = db.Column(db.String(64), nullable=True, index=True) + # Where the file came from: None/'upload' for manual, 'brickset' for the + # automatic fetcher. + source = db.Column(db.String(20), nullable=True) # For image sequences page_number = db.Column(db.Integer, default=1) @@ -40,6 +47,7 @@ class Instruction(db.Model): 'file_name': self.file_name, 'file_size': self.file_size, 'page_number': self.page_number, + 'source': self.source, 'file_url': f'/static/uploads/{clean_path}', 'thumbnail_url': f'/static/uploads/{thumbnail_clean}' if thumbnail_clean else None, 'uploaded_at': self.uploaded_at.isoformat() diff --git a/app/models/instruction_fetch.py b/app/models/instruction_fetch.py new file mode 100644 index 0000000..3ad887f --- /dev/null +++ b/app/models/instruction_fetch.py @@ -0,0 +1,121 @@ +""" +Queue of "go and fetch the instructions" jobs for official sets. + +One row per set per attempt run. The queue lives in the database rather than +in memory so a restart doesn't lose the backlog, and so the admin page can +show what happened to every set without scraping the log file. + +Only official sets ever get queued — MOCs have no Brickset entry, so there is +nothing to fetch. + +Created by: Jess Rogerson (yelling commands at Claude.AI) +""" + +from datetime import datetime + +from app import db + +# queued - waiting its turn +# running - being worked on right now +# done - at least one instruction file was saved +# empty - Brickset has no instructions on file for this set +# failed - gave up after max attempts +# cancelled - removed from the queue by an admin +STATUSES = ('queued', 'running', 'done', 'empty', 'failed', 'cancelled') + +OPEN_STATUSES = ('queued', 'running') + +STATUS_BADGES = { + 'queued': 'secondary', + 'running': 'primary', + 'done': 'success', + 'empty': 'warning', + 'failed': 'danger', + 'cancelled': 'dark', +} + + +class InstructionFetchJob(db.Model): + """A single set queued for an automatic instruction download.""" + + __tablename__ = 'instruction_fetch_jobs' + + id = db.Column(db.Integer, primary_key=True) + set_id = db.Column(db.Integer, db.ForeignKey('sets.id'), nullable=False, + index=True) + + status = db.Column(db.String(12), nullable=False, default='queued', + server_default='queued', index=True) + attempts = db.Column(db.Integer, nullable=False, default=0, + server_default='0') + files_added = db.Column(db.Integer, nullable=False, default=0, + server_default='0') + # 'brickset' for now; the column exists so a future LEGO-direct or + # Rebrickable path can be told apart in the history. + source = db.Column(db.String(20), nullable=True) + message = db.Column(db.Text, nullable=True) + + queued_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow, + index=True) + # Retry backoff: a job is invisible to the worker until this passes. + not_before = db.Column(db.DateTime, nullable=True) + started_at = db.Column(db.DateTime, nullable=True) + finished_at = db.Column(db.DateTime, nullable=True) + + queued_by_id = db.Column(db.Integer, db.ForeignKey('users.id'), + nullable=True) + + lego_set = db.relationship( + 'Set', + backref=db.backref('fetch_jobs', lazy='dynamic', + cascade='all, delete-orphan')) + queued_by = db.relationship('User', foreign_keys=[queued_by_id]) + + def __repr__(self): + return f'' + + # -- helpers ------------------------------------------------------------ + + @property + def badge(self): + return STATUS_BADGES.get(self.status, 'secondary') + + @property + def is_open(self): + return self.status in OPEN_STATUSES + + @classmethod + def open_for_set(cls, set_id): + """The queued/running job for a set, if there is one.""" + return cls.query.filter(cls.set_id == set_id, + cls.status.in_(OPEN_STATUSES)).first() + + @classmethod + def counts(cls): + """Row counts per status, with every status present.""" + from sqlalchemy import func + rows = db.session.query(cls.status, func.count(cls.id)).group_by( + cls.status).all() + counts = {status: 0 for status in STATUSES} + for status, count in rows: + counts[status] = count + counts['total'] = sum(counts[s] for s in STATUSES) + return counts + + def to_dict(self): + lego_set = self.lego_set + return { + 'id': self.id, + 'set_id': self.set_id, + 'set_number': lego_set.set_number if lego_set else None, + 'set_name': lego_set.set_name if lego_set else None, + 'status': self.status, + 'badge': self.badge, + 'attempts': self.attempts, + 'files_added': self.files_added, + 'source': self.source, + 'message': self.message, + 'queued_at': self.queued_at.isoformat() if self.queued_at else None, + 'finished_at': (self.finished_at.isoformat() + if self.finished_at else None), + } diff --git a/app/models/site_setting.py b/app/models/site_setting.py index d4cb5ba..e446ac9 100644 --- a/app/models/site_setting.py +++ b/app/models/site_setting.py @@ -22,6 +22,16 @@ DEFAULTS = { # Send logged-in users straight to their dashboard instead of the # marketing homepage. 'home_redirect_dashboard': 'true', + + # Automatic instruction fetcher (see services/instruction_fetcher.py). + # Off by default: an admin has to switch it on deliberately. + 'fetch_enabled': 'false', + 'fetch_interval_minutes': '10', + 'fetch_file_delay_seconds': '5', + 'fetch_max_attempts': '3', + 'fetch_skip_sets_with_pdfs': 'true', + 'fetch_allow_setid_lookup': 'true', + 'fetch_last_run': '', } VISIBILITIES = ('private', 'shared', 'public') diff --git a/app/routes/__init__.py b/app/routes/__init__.py index cda479c..b4e1738 100644 --- a/app/routes/__init__.py +++ b/app/routes/__init__.py @@ -3,5 +3,7 @@ from app.routes.main import main_bp from app.routes.sets import sets_bp from app.routes.instructions import instructions_bp from app.routes.moc import bp as moc_bp +from app.routes.instruction_fetch import instruction_fetch_bp -__all__ = ['auth_bp', 'main_bp', 'sets_bp', 'instructions_bp', 'moc_bp'] +__all__ = ['auth_bp', 'main_bp', 'sets_bp', 'instructions_bp', 'moc_bp', + 'instruction_fetch_bp'] diff --git a/app/routes/instruction_fetch.py b/app/routes/instruction_fetch.py new file mode 100644 index 0000000..618289e --- /dev/null +++ b/app/routes/instruction_fetch.py @@ -0,0 +1,197 @@ +""" +Admin pages for the automatic instruction fetcher. + +Everything here is admin-only and reuses admin.admin_required so there is one +definition of "is this person allowed in". + +Created by: Jess Rogerson (yelling commands at Claude.AI) +""" + +from flask import (Blueprint, current_app, flash, jsonify, redirect, + render_template, request, url_for) +from flask_login import current_user, login_required + +from app import db +from app.models.instruction_fetch import InstructionFetchJob +from app.models.set import Set +from app.models.site_setting import SiteSetting +from app.routes.admin import admin_required +from app.services import fetch_worker +from app.services import instruction_fetcher as fetcher + +instruction_fetch_bp = Blueprint('instruction_fetch', __name__, + url_prefix='/admin/instruction-fetch') + + +def _int_form(name, default, low, high): + try: + value = int(request.form.get(name, default)) + except (TypeError, ValueError): + value = default + return max(low, min(high, value)) + + +@instruction_fetch_bp.route('/') +@login_required +@admin_required +def index(): + """Control panel for the fetcher.""" + summary = fetcher.status_summary() + jobs = InstructionFetchJob.query.order_by( + InstructionFetchJob.id.desc()).limit(50).all() + return render_template('admin/instruction_fetch.html', + summary=summary, + jobs=jobs, + worker_running=fetch_worker.is_running()) + + +@instruction_fetch_bp.route('/status') +@login_required +@admin_required +def status(): + """JSON for the page's auto-refresh.""" + summary = fetcher.status_summary() + summary['worker_running'] = fetch_worker.is_running() + return jsonify(summary) + + +@instruction_fetch_bp.route('/settings', methods=['POST']) +@login_required +@admin_required +def save_settings(): + SiteSetting.set('fetch_enabled', + bool(request.form.get('fetch_enabled'))) + SiteSetting.set('fetch_interval_minutes', + str(_int_form('fetch_interval_minutes', 10, + fetcher.MIN_INTERVAL_MINUTES, + fetcher.MAX_INTERVAL_MINUTES))) + SiteSetting.set('fetch_file_delay_seconds', + str(_int_form('fetch_file_delay_seconds', 5, 0, 300))) + SiteSetting.set('fetch_max_attempts', + str(_int_form('fetch_max_attempts', 3, 1, 10))) + SiteSetting.set('fetch_skip_sets_with_pdfs', + bool(request.form.get('fetch_skip_sets_with_pdfs'))) + SiteSetting.set('fetch_allow_setid_lookup', + bool(request.form.get('fetch_allow_setid_lookup'))) + db.session.commit() + + flash('Fetcher settings saved.', 'success') + return redirect(url_for('instruction_fetch.index')) + + +@instruction_fetch_bp.route('/queue', methods=['POST']) +@login_required +@admin_required +def queue(): + """Add official sets to the queue.""" + scope = request.form.get('scope', 'missing') + limit = request.form.get('limit', '').strip() + limit = int(limit) if limit.isdigit() and int(limit) > 0 else None + + if scope == 'all': + added = fetcher.queue_all_official(current_user.id, limit) + label = 'official set(s)' + elif scope == 'retry-failed': + added = fetcher.requeue_failed(current_user.id) + label = 'failed job(s) requeued' + else: + added = fetcher.queue_missing(current_user.id, limit) + label = 'set(s) with no PDF yet' + + if added: + minutes = added * fetcher.interval_minutes() + hours = round(minutes / 60, 1) + flash(f'Queued {added} {label}. At one every ' + f'{fetcher.interval_minutes()} minutes that is about ' + f'{hours} hours of background work.', 'success') + if not fetcher.is_enabled(): + flash('The fetcher is currently paused — switch it on below to ' + 'start working through the queue.', 'warning') + else: + flash('Nothing new to queue.', 'info') + + return redirect(url_for('instruction_fetch.index')) + + +@instruction_fetch_bp.route('/queue-set/', methods=['POST']) +@login_required +@admin_required +def queue_set(set_id): + """Queue one specific set (used from the set page).""" + lego_set = Set.query.get_or_404(set_id) + if lego_set.is_moc: + flash('MOCs have no Brickset entry to fetch from.', 'warning') + elif fetcher.queue_sets([lego_set], current_user.id): + flash(f'{lego_set.set_number} queued for an instruction fetch.', + 'success') + else: + flash(f'{lego_set.set_number} is already in the queue.', 'info') + + return redirect(request.referrer or url_for('instruction_fetch.index')) + + +@instruction_fetch_bp.route('/run-now', methods=['POST']) +@login_required +@admin_required +def run_now(): + """Process one queued set immediately, in the background.""" + force = bool(request.form.get('force')) + + if not fetcher.is_due() and not force: + wait = fetcher.seconds_until_due() + flash(f'Still cooling down — {wait // 60}m {wait % 60}s until the ' + f'next allowed fetch. Tick "ignore the timer" to override.', + 'warning') + return redirect(url_for('instruction_fetch.index')) + + counts = InstructionFetchJob.counts() + if not counts['queued']: + flash('The queue is empty.', 'info') + return redirect(url_for('instruction_fetch.index')) + + fetch_worker.run_once_async(current_app._get_current_object(), force=force) + flash('Fetching one set now — the table below will update shortly.', + 'info') + return redirect(url_for('instruction_fetch.index')) + + +@instruction_fetch_bp.route('/jobs//cancel', methods=['POST']) +@login_required +@admin_required +def cancel_job(job_id): + job = InstructionFetchJob.query.get_or_404(job_id) + if job.is_open: + job.status = 'cancelled' + job.message = 'cancelled by admin' + db.session.commit() + flash('Job cancelled.', 'success') + else: + flash('That job has already finished.', 'info') + return redirect(url_for('instruction_fetch.index')) + + +@instruction_fetch_bp.route('/jobs//retry', methods=['POST']) +@login_required +@admin_required +def retry_job(job_id): + job = InstructionFetchJob.query.get_or_404(job_id) + if InstructionFetchJob.open_for_set(job.set_id): + flash('That set is already in the queue.', 'info') + else: + job.status = 'queued' + job.attempts = 0 + job.message = None + job.not_before = None + db.session.commit() + flash('Job requeued.', 'success') + return redirect(url_for('instruction_fetch.index')) + + +@instruction_fetch_bp.route('/clear', methods=['POST']) +@login_required +@admin_required +def clear(): + scope = request.form.get('scope', 'finished') + removed = fetcher.clear_jobs(scope) + flash(f'Removed {removed} job record(s).', 'success') + return redirect(url_for('instruction_fetch.index')) diff --git a/app/services/brickset_api.py b/app/services/brickset_api.py index c6fdcc9..35baccd 100644 --- a/app/services/brickset_api.py +++ b/app/services/brickset_api.py @@ -1,19 +1,39 @@ +""" +Brickset API v3 client. + +https://brickset.com/article/52664/api-version-3-documentation + +Two things worth knowing about the API, both of which shape this file: + + * getSets takes its search terms as a JSON string. Python's str() on a dict + produces single quotes, which is not JSON — hence json.dumps below. + * getInstructions takes a Brickset setID; getInstructions2 takes a set + number like "10497-1". They are different methods and passing a set + number to the first one quietly returns nothing. + +Only getSets counts against the daily key quota. + +Created by: Jess Rogerson (yelling commands at Claude.AI) +""" + +import json +from typing import Any, Dict, List, Optional + import requests from flask import current_app -from typing import Optional, List, Dict, Any class BricksetAPI: """Service for interacting with the Brickset API v3.""" - + BASE_URL = 'https://brickset.com/api/v3.asmx' - + def __init__(self): self.api_key = current_app.config.get('BRICKSET_API_KEY') self.username = current_app.config.get('BRICKSET_USERNAME') self.password = current_app.config.get('BRICKSET_PASSWORD') self._user_hash = None - + @staticmethod def is_configured() -> bool: """Check if Brickset API is properly configured.""" @@ -22,16 +42,16 @@ class BricksetAPI: current_app.config.get('BRICKSET_USERNAME') and current_app.config.get('BRICKSET_PASSWORD') ) - + def _get_user_hash(self) -> Optional[str]: """Authenticate and get user hash token.""" if self._user_hash: return self._user_hash - + if not all([self.api_key, self.username, self.password]): current_app.logger.warning('Brickset API credentials not configured') return None - + try: response = requests.get( f'{self.BASE_URL}/login', @@ -44,47 +64,43 @@ class BricksetAPI: ) response.raise_for_status() data = response.json() - + if data.get('status') == 'success': self._user_hash = data.get('hash') return self._user_hash else: current_app.logger.error(f"Brickset login failed: {data.get('message')}") return None - + except Exception as e: current_app.logger.error(f'Brickset API authentication error: {str(e)}') return None - - def search_sets(self, - set_number: Optional[str] = None, - theme: Optional[str] = None, - year: Optional[int] = None, - query: Optional[str] = None) -> List[Dict[str, Any]]: + + def search_sets(self, + set_number: Optional[str] = None, + theme: Optional[str] = None, + year: Optional[int] = None, + query: Optional[str] = None, + page_size: Optional[int] = None) -> List[Dict[str, Any]]: """ Search for LEGO sets using various parameters. - + Args: set_number: Specific set number to search for theme: Theme name to filter by year: Year released query: General search query - + page_size: Number of records to return (max 500) + Returns: List of set dictionaries """ user_hash = self._get_user_hash() if not user_hash: return [] - - params = { - 'apiKey': self.api_key, - 'userHash': user_hash, - 'params': '{}' # JSON params object - } - + # Build search parameters - search_params = {} + search_params: Dict[str, Any] = {} if set_number: search_params['setNumber'] = set_number if theme: @@ -93,9 +109,20 @@ class BricksetAPI: search_params['year'] = year if query: search_params['query'] = query - - params['params'] = str(search_params) - + if page_size: + search_params['pageSize'] = page_size + + if not search_params: + return [] + + params = { + 'apiKey': self.api_key, + 'userHash': user_hash, + # Must be a JSON string — str(dict) gives single quotes and is + # rejected with "Parameter error". + 'params': json.dumps(search_params), + } + try: response = requests.get( f'{self.BASE_URL}/getSets', @@ -104,41 +131,49 @@ class BricksetAPI: ) response.raise_for_status() data = response.json() - + if data.get('status') == 'success': return data.get('sets', []) - else: - current_app.logger.error(f"Brickset search failed: {data.get('message')}") - return [] - + + message = data.get('message', 'unknown error') + current_app.logger.error(f'Brickset search failed: {message}') + if 'limit exceeded' in message.lower(): + raise RuntimeError('Brickset API limit exceeded') + return [] + + except RuntimeError: + raise except Exception as e: current_app.logger.error(f'Brickset API search error: {str(e)}') return [] - + def get_set_by_number(self, set_number: str) -> Optional[Dict[str, Any]]: """ Get detailed information for a specific set by its number. - + Args: - set_number: The LEGO set number (e.g., "10497") - + set_number: The LEGO set number (e.g., "10497" or "10497-1") + Returns: Dictionary with set information or None """ results = self.search_sets(set_number=set_number) + if not results and '-' not in set_number: + # Brickset stores numbers with a variant suffix. + results = self.search_sets(set_number=f'{set_number}-1') return results[0] if results else None - + def get_themes(self) -> List[str]: """ Get list of all available LEGO themes. - + Returns: List of theme names """ user_hash = self._get_user_hash() if not user_hash: return [] - + try: response = requests.get( f'{self.BASE_URL}/getThemes', @@ -150,49 +185,76 @@ class BricksetAPI: ) response.raise_for_status() data = response.json() - + if data.get('status') == 'success': themes = data.get('themes', []) return [theme.get('theme') for theme in themes if theme.get('theme')] else: return [] - + except Exception as e: current_app.logger.error(f'Brickset API themes error: {str(e)}') return [] - - def get_instructions(self, set_number: str) -> List[Dict[str, Any]]: - """ - Get instruction information for a specific set. - - Args: - set_number: The LEGO set number - - Returns: - List of instruction dictionaries - """ - user_hash = self._get_user_hash() - if not user_hash: - return [] - + + # -- instructions ------------------------------------------------------- + + def _instructions_request(self, method: str, + params: Dict[str, Any]) -> List[Dict[str, Any]]: + params = dict(params, apiKey=self.api_key) try: - response = requests.get( - f'{self.BASE_URL}/getInstructions', - params={ - 'apiKey': self.api_key, - 'userHash': user_hash, - 'setNumber': set_number - }, - timeout=10 - ) + response = requests.get(f'{self.BASE_URL}/{method}', + params=params, timeout=20) response.raise_for_status() data = response.json() - + if data.get('status') == 'success': - return data.get('instructions', []) - else: - return [] - + return data.get('instructions', []) or [] + + current_app.logger.error( + f"Brickset {method} failed: {data.get('message')}") + return [] + except Exception as e: - current_app.logger.error(f'Brickset API instructions error: {str(e)}') + current_app.logger.error(f'Brickset {method} error: {str(e)}') + return [] + + def get_instructions(self, set_number: str) -> List[Dict[str, Any]]: + """ + Instructions for a set number, e.g. "10497-1" (getInstructions2). + + Returns a list of {'URL': ..., 'description': ...} dictionaries. + Does not need a user hash and does not count against the key quota. + """ + if not self.api_key or not set_number: + return [] + return self._instructions_request('getInstructions2', + {'setNumber': set_number}) + + def get_instructions_by_id(self, set_id: int) -> List[Dict[str, Any]]: + """Instructions for an internal Brickset set ID (getInstructions).""" + if not self.api_key or not set_id: + return [] + return self._instructions_request('getInstructions', {'setID': set_id}) + + # -- quota -------------------------------------------------------------- + + def get_key_usage(self) -> List[Dict[str, Any]]: + """ + Daily getSets usage for the last 30 days. + + Returns a list of {'dateStamp': ..., 'count': ...} dictionaries. + """ + if not self.api_key: + return [] + try: + response = requests.get(f'{self.BASE_URL}/getKeyUsageStats', + params={'apiKey': self.api_key}, + timeout=10) + response.raise_for_status() + data = response.json() + if data.get('status') == 'success': + return data.get('apiKeyUsage', []) or [] + return [] + except Exception as e: + current_app.logger.error(f'Brickset key usage error: {str(e)}') return [] diff --git a/app/services/fetch_worker.py b/app/services/fetch_worker.py new file mode 100644 index 0000000..ff1e616 --- /dev/null +++ b/app/services/fetch_worker.py @@ -0,0 +1,109 @@ +""" +The background ticker behind the instruction fetcher. + +A daemon thread wakes every few seconds, asks the fetcher whether the +interval has elapsed, and if so processes exactly one set. All of the actual +pacing lives in instruction_fetcher (and in the database), so pausing the +worker or changing the interval from the admin page takes effect on the next +tick without a restart. + +Deliberately a plain thread rather than a scheduler dependency: LIM runs +single-process under waitress, and Python 3.14 has been quite enough +dependency excitement for one project. + +Run one process. If you ever scale LIM horizontally, turn the worker off with +INSTRUCTION_FETCH_WORKER=0 everywhere and drive it from cron/Task Scheduler +with `flask instructions run` instead. + +Created by: Jess Rogerson (yelling commands at Claude.AI) +""" + +import os +import sys +import threading + +_thread = None +_stop = threading.Event() +_lock = threading.Lock() + + +def _is_cli_command(): + """True when the app was loaded by `flask ` rather than to serve.""" + if os.environ.get('FLASK_RUN_FROM_CLI') != 'true': + return False + args = [arg for arg in sys.argv[1:] if not arg.startswith('-')] + return bool(args) and args[0] != 'run' + + +def start(app): + """Start the ticker unless it's disabled or already running.""" + if not app.config.get('INSTRUCTION_FETCH_WORKER', True): + app.logger.info('Instruction fetch worker disabled by config') + return False + + # `flask instructions run`, `flask backup restore` and friends shouldn't + # quietly spin up a fetcher of their own. + if _is_cli_command(): + return False + + # Flask's reloader imports the app twice in debug; only the child should + # own the thread, or two of them race for the same job. + if app.debug and os.environ.get('WERKZEUG_RUN_MAIN') != 'true': + return False + + global _thread + with _lock: + if _thread is not None and _thread.is_alive(): + return False + _stop.clear() + _thread = threading.Thread(target=_loop, args=(app,), + name='lim-instruction-fetcher', + daemon=True) + _thread.start() + + app.logger.info('Instruction fetch worker started') + return True + + +def stop(timeout=5): + """Ask the ticker to finish (used by tests and the CLI).""" + _stop.set() + thread = _thread + if thread is not None and thread.is_alive(): + thread.join(timeout=timeout) + + +def is_running(): + return _thread is not None and _thread.is_alive() + + +def _loop(app): + tick = app.config.get('INSTRUCTION_FETCH_TICK_SECONDS', 20) + from app.services import instruction_fetcher as fetcher + + while not _stop.wait(tick): + try: + with app.app_context(): + fetcher.run_one() + except Exception: + app.logger.exception('Instruction fetch worker tick failed') + + +def run_once_async(app, force=False): + """ + Fire off a single job in the background. + + Used by the admin page's "Run one now" button so the request returns + straight away instead of holding the browser open for a 40 MB PDF. + """ + def _work(): + try: + with app.app_context(): + from app.services import instruction_fetcher as fetcher + fetcher.run_one(force=force) + except Exception: + app.logger.exception('Manual instruction fetch failed') + + thread = threading.Thread(target=_work, name='lim-fetch-once', daemon=True) + thread.start() + return thread diff --git a/app/services/instruction_fetcher.py b/app/services/instruction_fetcher.py new file mode 100644 index 0000000..8f0729f --- /dev/null +++ b/app/services/instruction_fetcher.py @@ -0,0 +1,562 @@ +""" +Automatic instruction downloads for official sets. + +Brickset's getInstructions/getInstructions2 methods hand back the URLs of the +official PDFs, which live on LEGO's own servers. So "download from Brickset or +LEGO" is really one flow: ask Brickset where the file is, then pull the PDF +from LEGO. + +The whole point of this module is politeness. Work is queued in the database +and exactly one set is processed per interval (10 minutes by default), with a +short pause between the individual PDFs of a multi-book set. Nothing here ever +loops over the whole library in one go. + +Notes on quotas: + * Only getSets counts against a Brickset API key's daily limit, so the + set-ID lookup is the expensive call and is skipped whenever the set + already has a brickset_id stored. + * getInstructions2 needs nothing but the API key and a set number. + +Created by: Jess Rogerson (yelling commands at Claude.AI) +""" + +import hashlib +import os +import re +import tempfile +import time +from datetime import datetime, timedelta + +import requests +from flask import current_app + +from app import db +from app.models.instruction import Instruction +from app.models.instruction_fetch import InstructionFetchJob +from app.models.set import Set +from app.models.site_setting import SiteSetting +from app.services.brickset_api import BricksetAPI +from app.services.file_handler import FileHandler + +USER_AGENT = ('LegoInstructionsManager/1.0 ' + '(+https://gitea.hideawaygaming.com.au/jessikitty/' + 'lego-instructions-manager)') + +# Settings surfaced on the admin page. Values are strings in site_settings. +SETTING_DEFAULTS = { + 'fetch_enabled': 'false', + 'fetch_interval_minutes': '10', + 'fetch_file_delay_seconds': '5', + 'fetch_max_attempts': '3', + 'fetch_skip_sets_with_pdfs': 'true', + 'fetch_allow_setid_lookup': 'true', + 'fetch_last_run': '', +} + +MIN_INTERVAL_MINUTES = 1 +MAX_INTERVAL_MINUTES = 1440 + + +# --------------------------------------------------------------------------- +# settings +# --------------------------------------------------------------------------- + +def _int_setting(key, fallback, low=None, high=None): + raw = SiteSetting.get(key, SETTING_DEFAULTS.get(key)) + try: + value = int(str(raw).strip()) + except (TypeError, ValueError): + value = fallback + if low is not None: + value = max(low, value) + if high is not None: + value = min(high, value) + return value + + +def is_enabled(): + return SiteSetting.get_bool('fetch_enabled', False) + + +def interval_minutes(): + return _int_setting('fetch_interval_minutes', 10, + MIN_INTERVAL_MINUTES, MAX_INTERVAL_MINUTES) + + +def file_delay_seconds(): + return _int_setting('fetch_file_delay_seconds', 5, 0, 300) + + +def max_attempts(): + return _int_setting('fetch_max_attempts', 3, 1, 10) + + +def skip_sets_with_pdfs(): + return SiteSetting.get_bool('fetch_skip_sets_with_pdfs', True) + + +def allow_setid_lookup(): + return SiteSetting.get_bool('fetch_allow_setid_lookup', True) + + +def last_run(): + raw = SiteSetting.get('fetch_last_run', '') + if not raw: + return None + try: + return datetime.fromisoformat(raw) + except ValueError: + return None + + +def _mark_ran(when=None): + SiteSetting.set('fetch_last_run', (when or datetime.utcnow()).isoformat()) + db.session.commit() + + +def next_run_at(): + """When the worker is next allowed to touch the network.""" + previous = last_run() + if previous is None: + return datetime.utcnow() + return previous + timedelta(minutes=interval_minutes()) + + +def seconds_until_due(): + return max(0, int((next_run_at() - datetime.utcnow()).total_seconds())) + + +def is_due(): + return seconds_until_due() <= 0 + + +# --------------------------------------------------------------------------- +# queueing +# --------------------------------------------------------------------------- + +def official_sets_query(): + """Official sets only — MOCs have nothing to look up.""" + return Set.query.filter(Set.is_moc.is_(False)) + + +def sets_missing_pdfs(): + """Official sets with no PDF instruction on file.""" + have_pdfs = db.session.query(Instruction.set_id).filter( + Instruction.file_type == 'PDF').distinct() + return official_sets_query().filter(~Set.id.in_(have_pdfs)) + + +def queue_sets(sets, user_id=None): + """Queue an iterable of sets, skipping any already waiting. -> count.""" + queued = 0 + for lego_set in sets: + if lego_set.is_moc: + continue + if InstructionFetchJob.open_for_set(lego_set.id): + continue + db.session.add(InstructionFetchJob(set_id=lego_set.id, + queued_by_id=user_id)) + queued += 1 + if queued: + db.session.commit() + return queued + + +def queue_missing(user_id=None, limit=None): + query = sets_missing_pdfs().order_by(Set.year_released.desc(), + Set.set_number) + if limit: + query = query.limit(limit) + return queue_sets(query.all(), user_id) + + +def queue_all_official(user_id=None, limit=None): + query = official_sets_query().order_by(Set.year_released.desc(), + Set.set_number) + if limit: + query = query.limit(limit) + return queue_sets(query.all(), user_id) + + +def requeue_failed(user_id=None): + """Put every failed job back at the end of the queue.""" + failed = InstructionFetchJob.query.filter_by(status='failed').all() + count = 0 + for job in failed: + if InstructionFetchJob.open_for_set(job.set_id): + continue + job.status = 'queued' + job.attempts = 0 + job.message = None + job.not_before = None + job.queued_at = datetime.utcnow() + job.queued_by_id = user_id or job.queued_by_id + count += 1 + if count: + db.session.commit() + return count + + +def clear_jobs(scope='finished'): + """Tidy the history. scope: finished | failed | all.""" + query = InstructionFetchJob.query + if scope == 'finished': + query = query.filter(InstructionFetchJob.status.in_( + ('done', 'empty', 'cancelled'))) + elif scope == 'failed': + query = query.filter_by(status='failed') + elif scope != 'all': + return 0 + removed = query.delete(synchronize_session=False) + db.session.commit() + return removed + + +# --------------------------------------------------------------------------- +# the actual fetching +# --------------------------------------------------------------------------- + +def brickset_number(set_number): + """Brickset wants {number}-{variant}; the library usually stores {number}.""" + number = (set_number or '').strip() + if not number: + return None + if re.search(r'-\d+$', number): + return number + return f'{number}-1' + + +def _instruction_urls(api, lego_set): + """(list of {url, description}, source, note).""" + note = None + + if lego_set.brickset_id: + entries = api.get_instructions_by_id(lego_set.brickset_id) + if entries: + return _normalise(entries), 'brickset', note + + number = brickset_number(lego_set.set_number) + if number: + entries = api.get_instructions(number) + if entries: + return _normalise(entries), 'brickset', note + + # Last resort: resolve the Brickset set ID, then ask again. This is the + # only call that counts against the daily API key quota, so it is both + # optional and cached back onto the set for next time. + if allow_setid_lookup() and not lego_set.brickset_id and number: + data = api.get_set_by_number(number) + if data and data.get('setID'): + lego_set.brickset_id = data['setID'] + db.session.commit() + entries = api.get_instructions_by_id(lego_set.brickset_id) + if entries: + return _normalise(entries), 'brickset', 'resolved via getSets' + note = 'set found, no instructions listed' + else: + note = 'set number not found on Brickset' + + return [], 'brickset', note + + +def _normalise(entries): + out = [] + for entry in entries or []: + url = entry.get('URL') or entry.get('url') + if not url: + continue + out.append({ + 'url': url, + 'description': (entry.get('description') + or entry.get('Description') or ''), + }) + return out + + +def _safe_name(lego_set, description, index): + """A readable file name, e.g. '10497-building-instructions-book-1.pdf'.""" + number = re.sub(r'[^A-Za-z0-9\-]+', '', lego_set.set_number or 'set') + base = re.sub(r'[^A-Za-z0-9]+', '-', description or '').strip('-').lower() + base = re.sub(r'-{2,}', '-', base)[:80].strip('-') + if not base: + return f'{number}-instructions-{index}.pdf' + # Brickset descriptions usually lead with the set number already. + if base.startswith(number.lower()): + return f'{base}.pdf' + return f'{number}-{base}.pdf' + + +class _LocalFile: + """Adapter so FileHandler.save_file can take a file we downloaded.""" + + def __init__(self, path, filename): + self._path = path + self.filename = filename + + def save(self, destination): + import shutil + shutil.move(self._path, destination) + + +def _download_pdf(url, max_bytes, timeout): + """Stream to a temp file. -> (path, size, sha256). Raises on refusal.""" + headers = {'User-Agent': USER_AGENT, 'Accept': 'application/pdf,*/*'} + digest = hashlib.sha256() + size = 0 + + handle, temp_path = tempfile.mkstemp(suffix='.pdf') + os.close(handle) + + try: + with requests.get(url, headers=headers, stream=True, + timeout=timeout) as response: + response.raise_for_status() + + content_type = (response.headers.get('Content-Type') or '').lower() + declared = response.headers.get('Content-Length') + if declared and declared.isdigit() and int(declared) > max_bytes: + raise ValueError( + f'file is {int(declared) // (1024 * 1024)} MB, over the limit') + + first = True + with open(temp_path, 'wb') as out: + for chunk in response.iter_content(chunk_size=64 * 1024): + if not chunk: + continue + if first: + if not chunk.startswith(b'%PDF'): + raise ValueError( + f'not a PDF (content-type {content_type or "unknown"})') + first = False + size += len(chunk) + if size > max_bytes: + raise ValueError('file exceeded the size limit mid-download') + digest.update(chunk) + out.write(chunk) + + if size == 0: + raise ValueError('empty response') + return temp_path, size, digest.hexdigest() + + except Exception: + if os.path.exists(temp_path): + os.remove(temp_path) + raise + + +def _existing_hashes(set_id): + rows = db.session.query(Instruction.sha256).filter( + Instruction.set_id == set_id, Instruction.sha256.isnot(None)).all() + return {row[0] for row in rows if row[0]} + + +def _next_page_number(set_id): + from sqlalchemy import func + highest = db.session.query(func.max(Instruction.page_number)).filter_by( + set_id=set_id).scalar() + return (highest or 0) + 1 + + +def fetch_for_set(lego_set, delay_between_files=None): + """ + Download every instruction Brickset lists for one official set. + + Returns (added, skipped, source, message). Never raises for the normal + failure modes — the message explains what happened. + """ + if lego_set.is_moc: + return 0, 0, None, 'MOC — nothing to look up' + + if not BricksetAPI.is_configured(): + raise RuntimeError('Brickset API credentials are not configured') + + max_bytes = current_app.config.get('FETCH_MAX_FILE_MB', 150) * 1024 * 1024 + timeout = current_app.config.get('FETCH_TIMEOUT_SECONDS', 60) + pause = (file_delay_seconds() if delay_between_files is None + else delay_between_files) + + api = BricksetAPI() + entries, source, note = _instruction_urls(api, lego_set) + + if not entries: + return 0, 0, source, note or 'no instructions listed on Brickset' + + known = _existing_hashes(lego_set.id) + page = _next_page_number(lego_set.id) + added = 0 + skipped = 0 + problems = [] + + for index, entry in enumerate(entries, start=1): + if index > 1 and pause: + time.sleep(pause) + + try: + temp_path, size, sha = _download_pdf(entry['url'], max_bytes, + timeout) + except Exception as exc: + problems.append(f'{entry["description"] or entry["url"]}: {exc}') + continue + + if sha in known: + os.remove(temp_path) + skipped += 1 + continue + + try: + filename = _safe_name(lego_set, entry['description'], index) + stored = _LocalFile(temp_path, filename) + rel_path, file_size, thumb = FileHandler.save_file( + stored, lego_set.set_number, 'PDF') + + instruction = Instruction( + set_id=lego_set.id, + file_type='PDF', + file_path=rel_path, + file_name=filename, + file_size=file_size or size, + thumbnail_path=thumb, + page_number=page, + sha256=sha, + source=source, + user_id=lego_set.user_id, + ) + db.session.add(instruction) + db.session.commit() + + known.add(sha) + page += 1 + added += 1 + + except Exception as exc: + db.session.rollback() + if os.path.exists(temp_path): + os.remove(temp_path) + problems.append(f'{filename}: saving failed — {exc}') + + parts = [] + if added: + parts.append(f'{added} file(s) downloaded') + if skipped: + parts.append(f'{skipped} already on file') + if problems: + parts.append('; '.join(problems[:3])) + if note: + parts.append(note) + + return added, skipped, source, ' | '.join(parts) or 'nothing to do' + + +# --------------------------------------------------------------------------- +# the queue runner +# --------------------------------------------------------------------------- + +def _claim_next_job(): + """Take the oldest eligible job and mark it running.""" + now = datetime.utcnow() + job = InstructionFetchJob.query.filter( + InstructionFetchJob.status == 'queued', + db.or_(InstructionFetchJob.not_before.is_(None), + InstructionFetchJob.not_before <= now), + ).order_by(InstructionFetchJob.queued_at).first() + + if job is None: + return None + + job.status = 'running' + job.started_at = now + job.attempts = (job.attempts or 0) + 1 + db.session.commit() + return job + + +def run_one(force=False): + """ + Process a single queued set, if the interval has elapsed. + + Returns a dict describing what happened, or None when there was nothing + to do (disabled, not due yet, or an empty queue). + """ + if not force and not is_enabled(): + return None + if not force and not is_due(): + return None + + job = _claim_next_job() + if job is None: + return None + + # Claim the slot up front: even a failure counts as network contact, so + # the next attempt still waits a full interval. + _mark_ran() + + lego_set = job.lego_set + if lego_set is None: + job.status = 'cancelled' + job.message = 'set no longer exists' + job.finished_at = datetime.utcnow() + db.session.commit() + return job.to_dict() + + if skip_sets_with_pdfs() and lego_set.instructions.filter_by( + file_type='PDF').count(): + job.status = 'done' + job.message = 'skipped — set already has PDF instructions' + job.finished_at = datetime.utcnow() + db.session.commit() + return job.to_dict() + + try: + added, skipped, source, message = fetch_for_set(lego_set) + job.files_added = added + job.source = source + job.message = message + if added: + job.status = 'done' + elif skipped: + job.status = 'done' + else: + job.status = 'empty' + + except Exception as exc: + current_app.logger.exception('Instruction fetch failed for %s', + lego_set.set_number) + job.message = str(exc)[:500] + if job.attempts >= max_attempts(): + job.status = 'failed' + else: + # Back off a couple of intervals before trying again. + job.status = 'queued' + job.not_before = datetime.utcnow() + timedelta( + minutes=interval_minutes() * 2) + + job.finished_at = datetime.utcnow() + db.session.commit() + + current_app.logger.info('Instruction fetch %s for %s: %s', job.status, + lego_set.set_number, job.message) + return job.to_dict() + + +def status_summary(): + """Everything the admin page and its poller need, in one dict.""" + counts = InstructionFetchJob.counts() + configured = BricksetAPI.is_configured() + recent = InstructionFetchJob.query.order_by( + InstructionFetchJob.id.desc()).limit(25).all() + + return { + 'enabled': is_enabled(), + 'brickset_configured': configured, + 'interval_minutes': interval_minutes(), + 'file_delay_seconds': file_delay_seconds(), + 'max_attempts': max_attempts(), + 'skip_sets_with_pdfs': skip_sets_with_pdfs(), + 'allow_setid_lookup': allow_setid_lookup(), + 'last_run': last_run().isoformat() if last_run() else None, + 'next_run_in_seconds': seconds_until_due(), + 'counts': counts, + 'official_sets': official_sets_query().count(), + 'sets_missing_pdfs': sets_missing_pdfs().count(), + 'jobs': [job.to_dict() for job in recent], + } diff --git a/app/services/schema_upgrade.py b/app/services/schema_upgrade.py index 2bba207..3eb2570 100644 --- a/app/services/schema_upgrade.py +++ b/app/services/schema_upgrade.py @@ -24,8 +24,18 @@ ADDITIVE_COLUMNS = { # site default instead. 'visibility': "VARCHAR(10) NOT NULL DEFAULT 'public'", }, + 'instructions': { + # Written by the automatic fetcher; null on everything uploaded + # before it existed, which is fine - dedupe just can't help there. + 'sha256': 'VARCHAR(64)', + 'source': 'VARCHAR(20)', + }, } +# Tables added by later versions. create_all() is not called at boot, so new +# tables need an explicit (and idempotent) nudge. +NEW_TABLES = ('app.models.instruction_fetch.InstructionFetchJob',) + def ensure_schema(app): """Add any missing columns. Returns a list of changes made.""" @@ -38,6 +48,20 @@ def ensure_schema(app): app.logger.warning('Schema check skipped: %s', exc) return applied + for dotted in NEW_TABLES: + module_path, _, class_name = dotted.rpartition('.') + try: + module = __import__(module_path, fromlist=[class_name]) + model = getattr(module, class_name) + if model.__tablename__ not in existing_tables: + model.__table__.create(db.engine, checkfirst=True) + applied.append(f'table {model.__tablename__}') + app.logger.info('Schema upgrade: created table %s', + model.__tablename__) + except Exception as exc: + app.logger.error('Schema upgrade failed for %s: %s', + dotted, exc) + for table, columns in ADDITIVE_COLUMNS.items(): if table not in existing_tables: continue # create_all() will build it with everything present diff --git a/app/templates/admin/dashboard.html b/app/templates/admin/dashboard.html index 39efa9b..09b8a57 100644 --- a/app/templates/admin/dashboard.html +++ b/app/templates/admin/dashboard.html @@ -255,6 +255,11 @@
+
+ + Instruction Fetcher + +
Manage Users diff --git a/app/templates/admin/instruction_fetch.html b/app/templates/admin/instruction_fetch.html new file mode 100644 index 0000000..e1f6e77 --- /dev/null +++ b/app/templates/admin/instruction_fetch.html @@ -0,0 +1,352 @@ +{# Instruction fetcher control panel. + Created by: Jess Rogerson (yelling commands at Claude.AI) #} +{% extends "base.html" %} + +{% block title %}Instruction Fetcher - Admin{% endblock %} + +{% block content %} +
+
+

Instruction Fetcher

+

+ Downloads official building instructions from LEGO, using Brickset + to find them. Official sets only — one set every + {{ summary.interval_minutes }} minutes so nobody's API gets upset. +

+
+
+ + Back + +
+
+ +{% if not summary.brickset_configured %} +
+ + Brickset API not configured. + Add BRICKSET_API_KEY, BRICKSET_USERNAME and + BRICKSET_PASSWORD to your .env and restart. +
+{% endif %} + +{% if not worker_running %} +
+ + The background worker thread isn't running in this process. Queued sets + will only move when you press Run one now, or when + flask instructions run is called from Task Scheduler. +
+{% endif %} + + +
+
+
+
+
Status
+
+ {% if summary.enabled %} + Running + {% else %} + Paused + {% endif %} +
+
+
+
+
+
+
+
Next fetch in
+
—
+
+
+
+
+
+
+
In the queue
+
{{ summary.counts.queued }}
+
+
+
+
+
+
+
Official sets without a PDF
+
{{ summary.sets_missing_pdfs }}
+
+
+
+
+ +
+ +
+
+
+
Build the queue
+
+
+

+ {{ summary.official_sets }} official set(s) in the library, + {{ summary.sets_missing_pdfs }} with no PDF instructions yet. +

+ +
+ +
+ Limit + + +
+
Newest sets first. Skips anything already queued.
+
+ +
+ + +
+ +
+ + +
+ +
+ +
+
+ + +
+ +
+
+
+
+ + +
+
+
+
Pacing
+
+
+
+
+ + +
+ Turn off to pause. The queue is kept either way. +
+
+ +
+ + +
+ 10 is a polite default (144 sets a day). The clock is stored + in the database, so a restart doesn't reset it. +
+
+ +
+ + +
+ Big sets have several books; this spaces those downloads out. +
+
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ This is the only call that counts against the Brickset + daily quota. The ID is saved to the set afterwards. +
+
+ + +
+
+
+
+
+ + +
+
+
Recent jobs
+
+
+ + +
+
+
+
+ + + + + + + + + + + + + {% for job in jobs %} + + + + + + + + + {% else %} + + + + {% endfor %} + +
SetStatusFilesTriesResult
+ {% if job.lego_set %} + + {{ job.lego_set.set_number }} + +
{{ job.lego_set.set_name }}
+ {% else %} + deleted set + {% endif %} + 		{{ job.status }}{{ job.files_added }}{{ job.attempts }}{{ job.message or '—' }} + {% if job.is_open %} +
+ +
+ {% else %} +
+ +
+ {% endif %} +
+ Nothing queued yet. +
+
+
+ + +{% endblock %} diff --git a/app/templates/admin/settings.html b/app/templates/admin/settings.html index 1f37b36..35d5215 100644 --- a/app/templates/admin/settings.html +++ b/app/templates/admin/settings.html @@ -8,6 +8,9 @@

Site Settings

+ + Instruction Fetcher + Backup & Restore diff --git a/docker-compose.yml b/docker-compose.yml index e1c50fd..c3ec854 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,6 +51,13 @@ services: BRICKSET_API_KEY: ${BRICKSET_API_KEY:-} BRICKSET_USERNAME: ${BRICKSET_USERNAME:-} BRICKSET_PASSWORD: ${BRICKSET_PASSWORD:-} + # Automatic instruction fetcher. Safe to leave on: gunicorn runs a + # single worker here, so there is exactly one ticker, and it does + # nothing until an admin enables it in the UI. + INSTRUCTION_FETCH_WORKER: ${INSTRUCTION_FETCH_WORKER:-1} + INSTRUCTION_FETCH_TICK_SECONDS: ${INSTRUCTION_FETCH_TICK_SECONDS:-20} + FETCH_MAX_FILE_MB: ${FETCH_MAX_FILE_MB:-150} + FETCH_TIMEOUT_SECONDS: ${FETCH_TIMEOUT_SECONDS:-60} volumes: - lim-data:/data ports: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 8ace215..41edbcc 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -52,7 +52,8 @@ fi # migration — see docs/DOCKER_DEPLOYMENT.md. if [ "${SKIP_DB_INIT:-0}" != "1" ]; then log "Ensuring schema exists..." - python - <<'PY' + # No fetcher thread in this short-lived bootstrap process. + INSTRUCTION_FETCH_WORKER=0 python - <<'PY' import os from app import create_app, db app = create_app(os.getenv('FLASK_ENV', 'production')) diff --git a/docs/INSTRUCTION_FETCHER.md b/docs/INSTRUCTION_FETCHER.md new file mode 100644 index 0000000..94f3f93 --- /dev/null +++ b/docs/INSTRUCTION_FETCHER.md @@ -0,0 +1,181 @@ +# Automatic Instruction Fetcher + +Downloads official building instruction PDFs for **official sets only**, one +set at a time, on a timer you control. MOCs are never touched — they have no +Brickset entry to look up. + +Admin → **Instruction Fetcher** (`/admin/instruction-fetch`). + +--- + +## Where the files actually come from + +Brickset's API doesn't host the PDFs; it tells you where LEGO keeps them. + +``` +getInstructions2(setNumber) -> [{ URL, description }, ...] (brickset.com) + | + v +GET https://www.lego.com/.../6876-1.pdf (lego.com) +``` + +So "from Brickset or LEGO" is one flow: Brickset for the lookup, LEGO for the +file. Saved PDFs land in the normal uploads tree +(`uploads/pdfs//`) with a thumbnail, exactly like a manual upload, +and appear on the set page as ordinary instructions. + +### API quota + +Per Brickset's documentation, **only `getSets` counts against the daily key +limit**. `getInstructions` and `getInstructions2` do not. The fetcher is built +around that: + +| Call | When | Counts against quota | +|------|------|----------------------| +| `getInstructions` (by setID) | set already has `brickset_id` | no | +| `getInstructions2` (by number) | normal path | no | +| `getSets` | only if the number isn't found and set-ID lookup is enabled | **yes** | + +When a `getSets` lookup does happen, the resulting Brickset ID is written back +to the set, so it never happens twice for the same set. + +--- + +## Pacing + +One set per interval, 10 minutes by default (≈144 sets/day). Within a set, +multi-book instructions are spaced by a few seconds each. + +The "last run" timestamp is stored in `site_settings`, not in memory, so: + +* restarting LIM doesn't reset the clock and let it stampede, +* changing the interval takes effect on the next tick, no restart needed, +* pausing keeps the queue intact. + +All of it is configurable on the page: + +| Setting | Default | Notes | +|---------|---------|-------| +| Work through the queue automatically | off | master switch; off = nothing happens | +| Minutes between sets | 10 | 1–1440 | +| Seconds between PDFs of the same set | 5 | 0–300 | +| Attempts before giving up | 3 | failures back off two intervals, then retry | +| Skip sets that already have a PDF | on | | +| Allow set-ID lookups | on | the only quota-consuming call | + +--- + +## Using it + +1. **Queue sets missing instructions** — newest first. An optional limit lets + you try 10 before committing to 900. +2. Switch **Work through the queue automatically** on. +3. Walk away. The page shows a live countdown and refreshes the job table as + each set finishes. + +**Run one now** processes a single set immediately in the background (tick +*ignore the timer* to skip the cooldown). Useful for checking credentials +before queueing hundreds. + +### Job statuses + +| Status | Meaning | +|--------|---------| +| `queued` | waiting its turn | +| `running` | being fetched right now | +| `done` | files saved, or already had them | +| `empty` | Brickset lists no instructions for this set | +| `failed` | gave up after the configured attempts | +| `cancelled` | stopped by an admin, or the set was deleted | + +--- + +## Safety rails + +* **PDF only.** The first bytes must be `%PDF`; anything else is rejected with + the reason recorded on the job. +* **Size cap.** `FETCH_MAX_FILE_MB` (default 150) is checked both from the + `Content-Length` header and while streaming. +* **Deduplication.** Every downloaded file is hashed (SHA-256, stored on + `instructions.sha256`). A file already on the set is skipped, so re-running + a set never creates duplicates. +* **Files are only written after a successful download** — partial downloads + go to a temp file and are deleted. +* **Official sets only**, enforced in the query, the queue, and the fetch + itself. + +--- + +## Running it from Task Scheduler instead + +The background thread lives in the web process. If you'd rather drive it +externally, set `INSTRUCTION_FETCH_WORKER=0` in `.env` and schedule: + +```bat +flask instructions run --count 1 --force +``` + +Other commands: + +```bat +flask instructions status :: queue counts + countdown +flask instructions queue --missing --limit 50 :: queue sets with no PDF +flask instructions queue --all :: queue every official set +flask instructions run --count 3 :: respect the interval +``` + +`--force` ignores both the timer and the enabled switch, which is what you +want when the schedule itself provides the pacing. + +> Run **one** worker. If the web process has the thread running *and* Task +> Scheduler calls the CLI, two of them can claim jobs at once. The CLI won't +> start a worker thread of its own, so `INSTRUCTION_FETCH_WORKER=0` is the +> only switch you need. + +--- + +## Environment variables + +```ini +INSTRUCTION_FETCH_WORKER=1 # 0 to disable the in-process worker +INSTRUCTION_FETCH_TICK_SECONDS=20 # how often it checks whether it's due +FETCH_MAX_FILE_MB=150 # reject anything larger +FETCH_TIMEOUT_SECONDS=60 +``` + +Everything else is in the admin UI, not the environment. + +--- + +## Database changes + +Applied automatically at startup by `services/schema_upgrade.py`; no manual +migration: + +* new table `instruction_fetch_jobs` +* new columns `instructions.sha256`, `instructions.source` + +Existing instruction rows keep `sha256 = NULL`, which simply means dedupe +can't compare against them. + +--- + +## Troubleshooting + +**"Brickset API not configured"** — `BRICKSET_API_KEY`, `BRICKSET_USERNAME` +and `BRICKSET_PASSWORD` in `.env`, then restart. + +**Everything comes back `empty`** — check the set numbers. Brickset stores +them as `{number}-{variant}`; the fetcher appends `-1` automatically, but an +unusual variant (`-2`, `-3`) has to be in the set number itself. + +**"The background worker thread isn't running"** — either +`INSTRUCTION_FETCH_WORKER=0`, or LIM is being served in a way that loads the +app without starting it. *Run one now* still works. + +**Downloads fail with a connection error** — the server needs outbound HTTPS +to `brickset.com` and `lego.com`. + +--- + +Created by: Jess Rogerson (yelling commands at Claude.AI) diff --git a/install_instruction_fetcher.bat b/install_instruction_fetcher.bat new file mode 100644 index 0000000..74ea239 --- /dev/null +++ b/install_instruction_fetcher.bat @@ -0,0 +1,211 @@ +@echo off +REM =========================================================================== +REM LEGO Instructions Manager - Instruction Fetcher installer +REM +REM RUN THIS FROM THE FOLDER YOU EXTRACTED THE UPDATE INTO, not from your +REM LIM install - the two cannot be the same folder. +REM +REM Copies 22 named files into an existing LIM install, backing up every +REM file it replaces first. Whole directories are never copied, so your +REM uploads and database are not involved. +REM +REM Usage: install_instruction_fetcher.bat [target] +REM Default target: E:\LIM +REM +REM Created by: Jess Rogerson (yelling commands at Claude.AI) +REM =========================================================================== + +setlocal EnableDelayedExpansion + +set "SOURCE=%~dp0" +set "TARGET=%~1" +if "%TARGET%"=="" set "TARGET=E:\LIM" +set "MISSING=0" +set "COPYFAIL=0" +set "SAVED=0" + +for %%A in ("%SOURCE%.") do set "SRCFULL=%%~fA" +for %%B in ("%TARGET%\.") do set "DSTFULL=%%~fB" + +echo. +echo Source : %SRCFULL% +echo Target : %DSTFULL% +echo. + +REM --- refuse to copy a folder onto itself ----------------------------------- +if /i "%SRCFULL%"=="%DSTFULL%" goto :samedir + +if not exist "%TARGET%\app\__init__.py" goto :nottarget + +REM --- are the update files actually here? ----------------------------------- +echo Checking the update files... +call :need ".env.docker.example" +call :need ".env.example" +call :need "README.md" +call :need "app\__init__.py" +call :need "app\cli.py" +call :need "app\config.py" +call :need "app\models\__init__.py" +call :need "app\models\instruction.py" +call :need "app\models\instruction_fetch.py" +call :need "app\models\site_setting.py" +call :need "app\routes\__init__.py" +call :need "app\routes\instruction_fetch.py" +call :need "app\services\brickset_api.py" +call :need "app\services\fetch_worker.py" +call :need "app\services\instruction_fetcher.py" +call :need "app\services\schema_upgrade.py" +call :need "app\templates\admin\dashboard.html" +call :need "app\templates\admin\instruction_fetch.html" +call :need "app\templates\admin\settings.html" +call :need "docker-compose.yml" +call :need "docker\entrypoint.sh" +call :need "docs\INSTRUCTION_FETCHER.md" +if not "%MISSING%"=="0" goto :incomplete +echo all 22 present. + +REM --- backup ------------------------------------------------------------------ +for /f "tokens=2 delims==" %%I in ('wmic os get localdatetime /value') do set "DT=%%I" +set "STAMP=!DT:~0,8!-!DT:~8,6!" +set "BACKUP=%TARGET%\_backup_fetcher_!STAMP!" + +echo. +echo Backing up the files about to be replaced: +echo !BACKUP! +echo. +mkdir "!BACKUP!" 2>nul +call :save ".env.docker.example" +call :save ".env.example" +call :save "README.md" +call :save "app\__init__.py" +call :save "app\cli.py" +call :save "app\config.py" +call :save "app\models\__init__.py" +call :save "app\models\instruction.py" +call :save "app\models\instruction_fetch.py" +call :save "app\models\site_setting.py" +call :save "app\routes\__init__.py" +call :save "app\routes\instruction_fetch.py" +call :save "app\services\brickset_api.py" +call :save "app\services\fetch_worker.py" +call :save "app\services\instruction_fetcher.py" +call :save "app\services\schema_upgrade.py" +call :save "app\templates\admin\dashboard.html" +call :save "app\templates\admin\instruction_fetch.html" +call :save "app\templates\admin\settings.html" +call :save "docker-compose.yml" +call :save "docker\entrypoint.sh" +call :save "docs\INSTRUCTION_FETCHER.md" +echo %SAVED% file^(s^) backed up. + +REM --- install ------------------------------------------------------------------- +echo. +echo Installing... +call :put ".env.docker.example" +call :put ".env.example" +call :put "README.md" +call :put "app\__init__.py" +call :put "app\cli.py" +call :put "app\config.py" +call :put "app\models\__init__.py" +call :put "app\models\instruction.py" +call :put "app\models\instruction_fetch.py" +call :put "app\models\site_setting.py" +call :put "app\routes\__init__.py" +call :put "app\routes\instruction_fetch.py" +call :put "app\services\brickset_api.py" +call :put "app\services\fetch_worker.py" +call :put "app\services\instruction_fetcher.py" +call :put "app\services\schema_upgrade.py" +call :put "app\templates\admin\dashboard.html" +call :put "app\templates\admin\instruction_fetch.html" +call :put "app\templates\admin\settings.html" +call :put "docker-compose.yml" +call :put "docker\entrypoint.sh" +call :put "docs\INSTRUCTION_FETCHER.md" +if not "%COPYFAIL%"=="0" goto :copyfailed + +echo. +echo ========================================================== +echo Done - 22 files installed. +echo. +echo No migration step: the new table and columns are created +echo the next time LIM starts. +echo. +echo 1. Restart LIM (start_lim.bat) +echo 2. Admin ^> Instruction Fetcher +echo 3. Queue sets, then switch the fetcher on +echo. +echo To push the same change to Gitea, run push_fetcher_update.bat +echo FROM THIS FOLDER (not from %TARGET%). +echo. +echo Rollback: copy the files in +echo !BACKUP! +echo back over %TARGET% +echo ========================================================== +echo. +pause +exit /b 0 + +REM --- exits --------------------------------------------------------------------- + +:samedir +echo ERROR: the source and target are the same folder. +echo. +echo You are running this script from inside your LIM install, so there +echo is nothing to copy - which is why an earlier run appeared to work +echo but changed nothing. +echo. +echo Open the folder you extracted the update into (the one containing +echo app\services\instruction_fetcher.py) and run it from there: +echo. +echo cd /d C:\path\to\lim-instruction-fetcher +echo install_instruction_fetcher.bat %TARGET% +goto :fail + +:nottarget +echo ERROR: %TARGET% does not look like a LIM install +echo ^(no app\__init__.py^). +echo. +echo Pass the right path, e.g. install_instruction_fetcher.bat D:\LIM +goto :fail + +:incomplete +echo. +echo ERROR: %MISSING% file^(s^) missing from +echo %SRCFULL% +echo. +echo This is not the extracted update folder. Look for the one +echo containing app\services\instruction_fetcher.py. +goto :fail + +:copyfailed +echo. +echo ERROR: %COPYFAIL% file^(s^) could not be copied. Is LIM still running? +echo Stop it and run this again; the backup in +echo !BACKUP! +echo has the originals. +goto :fail + +:need +if not exist "%SOURCE%%~1" echo MISSING: %~1 & set /a MISSING+=1 +goto :eof + +:save +if not exist "%TARGET%\%~1" goto :eof +for %%F in ("!BACKUP!\%~1") do if not exist "%%~dpF" mkdir "%%~dpF" +copy /Y "%TARGET%\%~1" "!BACKUP!\%~1" >nul +if not errorlevel 1 echo saved %~1 & set /a SAVED+=1 +goto :eof + +:put +for %%F in ("%TARGET%\%~1") do if not exist "%%~dpF" mkdir "%%~dpF" +copy /Y "%SOURCE%%~1" "%TARGET%\%~1" >nul +if errorlevel 1 echo FAILED: %~1 & set /a COPYFAIL+=1 +goto :eof + +:fail +echo. +echo Nothing was changed. +pause +exit /b 1