Add automatic instruction fetcher for official sets

Downloads official instruction PDFs from LEGO, located via the Brickset API, one set per interval (10 minutes by default) so neither API gets hammered. Official sets only; MOCs are excluded throughout.

- New admin page at /admin/instruction-fetch: queue builder, pacing controls, live countdown and per-set job history

- Database-backed queue (instruction_fetch_jobs) and last-run timestamp, so pacing survives restarts

- Background worker thread, plus 'flask instructions queue/run/status' for Task Scheduler or cron

- SHA-256 dedupe (instructions.sha256), PDF header check, size cap and streaming downloads via temp files

- brickset_api: getInstructions2 by set number and getInstructions by setID (the old code sent a set number to the setID method); getSets params now proper JSON; getKeyUsageStats added

- Schema top-ups applied automatically at boot: new table plus instructions.sha256/source

- Docker: fetcher env vars passed through compose, no worker thread in the entrypoint bootstrap

.env.docker.example

@@ -24,3 +24,13 @@ SETS_PER_PAGE=20
 BRICKSET_API_KEY=
 BRICKSET_USERNAME=
 BRICKSET_PASSWORD=
+
+# --- instruction fetcher (optional) -----------------------------------------
+# Pacing (interval, delays, on/off) is set in the admin UI, not here.
+# Leave the worker on unless you scale the app container past one replica or
+# drive the queue yourself with `docker compose exec app flask instructions run`.
+INSTRUCTION_FETCH_WORKER=1
+INSTRUCTION_FETCH_TICK_SECONDS=20
+# Largest instruction PDF to accept, in megabytes.
+FETCH_MAX_FILE_MB=150
+FETCH_TIMEOUT_SECONDS=60

.env.example

@@ -26,3 +26,15 @@ BACKUP_KEEP=5
 
 # Application Configuration
 SETS_PER_PAGE=20
+
+# --- Automatic instruction fetcher -----------------------------------------
+# Pacing lives in the admin UI (Admin > Instruction Fetcher). These are the
+# process-level knobs.
+# Set to 0 to disable the background worker thread and drive the queue from
+# Task Scheduler / cron with `flask instructions run` instead.
+INSTRUCTION_FETCH_WORKER=1
+# How often the worker checks whether the interval has elapsed, in seconds.
+INSTRUCTION_FETCH_TICK_SECONDS=20
+# Largest instruction PDF to accept, in megabytes.
+FETCH_MAX_FILE_MB=150
+FETCH_TIMEOUT_SECONDS=60

README.md

@@ -18,6 +18,7 @@ A Python web application for organizing and managing your LEGO instruction manua
 - 🖼️ **Image Gallery**: View instruction images in an organized gallery
 - 📄 **PDF Support**: View and download PDF instructions directly
 - 💾 **Backup & Restore**: One-click archive of the database plus every uploaded file, ready to restore anywhere
+- ⬇️ **Automatic Instruction Fetcher**: Downloads official instruction PDFs from LEGO (found via Brickset) for official sets, one every 10 minutes so nobody's API gets hammered — see [docs/INSTRUCTION_FETCHER.md](docs/INSTRUCTION_FETCHER.md)
 
 ## 🚀 Quick Start
 
@@ -185,6 +186,11 @@ flask --app run.py shell
 ### Instruction
 - id, set_id, file_type, file_path, file_name, file_size, thumbnail_path
 - page_number, user_id, uploaded_at
+- sha256, source (set by the automatic fetcher; null for manual uploads)
+
+### InstructionFetchJob
+- id, set_id, status, attempts, files_added, source, message
+- queued_at, not_before, started_at, finished_at, queued_by_id
 
 ### ExtraFile
 - id, set_id, file_name, original_filename, file_path, file_type, file_size

app/__init__.py

@@ -47,6 +47,7 @@ def create_app(config_name='default'):
     from app.routes.extra_files import extra_files_bp
     from app.routes.moc import bp as moc_bp
     from app.routes.backup import backup_bp
+    from app.routes.instruction_fetch import instruction_fetch_bp
     
     app.register_blueprint(auth_bp)
     app.register_blueprint(main_bp)
@@ -56,6 +57,7 @@ def create_app(config_name='default'):
     app.register_blueprint(extra_files_bp)
     app.register_blueprint(moc_bp)
     app.register_blueprint(backup_bp)
+    app.register_blueprint(instruction_fetch_bp)
     
     # Import models to ensure they're registered with SQLAlchemy
     from app.models.user import User
@@ -64,6 +66,7 @@ def create_app(config_name='default'):
     from app.models.extra_file import ExtraFile
     from app.models.set_share import SetShare
     from app.models.site_setting import SiteSetting
+    from app.models.instruction_fetch import InstructionFetchJob
 
     # Add any columns missing from an older database (additive only).
     if app.config.get('AUTO_SCHEMA_UPGRADE', True):
@@ -96,9 +99,14 @@ def create_app(config_name='default'):
     def uploaded_file(filename):
         return send_from_directory(app.config['UPLOAD_FOLDER'], filename)
 
-    # Register CLI commands (flask backup create/list/verify/restore)
+    # Register CLI commands (flask backup ... / flask instructions ...)
     from app.cli import register_cli
     register_cli(app)
+
+    # Background ticker for the automatic instruction fetcher. It does
+    # nothing at all until an admin enables it on the fetcher page.
+    from app.services import fetch_worker
+    fetch_worker.start(app)
     
     # Context processor for global template variables
     @app.context_processor

app/cli.py

@@ -8,6 +8,10 @@ Flask CLI commands for LIM backup and restore.
     flask backup restore ARCHIVE [--mode fresh|merge] [--force]
                                  [--overwrite-files] [--skip-files]
 
+    flask instructions queue [--missing/--all] [--limit N]
+    flask instructions run [--count N] [--force]
+    flask instructions status
+
 `restore` is the path to use on a fresh Docker instance: no login required,
 and it works before any admin user exists.
 
@@ -181,5 +185,88 @@ def restore_command(archive, mode, force, overwrite_files, skip_files, yes):
     click.secho('Restore complete.', fg='green')
 
 
+instructions_cli = AppGroup(
+    'instructions', help='Queue and run automatic instruction downloads.')
+
+
+@instructions_cli.command('queue')
+@click.option('--missing/--all', 'missing_only', default=True,
+              help='Only official sets with no PDF yet (default), or every '
+                   'official set.')
+@click.option('--limit', default=None, type=int,
+              help='Queue at most this many sets.')
+def instructions_queue(missing_only, limit):
+    """Add official sets to the fetch queue."""
+    from app.services import instruction_fetcher as fetcher
+
+    if missing_only:
+        added = fetcher.queue_missing(limit=limit)
+    else:
+        added = fetcher.queue_all_official(limit=limit)
+
+    minutes = added * fetcher.interval_minutes()
+    click.secho(f'Queued {added} set(s).', fg='green')
+    if added:
+        click.echo(f'  At one every {fetcher.interval_minutes()} minutes that '
+                   f'is roughly {round(minutes / 60, 1)} hours of work.')
+
+
+@instructions_cli.command('run')
+@click.option('--count', default=1, type=int,
+              help='How many sets to process before exiting.')
+@click.option('--force', is_flag=True,
+              help='Ignore the interval timer and the enabled switch.')
+def instructions_run(count, force):
+    """
+    Process queued sets, respecting the configured interval.
+
+    Handy from Task Scheduler: `flask instructions run --count 1 --force`
+    every 10 minutes gives the same pacing without the worker thread.
+    """
+    from app.services import instruction_fetcher as fetcher
+
+    done = 0
+    for _ in range(max(1, count)):
+        result = fetcher.run_one(force=force)
+        if result is None:
+            if not fetcher.is_enabled() and not force:
+                click.secho('Fetcher is switched off (use --force to '
+                            'override).', fg='yellow')
+            elif not fetcher.is_due() and not force:
+                wait = fetcher.seconds_until_due()
+                click.secho(f'Not due for another {wait // 60}m '
+                            f'{wait % 60}s.', fg='yellow')
+            else:
+                click.echo('Queue is empty.')
+            break
+
+        done += 1
+        click.echo(f"  {result['set_number']}: {result['status']} "
+                   f"({result['files_added']} file(s)) - {result['message']}")
+
+    click.secho(f'Processed {done} set(s).', fg='green')
+
+
+@instructions_cli.command('status')
+def instructions_status():
+    """Show queue counts and when the next fetch is allowed."""
+    from app.services import instruction_fetcher as fetcher
+
+    summary = fetcher.status_summary()
+    counts = summary['counts']
+    wait = summary['next_run_in_seconds']
+
+    click.echo('  Enabled  : ' + ('yes' if summary['enabled'] else 'no'))
+    click.echo(f"  Interval : {summary['interval_minutes']} minutes")
+    click.echo(f"  Next run : {'now' if wait == 0 else f'in {wait // 60}m {wait % 60}s'}")
+    click.echo(f"  Brickset : {'configured' if summary['brickset_configured'] else 'NOT configured'}")
+    click.echo('  Queue    : ' + ', '.join(
+        f'{counts[key]} {key}' for key in
+        ('queued', 'running', 'done', 'empty', 'failed', 'cancelled')))
+    click.echo(f"  Library  : {summary['official_sets']} official set(s), "
+               f"{summary['sets_missing_pdfs']} without a PDF")
+
+
 def register_cli(app):
     app.cli.add_command(backup_cli)
+    app.cli.add_command(instructions_cli)

app/config.py

@@ -59,6 +59,23 @@ class Config:
     # stream straight to disk and never go through Flask's form parser.
     BACKUP_MAX_UPLOAD = int(os.environ.get('BACKUP_MAX_UPLOAD', 0))
 
+    # Automatic instruction fetcher
+    # Pacing (interval, delays, on/off) lives in site_settings so it can be
+    # changed from the admin page. These are the hard limits and the switch
+    # for the in-process worker thread.
+    #
+    # Turn the worker off (0) if you drive the queue from Task Scheduler or
+    # cron with `flask instructions run` instead, or if you ever run more
+    # than one LIM process against the same database.
+    INSTRUCTION_FETCH_WORKER = os.environ.get('INSTRUCTION_FETCH_WORKER', '1') != '0'
+    # How often the worker wakes to check whether the interval has elapsed.
+    INSTRUCTION_FETCH_TICK_SECONDS = int(
+        os.environ.get('INSTRUCTION_FETCH_TICK_SECONDS', 20))
+    # Refuse anything larger than this, in megabytes. Some Technic manuals
+    # are genuinely enormous, hence the generous default.
+    FETCH_MAX_FILE_MB = int(os.environ.get('FETCH_MAX_FILE_MB', 150))
+    FETCH_TIMEOUT_SECONDS = int(os.environ.get('FETCH_TIMEOUT_SECONDS', 60))
+
     # Add missing columns automatically at startup. Turn off if you'd
     # rather manage schema changes with Flask-Migrate yourself.
     AUTO_SCHEMA_UPGRADE = os.environ.get('AUTO_SCHEMA_UPGRADE', '1') != '0'

app/models/__init__.py

@@ -4,5 +4,7 @@ from app.models.instruction import Instruction
 from app.models.extra_file import ExtraFile
 from app.models.set_share import SetShare
 from app.models.site_setting import SiteSetting
+from app.models.instruction_fetch import InstructionFetchJob
 
-__all__ = ['User', 'Set', 'Instruction', 'ExtraFile', 'SetShare', 'SiteSetting']
+__all__ = ['User', 'Set', 'Instruction', 'ExtraFile', 'SetShare',
+           'SiteSetting', 'InstructionFetchJob']

app/models/instruction.py

@@ -16,6 +16,13 @@ class Instruction(db.Model):
     file_name = db.Column(db.String(200), nullable=False)
     file_size = db.Column(db.Integer)  # Size in bytes
     thumbnail_path = db.Column(db.String(500))  # Thumbnail preview (especially for PDFs)
+
+    # SHA-256 of the stored file. Set by the automatic fetcher so the same
+    # PDF is never downloaded twice; null for older manual uploads.
+    sha256 = db.Column(db.String(64), nullable=True, index=True)
+    # Where the file came from: None/'upload' for manual, 'brickset' for the
+    # automatic fetcher.
+    source = db.Column(db.String(20), nullable=True)
     
     # For image sequences
     page_number = db.Column(db.Integer, default=1)
@@ -40,6 +47,7 @@ class Instruction(db.Model):
             'file_name': self.file_name,
             'file_size': self.file_size,
             'page_number': self.page_number,
+            'source': self.source,
             'file_url': f'/static/uploads/{clean_path}',
             'thumbnail_url': f'/static/uploads/{thumbnail_clean}' if thumbnail_clean else None,
             'uploaded_at': self.uploaded_at.isoformat()

app/models/instruction_fetch.py

@@ -0,0 +1,121 @@
+"""
+Queue of "go and fetch the instructions" jobs for official sets.
+
+One row per set per attempt run. The queue lives in the database rather than
+in memory so a restart doesn't lose the backlog, and so the admin page can
+show what happened to every set without scraping the log file.
+
+Only official sets ever get queued — MOCs have no Brickset entry, so there is
+nothing to fetch.
+
+Created by: Jess Rogerson (yelling commands at Claude.AI)
+"""
+
+from datetime import datetime
+
+from app import db
+
+# queued    - waiting its turn
+# running   - being worked on right now
+# done      - at least one instruction file was saved
+# empty     - Brickset has no instructions on file for this set
+# failed    - gave up after max attempts
+# cancelled - removed from the queue by an admin
+STATUSES = ('queued', 'running', 'done', 'empty', 'failed', 'cancelled')
+
+OPEN_STATUSES = ('queued', 'running')
+
+STATUS_BADGES = {
+    'queued': 'secondary',
+    'running': 'primary',
+    'done': 'success',
+    'empty': 'warning',
+    'failed': 'danger',
+    'cancelled': 'dark',
+}
+
+
+class InstructionFetchJob(db.Model):
+    """A single set queued for an automatic instruction download."""
+
+    __tablename__ = 'instruction_fetch_jobs'
+
+    id = db.Column(db.Integer, primary_key=True)
+    set_id = db.Column(db.Integer, db.ForeignKey('sets.id'), nullable=False,
+                       index=True)
+
+    status = db.Column(db.String(12), nullable=False, default='queued',
+                       server_default='queued', index=True)
+    attempts = db.Column(db.Integer, nullable=False, default=0,
+                         server_default='0')
+    files_added = db.Column(db.Integer, nullable=False, default=0,
+                            server_default='0')
+    # 'brickset' for now; the column exists so a future LEGO-direct or
+    # Rebrickable path can be told apart in the history.
+    source = db.Column(db.String(20), nullable=True)
+    message = db.Column(db.Text, nullable=True)
+
+    queued_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow,
+                          index=True)
+    # Retry backoff: a job is invisible to the worker until this passes.
+    not_before = db.Column(db.DateTime, nullable=True)
+    started_at = db.Column(db.DateTime, nullable=True)
+    finished_at = db.Column(db.DateTime, nullable=True)
+
+    queued_by_id = db.Column(db.Integer, db.ForeignKey('users.id'),
+                             nullable=True)
+
+    lego_set = db.relationship(
+        'Set',
+        backref=db.backref('fetch_jobs', lazy='dynamic',
+                           cascade='all, delete-orphan'))
+    queued_by = db.relationship('User', foreign_keys=[queued_by_id])
+
+    def __repr__(self):
+        return f'<InstructionFetchJob set={self.set_id} {self.status}>'
+
+    # -- helpers ------------------------------------------------------------
+
+    @property
+    def badge(self):
+        return STATUS_BADGES.get(self.status, 'secondary')
+
+    @property
+    def is_open(self):
+        return self.status in OPEN_STATUSES
+
+    @classmethod
+    def open_for_set(cls, set_id):
+        """The queued/running job for a set, if there is one."""
+        return cls.query.filter(cls.set_id == set_id,
+                                cls.status.in_(OPEN_STATUSES)).first()
+
+    @classmethod
+    def counts(cls):
+        """Row counts per status, with every status present."""
+        from sqlalchemy import func
+        rows = db.session.query(cls.status, func.count(cls.id)).group_by(
+            cls.status).all()
+        counts = {status: 0 for status in STATUSES}
+        for status, count in rows:
+            counts[status] = count
+        counts['total'] = sum(counts[s] for s in STATUSES)
+        return counts
+
+    def to_dict(self):
+        lego_set = self.lego_set
+        return {
+            'id': self.id,
+            'set_id': self.set_id,
+            'set_number': lego_set.set_number if lego_set else None,
+            'set_name': lego_set.set_name if lego_set else None,
+            'status': self.status,
+            'badge': self.badge,
+            'attempts': self.attempts,
+            'files_added': self.files_added,
+            'source': self.source,
+            'message': self.message,
+            'queued_at': self.queued_at.isoformat() if self.queued_at else None,
+            'finished_at': (self.finished_at.isoformat()
+                            if self.finished_at else None),
+        }

app/models/site_setting.py

@@ -22,6 +22,16 @@ DEFAULTS = {
     # Send logged-in users straight to their dashboard instead of the
     # marketing homepage.
     'home_redirect_dashboard': 'true',
+
+    # Automatic instruction fetcher (see services/instruction_fetcher.py).
+    # Off by default: an admin has to switch it on deliberately.
+    'fetch_enabled': 'false',
+    'fetch_interval_minutes': '10',
+    'fetch_file_delay_seconds': '5',
+    'fetch_max_attempts': '3',
+    'fetch_skip_sets_with_pdfs': 'true',
+    'fetch_allow_setid_lookup': 'true',
+    'fetch_last_run': '',
 }
 
 VISIBILITIES = ('private', 'shared', 'public')

app/routes/__init__.py

@@ -3,5 +3,7 @@ from app.routes.main import main_bp
 from app.routes.sets import sets_bp
 from app.routes.instructions import instructions_bp
 from app.routes.moc import bp as moc_bp
+from app.routes.instruction_fetch import instruction_fetch_bp
 
-__all__ = ['auth_bp', 'main_bp', 'sets_bp', 'instructions_bp', 'moc_bp']
+__all__ = ['auth_bp', 'main_bp', 'sets_bp', 'instructions_bp', 'moc_bp',
+           'instruction_fetch_bp']

app/routes/instruction_fetch.py

@@ -0,0 +1,197 @@
+"""
+Admin pages for the automatic instruction fetcher.
+
+Everything here is admin-only and reuses admin.admin_required so there is one
+definition of "is this person allowed in".
+
+Created by: Jess Rogerson (yelling commands at Claude.AI)
+"""
+
+from flask import (Blueprint, current_app, flash, jsonify, redirect,
+                   render_template, request, url_for)
+from flask_login import current_user, login_required
+
+from app import db
+from app.models.instruction_fetch import InstructionFetchJob
+from app.models.set import Set
+from app.models.site_setting import SiteSetting
+from app.routes.admin import admin_required
+from app.services import fetch_worker
+from app.services import instruction_fetcher as fetcher
+
+instruction_fetch_bp = Blueprint('instruction_fetch', __name__,
+                                 url_prefix='/admin/instruction-fetch')
+
+
+def _int_form(name, default, low, high):
+    try:
+        value = int(request.form.get(name, default))
+    except (TypeError, ValueError):
+        value = default
+    return max(low, min(high, value))
+
+
+@instruction_fetch_bp.route('/')
+@login_required
+@admin_required
+def index():
+    """Control panel for the fetcher."""
+    summary = fetcher.status_summary()
+    jobs = InstructionFetchJob.query.order_by(
+        InstructionFetchJob.id.desc()).limit(50).all()
+    return render_template('admin/instruction_fetch.html',
+                           summary=summary,
+                           jobs=jobs,
+                           worker_running=fetch_worker.is_running())
+
+
+@instruction_fetch_bp.route('/status')
+@login_required
+@admin_required
+def status():
+    """JSON for the page's auto-refresh."""
+    summary = fetcher.status_summary()
+    summary['worker_running'] = fetch_worker.is_running()
+    return jsonify(summary)
+
+
+@instruction_fetch_bp.route('/settings', methods=['POST'])
+@login_required
+@admin_required
+def save_settings():
+    SiteSetting.set('fetch_enabled',
+                    bool(request.form.get('fetch_enabled')))
+    SiteSetting.set('fetch_interval_minutes',
+                    str(_int_form('fetch_interval_minutes', 10,
+                                  fetcher.MIN_INTERVAL_MINUTES,
+                                  fetcher.MAX_INTERVAL_MINUTES)))
+    SiteSetting.set('fetch_file_delay_seconds',
+                    str(_int_form('fetch_file_delay_seconds', 5, 0, 300)))
+    SiteSetting.set('fetch_max_attempts',
+                    str(_int_form('fetch_max_attempts', 3, 1, 10)))
+    SiteSetting.set('fetch_skip_sets_with_pdfs',
+                    bool(request.form.get('fetch_skip_sets_with_pdfs')))
+    SiteSetting.set('fetch_allow_setid_lookup',
+                    bool(request.form.get('fetch_allow_setid_lookup')))
+    db.session.commit()
+
+    flash('Fetcher settings saved.', 'success')
+    return redirect(url_for('instruction_fetch.index'))
+
+
+@instruction_fetch_bp.route('/queue', methods=['POST'])
+@login_required
+@admin_required
+def queue():
+    """Add official sets to the queue."""
+    scope = request.form.get('scope', 'missing')
+    limit = request.form.get('limit', '').strip()
+    limit = int(limit) if limit.isdigit() and int(limit) > 0 else None
+
+    if scope == 'all':
+        added = fetcher.queue_all_official(current_user.id, limit)
+        label = 'official set(s)'
+    elif scope == 'retry-failed':
+        added = fetcher.requeue_failed(current_user.id)
+        label = 'failed job(s) requeued'
+    else:
+        added = fetcher.queue_missing(current_user.id, limit)
+        label = 'set(s) with no PDF yet'
+
+    if added:
+        minutes = added * fetcher.interval_minutes()
+        hours = round(minutes / 60, 1)
+        flash(f'Queued {added} {label}. At one every '
+              f'{fetcher.interval_minutes()} minutes that is about '
+              f'{hours} hours of background work.', 'success')
+        if not fetcher.is_enabled():
+            flash('The fetcher is currently paused — switch it on below to '
+                  'start working through the queue.', 'warning')
+    else:
+        flash('Nothing new to queue.', 'info')
+
+    return redirect(url_for('instruction_fetch.index'))
+
+
+@instruction_fetch_bp.route('/queue-set/<int:set_id>', methods=['POST'])
+@login_required
+@admin_required
+def queue_set(set_id):
+    """Queue one specific set (used from the set page)."""
+    lego_set = Set.query.get_or_404(set_id)
+    if lego_set.is_moc:
+        flash('MOCs have no Brickset entry to fetch from.', 'warning')
+    elif fetcher.queue_sets([lego_set], current_user.id):
+        flash(f'{lego_set.set_number} queued for an instruction fetch.',
+              'success')
+    else:
+        flash(f'{lego_set.set_number} is already in the queue.', 'info')
+
+    return redirect(request.referrer or url_for('instruction_fetch.index'))
+
+
+@instruction_fetch_bp.route('/run-now', methods=['POST'])
+@login_required
+@admin_required
+def run_now():
+    """Process one queued set immediately, in the background."""
+    force = bool(request.form.get('force'))
+
+    if not fetcher.is_due() and not force:
+        wait = fetcher.seconds_until_due()
+        flash(f'Still cooling down — {wait // 60}m {wait % 60}s until the '
+              f'next allowed fetch. Tick "ignore the timer" to override.',
+              'warning')
+        return redirect(url_for('instruction_fetch.index'))
+
+    counts = InstructionFetchJob.counts()
+    if not counts['queued']:
+        flash('The queue is empty.', 'info')
+        return redirect(url_for('instruction_fetch.index'))
+
+    fetch_worker.run_once_async(current_app._get_current_object(), force=force)
+    flash('Fetching one set now — the table below will update shortly.',
+          'info')
+    return redirect(url_for('instruction_fetch.index'))
+
+
+@instruction_fetch_bp.route('/jobs/<int:job_id>/cancel', methods=['POST'])
+@login_required
+@admin_required
+def cancel_job(job_id):
+    job = InstructionFetchJob.query.get_or_404(job_id)
+    if job.is_open:
+        job.status = 'cancelled'
+        job.message = 'cancelled by admin'
+        db.session.commit()
+        flash('Job cancelled.', 'success')
+    else:
+        flash('That job has already finished.', 'info')
+    return redirect(url_for('instruction_fetch.index'))
+
+
+@instruction_fetch_bp.route('/jobs/<int:job_id>/retry', methods=['POST'])
+@login_required
+@admin_required
+def retry_job(job_id):
+    job = InstructionFetchJob.query.get_or_404(job_id)
+    if InstructionFetchJob.open_for_set(job.set_id):
+        flash('That set is already in the queue.', 'info')
+    else:
+        job.status = 'queued'
+        job.attempts = 0
+        job.message = None
+        job.not_before = None
+        db.session.commit()
+        flash('Job requeued.', 'success')
+    return redirect(url_for('instruction_fetch.index'))
+
+
+@instruction_fetch_bp.route('/clear', methods=['POST'])
+@login_required
+@admin_required
+def clear():
+    scope = request.form.get('scope', 'finished')
+    removed = fetcher.clear_jobs(scope)
+    flash(f'Removed {removed} job record(s).', 'success')
+    return redirect(url_for('instruction_fetch.index'))

app/services/brickset_api.py

@@ -1,19 +1,39 @@
+"""
+Brickset API v3 client.
+
+https://brickset.com/article/52664/api-version-3-documentation
+
+Two things worth knowing about the API, both of which shape this file:
+
+  * getSets takes its search terms as a JSON string. Python's str() on a dict
+    produces single quotes, which is not JSON — hence json.dumps below.
+  * getInstructions takes a Brickset setID; getInstructions2 takes a set
+    number like "10497-1". They are different methods and passing a set
+    number to the first one quietly returns nothing.
+
+Only getSets counts against the daily key quota.
+
+Created by: Jess Rogerson (yelling commands at Claude.AI)
+"""
+
+import json
+from typing import Any, Dict, List, Optional
+
 import requests
 from flask import current_app
-from typing import Optional, List, Dict, Any
 
 
 class BricksetAPI:
     """Service for interacting with the Brickset API v3."""
-    
+
     BASE_URL = 'https://brickset.com/api/v3.asmx'
-    
+
     def __init__(self):
         self.api_key = current_app.config.get('BRICKSET_API_KEY')
         self.username = current_app.config.get('BRICKSET_USERNAME')
         self.password = current_app.config.get('BRICKSET_PASSWORD')
         self._user_hash = None
-    
+
     @staticmethod
     def is_configured() -> bool:
         """Check if Brickset API is properly configured."""
@@ -22,16 +42,16 @@ class BricksetAPI:
             current_app.config.get('BRICKSET_USERNAME') and
             current_app.config.get('BRICKSET_PASSWORD')
         )
-    
+
     def _get_user_hash(self) -> Optional[str]:
         """Authenticate and get user hash token."""
         if self._user_hash:
             return self._user_hash
-        
+
         if not all([self.api_key, self.username, self.password]):
             current_app.logger.warning('Brickset API credentials not configured')
             return None
-        
+
         try:
             response = requests.get(
                 f'{self.BASE_URL}/login',
@@ -44,47 +64,43 @@ class BricksetAPI:
             )
             response.raise_for_status()
             data = response.json()
-            
+
             if data.get('status') == 'success':
                 self._user_hash = data.get('hash')
                 return self._user_hash
             else:
                 current_app.logger.error(f"Brickset login failed: {data.get('message')}")
                 return None
-                
+
         except Exception as e:
             current_app.logger.error(f'Brickset API authentication error: {str(e)}')
             return None
-    
-    def search_sets(self, 
-                   set_number: Optional[str] = None,
-                   theme: Optional[str] = None,
-                   year: Optional[int] = None,
-                   query: Optional[str] = None) -> List[Dict[str, Any]]:
+
+    def search_sets(self,
+                    set_number: Optional[str] = None,
+                    theme: Optional[str] = None,
+                    year: Optional[int] = None,
+                    query: Optional[str] = None,
+                    page_size: Optional[int] = None) -> List[Dict[str, Any]]:
         """
         Search for LEGO sets using various parameters.
-        
+
         Args:
             set_number: Specific set number to search for
             theme: Theme name to filter by
             year: Year released
             query: General search query
-            
+            page_size: Number of records to return (max 500)
+
         Returns:
             List of set dictionaries
         """
         user_hash = self._get_user_hash()
         if not user_hash:
             return []
-        
-        params = {
-            'apiKey': self.api_key,
-            'userHash': user_hash,
-            'params': '{}'  # JSON params object
-        }
-        
+
         # Build search parameters
-        search_params = {}
+        search_params: Dict[str, Any] = {}
         if set_number:
             search_params['setNumber'] = set_number
         if theme:
@@ -93,9 +109,20 @@ class BricksetAPI:
             search_params['year'] = year
         if query:
             search_params['query'] = query
-        
-        params['params'] = str(search_params)
-        
+        if page_size:
+            search_params['pageSize'] = page_size
+
+        if not search_params:
+            return []
+
+        params = {
+            'apiKey': self.api_key,
+            'userHash': user_hash,
+            # Must be a JSON string — str(dict) gives single quotes and is
+            # rejected with "Parameter error".
+            'params': json.dumps(search_params),
+        }
+
         try:
             response = requests.get(
                 f'{self.BASE_URL}/getSets',
@@ -104,41 +131,49 @@ class BricksetAPI:
             )
             response.raise_for_status()
             data = response.json()
-            
+
             if data.get('status') == 'success':
                 return data.get('sets', [])
-            else:
-                current_app.logger.error(f"Brickset search failed: {data.get('message')}")
-                return []
-                
+
+            message = data.get('message', 'unknown error')
+            current_app.logger.error(f'Brickset search failed: {message}')
+            if 'limit exceeded' in message.lower():
+                raise RuntimeError('Brickset API limit exceeded')
+            return []
+
+        except RuntimeError:
+            raise
         except Exception as e:
             current_app.logger.error(f'Brickset API search error: {str(e)}')
             return []
-    
+
     def get_set_by_number(self, set_number: str) -> Optional[Dict[str, Any]]:
         """
         Get detailed information for a specific set by its number.
-        
+
         Args:
-            set_number: The LEGO set number (e.g., "10497")
-            
+            set_number: The LEGO set number (e.g., "10497" or "10497-1")
+
         Returns:
             Dictionary with set information or None
         """
         results = self.search_sets(set_number=set_number)
+        if not results and '-' not in set_number:
+            # Brickset stores numbers with a variant suffix.
+            results = self.search_sets(set_number=f'{set_number}-1')
         return results[0] if results else None
-    
+
     def get_themes(self) -> List[str]:
         """
         Get list of all available LEGO themes.
-        
+
         Returns:
             List of theme names
         """
         user_hash = self._get_user_hash()
         if not user_hash:
             return []
-        
+
         try:
             response = requests.get(
                 f'{self.BASE_URL}/getThemes',
@@ -150,49 +185,76 @@ class BricksetAPI:
             )
             response.raise_for_status()
             data = response.json()
-            
+
             if data.get('status') == 'success':
                 themes = data.get('themes', [])
                 return [theme.get('theme') for theme in themes if theme.get('theme')]
             else:
                 return []
-                
+
         except Exception as e:
             current_app.logger.error(f'Brickset API themes error: {str(e)}')
             return []
-    
-    def get_instructions(self, set_number: str) -> List[Dict[str, Any]]:
-        """
-        Get instruction information for a specific set.
-        
-        Args:
-            set_number: The LEGO set number
-            
-        Returns:
-            List of instruction dictionaries
-        """
-        user_hash = self._get_user_hash()
-        if not user_hash:
-            return []
-        
+
+    # -- instructions -------------------------------------------------------
+
+    def _instructions_request(self, method: str,
+                              params: Dict[str, Any]) -> List[Dict[str, Any]]:
+        params = dict(params, apiKey=self.api_key)
         try:
-            response = requests.get(
-                f'{self.BASE_URL}/getInstructions',
-                params={
-                    'apiKey': self.api_key,
-                    'userHash': user_hash,
-                    'setNumber': set_number
-                },
-                timeout=10
-            )
+            response = requests.get(f'{self.BASE_URL}/{method}',
+                                    params=params, timeout=20)
             response.raise_for_status()
             data = response.json()
-            
+
             if data.get('status') == 'success':
-                return data.get('instructions', [])
-            else:
-                return []
-                
+                return data.get('instructions', []) or []
+
+            current_app.logger.error(
+                f"Brickset {method} failed: {data.get('message')}")
+            return []
+
         except Exception as e:
-            current_app.logger.error(f'Brickset API instructions error: {str(e)}')
+            current_app.logger.error(f'Brickset {method} error: {str(e)}')
+            return []
+
+    def get_instructions(self, set_number: str) -> List[Dict[str, Any]]:
+        """
+        Instructions for a set number, e.g. "10497-1" (getInstructions2).
+
+        Returns a list of {'URL': ..., 'description': ...} dictionaries.
+        Does not need a user hash and does not count against the key quota.
+        """
+        if not self.api_key or not set_number:
+            return []
+        return self._instructions_request('getInstructions2',
+                                          {'setNumber': set_number})
+
+    def get_instructions_by_id(self, set_id: int) -> List[Dict[str, Any]]:
+        """Instructions for an internal Brickset set ID (getInstructions)."""
+        if not self.api_key or not set_id:
+            return []
+        return self._instructions_request('getInstructions', {'setID': set_id})
+
+    # -- quota --------------------------------------------------------------
+
+    def get_key_usage(self) -> List[Dict[str, Any]]:
+        """
+        Daily getSets usage for the last 30 days.
+
+        Returns a list of {'dateStamp': ..., 'count': ...} dictionaries.
+        """
+        if not self.api_key:
+            return []
+        try:
+            response = requests.get(f'{self.BASE_URL}/getKeyUsageStats',
+                                    params={'apiKey': self.api_key},
+                                    timeout=10)
+            response.raise_for_status()
+            data = response.json()
+            if data.get('status') == 'success':
+                return data.get('apiKeyUsage', []) or []
+            return []
+        except Exception as e:
+            current_app.logger.error(f'Brickset key usage error: {str(e)}')
             return []

app/services/fetch_worker.py

@@ -0,0 +1,109 @@
+"""
+The background ticker behind the instruction fetcher.
+
+A daemon thread wakes every few seconds, asks the fetcher whether the
+interval has elapsed, and if so processes exactly one set. All of the actual
+pacing lives in instruction_fetcher (and in the database), so pausing the
+worker or changing the interval from the admin page takes effect on the next
+tick without a restart.
+
+Deliberately a plain thread rather than a scheduler dependency: LIM runs
+single-process under waitress, and Python 3.14 has been quite enough
+dependency excitement for one project.
+
+Run one process. If you ever scale LIM horizontally, turn the worker off with
+INSTRUCTION_FETCH_WORKER=0 everywhere and drive it from cron/Task Scheduler
+with `flask instructions run` instead.
+
+Created by: Jess Rogerson (yelling commands at Claude.AI)
+"""
+
+import os
+import sys
+import threading
+
+_thread = None
+_stop = threading.Event()
+_lock = threading.Lock()
+
+
+def _is_cli_command():
+    """True when the app was loaded by `flask <cmd>` rather than to serve."""
+    if os.environ.get('FLASK_RUN_FROM_CLI') != 'true':
+        return False
+    args = [arg for arg in sys.argv[1:] if not arg.startswith('-')]
+    return bool(args) and args[0] != 'run'
+
+
+def start(app):
+    """Start the ticker unless it's disabled or already running."""
+    if not app.config.get('INSTRUCTION_FETCH_WORKER', True):
+        app.logger.info('Instruction fetch worker disabled by config')
+        return False
+
+    # `flask instructions run`, `flask backup restore` and friends shouldn't
+    # quietly spin up a fetcher of their own.
+    if _is_cli_command():
+        return False
+
+    # Flask's reloader imports the app twice in debug; only the child should
+    # own the thread, or two of them race for the same job.
+    if app.debug and os.environ.get('WERKZEUG_RUN_MAIN') != 'true':
+        return False
+
+    global _thread
+    with _lock:
+        if _thread is not None and _thread.is_alive():
+            return False
+        _stop.clear()
+        _thread = threading.Thread(target=_loop, args=(app,),
+                                   name='lim-instruction-fetcher',
+                                   daemon=True)
+        _thread.start()
+
+    app.logger.info('Instruction fetch worker started')
+    return True
+
+
+def stop(timeout=5):
+    """Ask the ticker to finish (used by tests and the CLI)."""
+    _stop.set()
+    thread = _thread
+    if thread is not None and thread.is_alive():
+        thread.join(timeout=timeout)
+
+
+def is_running():
+    return _thread is not None and _thread.is_alive()
+
+
+def _loop(app):
+    tick = app.config.get('INSTRUCTION_FETCH_TICK_SECONDS', 20)
+    from app.services import instruction_fetcher as fetcher
+
+    while not _stop.wait(tick):
+        try:
+            with app.app_context():
+                fetcher.run_one()
+        except Exception:
+            app.logger.exception('Instruction fetch worker tick failed')
+
+
+def run_once_async(app, force=False):
+    """
+    Fire off a single job in the background.
+
+    Used by the admin page's "Run one now" button so the request returns
+    straight away instead of holding the browser open for a 40 MB PDF.
+    """
+    def _work():
+        try:
+            with app.app_context():
+                from app.services import instruction_fetcher as fetcher
+                fetcher.run_one(force=force)
+        except Exception:
+            app.logger.exception('Manual instruction fetch failed')
+
+    thread = threading.Thread(target=_work, name='lim-fetch-once', daemon=True)
+    thread.start()
+    return thread

app/services/instruction_fetcher.py

@@ -0,0 +1,562 @@
+"""
+Automatic instruction downloads for official sets.
+
+Brickset's getInstructions/getInstructions2 methods hand back the URLs of the
+official PDFs, which live on LEGO's own servers. So "download from Brickset or
+LEGO" is really one flow: ask Brickset where the file is, then pull the PDF
+from LEGO.
+
+The whole point of this module is politeness. Work is queued in the database
+and exactly one set is processed per interval (10 minutes by default), with a
+short pause between the individual PDFs of a multi-book set. Nothing here ever
+loops over the whole library in one go.
+
+Notes on quotas:
+  * Only getSets counts against a Brickset API key's daily limit, so the
+    set-ID lookup is the expensive call and is skipped whenever the set
+    already has a brickset_id stored.
+  * getInstructions2 needs nothing but the API key and a set number.
+
+Created by: Jess Rogerson (yelling commands at Claude.AI)
+"""
+
+import hashlib
+import os
+import re
+import tempfile
+import time
+from datetime import datetime, timedelta
+
+import requests
+from flask import current_app
+
+from app import db
+from app.models.instruction import Instruction
+from app.models.instruction_fetch import InstructionFetchJob
+from app.models.set import Set
+from app.models.site_setting import SiteSetting
+from app.services.brickset_api import BricksetAPI
+from app.services.file_handler import FileHandler
+
+USER_AGENT = ('LegoInstructionsManager/1.0 '
+              '(+https://gitea.hideawaygaming.com.au/jessikitty/'
+              'lego-instructions-manager)')
+
+# Settings surfaced on the admin page. Values are strings in site_settings.
+SETTING_DEFAULTS = {
+    'fetch_enabled': 'false',
+    'fetch_interval_minutes': '10',
+    'fetch_file_delay_seconds': '5',
+    'fetch_max_attempts': '3',
+    'fetch_skip_sets_with_pdfs': 'true',
+    'fetch_allow_setid_lookup': 'true',
+    'fetch_last_run': '',
+}
+
+MIN_INTERVAL_MINUTES = 1
+MAX_INTERVAL_MINUTES = 1440
+
+
+# ---------------------------------------------------------------------------
+# settings
+# ---------------------------------------------------------------------------
+
+def _int_setting(key, fallback, low=None, high=None):
+    raw = SiteSetting.get(key, SETTING_DEFAULTS.get(key))
+    try:
+        value = int(str(raw).strip())
+    except (TypeError, ValueError):
+        value = fallback
+    if low is not None:
+        value = max(low, value)
+    if high is not None:
+        value = min(high, value)
+    return value
+
+
+def is_enabled():
+    return SiteSetting.get_bool('fetch_enabled', False)
+
+
+def interval_minutes():
+    return _int_setting('fetch_interval_minutes', 10,
+                        MIN_INTERVAL_MINUTES, MAX_INTERVAL_MINUTES)
+
+
+def file_delay_seconds():
+    return _int_setting('fetch_file_delay_seconds', 5, 0, 300)
+
+
+def max_attempts():
+    return _int_setting('fetch_max_attempts', 3, 1, 10)
+
+
+def skip_sets_with_pdfs():
+    return SiteSetting.get_bool('fetch_skip_sets_with_pdfs', True)
+
+
+def allow_setid_lookup():
+    return SiteSetting.get_bool('fetch_allow_setid_lookup', True)
+
+
+def last_run():
+    raw = SiteSetting.get('fetch_last_run', '')
+    if not raw:
+        return None
+    try:
+        return datetime.fromisoformat(raw)
+    except ValueError:
+        return None
+
+
+def _mark_ran(when=None):
+    SiteSetting.set('fetch_last_run', (when or datetime.utcnow()).isoformat())
+    db.session.commit()
+
+
+def next_run_at():
+    """When the worker is next allowed to touch the network."""
+    previous = last_run()
+    if previous is None:
+        return datetime.utcnow()
+    return previous + timedelta(minutes=interval_minutes())
+
+
+def seconds_until_due():
+    return max(0, int((next_run_at() - datetime.utcnow()).total_seconds()))
+
+
+def is_due():
+    return seconds_until_due() <= 0
+
+
+# ---------------------------------------------------------------------------
+# queueing
+# ---------------------------------------------------------------------------
+
+def official_sets_query():
+    """Official sets only — MOCs have nothing to look up."""
+    return Set.query.filter(Set.is_moc.is_(False))
+
+
+def sets_missing_pdfs():
+    """Official sets with no PDF instruction on file."""
+    have_pdfs = db.session.query(Instruction.set_id).filter(
+        Instruction.file_type == 'PDF').distinct()
+    return official_sets_query().filter(~Set.id.in_(have_pdfs))
+
+
+def queue_sets(sets, user_id=None):
+    """Queue an iterable of sets, skipping any already waiting. -> count."""
+    queued = 0
+    for lego_set in sets:
+        if lego_set.is_moc:
+            continue
+        if InstructionFetchJob.open_for_set(lego_set.id):
+            continue
+        db.session.add(InstructionFetchJob(set_id=lego_set.id,
+                                           queued_by_id=user_id))
+        queued += 1
+    if queued:
+        db.session.commit()
+    return queued
+
+
+def queue_missing(user_id=None, limit=None):
+    query = sets_missing_pdfs().order_by(Set.year_released.desc(),
+                                         Set.set_number)
+    if limit:
+        query = query.limit(limit)
+    return queue_sets(query.all(), user_id)
+
+
+def queue_all_official(user_id=None, limit=None):
+    query = official_sets_query().order_by(Set.year_released.desc(),
+                                           Set.set_number)
+    if limit:
+        query = query.limit(limit)
+    return queue_sets(query.all(), user_id)
+
+
+def requeue_failed(user_id=None):
+    """Put every failed job back at the end of the queue."""
+    failed = InstructionFetchJob.query.filter_by(status='failed').all()
+    count = 0
+    for job in failed:
+        if InstructionFetchJob.open_for_set(job.set_id):
+            continue
+        job.status = 'queued'
+        job.attempts = 0
+        job.message = None
+        job.not_before = None
+        job.queued_at = datetime.utcnow()
+        job.queued_by_id = user_id or job.queued_by_id
+        count += 1
+    if count:
+        db.session.commit()
+    return count
+
+
+def clear_jobs(scope='finished'):
+    """Tidy the history. scope: finished | failed | all."""
+    query = InstructionFetchJob.query
+    if scope == 'finished':
+        query = query.filter(InstructionFetchJob.status.in_(
+            ('done', 'empty', 'cancelled')))
+    elif scope == 'failed':
+        query = query.filter_by(status='failed')
+    elif scope != 'all':
+        return 0
+    removed = query.delete(synchronize_session=False)
+    db.session.commit()
+    return removed
+
+
+# ---------------------------------------------------------------------------
+# the actual fetching
+# ---------------------------------------------------------------------------
+
+def brickset_number(set_number):
+    """Brickset wants {number}-{variant}; the library usually stores {number}."""
+    number = (set_number or '').strip()
+    if not number:
+        return None
+    if re.search(r'-\d+$', number):
+        return number
+    return f'{number}-1'
+
+
+def _instruction_urls(api, lego_set):
+    """(list of {url, description}, source, note)."""
+    note = None
+
+    if lego_set.brickset_id:
+        entries = api.get_instructions_by_id(lego_set.brickset_id)
+        if entries:
+            return _normalise(entries), 'brickset', note
+
+    number = brickset_number(lego_set.set_number)
+    if number:
+        entries = api.get_instructions(number)
+        if entries:
+            return _normalise(entries), 'brickset', note
+
+    # Last resort: resolve the Brickset set ID, then ask again. This is the
+    # only call that counts against the daily API key quota, so it is both
+    # optional and cached back onto the set for next time.
+    if allow_setid_lookup() and not lego_set.brickset_id and number:
+        data = api.get_set_by_number(number)
+        if data and data.get('setID'):
+            lego_set.brickset_id = data['setID']
+            db.session.commit()
+            entries = api.get_instructions_by_id(lego_set.brickset_id)
+            if entries:
+                return _normalise(entries), 'brickset', 'resolved via getSets'
+            note = 'set found, no instructions listed'
+        else:
+            note = 'set number not found on Brickset'
+
+    return [], 'brickset', note
+
+
+def _normalise(entries):
+    out = []
+    for entry in entries or []:
+        url = entry.get('URL') or entry.get('url')
+        if not url:
+            continue
+        out.append({
+            'url': url,
+            'description': (entry.get('description')
+                            or entry.get('Description') or ''),
+        })
+    return out
+
+
+def _safe_name(lego_set, description, index):
+    """A readable file name, e.g. '10497-building-instructions-book-1.pdf'."""
+    number = re.sub(r'[^A-Za-z0-9\-]+', '', lego_set.set_number or 'set')
+    base = re.sub(r'[^A-Za-z0-9]+', '-', description or '').strip('-').lower()
+    base = re.sub(r'-{2,}', '-', base)[:80].strip('-')
+    if not base:
+        return f'{number}-instructions-{index}.pdf'
+    # Brickset descriptions usually lead with the set number already.
+    if base.startswith(number.lower()):
+        return f'{base}.pdf'
+    return f'{number}-{base}.pdf'
+
+
+class _LocalFile:
+    """Adapter so FileHandler.save_file can take a file we downloaded."""
+
+    def __init__(self, path, filename):
+        self._path = path
+        self.filename = filename
+
+    def save(self, destination):
+        import shutil
+        shutil.move(self._path, destination)
+
+
+def _download_pdf(url, max_bytes, timeout):
+    """Stream to a temp file. -> (path, size, sha256). Raises on refusal."""
+    headers = {'User-Agent': USER_AGENT, 'Accept': 'application/pdf,*/*'}
+    digest = hashlib.sha256()
+    size = 0
+
+    handle, temp_path = tempfile.mkstemp(suffix='.pdf')
+    os.close(handle)
+
+    try:
+        with requests.get(url, headers=headers, stream=True,
+                          timeout=timeout) as response:
+            response.raise_for_status()
+
+            content_type = (response.headers.get('Content-Type') or '').lower()
+            declared = response.headers.get('Content-Length')
+            if declared and declared.isdigit() and int(declared) > max_bytes:
+                raise ValueError(
+                    f'file is {int(declared) // (1024 * 1024)} MB, over the limit')
+
+            first = True
+            with open(temp_path, 'wb') as out:
+                for chunk in response.iter_content(chunk_size=64 * 1024):
+                    if not chunk:
+                        continue
+                    if first:
+                        if not chunk.startswith(b'%PDF'):
+                            raise ValueError(
+                                f'not a PDF (content-type {content_type or "unknown"})')
+                        first = False
+                    size += len(chunk)
+                    if size > max_bytes:
+                        raise ValueError('file exceeded the size limit mid-download')
+                    digest.update(chunk)
+                    out.write(chunk)
+
+        if size == 0:
+            raise ValueError('empty response')
+        return temp_path, size, digest.hexdigest()
+
+    except Exception:
+        if os.path.exists(temp_path):
+            os.remove(temp_path)
+        raise
+
+
+def _existing_hashes(set_id):
+    rows = db.session.query(Instruction.sha256).filter(
+        Instruction.set_id == set_id, Instruction.sha256.isnot(None)).all()
+    return {row[0] for row in rows if row[0]}
+
+
+def _next_page_number(set_id):
+    from sqlalchemy import func
+    highest = db.session.query(func.max(Instruction.page_number)).filter_by(
+        set_id=set_id).scalar()
+    return (highest or 0) + 1
+
+
+def fetch_for_set(lego_set, delay_between_files=None):
+    """
+    Download every instruction Brickset lists for one official set.
+
+    Returns (added, skipped, source, message). Never raises for the normal
+    failure modes — the message explains what happened.
+    """
+    if lego_set.is_moc:
+        return 0, 0, None, 'MOC — nothing to look up'
+
+    if not BricksetAPI.is_configured():
+        raise RuntimeError('Brickset API credentials are not configured')
+
+    max_bytes = current_app.config.get('FETCH_MAX_FILE_MB', 150) * 1024 * 1024
+    timeout = current_app.config.get('FETCH_TIMEOUT_SECONDS', 60)
+    pause = (file_delay_seconds() if delay_between_files is None
+             else delay_between_files)
+
+    api = BricksetAPI()
+    entries, source, note = _instruction_urls(api, lego_set)
+
+    if not entries:
+        return 0, 0, source, note or 'no instructions listed on Brickset'
+
+    known = _existing_hashes(lego_set.id)
+    page = _next_page_number(lego_set.id)
+    added = 0
+    skipped = 0
+    problems = []
+
+    for index, entry in enumerate(entries, start=1):
+        if index > 1 and pause:
+            time.sleep(pause)
+
+        try:
+            temp_path, size, sha = _download_pdf(entry['url'], max_bytes,
+                                                 timeout)
+        except Exception as exc:
+            problems.append(f'{entry["description"] or entry["url"]}: {exc}')
+            continue
+
+        if sha in known:
+            os.remove(temp_path)
+            skipped += 1
+            continue
+
+        try:
+            filename = _safe_name(lego_set, entry['description'], index)
+            stored = _LocalFile(temp_path, filename)
+            rel_path, file_size, thumb = FileHandler.save_file(
+                stored, lego_set.set_number, 'PDF')
+
+            instruction = Instruction(
+                set_id=lego_set.id,
+                file_type='PDF',
+                file_path=rel_path,
+                file_name=filename,
+                file_size=file_size or size,
+                thumbnail_path=thumb,
+                page_number=page,
+                sha256=sha,
+                source=source,
+                user_id=lego_set.user_id,
+            )
+            db.session.add(instruction)
+            db.session.commit()
+
+            known.add(sha)
+            page += 1
+            added += 1
+
+        except Exception as exc:
+            db.session.rollback()
+            if os.path.exists(temp_path):
+                os.remove(temp_path)
+            problems.append(f'{filename}: saving failed — {exc}')
+
+    parts = []
+    if added:
+        parts.append(f'{added} file(s) downloaded')
+    if skipped:
+        parts.append(f'{skipped} already on file')
+    if problems:
+        parts.append('; '.join(problems[:3]))
+    if note:
+        parts.append(note)
+
+    return added, skipped, source, ' | '.join(parts) or 'nothing to do'
+
+
+# ---------------------------------------------------------------------------
+# the queue runner
+# ---------------------------------------------------------------------------
+
+def _claim_next_job():
+    """Take the oldest eligible job and mark it running."""
+    now = datetime.utcnow()
+    job = InstructionFetchJob.query.filter(
+        InstructionFetchJob.status == 'queued',
+        db.or_(InstructionFetchJob.not_before.is_(None),
+               InstructionFetchJob.not_before <= now),
+    ).order_by(InstructionFetchJob.queued_at).first()
+
+    if job is None:
+        return None
+
+    job.status = 'running'
+    job.started_at = now
+    job.attempts = (job.attempts or 0) + 1
+    db.session.commit()
+    return job
+
+
+def run_one(force=False):
+    """
+    Process a single queued set, if the interval has elapsed.
+
+    Returns a dict describing what happened, or None when there was nothing
+    to do (disabled, not due yet, or an empty queue).
+    """
+    if not force and not is_enabled():
+        return None
+    if not force and not is_due():
+        return None
+
+    job = _claim_next_job()
+    if job is None:
+        return None
+
+    # Claim the slot up front: even a failure counts as network contact, so
+    # the next attempt still waits a full interval.
+    _mark_ran()
+
+    lego_set = job.lego_set
+    if lego_set is None:
+        job.status = 'cancelled'
+        job.message = 'set no longer exists'
+        job.finished_at = datetime.utcnow()
+        db.session.commit()
+        return job.to_dict()
+
+    if skip_sets_with_pdfs() and lego_set.instructions.filter_by(
+            file_type='PDF').count():
+        job.status = 'done'
+        job.message = 'skipped — set already has PDF instructions'
+        job.finished_at = datetime.utcnow()
+        db.session.commit()
+        return job.to_dict()
+
+    try:
+        added, skipped, source, message = fetch_for_set(lego_set)
+        job.files_added = added
+        job.source = source
+        job.message = message
+        if added:
+            job.status = 'done'
+        elif skipped:
+            job.status = 'done'
+        else:
+            job.status = 'empty'
+
+    except Exception as exc:
+        current_app.logger.exception('Instruction fetch failed for %s',
+                                     lego_set.set_number)
+        job.message = str(exc)[:500]
+        if job.attempts >= max_attempts():
+            job.status = 'failed'
+        else:
+            # Back off a couple of intervals before trying again.
+            job.status = 'queued'
+            job.not_before = datetime.utcnow() + timedelta(
+                minutes=interval_minutes() * 2)
+
+    job.finished_at = datetime.utcnow()
+    db.session.commit()
+
+    current_app.logger.info('Instruction fetch %s for %s: %s', job.status,
+                            lego_set.set_number, job.message)
+    return job.to_dict()
+
+
+def status_summary():
+    """Everything the admin page and its poller need, in one dict."""
+    counts = InstructionFetchJob.counts()
+    configured = BricksetAPI.is_configured()
+    recent = InstructionFetchJob.query.order_by(
+        InstructionFetchJob.id.desc()).limit(25).all()
+
+    return {
+        'enabled': is_enabled(),
+        'brickset_configured': configured,
+        'interval_minutes': interval_minutes(),
+        'file_delay_seconds': file_delay_seconds(),
+        'max_attempts': max_attempts(),
+        'skip_sets_with_pdfs': skip_sets_with_pdfs(),
+        'allow_setid_lookup': allow_setid_lookup(),
+        'last_run': last_run().isoformat() if last_run() else None,
+        'next_run_in_seconds': seconds_until_due(),
+        'counts': counts,
+        'official_sets': official_sets_query().count(),
+        'sets_missing_pdfs': sets_missing_pdfs().count(),
+        'jobs': [job.to_dict() for job in recent],
+    }

app/services/schema_upgrade.py

@@ -24,8 +24,18 @@ ADDITIVE_COLUMNS = {
         # site default instead.
         'visibility': "VARCHAR(10) NOT NULL DEFAULT 'public'",
     },
+    'instructions': {
+        # Written by the automatic fetcher; null on everything uploaded
+        # before it existed, which is fine - dedupe just can't help there.
+        'sha256': 'VARCHAR(64)',
+        'source': 'VARCHAR(20)',
+    },
 }
 
+# Tables added by later versions. create_all() is not called at boot, so new
+# tables need an explicit (and idempotent) nudge.
+NEW_TABLES = ('app.models.instruction_fetch.InstructionFetchJob',)
+
 
 def ensure_schema(app):
     """Add any missing columns. Returns a list of changes made."""
@@ -38,6 +48,20 @@ def ensure_schema(app):
             app.logger.warning('Schema check skipped: %s', exc)
             return applied
 
+        for dotted in NEW_TABLES:
+            module_path, _, class_name = dotted.rpartition('.')
+            try:
+                module = __import__(module_path, fromlist=[class_name])
+                model = getattr(module, class_name)
+                if model.__tablename__ not in existing_tables:
+                    model.__table__.create(db.engine, checkfirst=True)
+                    applied.append(f'table {model.__tablename__}')
+                    app.logger.info('Schema upgrade: created table %s',
+                                    model.__tablename__)
+            except Exception as exc:
+                app.logger.error('Schema upgrade failed for %s: %s',
+                                 dotted, exc)
+
         for table, columns in ADDITIVE_COLUMNS.items():
             if table not in existing_tables:
                 continue  # create_all() will build it with everything present

app/templates/admin/dashboard.html

@@ -255,6 +255,11 @@
             </div>
             <div class="card-body">
                 <div class="row">
+                    <div class="col-md-3 mb-2">
+                        <a href="{{ url_for('instruction_fetch.index') }}" class="btn btn-outline-warning w-100">
+                            <i class="bi bi-cloud-arrow-down"></i> Instruction Fetcher
+                        </a>
+                    </div>
                     <div class="col-md-3 mb-2">
                         <a href="{{ url_for('admin.users') }}" class="btn btn-outline-primary w-100">
                             <i class="bi bi-people"></i> Manage Users

app/templates/admin/instruction_fetch.html

@@ -0,0 +1,352 @@
+{# Instruction fetcher control panel.
+   Created by: Jess Rogerson (yelling commands at Claude.AI) #}
+{% extends "base.html" %}
+
+{% block title %}Instruction Fetcher - Admin{% endblock %}
+
+{% block content %}
+<div class="row mb-4">
+    <div class="col">
+        <h1><i class="bi bi-cloud-arrow-down"></i> Instruction Fetcher</h1>
+        <p class="text-muted mb-0">
+            Downloads official building instructions from LEGO, using Brickset
+            to find them. Official sets only &mdash; one set every
+            {{ summary.interval_minutes }} minutes so nobody's API gets upset.
+        </p>
+    </div>
+    <div class="col-auto">
+        <a href="{{ url_for('admin.dashboard') }}" class="btn btn-outline-secondary">
+            <i class="bi bi-arrow-left"></i> Back
+        </a>
+    </div>
+</div>
+
+{% if not summary.brickset_configured %}
+<div class="alert alert-danger">
+    <i class="bi bi-exclamation-triangle"></i>
+    <strong>Brickset API not configured.</strong>
+    Add <code>BRICKSET_API_KEY</code>, <code>BRICKSET_USERNAME</code> and
+    <code>BRICKSET_PASSWORD</code> to your <code>.env</code> and restart.
+</div>
+{% endif %}
+
+{% if not worker_running %}
+<div class="alert alert-warning">
+    <i class="bi bi-info-circle"></i>
+    The background worker thread isn't running in this process. Queued sets
+    will only move when you press <em>Run one now</em>, or when
+    <code>flask instructions run</code> is called from Task Scheduler.
+</div>
+{% endif %}
+
+<!-- Status strip -->
+<div class="row g-3 mb-4">
+    <div class="col-md-3">
+        <div class="card h-100">
+            <div class="card-body">
+                <div class="text-muted small text-uppercase">Status</div>
+                <div class="h4 mb-0" id="stat-status">
+                    {% if summary.enabled %}
+                        <span class="badge bg-success">Running</span>
+                    {% else %}
+                        <span class="badge bg-secondary">Paused</span>
+                    {% endif %}
+                </div>
+            </div>
+        </div>
+    </div>
+    <div class="col-md-3">
+        <div class="card h-100">
+            <div class="card-body">
+                <div class="text-muted small text-uppercase">Next fetch in</div>
+                <div class="h4 mb-0" id="stat-countdown">&mdash;</div>
+            </div>
+        </div>
+    </div>
+    <div class="col-md-3">
+        <div class="card h-100">
+            <div class="card-body">
+                <div class="text-muted small text-uppercase">In the queue</div>
+                <div class="h4 mb-0" id="stat-queued">{{ summary.counts.queued }}</div>
+            </div>
+        </div>
+    </div>
+    <div class="col-md-3">
+        <div class="card h-100">
+            <div class="card-body">
+                <div class="text-muted small text-uppercase">Official sets without a PDF</div>
+                <div class="h4 mb-0" id="stat-missing">{{ summary.sets_missing_pdfs }}</div>
+            </div>
+        </div>
+    </div>
+</div>
+
+<div class="row">
+    <!-- Queue controls -->
+    <div class="col-lg-6 mb-4">
+        <div class="card h-100">
+            <div class="card-header">
+                <h5 class="mb-0"><i class="bi bi-list-ol"></i> Build the queue</h5>
+            </div>
+            <div class="card-body">
+                <p class="text-muted">
+                    {{ summary.official_sets }} official set(s) in the library,
+                    {{ summary.sets_missing_pdfs }} with no PDF instructions yet.
+                </p>
+
+                <form method="POST" action="{{ url_for('instruction_fetch.queue') }}"
+                      class="mb-3">
+                    <input type="hidden" name="scope" value="missing">
+                    <div class="input-group">
+                        <span class="input-group-text">Limit</span>
+                        <input type="number" class="form-control" name="limit"
+                               min="1" placeholder="all" style="max-width: 120px;">
+                        <button type="submit" class="btn btn-primary">
+                            <i class="bi bi-plus-circle"></i> Queue sets missing instructions
+                        </button>
+                    </div>
+                    <div class="form-text">Newest sets first. Skips anything already queued.</div>
+                </form>
+
+                <form method="POST" action="{{ url_for('instruction_fetch.queue') }}"
+                      class="mb-3"
+                      onsubmit="return confirm('Queue every official set, including ones that already have instructions?');">
+                    <input type="hidden" name="scope" value="all">
+                    <button type="submit" class="btn btn-outline-secondary btn-sm">
+                        <i class="bi bi-arrow-repeat"></i> Queue all official sets
+                    </button>
+                </form>
+
+                <form method="POST" action="{{ url_for('instruction_fetch.queue') }}">
+                    <input type="hidden" name="scope" value="retry-failed">
+                    <button type="submit" class="btn btn-outline-warning btn-sm">
+                        <i class="bi bi-arrow-counterclockwise"></i>
+                        Retry failed ({{ summary.counts.failed }})
+                    </button>
+                </form>
+
+                <hr>
+
+                <form method="POST" action="{{ url_for('instruction_fetch.run_now') }}">
+                    <div class="form-check mb-2">
+                        <input class="form-check-input" type="checkbox"
+                               id="force" name="force" value="1">
+                        <label class="form-check-label" for="force">
+                            Ignore the timer for this one
+                        </label>
+                    </div>
+                    <button type="submit" class="btn btn-success"
+                            {% if not summary.brickset_configured %}disabled{% endif %}>
+                        <i class="bi bi-play-fill"></i> Run one now
+                    </button>
+                </form>
+            </div>
+        </div>
+    </div>
+
+    <!-- Settings -->
+    <div class="col-lg-6 mb-4">
+        <div class="card h-100">
+            <div class="card-header">
+                <h5 class="mb-0"><i class="bi bi-sliders"></i> Pacing</h5>
+            </div>
+            <div class="card-body">
+                <form method="POST" action="{{ url_for('instruction_fetch.save_settings') }}">
+                    <div class="form-check form-switch mb-3">
+                        <input class="form-check-input" type="checkbox" role="switch"
+                               id="fetch_enabled" name="fetch_enabled" value="1"
+                               {% if summary.enabled %}checked{% endif %}>
+                        <label class="form-check-label" for="fetch_enabled">
+                            <strong>Work through the queue automatically</strong>
+                        </label>
+                        <div class="form-text">
+                            Turn off to pause. The queue is kept either way.
+                        </div>
+                    </div>
+
+                    <div class="mb-3">
+                        <label class="form-label" for="fetch_interval_minutes">
+                            <strong>Minutes between sets</strong>
+                        </label>
+                        <input type="number" class="form-control" min="1" max="1440"
+                               id="fetch_interval_minutes" name="fetch_interval_minutes"
+                               value="{{ summary.interval_minutes }}">
+                        <div class="form-text">
+                            10 is a polite default (144 sets a day). The clock is stored
+                            in the database, so a restart doesn't reset it.
+                        </div>
+                    </div>
+
+                    <div class="mb-3">
+                        <label class="form-label" for="fetch_file_delay_seconds">
+                            <strong>Seconds between PDFs of the same set</strong>
+                        </label>
+                        <input type="number" class="form-control" min="0" max="300"
+                               id="fetch_file_delay_seconds" name="fetch_file_delay_seconds"
+                               value="{{ summary.file_delay_seconds }}">
+                        <div class="form-text">
+                            Big sets have several books; this spaces those downloads out.
+                        </div>
+                    </div>
+
+                    <div class="mb-3">
+                        <label class="form-label" for="fetch_max_attempts">
+                            <strong>Attempts before giving up</strong>
+                        </label>
+                        <input type="number" class="form-control" min="1" max="10"
+                               id="fetch_max_attempts" name="fetch_max_attempts"
+                               value="{{ summary.max_attempts }}">
+                    </div>
+
+                    <div class="form-check mb-2">
+                        <input class="form-check-input" type="checkbox"
+                               id="fetch_skip_sets_with_pdfs" name="fetch_skip_sets_with_pdfs"
+                               value="1" {% if summary.skip_sets_with_pdfs %}checked{% endif %}>
+                        <label class="form-check-label" for="fetch_skip_sets_with_pdfs">
+                            Skip sets that already have a PDF
+                        </label>
+                    </div>
+
+                    <div class="form-check mb-3">
+                        <input class="form-check-input" type="checkbox"
+                               id="fetch_allow_setid_lookup" name="fetch_allow_setid_lookup"
+                               value="1" {% if summary.allow_setid_lookup %}checked{% endif %}>
+                        <label class="form-check-label" for="fetch_allow_setid_lookup">
+                            Allow set-ID lookups when a number isn't found
+                        </label>
+                        <div class="form-text">
+                            This is the only call that counts against the Brickset
+                            daily quota. The ID is saved to the set afterwards.
+                        </div>
+                    </div>
+
+                    <button type="submit" class="btn btn-primary">
+                        <i class="bi bi-save"></i> Save
+                    </button>
+                </form>
+            </div>
+        </div>
+    </div>
+</div>
+
+<!-- History -->
+<div class="card">
+    <div class="card-header d-flex justify-content-between align-items-center">
+        <h5 class="mb-0"><i class="bi bi-clock-history"></i> Recent jobs</h5>
+        <div>
+            <form method="POST" action="{{ url_for('instruction_fetch.clear') }}"
+                  class="d-inline">
+                <input type="hidden" name="scope" value="finished">
+                <button type="submit" class="btn btn-sm btn-outline-secondary">
+                    Clear finished
+                </button>
+            </form>
+        </div>
+    </div>
+    <div class="table-responsive">
+        <table class="table table-hover mb-0 align-middle">
+            <thead>
+                <tr>
+                    <th>Set</th>
+                    <th>Status</th>
+                    <th class="text-center">Files</th>
+                    <th class="text-center">Tries</th>
+                    <th>Result</th>
+                    <th></th>
+                </tr>
+            </thead>
+            <tbody>
+                {% for job in jobs %}
+                <tr>
+                    <td>
+                        {% if job.lego_set %}
+                        <a href="{{ url_for('sets.view_set', set_id=job.set_id) }}">
+                            <strong>{{ job.lego_set.set_number }}</strong>
+                        </a>
+                        <div class="small text-muted">{{ job.lego_set.set_name }}</div>
+                        {% else %}
+                        <span class="text-muted">deleted set</span>
+                        {% endif %}
+                    </td>
+                    <td><span class="badge bg-{{ job.badge }}">{{ job.status }}</span></td>
+                    <td class="text-center">{{ job.files_added }}</td>
+                    <td class="text-center">{{ job.attempts }}</td>
+                    <td class="small">{{ job.message or '—' }}</td>
+                    <td class="text-end">
+                        {% if job.is_open %}
+                        <form method="POST"
+                              action="{{ url_for('instruction_fetch.cancel_job', job_id=job.id) }}"
+                              class="d-inline">
+                            <button class="btn btn-sm btn-outline-danger">Cancel</button>
+                        </form>
+                        {% else %}
+                        <form method="POST"
+                              action="{{ url_for('instruction_fetch.retry_job', job_id=job.id) }}"
+                              class="d-inline">
+                            <button class="btn btn-sm btn-outline-secondary">Retry</button>
+                        </form>
+                        {% endif %}
+                    </td>
+                </tr>
+                {% else %}
+                <tr>
+                    <td colspan="6" class="text-center text-muted py-4">
+                        Nothing queued yet.
+                    </td>
+                </tr>
+                {% endfor %}
+            </tbody>
+        </table>
+    </div>
+</div>
+
+<script>
+(function () {
+    const statusUrl = "{{ url_for('instruction_fetch.status') }}";
+    let remaining = {{ summary.next_run_in_seconds }};
+    let enabled = {{ 'true' if summary.enabled else 'false' }};
+    let lastFinished = {{ summary.counts.done + summary.counts.empty + summary.counts.failed }};
+
+    function paintCountdown() {
+        const el = document.getElementById('stat-countdown');
+        if (!el) return;
+        if (!enabled) { el.textContent = 'paused'; return; }
+        if (remaining <= 0) { el.textContent = 'due now'; return; }
+        const m = Math.floor(remaining / 60);
+        const s = remaining % 60;
+        el.textContent = m + 'm ' + String(s).padStart(2, '0') + 's';
+    }
+
+    setInterval(function () {
+        if (remaining > 0) { remaining -= 1; }
+        paintCountdown();
+    }, 1000);
+
+    async function poll() {
+        try {
+            const res = await fetch(statusUrl, {headers: {'Accept': 'application/json'}});
+            if (!res.ok) return;
+            const data = await res.json();
+            remaining = data.next_run_in_seconds;
+            enabled = data.enabled;
+            document.getElementById('stat-queued').textContent = data.counts.queued;
+            document.getElementById('stat-missing').textContent = data.sets_missing_pdfs;
+            document.getElementById('stat-status').innerHTML = data.enabled
+                ? '<span class="badge bg-success">Running</span>'
+                : '<span class="badge bg-secondary">Paused</span>';
+            paintCountdown();
+
+            // Reload once a job finishes so the table stays honest.
+            const finished = data.counts.done + data.counts.empty + data.counts.failed;
+            if (finished !== lastFinished) {
+                lastFinished = finished;
+                window.location.reload();
+            }
+        } catch (err) { /* offline for a moment; try again next tick */ }
+    }
+
+    paintCountdown();
+    setInterval(poll, 15000);
+})();
+</script>
+{% endblock %}

app/templates/admin/settings.html

@@ -8,6 +8,9 @@
         <h1><i class="bi bi-sliders"></i> Site Settings</h1>
     </div>
     <div class="col-auto">
+        <a href="{{ url_for('instruction_fetch.index') }}" class="btn btn-outline-warning me-2">
+            <i class="bi bi-cloud-arrow-down"></i> Instruction Fetcher
+        </a>
         <a href="{{ url_for('backup.index') }}" class="btn btn-outline-danger me-2">
             <i class="bi bi-archive"></i> Backup &amp; Restore
         </a>

docker-compose.yml

@@ -51,6 +51,13 @@ services:
       BRICKSET_API_KEY: ${BRICKSET_API_KEY:-}
       BRICKSET_USERNAME: ${BRICKSET_USERNAME:-}
       BRICKSET_PASSWORD: ${BRICKSET_PASSWORD:-}
+      # Automatic instruction fetcher. Safe to leave on: gunicorn runs a
+      # single worker here, so there is exactly one ticker, and it does
+      # nothing until an admin enables it in the UI.
+      INSTRUCTION_FETCH_WORKER: ${INSTRUCTION_FETCH_WORKER:-1}
+      INSTRUCTION_FETCH_TICK_SECONDS: ${INSTRUCTION_FETCH_TICK_SECONDS:-20}
+      FETCH_MAX_FILE_MB: ${FETCH_MAX_FILE_MB:-150}
+      FETCH_TIMEOUT_SECONDS: ${FETCH_TIMEOUT_SECONDS:-60}
     volumes:
       - lim-data:/data
     ports:

docker/entrypoint.sh

@@ -52,7 +52,8 @@ fi
 # migration — see docs/DOCKER_DEPLOYMENT.md.
 if [ "${SKIP_DB_INIT:-0}" != "1" ]; then
     log "Ensuring schema exists..."
-    python - <<'PY'
+    # No fetcher thread in this short-lived bootstrap process.
+    INSTRUCTION_FETCH_WORKER=0 python - <<'PY'
 import os
 from app import create_app, db
 app = create_app(os.getenv('FLASK_ENV', 'production'))

docs/INSTRUCTION_FETCHER.md

@@ -0,0 +1,181 @@
+# Automatic Instruction Fetcher
+
+Downloads official building instruction PDFs for **official sets only**, one
+set at a time, on a timer you control. MOCs are never touched — they have no
+Brickset entry to look up.
+
+Admin → **Instruction Fetcher** (`/admin/instruction-fetch`).
+
+---
+
+## Where the files actually come from
+
+Brickset's API doesn't host the PDFs; it tells you where LEGO keeps them.
+
+```
+getInstructions2(setNumber)  ->  [{ URL, description }, ...]   (brickset.com)
+         |
+         v
+GET https://www.lego.com/.../6876-1.pdf                        (lego.com)
+```
+
+So "from Brickset or LEGO" is one flow: Brickset for the lookup, LEGO for the
+file. Saved PDFs land in the normal uploads tree
+(`uploads/pdfs/<set number>/`) with a thumbnail, exactly like a manual upload,
+and appear on the set page as ordinary instructions.
+
+### API quota
+
+Per Brickset's documentation, **only `getSets` counts against the daily key
+limit**. `getInstructions` and `getInstructions2` do not. The fetcher is built
+around that:
+
+| Call | When | Counts against quota |
+|------|------|----------------------|
+| `getInstructions` (by setID) | set already has `brickset_id` | no |
+| `getInstructions2` (by number) | normal path | no |
+| `getSets` | only if the number isn't found and set-ID lookup is enabled | **yes** |
+
+When a `getSets` lookup does happen, the resulting Brickset ID is written back
+to the set, so it never happens twice for the same set.
+
+---
+
+## Pacing
+
+One set per interval, 10 minutes by default (≈144 sets/day). Within a set,
+multi-book instructions are spaced by a few seconds each.
+
+The "last run" timestamp is stored in `site_settings`, not in memory, so:
+
+* restarting LIM doesn't reset the clock and let it stampede,
+* changing the interval takes effect on the next tick, no restart needed,
+* pausing keeps the queue intact.
+
+All of it is configurable on the page:
+
+| Setting | Default | Notes |
+|---------|---------|-------|
+| Work through the queue automatically | off | master switch; off = nothing happens |
+| Minutes between sets | 10 | 1–1440 |
+| Seconds between PDFs of the same set | 5 | 0–300 |
+| Attempts before giving up | 3 | failures back off two intervals, then retry |
+| Skip sets that already have a PDF | on | |
+| Allow set-ID lookups | on | the only quota-consuming call |
+
+---
+
+## Using it
+
+1. **Queue sets missing instructions** — newest first. An optional limit lets
+   you try 10 before committing to 900.
+2. Switch **Work through the queue automatically** on.
+3. Walk away. The page shows a live countdown and refreshes the job table as
+   each set finishes.
+
+**Run one now** processes a single set immediately in the background (tick
+*ignore the timer* to skip the cooldown). Useful for checking credentials
+before queueing hundreds.
+
+### Job statuses
+
+| Status | Meaning |
+|--------|---------|
+| `queued` | waiting its turn |
+| `running` | being fetched right now |
+| `done` | files saved, or already had them |
+| `empty` | Brickset lists no instructions for this set |
+| `failed` | gave up after the configured attempts |
+| `cancelled` | stopped by an admin, or the set was deleted |
+
+---
+
+## Safety rails
+
+* **PDF only.** The first bytes must be `%PDF`; anything else is rejected with
+  the reason recorded on the job.
+* **Size cap.** `FETCH_MAX_FILE_MB` (default 150) is checked both from the
+  `Content-Length` header and while streaming.
+* **Deduplication.** Every downloaded file is hashed (SHA-256, stored on
+  `instructions.sha256`). A file already on the set is skipped, so re-running
+  a set never creates duplicates.
+* **Files are only written after a successful download** — partial downloads
+  go to a temp file and are deleted.
+* **Official sets only**, enforced in the query, the queue, and the fetch
+  itself.
+
+---
+
+## Running it from Task Scheduler instead
+
+The background thread lives in the web process. If you'd rather drive it
+externally, set `INSTRUCTION_FETCH_WORKER=0` in `.env` and schedule:
+
+```bat
+flask instructions run --count 1 --force
+```
+
+Other commands:
+
+```bat
+flask instructions status                      :: queue counts + countdown
+flask instructions queue --missing --limit 50  :: queue sets with no PDF
+flask instructions queue --all                 :: queue every official set
+flask instructions run --count 3               :: respect the interval
+```
+
+`--force` ignores both the timer and the enabled switch, which is what you
+want when the schedule itself provides the pacing.
+
+> Run **one** worker. If the web process has the thread running *and* Task
+> Scheduler calls the CLI, two of them can claim jobs at once. The CLI won't
+> start a worker thread of its own, so `INSTRUCTION_FETCH_WORKER=0` is the
+> only switch you need.
+
+---
+
+## Environment variables
+
+```ini
+INSTRUCTION_FETCH_WORKER=1        # 0 to disable the in-process worker
+INSTRUCTION_FETCH_TICK_SECONDS=20 # how often it checks whether it's due
+FETCH_MAX_FILE_MB=150             # reject anything larger
+FETCH_TIMEOUT_SECONDS=60
+```
+
+Everything else is in the admin UI, not the environment.
+
+---
+
+## Database changes
+
+Applied automatically at startup by `services/schema_upgrade.py`; no manual
+migration:
+
+* new table `instruction_fetch_jobs`
+* new columns `instructions.sha256`, `instructions.source`
+
+Existing instruction rows keep `sha256 = NULL`, which simply means dedupe
+can't compare against them.
+
+---
+
+## Troubleshooting
+
+**"Brickset API not configured"** — `BRICKSET_API_KEY`, `BRICKSET_USERNAME`
+and `BRICKSET_PASSWORD` in `.env`, then restart.
+
+**Everything comes back `empty`** — check the set numbers. Brickset stores
+them as `{number}-{variant}`; the fetcher appends `-1` automatically, but an
+unusual variant (`-2`, `-3`) has to be in the set number itself.
+
+**"The background worker thread isn't running"** — either
+`INSTRUCTION_FETCH_WORKER=0`, or LIM is being served in a way that loads the
+app without starting it. *Run one now* still works.
+
+**Downloads fail with a connection error** — the server needs outbound HTTPS
+to `brickset.com` and `lego.com`.
+
+---
+
+Created by: Jess Rogerson (yelling commands at Claude.AI)

install_instruction_fetcher.bat

@@ -0,0 +1,211 @@
+@echo off
+REM ===========================================================================
+REM  LEGO Instructions Manager - Instruction Fetcher installer
+REM
+REM  RUN THIS FROM THE FOLDER YOU EXTRACTED THE UPDATE INTO, not from your
+REM  LIM install - the two cannot be the same folder.
+REM
+REM  Copies 22 named files into an existing LIM install, backing up every
+REM  file it replaces first. Whole directories are never copied, so your
+REM  uploads and database are not involved.
+REM
+REM  Usage:   install_instruction_fetcher.bat [target]
+REM  Default target: E:\LIM
+REM
+REM  Created by: Jess Rogerson (yelling commands at Claude.AI)
+REM ===========================================================================
+
+setlocal EnableDelayedExpansion
+
+set "SOURCE=%~dp0"
+set "TARGET=%~1"
+if "%TARGET%"=="" set "TARGET=E:\LIM"
+set "MISSING=0"
+set "COPYFAIL=0"
+set "SAVED=0"
+
+for %%A in ("%SOURCE%.") do set "SRCFULL=%%~fA"
+for %%B in ("%TARGET%\.") do set "DSTFULL=%%~fB"
+
+echo.
+echo  Source : %SRCFULL%
+echo  Target : %DSTFULL%
+echo.
+
+REM --- refuse to copy a folder onto itself -----------------------------------
+if /i "%SRCFULL%"=="%DSTFULL%" goto :samedir
+
+if not exist "%TARGET%\app\__init__.py" goto :nottarget
+
+REM --- are the update files actually here? -----------------------------------
+echo  Checking the update files...
+call :need ".env.docker.example"
+call :need ".env.example"
+call :need "README.md"
+call :need "app\__init__.py"
+call :need "app\cli.py"
+call :need "app\config.py"
+call :need "app\models\__init__.py"
+call :need "app\models\instruction.py"
+call :need "app\models\instruction_fetch.py"
+call :need "app\models\site_setting.py"
+call :need "app\routes\__init__.py"
+call :need "app\routes\instruction_fetch.py"
+call :need "app\services\brickset_api.py"
+call :need "app\services\fetch_worker.py"
+call :need "app\services\instruction_fetcher.py"
+call :need "app\services\schema_upgrade.py"
+call :need "app\templates\admin\dashboard.html"
+call :need "app\templates\admin\instruction_fetch.html"
+call :need "app\templates\admin\settings.html"
+call :need "docker-compose.yml"
+call :need "docker\entrypoint.sh"
+call :need "docs\INSTRUCTION_FETCHER.md"
+if not "%MISSING%"=="0" goto :incomplete
+echo    all 22 present.
+
+REM --- backup ------------------------------------------------------------------
+for /f "tokens=2 delims==" %%I in ('wmic os get localdatetime /value') do set "DT=%%I"
+set "STAMP=!DT:~0,8!-!DT:~8,6!"
+set "BACKUP=%TARGET%\_backup_fetcher_!STAMP!"
+
+echo.
+echo  Backing up the files about to be replaced:
+echo    !BACKUP!
+echo.
+mkdir "!BACKUP!" 2>nul
+call :save ".env.docker.example"
+call :save ".env.example"
+call :save "README.md"
+call :save "app\__init__.py"
+call :save "app\cli.py"
+call :save "app\config.py"
+call :save "app\models\__init__.py"
+call :save "app\models\instruction.py"
+call :save "app\models\instruction_fetch.py"
+call :save "app\models\site_setting.py"
+call :save "app\routes\__init__.py"
+call :save "app\routes\instruction_fetch.py"
+call :save "app\services\brickset_api.py"
+call :save "app\services\fetch_worker.py"
+call :save "app\services\instruction_fetcher.py"
+call :save "app\services\schema_upgrade.py"
+call :save "app\templates\admin\dashboard.html"
+call :save "app\templates\admin\instruction_fetch.html"
+call :save "app\templates\admin\settings.html"
+call :save "docker-compose.yml"
+call :save "docker\entrypoint.sh"
+call :save "docs\INSTRUCTION_FETCHER.md"
+echo    %SAVED% file^(s^) backed up.
+
+REM --- install -------------------------------------------------------------------
+echo.
+echo  Installing...
+call :put ".env.docker.example"
+call :put ".env.example"
+call :put "README.md"
+call :put "app\__init__.py"
+call :put "app\cli.py"
+call :put "app\config.py"
+call :put "app\models\__init__.py"
+call :put "app\models\instruction.py"
+call :put "app\models\instruction_fetch.py"
+call :put "app\models\site_setting.py"
+call :put "app\routes\__init__.py"
+call :put "app\routes\instruction_fetch.py"
+call :put "app\services\brickset_api.py"
+call :put "app\services\fetch_worker.py"
+call :put "app\services\instruction_fetcher.py"
+call :put "app\services\schema_upgrade.py"
+call :put "app\templates\admin\dashboard.html"
+call :put "app\templates\admin\instruction_fetch.html"
+call :put "app\templates\admin\settings.html"
+call :put "docker-compose.yml"
+call :put "docker\entrypoint.sh"
+call :put "docs\INSTRUCTION_FETCHER.md"
+if not "%COPYFAIL%"=="0" goto :copyfailed
+
+echo.
+echo  ==========================================================
+echo   Done - 22 files installed.
+echo.
+echo   No migration step: the new table and columns are created
+echo   the next time LIM starts.
+echo.
+echo     1. Restart LIM  (start_lim.bat)
+echo     2. Admin  ^>  Instruction Fetcher
+echo     3. Queue sets, then switch the fetcher on
+echo.
+echo   To push the same change to Gitea, run push_fetcher_update.bat
+echo   FROM THIS FOLDER (not from %TARGET%).
+echo.
+echo   Rollback: copy the files in
+echo     !BACKUP!
+echo   back over %TARGET%
+echo  ==========================================================
+echo.
+pause
+exit /b 0
+
+REM --- exits ---------------------------------------------------------------------
+
+:samedir
+echo  ERROR: the source and target are the same folder.
+echo.
+echo  You are running this script from inside your LIM install, so there
+echo  is nothing to copy - which is why an earlier run appeared to work
+echo  but changed nothing.
+echo.
+echo  Open the folder you extracted the update into (the one containing
+echo  app\services\instruction_fetcher.py) and run it from there:
+echo.
+echo      cd /d C:\path\to\lim-instruction-fetcher
+echo      install_instruction_fetcher.bat %TARGET%
+goto :fail
+
+:nottarget
+echo  ERROR: %TARGET% does not look like a LIM install
+echo         ^(no app\__init__.py^).
+echo.
+echo  Pass the right path, e.g.  install_instruction_fetcher.bat D:\LIM
+goto :fail
+
+:incomplete
+echo.
+echo  ERROR: %MISSING% file^(s^) missing from
+echo           %SRCFULL%
+echo.
+echo  This is not the extracted update folder. Look for the one
+echo  containing app\services\instruction_fetcher.py.
+goto :fail
+
+:copyfailed
+echo.
+echo  ERROR: %COPYFAIL% file^(s^) could not be copied. Is LIM still running?
+echo         Stop it and run this again; the backup in
+echo           !BACKUP!
+echo         has the originals.
+goto :fail
+
+:need
+if not exist "%SOURCE%%~1" echo    MISSING: %~1 & set /a MISSING+=1
+goto :eof
+
+:save
+if not exist "%TARGET%\%~1" goto :eof
+for %%F in ("!BACKUP!\%~1") do if not exist "%%~dpF" mkdir "%%~dpF"
+copy /Y "%TARGET%\%~1" "!BACKUP!\%~1" >nul
+if not errorlevel 1 echo    saved  %~1 & set /a SAVED+=1
+goto :eof
+
+:put
+for %%F in ("%TARGET%\%~1") do if not exist "%%~dpF" mkdir "%%~dpF"
+copy /Y "%SOURCE%%~1" "%TARGET%\%~1" >nul
+if errorlevel 1 echo    FAILED: %~1 & set /a COPYFAIL+=1
+goto :eof
+
+:fail
+echo.
+echo  Nothing was changed.
+pause
+exit /b 1