Add automatic instruction fetcher for official sets

Downloads official instruction PDFs from LEGO, located via the Brickset API, one set per interval (10 minutes by default) so neither API gets hammered. Official sets only; MOCs are excluded throughout.

- New admin page at /admin/instruction-fetch: queue builder, pacing controls, live countdown and per-set job history

- Database-backed queue (instruction_fetch_jobs) and last-run timestamp, so pacing survives restarts

- Background worker thread, plus 'flask instructions queue/run/status' for Task Scheduler or cron

- SHA-256 dedupe (instructions.sha256), PDF header check, size cap and streaming downloads via temp files

- brickset_api: getInstructions2 by set number and getInstructions by setID (the old code sent a set number to the setID method); getSets params now proper JSON; getKeyUsageStats added

- Schema top-ups applied automatically at boot: new table plus instructions.sha256/source

- Docker: fetcher env vars passed through compose, no worker thread in the entrypoint bootstrap

app/services/brickset_api.py

@@ -1,19 +1,39 @@
+"""
+Brickset API v3 client.
+
+https://brickset.com/article/52664/api-version-3-documentation
+
+Two things worth knowing about the API, both of which shape this file:
+
+  * getSets takes its search terms as a JSON string. Python's str() on a dict
+    produces single quotes, which is not JSON — hence json.dumps below.
+  * getInstructions takes a Brickset setID; getInstructions2 takes a set
+    number like "10497-1". They are different methods and passing a set
+    number to the first one quietly returns nothing.
+
+Only getSets counts against the daily key quota.
+
+Created by: Jess Rogerson (yelling commands at Claude.AI)
+"""
+
+import json
+from typing import Any, Dict, List, Optional
+
 import requests
 from flask import current_app
-from typing import Optional, List, Dict, Any
 
 
 class BricksetAPI:
     """Service for interacting with the Brickset API v3."""
-    
+
     BASE_URL = 'https://brickset.com/api/v3.asmx'
-    
+
     def __init__(self):
         self.api_key = current_app.config.get('BRICKSET_API_KEY')
         self.username = current_app.config.get('BRICKSET_USERNAME')
         self.password = current_app.config.get('BRICKSET_PASSWORD')
         self._user_hash = None
-    
+
     @staticmethod
     def is_configured() -> bool:
         """Check if Brickset API is properly configured."""
@@ -22,16 +42,16 @@ class BricksetAPI:
             current_app.config.get('BRICKSET_USERNAME') and
             current_app.config.get('BRICKSET_PASSWORD')
         )
-    
+
     def _get_user_hash(self) -> Optional[str]:
         """Authenticate and get user hash token."""
         if self._user_hash:
             return self._user_hash
-        
+
         if not all([self.api_key, self.username, self.password]):
             current_app.logger.warning('Brickset API credentials not configured')
             return None
-        
+
         try:
             response = requests.get(
                 f'{self.BASE_URL}/login',
@@ -44,47 +64,43 @@ class BricksetAPI:
             )
             response.raise_for_status()
             data = response.json()
-            
+
             if data.get('status') == 'success':
                 self._user_hash = data.get('hash')
                 return self._user_hash
             else:
                 current_app.logger.error(f"Brickset login failed: {data.get('message')}")
                 return None
-                
+
         except Exception as e:
             current_app.logger.error(f'Brickset API authentication error: {str(e)}')
             return None
-    
-    def search_sets(self, 
-                   set_number: Optional[str] = None,
-                   theme: Optional[str] = None,
-                   year: Optional[int] = None,
-                   query: Optional[str] = None) -> List[Dict[str, Any]]:
+
+    def search_sets(self,
+                    set_number: Optional[str] = None,
+                    theme: Optional[str] = None,
+                    year: Optional[int] = None,
+                    query: Optional[str] = None,
+                    page_size: Optional[int] = None) -> List[Dict[str, Any]]:
         """
         Search for LEGO sets using various parameters.
-        
+
         Args:
             set_number: Specific set number to search for
             theme: Theme name to filter by
             year: Year released
             query: General search query
-            
+            page_size: Number of records to return (max 500)
+
         Returns:
             List of set dictionaries
         """
         user_hash = self._get_user_hash()
         if not user_hash:
             return []
-        
-        params = {
-            'apiKey': self.api_key,
-            'userHash': user_hash,
-            'params': '{}'  # JSON params object
-        }
-        
+
         # Build search parameters
-        search_params = {}
+        search_params: Dict[str, Any] = {}
         if set_number:
             search_params['setNumber'] = set_number
         if theme:
@@ -93,9 +109,20 @@ class BricksetAPI:
             search_params['year'] = year
         if query:
             search_params['query'] = query
-        
-        params['params'] = str(search_params)
-        
+        if page_size:
+            search_params['pageSize'] = page_size
+
+        if not search_params:
+            return []
+
+        params = {
+            'apiKey': self.api_key,
+            'userHash': user_hash,
+            # Must be a JSON string — str(dict) gives single quotes and is
+            # rejected with "Parameter error".
+            'params': json.dumps(search_params),
+        }
+
         try:
             response = requests.get(
                 f'{self.BASE_URL}/getSets',
@@ -104,41 +131,49 @@ class BricksetAPI:
             )
             response.raise_for_status()
             data = response.json()
-            
+
             if data.get('status') == 'success':
                 return data.get('sets', [])
-            else:
-                current_app.logger.error(f"Brickset search failed: {data.get('message')}")
-                return []
-                
+
+            message = data.get('message', 'unknown error')
+            current_app.logger.error(f'Brickset search failed: {message}')
+            if 'limit exceeded' in message.lower():
+                raise RuntimeError('Brickset API limit exceeded')
+            return []
+
+        except RuntimeError:
+            raise
         except Exception as e:
             current_app.logger.error(f'Brickset API search error: {str(e)}')
             return []
-    
+
     def get_set_by_number(self, set_number: str) -> Optional[Dict[str, Any]]:
         """
         Get detailed information for a specific set by its number.
-        
+
         Args:
-            set_number: The LEGO set number (e.g., "10497")
-            
+            set_number: The LEGO set number (e.g., "10497" or "10497-1")
+
         Returns:
             Dictionary with set information or None
         """
         results = self.search_sets(set_number=set_number)
+        if not results and '-' not in set_number:
+            # Brickset stores numbers with a variant suffix.
+            results = self.search_sets(set_number=f'{set_number}-1')
         return results[0] if results else None
-    
+
     def get_themes(self) -> List[str]:
         """
         Get list of all available LEGO themes.
-        
+
         Returns:
             List of theme names
         """
         user_hash = self._get_user_hash()
         if not user_hash:
             return []
-        
+
         try:
             response = requests.get(
                 f'{self.BASE_URL}/getThemes',
@@ -150,49 +185,76 @@ class BricksetAPI:
             )
             response.raise_for_status()
             data = response.json()
-            
+
             if data.get('status') == 'success':
                 themes = data.get('themes', [])
                 return [theme.get('theme') for theme in themes if theme.get('theme')]
             else:
                 return []
-                
+
         except Exception as e:
             current_app.logger.error(f'Brickset API themes error: {str(e)}')
             return []
-    
-    def get_instructions(self, set_number: str) -> List[Dict[str, Any]]:
-        """
-        Get instruction information for a specific set.
-        
-        Args:
-            set_number: The LEGO set number
-            
-        Returns:
-            List of instruction dictionaries
-        """
-        user_hash = self._get_user_hash()
-        if not user_hash:
-            return []
-        
+
+    # -- instructions -------------------------------------------------------
+
+    def _instructions_request(self, method: str,
+                              params: Dict[str, Any]) -> List[Dict[str, Any]]:
+        params = dict(params, apiKey=self.api_key)
         try:
-            response = requests.get(
-                f'{self.BASE_URL}/getInstructions',
-                params={
-                    'apiKey': self.api_key,
-                    'userHash': user_hash,
-                    'setNumber': set_number
-                },
-                timeout=10
-            )
+            response = requests.get(f'{self.BASE_URL}/{method}',
+                                    params=params, timeout=20)
             response.raise_for_status()
             data = response.json()
-            
+
             if data.get('status') == 'success':
-                return data.get('instructions', [])
-            else:
-                return []
-                
+                return data.get('instructions', []) or []
+
+            current_app.logger.error(
+                f"Brickset {method} failed: {data.get('message')}")
+            return []
+
         except Exception as e:
-            current_app.logger.error(f'Brickset API instructions error: {str(e)}')
+            current_app.logger.error(f'Brickset {method} error: {str(e)}')
+            return []
+
+    def get_instructions(self, set_number: str) -> List[Dict[str, Any]]:
+        """
+        Instructions for a set number, e.g. "10497-1" (getInstructions2).
+
+        Returns a list of {'URL': ..., 'description': ...} dictionaries.
+        Does not need a user hash and does not count against the key quota.
+        """
+        if not self.api_key or not set_number:
+            return []
+        return self._instructions_request('getInstructions2',
+                                          {'setNumber': set_number})
+
+    def get_instructions_by_id(self, set_id: int) -> List[Dict[str, Any]]:
+        """Instructions for an internal Brickset set ID (getInstructions)."""
+        if not self.api_key or not set_id:
+            return []
+        return self._instructions_request('getInstructions', {'setID': set_id})
+
+    # -- quota --------------------------------------------------------------
+
+    def get_key_usage(self) -> List[Dict[str, Any]]:
+        """
+        Daily getSets usage for the last 30 days.
+
+        Returns a list of {'dateStamp': ..., 'count': ...} dictionaries.
+        """
+        if not self.api_key:
+            return []
+        try:
+            response = requests.get(f'{self.BASE_URL}/getKeyUsageStats',
+                                    params={'apiKey': self.api_key},
+                                    timeout=10)
+            response.raise_for_status()
+            data = response.json()
+            if data.get('status') == 'success':
+                return data.get('apiKeyUsage', []) or []
+            return []
+        except Exception as e:
+            current_app.logger.error(f'Brickset key usage error: {str(e)}')
             return []

app/services/fetch_worker.py

@@ -0,0 +1,109 @@
+"""
+The background ticker behind the instruction fetcher.
+
+A daemon thread wakes every few seconds, asks the fetcher whether the
+interval has elapsed, and if so processes exactly one set. All of the actual
+pacing lives in instruction_fetcher (and in the database), so pausing the
+worker or changing the interval from the admin page takes effect on the next
+tick without a restart.
+
+Deliberately a plain thread rather than a scheduler dependency: LIM runs
+single-process under waitress, and Python 3.14 has been quite enough
+dependency excitement for one project.
+
+Run one process. If you ever scale LIM horizontally, turn the worker off with
+INSTRUCTION_FETCH_WORKER=0 everywhere and drive it from cron/Task Scheduler
+with `flask instructions run` instead.
+
+Created by: Jess Rogerson (yelling commands at Claude.AI)
+"""
+
+import os
+import sys
+import threading
+
+_thread = None
+_stop = threading.Event()
+_lock = threading.Lock()
+
+
+def _is_cli_command():
+    """True when the app was loaded by `flask <cmd>` rather than to serve."""
+    if os.environ.get('FLASK_RUN_FROM_CLI') != 'true':
+        return False
+    args = [arg for arg in sys.argv[1:] if not arg.startswith('-')]
+    return bool(args) and args[0] != 'run'
+
+
+def start(app):
+    """Start the ticker unless it's disabled or already running."""
+    if not app.config.get('INSTRUCTION_FETCH_WORKER', True):
+        app.logger.info('Instruction fetch worker disabled by config')
+        return False
+
+    # `flask instructions run`, `flask backup restore` and friends shouldn't
+    # quietly spin up a fetcher of their own.
+    if _is_cli_command():
+        return False
+
+    # Flask's reloader imports the app twice in debug; only the child should
+    # own the thread, or two of them race for the same job.
+    if app.debug and os.environ.get('WERKZEUG_RUN_MAIN') != 'true':
+        return False
+
+    global _thread
+    with _lock:
+        if _thread is not None and _thread.is_alive():
+            return False
+        _stop.clear()
+        _thread = threading.Thread(target=_loop, args=(app,),
+                                   name='lim-instruction-fetcher',
+                                   daemon=True)
+        _thread.start()
+
+    app.logger.info('Instruction fetch worker started')
+    return True
+
+
+def stop(timeout=5):
+    """Ask the ticker to finish (used by tests and the CLI)."""
+    _stop.set()
+    thread = _thread
+    if thread is not None and thread.is_alive():
+        thread.join(timeout=timeout)
+
+
+def is_running():
+    return _thread is not None and _thread.is_alive()
+
+
+def _loop(app):
+    tick = app.config.get('INSTRUCTION_FETCH_TICK_SECONDS', 20)
+    from app.services import instruction_fetcher as fetcher
+
+    while not _stop.wait(tick):
+        try:
+            with app.app_context():
+                fetcher.run_one()
+        except Exception:
+            app.logger.exception('Instruction fetch worker tick failed')
+
+
+def run_once_async(app, force=False):
+    """
+    Fire off a single job in the background.
+
+    Used by the admin page's "Run one now" button so the request returns
+    straight away instead of holding the browser open for a 40 MB PDF.
+    """
+    def _work():
+        try:
+            with app.app_context():
+                from app.services import instruction_fetcher as fetcher
+                fetcher.run_one(force=force)
+        except Exception:
+            app.logger.exception('Manual instruction fetch failed')
+
+    thread = threading.Thread(target=_work, name='lim-fetch-once', daemon=True)
+    thread.start()
+    return thread

app/services/instruction_fetcher.py

@@ -0,0 +1,562 @@
+"""
+Automatic instruction downloads for official sets.
+
+Brickset's getInstructions/getInstructions2 methods hand back the URLs of the
+official PDFs, which live on LEGO's own servers. So "download from Brickset or
+LEGO" is really one flow: ask Brickset where the file is, then pull the PDF
+from LEGO.
+
+The whole point of this module is politeness. Work is queued in the database
+and exactly one set is processed per interval (10 minutes by default), with a
+short pause between the individual PDFs of a multi-book set. Nothing here ever
+loops over the whole library in one go.
+
+Notes on quotas:
+  * Only getSets counts against a Brickset API key's daily limit, so the
+    set-ID lookup is the expensive call and is skipped whenever the set
+    already has a brickset_id stored.
+  * getInstructions2 needs nothing but the API key and a set number.
+
+Created by: Jess Rogerson (yelling commands at Claude.AI)
+"""
+
+import hashlib
+import os
+import re
+import tempfile
+import time
+from datetime import datetime, timedelta
+
+import requests
+from flask import current_app
+
+from app import db
+from app.models.instruction import Instruction
+from app.models.instruction_fetch import InstructionFetchJob
+from app.models.set import Set
+from app.models.site_setting import SiteSetting
+from app.services.brickset_api import BricksetAPI
+from app.services.file_handler import FileHandler
+
+USER_AGENT = ('LegoInstructionsManager/1.0 '
+              '(+https://gitea.hideawaygaming.com.au/jessikitty/'
+              'lego-instructions-manager)')
+
+# Settings surfaced on the admin page. Values are strings in site_settings.
+SETTING_DEFAULTS = {
+    'fetch_enabled': 'false',
+    'fetch_interval_minutes': '10',
+    'fetch_file_delay_seconds': '5',
+    'fetch_max_attempts': '3',
+    'fetch_skip_sets_with_pdfs': 'true',
+    'fetch_allow_setid_lookup': 'true',
+    'fetch_last_run': '',
+}
+
+MIN_INTERVAL_MINUTES = 1
+MAX_INTERVAL_MINUTES = 1440
+
+
+# ---------------------------------------------------------------------------
+# settings
+# ---------------------------------------------------------------------------
+
+def _int_setting(key, fallback, low=None, high=None):
+    raw = SiteSetting.get(key, SETTING_DEFAULTS.get(key))
+    try:
+        value = int(str(raw).strip())
+    except (TypeError, ValueError):
+        value = fallback
+    if low is not None:
+        value = max(low, value)
+    if high is not None:
+        value = min(high, value)
+    return value
+
+
+def is_enabled():
+    return SiteSetting.get_bool('fetch_enabled', False)
+
+
+def interval_minutes():
+    return _int_setting('fetch_interval_minutes', 10,
+                        MIN_INTERVAL_MINUTES, MAX_INTERVAL_MINUTES)
+
+
+def file_delay_seconds():
+    return _int_setting('fetch_file_delay_seconds', 5, 0, 300)
+
+
+def max_attempts():
+    return _int_setting('fetch_max_attempts', 3, 1, 10)
+
+
+def skip_sets_with_pdfs():
+    return SiteSetting.get_bool('fetch_skip_sets_with_pdfs', True)
+
+
+def allow_setid_lookup():
+    return SiteSetting.get_bool('fetch_allow_setid_lookup', True)
+
+
+def last_run():
+    raw = SiteSetting.get('fetch_last_run', '')
+    if not raw:
+        return None
+    try:
+        return datetime.fromisoformat(raw)
+    except ValueError:
+        return None
+
+
+def _mark_ran(when=None):
+    SiteSetting.set('fetch_last_run', (when or datetime.utcnow()).isoformat())
+    db.session.commit()
+
+
+def next_run_at():
+    """When the worker is next allowed to touch the network."""
+    previous = last_run()
+    if previous is None:
+        return datetime.utcnow()
+    return previous + timedelta(minutes=interval_minutes())
+
+
+def seconds_until_due():
+    return max(0, int((next_run_at() - datetime.utcnow()).total_seconds()))
+
+
+def is_due():
+    return seconds_until_due() <= 0
+
+
+# ---------------------------------------------------------------------------
+# queueing
+# ---------------------------------------------------------------------------
+
+def official_sets_query():
+    """Official sets only — MOCs have nothing to look up."""
+    return Set.query.filter(Set.is_moc.is_(False))
+
+
+def sets_missing_pdfs():
+    """Official sets with no PDF instruction on file."""
+    have_pdfs = db.session.query(Instruction.set_id).filter(
+        Instruction.file_type == 'PDF').distinct()
+    return official_sets_query().filter(~Set.id.in_(have_pdfs))
+
+
+def queue_sets(sets, user_id=None):
+    """Queue an iterable of sets, skipping any already waiting. -> count."""
+    queued = 0
+    for lego_set in sets:
+        if lego_set.is_moc:
+            continue
+        if InstructionFetchJob.open_for_set(lego_set.id):
+            continue
+        db.session.add(InstructionFetchJob(set_id=lego_set.id,
+                                           queued_by_id=user_id))
+        queued += 1
+    if queued:
+        db.session.commit()
+    return queued
+
+
+def queue_missing(user_id=None, limit=None):
+    query = sets_missing_pdfs().order_by(Set.year_released.desc(),
+                                         Set.set_number)
+    if limit:
+        query = query.limit(limit)
+    return queue_sets(query.all(), user_id)
+
+
+def queue_all_official(user_id=None, limit=None):
+    query = official_sets_query().order_by(Set.year_released.desc(),
+                                           Set.set_number)
+    if limit:
+        query = query.limit(limit)
+    return queue_sets(query.all(), user_id)
+
+
+def requeue_failed(user_id=None):
+    """Put every failed job back at the end of the queue."""
+    failed = InstructionFetchJob.query.filter_by(status='failed').all()
+    count = 0
+    for job in failed:
+        if InstructionFetchJob.open_for_set(job.set_id):
+            continue
+        job.status = 'queued'
+        job.attempts = 0
+        job.message = None
+        job.not_before = None
+        job.queued_at = datetime.utcnow()
+        job.queued_by_id = user_id or job.queued_by_id
+        count += 1
+    if count:
+        db.session.commit()
+    return count
+
+
+def clear_jobs(scope='finished'):
+    """Tidy the history. scope: finished | failed | all."""
+    query = InstructionFetchJob.query
+    if scope == 'finished':
+        query = query.filter(InstructionFetchJob.status.in_(
+            ('done', 'empty', 'cancelled')))
+    elif scope == 'failed':
+        query = query.filter_by(status='failed')
+    elif scope != 'all':
+        return 0
+    removed = query.delete(synchronize_session=False)
+    db.session.commit()
+    return removed
+
+
+# ---------------------------------------------------------------------------
+# the actual fetching
+# ---------------------------------------------------------------------------
+
+def brickset_number(set_number):
+    """Brickset wants {number}-{variant}; the library usually stores {number}."""
+    number = (set_number or '').strip()
+    if not number:
+        return None
+    if re.search(r'-\d+$', number):
+        return number
+    return f'{number}-1'
+
+
+def _instruction_urls(api, lego_set):
+    """(list of {url, description}, source, note)."""
+    note = None
+
+    if lego_set.brickset_id:
+        entries = api.get_instructions_by_id(lego_set.brickset_id)
+        if entries:
+            return _normalise(entries), 'brickset', note
+
+    number = brickset_number(lego_set.set_number)
+    if number:
+        entries = api.get_instructions(number)
+        if entries:
+            return _normalise(entries), 'brickset', note
+
+    # Last resort: resolve the Brickset set ID, then ask again. This is the
+    # only call that counts against the daily API key quota, so it is both
+    # optional and cached back onto the set for next time.
+    if allow_setid_lookup() and not lego_set.brickset_id and number:
+        data = api.get_set_by_number(number)
+        if data and data.get('setID'):
+            lego_set.brickset_id = data['setID']
+            db.session.commit()
+            entries = api.get_instructions_by_id(lego_set.brickset_id)
+            if entries:
+                return _normalise(entries), 'brickset', 'resolved via getSets'
+            note = 'set found, no instructions listed'
+        else:
+            note = 'set number not found on Brickset'
+
+    return [], 'brickset', note
+
+
+def _normalise(entries):
+    out = []
+    for entry in entries or []:
+        url = entry.get('URL') or entry.get('url')
+        if not url:
+            continue
+        out.append({
+            'url': url,
+            'description': (entry.get('description')
+                            or entry.get('Description') or ''),
+        })
+    return out
+
+
+def _safe_name(lego_set, description, index):
+    """A readable file name, e.g. '10497-building-instructions-book-1.pdf'."""
+    number = re.sub(r'[^A-Za-z0-9\-]+', '', lego_set.set_number or 'set')
+    base = re.sub(r'[^A-Za-z0-9]+', '-', description or '').strip('-').lower()
+    base = re.sub(r'-{2,}', '-', base)[:80].strip('-')
+    if not base:
+        return f'{number}-instructions-{index}.pdf'
+    # Brickset descriptions usually lead with the set number already.
+    if base.startswith(number.lower()):
+        return f'{base}.pdf'
+    return f'{number}-{base}.pdf'
+
+
+class _LocalFile:
+    """Adapter so FileHandler.save_file can take a file we downloaded."""
+
+    def __init__(self, path, filename):
+        self._path = path
+        self.filename = filename
+
+    def save(self, destination):
+        import shutil
+        shutil.move(self._path, destination)
+
+
+def _download_pdf(url, max_bytes, timeout):
+    """Stream to a temp file. -> (path, size, sha256). Raises on refusal."""
+    headers = {'User-Agent': USER_AGENT, 'Accept': 'application/pdf,*/*'}
+    digest = hashlib.sha256()
+    size = 0
+
+    handle, temp_path = tempfile.mkstemp(suffix='.pdf')
+    os.close(handle)
+
+    try:
+        with requests.get(url, headers=headers, stream=True,
+                          timeout=timeout) as response:
+            response.raise_for_status()
+
+            content_type = (response.headers.get('Content-Type') or '').lower()
+            declared = response.headers.get('Content-Length')
+            if declared and declared.isdigit() and int(declared) > max_bytes:
+                raise ValueError(
+                    f'file is {int(declared) // (1024 * 1024)} MB, over the limit')
+
+            first = True
+            with open(temp_path, 'wb') as out:
+                for chunk in response.iter_content(chunk_size=64 * 1024):
+                    if not chunk:
+                        continue
+                    if first:
+                        if not chunk.startswith(b'%PDF'):
+                            raise ValueError(
+                                f'not a PDF (content-type {content_type or "unknown"})')
+                        first = False
+                    size += len(chunk)
+                    if size > max_bytes:
+                        raise ValueError('file exceeded the size limit mid-download')
+                    digest.update(chunk)
+                    out.write(chunk)
+
+        if size == 0:
+            raise ValueError('empty response')
+        return temp_path, size, digest.hexdigest()
+
+    except Exception:
+        if os.path.exists(temp_path):
+            os.remove(temp_path)
+        raise
+
+
+def _existing_hashes(set_id):
+    rows = db.session.query(Instruction.sha256).filter(
+        Instruction.set_id == set_id, Instruction.sha256.isnot(None)).all()
+    return {row[0] for row in rows if row[0]}
+
+
+def _next_page_number(set_id):
+    from sqlalchemy import func
+    highest = db.session.query(func.max(Instruction.page_number)).filter_by(
+        set_id=set_id).scalar()
+    return (highest or 0) + 1
+
+
+def fetch_for_set(lego_set, delay_between_files=None):
+    """
+    Download every instruction Brickset lists for one official set.
+
+    Returns (added, skipped, source, message). Never raises for the normal
+    failure modes — the message explains what happened.
+    """
+    if lego_set.is_moc:
+        return 0, 0, None, 'MOC — nothing to look up'
+
+    if not BricksetAPI.is_configured():
+        raise RuntimeError('Brickset API credentials are not configured')
+
+    max_bytes = current_app.config.get('FETCH_MAX_FILE_MB', 150) * 1024 * 1024
+    timeout = current_app.config.get('FETCH_TIMEOUT_SECONDS', 60)
+    pause = (file_delay_seconds() if delay_between_files is None
+             else delay_between_files)
+
+    api = BricksetAPI()
+    entries, source, note = _instruction_urls(api, lego_set)
+
+    if not entries:
+        return 0, 0, source, note or 'no instructions listed on Brickset'
+
+    known = _existing_hashes(lego_set.id)
+    page = _next_page_number(lego_set.id)
+    added = 0
+    skipped = 0
+    problems = []
+
+    for index, entry in enumerate(entries, start=1):
+        if index > 1 and pause:
+            time.sleep(pause)
+
+        try:
+            temp_path, size, sha = _download_pdf(entry['url'], max_bytes,
+                                                 timeout)
+        except Exception as exc:
+            problems.append(f'{entry["description"] or entry["url"]}: {exc}')
+            continue
+
+        if sha in known:
+            os.remove(temp_path)
+            skipped += 1
+            continue
+
+        try:
+            filename = _safe_name(lego_set, entry['description'], index)
+            stored = _LocalFile(temp_path, filename)
+            rel_path, file_size, thumb = FileHandler.save_file(
+                stored, lego_set.set_number, 'PDF')
+
+            instruction = Instruction(
+                set_id=lego_set.id,
+                file_type='PDF',
+                file_path=rel_path,
+                file_name=filename,
+                file_size=file_size or size,
+                thumbnail_path=thumb,
+                page_number=page,
+                sha256=sha,
+                source=source,
+                user_id=lego_set.user_id,
+            )
+            db.session.add(instruction)
+            db.session.commit()
+
+            known.add(sha)
+            page += 1
+            added += 1
+
+        except Exception as exc:
+            db.session.rollback()
+            if os.path.exists(temp_path):
+                os.remove(temp_path)
+            problems.append(f'{filename}: saving failed — {exc}')
+
+    parts = []
+    if added:
+        parts.append(f'{added} file(s) downloaded')
+    if skipped:
+        parts.append(f'{skipped} already on file')
+    if problems:
+        parts.append('; '.join(problems[:3]))
+    if note:
+        parts.append(note)
+
+    return added, skipped, source, ' | '.join(parts) or 'nothing to do'
+
+
+# ---------------------------------------------------------------------------
+# the queue runner
+# ---------------------------------------------------------------------------
+
+def _claim_next_job():
+    """Take the oldest eligible job and mark it running."""
+    now = datetime.utcnow()
+    job = InstructionFetchJob.query.filter(
+        InstructionFetchJob.status == 'queued',
+        db.or_(InstructionFetchJob.not_before.is_(None),
+               InstructionFetchJob.not_before <= now),
+    ).order_by(InstructionFetchJob.queued_at).first()
+
+    if job is None:
+        return None
+
+    job.status = 'running'
+    job.started_at = now
+    job.attempts = (job.attempts or 0) + 1
+    db.session.commit()
+    return job
+
+
+def run_one(force=False):
+    """
+    Process a single queued set, if the interval has elapsed.
+
+    Returns a dict describing what happened, or None when there was nothing
+    to do (disabled, not due yet, or an empty queue).
+    """
+    if not force and not is_enabled():
+        return None
+    if not force and not is_due():
+        return None
+
+    job = _claim_next_job()
+    if job is None:
+        return None
+
+    # Claim the slot up front: even a failure counts as network contact, so
+    # the next attempt still waits a full interval.
+    _mark_ran()
+
+    lego_set = job.lego_set
+    if lego_set is None:
+        job.status = 'cancelled'
+        job.message = 'set no longer exists'
+        job.finished_at = datetime.utcnow()
+        db.session.commit()
+        return job.to_dict()
+
+    if skip_sets_with_pdfs() and lego_set.instructions.filter_by(
+            file_type='PDF').count():
+        job.status = 'done'
+        job.message = 'skipped — set already has PDF instructions'
+        job.finished_at = datetime.utcnow()
+        db.session.commit()
+        return job.to_dict()
+
+    try:
+        added, skipped, source, message = fetch_for_set(lego_set)
+        job.files_added = added
+        job.source = source
+        job.message = message
+        if added:
+            job.status = 'done'
+        elif skipped:
+            job.status = 'done'
+        else:
+            job.status = 'empty'
+
+    except Exception as exc:
+        current_app.logger.exception('Instruction fetch failed for %s',
+                                     lego_set.set_number)
+        job.message = str(exc)[:500]
+        if job.attempts >= max_attempts():
+            job.status = 'failed'
+        else:
+            # Back off a couple of intervals before trying again.
+            job.status = 'queued'
+            job.not_before = datetime.utcnow() + timedelta(
+                minutes=interval_minutes() * 2)
+
+    job.finished_at = datetime.utcnow()
+    db.session.commit()
+
+    current_app.logger.info('Instruction fetch %s for %s: %s', job.status,
+                            lego_set.set_number, job.message)
+    return job.to_dict()
+
+
+def status_summary():
+    """Everything the admin page and its poller need, in one dict."""
+    counts = InstructionFetchJob.counts()
+    configured = BricksetAPI.is_configured()
+    recent = InstructionFetchJob.query.order_by(
+        InstructionFetchJob.id.desc()).limit(25).all()
+
+    return {
+        'enabled': is_enabled(),
+        'brickset_configured': configured,
+        'interval_minutes': interval_minutes(),
+        'file_delay_seconds': file_delay_seconds(),
+        'max_attempts': max_attempts(),
+        'skip_sets_with_pdfs': skip_sets_with_pdfs(),
+        'allow_setid_lookup': allow_setid_lookup(),
+        'last_run': last_run().isoformat() if last_run() else None,
+        'next_run_in_seconds': seconds_until_due(),
+        'counts': counts,
+        'official_sets': official_sets_query().count(),
+        'sets_missing_pdfs': sets_missing_pdfs().count(),
+        'jobs': [job.to_dict() for job in recent],
+    }

app/services/schema_upgrade.py

@@ -24,8 +24,18 @@ ADDITIVE_COLUMNS = {
         # site default instead.
         'visibility': "VARCHAR(10) NOT NULL DEFAULT 'public'",
     },
+    'instructions': {
+        # Written by the automatic fetcher; null on everything uploaded
+        # before it existed, which is fine - dedupe just can't help there.
+        'sha256': 'VARCHAR(64)',
+        'source': 'VARCHAR(20)',
+    },
 }
 
+# Tables added by later versions. create_all() is not called at boot, so new
+# tables need an explicit (and idempotent) nudge.
+NEW_TABLES = ('app.models.instruction_fetch.InstructionFetchJob',)
+
 
 def ensure_schema(app):
     """Add any missing columns. Returns a list of changes made."""
@@ -38,6 +48,20 @@ def ensure_schema(app):
             app.logger.warning('Schema check skipped: %s', exc)
             return applied
 
+        for dotted in NEW_TABLES:
+            module_path, _, class_name = dotted.rpartition('.')
+            try:
+                module = __import__(module_path, fromlist=[class_name])
+                model = getattr(module, class_name)
+                if model.__tablename__ not in existing_tables:
+                    model.__table__.create(db.engine, checkfirst=True)
+                    applied.append(f'table {model.__tablename__}')
+                    app.logger.info('Schema upgrade: created table %s',
+                                    model.__tablename__)
+            except Exception as exc:
+                app.logger.error('Schema upgrade failed for %s: %s',
+                                 dotted, exc)
+
         for table, columns in ADDITIVE_COLUMNS.items():
             if table not in existing_tables:
                 continue  # create_all() will build it with everything present