jessikitty/lego-instructions-manager
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add automatic instruction fetcher for official sets

Browse Source 
Downloads official instruction PDFs from LEGO, located via the Brickset API, one set per interval (10 minutes by default) so neither API gets hammered. Official sets only; MOCs are excluded throughout.

- New admin page at /admin/instruction-fetch: queue builder, pacing controls, live countdown and per-set job history

- Database-backed queue (instruction_fetch_jobs) and last-run timestamp, so pacing survives restarts

- Background worker thread, plus 'flask instructions queue/run/status' for Task Scheduler or cron

- SHA-256 dedupe (instructions.sha256), PDF header check, size cap and streaming downloads via temp files

- brickset_api: getInstructions2 by set number and getInstructions by setID (the old code sent a set number to the setID method); getSets params now proper JSON; getKeyUsageStats added

- Schema top-ups applied automatically at boot: new table plus instructions.sha256/source

- Docker: fetcher env vars passed through compose, no worker thread in the entrypoint bootstrap
This commit is contained in:
jessikitty committed 2026-09-26 13:43:17 +10:00
1 parent aac2b60517
commit 89f65e2d74
23 files changed
+2073 -76
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/services/instruction_fetcher.py
+562
View File
@@ -0,0 +1,562 @@
"""
Automatic instruction downloads for official sets.
Brickset's getInstructions/getInstructions2 methods hand back the URLs of the
official PDFs, which live on LEGO's own servers. So "download from Brickset or
LEGO" is really one flow: ask Brickset where the file is, then pull the PDF
from LEGO.
The whole point of this module is politeness. Work is queued in the database
and exactly one set is processed per interval (10 minutes by default), with a
short pause between the individual PDFs of a multi-book set. Nothing here ever
loops over the whole library in one go.
Notes on quotas:
* Only getSets counts against a Brickset API key's daily limit, so the
set-ID lookup is the expensive call and is skipped whenever the set
already has a brickset_id stored.
* getInstructions2 needs nothing but the API key and a set number.
Created by: Jess Rogerson (yelling commands at Claude.AI)
"""
import hashlib
import os
import re
import tempfile
import time
from datetime import datetime, timedelta
import requests
from flask import current_app
from app import db
from app.models.instruction import Instruction
from app.models.instruction_fetch import InstructionFetchJob
from app.models.set import Set
from app.models.site_setting import SiteSetting
from app.services.brickset_api import BricksetAPI
from app.services.file_handler import FileHandler
USER_AGENT = ('LegoInstructionsManager/1.0 '
'(+https://gitea.hideawaygaming.com.au/jessikitty/'
'lego-instructions-manager)')
# Settings surfaced on the admin page. Values are strings in site_settings.
SETTING_DEFAULTS = {
'fetch_enabled': 'false',
'fetch_interval_minutes': '10',
'fetch_file_delay_seconds': '5',
'fetch_max_attempts': '3',
'fetch_skip_sets_with_pdfs': 'true',
'fetch_allow_setid_lookup': 'true',
'fetch_last_run': '',
}
MIN_INTERVAL_MINUTES = 1
MAX_INTERVAL_MINUTES = 1440
# ---------------------------------------------------------------------------
# settings
# ---------------------------------------------------------------------------
def _int_setting(key, fallback, low=None, high=None):
raw = SiteSetting.get(key, SETTING_DEFAULTS.get(key))
try:
value = int(str(raw).strip())
except (TypeError, ValueError):
value = fallback
if low is not None:
value = max(low, value)
if high is not None:
value = min(high, value)
return value
def is_enabled():
return SiteSetting.get_bool('fetch_enabled', False)
def interval_minutes():
return _int_setting('fetch_interval_minutes', 10,
MIN_INTERVAL_MINUTES, MAX_INTERVAL_MINUTES)
def file_delay_seconds():
return _int_setting('fetch_file_delay_seconds', 5, 0, 300)
def max_attempts():
return _int_setting('fetch_max_attempts', 3, 1, 10)
def skip_sets_with_pdfs():
return SiteSetting.get_bool('fetch_skip_sets_with_pdfs', True)
def allow_setid_lookup():
return SiteSetting.get_bool('fetch_allow_setid_lookup', True)
def last_run():
raw = SiteSetting.get('fetch_last_run', '')
if not raw:
return None
try:
return datetime.fromisoformat(raw)
except ValueError:
return None
def _mark_ran(when=None):
SiteSetting.set('fetch_last_run', (when or datetime.utcnow()).isoformat())
db.session.commit()
def next_run_at():
"""When the worker is next allowed to touch the network."""
previous = last_run()
if previous is None:
return datetime.utcnow()
return previous + timedelta(minutes=interval_minutes())
def seconds_until_due():
return max(0, int((next_run_at() - datetime.utcnow()).total_seconds()))
def is_due():
return seconds_until_due() <= 0
# ---------------------------------------------------------------------------
# queueing
# ---------------------------------------------------------------------------
def official_sets_query():
"""Official sets only — MOCs have nothing to look up."""
return Set.query.filter(Set.is_moc.is_(False))
def sets_missing_pdfs():
"""Official sets with no PDF instruction on file."""
have_pdfs = db.session.query(Instruction.set_id).filter(
Instruction.file_type == 'PDF').distinct()
return official_sets_query().filter(~Set.id.in_(have_pdfs))
def queue_sets(sets, user_id=None):
"""Queue an iterable of sets, skipping any already waiting. -> count."""
queued = 0
for lego_set in sets:
if lego_set.is_moc:
continue
if InstructionFetchJob.open_for_set(lego_set.id):
continue
db.session.add(InstructionFetchJob(set_id=lego_set.id,
queued_by_id=user_id))
queued += 1
if queued:
db.session.commit()
return queued
def queue_missing(user_id=None, limit=None):
query = sets_missing_pdfs().order_by(Set.year_released.desc(),
Set.set_number)
if limit:
query = query.limit(limit)
return queue_sets(query.all(), user_id)
def queue_all_official(user_id=None, limit=None):
query = official_sets_query().order_by(Set.year_released.desc(),
Set.set_number)
if limit:
query = query.limit(limit)
return queue_sets(query.all(), user_id)
def requeue_failed(user_id=None):
"""Put every failed job back at the end of the queue."""
failed = InstructionFetchJob.query.filter_by(status='failed').all()
count = 0
for job in failed:
if InstructionFetchJob.open_for_set(job.set_id):
continue
job.status = 'queued'
job.attempts = 0
job.message = None
job.not_before = None
job.queued_at = datetime.utcnow()
job.queued_by_id = user_id or job.queued_by_id
count += 1
if count:
db.session.commit()
return count
def clear_jobs(scope='finished'):
"""Tidy the history. scope: finished | failed | all."""
query = InstructionFetchJob.query
if scope == 'finished':
query = query.filter(InstructionFetchJob.status.in_(
('done', 'empty', 'cancelled')))
elif scope == 'failed':
query = query.filter_by(status='failed')
elif scope != 'all':
return 0
removed = query.delete(synchronize_session=False)
db.session.commit()
return removed
# ---------------------------------------------------------------------------
# the actual fetching
# ---------------------------------------------------------------------------
def brickset_number(set_number):
"""Brickset wants {number}-{variant}; the library usually stores {number}."""
number = (set_number or '').strip()
if not number:
return None
if re.search(r'-\d+$', number):
return number
return f'{number}-1'
def _instruction_urls(api, lego_set):
"""(list of {url, description}, source, note)."""
note = None
if lego_set.brickset_id:
entries = api.get_instructions_by_id(lego_set.brickset_id)
if entries:
return _normalise(entries), 'brickset', note
number = brickset_number(lego_set.set_number)
if number:
entries = api.get_instructions(number)
if entries:
return _normalise(entries), 'brickset', note
# Last resort: resolve the Brickset set ID, then ask again. This is the
# only call that counts against the daily API key quota, so it is both
# optional and cached back onto the set for next time.
if allow_setid_lookup() and not lego_set.brickset_id and number:
data = api.get_set_by_number(number)
if data and data.get('setID'):
lego_set.brickset_id = data['setID']
db.session.commit()
entries = api.get_instructions_by_id(lego_set.brickset_id)
if entries:
return _normalise(entries), 'brickset', 'resolved via getSets'
note = 'set found, no instructions listed'
else:
note = 'set number not found on Brickset'
return [], 'brickset', note
def _normalise(entries):
out = []
for entry in entries or []:
url = entry.get('URL') or entry.get('url')
if not url:
continue
out.append({
'url': url,
'description': (entry.get('description')
or entry.get('Description') or ''),
})
return out
def _safe_name(lego_set, description, index):
"""A readable file name, e.g. '10497-building-instructions-book-1.pdf'."""
number = re.sub(r'[^A-Za-z0-9\-]+', '', lego_set.set_number or 'set')
base = re.sub(r'[^A-Za-z0-9]+', '-', description or '').strip('-').lower()
base = re.sub(r'-{2,}', '-', base)[:80].strip('-')
if not base:
return f'{number}-instructions-{index}.pdf'
# Brickset descriptions usually lead with the set number already.
if base.startswith(number.lower()):
return f'{base}.pdf'
return f'{number}-{base}.pdf'
class _LocalFile:
"""Adapter so FileHandler.save_file can take a file we downloaded."""
def __init__(self, path, filename):
self._path = path
self.filename = filename
def save(self, destination):
import shutil
shutil.move(self._path, destination)
def _download_pdf(url, max_bytes, timeout):
"""Stream to a temp file. -> (path, size, sha256). Raises on refusal."""
headers = {'User-Agent': USER_AGENT, 'Accept': 'application/pdf,*/*'}
digest = hashlib.sha256()
size = 0
handle, temp_path = tempfile.mkstemp(suffix='.pdf')
os.close(handle)
try:
with requests.get(url, headers=headers, stream=True,
timeout=timeout) as response:
response.raise_for_status()
content_type = (response.headers.get('Content-Type') or '').lower()
declared = response.headers.get('Content-Length')
if declared and declared.isdigit() and int(declared) > max_bytes:
raise ValueError(
f'file is {int(declared) // (1024 * 1024)} MB, over the limit')
first = True
with open(temp_path, 'wb') as out:
for chunk in response.iter_content(chunk_size=64 * 1024):
if not chunk:
continue
if first:
if not chunk.startswith(b'%PDF'):
raise ValueError(
f'not a PDF (content-type {content_type or "unknown"})')
first = False
size += len(chunk)
if size > max_bytes:
raise ValueError('file exceeded the size limit mid-download')
digest.update(chunk)
out.write(chunk)
if size == 0:
raise ValueError('empty response')
return temp_path, size, digest.hexdigest()
except Exception:
if os.path.exists(temp_path):
os.remove(temp_path)
raise
def _existing_hashes(set_id):
rows = db.session.query(Instruction.sha256).filter(
Instruction.set_id == set_id, Instruction.sha256.isnot(None)).all()
return {row[0] for row in rows if row[0]}
def _next_page_number(set_id):
from sqlalchemy import func
highest = db.session.query(func.max(Instruction.page_number)).filter_by(
set_id=set_id).scalar()
return (highest or 0) + 1
def fetch_for_set(lego_set, delay_between_files=None):
"""
Download every instruction Brickset lists for one official set.
Returns (added, skipped, source, message). Never raises for the normal
failure modes — the message explains what happened.
"""
if lego_set.is_moc:
return 0, 0, None, 'MOC — nothing to look up'
if not BricksetAPI.is_configured():
raise RuntimeError('Brickset API credentials are not configured')
max_bytes = current_app.config.get('FETCH_MAX_FILE_MB', 150) * 1024 * 1024
timeout = current_app.config.get('FETCH_TIMEOUT_SECONDS', 60)
pause = (file_delay_seconds() if delay_between_files is None
else delay_between_files)
api = BricksetAPI()
entries, source, note = _instruction_urls(api, lego_set)
if not entries:
return 0, 0, source, note or 'no instructions listed on Brickset'
known = _existing_hashes(lego_set.id)
page = _next_page_number(lego_set.id)
added = 0
skipped = 0
problems = []
for index, entry in enumerate(entries, start=1):
if index > 1 and pause:
time.sleep(pause)
try:
temp_path, size, sha = _download_pdf(entry['url'], max_bytes,
timeout)
except Exception as exc:
problems.append(f'{entry["description"] or entry["url"]}: {exc}')
continue
if sha in known:
os.remove(temp_path)
skipped += 1
continue
try:
filename = _safe_name(lego_set, entry['description'], index)
stored = _LocalFile(temp_path, filename)
rel_path, file_size, thumb = FileHandler.save_file(
stored, lego_set.set_number, 'PDF')
instruction = Instruction(
set_id=lego_set.id,
file_type='PDF',
file_path=rel_path,
file_name=filename,
file_size=file_size or size,
thumbnail_path=thumb,
page_number=page,
sha256=sha,
source=source,
user_id=lego_set.user_id,
)
db.session.add(instruction)
db.session.commit()
known.add(sha)
page += 1
added += 1
except Exception as exc:
db.session.rollback()
if os.path.exists(temp_path):
os.remove(temp_path)
problems.append(f'{filename}: saving failed — {exc}')
parts = []
if added:
parts.append(f'{added} file(s) downloaded')
if skipped:
parts.append(f'{skipped} already on file')
if problems:
parts.append('; '.join(problems[:3]))
if note:
parts.append(note)
return added, skipped, source, ' | '.join(parts) or 'nothing to do'
# ---------------------------------------------------------------------------
# the queue runner
# ---------------------------------------------------------------------------
def _claim_next_job():
"""Take the oldest eligible job and mark it running."""
now = datetime.utcnow()
job = InstructionFetchJob.query.filter(
InstructionFetchJob.status == 'queued',
db.or_(InstructionFetchJob.not_before.is_(None),
InstructionFetchJob.not_before <= now),
).order_by(InstructionFetchJob.queued_at).first()
if job is None:
return None
job.status = 'running'
job.started_at = now
job.attempts = (job.attempts or 0) + 1
db.session.commit()
return job
def run_one(force=False):
"""
Process a single queued set, if the interval has elapsed.
Returns a dict describing what happened, or None when there was nothing
to do (disabled, not due yet, or an empty queue).
"""
if not force and not is_enabled():
return None
if not force and not is_due():
return None
job = _claim_next_job()
if job is None:
return None
# Claim the slot up front: even a failure counts as network contact, so
# the next attempt still waits a full interval.
_mark_ran()
lego_set = job.lego_set
if lego_set is None:
job.status = 'cancelled'
job.message = 'set no longer exists'
job.finished_at = datetime.utcnow()
db.session.commit()
return job.to_dict()
if skip_sets_with_pdfs() and lego_set.instructions.filter_by(
file_type='PDF').count():
job.status = 'done'
job.message = 'skipped — set already has PDF instructions'
job.finished_at = datetime.utcnow()
db.session.commit()
return job.to_dict()
try:
added, skipped, source, message = fetch_for_set(lego_set)
job.files_added = added
job.source = source
job.message = message
if added:
job.status = 'done'
elif skipped:
job.status = 'done'
else:
job.status = 'empty'
except Exception as exc:
current_app.logger.exception('Instruction fetch failed for %s',
lego_set.set_number)
job.message = str(exc)[:500]
if job.attempts >= max_attempts():
job.status = 'failed'
else:
# Back off a couple of intervals before trying again.
job.status = 'queued'
job.not_before = datetime.utcnow() + timedelta(
minutes=interval_minutes() * 2)
job.finished_at = datetime.utcnow()
db.session.commit()
current_app.logger.info('Instruction fetch %s for %s: %s', job.status,
lego_set.set_number, job.message)
return job.to_dict()
def status_summary():
"""Everything the admin page and its poller need, in one dict."""
counts = InstructionFetchJob.counts()
configured = BricksetAPI.is_configured()
recent = InstructionFetchJob.query.order_by(
InstructionFetchJob.id.desc()).limit(25).all()
return {
'enabled': is_enabled(),
'brickset_configured': configured,
'interval_minutes': interval_minutes(),
'file_delay_seconds': file_delay_seconds(),
'max_attempts': max_attempts(),
'skip_sets_with_pdfs': skip_sets_with_pdfs(),
'allow_setid_lookup': allow_setid_lookup(),
'last_run': last_run().isoformat() if last_run() else None,
'next_run_in_seconds': seconds_until_due(),
'counts': counts,
'official_sets': official_sets_query().count(),
'sets_missing_pdfs': sets_missing_pdfs().count(),
'jobs': [job.to_dict() for job in recent],
}