jessikitty
89f65e2d74
Downloads official instruction PDFs from LEGO, located via the Brickset API, one set per interval (10 minutes by default) so neither API gets hammered. Official sets only; MOCs are excluded throughout. - New admin page at /admin/instruction-fetch: queue builder, pacing controls, live countdown and per-set job history - Database-backed queue (instruction_fetch_jobs) and last-run timestamp, so pacing survives restarts - Background worker thread, plus 'flask instructions queue/run/status' for Task Scheduler or cron - SHA-256 dedupe (instructions.sha256), PDF header check, size cap and streaming downloads via temp files - brickset_api: getInstructions2 by set number and getInstructions by setID (the old code sent a set number to the setID method); getSets params now proper JSON; getKeyUsageStats added - Schema top-ups applied automatically at boot: new table plus instructions.sha256/source - Docker: fetcher env vars passed through compose, no worker thread in the entrypoint bootstrap
563 lines
18 KiB
Python
563 lines
18 KiB
Python
"""
|
|
Automatic instruction downloads for official sets.
|
|
|
|
Brickset's getInstructions/getInstructions2 methods hand back the URLs of the
|
|
official PDFs, which live on LEGO's own servers. So "download from Brickset or
|
|
LEGO" is really one flow: ask Brickset where the file is, then pull the PDF
|
|
from LEGO.
|
|
|
|
The whole point of this module is politeness. Work is queued in the database
|
|
and exactly one set is processed per interval (10 minutes by default), with a
|
|
short pause between the individual PDFs of a multi-book set. Nothing here ever
|
|
loops over the whole library in one go.
|
|
|
|
Notes on quotas:
|
|
* Only getSets counts against a Brickset API key's daily limit, so the
|
|
set-ID lookup is the expensive call and is skipped whenever the set
|
|
already has a brickset_id stored.
|
|
* getInstructions2 needs nothing but the API key and a set number.
|
|
|
|
Created by: Jess Rogerson (yelling commands at Claude.AI)
|
|
"""
|
|
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import tempfile
|
|
import time
|
|
from datetime import datetime, timedelta
|
|
|
|
import requests
|
|
from flask import current_app
|
|
|
|
from app import db
|
|
from app.models.instruction import Instruction
|
|
from app.models.instruction_fetch import InstructionFetchJob
|
|
from app.models.set import Set
|
|
from app.models.site_setting import SiteSetting
|
|
from app.services.brickset_api import BricksetAPI
|
|
from app.services.file_handler import FileHandler
|
|
|
|
USER_AGENT = ('LegoInstructionsManager/1.0 '
|
|
'(+https://gitea.hideawaygaming.com.au/jessikitty/'
|
|
'lego-instructions-manager)')
|
|
|
|
# Settings surfaced on the admin page. Values are strings in site_settings.
|
|
SETTING_DEFAULTS = {
|
|
'fetch_enabled': 'false',
|
|
'fetch_interval_minutes': '10',
|
|
'fetch_file_delay_seconds': '5',
|
|
'fetch_max_attempts': '3',
|
|
'fetch_skip_sets_with_pdfs': 'true',
|
|
'fetch_allow_setid_lookup': 'true',
|
|
'fetch_last_run': '',
|
|
}
|
|
|
|
MIN_INTERVAL_MINUTES = 1
|
|
MAX_INTERVAL_MINUTES = 1440
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# settings
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _int_setting(key, fallback, low=None, high=None):
|
|
raw = SiteSetting.get(key, SETTING_DEFAULTS.get(key))
|
|
try:
|
|
value = int(str(raw).strip())
|
|
except (TypeError, ValueError):
|
|
value = fallback
|
|
if low is not None:
|
|
value = max(low, value)
|
|
if high is not None:
|
|
value = min(high, value)
|
|
return value
|
|
|
|
|
|
def is_enabled():
|
|
return SiteSetting.get_bool('fetch_enabled', False)
|
|
|
|
|
|
def interval_minutes():
|
|
return _int_setting('fetch_interval_minutes', 10,
|
|
MIN_INTERVAL_MINUTES, MAX_INTERVAL_MINUTES)
|
|
|
|
|
|
def file_delay_seconds():
|
|
return _int_setting('fetch_file_delay_seconds', 5, 0, 300)
|
|
|
|
|
|
def max_attempts():
|
|
return _int_setting('fetch_max_attempts', 3, 1, 10)
|
|
|
|
|
|
def skip_sets_with_pdfs():
|
|
return SiteSetting.get_bool('fetch_skip_sets_with_pdfs', True)
|
|
|
|
|
|
def allow_setid_lookup():
|
|
return SiteSetting.get_bool('fetch_allow_setid_lookup', True)
|
|
|
|
|
|
def last_run():
|
|
raw = SiteSetting.get('fetch_last_run', '')
|
|
if not raw:
|
|
return None
|
|
try:
|
|
return datetime.fromisoformat(raw)
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def _mark_ran(when=None):
|
|
SiteSetting.set('fetch_last_run', (when or datetime.utcnow()).isoformat())
|
|
db.session.commit()
|
|
|
|
|
|
def next_run_at():
|
|
"""When the worker is next allowed to touch the network."""
|
|
previous = last_run()
|
|
if previous is None:
|
|
return datetime.utcnow()
|
|
return previous + timedelta(minutes=interval_minutes())
|
|
|
|
|
|
def seconds_until_due():
|
|
return max(0, int((next_run_at() - datetime.utcnow()).total_seconds()))
|
|
|
|
|
|
def is_due():
|
|
return seconds_until_due() <= 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# queueing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def official_sets_query():
|
|
"""Official sets only — MOCs have nothing to look up."""
|
|
return Set.query.filter(Set.is_moc.is_(False))
|
|
|
|
|
|
def sets_missing_pdfs():
|
|
"""Official sets with no PDF instruction on file."""
|
|
have_pdfs = db.session.query(Instruction.set_id).filter(
|
|
Instruction.file_type == 'PDF').distinct()
|
|
return official_sets_query().filter(~Set.id.in_(have_pdfs))
|
|
|
|
|
|
def queue_sets(sets, user_id=None):
|
|
"""Queue an iterable of sets, skipping any already waiting. -> count."""
|
|
queued = 0
|
|
for lego_set in sets:
|
|
if lego_set.is_moc:
|
|
continue
|
|
if InstructionFetchJob.open_for_set(lego_set.id):
|
|
continue
|
|
db.session.add(InstructionFetchJob(set_id=lego_set.id,
|
|
queued_by_id=user_id))
|
|
queued += 1
|
|
if queued:
|
|
db.session.commit()
|
|
return queued
|
|
|
|
|
|
def queue_missing(user_id=None, limit=None):
|
|
query = sets_missing_pdfs().order_by(Set.year_released.desc(),
|
|
Set.set_number)
|
|
if limit:
|
|
query = query.limit(limit)
|
|
return queue_sets(query.all(), user_id)
|
|
|
|
|
|
def queue_all_official(user_id=None, limit=None):
|
|
query = official_sets_query().order_by(Set.year_released.desc(),
|
|
Set.set_number)
|
|
if limit:
|
|
query = query.limit(limit)
|
|
return queue_sets(query.all(), user_id)
|
|
|
|
|
|
def requeue_failed(user_id=None):
|
|
"""Put every failed job back at the end of the queue."""
|
|
failed = InstructionFetchJob.query.filter_by(status='failed').all()
|
|
count = 0
|
|
for job in failed:
|
|
if InstructionFetchJob.open_for_set(job.set_id):
|
|
continue
|
|
job.status = 'queued'
|
|
job.attempts = 0
|
|
job.message = None
|
|
job.not_before = None
|
|
job.queued_at = datetime.utcnow()
|
|
job.queued_by_id = user_id or job.queued_by_id
|
|
count += 1
|
|
if count:
|
|
db.session.commit()
|
|
return count
|
|
|
|
|
|
def clear_jobs(scope='finished'):
|
|
"""Tidy the history. scope: finished | failed | all."""
|
|
query = InstructionFetchJob.query
|
|
if scope == 'finished':
|
|
query = query.filter(InstructionFetchJob.status.in_(
|
|
('done', 'empty', 'cancelled')))
|
|
elif scope == 'failed':
|
|
query = query.filter_by(status='failed')
|
|
elif scope != 'all':
|
|
return 0
|
|
removed = query.delete(synchronize_session=False)
|
|
db.session.commit()
|
|
return removed
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# the actual fetching
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def brickset_number(set_number):
|
|
"""Brickset wants {number}-{variant}; the library usually stores {number}."""
|
|
number = (set_number or '').strip()
|
|
if not number:
|
|
return None
|
|
if re.search(r'-\d+$', number):
|
|
return number
|
|
return f'{number}-1'
|
|
|
|
|
|
def _instruction_urls(api, lego_set):
|
|
"""(list of {url, description}, source, note)."""
|
|
note = None
|
|
|
|
if lego_set.brickset_id:
|
|
entries = api.get_instructions_by_id(lego_set.brickset_id)
|
|
if entries:
|
|
return _normalise(entries), 'brickset', note
|
|
|
|
number = brickset_number(lego_set.set_number)
|
|
if number:
|
|
entries = api.get_instructions(number)
|
|
if entries:
|
|
return _normalise(entries), 'brickset', note
|
|
|
|
# Last resort: resolve the Brickset set ID, then ask again. This is the
|
|
# only call that counts against the daily API key quota, so it is both
|
|
# optional and cached back onto the set for next time.
|
|
if allow_setid_lookup() and not lego_set.brickset_id and number:
|
|
data = api.get_set_by_number(number)
|
|
if data and data.get('setID'):
|
|
lego_set.brickset_id = data['setID']
|
|
db.session.commit()
|
|
entries = api.get_instructions_by_id(lego_set.brickset_id)
|
|
if entries:
|
|
return _normalise(entries), 'brickset', 'resolved via getSets'
|
|
note = 'set found, no instructions listed'
|
|
else:
|
|
note = 'set number not found on Brickset'
|
|
|
|
return [], 'brickset', note
|
|
|
|
|
|
def _normalise(entries):
|
|
out = []
|
|
for entry in entries or []:
|
|
url = entry.get('URL') or entry.get('url')
|
|
if not url:
|
|
continue
|
|
out.append({
|
|
'url': url,
|
|
'description': (entry.get('description')
|
|
or entry.get('Description') or ''),
|
|
})
|
|
return out
|
|
|
|
|
|
def _safe_name(lego_set, description, index):
|
|
"""A readable file name, e.g. '10497-building-instructions-book-1.pdf'."""
|
|
number = re.sub(r'[^A-Za-z0-9\-]+', '', lego_set.set_number or 'set')
|
|
base = re.sub(r'[^A-Za-z0-9]+', '-', description or '').strip('-').lower()
|
|
base = re.sub(r'-{2,}', '-', base)[:80].strip('-')
|
|
if not base:
|
|
return f'{number}-instructions-{index}.pdf'
|
|
# Brickset descriptions usually lead with the set number already.
|
|
if base.startswith(number.lower()):
|
|
return f'{base}.pdf'
|
|
return f'{number}-{base}.pdf'
|
|
|
|
|
|
class _LocalFile:
|
|
"""Adapter so FileHandler.save_file can take a file we downloaded."""
|
|
|
|
def __init__(self, path, filename):
|
|
self._path = path
|
|
self.filename = filename
|
|
|
|
def save(self, destination):
|
|
import shutil
|
|
shutil.move(self._path, destination)
|
|
|
|
|
|
def _download_pdf(url, max_bytes, timeout):
|
|
"""Stream to a temp file. -> (path, size, sha256). Raises on refusal."""
|
|
headers = {'User-Agent': USER_AGENT, 'Accept': 'application/pdf,*/*'}
|
|
digest = hashlib.sha256()
|
|
size = 0
|
|
|
|
handle, temp_path = tempfile.mkstemp(suffix='.pdf')
|
|
os.close(handle)
|
|
|
|
try:
|
|
with requests.get(url, headers=headers, stream=True,
|
|
timeout=timeout) as response:
|
|
response.raise_for_status()
|
|
|
|
content_type = (response.headers.get('Content-Type') or '').lower()
|
|
declared = response.headers.get('Content-Length')
|
|
if declared and declared.isdigit() and int(declared) > max_bytes:
|
|
raise ValueError(
|
|
f'file is {int(declared) // (1024 * 1024)} MB, over the limit')
|
|
|
|
first = True
|
|
with open(temp_path, 'wb') as out:
|
|
for chunk in response.iter_content(chunk_size=64 * 1024):
|
|
if not chunk:
|
|
continue
|
|
if first:
|
|
if not chunk.startswith(b'%PDF'):
|
|
raise ValueError(
|
|
f'not a PDF (content-type {content_type or "unknown"})')
|
|
first = False
|
|
size += len(chunk)
|
|
if size > max_bytes:
|
|
raise ValueError('file exceeded the size limit mid-download')
|
|
digest.update(chunk)
|
|
out.write(chunk)
|
|
|
|
if size == 0:
|
|
raise ValueError('empty response')
|
|
return temp_path, size, digest.hexdigest()
|
|
|
|
except Exception:
|
|
if os.path.exists(temp_path):
|
|
os.remove(temp_path)
|
|
raise
|
|
|
|
|
|
def _existing_hashes(set_id):
|
|
rows = db.session.query(Instruction.sha256).filter(
|
|
Instruction.set_id == set_id, Instruction.sha256.isnot(None)).all()
|
|
return {row[0] for row in rows if row[0]}
|
|
|
|
|
|
def _next_page_number(set_id):
|
|
from sqlalchemy import func
|
|
highest = db.session.query(func.max(Instruction.page_number)).filter_by(
|
|
set_id=set_id).scalar()
|
|
return (highest or 0) + 1
|
|
|
|
|
|
def fetch_for_set(lego_set, delay_between_files=None):
|
|
"""
|
|
Download every instruction Brickset lists for one official set.
|
|
|
|
Returns (added, skipped, source, message). Never raises for the normal
|
|
failure modes — the message explains what happened.
|
|
"""
|
|
if lego_set.is_moc:
|
|
return 0, 0, None, 'MOC — nothing to look up'
|
|
|
|
if not BricksetAPI.is_configured():
|
|
raise RuntimeError('Brickset API credentials are not configured')
|
|
|
|
max_bytes = current_app.config.get('FETCH_MAX_FILE_MB', 150) * 1024 * 1024
|
|
timeout = current_app.config.get('FETCH_TIMEOUT_SECONDS', 60)
|
|
pause = (file_delay_seconds() if delay_between_files is None
|
|
else delay_between_files)
|
|
|
|
api = BricksetAPI()
|
|
entries, source, note = _instruction_urls(api, lego_set)
|
|
|
|
if not entries:
|
|
return 0, 0, source, note or 'no instructions listed on Brickset'
|
|
|
|
known = _existing_hashes(lego_set.id)
|
|
page = _next_page_number(lego_set.id)
|
|
added = 0
|
|
skipped = 0
|
|
problems = []
|
|
|
|
for index, entry in enumerate(entries, start=1):
|
|
if index > 1 and pause:
|
|
time.sleep(pause)
|
|
|
|
try:
|
|
temp_path, size, sha = _download_pdf(entry['url'], max_bytes,
|
|
timeout)
|
|
except Exception as exc:
|
|
problems.append(f'{entry["description"] or entry["url"]}: {exc}')
|
|
continue
|
|
|
|
if sha in known:
|
|
os.remove(temp_path)
|
|
skipped += 1
|
|
continue
|
|
|
|
try:
|
|
filename = _safe_name(lego_set, entry['description'], index)
|
|
stored = _LocalFile(temp_path, filename)
|
|
rel_path, file_size, thumb = FileHandler.save_file(
|
|
stored, lego_set.set_number, 'PDF')
|
|
|
|
instruction = Instruction(
|
|
set_id=lego_set.id,
|
|
file_type='PDF',
|
|
file_path=rel_path,
|
|
file_name=filename,
|
|
file_size=file_size or size,
|
|
thumbnail_path=thumb,
|
|
page_number=page,
|
|
sha256=sha,
|
|
source=source,
|
|
user_id=lego_set.user_id,
|
|
)
|
|
db.session.add(instruction)
|
|
db.session.commit()
|
|
|
|
known.add(sha)
|
|
page += 1
|
|
added += 1
|
|
|
|
except Exception as exc:
|
|
db.session.rollback()
|
|
if os.path.exists(temp_path):
|
|
os.remove(temp_path)
|
|
problems.append(f'{filename}: saving failed — {exc}')
|
|
|
|
parts = []
|
|
if added:
|
|
parts.append(f'{added} file(s) downloaded')
|
|
if skipped:
|
|
parts.append(f'{skipped} already on file')
|
|
if problems:
|
|
parts.append('; '.join(problems[:3]))
|
|
if note:
|
|
parts.append(note)
|
|
|
|
return added, skipped, source, ' | '.join(parts) or 'nothing to do'
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# the queue runner
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _claim_next_job():
|
|
"""Take the oldest eligible job and mark it running."""
|
|
now = datetime.utcnow()
|
|
job = InstructionFetchJob.query.filter(
|
|
InstructionFetchJob.status == 'queued',
|
|
db.or_(InstructionFetchJob.not_before.is_(None),
|
|
InstructionFetchJob.not_before <= now),
|
|
).order_by(InstructionFetchJob.queued_at).first()
|
|
|
|
if job is None:
|
|
return None
|
|
|
|
job.status = 'running'
|
|
job.started_at = now
|
|
job.attempts = (job.attempts or 0) + 1
|
|
db.session.commit()
|
|
return job
|
|
|
|
|
|
def run_one(force=False):
|
|
"""
|
|
Process a single queued set, if the interval has elapsed.
|
|
|
|
Returns a dict describing what happened, or None when there was nothing
|
|
to do (disabled, not due yet, or an empty queue).
|
|
"""
|
|
if not force and not is_enabled():
|
|
return None
|
|
if not force and not is_due():
|
|
return None
|
|
|
|
job = _claim_next_job()
|
|
if job is None:
|
|
return None
|
|
|
|
# Claim the slot up front: even a failure counts as network contact, so
|
|
# the next attempt still waits a full interval.
|
|
_mark_ran()
|
|
|
|
lego_set = job.lego_set
|
|
if lego_set is None:
|
|
job.status = 'cancelled'
|
|
job.message = 'set no longer exists'
|
|
job.finished_at = datetime.utcnow()
|
|
db.session.commit()
|
|
return job.to_dict()
|
|
|
|
if skip_sets_with_pdfs() and lego_set.instructions.filter_by(
|
|
file_type='PDF').count():
|
|
job.status = 'done'
|
|
job.message = 'skipped — set already has PDF instructions'
|
|
job.finished_at = datetime.utcnow()
|
|
db.session.commit()
|
|
return job.to_dict()
|
|
|
|
try:
|
|
added, skipped, source, message = fetch_for_set(lego_set)
|
|
job.files_added = added
|
|
job.source = source
|
|
job.message = message
|
|
if added:
|
|
job.status = 'done'
|
|
elif skipped:
|
|
job.status = 'done'
|
|
else:
|
|
job.status = 'empty'
|
|
|
|
except Exception as exc:
|
|
current_app.logger.exception('Instruction fetch failed for %s',
|
|
lego_set.set_number)
|
|
job.message = str(exc)[:500]
|
|
if job.attempts >= max_attempts():
|
|
job.status = 'failed'
|
|
else:
|
|
# Back off a couple of intervals before trying again.
|
|
job.status = 'queued'
|
|
job.not_before = datetime.utcnow() + timedelta(
|
|
minutes=interval_minutes() * 2)
|
|
|
|
job.finished_at = datetime.utcnow()
|
|
db.session.commit()
|
|
|
|
current_app.logger.info('Instruction fetch %s for %s: %s', job.status,
|
|
lego_set.set_number, job.message)
|
|
return job.to_dict()
|
|
|
|
|
|
def status_summary():
|
|
"""Everything the admin page and its poller need, in one dict."""
|
|
counts = InstructionFetchJob.counts()
|
|
configured = BricksetAPI.is_configured()
|
|
recent = InstructionFetchJob.query.order_by(
|
|
InstructionFetchJob.id.desc()).limit(25).all()
|
|
|
|
return {
|
|
'enabled': is_enabled(),
|
|
'brickset_configured': configured,
|
|
'interval_minutes': interval_minutes(),
|
|
'file_delay_seconds': file_delay_seconds(),
|
|
'max_attempts': max_attempts(),
|
|
'skip_sets_with_pdfs': skip_sets_with_pdfs(),
|
|
'allow_setid_lookup': allow_setid_lookup(),
|
|
'last_run': last_run().isoformat() if last_run() else None,
|
|
'next_run_in_seconds': seconds_until_due(),
|
|
'counts': counts,
|
|
'official_sets': official_sets_query().count(),
|
|
'sets_missing_pdfs': sets_missing_pdfs().count(),
|
|
'jobs': [job.to_dict() for job in recent],
|
|
}
|