jessikitty
89f65e2d74
Downloads official instruction PDFs from LEGO, located via the Brickset API, one set per interval (10 minutes by default) so neither API gets hammered. Official sets only; MOCs are excluded throughout. - New admin page at /admin/instruction-fetch: queue builder, pacing controls, live countdown and per-set job history - Database-backed queue (instruction_fetch_jobs) and last-run timestamp, so pacing survives restarts - Background worker thread, plus 'flask instructions queue/run/status' for Task Scheduler or cron - SHA-256 dedupe (instructions.sha256), PDF header check, size cap and streaming downloads via temp files - brickset_api: getInstructions2 by set number and getInstructions by setID (the old code sent a set number to the setID method); getSets params now proper JSON; getKeyUsageStats added - Schema top-ups applied automatically at boot: new table plus instructions.sha256/source - Docker: fetcher env vars passed through compose, no worker thread in the entrypoint bootstrap
122 lines
4.3 KiB
Python
122 lines
4.3 KiB
Python
"""
|
|
Queue of "go and fetch the instructions" jobs for official sets.
|
|
|
|
One row per set per attempt run. The queue lives in the database rather than
|
|
in memory so a restart doesn't lose the backlog, and so the admin page can
|
|
show what happened to every set without scraping the log file.
|
|
|
|
Only official sets ever get queued — MOCs have no Brickset entry, so there is
|
|
nothing to fetch.
|
|
|
|
Created by: Jess Rogerson (yelling commands at Claude.AI)
|
|
"""
|
|
|
|
from datetime import datetime
|
|
|
|
from app import db
|
|
|
|
# queued - waiting its turn
|
|
# running - being worked on right now
|
|
# done - at least one instruction file was saved
|
|
# empty - Brickset has no instructions on file for this set
|
|
# failed - gave up after max attempts
|
|
# cancelled - removed from the queue by an admin
|
|
STATUSES = ('queued', 'running', 'done', 'empty', 'failed', 'cancelled')
|
|
|
|
OPEN_STATUSES = ('queued', 'running')
|
|
|
|
STATUS_BADGES = {
|
|
'queued': 'secondary',
|
|
'running': 'primary',
|
|
'done': 'success',
|
|
'empty': 'warning',
|
|
'failed': 'danger',
|
|
'cancelled': 'dark',
|
|
}
|
|
|
|
|
|
class InstructionFetchJob(db.Model):
|
|
"""A single set queued for an automatic instruction download."""
|
|
|
|
__tablename__ = 'instruction_fetch_jobs'
|
|
|
|
id = db.Column(db.Integer, primary_key=True)
|
|
set_id = db.Column(db.Integer, db.ForeignKey('sets.id'), nullable=False,
|
|
index=True)
|
|
|
|
status = db.Column(db.String(12), nullable=False, default='queued',
|
|
server_default='queued', index=True)
|
|
attempts = db.Column(db.Integer, nullable=False, default=0,
|
|
server_default='0')
|
|
files_added = db.Column(db.Integer, nullable=False, default=0,
|
|
server_default='0')
|
|
# 'brickset' for now; the column exists so a future LEGO-direct or
|
|
# Rebrickable path can be told apart in the history.
|
|
source = db.Column(db.String(20), nullable=True)
|
|
message = db.Column(db.Text, nullable=True)
|
|
|
|
queued_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow,
|
|
index=True)
|
|
# Retry backoff: a job is invisible to the worker until this passes.
|
|
not_before = db.Column(db.DateTime, nullable=True)
|
|
started_at = db.Column(db.DateTime, nullable=True)
|
|
finished_at = db.Column(db.DateTime, nullable=True)
|
|
|
|
queued_by_id = db.Column(db.Integer, db.ForeignKey('users.id'),
|
|
nullable=True)
|
|
|
|
lego_set = db.relationship(
|
|
'Set',
|
|
backref=db.backref('fetch_jobs', lazy='dynamic',
|
|
cascade='all, delete-orphan'))
|
|
queued_by = db.relationship('User', foreign_keys=[queued_by_id])
|
|
|
|
def __repr__(self):
|
|
return f'<InstructionFetchJob set={self.set_id} {self.status}>'
|
|
|
|
# -- helpers ------------------------------------------------------------
|
|
|
|
@property
|
|
def badge(self):
|
|
return STATUS_BADGES.get(self.status, 'secondary')
|
|
|
|
@property
|
|
def is_open(self):
|
|
return self.status in OPEN_STATUSES
|
|
|
|
@classmethod
|
|
def open_for_set(cls, set_id):
|
|
"""The queued/running job for a set, if there is one."""
|
|
return cls.query.filter(cls.set_id == set_id,
|
|
cls.status.in_(OPEN_STATUSES)).first()
|
|
|
|
@classmethod
|
|
def counts(cls):
|
|
"""Row counts per status, with every status present."""
|
|
from sqlalchemy import func
|
|
rows = db.session.query(cls.status, func.count(cls.id)).group_by(
|
|
cls.status).all()
|
|
counts = {status: 0 for status in STATUSES}
|
|
for status, count in rows:
|
|
counts[status] = count
|
|
counts['total'] = sum(counts[s] for s in STATUSES)
|
|
return counts
|
|
|
|
def to_dict(self):
|
|
lego_set = self.lego_set
|
|
return {
|
|
'id': self.id,
|
|
'set_id': self.set_id,
|
|
'set_number': lego_set.set_number if lego_set else None,
|
|
'set_name': lego_set.set_name if lego_set else None,
|
|
'status': self.status,
|
|
'badge': self.badge,
|
|
'attempts': self.attempts,
|
|
'files_added': self.files_added,
|
|
'source': self.source,
|
|
'message': self.message,
|
|
'queued_at': self.queued_at.isoformat() if self.queued_at else None,
|
|
'finished_at': (self.finished_at.isoformat()
|
|
if self.finished_at else None),
|
|
}
|