Add automatic instruction fetcher for official sets

Downloads official instruction PDFs from LEGO, located via the Brickset API, one set per interval (10 minutes by default) so neither API gets hammered. Official sets only; MOCs are excluded throughout. - New admin page at /admin/instruction-fetch: queue builder, pacing controls, live countdown and per-set job history - Database-backed queue (instruction_fetch_jobs) and last-run timestamp, so pacing survives restarts - Background worker thread, plus 'flask instructions queue/run/status' for Task Scheduler or cron - SHA-256 dedupe (instructions.sha256), PDF header check, size cap and streaming downloads via temp files - brickset_api: getInstructions2 by set number and getInstructions by setID (the old code sent a set number to the setID method); getSets params now proper JSON; getKeyUsageStats added - Schema top-ups applied automatically at boot: new table plus instructions.sha256/source - Docker: fetcher env vars passed through compose, no worker thread in the entrypoint bootstrap
2026-09-26 13:43:17 +10:00
1 parent aac2b60517
commit 89f65e2d74
@@ -0,0 +1,121 @@
+"""
+Queue of "go and fetch the instructions" jobs for official sets.
+
+One row per set per attempt run. The queue lives in the database rather than
+in memory so a restart doesn't lose the backlog, and so the admin page can
+show what happened to every set without scraping the log file.
+
+Only official sets ever get queued — MOCs have no Brickset entry, so there is
+nothing to fetch.
+
+Created by: Jess Rogerson (yelling commands at Claude.AI)
+"""
+
+from datetime import datetime
+
+from app import db
+
+# queued    - waiting its turn
+# running   - being worked on right now
+# done      - at least one instruction file was saved
+# empty     - Brickset has no instructions on file for this set
+# failed    - gave up after max attempts
+# cancelled - removed from the queue by an admin
+STATUSES = ('queued', 'running', 'done', 'empty', 'failed', 'cancelled')
+
+OPEN_STATUSES = ('queued', 'running')
+
+STATUS_BADGES = {
+    'queued': 'secondary',
+    'running': 'primary',
+    'done': 'success',
+    'empty': 'warning',
+    'failed': 'danger',
+    'cancelled': 'dark',
+}
+
+
+class InstructionFetchJob(db.Model):
+    """A single set queued for an automatic instruction download."""
+
+    __tablename__ = 'instruction_fetch_jobs'
+
+    id = db.Column(db.Integer, primary_key=True)
+    set_id = db.Column(db.Integer, db.ForeignKey('sets.id'), nullable=False,
+                       index=True)
+
+    status = db.Column(db.String(12), nullable=False, default='queued',
+                       server_default='queued', index=True)
+    attempts = db.Column(db.Integer, nullable=False, default=0,
+                         server_default='0')
+    files_added = db.Column(db.Integer, nullable=False, default=0,
+                            server_default='0')
+    # 'brickset' for now; the column exists so a future LEGO-direct or
+    # Rebrickable path can be told apart in the history.
+    source = db.Column(db.String(20), nullable=True)
+    message = db.Column(db.Text, nullable=True)
+
+    queued_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow,
+                          index=True)
+    # Retry backoff: a job is invisible to the worker until this passes.
+    not_before = db.Column(db.DateTime, nullable=True)
+    started_at = db.Column(db.DateTime, nullable=True)
+    finished_at = db.Column(db.DateTime, nullable=True)
+
+    queued_by_id = db.Column(db.Integer, db.ForeignKey('users.id'),
+                             nullable=True)
+
+    lego_set = db.relationship(
+        'Set',
+        backref=db.backref('fetch_jobs', lazy='dynamic',
+                           cascade='all, delete-orphan'))
+    queued_by = db.relationship('User', foreign_keys=[queued_by_id])
+
+    def __repr__(self):
+        return f'<InstructionFetchJob set={self.set_id} {self.status}>'
+
+    # -- helpers ------------------------------------------------------------
+
+    @property
+    def badge(self):
+        return STATUS_BADGES.get(self.status, 'secondary')
+
+    @property
+    def is_open(self):
+        return self.status in OPEN_STATUSES
+
+    @classmethod
+    def open_for_set(cls, set_id):
+        """The queued/running job for a set, if there is one."""
+        return cls.query.filter(cls.set_id == set_id,
+                                cls.status.in_(OPEN_STATUSES)).first()
+
+    @classmethod
+    def counts(cls):
+        """Row counts per status, with every status present."""
+        from sqlalchemy import func
+        rows = db.session.query(cls.status, func.count(cls.id)).group_by(
+            cls.status).all()
+        counts = {status: 0 for status in STATUSES}
+        for status, count in rows:
+            counts[status] = count
+        counts['total'] = sum(counts[s] for s in STATUSES)
+        return counts
+
+    def to_dict(self):
+        lego_set = self.lego_set
+        return {
+            'id': self.id,
+            'set_id': self.set_id,
+            'set_number': lego_set.set_number if lego_set else None,
+            'set_name': lego_set.set_name if lego_set else None,
+            'status': self.status,
+            'badge': self.badge,
+            'attempts': self.attempts,
+            'files_added': self.files_added,
+            'source': self.source,
+            'message': self.message,
+            'queued_at': self.queued_at.isoformat() if self.queued_at else None,
+            'finished_at': (self.finished_at.isoformat()
+                            if self.finished_at else None),
+        }