Add automatic instruction fetcher for official sets
Downloads official instruction PDFs from LEGO, located via the Brickset API, one set per interval (10 minutes by default) so neither API gets hammered. Official sets only; MOCs are excluded throughout. - New admin page at /admin/instruction-fetch: queue builder, pacing controls, live countdown and per-set job history - Database-backed queue (instruction_fetch_jobs) and last-run timestamp, so pacing survives restarts - Background worker thread, plus 'flask instructions queue/run/status' for Task Scheduler or cron - SHA-256 dedupe (instructions.sha256), PDF header check, size cap and streaming downloads via temp files - brickset_api: getInstructions2 by set number and getInstructions by setID (the old code sent a set number to the setID method); getSets params now proper JSON; getKeyUsageStats added - Schema top-ups applied automatically at boot: new table plus instructions.sha256/source - Docker: fetcher env vars passed through compose, no worker thread in the entrypoint bootstrap
This commit is contained in:
1 parent
aac2b60517
commit
89f65e2d74
23 files changed
+2073
-76
No files matched your search
@@ -24,8 +24,18 @@ ADDITIVE_COLUMNS = {
|
||||
# site default instead.
|
||||
'visibility': "VARCHAR(10) NOT NULL DEFAULT 'public'",
|
||||
},
|
||||
'instructions': {
|
||||
# Written by the automatic fetcher; null on everything uploaded
|
||||
# before it existed, which is fine - dedupe just can't help there.
|
||||
'sha256': 'VARCHAR(64)',
|
||||
'source': 'VARCHAR(20)',
|
||||
},
|
||||
}
|
||||
|
||||
# Tables added by later versions. create_all() is not called at boot, so new
|
||||
# tables need an explicit (and idempotent) nudge.
|
||||
NEW_TABLES = ('app.models.instruction_fetch.InstructionFetchJob',)
|
||||
|
||||
|
||||
def ensure_schema(app):
|
||||
"""Add any missing columns. Returns a list of changes made."""
|
||||
@@ -38,6 +48,20 @@ def ensure_schema(app):
|
||||
app.logger.warning('Schema check skipped: %s', exc)
|
||||
return applied
|
||||
|
||||
for dotted in NEW_TABLES:
|
||||
module_path, _, class_name = dotted.rpartition('.')
|
||||
try:
|
||||
module = __import__(module_path, fromlist=[class_name])
|
||||
model = getattr(module, class_name)
|
||||
if model.__tablename__ not in existing_tables:
|
||||
model.__table__.create(db.engine, checkfirst=True)
|
||||
applied.append(f'table {model.__tablename__}')
|
||||
app.logger.info('Schema upgrade: created table %s',
|
||||
model.__tablename__)
|
||||
except Exception as exc:
|
||||
app.logger.error('Schema upgrade failed for %s: %s',
|
||||
dotted, exc)
|
||||
|
||||
for table, columns in ADDITIVE_COLUMNS.items():
|
||||
if table not in existing_tables:
|
||||
continue # create_all() will build it with everything present
|
||||
|
||||
Reference in new issue
Block a user