Files
lego-instructions-manager/app/templates/admin/instruction_fetch.html
T
jessikitty 89f65e2d74 Add automatic instruction fetcher for official sets
Downloads official instruction PDFs from LEGO, located via the Brickset API, one set per interval (10 minutes by default) so neither API gets hammered. Official sets only; MOCs are excluded throughout.

- New admin page at /admin/instruction-fetch: queue builder, pacing controls, live countdown and per-set job history

- Database-backed queue (instruction_fetch_jobs) and last-run timestamp, so pacing survives restarts

- Background worker thread, plus 'flask instructions queue/run/status' for Task Scheduler or cron

- SHA-256 dedupe (instructions.sha256), PDF header check, size cap and streaming downloads via temp files

- brickset_api: getInstructions2 by set number and getInstructions by setID (the old code sent a set number to the setID method); getSets params now proper JSON; getKeyUsageStats added

- Schema top-ups applied automatically at boot: new table plus instructions.sha256/source

- Docker: fetcher env vars passed through compose, no worker thread in the entrypoint bootstrap
2026-09-26 13:43:17 +10:00

353 lines
15 KiB
HTML

{# Instruction fetcher control panel.
Created by: Jess Rogerson (yelling commands at Claude.AI) #}
{% extends "base.html" %}
{% block title %}Instruction Fetcher - Admin{% endblock %}
{% block content %}
<div class="row mb-4">
<div class="col">
<h1><i class="bi bi-cloud-arrow-down"></i> Instruction Fetcher</h1>
<p class="text-muted mb-0">
Downloads official building instructions from LEGO, using Brickset
to find them. Official sets only &mdash; one set every
{{ summary.interval_minutes }} minutes so nobody's API gets upset.
</p>
</div>
<div class="col-auto">
<a href="{{ url_for('admin.dashboard') }}" class="btn btn-outline-secondary">
<i class="bi bi-arrow-left"></i> Back
</a>
</div>
</div>
{% if not summary.brickset_configured %}
<div class="alert alert-danger">
<i class="bi bi-exclamation-triangle"></i>
<strong>Brickset API not configured.</strong>
Add <code>BRICKSET_API_KEY</code>, <code>BRICKSET_USERNAME</code> and
<code>BRICKSET_PASSWORD</code> to your <code>.env</code> and restart.
</div>
{% endif %}
{% if not worker_running %}
<div class="alert alert-warning">
<i class="bi bi-info-circle"></i>
The background worker thread isn't running in this process. Queued sets
will only move when you press <em>Run one now</em>, or when
<code>flask instructions run</code> is called from Task Scheduler.
</div>
{% endif %}
<!-- Status strip -->
<div class="row g-3 mb-4">
<div class="col-md-3">
<div class="card h-100">
<div class="card-body">
<div class="text-muted small text-uppercase">Status</div>
<div class="h4 mb-0" id="stat-status">
{% if summary.enabled %}
<span class="badge bg-success">Running</span>
{% else %}
<span class="badge bg-secondary">Paused</span>
{% endif %}
</div>
</div>
</div>
</div>
<div class="col-md-3">
<div class="card h-100">
<div class="card-body">
<div class="text-muted small text-uppercase">Next fetch in</div>
<div class="h4 mb-0" id="stat-countdown">&mdash;</div>
</div>
</div>
</div>
<div class="col-md-3">
<div class="card h-100">
<div class="card-body">
<div class="text-muted small text-uppercase">In the queue</div>
<div class="h4 mb-0" id="stat-queued">{{ summary.counts.queued }}</div>
</div>
</div>
</div>
<div class="col-md-3">
<div class="card h-100">
<div class="card-body">
<div class="text-muted small text-uppercase">Official sets without a PDF</div>
<div class="h4 mb-0" id="stat-missing">{{ summary.sets_missing_pdfs }}</div>
</div>
</div>
</div>
</div>
<div class="row">
<!-- Queue controls -->
<div class="col-lg-6 mb-4">
<div class="card h-100">
<div class="card-header">
<h5 class="mb-0"><i class="bi bi-list-ol"></i> Build the queue</h5>
</div>
<div class="card-body">
<p class="text-muted">
{{ summary.official_sets }} official set(s) in the library,
{{ summary.sets_missing_pdfs }} with no PDF instructions yet.
</p>
<form method="POST" action="{{ url_for('instruction_fetch.queue') }}"
class="mb-3">
<input type="hidden" name="scope" value="missing">
<div class="input-group">
<span class="input-group-text">Limit</span>
<input type="number" class="form-control" name="limit"
min="1" placeholder="all" style="max-width: 120px;">
<button type="submit" class="btn btn-primary">
<i class="bi bi-plus-circle"></i> Queue sets missing instructions
</button>
</div>
<div class="form-text">Newest sets first. Skips anything already queued.</div>
</form>
<form method="POST" action="{{ url_for('instruction_fetch.queue') }}"
class="mb-3"
onsubmit="return confirm('Queue every official set, including ones that already have instructions?');">
<input type="hidden" name="scope" value="all">
<button type="submit" class="btn btn-outline-secondary btn-sm">
<i class="bi bi-arrow-repeat"></i> Queue all official sets
</button>
</form>
<form method="POST" action="{{ url_for('instruction_fetch.queue') }}">
<input type="hidden" name="scope" value="retry-failed">
<button type="submit" class="btn btn-outline-warning btn-sm">
<i class="bi bi-arrow-counterclockwise"></i>
Retry failed ({{ summary.counts.failed }})
</button>
</form>
<hr>
<form method="POST" action="{{ url_for('instruction_fetch.run_now') }}">
<div class="form-check mb-2">
<input class="form-check-input" type="checkbox"
id="force" name="force" value="1">
<label class="form-check-label" for="force">
Ignore the timer for this one
</label>
</div>
<button type="submit" class="btn btn-success"
{% if not summary.brickset_configured %}disabled{% endif %}>
<i class="bi bi-play-fill"></i> Run one now
</button>
</form>
</div>
</div>
</div>
<!-- Settings -->
<div class="col-lg-6 mb-4">
<div class="card h-100">
<div class="card-header">
<h5 class="mb-0"><i class="bi bi-sliders"></i> Pacing</h5>
</div>
<div class="card-body">
<form method="POST" action="{{ url_for('instruction_fetch.save_settings') }}">
<div class="form-check form-switch mb-3">
<input class="form-check-input" type="checkbox" role="switch"
id="fetch_enabled" name="fetch_enabled" value="1"
{% if summary.enabled %}checked{% endif %}>
<label class="form-check-label" for="fetch_enabled">
<strong>Work through the queue automatically</strong>
</label>
<div class="form-text">
Turn off to pause. The queue is kept either way.
</div>
</div>
<div class="mb-3">
<label class="form-label" for="fetch_interval_minutes">
<strong>Minutes between sets</strong>
</label>
<input type="number" class="form-control" min="1" max="1440"
id="fetch_interval_minutes" name="fetch_interval_minutes"
value="{{ summary.interval_minutes }}">
<div class="form-text">
10 is a polite default (144 sets a day). The clock is stored
in the database, so a restart doesn't reset it.
</div>
</div>
<div class="mb-3">
<label class="form-label" for="fetch_file_delay_seconds">
<strong>Seconds between PDFs of the same set</strong>
</label>
<input type="number" class="form-control" min="0" max="300"
id="fetch_file_delay_seconds" name="fetch_file_delay_seconds"
value="{{ summary.file_delay_seconds }}">
<div class="form-text">
Big sets have several books; this spaces those downloads out.
</div>
</div>
<div class="mb-3">
<label class="form-label" for="fetch_max_attempts">
<strong>Attempts before giving up</strong>
</label>
<input type="number" class="form-control" min="1" max="10"
id="fetch_max_attempts" name="fetch_max_attempts"
value="{{ summary.max_attempts }}">
</div>
<div class="form-check mb-2">
<input class="form-check-input" type="checkbox"
id="fetch_skip_sets_with_pdfs" name="fetch_skip_sets_with_pdfs"
value="1" {% if summary.skip_sets_with_pdfs %}checked{% endif %}>
<label class="form-check-label" for="fetch_skip_sets_with_pdfs">
Skip sets that already have a PDF
</label>
</div>
<div class="form-check mb-3">
<input class="form-check-input" type="checkbox"
id="fetch_allow_setid_lookup" name="fetch_allow_setid_lookup"
value="1" {% if summary.allow_setid_lookup %}checked{% endif %}>
<label class="form-check-label" for="fetch_allow_setid_lookup">
Allow set-ID lookups when a number isn't found
</label>
<div class="form-text">
This is the only call that counts against the Brickset
daily quota. The ID is saved to the set afterwards.
</div>
</div>
<button type="submit" class="btn btn-primary">
<i class="bi bi-save"></i> Save
</button>
</form>
</div>
</div>
</div>
</div>
<!-- History -->
<div class="card">
<div class="card-header d-flex justify-content-between align-items-center">
<h5 class="mb-0"><i class="bi bi-clock-history"></i> Recent jobs</h5>
<div>
<form method="POST" action="{{ url_for('instruction_fetch.clear') }}"
class="d-inline">
<input type="hidden" name="scope" value="finished">
<button type="submit" class="btn btn-sm btn-outline-secondary">
Clear finished
</button>
</form>
</div>
</div>
<div class="table-responsive">
<table class="table table-hover mb-0 align-middle">
<thead>
<tr>
<th>Set</th>
<th>Status</th>
<th class="text-center">Files</th>
<th class="text-center">Tries</th>
<th>Result</th>
<th></th>
</tr>
</thead>
<tbody>
{% for job in jobs %}
<tr>
<td>
{% if job.lego_set %}
<a href="{{ url_for('sets.view_set', set_id=job.set_id) }}">
<strong>{{ job.lego_set.set_number }}</strong>
</a>
<div class="small text-muted">{{ job.lego_set.set_name }}</div>
{% else %}
<span class="text-muted">deleted set</span>
{% endif %}
</td>
<td><span class="badge bg-{{ job.badge }}">{{ job.status }}</span></td>
<td class="text-center">{{ job.files_added }}</td>
<td class="text-center">{{ job.attempts }}</td>
<td class="small">{{ job.message or '—' }}</td>
<td class="text-end">
{% if job.is_open %}
<form method="POST"
action="{{ url_for('instruction_fetch.cancel_job', job_id=job.id) }}"
class="d-inline">
<button class="btn btn-sm btn-outline-danger">Cancel</button>
</form>
{% else %}
<form method="POST"
action="{{ url_for('instruction_fetch.retry_job', job_id=job.id) }}"
class="d-inline">
<button class="btn btn-sm btn-outline-secondary">Retry</button>
</form>
{% endif %}
</td>
</tr>
{% else %}
<tr>
<td colspan="6" class="text-center text-muted py-4">
Nothing queued yet.
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
<script>
(function () {
const statusUrl = "{{ url_for('instruction_fetch.status') }}";
let remaining = {{ summary.next_run_in_seconds }};
let enabled = {{ 'true' if summary.enabled else 'false' }};
let lastFinished = {{ summary.counts.done + summary.counts.empty + summary.counts.failed }};
function paintCountdown() {
const el = document.getElementById('stat-countdown');
if (!el) return;
if (!enabled) { el.textContent = 'paused'; return; }
if (remaining <= 0) { el.textContent = 'due now'; return; }
const m = Math.floor(remaining / 60);
const s = remaining % 60;
el.textContent = m + 'm ' + String(s).padStart(2, '0') + 's';
}
setInterval(function () {
if (remaining > 0) { remaining -= 1; }
paintCountdown();
}, 1000);
async function poll() {
try {
const res = await fetch(statusUrl, {headers: {'Accept': 'application/json'}});
if (!res.ok) return;
const data = await res.json();
remaining = data.next_run_in_seconds;
enabled = data.enabled;
document.getElementById('stat-queued').textContent = data.counts.queued;
document.getElementById('stat-missing').textContent = data.sets_missing_pdfs;
document.getElementById('stat-status').innerHTML = data.enabled
? '<span class="badge bg-success">Running</span>'
: '<span class="badge bg-secondary">Paused</span>';
paintCountdown();
// Reload once a job finishes so the table stays honest.
const finished = data.counts.done + data.counts.empty + data.counts.failed;
if (finished !== lastFinished) {
lastFinished = finished;
window.location.reload();
}
} catch (err) { /* offline for a moment; try again next tick */ }
}
paintCountdown();
setInterval(poll, 15000);
})();
</script>
{% endblock %}