jessikitty
f7c20de5ac
LIM configured no logging at all, so the instruction fetcher had no trail to follow. Adds a rotating file handler on the root logger plus two owner-only admin pages. - Admin > Logs: filter by level, source and text; tracebacks kept with their error; auto-refresh, raw download, rotated-file browsing and rotate-now - Admin > Logs > Brickset API calls: every outbound call recorded with method, params, caller, duration, result count and error, plus daily quota accounting (only getSets counts) and a comparison against getKeyUsageStats - New owner concept (services/ownership.py): a single account, distinct from is_admin, since these pages expose other admins activity. Defaults to the first admin; flask owner show/set to change - Services now log through module loggers; retention setting and pruning for call records - New CLI: flask owner show/set, flask logs path/prune - New table brickset_api_calls, created automatically at boot; LOG_* env vars, /data/logs in Docker
362 lines
16 KiB
HTML
362 lines
16 KiB
HTML
{# Instruction fetcher control panel.
|
|
Created by: Jess Rogerson (yelling commands at Claude.AI) #}
|
|
{% extends "base.html" %}
|
|
|
|
{% block title %}Instruction Fetcher - Admin{% endblock %}
|
|
|
|
{% block content %}
|
|
<div class="row mb-4">
|
|
<div class="col">
|
|
<h1><i class="bi bi-cloud-arrow-down"></i> Instruction Fetcher</h1>
|
|
<p class="text-muted mb-0">
|
|
Downloads official building instructions from LEGO, using Brickset
|
|
to find them. Official sets only — one set every
|
|
{{ summary.interval_minutes }} minutes so nobody's API gets upset.
|
|
</p>
|
|
</div>
|
|
<div class="col-auto">
|
|
{% if current_user_is_owner %}
|
|
<a href="{{ url_for('logs.index', source='app.services.instruction_fetcher') }}"
|
|
class="btn btn-outline-dark">
|
|
<i class="bi bi-journal-text"></i> Fetcher log
|
|
</a>
|
|
<a href="{{ url_for('logs.brickset') }}" class="btn btn-outline-dark">
|
|
<i class="bi bi-cloud"></i> API calls
|
|
</a>
|
|
{% endif %}
|
|
<a href="{{ url_for('admin.dashboard') }}" class="btn btn-outline-secondary">
|
|
<i class="bi bi-arrow-left"></i> Back
|
|
</a>
|
|
</div>
|
|
</div>
|
|
|
|
{% if not summary.brickset_configured %}
|
|
<div class="alert alert-danger">
|
|
<i class="bi bi-exclamation-triangle"></i>
|
|
<strong>Brickset API not configured.</strong>
|
|
Add <code>BRICKSET_API_KEY</code>, <code>BRICKSET_USERNAME</code> and
|
|
<code>BRICKSET_PASSWORD</code> to your <code>.env</code> and restart.
|
|
</div>
|
|
{% endif %}
|
|
|
|
{% if not worker_running %}
|
|
<div class="alert alert-warning">
|
|
<i class="bi bi-info-circle"></i>
|
|
The background worker thread isn't running in this process. Queued sets
|
|
will only move when you press <em>Run one now</em>, or when
|
|
<code>flask instructions run</code> is called from Task Scheduler.
|
|
</div>
|
|
{% endif %}
|
|
|
|
<!-- Status strip -->
|
|
<div class="row g-3 mb-4">
|
|
<div class="col-md-3">
|
|
<div class="card h-100">
|
|
<div class="card-body">
|
|
<div class="text-muted small text-uppercase">Status</div>
|
|
<div class="h4 mb-0" id="stat-status">
|
|
{% if summary.enabled %}
|
|
<span class="badge bg-success">Running</span>
|
|
{% else %}
|
|
<span class="badge bg-secondary">Paused</span>
|
|
{% endif %}
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="col-md-3">
|
|
<div class="card h-100">
|
|
<div class="card-body">
|
|
<div class="text-muted small text-uppercase">Next fetch in</div>
|
|
<div class="h4 mb-0" id="stat-countdown">—</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="col-md-3">
|
|
<div class="card h-100">
|
|
<div class="card-body">
|
|
<div class="text-muted small text-uppercase">In the queue</div>
|
|
<div class="h4 mb-0" id="stat-queued">{{ summary.counts.queued }}</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="col-md-3">
|
|
<div class="card h-100">
|
|
<div class="card-body">
|
|
<div class="text-muted small text-uppercase">Official sets without a PDF</div>
|
|
<div class="h4 mb-0" id="stat-missing">{{ summary.sets_missing_pdfs }}</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="row">
|
|
<!-- Queue controls -->
|
|
<div class="col-lg-6 mb-4">
|
|
<div class="card h-100">
|
|
<div class="card-header">
|
|
<h5 class="mb-0"><i class="bi bi-list-ol"></i> Build the queue</h5>
|
|
</div>
|
|
<div class="card-body">
|
|
<p class="text-muted">
|
|
{{ summary.official_sets }} official set(s) in the library,
|
|
{{ summary.sets_missing_pdfs }} with no PDF instructions yet.
|
|
</p>
|
|
|
|
<form method="POST" action="{{ url_for('instruction_fetch.queue') }}"
|
|
class="mb-3">
|
|
<input type="hidden" name="scope" value="missing">
|
|
<div class="input-group">
|
|
<span class="input-group-text">Limit</span>
|
|
<input type="number" class="form-control" name="limit"
|
|
min="1" placeholder="all" style="max-width: 120px;">
|
|
<button type="submit" class="btn btn-primary">
|
|
<i class="bi bi-plus-circle"></i> Queue sets missing instructions
|
|
</button>
|
|
</div>
|
|
<div class="form-text">Newest sets first. Skips anything already queued.</div>
|
|
</form>
|
|
|
|
<form method="POST" action="{{ url_for('instruction_fetch.queue') }}"
|
|
class="mb-3"
|
|
onsubmit="return confirm('Queue every official set, including ones that already have instructions?');">
|
|
<input type="hidden" name="scope" value="all">
|
|
<button type="submit" class="btn btn-outline-secondary btn-sm">
|
|
<i class="bi bi-arrow-repeat"></i> Queue all official sets
|
|
</button>
|
|
</form>
|
|
|
|
<form method="POST" action="{{ url_for('instruction_fetch.queue') }}">
|
|
<input type="hidden" name="scope" value="retry-failed">
|
|
<button type="submit" class="btn btn-outline-warning btn-sm">
|
|
<i class="bi bi-arrow-counterclockwise"></i>
|
|
Retry failed ({{ summary.counts.failed }})
|
|
</button>
|
|
</form>
|
|
|
|
<hr>
|
|
|
|
<form method="POST" action="{{ url_for('instruction_fetch.run_now') }}">
|
|
<div class="form-check mb-2">
|
|
<input class="form-check-input" type="checkbox"
|
|
id="force" name="force" value="1">
|
|
<label class="form-check-label" for="force">
|
|
Ignore the timer for this one
|
|
</label>
|
|
</div>
|
|
<button type="submit" class="btn btn-success"
|
|
{% if not summary.brickset_configured %}disabled{% endif %}>
|
|
<i class="bi bi-play-fill"></i> Run one now
|
|
</button>
|
|
</form>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Settings -->
|
|
<div class="col-lg-6 mb-4">
|
|
<div class="card h-100">
|
|
<div class="card-header">
|
|
<h5 class="mb-0"><i class="bi bi-sliders"></i> Pacing</h5>
|
|
</div>
|
|
<div class="card-body">
|
|
<form method="POST" action="{{ url_for('instruction_fetch.save_settings') }}">
|
|
<div class="form-check form-switch mb-3">
|
|
<input class="form-check-input" type="checkbox" role="switch"
|
|
id="fetch_enabled" name="fetch_enabled" value="1"
|
|
{% if summary.enabled %}checked{% endif %}>
|
|
<label class="form-check-label" for="fetch_enabled">
|
|
<strong>Work through the queue automatically</strong>
|
|
</label>
|
|
<div class="form-text">
|
|
Turn off to pause. The queue is kept either way.
|
|
</div>
|
|
</div>
|
|
|
|
<div class="mb-3">
|
|
<label class="form-label" for="fetch_interval_minutes">
|
|
<strong>Minutes between sets</strong>
|
|
</label>
|
|
<input type="number" class="form-control" min="1" max="1440"
|
|
id="fetch_interval_minutes" name="fetch_interval_minutes"
|
|
value="{{ summary.interval_minutes }}">
|
|
<div class="form-text">
|
|
10 is a polite default (144 sets a day). The clock is stored
|
|
in the database, so a restart doesn't reset it.
|
|
</div>
|
|
</div>
|
|
|
|
<div class="mb-3">
|
|
<label class="form-label" for="fetch_file_delay_seconds">
|
|
<strong>Seconds between PDFs of the same set</strong>
|
|
</label>
|
|
<input type="number" class="form-control" min="0" max="300"
|
|
id="fetch_file_delay_seconds" name="fetch_file_delay_seconds"
|
|
value="{{ summary.file_delay_seconds }}">
|
|
<div class="form-text">
|
|
Big sets have several books; this spaces those downloads out.
|
|
</div>
|
|
</div>
|
|
|
|
<div class="mb-3">
|
|
<label class="form-label" for="fetch_max_attempts">
|
|
<strong>Attempts before giving up</strong>
|
|
</label>
|
|
<input type="number" class="form-control" min="1" max="10"
|
|
id="fetch_max_attempts" name="fetch_max_attempts"
|
|
value="{{ summary.max_attempts }}">
|
|
</div>
|
|
|
|
<div class="form-check mb-2">
|
|
<input class="form-check-input" type="checkbox"
|
|
id="fetch_skip_sets_with_pdfs" name="fetch_skip_sets_with_pdfs"
|
|
value="1" {% if summary.skip_sets_with_pdfs %}checked{% endif %}>
|
|
<label class="form-check-label" for="fetch_skip_sets_with_pdfs">
|
|
Skip sets that already have a PDF
|
|
</label>
|
|
</div>
|
|
|
|
<div class="form-check mb-3">
|
|
<input class="form-check-input" type="checkbox"
|
|
id="fetch_allow_setid_lookup" name="fetch_allow_setid_lookup"
|
|
value="1" {% if summary.allow_setid_lookup %}checked{% endif %}>
|
|
<label class="form-check-label" for="fetch_allow_setid_lookup">
|
|
Allow set-ID lookups when a number isn't found
|
|
</label>
|
|
<div class="form-text">
|
|
This is the only call that counts against the Brickset
|
|
daily quota. The ID is saved to the set afterwards.
|
|
</div>
|
|
</div>
|
|
|
|
<button type="submit" class="btn btn-primary">
|
|
<i class="bi bi-save"></i> Save
|
|
</button>
|
|
</form>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- History -->
|
|
<div class="card">
|
|
<div class="card-header d-flex justify-content-between align-items-center">
|
|
<h5 class="mb-0"><i class="bi bi-clock-history"></i> Recent jobs</h5>
|
|
<div>
|
|
<form method="POST" action="{{ url_for('instruction_fetch.clear') }}"
|
|
class="d-inline">
|
|
<input type="hidden" name="scope" value="finished">
|
|
<button type="submit" class="btn btn-sm btn-outline-secondary">
|
|
Clear finished
|
|
</button>
|
|
</form>
|
|
</div>
|
|
</div>
|
|
<div class="table-responsive">
|
|
<table class="table table-hover mb-0 align-middle">
|
|
<thead>
|
|
<tr>
|
|
<th>Set</th>
|
|
<th>Status</th>
|
|
<th class="text-center">Files</th>
|
|
<th class="text-center">Tries</th>
|
|
<th>Result</th>
|
|
<th></th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{% for job in jobs %}
|
|
<tr>
|
|
<td>
|
|
{% if job.lego_set %}
|
|
<a href="{{ url_for('sets.view_set', set_id=job.set_id) }}">
|
|
<strong>{{ job.lego_set.set_number }}</strong>
|
|
</a>
|
|
<div class="small text-muted">{{ job.lego_set.set_name }}</div>
|
|
{% else %}
|
|
<span class="text-muted">deleted set</span>
|
|
{% endif %}
|
|
</td>
|
|
<td><span class="badge bg-{{ job.badge }}">{{ job.status }}</span></td>
|
|
<td class="text-center">{{ job.files_added }}</td>
|
|
<td class="text-center">{{ job.attempts }}</td>
|
|
<td class="small">{{ job.message or '—' }}</td>
|
|
<td class="text-end">
|
|
{% if job.is_open %}
|
|
<form method="POST"
|
|
action="{{ url_for('instruction_fetch.cancel_job', job_id=job.id) }}"
|
|
class="d-inline">
|
|
<button class="btn btn-sm btn-outline-danger">Cancel</button>
|
|
</form>
|
|
{% else %}
|
|
<form method="POST"
|
|
action="{{ url_for('instruction_fetch.retry_job', job_id=job.id) }}"
|
|
class="d-inline">
|
|
<button class="btn btn-sm btn-outline-secondary">Retry</button>
|
|
</form>
|
|
{% endif %}
|
|
</td>
|
|
</tr>
|
|
{% else %}
|
|
<tr>
|
|
<td colspan="6" class="text-center text-muted py-4">
|
|
Nothing queued yet.
|
|
</td>
|
|
</tr>
|
|
{% endfor %}
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</div>
|
|
|
|
<script>
|
|
(function () {
|
|
const statusUrl = "{{ url_for('instruction_fetch.status') }}";
|
|
let remaining = {{ summary.next_run_in_seconds }};
|
|
let enabled = {{ 'true' if summary.enabled else 'false' }};
|
|
let lastFinished = {{ summary.counts.done + summary.counts.empty + summary.counts.failed }};
|
|
|
|
function paintCountdown() {
|
|
const el = document.getElementById('stat-countdown');
|
|
if (!el) return;
|
|
if (!enabled) { el.textContent = 'paused'; return; }
|
|
if (remaining <= 0) { el.textContent = 'due now'; return; }
|
|
const m = Math.floor(remaining / 60);
|
|
const s = remaining % 60;
|
|
el.textContent = m + 'm ' + String(s).padStart(2, '0') + 's';
|
|
}
|
|
|
|
setInterval(function () {
|
|
if (remaining > 0) { remaining -= 1; }
|
|
paintCountdown();
|
|
}, 1000);
|
|
|
|
async function poll() {
|
|
try {
|
|
const res = await fetch(statusUrl, {headers: {'Accept': 'application/json'}});
|
|
if (!res.ok) return;
|
|
const data = await res.json();
|
|
remaining = data.next_run_in_seconds;
|
|
enabled = data.enabled;
|
|
document.getElementById('stat-queued').textContent = data.counts.queued;
|
|
document.getElementById('stat-missing').textContent = data.sets_missing_pdfs;
|
|
document.getElementById('stat-status').innerHTML = data.enabled
|
|
? '<span class="badge bg-success">Running</span>'
|
|
: '<span class="badge bg-secondary">Paused</span>';
|
|
paintCountdown();
|
|
|
|
// Reload once a job finishes so the table stays honest.
|
|
const finished = data.counts.done + data.counts.empty + data.counts.failed;
|
|
if (finished !== lastFinished) {
|
|
lastFinished = finished;
|
|
window.location.reload();
|
|
}
|
|
} catch (err) { /* offline for a moment; try again next tick */ }
|
|
}
|
|
|
|
paintCountdown();
|
|
setInterval(poll, 15000);
|
|
})();
|
|
</script>
|
|
{% endblock %}
|