Add owner-only logging and Brickset API call tracking

LIM configured no logging at all, so the instruction fetcher had no trail to follow. Adds a rotating file handler on the root logger plus two owner-only admin pages.

- Admin > Logs: filter by level, source and text; tracebacks kept with their error; auto-refresh, raw download, rotated-file browsing and rotate-now

- Admin > Logs > Brickset API calls: every outbound call recorded with method, params, caller, duration, result count and error, plus daily quota accounting (only getSets counts) and a comparison against getKeyUsageStats

- New owner concept (services/ownership.py): a single account, distinct from is_admin, since these pages expose other admins activity. Defaults to the first admin; flask owner show/set to change

- Services now log through module loggers; retention setting and pruning for call records

- New CLI: flask owner show/set, flask logs path/prune

- New table brickset_api_calls, created automatically at boot; LOG_* env vars, /data/logs in Docker
This commit is contained in:
jessikitty committed 2026-09-27 10:42:49 +10:00
1 parent 89f65e2d74
commit f7c20de5ac
27 files changed
+1914 -35

No files matched your search

+298
View File
@@ -0,0 +1,298 @@
{# Brickset API call history - owner only.
Created by: Jess Rogerson (yelling commands at Claude.AI) #}
{% extends "base.html" %}
{% block title %}Brickset API calls - Admin{% endblock %}
{% block content %}
<div class="row mb-3">
<div class="col">
<h1><i class="bi bi-cloud"></i> Brickset API calls</h1>
<p class="text-muted mb-0">
Every request LIM has made. Only <code>getSets</code> counts
against the daily key limit.
</p>
</div>
<div class="col-auto">
<a href="{{ url_for('admin.dashboard') }}" class="btn btn-outline-secondary">
<i class="bi bi-arrow-left"></i> Back
</a>
</div>
</div>
<ul class="nav nav-tabs mb-3">
<li class="nav-item">
<a class="nav-link" href="{{ url_for('logs.index') }}">
<i class="bi bi-journal-text"></i> Application log
</a>
</li>
<li class="nav-item">
<a class="nav-link active" href="{{ url_for('logs.brickset') }}">
<i class="bi bi-cloud"></i> Brickset API calls
</a>
</li>
</ul>
<div class="row g-3 mb-4">
<div class="col-md-3">
<div class="card h-100">
<div class="card-body">
<div class="text-muted small text-uppercase">Quota calls today</div>
<div class="h3 mb-0">{{ stats.today_quota }}</div>
<div class="small text-muted">getSets only</div>
</div>
</div>
</div>
<div class="col-md-3">
<div class="card h-100">
<div class="card-body">
<div class="text-muted small text-uppercase">All calls today</div>
<div class="h3 mb-0">{{ stats.today }}</div>
<div class="small text-muted">quota-free included</div>
</div>
</div>
</div>
<div class="col-md-3">
<div class="card h-100">
<div class="card-body">
<div class="text-muted small text-uppercase">Last {{ stats.days }} days</div>
<div class="h3 mb-0">{{ stats.window }}</div>
<div class="small text-muted">{{ stats.total }} recorded in total</div>
</div>
</div>
</div>
<div class="col-md-3">
<div class="card h-100">
<div class="card-body">
<div class="text-muted small text-uppercase">Failures</div>
<div class="h3 mb-0 {% if stats.window_failed %}text-danger{% endif %}">
{{ stats.window_failed }}
</div>
<div class="small text-muted">in the last {{ stats.days }} days</div>
</div>
</div>
</div>
</div>
<div class="row">
<div class="col-lg-5 mb-4">
<div class="card h-100">
<div class="card-header">
<h6 class="mb-0">By method, last {{ stats.days }} days</h6>
</div>
<div class="table-responsive">
<table class="table table-sm mb-0">
<thead>
<tr>
<th>Method</th>
<th class="text-end">Calls</th>
<th class="text-end">Avg</th>
<th class="text-center">Quota</th>
</tr>
</thead>
<tbody>
{% for row in stats.by_method %}
<tr>
<td><code>{{ row.method }}</code></td>
<td class="text-end">{{ row.count }}</td>
<td class="text-end text-muted">
{% if row.avg_ms %}{{ row.avg_ms }} ms{% else %}&mdash;{% endif %}
</td>
<td class="text-center">
{% if row.method == 'getSets' %}
<span class="badge bg-warning">yes</span>
{% else %}
<span class="text-muted">no</span>
{% endif %}
</td>
</tr>
{% else %}
<tr>
<td colspan="4" class="text-center text-muted py-3">
No calls recorded yet.
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
<div class="col-lg-4 mb-4">
<div class="card h-100">
<div class="card-header">
<h6 class="mb-0">Quota calls per day</h6>
</div>
<div class="table-responsive">
<table class="table table-sm mb-0">
<tbody>
{% for row in stats.quota_per_day %}
<tr>
<td>{{ row.date }}</td>
<td class="text-end"><strong>{{ row.count }}</strong></td>
</tr>
{% else %}
<tr>
<td class="text-center text-muted py-3">
No quota-counting calls in this window.
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
<div class="card-footer">
<form method="POST" action="{{ url_for('logs.brickset_quota') }}">
<button class="btn btn-sm btn-outline-primary w-100">
<i class="bi bi-arrow-repeat"></i> Compare with Brickset's own figures
</button>
</form>
</div>
</div>
</div>
<div class="col-lg-3 mb-4">
<div class="card h-100">
<div class="card-header">
<h6 class="mb-0">Retention</h6>
</div>
<div class="card-body">
<form method="POST" action="{{ url_for('logs.brickset_retention') }}"
class="mb-3">
<label class="form-label small" for="days">Keep records for</label>
<div class="input-group input-group-sm">
<input type="number" class="form-control" id="days" name="days"
value="{{ retention_days }}" min="1" max="3650">
<span class="input-group-text">days</span>
</div>
<button class="btn btn-sm btn-primary mt-2 w-100">Save</button>
</form>
<form method="POST" action="{{ url_for('logs.brickset_prune') }}"
onsubmit="return confirm('Delete records older than {{ retention_days }} days?');">
<button class="btn btn-sm btn-outline-danger w-100">
<i class="bi bi-trash"></i> Prune now
</button>
</form>
</div>
</div>
</div>
</div>
<div class="card">
<div class="card-header">
<form method="GET" class="row g-2 align-items-end">
<div class="col-auto">
<label class="form-label small mb-1" for="method">Method</label>
<select class="form-select form-select-sm" id="method" name="method">
<option value="">All</option>
{% for name in methods %}
<option value="{{ name }}" {% if name == method %}selected{% endif %}>
{{ name }}
</option>
{% endfor %}
</select>
</div>
<div class="col-auto">
<label class="form-label small mb-1" for="dayswin">Window</label>
<select class="form-select form-select-sm" id="dayswin" name="days">
{% for option in [1, 7, 30, 90] %}
<option value="{{ option }}" {% if option == days %}selected{% endif %}>
{{ option }} day{{ 's' if option > 1 }}
</option>
{% endfor %}
</select>
</div>
<div class="col-auto">
<div class="form-check mb-1">
<input class="form-check-input" type="checkbox" id="failed"
name="failed" value="1" {% if only_failed %}checked{% endif %}>
<label class="form-check-label small" for="failed">
failures only
</label>
</div>
</div>
<div class="col-auto">
<button class="btn btn-sm btn-primary">Filter</button>
</div>
</form>
</div>
<div class="table-responsive">
<table class="table table-sm table-hover mb-0" style="font-size: 0.85rem;">
<thead>
<tr>
<th>When (UTC)</th>
<th>Method</th>
<th>Asked</th>
<th>From</th>
<th class="text-center">Result</th>
<th class="text-end">Took</th>
</tr>
</thead>
<tbody>
{% for call in calls %}
<tr class="{% if not call.ok %}table-danger{% endif %}">
<td class="text-nowrap text-muted">
{{ call.called_at.strftime('%Y-%m-%d %H:%M:%S') }}
</td>
<td>
<code>{{ call.method }}</code>
{% if call.quota_counted %}
<span class="badge bg-warning" title="Counts against the daily limit">q</span>
{% endif %}
</td>
<td class="text-truncate" style="max-width: 280px;"
title="{{ call.params }}">{{ call.params or '—' }}</td>
<td class="text-muted">{{ call.context or '—' }}</td>
<td class="text-center">
{% if call.ok %}
<span class="badge bg-success">
{% if call.result_count is not none %}{{ call.result_count }}{% else %}ok{% endif %}
</span>
{% else %}
<span class="badge bg-danger" title="{{ call.message }}">failed</span>
{% endif %}
</td>
<td class="text-end text-muted">
{% if call.duration_ms %}{{ call.duration_ms }} ms{% else %}&mdash;{% endif %}
</td>
</tr>
{% if not call.ok and call.message %}
<tr class="table-danger">
<td></td>
<td colspan="5" class="small text-danger">{{ call.message }}</td>
</tr>
{% endif %}
{% else %}
<tr>
<td colspan="6" class="text-center text-muted py-4">
No calls recorded yet.
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
{% if pagination.pages > 1 %}
<div class="card-footer d-flex justify-content-between align-items-center">
<span class="text-muted small">
Page {{ pagination.page }} of {{ pagination.pages }}
</span>
<div>
{% if pagination.has_prev %}
<a class="btn btn-sm btn-outline-secondary"
href="{{ url_for('logs.brickset', page=pagination.prev_num, method=method, days=days, failed='1' if only_failed else None) }}">
Newer
</a>
{% endif %}
{% if pagination.has_next %}
<a class="btn btn-sm btn-outline-secondary"
href="{{ url_for('logs.brickset', page=pagination.next_num, method=method, days=days, failed='1' if only_failed else None) }}">
Older
</a>
{% endif %}
</div>
</div>
{% endif %}
</div>
{% endblock %}
+7
View File
@@ -260,6 +260,13 @@
<i class="bi bi-cloud-arrow-down"></i> Instruction Fetcher
</a>
</div>
{% if current_user_is_owner %}
<div class="col-md-3 mb-2">
<a href="{{ url_for('logs.index') }}" class="btn btn-outline-dark w-100">
<i class="bi bi-journal-text"></i> Logs
</a>
</div>
{% endif %}
<div class="col-md-3 mb-2">
<a href="{{ url_for('admin.users') }}" class="btn btn-outline-primary w-100">
<i class="bi bi-people"></i> Manage Users
@@ -15,6 +15,15 @@
</p>
</div>
<div class="col-auto">
{% if current_user_is_owner %}
<a href="{{ url_for('logs.index', source='app.services.instruction_fetcher') }}"
class="btn btn-outline-dark">
<i class="bi bi-journal-text"></i> Fetcher log
</a>
<a href="{{ url_for('logs.brickset') }}" class="btn btn-outline-dark">
<i class="bi bi-cloud"></i> API calls
</a>
{% endif %}
<a href="{{ url_for('admin.dashboard') }}" class="btn btn-outline-secondary">
<i class="bi bi-arrow-left"></i> Back
</a>
+222
View File
@@ -0,0 +1,222 @@
{# Application log viewer - owner only.
Created by: Jess Rogerson (yelling commands at Claude.AI) #}
{% extends "base.html" %}
{% block title %}Logs - Admin{% endblock %}
{% block content %}
<div class="row mb-3">
<div class="col">
<h1><i class="bi bi-journal-text"></i> Logs</h1>
<p class="text-muted mb-0">
<code>{{ log_path }}</code>
</p>
</div>
<div class="col-auto">
<a href="{{ url_for('admin.dashboard') }}" class="btn btn-outline-secondary">
<i class="bi bi-arrow-left"></i> Back
</a>
</div>
</div>
<ul class="nav nav-tabs mb-3">
<li class="nav-item">
<a class="nav-link active" href="{{ url_for('logs.index') }}">
<i class="bi bi-journal-text"></i> Application log
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{ url_for('logs.brickset') }}">
<i class="bi bi-cloud"></i> Brickset API calls
</a>
</li>
</ul>
{% if not configured %}
<div class="alert alert-warning">
<i class="bi bi-exclamation-triangle"></i>
No log file handler is active — check <code>LOG_FOLDER</code> is writable,
then restart LIM.
</div>
{% endif %}
<div class="card mb-3">
<div class="card-body">
<form method="GET" class="row g-2 align-items-end">
<div class="col-auto">
<label class="form-label small mb-1" for="level">Minimum level</label>
<select class="form-select form-select-sm" id="level" name="level">
{% for option in levels %}
<option value="{{ option }}" {% if option == level %}selected{% endif %}>
{{ option }}
</option>
{% endfor %}
</select>
</div>
<div class="col-auto">
<label class="form-label small mb-1" for="source">Source</label>
<select class="form-select form-select-sm" id="source" name="source">
<option value="">Everything</option>
<option value="app.services.instruction_fetcher"
{% if source == 'app.services.instruction_fetcher' %}selected{% endif %}>
Instruction fetcher
</option>
<option value="app.services.fetch_worker"
{% if source == 'app.services.fetch_worker' %}selected{% endif %}>
Fetch worker
</option>
<option value="app.services.brickset_api"
{% if source == 'app.services.brickset_api' %}selected{% endif %}>
Brickset API
</option>
<option value="app.services"
{% if source == 'app.services' %}selected{% endif %}>
All services
</option>
</select>
</div>
<div class="col">
<label class="form-label small mb-1" for="q">Contains</label>
<input type="text" class="form-control form-control-sm" id="q"
name="q" value="{{ search }}" placeholder="text to match">
</div>
<div class="col-auto">
<label class="form-label small mb-1" for="lines">Lines</label>
<input type="number" class="form-control form-control-sm" id="lines"
name="lines" value="{{ lines }}" min="25" max="2000"
style="width: 100px;">
</div>
{% if files|length > 1 %}
<div class="col-auto">
<label class="form-label small mb-1" for="file">File</label>
<select class="form-select form-select-sm" id="file" name="file">
{% for item in files %}
<option value="{{ item.name }}" {% if item.name == filename %}selected{% endif %}>
{{ item.name }} ({{ (item.size / 1024)|round(1) }} KB)
</option>
{% endfor %}
</select>
</div>
{% endif %}
<div class="col-auto">
<button type="submit" class="btn btn-sm btn-primary">
<i class="bi bi-funnel"></i> Apply
</button>
</div>
</form>
</div>
</div>
<div class="d-flex justify-content-between align-items-center mb-2">
<div class="text-muted small">
Showing <strong id="shown">{{ shown }}</strong> of {{ total }} entries in view
<span class="ms-3">
<input class="form-check-input" type="checkbox" id="autorefresh">
<label class="form-check-label" for="autorefresh">auto-refresh</label>
</span>
</div>
<div>
<a href="{{ url_for('logs.download', file=filename) }}"
class="btn btn-sm btn-outline-secondary">
<i class="bi bi-download"></i> Download
</a>
<form method="POST" action="{{ url_for('logs.rotate') }}" class="d-inline"
onsubmit="return confirm('Start a fresh log file?');">
<button class="btn btn-sm btn-outline-secondary">
<i class="bi bi-arrow-clockwise"></i> Rotate now
</button>
</form>
</div>
</div>
<div class="card">
<div class="table-responsive" style="max-height: 70vh; overflow-y: auto;">
<table class="table table-sm table-hover mb-0" style="font-family: var(--bs-font-monospace); font-size: 0.82rem;">
<thead class="sticky-top bg-body">
<tr>
<th style="width: 150px;">Time</th>
<th style="width: 90px;">Level</th>
<th style="width: 220px;">Source</th>
<th>Message</th>
</tr>
</thead>
<tbody id="logbody">
{% for entry in entries %}
<tr>
<td class="text-nowrap text-muted">{{ entry.time }}</td>
<td><span class="badge bg-{{ entry.badge }}">{{ entry.level }}</span></td>
<td class="text-muted text-truncate" style="max-width: 220px;"
title="{{ entry.logger }}">{{ entry.logger }}</td>
<td>
{{ entry.message }}
{% if entry.extra %}
<pre class="mb-0 mt-1 small text-danger">{{ entry.extra|join('\n') }}</pre>
{% endif %}
</td>
</tr>
{% else %}
<tr>
<td colspan="4" class="text-center text-muted py-4">
Nothing matches. Try a lower level or clear the filter.
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
<script>
(function () {
const params = new URLSearchParams(window.location.search);
const url = "{{ url_for('logs.tail_json') }}" + '?' + params.toString();
const body = document.getElementById('logbody');
const shown = document.getElementById('shown');
const toggle = document.getElementById('autorefresh');
let timer = null;
function escapeHtml(text) {
const div = document.createElement('div');
div.textContent = text;
return div.innerHTML;
}
const badges = {DEBUG: 'secondary', INFO: 'info', WARNING: 'warning',
ERROR: 'danger', CRITICAL: 'danger'};
async function refresh() {
try {
const res = await fetch(url, {headers: {'Accept': 'application/json'}});
if (!res.ok) return;
const data = await res.json();
shown.textContent = data.shown;
body.innerHTML = data.entries.map(function (e) {
const extra = e.extra && e.extra.length
? '<pre class="mb-0 mt-1 small text-danger">' +
escapeHtml(e.extra.join('\n')) + '</pre>'
: '';
return '<tr>' +
'<td class="text-nowrap text-muted">' + escapeHtml(e.time) + '</td>' +
'<td><span class="badge bg-' + (badges[e.level] || 'secondary') +
'">' + escapeHtml(e.level) + '</span></td>' +
'<td class="text-muted text-truncate" style="max-width:220px" title="' +
escapeHtml(e.logger) + '">' + escapeHtml(e.logger) + '</td>' +
'<td>' + escapeHtml(e.message) + extra + '</td>' +
'</tr>';
}).join('') ||
'<tr><td colspan="4" class="text-center text-muted py-4">Nothing matches.</td></tr>';
} catch (err) { /* try again next tick */ }
}
toggle.addEventListener('change', function () {
if (toggle.checked) {
refresh();
timer = setInterval(refresh, 10000);
} else if (timer) {
clearInterval(timer);
timer = null;
}
});
})();
</script>
{% endblock %}
+5
View File
@@ -11,6 +11,11 @@
<a href="{{ url_for('instruction_fetch.index') }}" class="btn btn-outline-warning me-2">
<i class="bi bi-cloud-arrow-down"></i> Instruction Fetcher
</a>
{% if current_user_is_owner %}
<a href="{{ url_for('logs.index') }}" class="btn btn-outline-dark me-2">
<i class="bi bi-journal-text"></i> Logs
</a>
{% endif %}
<a href="{{ url_for('backup.index') }}" class="btn btn-outline-danger me-2">
<i class="bi bi-archive"></i> Backup &amp; Restore
</a>