Add owner-only logging and Brickset API call tracking

LIM configured no logging at all, so the instruction fetcher had no trail to follow. Adds a rotating file handler on the root logger plus two owner-only admin pages.

- Admin > Logs: filter by level, source and text; tracebacks kept with their error; auto-refresh, raw download, rotated-file browsing and rotate-now

- Admin > Logs > Brickset API calls: every outbound call recorded with method, params, caller, duration, result count and error, plus daily quota accounting (only getSets counts) and a comparison against getKeyUsageStats

- New owner concept (services/ownership.py): a single account, distinct from is_admin, since these pages expose other admins activity. Defaults to the first admin; flask owner show/set to change

- Services now log through module loggers; retention setting and pruning for call records

- New CLI: flask owner show/set, flask logs path/prune

- New table brickset_api_calls, created automatically at boot; LOG_* env vars, /data/logs in Docker

.env.docker.example

@@ -34,3 +34,13 @@ INSTRUCTION_FETCH_TICK_SECONDS=20
 # Largest instruction PDF to accept, in megabytes.
 FETCH_MAX_FILE_MB=150
 FETCH_TIMEOUT_SECONDS=60
+
+# --- logging ----------------------------------------------------------------
+# The admin log viewer (owner only) reads these files.
+LOG_FOLDER=/data/logs
+LOG_LEVEL=INFO
+# Rotate at this size, keeping LOG_BACKUP_COUNT old copies.
+LOG_MAX_BYTES=2097152
+LOG_BACKUP_COUNT=5
+# 1 to include per-request access logging (noisy).
+LOG_ACCESS=0

.env.example

@@ -38,3 +38,13 @@ INSTRUCTION_FETCH_TICK_SECONDS=20
 # Largest instruction PDF to accept, in megabytes.
 FETCH_MAX_FILE_MB=150
 FETCH_TIMEOUT_SECONDS=60
+
+# --- logging ----------------------------------------------------------------
+# The admin log viewer (owner only) reads these files.
+LOG_FOLDER=logs
+LOG_LEVEL=INFO
+# Rotate at this size, keeping LOG_BACKUP_COUNT old copies.
+LOG_MAX_BYTES=2097152
+LOG_BACKUP_COUNT=5
+# 1 to include per-request access logging (noisy).
+LOG_ACCESS=0

README.md

@@ -18,6 +18,7 @@ A Python web application for organizing and managing your LEGO instruction manua
 - 🖼️ **Image Gallery**: View instruction images in an organized gallery
 - 📄 **PDF Support**: View and download PDF instructions directly
 - 💾 **Backup & Restore**: One-click archive of the database plus every uploaded file, ready to restore anywhere
+- 📋 **Logging & API tracking**: Owner-only log viewer and a record of every Brickset call with daily quota accounting — see [docs/LOGGING.md](docs/LOGGING.md)
 - ⬇️ **Automatic Instruction Fetcher**: Downloads official instruction PDFs from LEGO (found via Brickset) for official sets, one every 10 minutes so nobody's API gets hammered — see [docs/INSTRUCTION_FETCHER.md](docs/INSTRUCTION_FETCHER.md)
 
 ## 🚀 Quick Start
@@ -188,6 +189,10 @@ flask --app run.py shell
 - page_number, user_id, uploaded_at
 - sha256, source (set by the automatic fetcher; null for manual uploads)
 
+### BricksetApiCall
+- id, called_at, method, params, context, ok, http_status
+- result_count, duration_ms, quota_counted, message, user_id
+
 ### InstructionFetchJob
 - id, set_id, status, attempts, files_added, source, message
 - queued_at, not_before, started_at, finished_at, queued_by_id

app/__init__.py

@@ -20,6 +20,10 @@ def create_app(config_name='default'):
     # Load configuration
     app.config.from_object(config[config_name])
 
+    # Logging first, so anything below can report problems.
+    from app.services.logging_setup import configure_logging
+    configure_logging(app)
+    
     # Initialize extensions
     db.init_app(app)
     migrate.init_app(app, db)
@@ -48,6 +52,7 @@ def create_app(config_name='default'):
     from app.routes.moc import bp as moc_bp
     from app.routes.backup import backup_bp
     from app.routes.instruction_fetch import instruction_fetch_bp
+    from app.routes.logs import logs_bp
     
     app.register_blueprint(auth_bp)
     app.register_blueprint(main_bp)
@@ -58,6 +63,7 @@ def create_app(config_name='default'):
     app.register_blueprint(moc_bp)
     app.register_blueprint(backup_bp)
     app.register_blueprint(instruction_fetch_bp)
+    app.register_blueprint(logs_bp)
     
     # Import models to ensure they're registered with SQLAlchemy
     from app.models.user import User
@@ -67,6 +73,7 @@ def create_app(config_name='default'):
     from app.models.set_share import SetShare
     from app.models.site_setting import SiteSetting
     from app.models.instruction_fetch import InstructionFetchJob
+    from app.models.api_call import BricksetApiCall
 
     # Add any columns missing from an older database (additive only).
     if app.config.get('AUTO_SCHEMA_UPGRADE', True):
@@ -103,6 +110,15 @@ def create_app(config_name='default'):
     from app.cli import register_cli
     register_cli(app)
 
+    # Templates need to know whether to show owner-only links.
+    @app.context_processor
+    def inject_ownership():
+        from app.services.ownership import is_owner
+        try:
+            return {'current_user_is_owner': is_owner()}
+        except Exception:
+            return {'current_user_is_owner': False}
+
     # Background ticker for the automatic instruction fetcher. It does
     # nothing at all until an admin enables it on the fetcher page.
     from app.services import fetch_worker

app/cli.py

@@ -12,6 +12,11 @@ Flask CLI commands for LIM backup and restore.
     flask instructions run [--count N] [--force]
     flask instructions status
 
+    flask owner show
+    flask owner set USERNAME
+    flask logs path [--tail N]
+    flask logs prune [--days N]
+
 `restore` is the path to use on a fresh Docker instance: no login required,
 and it works before any admin user exists.
 
@@ -267,6 +272,88 @@ def instructions_status():
                f"{summary['sets_missing_pdfs']} without a PDF")
 
 
+owner_cli = AppGroup('owner', help='Who owns this instance.')
+
+
+@owner_cli.command('show')
+def owner_show():
+    """Show the owner account (the only one who can read the logs)."""
+    from app.services.ownership import get_owner
+
+    owner = get_owner()
+    if owner is None:
+        click.secho('No users exist yet.', fg='yellow')
+        return
+    click.secho(f'Owner: {owner.username} <{owner.email}> (id {owner.id})',
+                fg='green')
+
+
+@owner_cli.command('set')
+@click.argument('username')
+def owner_set(username):
+    """Hand ownership to USERNAME (promotes them to admin if needed)."""
+    from app.models.user import User
+    from app.services.ownership import set_owner
+
+    user = User.query.filter_by(username=username).first()
+    if user is None:
+        click.secho(f'No user called {username!r}.', fg='red')
+        raise SystemExit(1)
+
+    set_owner(user)
+    click.secho(f'{user.username} is now the owner.', fg='green')
+
+
+logs_cli = AppGroup('logs', help='Application log file.')
+
+
+@logs_cli.command('path')
+@click.option('--tail', default=0, type=int,
+              help='Also print the last N lines.')
+def logs_path(tail):
+    """Where the log lives, and optionally its tail."""
+    import os
+
+    from flask import current_app
+
+    from app.services import logging_setup
+
+    path = logging_setup.log_path(current_app)
+    exists = os.path.isfile(path)
+    click.echo(f'  {path}')
+    click.echo(f'  exists: {exists}'
+               + (f'  size: {os.path.getsize(path)} bytes' if exists else ''))
+    click.echo(f'  handler active: {logging_setup.get_handler() is not None}')
+
+    if tail and exists:
+        entries, total = logging_setup.read_entries(path, max_entries=tail)
+        click.echo(f'\n  last {len(entries)} of {total} entries:\n')
+        for entry in entries:
+            click.echo(f"  {entry['time']} {entry['level']:<8} "
+                       f"{entry['logger']} | {entry['message']}")
+
+
+@logs_cli.command('prune')
+@click.option('--days', default=None, type=int,
+              help='Override the configured retention.')
+def logs_prune(days):
+    """Delete old Brickset API call records."""
+    from app.models.api_call import BricksetApiCall
+    from app.models.site_setting import SiteSetting
+
+    if days is None:
+        try:
+            days = int(SiteSetting.get('api_log_retention_days', '90'))
+        except (TypeError, ValueError):
+            days = 90
+
+    removed = BricksetApiCall.prune(days)
+    click.secho(f'Removed {removed} record(s) older than {days} days.',
+                fg='green')
+
+
 def register_cli(app):
     app.cli.add_command(backup_cli)
     app.cli.add_command(instructions_cli)
+    app.cli.add_command(owner_cli)
+    app.cli.add_command(logs_cli)

app/config.py

@@ -59,6 +59,17 @@ class Config:
     # stream straight to disk and never go through Flask's form parser.
     BACKUP_MAX_UPLOAD = int(os.environ.get('BACKUP_MAX_UPLOAD', 0))
 
+    # Logging
+    # Rotating file handler; the admin log viewer reads these files.
+    # Docker points LOG_FOLDER at the mounted volume (/data/logs).
+    LOG_FOLDER = _resolve_path(os.environ.get('LOG_FOLDER'),
+                               os.path.join(projectdir, 'logs'))
+    LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO')
+    LOG_MAX_BYTES = int(os.environ.get('LOG_MAX_BYTES', 2097152))  # 2MB
+    LOG_BACKUP_COUNT = int(os.environ.get('LOG_BACKUP_COUNT', 5))
+    # Per-request access logging is noisy; off unless you ask for it.
+    LOG_ACCESS = os.environ.get('LOG_ACCESS', '0') == '1'
+
     # Automatic instruction fetcher
     # Pacing (interval, delays, on/off) lives in site_settings so it can be
     # changed from the admin page. These are the hard limits and the switch

app/models/__init__.py

@@ -5,6 +5,7 @@ from app.models.extra_file import ExtraFile
 from app.models.set_share import SetShare
 from app.models.site_setting import SiteSetting
 from app.models.instruction_fetch import InstructionFetchJob
+from app.models.api_call import BricksetApiCall
 
 __all__ = ['User', 'Set', 'Instruction', 'ExtraFile', 'SetShare',
-           'SiteSetting', 'InstructionFetchJob']
+           'SiteSetting', 'InstructionFetchJob', 'BricksetApiCall']

app/models/api_call.py

@@ -0,0 +1,157 @@
+"""
+A record of every call LIM makes to the Brickset API.
+
+Two reasons this exists. First, Brickset's daily key limit applies to getSets
+and nothing else, so knowing how many of those went out today is the only
+number that matters for staying under it — and Brickset's own
+getKeyUsageStats only reports whole days, after the fact. Second, when the
+instruction fetcher comes back empty it is useful to see exactly what was
+asked and what came back.
+
+Rows are small and pruned on a retention setting (90 days by default).
+
+Created by: Jess Rogerson (yelling commands at Claude.AI)
+"""
+
+from datetime import datetime, timedelta
+
+from app import db
+
+# The only method that counts against the daily API key quota.
+QUOTA_METHODS = {'getSets'}
+
+
+class BricksetApiCall(db.Model):
+    """One outbound request to brickset.com."""
+
+    __tablename__ = 'brickset_api_calls'
+
+    id = db.Column(db.Integer, primary_key=True)
+    called_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow,
+                          index=True)
+
+    method = db.Column(db.String(40), nullable=False, index=True)
+    # Short human-readable summary of what was asked, never credentials.
+    params = db.Column(db.String(300), nullable=True)
+    # Which part of LIM made the call: fetcher, search, bulk-import, ...
+    context = db.Column(db.String(40), nullable=True, index=True)
+
+    ok = db.Column(db.Boolean, nullable=False, default=True)
+    http_status = db.Column(db.Integer, nullable=True)
+    result_count = db.Column(db.Integer, nullable=True)
+    duration_ms = db.Column(db.Integer, nullable=True)
+    quota_counted = db.Column(db.Boolean, nullable=False, default=False,
+                              index=True)
+    message = db.Column(db.Text, nullable=True)
+
+    user_id = db.Column(db.Integer, db.ForeignKey('users.id'), nullable=True)
+    user = db.relationship('User', foreign_keys=[user_id])
+
+    def __repr__(self):
+        return f'<BricksetApiCall {self.method} {"ok" if self.ok else "fail"}>'
+
+    # -- writing ------------------------------------------------------------
+
+    @classmethod
+    def record(cls, method, params=None, ok=True, http_status=None,
+               result_count=None, duration_ms=None, message=None,
+               context=None, user_id=None):
+        """
+        Log a call. Never raises — a logging failure must not break a fetch.
+        """
+        try:
+            call = cls(
+                method=method,
+                params=(params or '')[:300] or None,
+                context=(context or '')[:40] or None,
+                ok=bool(ok),
+                http_status=http_status,
+                result_count=result_count,
+                duration_ms=duration_ms,
+                quota_counted=method in QUOTA_METHODS,
+                message=(message or None),
+                user_id=user_id,
+            )
+            db.session.add(call)
+            db.session.commit()
+            return call
+        except Exception:
+            try:
+                db.session.rollback()
+            except Exception:
+                pass
+            return None
+
+    # -- reading ------------------------------------------------------------
+
+    @classmethod
+    def stats(cls, days=7):
+        """Totals for the dashboard at the top of the Brickset log page."""
+        from sqlalchemy import func
+
+        now = datetime.utcnow()
+        midnight = datetime(now.year, now.month, now.day)
+        since = now - timedelta(days=days)
+
+        def count(query):
+            return query.scalar() or 0
+
+        today = db.session.query(func.count(cls.id)).filter(
+            cls.called_at >= midnight)
+        today_quota = db.session.query(func.count(cls.id)).filter(
+            cls.called_at >= midnight, cls.quota_counted.is_(True))
+        window = db.session.query(func.count(cls.id)).filter(
+            cls.called_at >= since)
+        window_failed = db.session.query(func.count(cls.id)).filter(
+            cls.called_at >= since, cls.ok.is_(False))
+
+        by_method = db.session.query(
+            cls.method, func.count(cls.id), func.avg(cls.duration_ms)
+        ).filter(cls.called_at >= since).group_by(cls.method).order_by(
+            func.count(cls.id).desc()).all()
+
+        per_day = db.session.query(
+            func.date(cls.called_at), func.count(cls.id)
+        ).filter(cls.called_at >= since, cls.quota_counted.is_(True)
+                 ).group_by(func.date(cls.called_at)).all()
+
+        return {
+            'days': days,
+            'today': count(today),
+            'today_quota': count(today_quota),
+            'window': count(window),
+            'window_failed': count(window_failed),
+            'total': count(db.session.query(func.count(cls.id))),
+            'by_method': [
+                {'method': method, 'count': number,
+                 'avg_ms': int(avg) if avg else None}
+                for method, number, avg in by_method
+            ],
+            'quota_per_day': [
+                {'date': str(day), 'count': number} for day, number in per_day
+            ],
+        }
+
+    @classmethod
+    def prune(cls, days=90):
+        """Delete records older than `days`. Returns how many went."""
+        cutoff = datetime.utcnow() - timedelta(days=days)
+        removed = cls.query.filter(cls.called_at < cutoff).delete(
+            synchronize_session=False)
+        db.session.commit()
+        return removed
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'called_at': self.called_at.isoformat() if self.called_at else None,
+            'method': self.method,
+            'params': self.params,
+            'context': self.context,
+            'ok': self.ok,
+            'http_status': self.http_status,
+            'result_count': self.result_count,
+            'duration_ms': self.duration_ms,
+            'quota_counted': self.quota_counted,
+            'message': self.message,
+        }

app/models/site_setting.py

@@ -32,6 +32,10 @@ DEFAULTS = {
     'fetch_skip_sets_with_pdfs': 'true',
     'fetch_allow_setid_lookup': 'true',
     'fetch_last_run': '',
+
+    # Logging / API call history
+    'owner_user_id': '',
+    'api_log_retention_days': '90',
 }
 
 VISIBILITIES = ('private', 'shared', 'public')

app/routes/__init__.py

@@ -4,6 +4,7 @@ from app.routes.sets import sets_bp
 from app.routes.instructions import instructions_bp
 from app.routes.moc import bp as moc_bp
 from app.routes.instruction_fetch import instruction_fetch_bp
+from app.routes.logs import logs_bp
 
 __all__ = ['auth_bp', 'main_bp', 'sets_bp', 'instructions_bp', 'moc_bp',
-           'instruction_fetch_bp']
+           'instruction_fetch_bp', 'logs_bp']

app/routes/logs.py

@@ -0,0 +1,223 @@
+"""
+Owner-only log viewer and Brickset API call history.
+
+These pages show everything happening inside the instance, including other
+admins' activity, so they sit behind owner_required rather than
+admin_required.
+
+Created by: Jess Rogerson (yelling commands at Claude.AI)
+"""
+
+import os
+
+from flask import (Blueprint, current_app, flash, jsonify, redirect,
+                   render_template, request, send_file, url_for)
+from flask_login import login_required
+
+from app import db
+from app.models.api_call import BricksetApiCall
+from app.models.site_setting import SiteSetting
+from app.services import logging_setup
+from app.services.ownership import owner_required
+
+logs_bp = Blueprint('logs', __name__, url_prefix='/admin/logs')
+
+DEFAULT_LINES = 300
+MAX_LINES = 2000
+
+
+def _int_arg(name, default, low, high):
+    try:
+        value = int(request.args.get(name, default))
+    except (TypeError, ValueError):
+        value = default
+    return max(low, min(high, value))
+
+
+def _retention_days():
+    raw = SiteSetting.get('api_log_retention_days', '90')
+    try:
+        return max(1, min(3650, int(str(raw).strip())))
+    except (TypeError, ValueError):
+        return 90
+
+
+# ---------------------------------------------------------------------------
+# application log
+# ---------------------------------------------------------------------------
+
+@logs_bp.route('/')
+@login_required
+@owner_required
+def index():
+    """Tail of the application log, filtered."""
+    level = request.args.get('level', 'INFO')
+    search = request.args.get('q', '').strip()
+    source = request.args.get('source', '')
+    lines = _int_arg('lines', DEFAULT_LINES, 25, MAX_LINES)
+    filename = request.args.get('file', logging_setup.LOG_FILENAME)
+
+    path = logging_setup.resolve_file(current_app, filename)
+    entries, total = logging_setup.read_entries(
+        path, max_entries=lines, level=level, search=search or None,
+        logger_prefix=source or None)
+
+    files = logging_setup.available_files(current_app)
+    configured = logging_setup.get_handler() is not None
+
+    return render_template(
+        'admin/logs.html',
+        entries=list(reversed(entries)),   # newest first on screen
+        shown=len(entries),
+        total=total,
+        level=level,
+        search=search,
+        source=source,
+        lines=lines,
+        filename=filename,
+        files=files,
+        levels=logging_setup.LEVELS,
+        configured=configured,
+        log_path=logging_setup.log_path(current_app),
+    )
+
+
+@logs_bp.route('/tail.json')
+@login_required
+@owner_required
+def tail_json():
+    """Same data as the page, for auto-refresh."""
+    level = request.args.get('level', 'INFO')
+    search = request.args.get('q', '').strip()
+    source = request.args.get('source', '')
+    lines = _int_arg('lines', DEFAULT_LINES, 25, MAX_LINES)
+    path = logging_setup.resolve_file(
+        current_app, request.args.get('file', logging_setup.LOG_FILENAME))
+
+    entries, total = logging_setup.read_entries(
+        path, max_entries=lines, level=level, search=search or None,
+        logger_prefix=source or None)
+
+    return jsonify({
+        'entries': list(reversed(entries)),
+        'shown': len(entries),
+        'total': total,
+    })
+
+
+@logs_bp.route('/download')
+@login_required
+@owner_required
+def download():
+    path = logging_setup.resolve_file(
+        current_app, request.args.get('file', logging_setup.LOG_FILENAME))
+    if not path:
+        flash('No such log file.', 'warning')
+        return redirect(url_for('logs.index'))
+    return send_file(path, as_attachment=True,
+                     download_name=os.path.basename(path))
+
+
+@logs_bp.route('/rotate', methods=['POST'])
+@login_required
+@owner_required
+def rotate():
+    """Start a fresh log file, keeping the current one as a backup."""
+    if logging_setup.rotate_now():
+        current_app.logger.info('Log rotated by the owner')
+        flash('Log rotated — the previous contents are now in lim.log.1.',
+              'success')
+    else:
+        flash('No file handler is active, so there is nothing to rotate.',
+              'warning')
+    return redirect(url_for('logs.index'))
+
+
+# ---------------------------------------------------------------------------
+# Brickset API calls
+# ---------------------------------------------------------------------------
+
+@logs_bp.route('/brickset')
+@login_required
+@owner_required
+def brickset():
+    """Every call LIM has made to the Brickset API."""
+    days = _int_arg('days', 7, 1, 90)
+    page = _int_arg('page', 1, 1, 10000)
+    method = request.args.get('method', '')
+    only_failed = request.args.get('failed') == '1'
+
+    query = BricksetApiCall.query
+    if method:
+        query = query.filter(BricksetApiCall.method == method)
+    if only_failed:
+        query = query.filter(BricksetApiCall.ok.is_(False))
+
+    pagination = query.order_by(BricksetApiCall.id.desc()).paginate(
+        page=page, per_page=50, error_out=False)
+
+    methods = [row[0] for row in db.session.query(
+        BricksetApiCall.method).distinct().order_by(
+        BricksetApiCall.method).all()]
+
+    return render_template(
+        'admin/brickset_calls.html',
+        stats=BricksetApiCall.stats(days),
+        calls=pagination.items,
+        pagination=pagination,
+        days=days,
+        method=method,
+        only_failed=only_failed,
+        methods=methods,
+        retention_days=_retention_days(),
+        quota_note=None,
+    )
+
+
+@logs_bp.route('/brickset/quota', methods=['POST'])
+@login_required
+@owner_required
+def brickset_quota():
+    """
+    Ask Brickset what it thinks our usage has been.
+
+    getKeyUsageStats is free — it doesn't count against the quota it reports.
+    """
+    from app.services.brickset_api import BricksetAPI
+
+    if not BricksetAPI.is_configured():
+        flash('Brickset credentials are not configured.', 'warning')
+        return redirect(url_for('logs.brickset'))
+
+    usage = BricksetAPI(context='quota-check').get_key_usage()
+    if not usage:
+        flash('Brickset returned no usage data.', 'warning')
+    else:
+        recent = ', '.join(
+            f"{row.get('dateStamp', '?')}: {row.get('count', 0)}"
+            for row in usage[:7])
+        flash(f'Brickset reports (most recent first) — {recent}', 'info')
+    return redirect(url_for('logs.brickset'))
+
+
+@logs_bp.route('/brickset/retention', methods=['POST'])
+@login_required
+@owner_required
+def brickset_retention():
+    try:
+        days = max(1, min(3650, int(request.form.get('days', 90))))
+    except (TypeError, ValueError):
+        days = 90
+    SiteSetting.set('api_log_retention_days', str(days))
+    db.session.commit()
+    flash(f'API call records are now kept for {days} days.', 'success')
+    return redirect(url_for('logs.brickset'))
+
+
+@logs_bp.route('/brickset/prune', methods=['POST'])
+@login_required
+@owner_required
+def brickset_prune():
+    removed = BricksetApiCall.prune(_retention_days())
+    flash(f'Removed {removed} old API call record(s).', 'success')
+    return redirect(url_for('logs.brickset'))

app/services/brickset_api.py

@@ -17,10 +17,15 @@ Created by: Jess Rogerson (yelling commands at Claude.AI)
 """
 
 import json
+import logging
+import time
 from typing import Any, Dict, List, Optional
 
 import requests
-from flask import current_app
+from flask import current_app, has_request_context, request
+from flask_login import current_user
+
+logger = logging.getLogger(__name__)
 
 
 class BricksetAPI:
@@ -28,11 +33,49 @@ class BricksetAPI:
 
     BASE_URL = 'https://brickset.com/api/v3.asmx'
 
-    def __init__(self):
+    def __init__(self, context: Optional[str] = None):
         self.api_key = current_app.config.get('BRICKSET_API_KEY')
         self.username = current_app.config.get('BRICKSET_USERNAME')
         self.password = current_app.config.get('BRICKSET_PASSWORD')
         self._user_hash = None
+        # Recorded against every call so the log shows which part of LIM
+        # was responsible. Falls back to the endpoint name in a request.
+        self.context = context or self._default_context()
+
+    @staticmethod
+    def _default_context() -> str:
+        if has_request_context() and request.endpoint:
+            return request.endpoint.split('.')[-1][:40]
+        return 'background'
+
+    def _log_call(self, method, params=None, ok=True, http_status=None,
+                  result_count=None, started=None, message=None):
+        """Record one call. Never raises."""
+        duration = None
+        if started is not None:
+            duration = int((time.monotonic() - started) * 1000)
+
+        if ok:
+            logger.info('Brickset %s (%s) -> %s result(s) in %sms',
+                        method, params or '-',
+                        '?' if result_count is None else result_count,
+                        duration)
+        else:
+            logger.warning('Brickset %s (%s) failed: %s',
+                           method, params or '-', message)
+
+        try:
+            from app.models.api_call import BricksetApiCall
+            user_id = None
+            if has_request_context() and getattr(current_user,
+                                                 'is_authenticated', False):
+                user_id = current_user.id
+            BricksetApiCall.record(
+                method=method, params=params, ok=ok, http_status=http_status,
+                result_count=result_count, duration_ms=duration,
+                message=message, context=self.context, user_id=user_id)
+        except Exception:
+            logger.debug('Could not record Brickset call', exc_info=True)
 
     @staticmethod
     def is_configured() -> bool:
@@ -52,6 +95,7 @@ class BricksetAPI:
             current_app.logger.warning('Brickset API credentials not configured')
             return None
 
+        started = time.monotonic()
         try:
             response = requests.get(
                 f'{self.BASE_URL}/login',
@@ -67,13 +111,19 @@ class BricksetAPI:
 
             if data.get('status') == 'success':
                 self._user_hash = data.get('hash')
+                self._log_call('login', self.username, ok=True,
+                               http_status=response.status_code,
+                               started=started)
                 return self._user_hash
-            else:
-                current_app.logger.error(f"Brickset login failed: {data.get('message')}")
+
+            self._log_call('login', self.username, ok=False,
+                           http_status=response.status_code, started=started,
+                           message=str(data.get('message')))
             return None
 
         except Exception as e:
-            current_app.logger.error(f'Brickset API authentication error: {str(e)}')
+            self._log_call('login', self.username, ok=False, started=started,
+                           message=f'{type(e).__name__}: {e}')
             return None
 
     def search_sets(self,
@@ -123,6 +173,9 @@ class BricksetAPI:
             'params': json.dumps(search_params),
         }
 
+        summary = json.dumps(search_params, separators=(',', ':'))
+        started = time.monotonic()
+
         try:
             response = requests.get(
                 f'{self.BASE_URL}/getSets',
@@ -133,10 +186,16 @@ class BricksetAPI:
             data = response.json()
 
             if data.get('status') == 'success':
-                return data.get('sets', [])
+                sets = data.get('sets', [])
+                self._log_call('getSets', summary, ok=True,
+                               http_status=response.status_code,
+                               result_count=len(sets), started=started)
+                return sets
 
             message = data.get('message', 'unknown error')
-            current_app.logger.error(f'Brickset search failed: {message}')
+            self._log_call('getSets', summary, ok=False,
+                           http_status=response.status_code, started=started,
+                           message=message)
             if 'limit exceeded' in message.lower():
                 raise RuntimeError('Brickset API limit exceeded')
             return []
@@ -144,7 +203,8 @@ class BricksetAPI:
         except RuntimeError:
             raise
         except Exception as e:
-            current_app.logger.error(f'Brickset API search error: {str(e)}')
+            self._log_call('getSets', summary, ok=False, started=started,
+                           message=f'{type(e).__name__}: {e}')
             return []
 
     def get_set_by_number(self, set_number: str) -> Optional[Dict[str, Any]]:
@@ -174,6 +234,7 @@ class BricksetAPI:
         if not user_hash:
             return []
 
+        started = time.monotonic()
         try:
             response = requests.get(
                 f'{self.BASE_URL}/getThemes',
@@ -188,19 +249,31 @@ class BricksetAPI:
 
             if data.get('status') == 'success':
                 themes = data.get('themes', [])
-                return [theme.get('theme') for theme in themes if theme.get('theme')]
-            else:
+                names = [theme.get('theme') for theme in themes
+                         if theme.get('theme')]
+                self._log_call('getThemes', ok=True,
+                               http_status=response.status_code,
+                               result_count=len(names), started=started)
+                return names
+
+            self._log_call('getThemes', ok=False,
+                           http_status=response.status_code, started=started,
+                           message=str(data.get('message')))
             return []
 
         except Exception as e:
-            current_app.logger.error(f'Brickset API themes error: {str(e)}')
+            self._log_call('getThemes', ok=False, started=started,
+                           message=f'{type(e).__name__}: {e}')
             return []
 
     # -- instructions -------------------------------------------------------
 
     def _instructions_request(self, method: str,
                               params: Dict[str, Any]) -> List[Dict[str, Any]]:
+        summary = ', '.join(f'{k}={v}' for k, v in params.items())
         params = dict(params, apiKey=self.api_key)
+        started = time.monotonic()
+
         try:
             response = requests.get(f'{self.BASE_URL}/{method}',
                                     params=params, timeout=20)
@@ -208,14 +281,20 @@ class BricksetAPI:
             data = response.json()
 
             if data.get('status') == 'success':
-                return data.get('instructions', []) or []
+                instructions = data.get('instructions', []) or []
+                self._log_call(method, summary, ok=True,
+                               http_status=response.status_code,
+                               result_count=len(instructions), started=started)
+                return instructions
 
-            current_app.logger.error(
-                f"Brickset {method} failed: {data.get('message')}")
+            self._log_call(method, summary, ok=False,
+                           http_status=response.status_code, started=started,
+                           message=str(data.get('message')))
             return []
 
         except Exception as e:
-            current_app.logger.error(f'Brickset {method} error: {str(e)}')
+            self._log_call(method, summary, ok=False, started=started,
+                           message=f'{type(e).__name__}: {e}')
             return []
 
     def get_instructions(self, set_number: str) -> List[Dict[str, Any]]:
@@ -246,6 +325,7 @@ class BricksetAPI:
         """
         if not self.api_key:
             return []
+        started = time.monotonic()
         try:
             response = requests.get(f'{self.BASE_URL}/getKeyUsageStats',
                                     params={'apiKey': self.api_key},
@@ -253,8 +333,17 @@ class BricksetAPI:
             response.raise_for_status()
             data = response.json()
             if data.get('status') == 'success':
-                return data.get('apiKeyUsage', []) or []
+                usage = data.get('apiKeyUsage', []) or []
+                self._log_call('getKeyUsageStats', ok=True,
+                               http_status=response.status_code,
+                               result_count=len(usage), started=started)
+                return usage
+
+            self._log_call('getKeyUsageStats', ok=False,
+                           http_status=response.status_code, started=started,
+                           message=str(data.get('message')))
             return []
         except Exception as e:
-            current_app.logger.error(f'Brickset key usage error: {str(e)}')
+            self._log_call('getKeyUsageStats', ok=False, started=started,
+                           message=f'{type(e).__name__}: {e}')
             return []

app/services/fetch_worker.py

@@ -18,10 +18,13 @@ with `flask instructions run` instead.
 Created by: Jess Rogerson (yelling commands at Claude.AI)
 """
 
+import logging
 import os
 import sys
 import threading
 
+logger = logging.getLogger(__name__)
+
 _thread = None
 _stop = threading.Event()
 _lock = threading.Lock()
@@ -38,7 +41,7 @@ def _is_cli_command():
 def start(app):
     """Start the ticker unless it's disabled or already running."""
     if not app.config.get('INSTRUCTION_FETCH_WORKER', True):
-        app.logger.info('Instruction fetch worker disabled by config')
+        logger.info('Instruction fetch worker disabled by config')
         return False
 
     # `flask instructions run`, `flask backup restore` and friends shouldn't
@@ -61,7 +64,8 @@ def start(app):
                                    daemon=True)
         _thread.start()
 
-    app.logger.info('Instruction fetch worker started')
+    logger.info('Instruction fetch worker started (tick %ss)',
+                app.config.get('INSTRUCTION_FETCH_TICK_SECONDS', 20))
     return True
 
 
@@ -86,7 +90,7 @@ def _loop(app):
             with app.app_context():
                 fetcher.run_one()
         except Exception:
-            app.logger.exception('Instruction fetch worker tick failed')
+            logger.exception('Instruction fetch worker tick failed')
 
 
 def run_once_async(app, force=False):
@@ -102,7 +106,7 @@ def run_once_async(app, force=False):
                 from app.services import instruction_fetcher as fetcher
                 fetcher.run_one(force=force)
         except Exception:
-            app.logger.exception('Manual instruction fetch failed')
+            logger.exception('Manual instruction fetch failed')
 
     thread = threading.Thread(target=_work, name='lim-fetch-once', daemon=True)
     thread.start()

app/services/instruction_fetcher.py

@@ -21,6 +21,7 @@ Created by: Jess Rogerson (yelling commands at Claude.AI)
 """
 
 import hashlib
+import logging
 import os
 import re
 import tempfile
@@ -30,6 +31,8 @@ from datetime import datetime, timedelta
 import requests
 from flask import current_app
 
+logger = logging.getLogger(__name__)
+
 from app import db
 from app.models.instruction import Instruction
 from app.models.instruction_fetch import InstructionFetchJob
@@ -375,8 +378,11 @@ def fetch_for_set(lego_set, delay_between_files=None):
     pause = (file_delay_seconds() if delay_between_files is None
              else delay_between_files)
 
-    api = BricksetAPI()
+    api = BricksetAPI(context='fetcher')
     entries, source, note = _instruction_urls(api, lego_set)
+    logger.info('Set %s: Brickset lists %d instruction file(s)%s',
+                lego_set.set_number, len(entries),
+                f' ({note})' if note else '')
 
     if not entries:
         return 0, 0, source, note or 'no instructions listed on Brickset'
@@ -395,6 +401,8 @@ def fetch_for_set(lego_set, delay_between_files=None):
             temp_path, size, sha = _download_pdf(entry['url'], max_bytes,
                                                  timeout)
         except Exception as exc:
+            logger.warning('Set %s: download failed for %s - %s',
+                           lego_set.set_number, entry['url'], exc)
             problems.append(f'{entry["description"] or entry["url"]}: {exc}')
             continue
 
@@ -427,6 +435,8 @@ def fetch_for_set(lego_set, delay_between_files=None):
             known.add(sha)
             page += 1
             added += 1
+            logger.info('Set %s: saved %s (%d bytes)',
+                        lego_set.set_number, filename, file_size or size)
 
         except Exception as exc:
             db.session.rollback()
@@ -519,7 +529,7 @@ def run_one(force=False):
             job.status = 'empty'
 
     except Exception as exc:
-        current_app.logger.exception('Instruction fetch failed for %s',
+        logger.exception('Instruction fetch failed for %s',
                          lego_set.set_number)
         job.message = str(exc)[:500]
         if job.attempts >= max_attempts():
@@ -533,7 +543,7 @@ def run_one(force=False):
     job.finished_at = datetime.utcnow()
     db.session.commit()
 
-    current_app.logger.info('Instruction fetch %s for %s: %s', job.status,
+    logger.info('Job #%s %s for %s: %s', job.id, job.status,
                 lego_set.set_number, job.message)
     return job.to_dict()
 

app/services/logging_setup.py

@@ -0,0 +1,218 @@
+"""
+Logging setup and log reading.
+
+LIM previously configured no logging at all, which left Flask's logger at
+WARNING and wrote nothing to disk — so a quiet background feature like the
+instruction fetcher had no trail to follow. This attaches a rotating file
+handler to the root logger, so every module logger (app.services.*, and the
+fetcher in particular) lands in one file, and provides the parsing the admin
+log viewer needs.
+
+Rotation is by size, not date: LOG_MAX_BYTES per file, LOG_BACKUP_COUNT old
+copies, so disk use is bounded no matter how chatty things get.
+
+Created by: Jess Rogerson (yelling commands at Claude.AI)
+"""
+
+import io
+import logging
+import os
+import re
+from logging.handlers import RotatingFileHandler
+
+LOG_FILENAME = 'lim.log'
+
+LEVELS = ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL')
+
+LEVEL_BADGES = {
+    'DEBUG': 'secondary',
+    'INFO': 'info',
+    'WARNING': 'warning',
+    'ERROR': 'danger',
+    'CRITICAL': 'danger',
+}
+
+# 2026-09-26 13:38:00 INFO     app.services.instruction_fetcher | message
+LINE_RE = re.compile(
+    r'^(?P<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\s+'
+    r'(?P<level>[A-Z]+)\s+'
+    r'(?P<logger>\S+)\s+\|\s'
+    r'(?P<message>.*)$'
+)
+
+_FORMAT = '%(asctime)s %(levelname)-8s %(name)s | %(message)s'
+_DATEFMT = '%Y-%m-%d %H:%M:%S'
+
+
+def log_path(app):
+    return os.path.join(app.config['LOG_FOLDER'], LOG_FILENAME)
+
+
+def configure_logging(app):
+    """
+    Attach the rotating file handler. Idempotent — safe on a reloader restart.
+    """
+    folder = app.config['LOG_FOLDER']
+    try:
+        os.makedirs(folder, exist_ok=True)
+    except Exception as exc:
+        # A read-only or missing volume shouldn't stop the app booting.
+        app.logger.warning('Could not create log folder %s: %s', folder, exc)
+        return None
+
+    level = getattr(logging, str(app.config.get('LOG_LEVEL', 'INFO')).upper(),
+                    logging.INFO)
+    path = log_path(app)
+
+    root = logging.getLogger()
+    for existing in root.handlers:
+        if getattr(existing, '_lim_handler', False):
+            existing.setLevel(level)
+            root.setLevel(level)
+            app.logger.setLevel(level)
+            return path
+
+    try:
+        handler = RotatingFileHandler(
+            path,
+            maxBytes=int(app.config.get('LOG_MAX_BYTES', 2 * 1024 * 1024)),
+            backupCount=int(app.config.get('LOG_BACKUP_COUNT', 5)),
+            encoding='utf-8',
+        )
+    except Exception as exc:
+        app.logger.warning('Could not open log file %s: %s', path, exc)
+        return None
+
+    handler.setFormatter(logging.Formatter(_FORMAT, datefmt=_DATEFMT))
+    handler.setLevel(level)
+    handler._lim_handler = True
+
+    root.addHandler(handler)
+    root.setLevel(level)
+    app.logger.setLevel(level)
+
+    # Request access logs are noisy and rarely what you're looking for.
+    if not app.config.get('LOG_ACCESS', False):
+        logging.getLogger('werkzeug').setLevel(logging.WARNING)
+
+    app.logger.info('Logging to %s at %s', path,
+                    logging.getLevelName(level))
+    return path
+
+
+def get_handler():
+    for handler in logging.getLogger().handlers:
+        if getattr(handler, '_lim_handler', False):
+            return handler
+    return None
+
+
+def rotate_now():
+    """Force a rollover. Returns True if it happened."""
+    handler = get_handler()
+    if handler is None:
+        return False
+    handler.doRollover()
+    return True
+
+
+def available_files(app):
+    """The live log plus its rotated backups, newest first."""
+    folder = app.config['LOG_FOLDER']
+    if not os.path.isdir(folder):
+        return []
+
+    found = []
+    for name in os.listdir(folder):
+        if name == LOG_FILENAME or name.startswith(LOG_FILENAME + '.'):
+            full = os.path.join(folder, name)
+            try:
+                found.append({
+                    'name': name,
+                    'size': os.path.getsize(full),
+                    'current': name == LOG_FILENAME,
+                })
+            except OSError:
+                continue
+
+    # lim.log first, then lim.log.1, lim.log.2, ...
+    found.sort(key=lambda item: (not item['current'], item['name']))
+    return found
+
+
+def resolve_file(app, name):
+    """Turn a user-supplied file name into a safe path inside LOG_FOLDER."""
+    if not name:
+        name = LOG_FILENAME
+    if name != LOG_FILENAME and not name.startswith(LOG_FILENAME + '.'):
+        return None
+    if os.path.sep in name or '/' in name or '..' in name:
+        return None
+    path = os.path.join(app.config['LOG_FOLDER'], name)
+    return path if os.path.isfile(path) else None
+
+
+def _tail_bytes(path, limit):
+    """Last `limit` bytes of a file, decoded leniently."""
+    size = os.path.getsize(path)
+    with io.open(path, 'rb') as handle:
+        if size > limit:
+            handle.seek(size - limit)
+            handle.readline()  # discard the partial first line
+        return handle.read().decode('utf-8', errors='replace')
+
+
+def read_entries(path, max_entries=300, level=None, search=None,
+                 logger_prefix=None, byte_limit=1024 * 1024):
+    """
+    Parse the tail of a log file into entries, newest last.
+
+    Lines that don't match the format (traceback continuations) are attached
+    to the entry above them, so a stack trace stays with its error.
+    """
+    if not path or not os.path.isfile(path):
+        return [], 0
+
+    text = _tail_bytes(path, byte_limit)
+    entries = []
+
+    for line in text.splitlines():
+        match = LINE_RE.match(line)
+        if match:
+            entries.append({
+                'time': match.group('time'),
+                'level': match.group('level'),
+                'logger': match.group('logger'),
+                'message': match.group('message'),
+                'extra': [],
+            })
+        elif entries:
+            entries[-1]['extra'].append(line)
+
+    total = len(entries)
+
+    if level:
+        wanted = str(level).upper()
+        if wanted in LEVELS:
+            floor = LEVELS.index(wanted)
+            entries = [entry for entry in entries
+                       if entry['level'] in LEVELS
+                       and LEVELS.index(entry['level']) >= floor]
+
+    if logger_prefix:
+        entries = [entry for entry in entries
+                   if entry['logger'].startswith(logger_prefix)]
+
+    if search:
+        needle = search.lower()
+        entries = [
+            entry for entry in entries
+            if needle in entry['message'].lower()
+            or needle in entry['logger'].lower()
+            or any(needle in line.lower() for line in entry['extra'])
+        ]
+
+    for entry in entries:
+        entry['badge'] = LEVEL_BADGES.get(entry['level'], 'secondary')
+
+    return entries[-max_entries:], total

app/services/ownership.py

@@ -0,0 +1,86 @@
+"""
+Who owns this LIM instance.
+
+LIM only had is_admin, which is fine for day-to-day administration but too
+broad for things that expose the whole instance: the log viewer shows every
+request, every error and every API call, including other admins' activity.
+
+The owner is a single account:
+
+  * whatever `owner_user_id` in site_settings points at, if that user exists;
+  * otherwise the lowest-numbered admin (in practice the account created by
+    `flask init_db`), which is then remembered.
+
+Change it with `flask owner set <username>`.
+
+Created by: Jess Rogerson (yelling commands at Claude.AI)
+"""
+
+from functools import wraps
+
+from flask import flash, redirect, url_for
+from flask_login import current_user
+
+from app import db
+from app.models.site_setting import SiteSetting
+from app.models.user import User
+
+OWNER_SETTING = 'owner_user_id'
+
+
+def get_owner():
+    """The User who owns this instance, or None on an empty database."""
+    raw = SiteSetting.get(OWNER_SETTING, '')
+    if raw and str(raw).strip().isdigit():
+        owner = db.session.get(User, int(str(raw).strip()))
+        if owner is not None:
+            return owner
+
+    # Nothing stored (or it points at a deleted account): fall back to the
+    # first admin and remember the answer.
+    owner = (User.query.filter_by(is_admin=True).order_by(User.id).first()
+             or User.query.order_by(User.id).first())
+    if owner is not None:
+        try:
+            SiteSetting.set(OWNER_SETTING, str(owner.id))
+            db.session.commit()
+        except Exception:
+            db.session.rollback()
+    return owner
+
+
+def get_owner_id():
+    owner = get_owner()
+    return owner.id if owner else None
+
+
+def set_owner(user):
+    """Hand ownership to a user. Returns the user."""
+    SiteSetting.set(OWNER_SETTING, str(user.id))
+    if not user.is_admin:
+        user.is_admin = True
+    db.session.commit()
+    return user
+
+
+def is_owner(user=None):
+    """Is this user (default: the logged-in one) the owner?"""
+    user = user or current_user
+    if user is None or not getattr(user, 'is_authenticated', False):
+        return False
+    owner_id = get_owner_id()
+    return owner_id is not None and user.id == owner_id
+
+
+def owner_required(f):
+    """Stricter than admin_required: this account only."""
+    @wraps(f)
+    def decorated_function(*args, **kwargs):
+        if not current_user.is_authenticated:
+            flash('Please log in to access this page.', 'warning')
+            return redirect(url_for('auth.login'))
+        if not is_owner(current_user):
+            flash('That area is restricted to the instance owner.', 'danger')
+            return redirect(url_for('main.index'))
+        return f(*args, **kwargs)
+    return decorated_function

app/services/schema_upgrade.py

@@ -34,7 +34,10 @@ ADDITIVE_COLUMNS = {
 
 # Tables added by later versions. create_all() is not called at boot, so new
 # tables need an explicit (and idempotent) nudge.
-NEW_TABLES = ('app.models.instruction_fetch.InstructionFetchJob',)
+NEW_TABLES = (
+    'app.models.instruction_fetch.InstructionFetchJob',
+    'app.models.api_call.BricksetApiCall',
+)
 
 
 def ensure_schema(app):

app/templates/admin/brickset_calls.html

@@ -0,0 +1,298 @@
+{# Brickset API call history - owner only.
+   Created by: Jess Rogerson (yelling commands at Claude.AI) #}
+{% extends "base.html" %}
+
+{% block title %}Brickset API calls - Admin{% endblock %}
+
+{% block content %}
+<div class="row mb-3">
+    <div class="col">
+        <h1><i class="bi bi-cloud"></i> Brickset API calls</h1>
+        <p class="text-muted mb-0">
+            Every request LIM has made. Only <code>getSets</code> counts
+            against the daily key limit.
+        </p>
+    </div>
+    <div class="col-auto">
+        <a href="{{ url_for('admin.dashboard') }}" class="btn btn-outline-secondary">
+            <i class="bi bi-arrow-left"></i> Back
+        </a>
+    </div>
+</div>
+
+<ul class="nav nav-tabs mb-3">
+    <li class="nav-item">
+        <a class="nav-link" href="{{ url_for('logs.index') }}">
+            <i class="bi bi-journal-text"></i> Application log
+        </a>
+    </li>
+    <li class="nav-item">
+        <a class="nav-link active" href="{{ url_for('logs.brickset') }}">
+            <i class="bi bi-cloud"></i> Brickset API calls
+        </a>
+    </li>
+</ul>
+
+<div class="row g-3 mb-4">
+    <div class="col-md-3">
+        <div class="card h-100">
+            <div class="card-body">
+                <div class="text-muted small text-uppercase">Quota calls today</div>
+                <div class="h3 mb-0">{{ stats.today_quota }}</div>
+                <div class="small text-muted">getSets only</div>
+            </div>
+        </div>
+    </div>
+    <div class="col-md-3">
+        <div class="card h-100">
+            <div class="card-body">
+                <div class="text-muted small text-uppercase">All calls today</div>
+                <div class="h3 mb-0">{{ stats.today }}</div>
+                <div class="small text-muted">quota-free included</div>
+            </div>
+        </div>
+    </div>
+    <div class="col-md-3">
+        <div class="card h-100">
+            <div class="card-body">
+                <div class="text-muted small text-uppercase">Last {{ stats.days }} days</div>
+                <div class="h3 mb-0">{{ stats.window }}</div>
+                <div class="small text-muted">{{ stats.total }} recorded in total</div>
+            </div>
+        </div>
+    </div>
+    <div class="col-md-3">
+        <div class="card h-100">
+            <div class="card-body">
+                <div class="text-muted small text-uppercase">Failures</div>
+                <div class="h3 mb-0 {% if stats.window_failed %}text-danger{% endif %}">
+                    {{ stats.window_failed }}
+                </div>
+                <div class="small text-muted">in the last {{ stats.days }} days</div>
+            </div>
+        </div>
+    </div>
+</div>
+
+<div class="row">
+    <div class="col-lg-5 mb-4">
+        <div class="card h-100">
+            <div class="card-header">
+                <h6 class="mb-0">By method, last {{ stats.days }} days</h6>
+            </div>
+            <div class="table-responsive">
+                <table class="table table-sm mb-0">
+                    <thead>
+                        <tr>
+                            <th>Method</th>
+                            <th class="text-end">Calls</th>
+                            <th class="text-end">Avg</th>
+                            <th class="text-center">Quota</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {% for row in stats.by_method %}
+                        <tr>
+                            <td><code>{{ row.method }}</code></td>
+                            <td class="text-end">{{ row.count }}</td>
+                            <td class="text-end text-muted">
+                                {% if row.avg_ms %}{{ row.avg_ms }} ms{% else %}&mdash;{% endif %}
+                            </td>
+                            <td class="text-center">
+                                {% if row.method == 'getSets' %}
+                                <span class="badge bg-warning">yes</span>
+                                {% else %}
+                                <span class="text-muted">no</span>
+                                {% endif %}
+                            </td>
+                        </tr>
+                        {% else %}
+                        <tr>
+                            <td colspan="4" class="text-center text-muted py-3">
+                                No calls recorded yet.
+                            </td>
+                        </tr>
+                        {% endfor %}
+                    </tbody>
+                </table>
+            </div>
+        </div>
+    </div>
+
+    <div class="col-lg-4 mb-4">
+        <div class="card h-100">
+            <div class="card-header">
+                <h6 class="mb-0">Quota calls per day</h6>
+            </div>
+            <div class="table-responsive">
+                <table class="table table-sm mb-0">
+                    <tbody>
+                        {% for row in stats.quota_per_day %}
+                        <tr>
+                            <td>{{ row.date }}</td>
+                            <td class="text-end"><strong>{{ row.count }}</strong></td>
+                        </tr>
+                        {% else %}
+                        <tr>
+                            <td class="text-center text-muted py-3">
+                                No quota-counting calls in this window.
+                            </td>
+                        </tr>
+                        {% endfor %}
+                    </tbody>
+                </table>
+            </div>
+            <div class="card-footer">
+                <form method="POST" action="{{ url_for('logs.brickset_quota') }}">
+                    <button class="btn btn-sm btn-outline-primary w-100">
+                        <i class="bi bi-arrow-repeat"></i> Compare with Brickset's own figures
+                    </button>
+                </form>
+            </div>
+        </div>
+    </div>
+
+    <div class="col-lg-3 mb-4">
+        <div class="card h-100">
+            <div class="card-header">
+                <h6 class="mb-0">Retention</h6>
+            </div>
+            <div class="card-body">
+                <form method="POST" action="{{ url_for('logs.brickset_retention') }}"
+                      class="mb-3">
+                    <label class="form-label small" for="days">Keep records for</label>
+                    <div class="input-group input-group-sm">
+                        <input type="number" class="form-control" id="days" name="days"
+                               value="{{ retention_days }}" min="1" max="3650">
+                        <span class="input-group-text">days</span>
+                    </div>
+                    <button class="btn btn-sm btn-primary mt-2 w-100">Save</button>
+                </form>
+                <form method="POST" action="{{ url_for('logs.brickset_prune') }}"
+                      onsubmit="return confirm('Delete records older than {{ retention_days }} days?');">
+                    <button class="btn btn-sm btn-outline-danger w-100">
+                        <i class="bi bi-trash"></i> Prune now
+                    </button>
+                </form>
+            </div>
+        </div>
+    </div>
+</div>
+
+<div class="card">
+    <div class="card-header">
+        <form method="GET" class="row g-2 align-items-end">
+            <div class="col-auto">
+                <label class="form-label small mb-1" for="method">Method</label>
+                <select class="form-select form-select-sm" id="method" name="method">
+                    <option value="">All</option>
+                    {% for name in methods %}
+                    <option value="{{ name }}" {% if name == method %}selected{% endif %}>
+                        {{ name }}
+                    </option>
+                    {% endfor %}
+                </select>
+            </div>
+            <div class="col-auto">
+                <label class="form-label small mb-1" for="dayswin">Window</label>
+                <select class="form-select form-select-sm" id="dayswin" name="days">
+                    {% for option in [1, 7, 30, 90] %}
+                    <option value="{{ option }}" {% if option == days %}selected{% endif %}>
+                        {{ option }} day{{ 's' if option > 1 }}
+                    </option>
+                    {% endfor %}
+                </select>
+            </div>
+            <div class="col-auto">
+                <div class="form-check mb-1">
+                    <input class="form-check-input" type="checkbox" id="failed"
+                           name="failed" value="1" {% if only_failed %}checked{% endif %}>
+                    <label class="form-check-label small" for="failed">
+                        failures only
+                    </label>
+                </div>
+            </div>
+            <div class="col-auto">
+                <button class="btn btn-sm btn-primary">Filter</button>
+            </div>
+        </form>
+    </div>
+    <div class="table-responsive">
+        <table class="table table-sm table-hover mb-0" style="font-size: 0.85rem;">
+            <thead>
+                <tr>
+                    <th>When (UTC)</th>
+                    <th>Method</th>
+                    <th>Asked</th>
+                    <th>From</th>
+                    <th class="text-center">Result</th>
+                    <th class="text-end">Took</th>
+                </tr>
+            </thead>
+            <tbody>
+                {% for call in calls %}
+                <tr class="{% if not call.ok %}table-danger{% endif %}">
+                    <td class="text-nowrap text-muted">
+                        {{ call.called_at.strftime('%Y-%m-%d %H:%M:%S') }}
+                    </td>
+                    <td>
+                        <code>{{ call.method }}</code>
+                        {% if call.quota_counted %}
+                        <span class="badge bg-warning" title="Counts against the daily limit">q</span>
+                        {% endif %}
+                    </td>
+                    <td class="text-truncate" style="max-width: 280px;"
+                        title="{{ call.params }}">{{ call.params or '—' }}</td>
+                    <td class="text-muted">{{ call.context or '—' }}</td>
+                    <td class="text-center">
+                        {% if call.ok %}
+                        <span class="badge bg-success">
+                            {% if call.result_count is not none %}{{ call.result_count }}{% else %}ok{% endif %}
+                        </span>
+                        {% else %}
+                        <span class="badge bg-danger" title="{{ call.message }}">failed</span>
+                        {% endif %}
+                    </td>
+                    <td class="text-end text-muted">
+                        {% if call.duration_ms %}{{ call.duration_ms }} ms{% else %}&mdash;{% endif %}
+                    </td>
+                </tr>
+                {% if not call.ok and call.message %}
+                <tr class="table-danger">
+                    <td></td>
+                    <td colspan="5" class="small text-danger">{{ call.message }}</td>
+                </tr>
+                {% endif %}
+                {% else %}
+                <tr>
+                    <td colspan="6" class="text-center text-muted py-4">
+                        No calls recorded yet.
+                    </td>
+                </tr>
+                {% endfor %}
+            </tbody>
+        </table>
+    </div>
+    {% if pagination.pages > 1 %}
+    <div class="card-footer d-flex justify-content-between align-items-center">
+        <span class="text-muted small">
+            Page {{ pagination.page }} of {{ pagination.pages }}
+        </span>
+        <div>
+            {% if pagination.has_prev %}
+            <a class="btn btn-sm btn-outline-secondary"
+               href="{{ url_for('logs.brickset', page=pagination.prev_num, method=method, days=days, failed='1' if only_failed else None) }}">
+                Newer
+            </a>
+            {% endif %}
+            {% if pagination.has_next %}
+            <a class="btn btn-sm btn-outline-secondary"
+               href="{{ url_for('logs.brickset', page=pagination.next_num, method=method, days=days, failed='1' if only_failed else None) }}">
+                Older
+            </a>
+            {% endif %}
+        </div>
+    </div>
+    {% endif %}
+</div>
+{% endblock %}

app/templates/admin/dashboard.html

@@ -260,6 +260,13 @@
                             <i class="bi bi-cloud-arrow-down"></i> Instruction Fetcher
                         </a>
                     </div>
+                    {% if current_user_is_owner %}
+                    <div class="col-md-3 mb-2">
+                        <a href="{{ url_for('logs.index') }}" class="btn btn-outline-dark w-100">
+                            <i class="bi bi-journal-text"></i> Logs
+                        </a>
+                    </div>
+                    {% endif %}
                     <div class="col-md-3 mb-2">
                         <a href="{{ url_for('admin.users') }}" class="btn btn-outline-primary w-100">
                             <i class="bi bi-people"></i> Manage Users

app/templates/admin/instruction_fetch.html

@@ -15,6 +15,15 @@
         </p>
     </div>
     <div class="col-auto">
+        {% if current_user_is_owner %}
+        <a href="{{ url_for('logs.index', source='app.services.instruction_fetcher') }}"
+           class="btn btn-outline-dark">
+            <i class="bi bi-journal-text"></i> Fetcher log
+        </a>
+        <a href="{{ url_for('logs.brickset') }}" class="btn btn-outline-dark">
+            <i class="bi bi-cloud"></i> API calls
+        </a>
+        {% endif %}
         <a href="{{ url_for('admin.dashboard') }}" class="btn btn-outline-secondary">
             <i class="bi bi-arrow-left"></i> Back
         </a>

app/templates/admin/logs.html

@@ -0,0 +1,222 @@
+{# Application log viewer - owner only.
+   Created by: Jess Rogerson (yelling commands at Claude.AI) #}
+{% extends "base.html" %}
+
+{% block title %}Logs - Admin{% endblock %}
+
+{% block content %}
+<div class="row mb-3">
+    <div class="col">
+        <h1><i class="bi bi-journal-text"></i> Logs</h1>
+        <p class="text-muted mb-0">
+            <code>{{ log_path }}</code>
+        </p>
+    </div>
+    <div class="col-auto">
+        <a href="{{ url_for('admin.dashboard') }}" class="btn btn-outline-secondary">
+            <i class="bi bi-arrow-left"></i> Back
+        </a>
+    </div>
+</div>
+
+<ul class="nav nav-tabs mb-3">
+    <li class="nav-item">
+        <a class="nav-link active" href="{{ url_for('logs.index') }}">
+            <i class="bi bi-journal-text"></i> Application log
+        </a>
+    </li>
+    <li class="nav-item">
+        <a class="nav-link" href="{{ url_for('logs.brickset') }}">
+            <i class="bi bi-cloud"></i> Brickset API calls
+        </a>
+    </li>
+</ul>
+
+{% if not configured %}
+<div class="alert alert-warning">
+    <i class="bi bi-exclamation-triangle"></i>
+    No log file handler is active — check <code>LOG_FOLDER</code> is writable,
+    then restart LIM.
+</div>
+{% endif %}
+
+<div class="card mb-3">
+    <div class="card-body">
+        <form method="GET" class="row g-2 align-items-end">
+            <div class="col-auto">
+                <label class="form-label small mb-1" for="level">Minimum level</label>
+                <select class="form-select form-select-sm" id="level" name="level">
+                    {% for option in levels %}
+                    <option value="{{ option }}" {% if option == level %}selected{% endif %}>
+                        {{ option }}
+                    </option>
+                    {% endfor %}
+                </select>
+            </div>
+            <div class="col-auto">
+                <label class="form-label small mb-1" for="source">Source</label>
+                <select class="form-select form-select-sm" id="source" name="source">
+                    <option value="">Everything</option>
+                    <option value="app.services.instruction_fetcher"
+                            {% if source == 'app.services.instruction_fetcher' %}selected{% endif %}>
+                        Instruction fetcher
+                    </option>
+                    <option value="app.services.fetch_worker"
+                            {% if source == 'app.services.fetch_worker' %}selected{% endif %}>
+                        Fetch worker
+                    </option>
+                    <option value="app.services.brickset_api"
+                            {% if source == 'app.services.brickset_api' %}selected{% endif %}>
+                        Brickset API
+                    </option>
+                    <option value="app.services"
+                            {% if source == 'app.services' %}selected{% endif %}>
+                        All services
+                    </option>
+                </select>
+            </div>
+            <div class="col">
+                <label class="form-label small mb-1" for="q">Contains</label>
+                <input type="text" class="form-control form-control-sm" id="q"
+                       name="q" value="{{ search }}" placeholder="text to match">
+            </div>
+            <div class="col-auto">
+                <label class="form-label small mb-1" for="lines">Lines</label>
+                <input type="number" class="form-control form-control-sm" id="lines"
+                       name="lines" value="{{ lines }}" min="25" max="2000"
+                       style="width: 100px;">
+            </div>
+            {% if files|length > 1 %}
+            <div class="col-auto">
+                <label class="form-label small mb-1" for="file">File</label>
+                <select class="form-select form-select-sm" id="file" name="file">
+                    {% for item in files %}
+                    <option value="{{ item.name }}" {% if item.name == filename %}selected{% endif %}>
+                        {{ item.name }} ({{ (item.size / 1024)|round(1) }} KB)
+                    </option>
+                    {% endfor %}
+                </select>
+            </div>
+            {% endif %}
+            <div class="col-auto">
+                <button type="submit" class="btn btn-sm btn-primary">
+                    <i class="bi bi-funnel"></i> Apply
+                </button>
+            </div>
+        </form>
+    </div>
+</div>
+
+<div class="d-flex justify-content-between align-items-center mb-2">
+    <div class="text-muted small">
+        Showing <strong id="shown">{{ shown }}</strong> of {{ total }} entries in view
+        <span class="ms-3">
+            <input class="form-check-input" type="checkbox" id="autorefresh">
+            <label class="form-check-label" for="autorefresh">auto-refresh</label>
+        </span>
+    </div>
+    <div>
+        <a href="{{ url_for('logs.download', file=filename) }}"
+           class="btn btn-sm btn-outline-secondary">
+            <i class="bi bi-download"></i> Download
+        </a>
+        <form method="POST" action="{{ url_for('logs.rotate') }}" class="d-inline"
+              onsubmit="return confirm('Start a fresh log file?');">
+            <button class="btn btn-sm btn-outline-secondary">
+                <i class="bi bi-arrow-clockwise"></i> Rotate now
+            </button>
+        </form>
+    </div>
+</div>
+
+<div class="card">
+    <div class="table-responsive" style="max-height: 70vh; overflow-y: auto;">
+        <table class="table table-sm table-hover mb-0" style="font-family: var(--bs-font-monospace); font-size: 0.82rem;">
+            <thead class="sticky-top bg-body">
+                <tr>
+                    <th style="width: 150px;">Time</th>
+                    <th style="width: 90px;">Level</th>
+                    <th style="width: 220px;">Source</th>
+                    <th>Message</th>
+                </tr>
+            </thead>
+            <tbody id="logbody">
+                {% for entry in entries %}
+                <tr>
+                    <td class="text-nowrap text-muted">{{ entry.time }}</td>
+                    <td><span class="badge bg-{{ entry.badge }}">{{ entry.level }}</span></td>
+                    <td class="text-muted text-truncate" style="max-width: 220px;"
+                        title="{{ entry.logger }}">{{ entry.logger }}</td>
+                    <td>
+                        {{ entry.message }}
+                        {% if entry.extra %}
+                        <pre class="mb-0 mt-1 small text-danger">{{ entry.extra|join('\n') }}</pre>
+                        {% endif %}
+                    </td>
+                </tr>
+                {% else %}
+                <tr>
+                    <td colspan="4" class="text-center text-muted py-4">
+                        Nothing matches. Try a lower level or clear the filter.
+                    </td>
+                </tr>
+                {% endfor %}
+            </tbody>
+        </table>
+    </div>
+</div>
+
+<script>
+(function () {
+    const params = new URLSearchParams(window.location.search);
+    const url = "{{ url_for('logs.tail_json') }}" + '?' + params.toString();
+    const body = document.getElementById('logbody');
+    const shown = document.getElementById('shown');
+    const toggle = document.getElementById('autorefresh');
+    let timer = null;
+
+    function escapeHtml(text) {
+        const div = document.createElement('div');
+        div.textContent = text;
+        return div.innerHTML;
+    }
+
+    const badges = {DEBUG: 'secondary', INFO: 'info', WARNING: 'warning',
+                    ERROR: 'danger', CRITICAL: 'danger'};
+
+    async function refresh() {
+        try {
+            const res = await fetch(url, {headers: {'Accept': 'application/json'}});
+            if (!res.ok) return;
+            const data = await res.json();
+            shown.textContent = data.shown;
+            body.innerHTML = data.entries.map(function (e) {
+                const extra = e.extra && e.extra.length
+                    ? '<pre class="mb-0 mt-1 small text-danger">' +
+                      escapeHtml(e.extra.join('\n')) + '</pre>'
+                    : '';
+                return '<tr>' +
+                    '<td class="text-nowrap text-muted">' + escapeHtml(e.time) + '</td>' +
+                    '<td><span class="badge bg-' + (badges[e.level] || 'secondary') +
+                        '">' + escapeHtml(e.level) + '</span></td>' +
+                    '<td class="text-muted text-truncate" style="max-width:220px" title="' +
+                        escapeHtml(e.logger) + '">' + escapeHtml(e.logger) + '</td>' +
+                    '<td>' + escapeHtml(e.message) + extra + '</td>' +
+                    '</tr>';
+            }).join('') ||
+            '<tr><td colspan="4" class="text-center text-muted py-4">Nothing matches.</td></tr>';
+        } catch (err) { /* try again next tick */ }
+    }
+
+    toggle.addEventListener('change', function () {
+        if (toggle.checked) {
+            refresh();
+            timer = setInterval(refresh, 10000);
+        } else if (timer) {
+            clearInterval(timer);
+            timer = null;
+        }
+    });
+})();
+</script>
+{% endblock %}

app/templates/admin/settings.html

@@ -11,6 +11,11 @@
         <a href="{{ url_for('instruction_fetch.index') }}" class="btn btn-outline-warning me-2">
             <i class="bi bi-cloud-arrow-down"></i> Instruction Fetcher
         </a>
+        {% if current_user_is_owner %}
+        <a href="{{ url_for('logs.index') }}" class="btn btn-outline-dark me-2">
+            <i class="bi bi-journal-text"></i> Logs
+        </a>
+        {% endif %}
         <a href="{{ url_for('backup.index') }}" class="btn btn-outline-danger me-2">
             <i class="bi bi-archive"></i> Backup &amp; Restore
         </a>

diagnose_fetcher.py

@@ -0,0 +1,228 @@
+"""
+Instruction fetcher diagnostics - read-only.
+
+Drop this in your LIM folder (next to run.py) and run it. It changes nothing;
+it just reports what the fetcher can see, in the order things have to be true
+for a download to happen.
+
+    E:\\LIM> venv\\Scripts\\activate
+    E:\\LIM> python diagnose_fetcher.py
+
+Optionally test a live Brickset lookup for one set (no files are saved):
+
+    E:\\LIM> python diagnose_fetcher.py 10497
+
+Docker:
+
+    docker compose exec app python diagnose_fetcher.py
+    docker compose exec app python diagnose_fetcher.py 10497
+
+Created by: Jess Rogerson (yelling commands at Claude.AI)
+"""
+
+import os
+import sys
+
+os.environ.setdefault('INSTRUCTION_FETCH_WORKER', '0')
+
+
+def line(char='-'):
+    print(char * 68)
+
+
+def head(text):
+    print()
+    line('=')
+    print(f' {text}')
+    line('=')
+
+
+def main():
+    test_set = sys.argv[1] if len(sys.argv) > 1 else None
+
+    try:
+        from app import create_app, db
+    except Exception as exc:
+        print(f'FAILED to import the app: {exc}')
+        print('Are you in the LIM folder with the virtualenv activated?')
+        return 1
+
+    app = create_app(os.environ.get('FLASK_ENV', 'production'))
+
+    with app.app_context():
+        # -- is the update even installed? ---------------------------------
+        head('1. Is the fetcher installed?')
+        try:
+            from app.models.instruction_fetch import InstructionFetchJob
+            from app.services import instruction_fetcher as fetcher
+            print('  code               : present')
+        except Exception as exc:
+            print(f'  code               : MISSING ({exc})')
+            print('\n  The update is not installed in this folder.')
+            return 1
+
+        from sqlalchemy import inspect
+        inspector = inspect(db.engine)
+        tables = inspector.get_table_names()
+        print(f"  jobs table         : "
+              f"{'present' if 'instruction_fetch_jobs' in tables else 'MISSING - restart LIM'}")
+        cols = [c['name'] for c in inspector.get_columns('instructions')]
+        print(f"  instructions.sha256: {'present' if 'sha256' in cols else 'MISSING - restart LIM'}")
+
+        routes = [r.rule for r in app.url_map.iter_rules()
+                  if 'instruction-fetch' in r.rule]
+        print(f'  admin routes       : {len(routes)} registered')
+
+        # -- credentials ----------------------------------------------------
+        head('2. Brickset credentials')
+        from app.services.brickset_api import BricksetAPI
+        configured = BricksetAPI.is_configured()
+        for key in ('BRICKSET_API_KEY', 'BRICKSET_USERNAME', 'BRICKSET_PASSWORD'):
+            value = app.config.get(key)
+            shown = ('set (%d chars)' % len(value)) if value else 'NOT SET'
+            print(f'  {key:<18} : {shown}')
+        print(f'  is_configured()    : {configured}')
+
+        # -- settings --------------------------------------------------------
+        head('3. Fetcher settings')
+        from app.models.site_setting import SiteSetting
+        for key in sorted(fetcher.SETTING_DEFAULTS):
+            print(f'  {key:<26} : {SiteSetting.get(key, "(unset)")!r}')
+        print()
+        print(f'  is_enabled()               : {fetcher.is_enabled()}')
+        print(f'  interval_minutes()         : {fetcher.interval_minutes()}')
+        print(f'  is_due()                   : {fetcher.is_due()}')
+        print(f'  seconds_until_due()        : {fetcher.seconds_until_due()}')
+        print(f'  last_run()                 : {fetcher.last_run()}')
+
+        # -- the library ------------------------------------------------------
+        head('4. What the queue builder can see')
+        from app.models.set import Set
+        from app.models.instruction import Instruction
+
+        total = Set.query.count()
+        official = fetcher.official_sets_query().count()
+        mocs = Set.query.filter(Set.is_moc.is_(True)).count()
+        null_moc = Set.query.filter(Set.is_moc.is_(None)).count()
+        missing = fetcher.sets_missing_pdfs().count()
+
+        print(f'  sets total                 : {total}')
+        print(f'  official (is_moc = false)  : {official}')
+        print(f'  MOCs     (is_moc = true)   : {mocs}')
+        print(f'  is_moc IS NULL             : {null_moc}   '
+              f'{"<-- these are invisible to the fetcher!" if null_moc else ""}')
+        print(f'  official with no PDF       : {missing}  <-- "Queue sets missing instructions" queues this many')
+
+        print()
+        print('  file_type values in the instructions table:')
+        rows = db.session.query(
+            Instruction.file_type, db.func.count(Instruction.id)
+        ).group_by(Instruction.file_type).all()
+        if rows:
+            for file_type, count in rows:
+                flag = '' if file_type == 'PDF' else ''
+                print(f'    {file_type!r:<12} {count}{flag}')
+        else:
+            print('    (no instruction files at all)')
+
+        if official:
+            print()
+            print('  first few official sets:')
+            for lego_set in fetcher.official_sets_query().limit(5):
+                pdfs = lego_set.instructions.filter_by(file_type='PDF').count()
+                print(f'    {lego_set.set_number:<12} {(lego_set.set_name or "")[:32]:<34} '
+                      f'PDFs={pdfs}  brickset_id={lego_set.brickset_id}')
+
+        # -- the queue ---------------------------------------------------------
+        head('5. The queue')
+        counts = InstructionFetchJob.counts()
+        for key, value in counts.items():
+            print(f'  {key:<12} : {value}')
+
+        recent = InstructionFetchJob.query.order_by(
+            InstructionFetchJob.id.desc()).limit(10).all()
+        if recent:
+            print()
+            print('  most recent jobs:')
+            for job in recent:
+                number = job.lego_set.set_number if job.lego_set else '(deleted)'
+                print(f'    #{job.id:<5} {number:<12} {job.status:<10} '
+                      f'files={job.files_added} tries={job.attempts}')
+                if job.message:
+                    print(f'           {job.message[:100]}')
+        else:
+            print()
+            print('  No jobs have ever been queued.')
+
+        # -- verdict -------------------------------------------------------------
+        head('6. Verdict')
+        problems = []
+        if not configured:
+            problems.append('Brickset credentials are not set in .env')
+        if null_moc:
+            problems.append(f'{null_moc} set(s) have is_moc = NULL and are skipped')
+        if not official:
+            problems.append('there are no official sets in the library')
+        elif not missing and counts['total'] == 0:
+            problems.append('every official set already has a PDF, so '
+                            '"queue missing" finds nothing - use '
+                            '"Queue all official sets" instead')
+        if not fetcher.is_enabled():
+            problems.append('the fetcher is switched OFF (admin page, '
+                            '"Work through the queue automatically")')
+        if counts['queued'] and not fetcher.is_enabled():
+            problems.append(f"{counts['queued']} job(s) are waiting but "
+                            f"nothing will run until it is switched on")
+        if not app.config.get('INSTRUCTION_FETCH_WORKER', True):
+            problems.append('INSTRUCTION_FETCH_WORKER is 0, so the web '
+                            'process runs no worker thread')
+
+        if problems:
+            for problem in problems:
+                print(f'  * {problem}')
+        else:
+            print('  Nothing obviously wrong. If the queue has jobs and the '
+                  'fetcher is on,')
+            print('  the next one runs in '
+                  f'{fetcher.seconds_until_due()} seconds.')
+
+        # -- live test ------------------------------------------------------------
+        if test_set:
+            head(f'7. Live Brickset lookup for {test_set}')
+            number = fetcher.brickset_number(test_set)
+            print(f'  asking Brickset for : {number}')
+            api = BricksetAPI()
+            try:
+                entries = api.get_instructions(number)
+                print(f'  getInstructions2    : {len(entries)} result(s)')
+                for entry in entries:
+                    print(f'    {entry.get("description", "")}')
+                    print(f'      {entry.get("URL", "")}')
+                if not entries:
+                    print('  Nothing returned. Either Brickset has no '
+                          'instructions for this set,')
+                    print('  or the credentials/number are wrong.')
+                else:
+                    import requests
+                    url = entries[0].get('URL')
+                    print()
+                    print('  testing the first download (headers only)...')
+                    response = requests.get(
+                        url, stream=True, timeout=30,
+                        headers={'User-Agent': fetcher.USER_AGENT})
+                    print(f'    HTTP {response.status_code}  '
+                          f'{response.headers.get("Content-Type")}  '
+                          f'{response.headers.get("Content-Length")} bytes')
+                    response.close()
+            except Exception as exc:
+                print(f'  FAILED: {type(exc).__name__}: {exc}')
+
+        print()
+        line('=')
+        print(' end of report')
+        line('=')
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

docker-compose.yml

@@ -54,6 +54,11 @@ services:
       # Automatic instruction fetcher. Safe to leave on: gunicorn runs a
       # single worker here, so there is exactly one ticker, and it does
       # nothing until an admin enables it in the UI.
+      LOG_FOLDER: /data/logs
+      LOG_LEVEL: ${LOG_LEVEL:-INFO}
+      LOG_MAX_BYTES: ${LOG_MAX_BYTES:-2097152}
+      LOG_BACKUP_COUNT: ${LOG_BACKUP_COUNT:-5}
+      LOG_ACCESS: ${LOG_ACCESS:-0}
       INSTRUCTION_FETCH_WORKER: ${INSTRUCTION_FETCH_WORKER:-1}
       INSTRUCTION_FETCH_TICK_SECONDS: ${INSTRUCTION_FETCH_TICK_SECONDS:-20}
       FETCH_MAX_FILE_MB: ${FETCH_MAX_FILE_MB:-150}

docker/entrypoint.sh

@@ -15,7 +15,8 @@ log() { echo "[entrypoint] $*"; }
 # A freshly created named volume mounts as root:root, so the app user cannot
 # write to it until we hand it over. Cheap to repeat on every start.
 if [ "$(id -u)" = '0' ]; then
-    mkdir -p "${UPLOAD_FOLDER:-/data/uploads}" "${BACKUP_FOLDER:-/data/backups}"
+    mkdir -p "${UPLOAD_FOLDER:-/data/uploads}" "${BACKUP_FOLDER:-/data/backups}" \
+             "${LOG_FOLDER:-/data/logs}"
     chown -R "${APP_USER}:${APP_USER}" /data
 fi
 

docs/LOGGING.md

@@ -0,0 +1,145 @@
+# Logging and API call tracking
+
+Two owner-only pages under **Admin → Logs**:
+
+* **Application log** — the rotating file log, filterable and searchable.
+* **Brickset API calls** — every outbound request, with quota accounting.
+
+---
+
+## Owner, not admin
+
+These pages show everything happening inside the instance, including other
+admins' activity, so `is_admin` isn't enough. They're restricted to a single
+**owner** account.
+
+The owner is whoever `owner_user_id` in `site_settings` points at. If that's
+unset (or points at a deleted account) it falls back to the lowest-numbered
+admin — in practice the account `flask init_db` created — and remembers that
+answer.
+
+```bat
+flask owner show
+flask owner set jessikitty     :: promotes them to admin if needed
+```
+
+Non-owner admins don't get a permission error on the dashboard; the Logs tile
+simply isn't rendered for them.
+
+---
+
+## Application log
+
+LIM previously configured no logging at all, which left Flask's logger at
+WARNING with nothing written to disk. Now a rotating file handler is attached
+to the **root** logger, so every module logger lands in one file:
+
+```
+2026-09-26 13:38:00 INFO     app.services.instruction_fetcher | Set 10497: saved 10497-1-building-instructions.pdf (48261234 bytes)
+```
+
+Rotation is by size, so disk use is bounded no matter how chatty things get:
+`LOG_MAX_BYTES` per file (2 MB), `LOG_BACKUP_COUNT` old copies (5). Worst case
+is 12 MB.
+
+### The viewer
+
+Filter by minimum level, by source, and by free text; the source dropdown has
+shortcuts for the fetcher, the worker and the Brickset client, so you can
+isolate one feature's activity. Multi-line entries (tracebacks) stay attached
+to the error that produced them rather than scattering.
+
+Also on the page: auto-refresh every 10s, download the raw file, switch to a
+rotated backup, and *Rotate now* to start a clean file before reproducing a
+problem.
+
+Reading is capped at the last 1 MB of the file, so a huge log won't stall the
+page.
+
+### Environment
+
+```ini
+LOG_FOLDER=logs          # /data/logs in Docker
+LOG_LEVEL=INFO           # DEBUG when chasing something
+LOG_MAX_BYTES=2097152
+LOG_BACKUP_COUNT=5
+LOG_ACCESS=0             # 1 to include per-request logging (noisy)
+```
+
+A relative `LOG_FOLDER` resolves against the project root, not the working
+directory, same as `UPLOAD_FOLDER`.
+
+If the folder can't be created or opened, LIM logs a warning and carries on
+rather than refusing to boot. The viewer says so at the top of the page.
+
+```bat
+flask logs path --tail 50
+```
+
+---
+
+## Brickset API calls
+
+Every call is recorded: which method, what was asked, which part of LIM asked,
+how long it took, how many results came back, and the error if it failed.
+Credentials are never stored — the `login` row records the username only.
+
+### Quota accounting
+
+Per Brickset's documentation, **only `getSets` counts against the daily key
+limit**. The page leads with that number for today, separately from the total,
+because the total is mostly quota-free traffic:
+
+| Method | Counts |
+|--------|--------|
+| `getSets` | **yes** |
+| `getInstructions`, `getInstructions2` | no |
+| `getThemes`, `login`, `getKeyUsageStats` | no |
+
+Rows that count are flagged `q` in the table and broken out per day.
+
+*Compare with Brickset's own figures* calls `getKeyUsageStats`, which reports
+what Brickset thinks you've used over the last 30 days. It's free, and useful
+for confirming LIM's count matches theirs — a gap means something else is
+using the same key.
+
+The instruction fetcher is deliberately quota-light: it uses
+`getInstructions2` by default and only falls back to `getSets` when a set
+number can't be found, caching the returned Brickset ID so it never pays
+twice for the same set. On the calls page that shows up as a long run of
+`fetcher` rows with no `q` flags.
+
+### Retention
+
+90 days by default, changeable on the page. Pruning is manual (*Prune now*) or
+scheduled:
+
+```bat
+flask logs prune --days 30
+```
+
+---
+
+## Where to look when something's wrong
+
+**Fetcher downloaded nothing.** Logs → source *Instruction fetcher*. Each job
+logs how many files Brickset listed, and each download failure with its
+reason. The Brickset page shows whether the lookup even went out.
+
+**Suspected quota problems.** Brickset page, "Quota calls today". If that's
+low but Brickset is refusing you, something else is using the key — compare
+with their figures.
+
+**Something broke, no idea what.** Logs, level ERROR, no filter. Tracebacks
+are kept intact.
+
+---
+
+## Database
+
+`brickset_api_calls`, created automatically at startup by
+`services/schema_upgrade.py`. No migration step.
+
+---
+
+Created by: Jess Rogerson (yelling commands at Claude.AI)

install_instruction_fetcher.bat

@@ -1,6 +1,6 @@
 @echo off
 REM ===========================================================================
-REM  LEGO Instructions Manager - Instruction Fetcher installer
+REM  LEGO Instructions Manager - Instruction Fetcher + Logging installer
 REM
 REM  RUN THIS FROM THE FOLDER YOU EXTRACTED THE UPDATE INTO, not from your
 REM  LIM install - the two cannot be the same folder.
@@ -46,23 +46,31 @@ call :need "app\__init__.py"
 call :need "app\cli.py"
 call :need "app\config.py"
 call :need "app\models\__init__.py"
+call :need "app\models\api_call.py"
 call :need "app\models\instruction.py"
 call :need "app\models\instruction_fetch.py"
 call :need "app\models\site_setting.py"
 call :need "app\routes\__init__.py"
 call :need "app\routes\instruction_fetch.py"
+call :need "app\routes\logs.py"
 call :need "app\services\brickset_api.py"
 call :need "app\services\fetch_worker.py"
 call :need "app\services\instruction_fetcher.py"
+call :need "app\services\logging_setup.py"
+call :need "app\services\ownership.py"
 call :need "app\services\schema_upgrade.py"
+call :need "app\templates\admin\brickset_calls.html"
 call :need "app\templates\admin\dashboard.html"
 call :need "app\templates\admin\instruction_fetch.html"
+call :need "app\templates\admin\logs.html"
 call :need "app\templates\admin\settings.html"
+call :need "diagnose_fetcher.py"
 call :need "docker-compose.yml"
 call :need "docker\entrypoint.sh"
 call :need "docs\INSTRUCTION_FETCHER.md"
+call :need "docs\LOGGING.md"
 if not "%MISSING%"=="0" goto :incomplete
-echo    all 22 present.
+echo    all 30 present.
 
 REM --- backup ------------------------------------------------------------------
 for /f "tokens=2 delims==" %%I in ('wmic os get localdatetime /value') do set "DT=%%I"
@@ -81,21 +89,29 @@ call :save "app\__init__.py"
 call :save "app\cli.py"
 call :save "app\config.py"
 call :save "app\models\__init__.py"
+call :save "app\models\api_call.py"
 call :save "app\models\instruction.py"
 call :save "app\models\instruction_fetch.py"
 call :save "app\models\site_setting.py"
 call :save "app\routes\__init__.py"
 call :save "app\routes\instruction_fetch.py"
+call :save "app\routes\logs.py"
 call :save "app\services\brickset_api.py"
 call :save "app\services\fetch_worker.py"
 call :save "app\services\instruction_fetcher.py"
+call :save "app\services\logging_setup.py"
+call :save "app\services\ownership.py"
 call :save "app\services\schema_upgrade.py"
+call :save "app\templates\admin\brickset_calls.html"
 call :save "app\templates\admin\dashboard.html"
 call :save "app\templates\admin\instruction_fetch.html"
+call :save "app\templates\admin\logs.html"
 call :save "app\templates\admin\settings.html"
+call :save "diagnose_fetcher.py"
 call :save "docker-compose.yml"
 call :save "docker\entrypoint.sh"
 call :save "docs\INSTRUCTION_FETCHER.md"
+call :save "docs\LOGGING.md"
 echo    %SAVED% file^(s^) backed up.
 
 REM --- install -------------------------------------------------------------------
@@ -108,26 +124,34 @@ call :put "app\__init__.py"
 call :put "app\cli.py"
 call :put "app\config.py"
 call :put "app\models\__init__.py"
+call :put "app\models\api_call.py"
 call :put "app\models\instruction.py"
 call :put "app\models\instruction_fetch.py"
 call :put "app\models\site_setting.py"
 call :put "app\routes\__init__.py"
 call :put "app\routes\instruction_fetch.py"
+call :put "app\routes\logs.py"
 call :put "app\services\brickset_api.py"
 call :put "app\services\fetch_worker.py"
 call :put "app\services\instruction_fetcher.py"
+call :put "app\services\logging_setup.py"
+call :put "app\services\ownership.py"
 call :put "app\services\schema_upgrade.py"
+call :put "app\templates\admin\brickset_calls.html"
 call :put "app\templates\admin\dashboard.html"
 call :put "app\templates\admin\instruction_fetch.html"
+call :put "app\templates\admin\logs.html"
 call :put "app\templates\admin\settings.html"
+call :put "diagnose_fetcher.py"
 call :put "docker-compose.yml"
 call :put "docker\entrypoint.sh"
 call :put "docs\INSTRUCTION_FETCHER.md"
+call :put "docs\LOGGING.md"
 if not "%COPYFAIL%"=="0" goto :copyfailed
 
 echo.
 echo  ==========================================================
-echo   Done - 22 files installed.
+echo   Done - 30 files installed.
 echo.
 echo   No migration step: the new table and columns are created
 echo   the next time LIM starts.