Add owner-only logging and Brickset API call tracking
LIM configured no logging at all, so the instruction fetcher had no trail to follow. Adds a rotating file handler on the root logger plus two owner-only admin pages. - Admin > Logs: filter by level, source and text; tracebacks kept with their error; auto-refresh, raw download, rotated-file browsing and rotate-now - Admin > Logs > Brickset API calls: every outbound call recorded with method, params, caller, duration, result count and error, plus daily quota accounting (only getSets counts) and a comparison against getKeyUsageStats - New owner concept (services/ownership.py): a single account, distinct from is_admin, since these pages expose other admins activity. Defaults to the first admin; flask owner show/set to change - Services now log through module loggers; retention setting and pruning for call records - New CLI: flask owner show/set, flask logs path/prune - New table brickset_api_calls, created automatically at boot; LOG_* env vars, /data/logs in Docker
This commit is contained in:
1 parent
89f65e2d74
commit
f7c20de5ac
27 files changed
+1914
-35
No files matched your search
@@ -0,0 +1,157 @@
|
||||
"""
|
||||
A record of every call LIM makes to the Brickset API.
|
||||
|
||||
Two reasons this exists. First, Brickset's daily key limit applies to getSets
|
||||
and nothing else, so knowing how many of those went out today is the only
|
||||
number that matters for staying under it — and Brickset's own
|
||||
getKeyUsageStats only reports whole days, after the fact. Second, when the
|
||||
instruction fetcher comes back empty it is useful to see exactly what was
|
||||
asked and what came back.
|
||||
|
||||
Rows are small and pruned on a retention setting (90 days by default).
|
||||
|
||||
Created by: Jess Rogerson (yelling commands at Claude.AI)
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from app import db
|
||||
|
||||
# The only method that counts against the daily API key quota.
|
||||
QUOTA_METHODS = {'getSets'}
|
||||
|
||||
|
||||
class BricksetApiCall(db.Model):
|
||||
"""One outbound request to brickset.com."""
|
||||
|
||||
__tablename__ = 'brickset_api_calls'
|
||||
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
called_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow,
|
||||
index=True)
|
||||
|
||||
method = db.Column(db.String(40), nullable=False, index=True)
|
||||
# Short human-readable summary of what was asked, never credentials.
|
||||
params = db.Column(db.String(300), nullable=True)
|
||||
# Which part of LIM made the call: fetcher, search, bulk-import, ...
|
||||
context = db.Column(db.String(40), nullable=True, index=True)
|
||||
|
||||
ok = db.Column(db.Boolean, nullable=False, default=True)
|
||||
http_status = db.Column(db.Integer, nullable=True)
|
||||
result_count = db.Column(db.Integer, nullable=True)
|
||||
duration_ms = db.Column(db.Integer, nullable=True)
|
||||
quota_counted = db.Column(db.Boolean, nullable=False, default=False,
|
||||
index=True)
|
||||
message = db.Column(db.Text, nullable=True)
|
||||
|
||||
user_id = db.Column(db.Integer, db.ForeignKey('users.id'), nullable=True)
|
||||
user = db.relationship('User', foreign_keys=[user_id])
|
||||
|
||||
def __repr__(self):
|
||||
return f'<BricksetApiCall {self.method} {"ok" if self.ok else "fail"}>'
|
||||
|
||||
# -- writing ------------------------------------------------------------
|
||||
|
||||
@classmethod
|
||||
def record(cls, method, params=None, ok=True, http_status=None,
|
||||
result_count=None, duration_ms=None, message=None,
|
||||
context=None, user_id=None):
|
||||
"""
|
||||
Log a call. Never raises — a logging failure must not break a fetch.
|
||||
"""
|
||||
try:
|
||||
call = cls(
|
||||
method=method,
|
||||
params=(params or '')[:300] or None,
|
||||
context=(context or '')[:40] or None,
|
||||
ok=bool(ok),
|
||||
http_status=http_status,
|
||||
result_count=result_count,
|
||||
duration_ms=duration_ms,
|
||||
quota_counted=method in QUOTA_METHODS,
|
||||
message=(message or None),
|
||||
user_id=user_id,
|
||||
)
|
||||
db.session.add(call)
|
||||
db.session.commit()
|
||||
return call
|
||||
except Exception:
|
||||
try:
|
||||
db.session.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
# -- reading ------------------------------------------------------------
|
||||
|
||||
@classmethod
|
||||
def stats(cls, days=7):
|
||||
"""Totals for the dashboard at the top of the Brickset log page."""
|
||||
from sqlalchemy import func
|
||||
|
||||
now = datetime.utcnow()
|
||||
midnight = datetime(now.year, now.month, now.day)
|
||||
since = now - timedelta(days=days)
|
||||
|
||||
def count(query):
|
||||
return query.scalar() or 0
|
||||
|
||||
today = db.session.query(func.count(cls.id)).filter(
|
||||
cls.called_at >= midnight)
|
||||
today_quota = db.session.query(func.count(cls.id)).filter(
|
||||
cls.called_at >= midnight, cls.quota_counted.is_(True))
|
||||
window = db.session.query(func.count(cls.id)).filter(
|
||||
cls.called_at >= since)
|
||||
window_failed = db.session.query(func.count(cls.id)).filter(
|
||||
cls.called_at >= since, cls.ok.is_(False))
|
||||
|
||||
by_method = db.session.query(
|
||||
cls.method, func.count(cls.id), func.avg(cls.duration_ms)
|
||||
).filter(cls.called_at >= since).group_by(cls.method).order_by(
|
||||
func.count(cls.id).desc()).all()
|
||||
|
||||
per_day = db.session.query(
|
||||
func.date(cls.called_at), func.count(cls.id)
|
||||
).filter(cls.called_at >= since, cls.quota_counted.is_(True)
|
||||
).group_by(func.date(cls.called_at)).all()
|
||||
|
||||
return {
|
||||
'days': days,
|
||||
'today': count(today),
|
||||
'today_quota': count(today_quota),
|
||||
'window': count(window),
|
||||
'window_failed': count(window_failed),
|
||||
'total': count(db.session.query(func.count(cls.id))),
|
||||
'by_method': [
|
||||
{'method': method, 'count': number,
|
||||
'avg_ms': int(avg) if avg else None}
|
||||
for method, number, avg in by_method
|
||||
],
|
||||
'quota_per_day': [
|
||||
{'date': str(day), 'count': number} for day, number in per_day
|
||||
],
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def prune(cls, days=90):
|
||||
"""Delete records older than `days`. Returns how many went."""
|
||||
cutoff = datetime.utcnow() - timedelta(days=days)
|
||||
removed = cls.query.filter(cls.called_at < cutoff).delete(
|
||||
synchronize_session=False)
|
||||
db.session.commit()
|
||||
return removed
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'id': self.id,
|
||||
'called_at': self.called_at.isoformat() if self.called_at else None,
|
||||
'method': self.method,
|
||||
'params': self.params,
|
||||
'context': self.context,
|
||||
'ok': self.ok,
|
||||
'http_status': self.http_status,
|
||||
'result_count': self.result_count,
|
||||
'duration_ms': self.duration_ms,
|
||||
'quota_counted': self.quota_counted,
|
||||
'message': self.message,
|
||||
}
|
||||
Reference in new issue
Block a user