From c9ef418cdd77a5974b876f7951880c5631b577f7 Mon Sep 17 00:00:00 2001 From: jessikitty Date: Thu, 24 Sep 2026 14:31:07 +1000 Subject: [PATCH] Add backup/restore service: full archive of DB + all uploaded files --- app/services/backup_service.py | 779 +++++++++++++++++++++++++++++++++ 1 file changed, 779 insertions(+) create mode 100644 app/services/backup_service.py diff --git a/app/services/backup_service.py b/app/services/backup_service.py new file mode 100644 index 0000000..5e620af --- /dev/null +++ b/app/services/backup_service.py @@ -0,0 +1,779 @@ +""" +LEGO Instructions Manager - Backup & Restore service. + +Produces a single self-contained archive holding: + + manifest.json metadata, counts, integrity info + checksums.sha256 sha256 of every payload file in the archive + database/data.json portable row dump of every table + database/.db raw SQLite snapshot (SQLite deployments only) + uploads/... every uploaded PDF, image, cover, thumbnail + and extra file, with the on-disk tree intact + +The JSON dump is the thing that makes a move to Docker (and, if you want it, +Postgres) possible: it is engine-neutral and all stored paths are normalised +to forward slashes on the way out, so a Windows-created backup restores +cleanly onto Linux. + +Created by: Jess Rogerson (yelling commands at Claude.AI) +""" + +from __future__ import annotations + +import hashlib +import json +import os +import platform +import shutil +import sqlite3 +import sys +import tempfile +import threading +import uuid +import zipfile +from datetime import datetime, date, timezone + +from flask import current_app +from sqlalchemy import inspect as sa_inspect, types as sa_types + +from app import db +from app.models.user import User +from app.models.set import Set +from app.models.instruction import Instruction +from app.models.extra_file import ExtraFile + +BACKUP_FORMAT = 'lim-backup' +BACKUP_FORMAT_VERSION = 1 +GENERATOR = 'LEGO Instructions Manager backup service' + +# Restore order matters: parents before children. +TABLES = [ + ('users', User), + ('sets', Set), + ('instructions', Instruction), + ('extra_files', ExtraFile), +] + +# Columns holding a path relative to UPLOAD_FOLDER. These get normalised +# to forward slashes on backup and again on restore. +PATH_COLUMNS = { + 'sets': ['cover_image'], + 'instructions': ['file_path', 'thumbnail_path'], + 'extra_files': ['file_path'], +} + +UPLOAD_SUBDIRS = ['pdfs', 'images', 'covers', 'thumbnails', 'extra_files'] + +# Media that gains nothing from deflate; stored uncompressed so large +# collections back up quickly. +STORE_EXTENSIONS = { + '.pdf', '.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', + '.zip', '.7z', '.rar', '.gz', '.mp4', '.mov', '.io', +} + + +# -------------------------------------------------------------------------- +# Small helpers +# -------------------------------------------------------------------------- + +def _utcnow(): + return datetime.now(timezone.utc) + + +def norm_rel_path(value): + """Normalise a stored relative path to POSIX form.""" + if not value: + return value + cleaned = str(value).replace('\\', '/').strip() + while cleaned.startswith('./'): + cleaned = cleaned[2:] + return cleaned.lstrip('/') + + +def sha256_file(path, chunk_size=1024 * 1024): + digest = hashlib.sha256() + with open(path, 'rb') as handle: + for chunk in iter(lambda: handle.read(chunk_size), b''): + digest.update(chunk) + return digest.hexdigest() + + +def human_size(num_bytes): + size = float(num_bytes or 0) + for unit in ['B', 'KB', 'MB', 'GB', 'TB']: + if size < 1024.0: + return f'{size:.1f} {unit}' + size /= 1024.0 + return f'{size:.1f} PB' + + +def _compress_type(filename): + ext = os.path.splitext(filename)[1].lower() + return zipfile.ZIP_STORED if ext in STORE_EXTENSIONS else zipfile.ZIP_DEFLATED + + +def backup_folder(app=None): + app = app or current_app + folder = app.config.get('BACKUP_FOLDER') + if not folder: + folder = os.path.join(os.path.dirname(app.root_path), 'backups') + os.makedirs(folder, exist_ok=True) + return folder + + +def upload_folder(app=None): + app = app or current_app + return app.config['UPLOAD_FOLDER'] + + +def sqlite_db_path(app=None): + """Return the on-disk SQLite file backing this app, or None.""" + app = app or current_app + uri = app.config.get('SQLALCHEMY_DATABASE_URI', '') + if not uri.startswith('sqlite:'): + return None + if ':memory:' in uri: + return None + path = uri.split('sqlite:///', 1)[-1] + if not path or path == uri: + return None + return os.path.abspath(path) + + +def database_dialect(app=None): + app = app or current_app + uri = app.config.get('SQLALCHEMY_DATABASE_URI', '') + return uri.split(':', 1)[0] if ':' in uri else 'unknown' + + +# -------------------------------------------------------------------------- +# Serialisation +# -------------------------------------------------------------------------- + +def _serialise_value(value): + if isinstance(value, datetime): + return value.isoformat() + if isinstance(value, date): + return value.isoformat() + if isinstance(value, bytes): + return value.decode('utf-8', errors='replace') + return value + + +def _deserialise_value(column, value): + if value is None: + return None + if isinstance(column.type, sa_types.DateTime) and isinstance(value, str): + text = value.replace('Z', '+00:00') + parsed = datetime.fromisoformat(text) + # Stored naive-UTC throughout the app; drop any tzinfo for consistency. + if parsed.tzinfo is not None: + parsed = parsed.astimezone(timezone.utc).replace(tzinfo=None) + return parsed + if isinstance(column.type, sa_types.Date) and isinstance(value, str): + return date.fromisoformat(value) + return value + + +def dump_table(table_name, model): + """Return every row of a model as a list of plain dicts.""" + columns = [c.key for c in sa_inspect(model).columns] + path_cols = set(PATH_COLUMNS.get(table_name, [])) + rows = [] + for record in model.query.order_by(model.id).all(): + row = {} + for col in columns: + value = _serialise_value(getattr(record, col)) + if col in path_cols: + value = norm_rel_path(value) + row[col] = value + rows.append(row) + return rows + + +def referenced_files(): + """Every upload path the database expects to exist, relative to uploads.""" + paths = set() + for lego_set in Set.query.all(): + if lego_set.cover_image: + paths.add(norm_rel_path(lego_set.cover_image)) + for instruction in Instruction.query.all(): + if instruction.file_path: + paths.add(norm_rel_path(instruction.file_path)) + if instruction.thumbnail_path: + paths.add(norm_rel_path(instruction.thumbnail_path)) + for extra in ExtraFile.query.all(): + if extra.file_path: + paths.add(norm_rel_path(extra.file_path)) + return paths + + +def walk_uploads(root): + """Yield (absolute_path, relative_posix_path) for every uploaded file.""" + if not os.path.isdir(root): + return + for dirpath, _dirnames, filenames in os.walk(root): + for name in sorted(filenames): + absolute = os.path.join(dirpath, name) + relative = os.path.relpath(absolute, root).replace('\\', '/') + yield absolute, relative + + +# -------------------------------------------------------------------------- +# Create +# -------------------------------------------------------------------------- + +def create_backup(app=None, created_by=None, label=None, progress=None, + include_sqlite_file=True): + """ + Build a backup archive. Returns a dict describing the result. + + progress: optional callable(stage:str, done:int, total:int, note:str) + """ + app = app or current_app + progress = progress or (lambda *a, **k: None) + + uploads_root = upload_folder(app) + dest_dir = backup_folder(app) + + stamp = _utcnow().strftime('%Y%m%d-%H%M%S') + suffix = f'-{label}' if label else '' + safe_suffix = ''.join(ch for ch in suffix if ch.isalnum() or ch in '-_') + archive_name = f'lim-backup-{stamp}{safe_suffix}.zip' + archive_path = os.path.join(dest_dir, archive_name) + temp_path = archive_path + '.part' + + progress('scanning', 0, 0, 'Indexing uploads') + upload_files = list(walk_uploads(uploads_root)) + uploads_bytes = sum(os.path.getsize(p) for p, _ in upload_files) + total_steps = len(upload_files) + 4 + + checksums = [] + step = 0 + + try: + with zipfile.ZipFile(temp_path, 'w', compression=zipfile.ZIP_DEFLATED, + allowZip64=True) as archive: + + # --- database: portable JSON dump ------------------------------ + progress('database', step, total_steps, 'Exporting database rows') + data = {'format': BACKUP_FORMAT, + 'format_version': BACKUP_FORMAT_VERSION, + 'tables': {}} + counts = {} + for table_name, model in TABLES: + rows = dump_table(table_name, model) + data['tables'][table_name] = rows + counts[table_name] = len(rows) + + data_json = json.dumps(data, indent=2, ensure_ascii=False).encode('utf-8') + archive.writestr('database/data.json', data_json, + compress_type=zipfile.ZIP_DEFLATED) + checksums.append((hashlib.sha256(data_json).hexdigest(), + 'database/data.json')) + step += 1 + + # --- database: raw SQLite snapshot ----------------------------- + sqlite_path = sqlite_db_path(app) + sqlite_arcname = None + if include_sqlite_file and sqlite_path and os.path.exists(sqlite_path): + progress('database', step, total_steps, 'Snapshotting SQLite file') + snapshot = os.path.join(tempfile.gettempdir(), + f'lim-snapshot-{uuid.uuid4().hex}.db') + try: + source = sqlite3.connect(f'file:{sqlite_path}?mode=ro', uri=True) + target = sqlite3.connect(snapshot) + with target: + source.backup(target) + target.close() + source.close() + + sqlite_arcname = 'database/' + os.path.basename(sqlite_path) + archive.write(snapshot, sqlite_arcname, + compress_type=zipfile.ZIP_DEFLATED) + checksums.append((sha256_file(snapshot), sqlite_arcname)) + finally: + if os.path.exists(snapshot): + os.remove(snapshot) + step += 1 + + # --- uploads --------------------------------------------------- + on_disk = set() + for absolute, relative in upload_files: + arcname = 'uploads/' + relative + archive.write(absolute, arcname, + compress_type=_compress_type(relative)) + checksums.append((sha256_file(absolute), arcname)) + on_disk.add(relative) + step += 1 + if step % 25 == 0 or step == total_steps - 2: + progress('uploads', step, total_steps, relative) + + # --- integrity report ------------------------------------------ + progress('manifest', step, total_steps, 'Writing manifest') + expected = referenced_files() + missing = sorted(expected - on_disk) + orphaned = sorted(on_disk - expected) + + manifest = { + 'format': BACKUP_FORMAT, + 'format_version': BACKUP_FORMAT_VERSION, + 'generator': GENERATOR, + 'credit': 'Created by: Jess Rogerson (yelling commands at Claude.AI)', + 'created_at': _utcnow().isoformat(), + 'created_by': created_by, + 'label': label, + 'source': { + 'hostname': platform.node(), + 'platform': platform.platform(), + 'python': sys.version.split()[0], + 'database_dialect': database_dialect(app), + 'sqlite_file': sqlite_arcname, + 'upload_folder': uploads_root, + }, + 'counts': counts, + 'uploads': { + 'file_count': len(upload_files), + 'total_bytes': uploads_bytes, + 'total_human': human_size(uploads_bytes), + 'subdirs': [d for d in UPLOAD_SUBDIRS + if os.path.isdir(os.path.join(uploads_root, d))], + }, + 'integrity': { + 'missing_files': missing, + 'missing_count': len(missing), + 'orphaned_files': orphaned[:500], + 'orphaned_count': len(orphaned), + }, + } + manifest_json = json.dumps(manifest, indent=2, + ensure_ascii=False).encode('utf-8') + archive.writestr('manifest.json', manifest_json, + compress_type=zipfile.ZIP_DEFLATED) + step += 1 + + checksum_body = '\n'.join(f'{digest} {name}' + for digest, name in checksums) + '\n' + archive.writestr('checksums.sha256', checksum_body.encode('utf-8'), + compress_type=zipfile.ZIP_DEFLATED) + step += 1 + + os.replace(temp_path, archive_path) + + except Exception: + if os.path.exists(temp_path): + try: + os.remove(temp_path) + except OSError: + pass + raise + + archive_size = os.path.getsize(archive_path) + progress('done', total_steps, total_steps, archive_name) + + return { + 'filename': archive_name, + 'path': archive_path, + 'size_bytes': archive_size, + 'size_human': human_size(archive_size), + 'manifest': manifest, + } + + +# -------------------------------------------------------------------------- +# Inspect / verify +# -------------------------------------------------------------------------- + +def read_manifest(archive_path): + with zipfile.ZipFile(archive_path) as archive: + with archive.open('manifest.json') as handle: + return json.load(handle) + + +def verify_backup(archive_path): + """Recompute every checksum in the archive. Returns a report dict.""" + report = {'ok': True, 'checked': 0, 'failed': [], 'missing': [], + 'errors': []} + try: + manifest = read_manifest(archive_path) + except Exception as exc: + return {'ok': False, 'checked': 0, 'failed': [], 'missing': [], + 'errors': [f'Cannot read manifest: {exc}']} + + report['manifest'] = manifest + + with zipfile.ZipFile(archive_path) as archive: + names = set(archive.namelist()) + if 'checksums.sha256' not in names: + report['ok'] = False + report['errors'].append('No checksums.sha256 in archive') + return report + + expected = {} + with archive.open('checksums.sha256') as handle: + for line in handle.read().decode('utf-8').splitlines(): + if not line.strip(): + continue + digest, _, name = line.partition(' ') + expected[name] = digest + + for name, digest in expected.items(): + if name not in names: + report['missing'].append(name) + report['ok'] = False + continue + hasher = hashlib.sha256() + with archive.open(name) as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b''): + hasher.update(chunk) + report['checked'] += 1 + if hasher.hexdigest() != digest: + report['failed'].append(name) + report['ok'] = False + + return report + + +def list_backups(app=None): + """Return archives in the backup folder, newest first.""" + app = app or current_app + folder = backup_folder(app) + entries = [] + for name in os.listdir(folder): + if not name.lower().endswith('.zip'): + continue + path = os.path.join(folder, name) + if not os.path.isfile(path): + continue + stat = os.stat(path) + entry = { + 'filename': name, + 'path': path, + 'size_bytes': stat.st_size, + 'size_human': human_size(stat.st_size), + 'modified': datetime.fromtimestamp(stat.st_mtime), + 'manifest': None, + 'valid': False, + } + try: + entry['manifest'] = read_manifest(path) + entry['valid'] = entry['manifest'].get('format') == BACKUP_FORMAT + except Exception: + pass + entries.append(entry) + entries.sort(key=lambda e: e['modified'], reverse=True) + return entries + + +def delete_backup(filename, app=None): + app = app or current_app + folder = backup_folder(app) + safe = os.path.basename(filename) + path = os.path.join(folder, safe) + if os.path.commonpath([os.path.abspath(path), folder]) != folder: + raise ValueError('Refusing to delete outside the backup folder') + if not os.path.isfile(path): + raise FileNotFoundError(safe) + os.remove(path) + return safe + + +def prune_backups(keep, app=None): + """Delete all but the newest `keep` archives. Returns removed names.""" + if not keep or keep < 1: + return [] + removed = [] + for entry in list_backups(app)[keep:]: + try: + os.remove(entry['path']) + removed.append(entry['filename']) + except OSError: + pass + return removed + + +# -------------------------------------------------------------------------- +# Restore +# -------------------------------------------------------------------------- + +def _safe_extract_target(root, relative): + target = os.path.abspath(os.path.join(root, relative)) + if os.path.commonpath([target, os.path.abspath(root)]) != os.path.abspath(root): + raise ValueError(f'Unsafe path in archive: {relative}') + return target + + +def _table_is_empty(model): + return db.session.query(model.id).first() is None + + +def _reset_sequences(): + """Postgres keeps its own sequences; realign them after preserving IDs.""" + if db.engine.dialect.name != 'postgresql': + return + from sqlalchemy import text + for table_name, model in TABLES: + db.session.execute(text( + "SELECT setval(pg_get_serial_sequence(:t, 'id'), " + "COALESCE((SELECT MAX(id) FROM " + table_name + "), 1), true)" + ), {'t': table_name}) + db.session.commit() + + +def restore_backup(archive_path, mode='fresh', force=False, overwrite_files=False, + skip_files=False, app=None, progress=None): + """ + Restore an archive into the current instance. + + mode='fresh' target tables must be empty (or force=True to wipe them); + primary keys are preserved exactly. + mode='merge' keeps existing data; users are matched by username and sets + by set number. Sets that already exist are skipped whole. + + Returns a summary dict. + """ + app = app or current_app + progress = progress or (lambda *a, **k: None) + + if mode not in ('fresh', 'merge'): + raise ValueError("mode must be 'fresh' or 'merge'") + + manifest = read_manifest(archive_path) + if manifest.get('format') != BACKUP_FORMAT: + raise ValueError('Not a LIM backup archive') + if manifest.get('format_version', 0) > BACKUP_FORMAT_VERSION: + raise ValueError( + f"Archive format v{manifest['format_version']} is newer than this " + f"app supports (v{BACKUP_FORMAT_VERSION}). Upgrade LIM first." + ) + + summary = { + 'mode': mode, + 'manifest': manifest, + 'inserted': {name: 0 for name, _ in TABLES}, + 'skipped': {name: 0 for name, _ in TABLES}, + 'files_written': 0, + 'files_skipped': 0, + 'files_bytes': 0, + 'warnings': [], + } + + with zipfile.ZipFile(archive_path) as archive: + with archive.open('database/data.json') as handle: + data = json.load(handle) + tables = data.get('tables', {}) + + # ---- database ----------------------------------------------------- + progress('database', 0, 0, 'Restoring rows') + + if mode == 'fresh': + non_empty = [name for name, model in TABLES if not _table_is_empty(model)] + if non_empty and not force: + raise RuntimeError( + 'Target database is not empty (' + ', '.join(non_empty) + + '). Re-run with force to wipe these tables first.' + ) + if non_empty: + for table_name, model in reversed(TABLES): + deleted = model.query.delete() + summary['warnings'].append( + f'Wiped {deleted} existing row(s) from {table_name}') + db.session.commit() + + for table_name, model in TABLES: + columns = {c.key: c for c in sa_inspect(model).columns} + path_cols = set(PATH_COLUMNS.get(table_name, [])) + for row in tables.get(table_name, []): + kwargs = {} + for key, value in row.items(): + column = columns.get(key) + if column is None: + continue + if key in path_cols: + value = norm_rel_path(value) + kwargs[key] = _deserialise_value(column, value) + db.session.add(model(**kwargs)) + summary['inserted'][table_name] += 1 + db.session.flush() + db.session.commit() + _reset_sequences() + + else: # merge + user_map = {} + set_map = {} + + # users + columns = {c.key: c for c in sa_inspect(User).columns} + for row in tables.get('users', []): + existing = User.query.filter_by(username=row['username']).first() \ + or User.query.filter_by(email=row['email']).first() + if existing: + user_map[row['id']] = existing.id + summary['skipped']['users'] += 1 + continue + kwargs = {k: _deserialise_value(columns[k], v) + for k, v in row.items() if k in columns and k != 'id'} + user = User(**kwargs) + db.session.add(user) + db.session.flush() + user_map[row['id']] = user.id + summary['inserted']['users'] += 1 + + fallback_user = next(iter(user_map.values()), None) + if fallback_user is None: + first_user = User.query.order_by(User.id).first() + fallback_user = first_user.id if first_user else None + + # sets + columns = {c.key: c for c in sa_inspect(Set).columns} + skipped_set_ids = set() + for row in tables.get('sets', []): + existing = Set.query.filter_by(set_number=row['set_number']).first() + if existing: + set_map[row['id']] = existing.id + skipped_set_ids.add(row['id']) + summary['skipped']['sets'] += 1 + continue + kwargs = {} + for key, value in row.items(): + if key not in columns or key == 'id': + continue + if key in PATH_COLUMNS.get('sets', []): + value = norm_rel_path(value) + kwargs[key] = _deserialise_value(columns[key], value) + kwargs['user_id'] = user_map.get(row.get('user_id'), fallback_user) + if kwargs.get('brickset_id') is not None: + clash = Set.query.filter_by( + brickset_id=kwargs['brickset_id']).first() + if clash: + kwargs['brickset_id'] = None + lego_set = Set(**kwargs) + db.session.add(lego_set) + db.session.flush() + set_map[row['id']] = lego_set.id + summary['inserted']['sets'] += 1 + + # children of newly added sets only + for table_name, model, set_key in ( + ('instructions', Instruction, 'set_id'), + ('extra_files', ExtraFile, 'set_id'), + ): + columns = {c.key: c for c in sa_inspect(model).columns} + path_cols = set(PATH_COLUMNS.get(table_name, [])) + for row in tables.get(table_name, []): + old_set = row.get(set_key) + if old_set in skipped_set_ids or old_set not in set_map: + summary['skipped'][table_name] += 1 + continue + kwargs = {} + for key, value in row.items(): + if key not in columns or key == 'id': + continue + if key in path_cols: + value = norm_rel_path(value) + kwargs[key] = _deserialise_value(columns[key], value) + kwargs[set_key] = set_map[old_set] + for owner_key in ('user_id', 'uploaded_by'): + if owner_key in columns and row.get(owner_key) is not None: + kwargs[owner_key] = user_map.get(row[owner_key], + fallback_user) + db.session.add(model(**kwargs)) + summary['inserted'][table_name] += 1 + + db.session.commit() + + # ---- uploads ------------------------------------------------------ + if skip_files: + summary['warnings'].append('File restore skipped by request') + else: + root = upload_folder(app) + os.makedirs(root, exist_ok=True) + members = [n for n in archive.namelist() + if n.startswith('uploads/') and not n.endswith('/')] + total = len(members) + for index, name in enumerate(members, 1): + relative = name[len('uploads/'):] + if not relative: + continue + target = _safe_extract_target(root, relative) + if os.path.exists(target) and not overwrite_files: + summary['files_skipped'] += 1 + continue + os.makedirs(os.path.dirname(target), exist_ok=True) + with archive.open(name) as source, open(target, 'wb') as dest: + shutil.copyfileobj(source, dest, 1024 * 1024) + summary['files_written'] += 1 + summary['files_bytes'] += os.path.getsize(target) + if index % 25 == 0 or index == total: + progress('uploads', index, total, relative) + + summary['files_bytes_human'] = human_size(summary['files_bytes']) + return summary + + +# -------------------------------------------------------------------------- +# Background job tracking (used by the admin UI) +# -------------------------------------------------------------------------- + +_JOBS = {} +_JOBS_LOCK = threading.Lock() + + +def _set_job(job_id, **fields): + with _JOBS_LOCK: + job = _JOBS.setdefault(job_id, {'id': job_id}) + job.update(fields) + return dict(job) + + +def get_job(job_id): + with _JOBS_LOCK: + job = _JOBS.get(job_id) + return dict(job) if job else None + + +def start_backup_job(app, created_by=None, label=None, keep=None): + """Kick off a backup in a worker thread and return its job id.""" + job_id = uuid.uuid4().hex + _set_job(job_id, state='queued', stage='queued', done=0, total=0, + note='Waiting to start', started_at=_utcnow().isoformat(), + result=None, error=None) + + def worker(): + with app.app_context(): + try: + _set_job(job_id, state='running') + + def on_progress(stage, done, total, note): + _set_job(job_id, stage=stage, done=done, total=total, + note=note) + + result = create_backup(app=app, created_by=created_by, + label=label, progress=on_progress) + pruned = prune_backups(keep, app=app) if keep else [] + _set_job(job_id, state='complete', stage='done', + note=result['filename'], + result={ + 'filename': result['filename'], + 'size_human': result['size_human'], + 'size_bytes': result['size_bytes'], + 'counts': result['manifest']['counts'], + 'uploads': result['manifest']['uploads'], + 'integrity': { + 'missing_count': + result['manifest']['integrity']['missing_count'], + 'orphaned_count': + result['manifest']['integrity']['orphaned_count'], + }, + 'pruned': pruned, + }, + finished_at=_utcnow().isoformat()) + except Exception as exc: # noqa: BLE001 - surfaced to the UI + app.logger.exception('Backup job failed') + _set_job(job_id, state='failed', error=str(exc), + finished_at=_utcnow().isoformat()) + + thread = threading.Thread(target=worker, name=f'lim-backup-{job_id[:8]}', + daemon=True) + thread.start() + return job_id