""" LEGO Instructions Manager - Backup & Restore service. Produces a single self-contained archive holding: manifest.json metadata, counts, integrity info checksums.sha256 sha256 of every payload file in the archive database/data.json portable row dump of every table database/.db raw SQLite snapshot (SQLite deployments only) uploads/... every uploaded PDF, image, cover, thumbnail and extra file, with the on-disk tree intact The JSON dump is the thing that makes a move to Docker (and, if you want it, Postgres) possible: it is engine-neutral and all stored paths are normalised to forward slashes on the way out, so a Windows-created backup restores cleanly onto Linux. Created by: Jess Rogerson (yelling commands at Claude.AI) """ from __future__ import annotations import hashlib import json import os import platform import shutil import sqlite3 import sys import tempfile import threading import uuid import zipfile from datetime import datetime, date, timezone from flask import current_app from sqlalchemy import inspect as sa_inspect, types as sa_types from app import db from app.models.user import User from app.models.set import Set from app.models.instruction import Instruction from app.models.extra_file import ExtraFile BACKUP_FORMAT = 'lim-backup' BACKUP_FORMAT_VERSION = 1 GENERATOR = 'LEGO Instructions Manager backup service' # Restore order matters: parents before children. TABLES = [ ('users', User), ('sets', Set), ('instructions', Instruction), ('extra_files', ExtraFile), ] # Columns holding a path relative to UPLOAD_FOLDER. These get normalised # to forward slashes on backup and again on restore. PATH_COLUMNS = { 'sets': ['cover_image'], 'instructions': ['file_path', 'thumbnail_path'], 'extra_files': ['file_path'], } UPLOAD_SUBDIRS = ['pdfs', 'images', 'covers', 'thumbnails', 'extra_files'] # Media that gains nothing from deflate; stored uncompressed so large # collections back up quickly. STORE_EXTENSIONS = { '.pdf', '.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.zip', '.7z', '.rar', '.gz', '.mp4', '.mov', '.io', } # -------------------------------------------------------------------------- # Small helpers # -------------------------------------------------------------------------- def _utcnow(): return datetime.now(timezone.utc) def norm_rel_path(value): """Normalise a stored relative path to POSIX form.""" if not value: return value cleaned = str(value).replace('\\', '/').strip() while cleaned.startswith('./'): cleaned = cleaned[2:] return cleaned.lstrip('/') def sha256_file(path, chunk_size=1024 * 1024): digest = hashlib.sha256() with open(path, 'rb') as handle: for chunk in iter(lambda: handle.read(chunk_size), b''): digest.update(chunk) return digest.hexdigest() def human_size(num_bytes): size = float(num_bytes or 0) for unit in ['B', 'KB', 'MB', 'GB', 'TB']: if size < 1024.0: return f'{size:.1f} {unit}' size /= 1024.0 return f'{size:.1f} PB' def _compress_type(filename): ext = os.path.splitext(filename)[1].lower() return zipfile.ZIP_STORED if ext in STORE_EXTENSIONS else zipfile.ZIP_DEFLATED def backup_folder(app=None): app = app or current_app folder = app.config.get('BACKUP_FOLDER') if not folder: folder = os.path.join(os.path.dirname(app.root_path), 'backups') os.makedirs(folder, exist_ok=True) return folder def upload_folder(app=None): app = app or current_app return app.config['UPLOAD_FOLDER'] def sqlite_db_path(app=None): """Return the on-disk SQLite file backing this app, or None.""" app = app or current_app uri = app.config.get('SQLALCHEMY_DATABASE_URI', '') if not uri.startswith('sqlite:'): return None if ':memory:' in uri: return None path = uri.split('sqlite:///', 1)[-1] if not path or path == uri: return None return os.path.abspath(path) def database_dialect(app=None): app = app or current_app uri = app.config.get('SQLALCHEMY_DATABASE_URI', '') return uri.split(':', 1)[0] if ':' in uri else 'unknown' # -------------------------------------------------------------------------- # Serialisation # -------------------------------------------------------------------------- def _serialise_value(value): if isinstance(value, datetime): return value.isoformat() if isinstance(value, date): return value.isoformat() if isinstance(value, bytes): return value.decode('utf-8', errors='replace') return value def _deserialise_value(column, value): if value is None: return None if isinstance(column.type, sa_types.DateTime) and isinstance(value, str): text = value.replace('Z', '+00:00') parsed = datetime.fromisoformat(text) # Stored naive-UTC throughout the app; drop any tzinfo for consistency. if parsed.tzinfo is not None: parsed = parsed.astimezone(timezone.utc).replace(tzinfo=None) return parsed if isinstance(column.type, sa_types.Date) and isinstance(value, str): return date.fromisoformat(value) return value def dump_table(table_name, model): """Return every row of a model as a list of plain dicts.""" columns = [c.key for c in sa_inspect(model).columns] path_cols = set(PATH_COLUMNS.get(table_name, [])) rows = [] for record in model.query.order_by(model.id).all(): row = {} for col in columns: value = _serialise_value(getattr(record, col)) if col in path_cols: value = norm_rel_path(value) row[col] = value rows.append(row) return rows def referenced_files(): """Every upload path the database expects to exist, relative to uploads.""" paths = set() for lego_set in Set.query.all(): if lego_set.cover_image: paths.add(norm_rel_path(lego_set.cover_image)) for instruction in Instruction.query.all(): if instruction.file_path: paths.add(norm_rel_path(instruction.file_path)) if instruction.thumbnail_path: paths.add(norm_rel_path(instruction.thumbnail_path)) for extra in ExtraFile.query.all(): if extra.file_path: paths.add(norm_rel_path(extra.file_path)) return paths def walk_uploads(root): """Yield (absolute_path, relative_posix_path) for every uploaded file.""" if not os.path.isdir(root): return for dirpath, _dirnames, filenames in os.walk(root): for name in sorted(filenames): absolute = os.path.join(dirpath, name) relative = os.path.relpath(absolute, root).replace('\\', '/') yield absolute, relative # -------------------------------------------------------------------------- # Create # -------------------------------------------------------------------------- def create_backup(app=None, created_by=None, label=None, progress=None, include_sqlite_file=True): """ Build a backup archive. Returns a dict describing the result. progress: optional callable(stage:str, done:int, total:int, note:str) """ app = app or current_app progress = progress or (lambda *a, **k: None) uploads_root = upload_folder(app) dest_dir = backup_folder(app) stamp = _utcnow().strftime('%Y%m%d-%H%M%S') suffix = f'-{label}' if label else '' safe_suffix = ''.join(ch for ch in suffix if ch.isalnum() or ch in '-_') archive_name = f'lim-backup-{stamp}{safe_suffix}.zip' archive_path = os.path.join(dest_dir, archive_name) temp_path = archive_path + '.part' progress('scanning', 0, 0, 'Indexing uploads') upload_files = list(walk_uploads(uploads_root)) uploads_bytes = sum(os.path.getsize(p) for p, _ in upload_files) total_steps = len(upload_files) + 4 checksums = [] step = 0 try: with zipfile.ZipFile(temp_path, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as archive: # --- database: portable JSON dump ------------------------------ progress('database', step, total_steps, 'Exporting database rows') data = {'format': BACKUP_FORMAT, 'format_version': BACKUP_FORMAT_VERSION, 'tables': {}} counts = {} for table_name, model in TABLES: rows = dump_table(table_name, model) data['tables'][table_name] = rows counts[table_name] = len(rows) data_json = json.dumps(data, indent=2, ensure_ascii=False).encode('utf-8') archive.writestr('database/data.json', data_json, compress_type=zipfile.ZIP_DEFLATED) checksums.append((hashlib.sha256(data_json).hexdigest(), 'database/data.json')) step += 1 # --- database: raw SQLite snapshot ----------------------------- sqlite_path = sqlite_db_path(app) sqlite_arcname = None if include_sqlite_file and sqlite_path and os.path.exists(sqlite_path): progress('database', step, total_steps, 'Snapshotting SQLite file') snapshot = os.path.join(tempfile.gettempdir(), f'lim-snapshot-{uuid.uuid4().hex}.db') try: source = sqlite3.connect(f'file:{sqlite_path}?mode=ro', uri=True) target = sqlite3.connect(snapshot) with target: source.backup(target) target.close() source.close() sqlite_arcname = 'database/' + os.path.basename(sqlite_path) archive.write(snapshot, sqlite_arcname, compress_type=zipfile.ZIP_DEFLATED) checksums.append((sha256_file(snapshot), sqlite_arcname)) finally: if os.path.exists(snapshot): os.remove(snapshot) step += 1 # --- uploads --------------------------------------------------- on_disk = set() for absolute, relative in upload_files: arcname = 'uploads/' + relative archive.write(absolute, arcname, compress_type=_compress_type(relative)) checksums.append((sha256_file(absolute), arcname)) on_disk.add(relative) step += 1 if step % 25 == 0 or step == total_steps - 2: progress('uploads', step, total_steps, relative) # --- integrity report ------------------------------------------ progress('manifest', step, total_steps, 'Writing manifest') expected = referenced_files() missing = sorted(expected - on_disk) orphaned = sorted(on_disk - expected) manifest = { 'format': BACKUP_FORMAT, 'format_version': BACKUP_FORMAT_VERSION, 'generator': GENERATOR, 'credit': 'Created by: Jess Rogerson (yelling commands at Claude.AI)', 'created_at': _utcnow().isoformat(), 'created_by': created_by, 'label': label, 'source': { 'hostname': platform.node(), 'platform': platform.platform(), 'python': sys.version.split()[0], 'database_dialect': database_dialect(app), 'sqlite_file': sqlite_arcname, 'upload_folder': uploads_root, }, 'counts': counts, 'uploads': { 'file_count': len(upload_files), 'total_bytes': uploads_bytes, 'total_human': human_size(uploads_bytes), 'subdirs': [d for d in UPLOAD_SUBDIRS if os.path.isdir(os.path.join(uploads_root, d))], }, 'integrity': { 'missing_files': missing, 'missing_count': len(missing), 'orphaned_files': orphaned[:500], 'orphaned_count': len(orphaned), }, } manifest_json = json.dumps(manifest, indent=2, ensure_ascii=False).encode('utf-8') archive.writestr('manifest.json', manifest_json, compress_type=zipfile.ZIP_DEFLATED) step += 1 checksum_body = '\n'.join(f'{digest} {name}' for digest, name in checksums) + '\n' archive.writestr('checksums.sha256', checksum_body.encode('utf-8'), compress_type=zipfile.ZIP_DEFLATED) step += 1 os.replace(temp_path, archive_path) except Exception: if os.path.exists(temp_path): try: os.remove(temp_path) except OSError: pass raise archive_size = os.path.getsize(archive_path) progress('done', total_steps, total_steps, archive_name) return { 'filename': archive_name, 'path': archive_path, 'size_bytes': archive_size, 'size_human': human_size(archive_size), 'manifest': manifest, } # -------------------------------------------------------------------------- # Inspect / verify # -------------------------------------------------------------------------- def read_manifest(archive_path): with zipfile.ZipFile(archive_path) as archive: with archive.open('manifest.json') as handle: return json.load(handle) def verify_backup(archive_path): """Recompute every checksum in the archive. Returns a report dict.""" report = {'ok': True, 'checked': 0, 'failed': [], 'missing': [], 'errors': []} try: manifest = read_manifest(archive_path) except Exception as exc: return {'ok': False, 'checked': 0, 'failed': [], 'missing': [], 'errors': [f'Cannot read manifest: {exc}']} report['manifest'] = manifest with zipfile.ZipFile(archive_path) as archive: names = set(archive.namelist()) if 'checksums.sha256' not in names: report['ok'] = False report['errors'].append('No checksums.sha256 in archive') return report expected = {} with archive.open('checksums.sha256') as handle: for line in handle.read().decode('utf-8').splitlines(): if not line.strip(): continue digest, _, name = line.partition(' ') expected[name] = digest for name, digest in expected.items(): if name not in names: report['missing'].append(name) report['ok'] = False continue hasher = hashlib.sha256() with archive.open(name) as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b''): hasher.update(chunk) report['checked'] += 1 if hasher.hexdigest() != digest: report['failed'].append(name) report['ok'] = False return report def list_backups(app=None): """Return archives in the backup folder, newest first.""" app = app or current_app folder = backup_folder(app) entries = [] for name in os.listdir(folder): if not name.lower().endswith('.zip'): continue path = os.path.join(folder, name) if not os.path.isfile(path): continue stat = os.stat(path) entry = { 'filename': name, 'path': path, 'size_bytes': stat.st_size, 'size_human': human_size(stat.st_size), 'modified': datetime.fromtimestamp(stat.st_mtime), 'manifest': None, 'valid': False, } try: entry['manifest'] = read_manifest(path) entry['valid'] = entry['manifest'].get('format') == BACKUP_FORMAT except Exception: pass entries.append(entry) entries.sort(key=lambda e: e['modified'], reverse=True) return entries def delete_backup(filename, app=None): app = app or current_app folder = backup_folder(app) safe = os.path.basename(filename) path = os.path.join(folder, safe) if os.path.commonpath([os.path.abspath(path), folder]) != folder: raise ValueError('Refusing to delete outside the backup folder') if not os.path.isfile(path): raise FileNotFoundError(safe) os.remove(path) return safe def prune_backups(keep, app=None): """Delete all but the newest `keep` archives. Returns removed names.""" if not keep or keep < 1: return [] removed = [] for entry in list_backups(app)[keep:]: try: os.remove(entry['path']) removed.append(entry['filename']) except OSError: pass return removed # -------------------------------------------------------------------------- # Restore # -------------------------------------------------------------------------- def _safe_extract_target(root, relative): target = os.path.abspath(os.path.join(root, relative)) if os.path.commonpath([target, os.path.abspath(root)]) != os.path.abspath(root): raise ValueError(f'Unsafe path in archive: {relative}') return target def _table_is_empty(model): return db.session.query(model.id).first() is None def _reset_sequences(): """Postgres keeps its own sequences; realign them after preserving IDs.""" if db.engine.dialect.name != 'postgresql': return from sqlalchemy import text for table_name, model in TABLES: db.session.execute(text( "SELECT setval(pg_get_serial_sequence(:t, 'id'), " "COALESCE((SELECT MAX(id) FROM " + table_name + "), 1), true)" ), {'t': table_name}) db.session.commit() def restore_backup(archive_path, mode='fresh', force=False, overwrite_files=False, skip_files=False, app=None, progress=None): """ Restore an archive into the current instance. mode='fresh' target tables must be empty (or force=True to wipe them); primary keys are preserved exactly. mode='merge' keeps existing data; users are matched by username and sets by set number. Sets that already exist are skipped whole. Returns a summary dict. """ app = app or current_app progress = progress or (lambda *a, **k: None) if mode not in ('fresh', 'merge'): raise ValueError("mode must be 'fresh' or 'merge'") manifest = read_manifest(archive_path) if manifest.get('format') != BACKUP_FORMAT: raise ValueError('Not a LIM backup archive') if manifest.get('format_version', 0) > BACKUP_FORMAT_VERSION: raise ValueError( f"Archive format v{manifest['format_version']} is newer than this " f"app supports (v{BACKUP_FORMAT_VERSION}). Upgrade LIM first." ) summary = { 'mode': mode, 'manifest': manifest, 'inserted': {name: 0 for name, _ in TABLES}, 'skipped': {name: 0 for name, _ in TABLES}, 'files_written': 0, 'files_skipped': 0, 'files_bytes': 0, 'warnings': [], } with zipfile.ZipFile(archive_path) as archive: with archive.open('database/data.json') as handle: data = json.load(handle) tables = data.get('tables', {}) # ---- database ----------------------------------------------------- progress('database', 0, 0, 'Restoring rows') if mode == 'fresh': non_empty = [name for name, model in TABLES if not _table_is_empty(model)] if non_empty and not force: raise RuntimeError( 'Target database is not empty (' + ', '.join(non_empty) + '). Re-run with force to wipe these tables first.' ) if non_empty: for table_name, model in reversed(TABLES): deleted = model.query.delete() summary['warnings'].append( f'Wiped {deleted} existing row(s) from {table_name}') db.session.commit() for table_name, model in TABLES: columns = {c.key: c for c in sa_inspect(model).columns} path_cols = set(PATH_COLUMNS.get(table_name, [])) for row in tables.get(table_name, []): kwargs = {} for key, value in row.items(): column = columns.get(key) if column is None: continue if key in path_cols: value = norm_rel_path(value) kwargs[key] = _deserialise_value(column, value) db.session.add(model(**kwargs)) summary['inserted'][table_name] += 1 db.session.flush() db.session.commit() _reset_sequences() else: # merge user_map = {} set_map = {} # users columns = {c.key: c for c in sa_inspect(User).columns} for row in tables.get('users', []): existing = User.query.filter_by(username=row['username']).first() \ or User.query.filter_by(email=row['email']).first() if existing: user_map[row['id']] = existing.id summary['skipped']['users'] += 1 continue kwargs = {k: _deserialise_value(columns[k], v) for k, v in row.items() if k in columns and k != 'id'} user = User(**kwargs) db.session.add(user) db.session.flush() user_map[row['id']] = user.id summary['inserted']['users'] += 1 fallback_user = next(iter(user_map.values()), None) if fallback_user is None: first_user = User.query.order_by(User.id).first() fallback_user = first_user.id if first_user else None # sets columns = {c.key: c for c in sa_inspect(Set).columns} skipped_set_ids = set() for row in tables.get('sets', []): existing = Set.query.filter_by(set_number=row['set_number']).first() if existing: set_map[row['id']] = existing.id skipped_set_ids.add(row['id']) summary['skipped']['sets'] += 1 continue kwargs = {} for key, value in row.items(): if key not in columns or key == 'id': continue if key in PATH_COLUMNS.get('sets', []): value = norm_rel_path(value) kwargs[key] = _deserialise_value(columns[key], value) kwargs['user_id'] = user_map.get(row.get('user_id'), fallback_user) if kwargs.get('brickset_id') is not None: clash = Set.query.filter_by( brickset_id=kwargs['brickset_id']).first() if clash: kwargs['brickset_id'] = None lego_set = Set(**kwargs) db.session.add(lego_set) db.session.flush() set_map[row['id']] = lego_set.id summary['inserted']['sets'] += 1 # children of newly added sets only for table_name, model, set_key in ( ('instructions', Instruction, 'set_id'), ('extra_files', ExtraFile, 'set_id'), ): columns = {c.key: c for c in sa_inspect(model).columns} path_cols = set(PATH_COLUMNS.get(table_name, [])) for row in tables.get(table_name, []): old_set = row.get(set_key) if old_set in skipped_set_ids or old_set not in set_map: summary['skipped'][table_name] += 1 continue kwargs = {} for key, value in row.items(): if key not in columns or key == 'id': continue if key in path_cols: value = norm_rel_path(value) kwargs[key] = _deserialise_value(columns[key], value) kwargs[set_key] = set_map[old_set] for owner_key in ('user_id', 'uploaded_by'): if owner_key in columns and row.get(owner_key) is not None: kwargs[owner_key] = user_map.get(row[owner_key], fallback_user) db.session.add(model(**kwargs)) summary['inserted'][table_name] += 1 db.session.commit() # ---- uploads ------------------------------------------------------ if skip_files: summary['warnings'].append('File restore skipped by request') else: root = upload_folder(app) os.makedirs(root, exist_ok=True) members = [n for n in archive.namelist() if n.startswith('uploads/') and not n.endswith('/')] total = len(members) for index, name in enumerate(members, 1): relative = name[len('uploads/'):] if not relative: continue target = _safe_extract_target(root, relative) if os.path.exists(target) and not overwrite_files: summary['files_skipped'] += 1 continue os.makedirs(os.path.dirname(target), exist_ok=True) with archive.open(name) as source, open(target, 'wb') as dest: shutil.copyfileobj(source, dest, 1024 * 1024) summary['files_written'] += 1 summary['files_bytes'] += os.path.getsize(target) if index % 25 == 0 or index == total: progress('uploads', index, total, relative) summary['files_bytes_human'] = human_size(summary['files_bytes']) return summary # -------------------------------------------------------------------------- # Background job tracking (used by the admin UI) # -------------------------------------------------------------------------- _JOBS = {} _JOBS_LOCK = threading.Lock() def _set_job(job_id, **fields): with _JOBS_LOCK: job = _JOBS.setdefault(job_id, {'id': job_id}) job.update(fields) return dict(job) def get_job(job_id): with _JOBS_LOCK: job = _JOBS.get(job_id) return dict(job) if job else None def start_backup_job(app, created_by=None, label=None, keep=None): """Kick off a backup in a worker thread and return its job id.""" job_id = uuid.uuid4().hex _set_job(job_id, state='queued', stage='queued', done=0, total=0, note='Waiting to start', started_at=_utcnow().isoformat(), result=None, error=None) def worker(): with app.app_context(): try: _set_job(job_id, state='running') def on_progress(stage, done, total, note): _set_job(job_id, stage=stage, done=done, total=total, note=note) result = create_backup(app=app, created_by=created_by, label=label, progress=on_progress) pruned = prune_backups(keep, app=app) if keep else [] _set_job(job_id, state='complete', stage='done', note=result['filename'], result={ 'filename': result['filename'], 'size_human': result['size_human'], 'size_bytes': result['size_bytes'], 'counts': result['manifest']['counts'], 'uploads': result['manifest']['uploads'], 'integrity': { 'missing_count': result['manifest']['integrity']['missing_count'], 'orphaned_count': result['manifest']['integrity']['orphaned_count'], }, 'pruned': pruned, }, finished_at=_utcnow().isoformat()) except Exception as exc: # noqa: BLE001 - surfaced to the UI app.logger.exception('Backup job failed') _set_job(job_id, state='failed', error=str(exc), finished_at=_utcnow().isoformat()) thread = threading.Thread(target=worker, name=f'lim-backup-{job_id[:8]}', daemon=True) thread.start() return job_id