780 lines
29 KiB
Python
780 lines
29 KiB
Python
"""
|
|
LEGO Instructions Manager - Backup & Restore service.
|
|
|
|
Produces a single self-contained archive holding:
|
|
|
|
manifest.json metadata, counts, integrity info
|
|
checksums.sha256 sha256 of every payload file in the archive
|
|
database/data.json portable row dump of every table
|
|
database/<name>.db raw SQLite snapshot (SQLite deployments only)
|
|
uploads/... every uploaded PDF, image, cover, thumbnail
|
|
and extra file, with the on-disk tree intact
|
|
|
|
The JSON dump is the thing that makes a move to Docker (and, if you want it,
|
|
Postgres) possible: it is engine-neutral and all stored paths are normalised
|
|
to forward slashes on the way out, so a Windows-created backup restores
|
|
cleanly onto Linux.
|
|
|
|
Created by: Jess Rogerson (yelling commands at Claude.AI)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import platform
|
|
import shutil
|
|
import sqlite3
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import uuid
|
|
import zipfile
|
|
from datetime import datetime, date, timezone
|
|
|
|
from flask import current_app
|
|
from sqlalchemy import inspect as sa_inspect, types as sa_types
|
|
|
|
from app import db
|
|
from app.models.user import User
|
|
from app.models.set import Set
|
|
from app.models.instruction import Instruction
|
|
from app.models.extra_file import ExtraFile
|
|
|
|
BACKUP_FORMAT = 'lim-backup'
|
|
BACKUP_FORMAT_VERSION = 1
|
|
GENERATOR = 'LEGO Instructions Manager backup service'
|
|
|
|
# Restore order matters: parents before children.
|
|
TABLES = [
|
|
('users', User),
|
|
('sets', Set),
|
|
('instructions', Instruction),
|
|
('extra_files', ExtraFile),
|
|
]
|
|
|
|
# Columns holding a path relative to UPLOAD_FOLDER. These get normalised
|
|
# to forward slashes on backup and again on restore.
|
|
PATH_COLUMNS = {
|
|
'sets': ['cover_image'],
|
|
'instructions': ['file_path', 'thumbnail_path'],
|
|
'extra_files': ['file_path'],
|
|
}
|
|
|
|
UPLOAD_SUBDIRS = ['pdfs', 'images', 'covers', 'thumbnails', 'extra_files']
|
|
|
|
# Media that gains nothing from deflate; stored uncompressed so large
|
|
# collections back up quickly.
|
|
STORE_EXTENSIONS = {
|
|
'.pdf', '.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp',
|
|
'.zip', '.7z', '.rar', '.gz', '.mp4', '.mov', '.io',
|
|
}
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Small helpers
|
|
# --------------------------------------------------------------------------
|
|
|
|
def _utcnow():
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
def norm_rel_path(value):
|
|
"""Normalise a stored relative path to POSIX form."""
|
|
if not value:
|
|
return value
|
|
cleaned = str(value).replace('\\', '/').strip()
|
|
while cleaned.startswith('./'):
|
|
cleaned = cleaned[2:]
|
|
return cleaned.lstrip('/')
|
|
|
|
|
|
def sha256_file(path, chunk_size=1024 * 1024):
|
|
digest = hashlib.sha256()
|
|
with open(path, 'rb') as handle:
|
|
for chunk in iter(lambda: handle.read(chunk_size), b''):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def human_size(num_bytes):
|
|
size = float(num_bytes or 0)
|
|
for unit in ['B', 'KB', 'MB', 'GB', 'TB']:
|
|
if size < 1024.0:
|
|
return f'{size:.1f} {unit}'
|
|
size /= 1024.0
|
|
return f'{size:.1f} PB'
|
|
|
|
|
|
def _compress_type(filename):
|
|
ext = os.path.splitext(filename)[1].lower()
|
|
return zipfile.ZIP_STORED if ext in STORE_EXTENSIONS else zipfile.ZIP_DEFLATED
|
|
|
|
|
|
def backup_folder(app=None):
|
|
app = app or current_app
|
|
folder = app.config.get('BACKUP_FOLDER')
|
|
if not folder:
|
|
folder = os.path.join(os.path.dirname(app.root_path), 'backups')
|
|
os.makedirs(folder, exist_ok=True)
|
|
return folder
|
|
|
|
|
|
def upload_folder(app=None):
|
|
app = app or current_app
|
|
return app.config['UPLOAD_FOLDER']
|
|
|
|
|
|
def sqlite_db_path(app=None):
|
|
"""Return the on-disk SQLite file backing this app, or None."""
|
|
app = app or current_app
|
|
uri = app.config.get('SQLALCHEMY_DATABASE_URI', '')
|
|
if not uri.startswith('sqlite:'):
|
|
return None
|
|
if ':memory:' in uri:
|
|
return None
|
|
path = uri.split('sqlite:///', 1)[-1]
|
|
if not path or path == uri:
|
|
return None
|
|
return os.path.abspath(path)
|
|
|
|
|
|
def database_dialect(app=None):
|
|
app = app or current_app
|
|
uri = app.config.get('SQLALCHEMY_DATABASE_URI', '')
|
|
return uri.split(':', 1)[0] if ':' in uri else 'unknown'
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Serialisation
|
|
# --------------------------------------------------------------------------
|
|
|
|
def _serialise_value(value):
|
|
if isinstance(value, datetime):
|
|
return value.isoformat()
|
|
if isinstance(value, date):
|
|
return value.isoformat()
|
|
if isinstance(value, bytes):
|
|
return value.decode('utf-8', errors='replace')
|
|
return value
|
|
|
|
|
|
def _deserialise_value(column, value):
|
|
if value is None:
|
|
return None
|
|
if isinstance(column.type, sa_types.DateTime) and isinstance(value, str):
|
|
text = value.replace('Z', '+00:00')
|
|
parsed = datetime.fromisoformat(text)
|
|
# Stored naive-UTC throughout the app; drop any tzinfo for consistency.
|
|
if parsed.tzinfo is not None:
|
|
parsed = parsed.astimezone(timezone.utc).replace(tzinfo=None)
|
|
return parsed
|
|
if isinstance(column.type, sa_types.Date) and isinstance(value, str):
|
|
return date.fromisoformat(value)
|
|
return value
|
|
|
|
|
|
def dump_table(table_name, model):
|
|
"""Return every row of a model as a list of plain dicts."""
|
|
columns = [c.key for c in sa_inspect(model).columns]
|
|
path_cols = set(PATH_COLUMNS.get(table_name, []))
|
|
rows = []
|
|
for record in model.query.order_by(model.id).all():
|
|
row = {}
|
|
for col in columns:
|
|
value = _serialise_value(getattr(record, col))
|
|
if col in path_cols:
|
|
value = norm_rel_path(value)
|
|
row[col] = value
|
|
rows.append(row)
|
|
return rows
|
|
|
|
|
|
def referenced_files():
|
|
"""Every upload path the database expects to exist, relative to uploads."""
|
|
paths = set()
|
|
for lego_set in Set.query.all():
|
|
if lego_set.cover_image:
|
|
paths.add(norm_rel_path(lego_set.cover_image))
|
|
for instruction in Instruction.query.all():
|
|
if instruction.file_path:
|
|
paths.add(norm_rel_path(instruction.file_path))
|
|
if instruction.thumbnail_path:
|
|
paths.add(norm_rel_path(instruction.thumbnail_path))
|
|
for extra in ExtraFile.query.all():
|
|
if extra.file_path:
|
|
paths.add(norm_rel_path(extra.file_path))
|
|
return paths
|
|
|
|
|
|
def walk_uploads(root):
|
|
"""Yield (absolute_path, relative_posix_path) for every uploaded file."""
|
|
if not os.path.isdir(root):
|
|
return
|
|
for dirpath, _dirnames, filenames in os.walk(root):
|
|
for name in sorted(filenames):
|
|
absolute = os.path.join(dirpath, name)
|
|
relative = os.path.relpath(absolute, root).replace('\\', '/')
|
|
yield absolute, relative
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Create
|
|
# --------------------------------------------------------------------------
|
|
|
|
def create_backup(app=None, created_by=None, label=None, progress=None,
|
|
include_sqlite_file=True):
|
|
"""
|
|
Build a backup archive. Returns a dict describing the result.
|
|
|
|
progress: optional callable(stage:str, done:int, total:int, note:str)
|
|
"""
|
|
app = app or current_app
|
|
progress = progress or (lambda *a, **k: None)
|
|
|
|
uploads_root = upload_folder(app)
|
|
dest_dir = backup_folder(app)
|
|
|
|
stamp = _utcnow().strftime('%Y%m%d-%H%M%S')
|
|
suffix = f'-{label}' if label else ''
|
|
safe_suffix = ''.join(ch for ch in suffix if ch.isalnum() or ch in '-_')
|
|
archive_name = f'lim-backup-{stamp}{safe_suffix}.zip'
|
|
archive_path = os.path.join(dest_dir, archive_name)
|
|
temp_path = archive_path + '.part'
|
|
|
|
progress('scanning', 0, 0, 'Indexing uploads')
|
|
upload_files = list(walk_uploads(uploads_root))
|
|
uploads_bytes = sum(os.path.getsize(p) for p, _ in upload_files)
|
|
total_steps = len(upload_files) + 4
|
|
|
|
checksums = []
|
|
step = 0
|
|
|
|
try:
|
|
with zipfile.ZipFile(temp_path, 'w', compression=zipfile.ZIP_DEFLATED,
|
|
allowZip64=True) as archive:
|
|
|
|
# --- database: portable JSON dump ------------------------------
|
|
progress('database', step, total_steps, 'Exporting database rows')
|
|
data = {'format': BACKUP_FORMAT,
|
|
'format_version': BACKUP_FORMAT_VERSION,
|
|
'tables': {}}
|
|
counts = {}
|
|
for table_name, model in TABLES:
|
|
rows = dump_table(table_name, model)
|
|
data['tables'][table_name] = rows
|
|
counts[table_name] = len(rows)
|
|
|
|
data_json = json.dumps(data, indent=2, ensure_ascii=False).encode('utf-8')
|
|
archive.writestr('database/data.json', data_json,
|
|
compress_type=zipfile.ZIP_DEFLATED)
|
|
checksums.append((hashlib.sha256(data_json).hexdigest(),
|
|
'database/data.json'))
|
|
step += 1
|
|
|
|
# --- database: raw SQLite snapshot -----------------------------
|
|
sqlite_path = sqlite_db_path(app)
|
|
sqlite_arcname = None
|
|
if include_sqlite_file and sqlite_path and os.path.exists(sqlite_path):
|
|
progress('database', step, total_steps, 'Snapshotting SQLite file')
|
|
snapshot = os.path.join(tempfile.gettempdir(),
|
|
f'lim-snapshot-{uuid.uuid4().hex}.db')
|
|
try:
|
|
source = sqlite3.connect(f'file:{sqlite_path}?mode=ro', uri=True)
|
|
target = sqlite3.connect(snapshot)
|
|
with target:
|
|
source.backup(target)
|
|
target.close()
|
|
source.close()
|
|
|
|
sqlite_arcname = 'database/' + os.path.basename(sqlite_path)
|
|
archive.write(snapshot, sqlite_arcname,
|
|
compress_type=zipfile.ZIP_DEFLATED)
|
|
checksums.append((sha256_file(snapshot), sqlite_arcname))
|
|
finally:
|
|
if os.path.exists(snapshot):
|
|
os.remove(snapshot)
|
|
step += 1
|
|
|
|
# --- uploads ---------------------------------------------------
|
|
on_disk = set()
|
|
for absolute, relative in upload_files:
|
|
arcname = 'uploads/' + relative
|
|
archive.write(absolute, arcname,
|
|
compress_type=_compress_type(relative))
|
|
checksums.append((sha256_file(absolute), arcname))
|
|
on_disk.add(relative)
|
|
step += 1
|
|
if step % 25 == 0 or step == total_steps - 2:
|
|
progress('uploads', step, total_steps, relative)
|
|
|
|
# --- integrity report ------------------------------------------
|
|
progress('manifest', step, total_steps, 'Writing manifest')
|
|
expected = referenced_files()
|
|
missing = sorted(expected - on_disk)
|
|
orphaned = sorted(on_disk - expected)
|
|
|
|
manifest = {
|
|
'format': BACKUP_FORMAT,
|
|
'format_version': BACKUP_FORMAT_VERSION,
|
|
'generator': GENERATOR,
|
|
'credit': 'Created by: Jess Rogerson (yelling commands at Claude.AI)',
|
|
'created_at': _utcnow().isoformat(),
|
|
'created_by': created_by,
|
|
'label': label,
|
|
'source': {
|
|
'hostname': platform.node(),
|
|
'platform': platform.platform(),
|
|
'python': sys.version.split()[0],
|
|
'database_dialect': database_dialect(app),
|
|
'sqlite_file': sqlite_arcname,
|
|
'upload_folder': uploads_root,
|
|
},
|
|
'counts': counts,
|
|
'uploads': {
|
|
'file_count': len(upload_files),
|
|
'total_bytes': uploads_bytes,
|
|
'total_human': human_size(uploads_bytes),
|
|
'subdirs': [d for d in UPLOAD_SUBDIRS
|
|
if os.path.isdir(os.path.join(uploads_root, d))],
|
|
},
|
|
'integrity': {
|
|
'missing_files': missing,
|
|
'missing_count': len(missing),
|
|
'orphaned_files': orphaned[:500],
|
|
'orphaned_count': len(orphaned),
|
|
},
|
|
}
|
|
manifest_json = json.dumps(manifest, indent=2,
|
|
ensure_ascii=False).encode('utf-8')
|
|
archive.writestr('manifest.json', manifest_json,
|
|
compress_type=zipfile.ZIP_DEFLATED)
|
|
step += 1
|
|
|
|
checksum_body = '\n'.join(f'{digest} {name}'
|
|
for digest, name in checksums) + '\n'
|
|
archive.writestr('checksums.sha256', checksum_body.encode('utf-8'),
|
|
compress_type=zipfile.ZIP_DEFLATED)
|
|
step += 1
|
|
|
|
os.replace(temp_path, archive_path)
|
|
|
|
except Exception:
|
|
if os.path.exists(temp_path):
|
|
try:
|
|
os.remove(temp_path)
|
|
except OSError:
|
|
pass
|
|
raise
|
|
|
|
archive_size = os.path.getsize(archive_path)
|
|
progress('done', total_steps, total_steps, archive_name)
|
|
|
|
return {
|
|
'filename': archive_name,
|
|
'path': archive_path,
|
|
'size_bytes': archive_size,
|
|
'size_human': human_size(archive_size),
|
|
'manifest': manifest,
|
|
}
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Inspect / verify
|
|
# --------------------------------------------------------------------------
|
|
|
|
def read_manifest(archive_path):
|
|
with zipfile.ZipFile(archive_path) as archive:
|
|
with archive.open('manifest.json') as handle:
|
|
return json.load(handle)
|
|
|
|
|
|
def verify_backup(archive_path):
|
|
"""Recompute every checksum in the archive. Returns a report dict."""
|
|
report = {'ok': True, 'checked': 0, 'failed': [], 'missing': [],
|
|
'errors': []}
|
|
try:
|
|
manifest = read_manifest(archive_path)
|
|
except Exception as exc:
|
|
return {'ok': False, 'checked': 0, 'failed': [], 'missing': [],
|
|
'errors': [f'Cannot read manifest: {exc}']}
|
|
|
|
report['manifest'] = manifest
|
|
|
|
with zipfile.ZipFile(archive_path) as archive:
|
|
names = set(archive.namelist())
|
|
if 'checksums.sha256' not in names:
|
|
report['ok'] = False
|
|
report['errors'].append('No checksums.sha256 in archive')
|
|
return report
|
|
|
|
expected = {}
|
|
with archive.open('checksums.sha256') as handle:
|
|
for line in handle.read().decode('utf-8').splitlines():
|
|
if not line.strip():
|
|
continue
|
|
digest, _, name = line.partition(' ')
|
|
expected[name] = digest
|
|
|
|
for name, digest in expected.items():
|
|
if name not in names:
|
|
report['missing'].append(name)
|
|
report['ok'] = False
|
|
continue
|
|
hasher = hashlib.sha256()
|
|
with archive.open(name) as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b''):
|
|
hasher.update(chunk)
|
|
report['checked'] += 1
|
|
if hasher.hexdigest() != digest:
|
|
report['failed'].append(name)
|
|
report['ok'] = False
|
|
|
|
return report
|
|
|
|
|
|
def list_backups(app=None):
|
|
"""Return archives in the backup folder, newest first."""
|
|
app = app or current_app
|
|
folder = backup_folder(app)
|
|
entries = []
|
|
for name in os.listdir(folder):
|
|
if not name.lower().endswith('.zip'):
|
|
continue
|
|
path = os.path.join(folder, name)
|
|
if not os.path.isfile(path):
|
|
continue
|
|
stat = os.stat(path)
|
|
entry = {
|
|
'filename': name,
|
|
'path': path,
|
|
'size_bytes': stat.st_size,
|
|
'size_human': human_size(stat.st_size),
|
|
'modified': datetime.fromtimestamp(stat.st_mtime),
|
|
'manifest': None,
|
|
'valid': False,
|
|
}
|
|
try:
|
|
entry['manifest'] = read_manifest(path)
|
|
entry['valid'] = entry['manifest'].get('format') == BACKUP_FORMAT
|
|
except Exception:
|
|
pass
|
|
entries.append(entry)
|
|
entries.sort(key=lambda e: e['modified'], reverse=True)
|
|
return entries
|
|
|
|
|
|
def delete_backup(filename, app=None):
|
|
app = app or current_app
|
|
folder = backup_folder(app)
|
|
safe = os.path.basename(filename)
|
|
path = os.path.join(folder, safe)
|
|
if os.path.commonpath([os.path.abspath(path), folder]) != folder:
|
|
raise ValueError('Refusing to delete outside the backup folder')
|
|
if not os.path.isfile(path):
|
|
raise FileNotFoundError(safe)
|
|
os.remove(path)
|
|
return safe
|
|
|
|
|
|
def prune_backups(keep, app=None):
|
|
"""Delete all but the newest `keep` archives. Returns removed names."""
|
|
if not keep or keep < 1:
|
|
return []
|
|
removed = []
|
|
for entry in list_backups(app)[keep:]:
|
|
try:
|
|
os.remove(entry['path'])
|
|
removed.append(entry['filename'])
|
|
except OSError:
|
|
pass
|
|
return removed
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Restore
|
|
# --------------------------------------------------------------------------
|
|
|
|
def _safe_extract_target(root, relative):
|
|
target = os.path.abspath(os.path.join(root, relative))
|
|
if os.path.commonpath([target, os.path.abspath(root)]) != os.path.abspath(root):
|
|
raise ValueError(f'Unsafe path in archive: {relative}')
|
|
return target
|
|
|
|
|
|
def _table_is_empty(model):
|
|
return db.session.query(model.id).first() is None
|
|
|
|
|
|
def _reset_sequences():
|
|
"""Postgres keeps its own sequences; realign them after preserving IDs."""
|
|
if db.engine.dialect.name != 'postgresql':
|
|
return
|
|
from sqlalchemy import text
|
|
for table_name, model in TABLES:
|
|
db.session.execute(text(
|
|
"SELECT setval(pg_get_serial_sequence(:t, 'id'), "
|
|
"COALESCE((SELECT MAX(id) FROM " + table_name + "), 1), true)"
|
|
), {'t': table_name})
|
|
db.session.commit()
|
|
|
|
|
|
def restore_backup(archive_path, mode='fresh', force=False, overwrite_files=False,
|
|
skip_files=False, app=None, progress=None):
|
|
"""
|
|
Restore an archive into the current instance.
|
|
|
|
mode='fresh' target tables must be empty (or force=True to wipe them);
|
|
primary keys are preserved exactly.
|
|
mode='merge' keeps existing data; users are matched by username and sets
|
|
by set number. Sets that already exist are skipped whole.
|
|
|
|
Returns a summary dict.
|
|
"""
|
|
app = app or current_app
|
|
progress = progress or (lambda *a, **k: None)
|
|
|
|
if mode not in ('fresh', 'merge'):
|
|
raise ValueError("mode must be 'fresh' or 'merge'")
|
|
|
|
manifest = read_manifest(archive_path)
|
|
if manifest.get('format') != BACKUP_FORMAT:
|
|
raise ValueError('Not a LIM backup archive')
|
|
if manifest.get('format_version', 0) > BACKUP_FORMAT_VERSION:
|
|
raise ValueError(
|
|
f"Archive format v{manifest['format_version']} is newer than this "
|
|
f"app supports (v{BACKUP_FORMAT_VERSION}). Upgrade LIM first."
|
|
)
|
|
|
|
summary = {
|
|
'mode': mode,
|
|
'manifest': manifest,
|
|
'inserted': {name: 0 for name, _ in TABLES},
|
|
'skipped': {name: 0 for name, _ in TABLES},
|
|
'files_written': 0,
|
|
'files_skipped': 0,
|
|
'files_bytes': 0,
|
|
'warnings': [],
|
|
}
|
|
|
|
with zipfile.ZipFile(archive_path) as archive:
|
|
with archive.open('database/data.json') as handle:
|
|
data = json.load(handle)
|
|
tables = data.get('tables', {})
|
|
|
|
# ---- database -----------------------------------------------------
|
|
progress('database', 0, 0, 'Restoring rows')
|
|
|
|
if mode == 'fresh':
|
|
non_empty = [name for name, model in TABLES if not _table_is_empty(model)]
|
|
if non_empty and not force:
|
|
raise RuntimeError(
|
|
'Target database is not empty (' + ', '.join(non_empty) +
|
|
'). Re-run with force to wipe these tables first.'
|
|
)
|
|
if non_empty:
|
|
for table_name, model in reversed(TABLES):
|
|
deleted = model.query.delete()
|
|
summary['warnings'].append(
|
|
f'Wiped {deleted} existing row(s) from {table_name}')
|
|
db.session.commit()
|
|
|
|
for table_name, model in TABLES:
|
|
columns = {c.key: c for c in sa_inspect(model).columns}
|
|
path_cols = set(PATH_COLUMNS.get(table_name, []))
|
|
for row in tables.get(table_name, []):
|
|
kwargs = {}
|
|
for key, value in row.items():
|
|
column = columns.get(key)
|
|
if column is None:
|
|
continue
|
|
if key in path_cols:
|
|
value = norm_rel_path(value)
|
|
kwargs[key] = _deserialise_value(column, value)
|
|
db.session.add(model(**kwargs))
|
|
summary['inserted'][table_name] += 1
|
|
db.session.flush()
|
|
db.session.commit()
|
|
_reset_sequences()
|
|
|
|
else: # merge
|
|
user_map = {}
|
|
set_map = {}
|
|
|
|
# users
|
|
columns = {c.key: c for c in sa_inspect(User).columns}
|
|
for row in tables.get('users', []):
|
|
existing = User.query.filter_by(username=row['username']).first() \
|
|
or User.query.filter_by(email=row['email']).first()
|
|
if existing:
|
|
user_map[row['id']] = existing.id
|
|
summary['skipped']['users'] += 1
|
|
continue
|
|
kwargs = {k: _deserialise_value(columns[k], v)
|
|
for k, v in row.items() if k in columns and k != 'id'}
|
|
user = User(**kwargs)
|
|
db.session.add(user)
|
|
db.session.flush()
|
|
user_map[row['id']] = user.id
|
|
summary['inserted']['users'] += 1
|
|
|
|
fallback_user = next(iter(user_map.values()), None)
|
|
if fallback_user is None:
|
|
first_user = User.query.order_by(User.id).first()
|
|
fallback_user = first_user.id if first_user else None
|
|
|
|
# sets
|
|
columns = {c.key: c for c in sa_inspect(Set).columns}
|
|
skipped_set_ids = set()
|
|
for row in tables.get('sets', []):
|
|
existing = Set.query.filter_by(set_number=row['set_number']).first()
|
|
if existing:
|
|
set_map[row['id']] = existing.id
|
|
skipped_set_ids.add(row['id'])
|
|
summary['skipped']['sets'] += 1
|
|
continue
|
|
kwargs = {}
|
|
for key, value in row.items():
|
|
if key not in columns or key == 'id':
|
|
continue
|
|
if key in PATH_COLUMNS.get('sets', []):
|
|
value = norm_rel_path(value)
|
|
kwargs[key] = _deserialise_value(columns[key], value)
|
|
kwargs['user_id'] = user_map.get(row.get('user_id'), fallback_user)
|
|
if kwargs.get('brickset_id') is not None:
|
|
clash = Set.query.filter_by(
|
|
brickset_id=kwargs['brickset_id']).first()
|
|
if clash:
|
|
kwargs['brickset_id'] = None
|
|
lego_set = Set(**kwargs)
|
|
db.session.add(lego_set)
|
|
db.session.flush()
|
|
set_map[row['id']] = lego_set.id
|
|
summary['inserted']['sets'] += 1
|
|
|
|
# children of newly added sets only
|
|
for table_name, model, set_key in (
|
|
('instructions', Instruction, 'set_id'),
|
|
('extra_files', ExtraFile, 'set_id'),
|
|
):
|
|
columns = {c.key: c for c in sa_inspect(model).columns}
|
|
path_cols = set(PATH_COLUMNS.get(table_name, []))
|
|
for row in tables.get(table_name, []):
|
|
old_set = row.get(set_key)
|
|
if old_set in skipped_set_ids or old_set not in set_map:
|
|
summary['skipped'][table_name] += 1
|
|
continue
|
|
kwargs = {}
|
|
for key, value in row.items():
|
|
if key not in columns or key == 'id':
|
|
continue
|
|
if key in path_cols:
|
|
value = norm_rel_path(value)
|
|
kwargs[key] = _deserialise_value(columns[key], value)
|
|
kwargs[set_key] = set_map[old_set]
|
|
for owner_key in ('user_id', 'uploaded_by'):
|
|
if owner_key in columns and row.get(owner_key) is not None:
|
|
kwargs[owner_key] = user_map.get(row[owner_key],
|
|
fallback_user)
|
|
db.session.add(model(**kwargs))
|
|
summary['inserted'][table_name] += 1
|
|
|
|
db.session.commit()
|
|
|
|
# ---- uploads ------------------------------------------------------
|
|
if skip_files:
|
|
summary['warnings'].append('File restore skipped by request')
|
|
else:
|
|
root = upload_folder(app)
|
|
os.makedirs(root, exist_ok=True)
|
|
members = [n for n in archive.namelist()
|
|
if n.startswith('uploads/') and not n.endswith('/')]
|
|
total = len(members)
|
|
for index, name in enumerate(members, 1):
|
|
relative = name[len('uploads/'):]
|
|
if not relative:
|
|
continue
|
|
target = _safe_extract_target(root, relative)
|
|
if os.path.exists(target) and not overwrite_files:
|
|
summary['files_skipped'] += 1
|
|
continue
|
|
os.makedirs(os.path.dirname(target), exist_ok=True)
|
|
with archive.open(name) as source, open(target, 'wb') as dest:
|
|
shutil.copyfileobj(source, dest, 1024 * 1024)
|
|
summary['files_written'] += 1
|
|
summary['files_bytes'] += os.path.getsize(target)
|
|
if index % 25 == 0 or index == total:
|
|
progress('uploads', index, total, relative)
|
|
|
|
summary['files_bytes_human'] = human_size(summary['files_bytes'])
|
|
return summary
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Background job tracking (used by the admin UI)
|
|
# --------------------------------------------------------------------------
|
|
|
|
_JOBS = {}
|
|
_JOBS_LOCK = threading.Lock()
|
|
|
|
|
|
def _set_job(job_id, **fields):
|
|
with _JOBS_LOCK:
|
|
job = _JOBS.setdefault(job_id, {'id': job_id})
|
|
job.update(fields)
|
|
return dict(job)
|
|
|
|
|
|
def get_job(job_id):
|
|
with _JOBS_LOCK:
|
|
job = _JOBS.get(job_id)
|
|
return dict(job) if job else None
|
|
|
|
|
|
def start_backup_job(app, created_by=None, label=None, keep=None):
|
|
"""Kick off a backup in a worker thread and return its job id."""
|
|
job_id = uuid.uuid4().hex
|
|
_set_job(job_id, state='queued', stage='queued', done=0, total=0,
|
|
note='Waiting to start', started_at=_utcnow().isoformat(),
|
|
result=None, error=None)
|
|
|
|
def worker():
|
|
with app.app_context():
|
|
try:
|
|
_set_job(job_id, state='running')
|
|
|
|
def on_progress(stage, done, total, note):
|
|
_set_job(job_id, stage=stage, done=done, total=total,
|
|
note=note)
|
|
|
|
result = create_backup(app=app, created_by=created_by,
|
|
label=label, progress=on_progress)
|
|
pruned = prune_backups(keep, app=app) if keep else []
|
|
_set_job(job_id, state='complete', stage='done',
|
|
note=result['filename'],
|
|
result={
|
|
'filename': result['filename'],
|
|
'size_human': result['size_human'],
|
|
'size_bytes': result['size_bytes'],
|
|
'counts': result['manifest']['counts'],
|
|
'uploads': result['manifest']['uploads'],
|
|
'integrity': {
|
|
'missing_count':
|
|
result['manifest']['integrity']['missing_count'],
|
|
'orphaned_count':
|
|
result['manifest']['integrity']['orphaned_count'],
|
|
},
|
|
'pruned': pruned,
|
|
},
|
|
finished_at=_utcnow().isoformat())
|
|
except Exception as exc: # noqa: BLE001 - surfaced to the UI
|
|
app.logger.exception('Backup job failed')
|
|
_set_job(job_id, state='failed', error=str(exc),
|
|
finished_at=_utcnow().isoformat())
|
|
|
|
thread = threading.Thread(target=worker, name=f'lim-backup-{job_id[:8]}',
|
|
daemon=True)
|
|
thread.start()
|
|
return job_id
|