Added WIP Backup System
This commit is contained in:
st01765
@@ -1,8 +1,6 @@
|
||||
import datetime
|
||||
|
||||
from django.forms import model_to_dict
|
||||
|
||||
from coord.models import *
|
||||
from ..models import *
|
||||
|
||||
|
||||
def _to_date(date):
|
||||
@@ -10,6 +8,284 @@ def _to_date(date):
|
||||
return None
|
||||
return date.strftime("%Y-%m-%d")
|
||||
|
||||
def serialize_model(instance, ignore_fields=None, extra_mappings=None):
|
||||
|
||||
if ignore_fields is None:
|
||||
ignore_fields = []
|
||||
if extra_mappings is None:
|
||||
extra_mappings = {}
|
||||
|
||||
data = model_to_dict(instance)
|
||||
|
||||
for field in ignore_fields:
|
||||
data.pop(field, None)
|
||||
|
||||
for field, value in extra_mappings.items():
|
||||
data[field] = value
|
||||
|
||||
return data
|
||||
|
||||
def serialize_suburb(suburb):
|
||||
return serialize_model(
|
||||
suburb
|
||||
)
|
||||
|
||||
def serialize_school(school):
|
||||
return serialize_model(
|
||||
school,
|
||||
extra_mappings={
|
||||
"suburb": school.suburb.name if school.suburb else None,
|
||||
}
|
||||
)
|
||||
|
||||
def serialize_bus(bus):
|
||||
bus_dict = serialize_model(
|
||||
bus,
|
||||
ignore_fields=["company"],
|
||||
)
|
||||
|
||||
bus_dict["drivers"] = [
|
||||
serialize_model(driver, ignore_fields=["bus"])
|
||||
for driver in Driver.objects.filter(bus=bus)
|
||||
]
|
||||
|
||||
bus_dict["shuttles"] = [
|
||||
serialize_model(shuttle, ignore_fields=["bus"])
|
||||
for shuttle in Shuttle.objects.filter(bus=bus)
|
||||
]
|
||||
|
||||
bus_dict["bus_stops"] = [
|
||||
serialize_model(
|
||||
bus_stop,
|
||||
ignore_fields=["bus"],
|
||||
extra_mappings={
|
||||
"am_time": bus_stop.am_time.strftime("%H:%M:%S") if bus_stop.am_time else None,
|
||||
"pm_time": bus_stop.pm_time.strftime("%H:%M:%S") if bus_stop.pm_time else None,
|
||||
},
|
||||
)
|
||||
for bus_stop in BusStop.objects.filter(bus=bus)
|
||||
]
|
||||
|
||||
return bus_dict
|
||||
|
||||
|
||||
def serialize_company(company):
|
||||
company_dict = serialize_model(
|
||||
company,
|
||||
extra_mappings={
|
||||
"suburb": company.suburb.name if company.suburb else None,
|
||||
},
|
||||
)
|
||||
company_dict["buses"] = [serialize_bus(bus) for bus in Bus.objects.filter(company=company)]
|
||||
return company_dict
|
||||
|
||||
|
||||
def serialize_traveller(traveller):
|
||||
traveller_dict = serialize_model(
|
||||
traveller,
|
||||
ignore_fields=['travel_start_date', 'travel_end_date', 'is_active', 'is_archived', 'bus_stops', 'fee_per_term', 'address'],
|
||||
extra_mappings = {
|
||||
"school": traveller.school.name if traveller.school else None,
|
||||
"dob": _to_date(traveller.dob),
|
||||
"assessment_date": _to_date(traveller.assessment_date),
|
||||
"created_on": traveller.created_on.strftime("%Y-%m-%d %H:%M:%S %Z"),
|
||||
"last_edit": traveller.last_edit.strftime("%Y-%m-%d %H:%M:%S %Z"),
|
||||
}
|
||||
)
|
||||
|
||||
traveller_dict["families"] = [
|
||||
serialize_model(
|
||||
family,
|
||||
ignore_fields=["traveller"],
|
||||
extra_mappings={
|
||||
"residential_suburb": family.residential_suburb.name if family.residential_suburb else None,
|
||||
"postal_suburb": family.postal_suburb.name if family.postal_suburb else None,
|
||||
},
|
||||
)
|
||||
for family in Family.objects.filter(traveller=traveller)
|
||||
]
|
||||
|
||||
traveller_dict["bus_stops"] = [
|
||||
serialize_model(
|
||||
route,
|
||||
ignore_fields=["traveller"],
|
||||
extra_mappings={
|
||||
"travel_start_date": _to_date(traveller.travel_start_date),
|
||||
}
|
||||
)
|
||||
for route in TravellerRoute.objects.filter(traveller=traveller)
|
||||
]
|
||||
|
||||
return traveller_dict
|
||||
|
||||
|
||||
def serialize_all():
|
||||
return {
|
||||
"suburbs": [serialize_suburb(suburb) for suburb in Suburb.objects.all()],
|
||||
"schools": [serialize_school(school) for school in School.objects.all()],
|
||||
"companies": [serialize_company(company) for company in Company.objects.all()],
|
||||
"travellers": [serialize_traveller(traveller) for traveller in Traveller.objects.all()],
|
||||
}
|
||||
|
||||
|
||||
def compare_backup(backup_content):
|
||||
import json
|
||||
|
||||
# Parse uploaded file
|
||||
backup_data = json.loads(backup_content)
|
||||
|
||||
# Current DB state
|
||||
current_data = serialize_all()
|
||||
|
||||
def compare_dicts_shallow(current, backup, ignore_keys=None):
|
||||
"""Compare two dicts at a single level."""
|
||||
ignore_keys = ignore_keys or []
|
||||
modifications = []
|
||||
|
||||
keys = set(current.keys()) | set(backup.keys())
|
||||
|
||||
for key in keys:
|
||||
if key in ignore_keys:
|
||||
continue
|
||||
cur_val = current.get(key)
|
||||
bak_val = backup.get(key)
|
||||
|
||||
if cur_val != bak_val:
|
||||
if cur_val is None:
|
||||
mod_type = "New"
|
||||
elif bak_val is None:
|
||||
mod_type = "Removed"
|
||||
else:
|
||||
mod_type = "Updated"
|
||||
|
||||
modifications.append({
|
||||
"type": mod_type,
|
||||
"key": key,
|
||||
"old_value": cur_val,
|
||||
"new_value": bak_val
|
||||
})
|
||||
return modifications
|
||||
|
||||
def compare_backup_dicts(current, backup, key, nested_keys=None):
|
||||
current_list = current.get(key, [])
|
||||
backup_list = backup.get(key, [])
|
||||
|
||||
if not (current_list and backup_list):
|
||||
return []
|
||||
|
||||
nested_keys = nested_keys or []
|
||||
|
||||
current_map = {item[key]: item for item in current_list}
|
||||
backup_map = {item[key]: item for item in backup_list}
|
||||
|
||||
results = []
|
||||
|
||||
# Check for added items
|
||||
for name in backup_map:
|
||||
if name not in current_map:
|
||||
results.append({
|
||||
"name": name,
|
||||
"type": "New",
|
||||
"modifications": [],
|
||||
"inline_modifications": []
|
||||
})
|
||||
|
||||
# Check for removed items
|
||||
for name in current_map:
|
||||
if name not in backup_map:
|
||||
results.append({
|
||||
"name": name,
|
||||
"type": "Removed",
|
||||
"modifications": [],
|
||||
"inline_modifications": []
|
||||
})
|
||||
|
||||
# Check for updated items
|
||||
for name in current_map:
|
||||
if name in backup_map:
|
||||
cur_item = current_map[name]
|
||||
bak_item = backup_map[name]
|
||||
|
||||
modifications = compare_dicts_shallow(cur_item, bak_item, ignore_keys=nested_keys)
|
||||
inline_modifications = []
|
||||
|
||||
for nested_key in nested_keys:
|
||||
nested_diff = compare_backup_dicts(current_list, current_list, nested_key, nested_keys)
|
||||
if nested_diff:
|
||||
inline_modifications.append({
|
||||
"name": nested_key.capitalize(),
|
||||
"modifications": nested_diff
|
||||
})
|
||||
|
||||
if modifications or inline_modifications:
|
||||
results.append({
|
||||
"name": name,
|
||||
"type": "Updated",
|
||||
"modifications": modifications,
|
||||
"inline_modifications": inline_modifications
|
||||
})
|
||||
|
||||
return results
|
||||
|
||||
diff_suburbs = compare_backup_dicts(
|
||||
current_data,
|
||||
backup_data,
|
||||
"suburbs"
|
||||
)
|
||||
diff_schools = compare_backup_dicts(
|
||||
current_data,
|
||||
backup_data,
|
||||
"schools"
|
||||
)
|
||||
diff_companies = compare_backup_dicts(
|
||||
current_data,
|
||||
backup_data,
|
||||
"companies",
|
||||
["buses", "drivers", "shuttles", "bus_stops"]
|
||||
)
|
||||
diff_travellers = compare_backup_dicts(
|
||||
current_data,
|
||||
backup_data,
|
||||
"travellers",
|
||||
["families", "bus_stops"]
|
||||
)
|
||||
|
||||
# Helper function to compare lists by key
|
||||
def compare_lists(current, backup, key="name"):
|
||||
current_map = {item[key]: item for item in current}
|
||||
backup_map = {item[key]: item for item in backup}
|
||||
|
||||
added = [backup_map[k] for k in backup_map if k not in current_map]
|
||||
removed = [current_map[k] for k in current_map if k not in backup_map]
|
||||
modified = [
|
||||
(k, {"db": current_map[k], "backup": backup_map[k]})
|
||||
for k in current_map
|
||||
if k in backup_map and current_map[k] != backup_map[k]
|
||||
]
|
||||
|
||||
return {
|
||||
"add": len(added),
|
||||
"remove": len(removed),
|
||||
"modify": len(modified),
|
||||
"details": {
|
||||
"added": added,
|
||||
"removed": removed,
|
||||
"modified": modified,
|
||||
},
|
||||
}
|
||||
|
||||
results = {
|
||||
"suburbs": compare_lists(current_data["suburbs"], backup_data["suburbs"], key="name"),
|
||||
"schools": compare_lists(current_data["schools"], backup_data["schools"], key="name"),
|
||||
"companies": compare_lists(current_data["companies"], backup_data["companies"], key="name"),
|
||||
"travellers": compare_lists(current_data["travellers"], backup_data["travellers"], key="name"),
|
||||
}
|
||||
|
||||
# Summary counts only
|
||||
summary = {k: {m: v[m] for m in ["add", "remove", "modify"]} for k, v in results.items()}
|
||||
|
||||
return summary, results
|
||||
|
||||
|
||||
def get_export_dict():
|
||||
suburbs = []
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
{% extends 'admin/base.html' %}
|
||||
|
||||
{% block extrahead %}
|
||||
{{ block.super }}
|
||||
<style>
|
||||
/* Collapsible diff styles */
|
||||
.diff-list, .diff-list ul { list-style: none; margin-left: 1em; padding-left: 0; }
|
||||
.nested { display: none; }
|
||||
.toggle { cursor: pointer; user-select: none; font-weight: bold; }
|
||||
.new { color: green; }
|
||||
.removed { color: red; }
|
||||
.updated { color: orange; }
|
||||
</style>
|
||||
{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<h1>Backup & Restore</h1>
|
||||
|
||||
<!-- Export backup -->
|
||||
<section>
|
||||
<h2>Download Backup</h2>
|
||||
<form method="get" action="{% url 'settings:export_backup' %}">
|
||||
<button type="submit">Download New Backup</button>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
<hr>
|
||||
|
||||
<!-- Upload backup -->
|
||||
<section>
|
||||
<h2>Upload Backup</h2>
|
||||
<form id="upload-form" method="post" enctype="multipart/form-data" action="">
|
||||
{% csrf_token %}
|
||||
<input type="file" name="backup_file" required>
|
||||
<button type="submit">Upload & Compare Backup</button>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
<hr>
|
||||
|
||||
<!-- Compare summary & diffs -->
|
||||
{% if compare_summary %}
|
||||
<section>
|
||||
<h2>Comparison Summary</h2>
|
||||
<ul>
|
||||
{% for model, counts in compare_summary.items %}
|
||||
<li>{{ model }}: Added {{ counts.add }}, Removed {{ counts.remove }}, Modified {{ counts.modify }}</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
|
||||
<h2>Details</h2>
|
||||
{% include "diff_list.html" with items=diffs.suburbs title="Suburbs" %}
|
||||
{% include "diff_list.html" with items=diffs.schools title="Schools" %}
|
||||
{% include "diff_list.html" with items=diffs.travellers title="Travellers" %}
|
||||
{% include "diff_list.html" with items=diffs.companies title="Companies" %}
|
||||
|
||||
<!-- Execute Recovery -->
|
||||
<form method="post" action="">
|
||||
{% csrf_token %}
|
||||
<!-- Pass the uploaded file content for restore -->
|
||||
<input type="hidden" name="backup_content" value="{{ uploaded_file_content|escape }}">
|
||||
<button type="submit" name="execute_restore"
|
||||
onclick="return confirm('Are you sure you want to restore this backup? This cannot be undone.')">
|
||||
Execute Recovery
|
||||
</button>
|
||||
</form>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
{% endblock %}
|
||||
|
||||
{% block extrajs %}
|
||||
<script>
|
||||
document.addEventListener("DOMContentLoaded", () => {
|
||||
// Collapsible diffs
|
||||
document.querySelectorAll('.toggle').forEach(el => {
|
||||
el.addEventListener('click', () => {
|
||||
const nested = el.nextElementSibling;
|
||||
if (nested) nested.style.display = (nested.style.display === "none" || nested.style.display === "") ? "block" : "none";
|
||||
});
|
||||
});
|
||||
});
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,20 @@
|
||||
{% extends "admin/base_site.html" %}
|
||||
|
||||
{% block content %}
|
||||
<h1>Backup Comparison</h1>
|
||||
|
||||
<h2>Summary</h2>
|
||||
<ul>
|
||||
{% for model, counts in summary.items %}
|
||||
<li>{{ model }}: Added {{ counts.add }}, Removed {{ counts.remove }}, Modified {{ counts.modify }}</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
|
||||
<!-- Optional: show details -->
|
||||
<pre>{{ details|safe }}</pre>
|
||||
|
||||
<form method="post" action="{% url 'restore_backup' %}">
|
||||
{% csrf_token %}
|
||||
<button type="submit">Execute Recovery</button>
|
||||
</form>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,26 @@
|
||||
<h3>{{ title }}</h3>
|
||||
<ul class="diff-list">
|
||||
{% for item in items %}
|
||||
<li>
|
||||
<span class="toggle">{{ item.name }} ({{ item.type }})</span>
|
||||
|
||||
{% if item.modifications %}
|
||||
<ul class="nested">
|
||||
{% for mod in item.modifications %}
|
||||
<li class="{{ mod.type|lower }}">
|
||||
{{ mod.type }}: {{ mod.key }} → {{ mod.old_value }} → {{ mod.new_value }}
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
{% endif %}
|
||||
|
||||
{% if item.inline_modifications %}
|
||||
{% for inline in item.inline_modifications %}
|
||||
<ul class="nested">
|
||||
{% include "diff_list.html" with items=inline.modifications title=inline.name %}
|
||||
</ul>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
@@ -3,7 +3,7 @@
|
||||
{% block content %}
|
||||
<div><a href={% url 'settings:nightly_task' %}>Run nightly task</a></div>
|
||||
<div><a href={% url 'settings:sms_test' %}>Send SMS test message</a></div>
|
||||
<div><a href={% url 'settings:export' %}>Export</a></div>
|
||||
<div><a href={% url 'settings:export_backup' %}>Export</a></div>
|
||||
<div>
|
||||
<div class="flex-container">
|
||||
<label for="id_traveller_term_cost">
|
||||
|
||||
@@ -7,6 +7,6 @@ urlpatterns = [
|
||||
path('rollover', views_settings.rollover, name='rollover'),
|
||||
path('nightly_task', views_settings.nightly_task, name='nightly_task'),
|
||||
path('sms_test', views_settings.sms_test, name='sms_test'),
|
||||
path('export', views_settings.export, name='export'),
|
||||
path('export', views_settings.backup, name='export_backup'),
|
||||
path('', views_settings.settings, name='index'),
|
||||
]
|
||||
|
||||
@@ -2,13 +2,13 @@ import datetime
|
||||
import json
|
||||
|
||||
from django.contrib.admin.views.decorators import staff_member_required
|
||||
from django.http import HttpResponse
|
||||
from django.http import HttpResponse, JsonResponse
|
||||
from django.shortcuts import render, redirect
|
||||
|
||||
from .scheduled_tasks import nightly_check_active_status
|
||||
from .utils.rollover import RolloverForm, execute_rollover
|
||||
|
||||
from .backup.backup_helpers import get_export_dict
|
||||
from .backup.backup_helpers import get_export_dict, serialize_all, compare_backup
|
||||
from .utils.send_sms import SMSTestForm, send_sms_test
|
||||
|
||||
|
||||
@@ -49,8 +49,91 @@ def nightly_task(request):
|
||||
|
||||
|
||||
@staff_member_required
|
||||
def export(request):
|
||||
def backup(request):
|
||||
if request.method == "POST" and request.FILES.get("backup_file"):
|
||||
uploaded_file = request.FILES["backup_file"]
|
||||
request.session["backup_file"] = {
|
||||
"content": uploaded_file.read().decode("utf-8"),
|
||||
"timestamp": datetime.datetime.now().timestamp(),
|
||||
}
|
||||
return JsonResponse({"status": "success", "message": "Backup uploaded."})
|
||||
return render(request, "admin/backup.html")
|
||||
|
||||
|
||||
@staff_member_required
|
||||
def export_backup(request):
|
||||
date = datetime.date.today().strftime("%Y-%m-%d")
|
||||
response = HttpResponse(json.dumps(get_export_dict()))
|
||||
response['Content-Disposition'] = f'attachment; filename=busportal_export-{date}.json'
|
||||
data = serialize_all()
|
||||
|
||||
response = HttpResponse(
|
||||
json.dumps(data, indent=2), # pretty print for readability
|
||||
content_type="application/json",
|
||||
)
|
||||
response["Content-Disposition"] = f'attachment; filename="busportal_export-{date}.json"'
|
||||
return response
|
||||
|
||||
|
||||
def backup_view(request):
|
||||
context = {}
|
||||
|
||||
if request.method == "POST":
|
||||
if "backup_file" in request.FILES:
|
||||
uploaded_file = request.FILES["backup_file"]
|
||||
# Read JSON content from uploaded file
|
||||
backup_content = uploaded_file.read().decode("utf-8")
|
||||
|
||||
# Compare backup with current DB state
|
||||
summary, diffs = compare_backup(backup_content)
|
||||
context["compare_summary"] = summary
|
||||
context["diffs"] = diffs
|
||||
context["uploaded_file_name"] = uploaded_file.name
|
||||
|
||||
elif "execute_restore" in request.POST:
|
||||
# You can pass the JSON content in a hidden input if needed
|
||||
backup_content = request.POST.get("backup_content")
|
||||
# restore_backup(backup_content)
|
||||
context["success_message"] = "Backup restored successfully."
|
||||
|
||||
return render(request, "backup.html", context)
|
||||
|
||||
@staff_member_required
|
||||
|
||||
|
||||
@staff_member_required
|
||||
def upload_backup_view(request):
|
||||
if request.method == 'POST' and request.FILES.get("backup_file"):
|
||||
uploaded_file = request.FILES["backup_file"]
|
||||
request.session["backup_file_content"] = uploaded_file.read().decode("utf-8")
|
||||
return JsonResponse({"status": "success", "message": "File uploaded successfully"})
|
||||
return render(request, 'admin/upload_backup_view.html')
|
||||
|
||||
|
||||
# --- Delete uploaded backup ---
|
||||
def cancel_backup_upload(request):
|
||||
request.session.pop("backup_file", None)
|
||||
return JsonResponse({"status": "success", "message": "Uploaded backup cleared."})
|
||||
|
||||
|
||||
# --- Compare backup ---
|
||||
def compare_backup_view(request):
|
||||
backup_entry = request.session.get("backup_file")
|
||||
if not backup_entry:
|
||||
return JsonResponse({"error": "No backup uploaded."}, status=400)
|
||||
|
||||
backup_data = backup_entry["content"]
|
||||
|
||||
summary, details = compare_backup(backup_data)
|
||||
return render(request, "backup_compare.html", {"summary": summary, "details": details})
|
||||
|
||||
def apply_backup(backup_data):
|
||||
pass
|
||||
|
||||
# --- Execute restore ---
|
||||
def restore_backup_view(request):
|
||||
backup_entry = request.session.pop("backup_file", None)
|
||||
if not backup_entry:
|
||||
return JsonResponse({"error": "No backup uploaded."}, status=400)
|
||||
|
||||
backup_data = json.loads(backup_entry["content"])
|
||||
apply_backup(backup_data)
|
||||
return JsonResponse({"status": "success", "message": "Backup restored successfully."})
|
||||
Reference in New Issue
Block a user