feature: saved exports

initial - not feature complete
2025-02-09 17:14:04 +11:00
parent 2fce645066
commit ac24055365
35 changed files with 2244 additions and 156 deletions
@@ -11,8 +11,8 @@ namespace Disco.Services.Exporting
    {
        private IExport context;
        public override string TaskName { get => context?.Name ?? "Exporting"; }
-        public override bool SingleInstanceTask { get { return false; } }
-        public override bool CancelInitiallySupported { get { return false; } }
+        public override bool SingleInstanceTask { get; } = false;
+        public override bool CancelInitiallySupported { get; } = false;

        public static ExportTaskContext ScheduleNow(IExport export)
        {
@@ -16,35 +16,32 @@ namespace Disco.Services.Exporting
 {
    public static class Exporter
    {
-        public static ExportResult Export<T, R>(IExport<T, R> context, DiscoDataContext database, IScheduledTaskStatus status)
+        public static ExportResult Export<T, R>(IExport<T, R> export, DiscoDataContext database, IScheduledTaskStatus status)
            where T : IExportOptions, new()
            where R : IExportRecord
        {
            MemoryStream stream;
            string mimeType;

-            status.UpdateStatus(1, $"Exporting {context.Name}", "Gathering data");
+            status.UpdateStatus(1, $"Exporting {export.Name}", "Gathering data");

-            var records = context.BuildRecords(database, status);
+            var records = export.BuildRecords(database, status);

            status.UpdateStatus(70, "Building metadata");

-            var metadata = context.BuildMetadata(database, records, status);
+            var metadata = export.BuildMetadata(database, records, status);

            if (metadata.Count == 0)
-                throw new ArgumentException("At least one export field must be specified", nameof(context.Options));
+                throw new ArgumentException("At least one export field must be specified", nameof(export.Options));

            var filenameBuilder = new StringBuilder();
-            filenameBuilder.Append(context.SuggestedFilenamePrefix);
-            if (context.TimestampSuffix)
-            {
-                filenameBuilder.Append('-');
-                filenameBuilder.Append(status.StartedTimestamp.Value.ToString("yyyyMMdd-HHmmss"));
-            }
+            filenameBuilder.Append(export.FilenamePrefix);
+            filenameBuilder.Append('-');
+            filenameBuilder.Append(status.StartedTimestamp.Value.ToString("yyyyMMdd-HHmmss"));

            status.UpdateStatus(80, $"Rendering {records.Count} records for export");

-            switch (context.Options.Format)
+            switch (export.Options.Format)
            {
                case ExportFormat.Csv:
                    filenameBuilder.Append(".csv");
@@ -54,10 +51,10 @@ namespace Disco.Services.Exporting
                case ExportFormat.Xlsx:
                    filenameBuilder.Append(".xlsx");
                    mimeType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
-                    stream = WriteXlsx(context.ExcelWorksheetName, context.ExcelTableName, metadata, records);
+                    stream = WriteXlsx(export.ExcelWorksheetName, export.ExcelTableName, metadata, records);
                    break;
                default:
-                    throw new NotSupportedException($"Unsupported export format: {context.Options.Format}");
+                    throw new NotSupportedException($"Unsupported export format: {export.Options.Format}");
            }

            return new ExportResult()
@@ -99,7 +96,6 @@ namespace Disco.Services.Exporting
            stream.Position = 0;
            return stream;
        }
-
        private static MemoryStream WriteXlsx<T>(string worksheetName, string tableName, List<ExportMetadataField<T>> metadata, List<T> records) where T : IExportRecord
        {
            var stream = new MemoryStream();
@@ -151,7 +147,6 @@ namespace Disco.Services.Exporting

            metadata.Add(columnName, valueAccessor, csvValueEncoder);
        }
-
        public static void Add<T, V>(this ExportMetadata<T> metadata, string columnName, Func<T, V> valueAccessor, Func<object, string> csvValueEncoder = null)
            where T : IExportRecord
        {
@@ -11,8 +11,8 @@ namespace Disco.Services.Exporting
    public interface IExport
    {
        Guid Id { get; set; }
-        string Name { get; set; }
-        string Description { get; set; }
+        [JsonIgnore]
+        string Name { get; }

        ExportResult Export(DiscoDataContext database, IScheduledTaskStatus status);
    }
@@ -22,10 +22,8 @@ namespace Disco.Services.Exporting
        where T : IExportOptions, new()
        where R : IExportRecord
    {
-        bool TimestampSuffix { get; set; }
-
        [JsonIgnore]
-        string SuggestedFilenamePrefix { get; }
+        string FilenamePrefix { get; }
        [JsonIgnore]
        string ExcelWorksheetName { get; }
        [JsonIgnore]
@@ -0,0 +1,42 @@
+using Disco.Data.Repository;
+using Disco.Services.Tasks;
+using Quartz;
+using System;
+
+namespace Disco.Services.Exporting
+{
+    public class SavedExportTask : ScheduledTask
+    {
+        public override string TaskName { get; } = "Saved Export Scheduler";
+        public override bool SingleInstanceTask { get; } = true;
+        public override bool CancelInitiallySupported { get; } = false;
+        public override bool LogExceptionsOnly { get; } = true;
+
+        public override void InitalizeScheduledTask(DiscoDataContext Database)
+        {
+
+            // run in 30 seconds, then every hour on the hour
+            if (DateTime.Now.Minute != 59)
+            {
+                var immediateTrigger = TriggerBuilder.Create().StartAt(DateTimeOffset.Now.AddSeconds(30));
+                ScheduleTask(immediateTrigger);
+            }
+
+            var nextHourTicks = DateTime.UtcNow.Ticks;
+            nextHourTicks -= nextHourTicks % TimeSpan.TicksPerHour; // round down to the hour
+            var nextHour = new DateTime(nextHourTicks, DateTimeKind.Utc)
+                .AddHours(1)
+                .AddSeconds(1);
+            var hourlyTrigger = TriggerBuilder.Create()
+                .StartAt(nextHour)
+                .WithSchedule(SimpleScheduleBuilder.RepeatHourlyForever());
+            ScheduleTask(hourlyTrigger);
+        }
+
+        protected override void ExecuteTask()
+        {
+            SavedExports.EvaluateSavedExports();
+        }
+
+    }
+}
@@ -0,0 +1,319 @@
+using Disco.Data.Repository;
+using Disco.Models.Exporting;
+using Disco.Models.Repository;
+using Disco.Models.Services.Devices;
+using Disco.Models.Services.Devices.DeviceFlag;
+using Disco.Models.Services.Exporting;
+using Disco.Models.Services.Jobs;
+using Disco.Models.Services.Users.UserFlags;
+using Disco.Services.Authorization;
+using Disco.Services.Devices;
+using Disco.Services.Devices.DeviceFlags;
+using Disco.Services.Jobs;
+using Disco.Services.Logging;
+using Disco.Services.Tasks;
+using Disco.Services.Users.UserFlags;
+using Newtonsoft.Json;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+
+namespace Disco.Services.Exporting
+{
+    public static class SavedExports
+    {
+        private static Dictionary<string, (Type Type, string Name, Func<IExport, DiscoDataContext, IScheduledTaskStatus, ExportResult> ExporterDelegate)> exportTypes = new Dictionary<string, (Type, string, Func<IExport, DiscoDataContext, IScheduledTaskStatus, ExportResult>)>();
+
+        static SavedExports()
+        {
+            RegisterExportType<DeviceFlagExport, DeviceFlagExportOptions, DeviceFlagExportRecord>();
+            RegisterExportType<DeviceExport, DeviceExportOptions, DeviceExportRecord>();
+            RegisterExportType<JobExport, JobExportOptions, JobExportRecord>();
+            RegisterExportType<UserFlagExport, UserFlagExportOptions, UserFlagExportRecord>();
+        }
+
+        internal static void RegisterExportType<T, E, R>()
+            where T : IExport<E, R>, new()
+            where E : IExportOptions, new()
+            where R : IExportRecord
+        {
+            var type = typeof(T);
+
+            if (exportTypes.TryGetValue(type.Name, out var existing))
+            {
+                if (existing.Type != type)
+                    throw new InvalidOperationException($"Export type already registered ({type.FullName})");
+            }
+            else
+            {
+                var name = new T().Name;
+                exportTypes[type.Name] = (type, name, (i, d, s) => Exporter.Export((T)i, d, s));
+            }
+        }
+
+        public static SavedExport SaveExport<T, R>(IExport<T, R> export, DiscoDataContext database, User createdBy)
+            where T : IExportOptions, new()
+            where R : IExportRecord
+        {
+            var exportType = export.GetType();
+            if (!exportTypes.TryGetValue(exportType.Name, out var exportTypeRef) || exportType != exportTypeRef.Type)
+                throw new InvalidOperationException($"Export type not registered for saving ({exportType.FullName})");
+
+            var saved = new SavedExport()
+            {
+                Version = 1,
+                Id = Guid.NewGuid(),
+                CreatedOn = DateTime.Now,
+                CreatedBy = createdBy.UserId,
+                Type = exportType.Name,
+                Config = Convert.ToBase64String(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(export, Formatting.None))),
+                Enabled = false,
+            };
+
+            var exports = database.DiscoConfiguration.SavedExports;
+            exports.Add(saved);
+            database.DiscoConfiguration.SavedExports = exports;
+            database.SaveChanges();
+
+            return saved;
+        }
+
+        public static void DeleteSavedExport(DiscoDataContext database, Guid id)
+        {
+            var exports = database.DiscoConfiguration.SavedExports;
+            var existing = exports.FirstOrDefault(e => e.Id == id);
+            if (existing == null)
+                return;
+            exports.Remove(existing);
+            database.DiscoConfiguration.SavedExports = exports;
+            database.SaveChanges();
+        }
+
+        public static void UpdateSavedExport(DiscoDataContext database, SavedExport export)
+        {
+            var exports = database.DiscoConfiguration.SavedExports;
+            var existing = exports.First(e => e.Id == export.Id);
+
+            if (string.IsNullOrWhiteSpace(export.Name))
+                throw new InvalidOperationException("Export name is required");
+
+            existing.Name = export.Name;
+            existing.Description = export.Description;
+
+            if (string.IsNullOrWhiteSpace(export.FilePath) || export.Schedule == null || export.Schedule.WeekDays == 0)
+            {
+                existing.Schedule = null;
+                existing.FilePath = null;
+            }
+            else
+            {
+                // new file path - file cannot exist
+                if (existing.FilePath == null && File.Exists(export.FilePath))
+                    throw new InvalidOperationException("Export file path already exists, delete the file before configuring the saved export");
+
+                // directory must exist
+                if (!Directory.Exists(Path.GetDirectoryName(export.FilePath)))
+                    throw new InvalidOperationException("Invalid export file path, the directory does not exist");
+
+                existing.FilePath = export.FilePath;
+                existing.TimestampSuffix = export.TimestampSuffix;
+
+                existing.Schedule = new SavedExportSchedule()
+                {
+                    Version = 1,
+                    WeekDays = export.Schedule.WeekDays,
+                    StartHour = export.Schedule.EndHour.HasValue ? Math.Min(export.Schedule.StartHour, export.Schedule.EndHour.Value) : export.Schedule.StartHour,
+                    EndHour = !export.Schedule.EndHour.HasValue ? null : (export.Schedule.EndHour.Value == export.Schedule.StartHour ? (byte?)null : Math.Max(export.Schedule.StartHour, export.Schedule.EndHour.Value)),
+                };
+            }
+
+            if (existing.FilePath != null && existing.Schedule == null)
+                throw new InvalidOperationException("Export file path requires a schedule");
+
+            if (export.OnDemandPrincipals == null || export.OnDemandPrincipals.Count == 0)
+                existing.OnDemandPrincipals = null;
+            else
+                existing.OnDemandPrincipals = new List<string>(export.OnDemandPrincipals);
+
+            existing.Enabled = true;
+
+            database.DiscoConfiguration.SavedExports = exports;
+            database.SaveChanges();
+        }
+
+        public static SavedExport GetSavedExport(DiscoDataContext database, Guid id, out string exportTypeName)
+        {
+            var export = database.DiscoConfiguration.SavedExports.FirstOrDefault(e => e.Id == id);
+
+            if (export == null)
+            {
+                exportTypeName = null;
+                return null;
+            }
+
+            exportTypeName = exportTypes[export.Type].Name;
+            return export;
+        }
+
+        public static List<SavedExport> GetSavedExports(DiscoDataContext database, string type, out string exportTypeName)
+        {
+            var exports = database.DiscoConfiguration.SavedExports.Where(e => e.Type == type).ToList();
+
+            if (exports.Count == 0)
+            {
+                exportTypeName = null;
+                return null;
+            }
+
+            exportTypeName = exportTypes[type].Name;
+            return exports;
+        }
+
+        public static bool IsAuthorized(SavedExport savedExport, AuthorizationToken authorization)
+        {
+            if (authorization.Has(Claims.Config.ManageSavedExports))
+                return true;
+
+            if (savedExport.OnDemandPrincipals == null || savedExport.OnDemandPrincipals.Count == 0)
+                return false;
+
+            if (savedExport.OnDemandPrincipals.Contains(authorization.User.UserId, StringComparer.OrdinalIgnoreCase))
+                return true;
+
+            if (savedExport.OnDemandPrincipals.Any(p => authorization.GroupMembership.Contains(p, StringComparer.OrdinalIgnoreCase)))
+                return true;
+
+            return false;
+        }
+
+        public static void EvaluateSavedExports()
+        {
+            using (var database = new DiscoDataContext())
+            {
+                CleanupSavedExports(database);
+
+                var scheduledExports = GetScheduledExports(database).ToList();
+
+                foreach (var scheduledExport in scheduledExports)
+                {
+                    ExportResult exportResult = null;
+                    try
+                    {
+                        exportResult = EvaluateSavedExport(database, scheduledExport);
+                    }
+                    catch (Exception ex)
+                    {
+                        SystemLog.LogException($"Failed to generate saved '{scheduledExport.Name}' [{scheduledExport.Id}]", ex);
+                        continue;
+                    }
+
+                    var filePath = scheduledExport.FilePath;
+                    if (scheduledExport.TimestampSuffix)
+                    {
+                        var timestamp = DateTime.Now.ToString("yyyyMMdd-HH");
+                        var extension = Path.GetExtension(filePath);
+                        filePath = Path.Combine(Path.GetDirectoryName(filePath), $"{Path.GetFileNameWithoutExtension(filePath)}-{timestamp}{extension}");
+                    }
+
+                    try
+                    {
+                        using (var fileStream = new FileStream(filePath, FileMode.Create, FileAccess.Write))
+                        {
+                            exportResult.Result.CopyTo(fileStream);
+                        }
+
+                        SystemLog.LogInformation($"Saved '{scheduledExport.Name}' [{scheduledExport.Name}] wrote to '{filePath}'");
+                    }
+                    catch (Exception ex)
+                    {
+                        SystemLog.LogException($"Failed to write saved '{scheduledExport.Name}' [{scheduledExport.Id}] to '{filePath}'", ex);
+                    }
+                }
+            }
+        }
+
+        public static ExportResult EvaluateSavedExport(DiscoDataContext database, SavedExport savedExport)
+        {
+            var (exportType, _, exportDelegate) = exportTypes[savedExport.Type];
+            var export = (IExport)JsonConvert.DeserializeObject(Encoding.UTF8.GetString(Convert.FromBase64String(savedExport.Config)), exportType);
+
+            return exportDelegate(export, database, ScheduledTaskMockStatus.Create(export.Name));
+        }
+
+        private static void CleanupSavedExports(DiscoDataContext database)
+        {
+            var exports = database.DiscoConfiguration.SavedExports;
+            var changed = false;
+
+            for (int i = 0; i < exports.Count; i++)
+            {
+                var export = exports[i];
+                if (export.Enabled)
+                    continue;
+
+                if (export.CreatedOn.AddDays(1) < DateTime.Now)
+                {
+                    exports.RemoveAt(i);
+                    i--;
+                    changed = true;
+                }
+            }
+
+            if (changed)
+            {
+                database.DiscoConfiguration.SavedExports = exports;
+                database.SaveChanges();
+            }
+        }
+
+        private static IEnumerable<SavedExport> GetScheduledExports(DiscoDataContext database)
+        {
+            var exports = database.DiscoConfiguration.SavedExports;
+            var now = DateTime.Now;
+            var hour = now.Hour;
+            var day = (byte)(1 << (int)now.DayOfWeek);
+
+            foreach (var export in exports)
+            {
+                if (!export.Enabled)
+                    continue;
+
+                if (string.IsNullOrEmpty(export.FilePath))
+                    continue;
+
+                // skip unknown export types
+                if (!exportTypes.ContainsKey(export.Type))
+                    continue;
+
+                var schedule = export.Schedule;
+                if (schedule == null)
+                    continue;
+
+                // scheduled for today?
+                if ((schedule.WeekDays & day) == 0)
+                    continue;
+
+                // always run if scheduled earlier today? (potentially missed)
+                if (schedule.StartHour >= hour || export.LastRunOn.GetValueOrDefault().Date == DateTime.Today)
+                {
+                    // are we beyond the end hour?
+                    if (schedule.EndHour.HasValue && hour > schedule.EndHour.Value)
+                        continue;
+
+                    // if no end hour and not the start hour, skip
+                    if (!schedule.EndHour.HasValue && schedule.StartHour != hour)
+                        continue;
+
+                    // before the start hour, skip
+                    if (hour < schedule.StartHour)
+                        continue;
+                }
+
+                yield return export;
+            }
+        }
+
+    }
+}