From cc1f2244566492db491a245db64c70a589e06c93 Mon Sep 17 00:00:00 2001 From: Gary Sharp Date: Sun, 15 Jun 2025 17:56:13 +1000 Subject: [PATCH] feature: Upload Online Attachments --- Disco.Services/Disco.Services.csproj | 4 + .../DiscoServices/ActivationService.cs | 9 +- .../DiscoServices/DiscoServiceHelpers.cs | 18 +- .../OnlineServicesAuthenticatedHandler.cs | 18 ++ .../OnlineServicesAuthentication.cs | 4 +- .../DiscoServices/OnlineServicesConnect.cs | 2 +- .../Upload/UploadOnlineClient.cs | 126 +++++++++++ .../Upload/UploadOnlineService.cs | 27 +++ .../Upload/UploadOnlineSyncTask.cs | 209 ++++++++++++++++++ 9 files changed, 395 insertions(+), 22 deletions(-) create mode 100644 Disco.Services/Interop/DiscoServices/OnlineServicesAuthenticatedHandler.cs create mode 100644 Disco.Services/Interop/DiscoServices/Upload/UploadOnlineClient.cs create mode 100644 Disco.Services/Interop/DiscoServices/Upload/UploadOnlineService.cs create mode 100644 Disco.Services/Interop/DiscoServices/Upload/UploadOnlineSyncTask.cs diff --git a/Disco.Services/Disco.Services.csproj b/Disco.Services/Disco.Services.csproj index 185d8f98..a61517d0 100644 --- a/Disco.Services/Disco.Services.csproj +++ b/Disco.Services/Disco.Services.csproj @@ -484,11 +484,15 @@ + + + + diff --git a/Disco.Services/Interop/DiscoServices/ActivationService.cs b/Disco.Services/Interop/DiscoServices/ActivationService.cs index 3472ac45..40f69399 100644 --- a/Disco.Services/Interop/DiscoServices/ActivationService.cs +++ b/Disco.Services/Interop/DiscoServices/ActivationService.cs @@ -19,7 +19,6 @@ namespace Disco.Services.Interop.DiscoServices public class ActivationService { private static readonly byte[] onlineServicesActivationKey; - internal static readonly Uri BaseUrl = new Uri("https://activate.discoict.com.au"); private readonly DiscoDataContext database; static ActivationService() @@ -41,7 +40,7 @@ namespace Disco.Services.Interop.DiscoServices public bool RequiresCleanup => Directory.Exists(GetDataStoreLocation); public Uri GetCallbackUrl() - => new Uri(BaseUrl, "/api/callback"); + => new Uri(DiscoServiceHelpers.ActivationServiceUrl, "/api/callback"); /// /// Begin the activation process @@ -64,7 +63,7 @@ namespace Disco.Services.Interop.DiscoServices ChallengeResponse challenge; using (var httpClient = new HttpClient()) { - httpClient.BaseAddress = BaseUrl; + httpClient.BaseAddress = DiscoServiceHelpers.ActivationServiceUrl; httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); var body = new ChallengeRequest() @@ -124,7 +123,7 @@ namespace Disco.Services.Interop.DiscoServices TimeStamp = challenge.TimeStamp, ChallengeResponse = challengeResponse, ChallengeResponseIv = challengeResponseIv, - RedirectUrl = new Uri(BaseUrl, "/").ToString(), + RedirectUrl = new Uri(DiscoServiceHelpers.ActivationServiceUrl, "/").ToString(), }; // store activation @@ -174,7 +173,7 @@ namespace Disco.Services.Interop.DiscoServices using (var httpClient = new HttpClient()) { - httpClient.BaseAddress = BaseUrl; + httpClient.BaseAddress = DiscoServiceHelpers.ActivationServiceUrl; httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); var body = new CompleteRequest() diff --git a/Disco.Services/Interop/DiscoServices/DiscoServiceHelpers.cs b/Disco.Services/Interop/DiscoServices/DiscoServiceHelpers.cs index bb5a3863..ed1cc25b 100644 --- a/Disco.Services/Interop/DiscoServices/DiscoServiceHelpers.cs +++ b/Disco.Services/Interop/DiscoServices/DiscoServiceHelpers.cs @@ -4,18 +4,8 @@ namespace Disco.Services.Interop.DiscoServices { public static class DiscoServiceHelpers { - [Obsolete] - public static string CommunityUrl() - { - return "https://discoict.com.au/base/"; - } - - public static string ServicesUrl - { - get - { - return "https://services.discoict.com.au/"; - } - } + public static string ServicesUrl { get; } = "https://services.discoict.com.au/"; + public static Uri ActivationServiceUrl { get; } = new Uri("https://activate.discoict.com.au"); + public static Uri UploadOnlineUrl { get; } = new Uri("https://upload.discoict.com.au"); } -} \ No newline at end of file +} diff --git a/Disco.Services/Interop/DiscoServices/OnlineServicesAuthenticatedHandler.cs b/Disco.Services/Interop/DiscoServices/OnlineServicesAuthenticatedHandler.cs new file mode 100644 index 00000000..57eb277a --- /dev/null +++ b/Disco.Services/Interop/DiscoServices/OnlineServicesAuthenticatedHandler.cs @@ -0,0 +1,18 @@ +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; + +namespace Disco.Services.Interop.DiscoServices +{ + internal class OnlineServicesAuthenticatedHandler : HttpClientHandler + { + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var token = await OnlineServicesAuthentication.GetTokenAsync(); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token); + + return await base.SendAsync(request, cancellationToken); + } + } +} diff --git a/Disco.Services/Interop/DiscoServices/OnlineServicesAuthentication.cs b/Disco.Services/Interop/DiscoServices/OnlineServicesAuthentication.cs index 2f3eb31d..bbe99ebb 100644 --- a/Disco.Services/Interop/DiscoServices/OnlineServicesAuthentication.cs +++ b/Disco.Services/Interop/DiscoServices/OnlineServicesAuthentication.cs @@ -50,7 +50,7 @@ namespace Disco.Services.Interop.DiscoServices { localExpires = tokenExpires; localToken = token; - if (tokenExpires != null && tokenExpires.Value < DateTime.UtcNow && localToken != null) + if (tokenExpires != null && tokenExpires.Value > DateTime.UtcNow && localToken != null) return localToken; if (!IsActivated) @@ -58,7 +58,7 @@ namespace Disco.Services.Interop.DiscoServices using (var httpClient = new HttpClient()) { - httpClient.BaseAddress = ActivationService.BaseUrl; + httpClient.BaseAddress = DiscoServiceHelpers.ActivationServiceUrl; httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); var timeStamp = DateTime.UtcNow.ToUnixEpoc(); diff --git a/Disco.Services/Interop/DiscoServices/OnlineServicesConnect.cs b/Disco.Services/Interop/DiscoServices/OnlineServicesConnect.cs index 514ac6c1..6e0ec56f 100644 --- a/Disco.Services/Interop/DiscoServices/OnlineServicesConnect.cs +++ b/Disco.Services/Interop/DiscoServices/OnlineServicesConnect.cs @@ -33,7 +33,7 @@ namespace Disco.Services.Interop.DiscoServices static OnlineServicesConnect() { connection = new HubConnectionBuilder() - .WithUrl(new Uri(ActivationService.BaseUrl, "/connect"), options => + .WithUrl(new Uri(DiscoServiceHelpers.ActivationServiceUrl, "/connect"), options => { options.AccessTokenProvider = () => OnlineServicesAuthentication.GetTokenAsync(); }) diff --git a/Disco.Services/Interop/DiscoServices/Upload/UploadOnlineClient.cs b/Disco.Services/Interop/DiscoServices/Upload/UploadOnlineClient.cs new file mode 100644 index 00000000..bd602513 --- /dev/null +++ b/Disco.Services/Interop/DiscoServices/Upload/UploadOnlineClient.cs @@ -0,0 +1,126 @@ +using Disco.Data.Repository; +using Disco.Models.Repository; +using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; +using System; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; + +namespace Disco.Services.Interop.DiscoServices.Upload +{ + internal class UploadOnlineClient + { + private readonly string organisationName; + private readonly HttpClient httpClient; + private readonly JsonSerializerSettings serializerSettings = new JsonSerializerSettings() + { + ContractResolver = new CamelCasePropertyNamesContractResolver(), + }; + public UploadOnlineClient() + { + using (var database = new DiscoDataContext()) + organisationName = database.DiscoConfiguration.OrganisationName; + + httpClient = new HttpClient(new OnlineServicesAuthenticatedHandler()) + { + BaseAddress = new Uri(DiscoServiceHelpers.UploadOnlineUrl, "/api/v1/"), + }; + } + + public async Task<(Uri sessionUri, DateTime sessionExpiration)> CreateSession(User techUser, IAttachmentTarget attachmentTarget) + { + var model = new CreateSessionRequestModel() + { + CreatedBy = techUser.UserId, + OrganisationName = organisationName, + TargetType = attachmentTarget.HasAttachmentType, + TargetId = attachmentTarget.AttachmentReferenceId, + TargetDisplayName = GetAttachmentTargetDisplayName(attachmentTarget), + }; + + var modelJson = JsonConvert.SerializeObject(model, serializerSettings); + + using (var response = await httpClient.PostAsync("session", new StringContent(modelJson, Encoding.UTF8, "application/json"))) + { + response.EnsureSuccessStatusCode(); + + var responseJson = await response.Content.ReadAsStringAsync(); + var responseModel = JsonConvert.DeserializeObject(responseJson, serializerSettings) + ?? throw new InvalidOperationException("Failed to create upload session (empty response)"); + + if (!responseModel.Success) + throw new InvalidOperationException($"Failed to create upload session ({responseModel.ErrorMessage})"); + + var expiration = DateTime.Now.AddSeconds(responseModel.ExpiresInSeconds - 10 ?? 0); + var sessionUri = new Uri(responseModel.SessionUrl, UriKind.Absolute); + + return (sessionUri, expiration); + } + } + + public MemoryStream SyncUploads(string lastFileId, string hintFileId) + { + var response = Task.Run(() => httpClient.GetAsync($"sync?last={lastFileId}&hint={hintFileId}")).GetAwaiter().GetResult(); + try + { + response.EnsureSuccessStatusCode(); + + if (response.StatusCode == HttpStatusCode.NoContent) + return null; + + var stream = new MemoryStream(); + + Task.Run(() => response.Content.CopyToAsync(stream)).GetAwaiter().GetResult(); + + stream.Position = 0; + return stream; + } + finally + { + response.Dispose(); + } + } + + private string GetAttachmentTargetDisplayName(IAttachmentTarget attachmentTarget) + { + switch (attachmentTarget.HasAttachmentType) + { + case AttachmentTypes.Device: + return $"Device: {attachmentTarget.AttachmentReferenceId}"; + case AttachmentTypes.Job: + return $"Job #{attachmentTarget.AttachmentReferenceId}"; + case AttachmentTypes.User: + if (attachmentTarget is User user) + return $"User: {user.DisplayName} ({ActiveDirectory.ActiveDirectory.FriendlyAccountId(user.UserId)})"; + else + return $"User: {attachmentTarget.AttachmentReferenceId}"; + case AttachmentTypes.DeviceBatch: + if (attachmentTarget is DeviceBatch deviceBatch) + return $"Device Batch: {deviceBatch.Name} ({deviceBatch.Id})"; + else + return $"Device Batch {attachmentTarget.AttachmentReferenceId}"; + } + return $"{attachmentTarget.HasAttachmentType}: {attachmentTarget.AttachmentReferenceId}"; + } + + private class CreateSessionRequestModel + { + public string CreatedBy { get; set; } + public string OrganisationName { get; set; } + public AttachmentTypes TargetType { get; set; } + public string TargetId { get; set; } + public string TargetDisplayName { get; set; } + } + + private class CreateSessionResponseModel + { + public bool Success { get; set; } + public string SessionUrl { get; set; } + public int? ExpiresInSeconds { get; set; } + public string ErrorMessage { get; set; } + } + } +} diff --git a/Disco.Services/Interop/DiscoServices/Upload/UploadOnlineService.cs b/Disco.Services/Interop/DiscoServices/Upload/UploadOnlineService.cs new file mode 100644 index 00000000..7c466569 --- /dev/null +++ b/Disco.Services/Interop/DiscoServices/Upload/UploadOnlineService.cs @@ -0,0 +1,27 @@ +using Disco.Models.Repository; +using System; +using System.IO; +using System.Threading.Tasks; + +namespace Disco.Services.Interop.DiscoServices.Upload +{ + public static class UploadOnlineService + { + private static readonly UploadOnlineClient client; + + static UploadOnlineService() + { + client = new UploadOnlineClient(); + } + + public static Task<(Uri sessionUri, DateTime sessionExpiration)> CreateSession(User techUser, IAttachmentTarget attachmentTarget) + { + return client.CreateSession(techUser, attachmentTarget); + } + + internal static MemoryStream SyncUploads(string lastFileId, string hintFileId) + { + return client.SyncUploads(lastFileId, hintFileId); + } + } +} diff --git a/Disco.Services/Interop/DiscoServices/Upload/UploadOnlineSyncTask.cs b/Disco.Services/Interop/DiscoServices/Upload/UploadOnlineSyncTask.cs new file mode 100644 index 00000000..1088318c --- /dev/null +++ b/Disco.Services/Interop/DiscoServices/Upload/UploadOnlineSyncTask.cs @@ -0,0 +1,209 @@ +using Disco.Data.Repository; +using Disco.Models.Repository; +using Disco.Models.Services.Interop.DiscoServices; +using Disco.Services.Tasks; +using Disco.Services.Users; +using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; +using Quartz; +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; + +namespace Disco.Services.Interop.DiscoServices.Upload +{ + public class UploadOnlineSyncTask : ScheduledTask + { + private readonly static object runSerialLock = new object(); + private const int connectNotificationType = 1628986937; + private const string AttachmentHandlerId = "Upload"; + + public override string TaskName { get { return "Upload Online - Sync"; } } + public override bool SingleInstanceTask { get; } = false; + + public static ScheduledTaskStatus RunningStatus + => ScheduledTasks.GetTaskStatuses(typeof(UploadOnlineSyncTask)).Where(ts => ts.IsRunning).FirstOrDefault(); + + public override void InitalizeScheduledTask(DiscoDataContext Database) + { + + OnlineServicesConnect.SubscribeToNotifications(HandleConnectNotification, connectNotificationType); + } + + public static ScheduledTaskStatus ScheduleInOneHour() + { + var instance = new UploadOnlineSyncTask(); + var trigger = TriggerBuilder.Create() + .StartAt(DateTimeOffset.Now.AddHours(1)); + return instance.ScheduleTask(trigger); + } + + private static ScheduledTaskStatus ScheduleNow(string hintFileId) + { + var taskState = new JobDataMap() { { "hintFileId", hintFileId } }; + var instance = new UploadOnlineSyncTask(); + return instance.ScheduleTask(taskState); + } + + private static void HandleConnectNotification(IConnectNotification notification) + { + if (notification.Version == 1 && notification.Type == connectNotificationType && Guid.TryParse(notification.Content, out _)) + { + ScheduleNow(notification.Content); + } + } + + protected override void ExecuteTask() + { + var hintFileId = (string)(ExecutionContext.JobDetail?.JobDataMap?["hintFileId"]); + + using (var database = new DiscoDataContext()) + { + if (hintFileId != null && UploadAttachmentExists(database, hintFileId)) + { + Status.Finished("Hinted attachment has already been downloaded"); + return; + } + + lock (runSerialLock) + { + if (hintFileId != null && UploadAttachmentExists(database, hintFileId)) + { + Status.Finished("Hinted attachment has already been downloaded"); + return; + } + + var lastFileId = LastUploadFileId(database); + + Status.UpdateStatus(10, "Fetching attachments from Online Services"); + var archiveStream = UploadOnlineService.SyncUploads(lastFileId, hintFileId); + + if (archiveStream == null) + { + Status.Finished("No new uploads found"); + return; + } + + using (var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read)) + { + Status.UpdateStatus(45, "Reading manifest"); + + List manifest; + var manifestEntry = archive.GetEntry("manifest.json"); + using (var manifestStream = manifestEntry.Open()) + { + using (var manifestReader = new StreamReader(manifestStream)) + { + using (var jsonReader = new JsonTextReader(manifestReader)) + { + var serializerSettings = new JsonSerializerSettings() + { + ContractResolver = new CamelCasePropertyNamesContractResolver(), + }; + var serializer = JsonSerializer.Create(serializerSettings); + manifest = serializer.Deserialize>(jsonReader); + } + } + } + + if (manifest == null || manifest.Count == 0) + { + Status.Finished("No uploads found in the archive manifest"); + return; + } + + Status.UpdateStatus(50, $"Importing {manifest.Count} attachments"); + var attachmentStream = new MemoryStream(); + foreach (var upload in manifest) + { + var archiveEntry = archive.GetEntry(upload.Id); + if (archiveEntry == null) + continue; + + if (!UserService.TryGetUser(upload.CreatedBy, database, false, out var createdBy)) + continue; + + var createdOn = DateTimeOffset.FromUnixTimeMilliseconds(upload.CreatedOn).ToLocalTime().DateTime; + + using (var uploadStream = archiveEntry.Open()) + { + uploadStream.CopyTo(attachmentStream); + attachmentStream.Position = 0; + switch (upload.TargetType) + { + case AttachmentTypes.Device: + var device = database.Devices.Find(upload.TargetId); + if (device == null) + continue; + if (database.DeviceAttachments.Any(da => da.DeviceSerialNumber == device.SerialNumber && da.HandlerId == AttachmentHandlerId && da.HandlerReferenceId == upload.Id)) + continue; + device.CreateAttachment(database, createdBy, upload.FileName, createdOn, upload.MimeType, upload.Comments, attachmentStream, DocumentTemplate: null, PdfThumbnail: null, HandlerId: AttachmentHandlerId, HandlerReferenceId: upload.Id, HandlerData: null); + break; + case AttachmentTypes.Job: + var jobId = int.Parse(upload.TargetId); + var job = database.Jobs.Find(jobId); + if (job == null) + continue; + if (database.JobAttachments.Any(ja => ja.JobId == jobId && ja.HandlerId == AttachmentHandlerId && ja.HandlerReferenceId == upload.Id)) + continue; + job.CreateAttachment(database, createdBy, upload.FileName, createdOn, upload.MimeType, upload.Comments, attachmentStream, DocumentTemplate: null, PdfThumbnail: null, HandlerId: AttachmentHandlerId, HandlerReferenceId: upload.Id, HandlerData: null); + break; + case AttachmentTypes.User: + if (UserService.TryGetUser(upload.TargetId, database, true, out var targetUser)) + { + if (database.UserAttachments.Any(ua => ua.UserId == targetUser.UserId && ua.HandlerId == AttachmentHandlerId && ua.HandlerReferenceId == upload.Id)) + continue; + targetUser.CreateAttachment(database, createdBy, upload.FileName, createdOn, upload.MimeType, upload.Comments, attachmentStream, DocumentTemplate: null, PdfThumbnail: null, HandlerId: AttachmentHandlerId, HandlerReferenceId: upload.Id, HandlerData: null); + } + break; + } + attachmentStream.SetLength(0); + } + } + + Status.Finished("Sync completed successfully"); + } + } + } + } + + private static bool UploadAttachmentExists(DiscoDataContext database, string fileId) + { + return database.JobAttachments + .Any(ja => ja.HandlerId == AttachmentHandlerId && ja.HandlerReferenceId == fileId) || + database.DeviceAttachments + .Any(da => da.HandlerId == AttachmentHandlerId && da.HandlerReferenceId == fileId) || + database.UserAttachments + .Any(ua => ua.HandlerId == AttachmentHandlerId && ua.HandlerReferenceId == fileId); + } + + private static string LastUploadFileId(DiscoDataContext database) + { + var ids = new List(3) + { + database.JobAttachments.Where(ja => ja.HandlerId == AttachmentHandlerId).Max(ja => ja.HandlerReferenceId), + database.DeviceAttachments.Where(ja => ja.HandlerId == AttachmentHandlerId).Max(ja => ja.HandlerReferenceId), + database.UserAttachments.Where(ja => ja.HandlerId == AttachmentHandlerId).Max(ja => ja.HandlerReferenceId), + }; + ids.Sort(StringComparer.Ordinal); + if (ids[2] == null) + return null; + else + return ids[2]; + } + + private class SyncUploadModel + { + public string Id { get; set; } + public string CreatedBy { get; set; } + public long CreatedOn { get; set; } + public AttachmentTypes TargetType { get; set; } + public string TargetId { get; set; } + public string FileName { get; set; } + public string MimeType { get; set; } + public string Comments { get; set; } + } + } +}