feature: Upload Online Attachments

Disco.Services/Disco.Services.csproj

@@ -484,11 +484,15 @@
     <Compile Include="Interop\DiscoServices\DiscoServiceHelpers.cs" />
     <Compile Include="Interop\DiscoServices\Jobs.cs" />
     <Compile Include="Interop\DiscoServices\LicenseValidationTask.cs" />
+    <Compile Include="Interop\DiscoServices\OnlineServicesAuthenticatedHandler.cs" />
     <Compile Include="Interop\DiscoServices\OnlineServicesAuthentication.cs" />
     <Compile Include="Interop\DiscoServices\OnlineServicesConnect.cs" />
     <Compile Include="Interop\DiscoServices\OnlineServicesConnectStartTask.cs" />
     <Compile Include="Interop\DiscoServices\PluginLibrary.cs" />
     <Compile Include="Interop\DiscoServices\PluginLibraryUpdateTask.cs" />
+    <Compile Include="Interop\DiscoServices\Upload\UploadOnlineClient.cs" />
+    <Compile Include="Interop\DiscoServices\Upload\UploadOnlineService.cs" />
+    <Compile Include="Interop\DiscoServices\Upload\UploadOnlineSyncTask.cs" />
     <Compile Include="Interop\IIS\PreserveIisBindingsTask.cs" />
     <Compile Include="Interop\MimeTypes.cs" />
     <Compile Include="Interop\VicEduDept\VicSmart.cs" />

Disco.Services/Interop/DiscoServices/ActivationService.cs

@@ -19,7 +19,6 @@ namespace Disco.Services.Interop.DiscoServices
     public class ActivationService
     {
         private static readonly byte[] onlineServicesActivationKey;
-        internal static readonly Uri BaseUrl = new Uri("https://activate.discoict.com.au");
         private readonly DiscoDataContext database;
 
         static ActivationService()
@@ -41,7 +40,7 @@ namespace Disco.Services.Interop.DiscoServices
         public bool RequiresCleanup => Directory.Exists(GetDataStoreLocation);
 
         public Uri GetCallbackUrl()
-            => new Uri(BaseUrl, "/api/callback");
+            => new Uri(DiscoServiceHelpers.ActivationServiceUrl, "/api/callback");
 
         /// <summary>
         /// Begin the activation process
@@ -64,7 +63,7 @@ namespace Disco.Services.Interop.DiscoServices
             ChallengeResponse challenge;
             using (var httpClient = new HttpClient())
             {
-                httpClient.BaseAddress = BaseUrl;
+                httpClient.BaseAddress = DiscoServiceHelpers.ActivationServiceUrl;
                 httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
 
                 var body = new ChallengeRequest()
@@ -124,7 +123,7 @@ namespace Disco.Services.Interop.DiscoServices
                 TimeStamp = challenge.TimeStamp,
                 ChallengeResponse = challengeResponse,
                 ChallengeResponseIv = challengeResponseIv,
-                RedirectUrl = new Uri(BaseUrl, "/").ToString(),
+                RedirectUrl = new Uri(DiscoServiceHelpers.ActivationServiceUrl, "/").ToString(),
             };
 
             // store activation
@@ -174,7 +173,7 @@ namespace Disco.Services.Interop.DiscoServices
 
             using (var httpClient = new HttpClient())
             {
-                httpClient.BaseAddress = BaseUrl;
+                httpClient.BaseAddress = DiscoServiceHelpers.ActivationServiceUrl;
                 httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
 
                 var body = new CompleteRequest()

Disco.Services/Interop/DiscoServices/DiscoServiceHelpers.cs

@@ -4,18 +4,8 @@ namespace Disco.Services.Interop.DiscoServices
 {
     public static class DiscoServiceHelpers
     {
-        [Obsolete]
+        public static string ServicesUrl { get; } = "https://services.discoict.com.au/";
-        public static string CommunityUrl()
+        public static Uri ActivationServiceUrl { get; } = new Uri("https://activate.discoict.com.au");
-        {
+        public static Uri UploadOnlineUrl { get; } = new Uri("https://upload.discoict.com.au");
-            return "https://discoict.com.au/base/";
-        }
-
-        public static string ServicesUrl
-        {
-            get
-            {
-                return "https://services.discoict.com.au/";
-            }
-        }
     }
 }

Disco.Services/Interop/DiscoServices/OnlineServicesAuthenticatedHandler.cs

@@ -0,0 +1,18 @@
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Disco.Services.Interop.DiscoServices
+{
+    internal class OnlineServicesAuthenticatedHandler : HttpClientHandler
+    {
+        protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+        {
+            var token = await OnlineServicesAuthentication.GetTokenAsync();
+            request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token);
+
+            return await base.SendAsync(request, cancellationToken);
+        }
+    }
+}

Disco.Services/Interop/DiscoServices/OnlineServicesAuthentication.cs

@@ -50,7 +50,7 @@ namespace Disco.Services.Interop.DiscoServices
             {
                 localExpires = tokenExpires;
                 localToken = token;
-                if (tokenExpires != null && tokenExpires.Value < DateTime.UtcNow && localToken != null)
+                if (tokenExpires != null && tokenExpires.Value > DateTime.UtcNow && localToken != null)
                     return localToken;
 
                 if (!IsActivated)
@@ -58,7 +58,7 @@ namespace Disco.Services.Interop.DiscoServices
 
                 using (var httpClient = new HttpClient())
                 {
-                    httpClient.BaseAddress = ActivationService.BaseUrl;
+                    httpClient.BaseAddress = DiscoServiceHelpers.ActivationServiceUrl;
                     httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
 
                     var timeStamp = DateTime.UtcNow.ToUnixEpoc();

Disco.Services/Interop/DiscoServices/OnlineServicesConnect.cs

@@ -33,7 +33,7 @@ namespace Disco.Services.Interop.DiscoServices
         static OnlineServicesConnect()
         {
             connection = new HubConnectionBuilder()
-                .WithUrl(new Uri(ActivationService.BaseUrl, "/connect"), options =>
+                .WithUrl(new Uri(DiscoServiceHelpers.ActivationServiceUrl, "/connect"), options =>
                 {
                     options.AccessTokenProvider = () => OnlineServicesAuthentication.GetTokenAsync();
                 })

Disco.Services/Interop/DiscoServices/Upload/UploadOnlineClient.cs

@@ -0,0 +1,126 @@
+using Disco.Data.Repository;
+using Disco.Models.Repository;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Serialization;
+using System;
+using System.IO;
+using System.Net;
+using System.Net.Http;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Disco.Services.Interop.DiscoServices.Upload
+{
+    internal class UploadOnlineClient
+    {
+        private readonly string organisationName;
+        private readonly HttpClient httpClient;
+        private readonly JsonSerializerSettings serializerSettings = new JsonSerializerSettings()
+        {
+            ContractResolver = new CamelCasePropertyNamesContractResolver(),
+        };
+        public UploadOnlineClient()
+        {
+            using (var database = new DiscoDataContext())
+                organisationName = database.DiscoConfiguration.OrganisationName;
+
+            httpClient = new HttpClient(new OnlineServicesAuthenticatedHandler())
+            {
+                BaseAddress = new Uri(DiscoServiceHelpers.UploadOnlineUrl, "/api/v1/"),
+            };
+        }
+
+        public async Task<(Uri sessionUri, DateTime sessionExpiration)> CreateSession(User techUser, IAttachmentTarget attachmentTarget)
+        {
+            var model = new CreateSessionRequestModel()
+            {
+                CreatedBy = techUser.UserId,
+                OrganisationName = organisationName,
+                TargetType = attachmentTarget.HasAttachmentType,
+                TargetId = attachmentTarget.AttachmentReferenceId,
+                TargetDisplayName = GetAttachmentTargetDisplayName(attachmentTarget),
+            };
+
+            var modelJson = JsonConvert.SerializeObject(model, serializerSettings);
+
+            using (var response = await httpClient.PostAsync("session", new StringContent(modelJson, Encoding.UTF8, "application/json")))
+            {
+                response.EnsureSuccessStatusCode();
+
+                var responseJson = await response.Content.ReadAsStringAsync();
+                var responseModel = JsonConvert.DeserializeObject<CreateSessionResponseModel>(responseJson, serializerSettings)
+                    ?? throw new InvalidOperationException("Failed to create upload session (empty response)");
+
+                if (!responseModel.Success)
+                    throw new InvalidOperationException($"Failed to create upload session ({responseModel.ErrorMessage})");
+
+                var expiration = DateTime.Now.AddSeconds(responseModel.ExpiresInSeconds - 10 ?? 0);
+                var sessionUri = new Uri(responseModel.SessionUrl, UriKind.Absolute);
+
+                return (sessionUri, expiration);
+            }
+        }
+
+        public MemoryStream SyncUploads(string lastFileId, string hintFileId)
+        {
+            var response = Task.Run(() => httpClient.GetAsync($"sync?last={lastFileId}&hint={hintFileId}")).GetAwaiter().GetResult();
+            try
+            {
+                response.EnsureSuccessStatusCode();
+
+                if (response.StatusCode == HttpStatusCode.NoContent)
+                    return null;
+
+                var stream = new MemoryStream();
+
+                Task.Run(() => response.Content.CopyToAsync(stream)).GetAwaiter().GetResult();
+
+                stream.Position = 0;
+                return stream;
+            }
+            finally
+            {
+                response.Dispose();
+            }
+        }
+
+        private string GetAttachmentTargetDisplayName(IAttachmentTarget attachmentTarget)
+        {
+            switch (attachmentTarget.HasAttachmentType)
+            {
+                case AttachmentTypes.Device:
+                    return $"Device: {attachmentTarget.AttachmentReferenceId}";
+                case AttachmentTypes.Job:
+                    return $"Job #{attachmentTarget.AttachmentReferenceId}";
+                case AttachmentTypes.User:
+                    if (attachmentTarget is User user)
+                        return $"User: {user.DisplayName} ({ActiveDirectory.ActiveDirectory.FriendlyAccountId(user.UserId)})";
+                    else
+                        return $"User: {attachmentTarget.AttachmentReferenceId}";
+                case AttachmentTypes.DeviceBatch:
+                    if (attachmentTarget is DeviceBatch deviceBatch)
+                        return $"Device Batch: {deviceBatch.Name} ({deviceBatch.Id})";
+                    else
+                        return $"Device Batch {attachmentTarget.AttachmentReferenceId}";
+            }
+            return $"{attachmentTarget.HasAttachmentType}: {attachmentTarget.AttachmentReferenceId}";
+        }
+
+        private class CreateSessionRequestModel
+        {
+            public string CreatedBy { get; set; }
+            public string OrganisationName { get; set; }
+            public AttachmentTypes TargetType { get; set; }
+            public string TargetId { get; set; }
+            public string TargetDisplayName { get; set; }
+        }
+
+        private class CreateSessionResponseModel
+        {
+            public bool Success { get; set; }
+            public string SessionUrl { get; set; }
+            public int? ExpiresInSeconds { get; set; }
+            public string ErrorMessage { get; set; }
+        }
+    }
+}

Disco.Services/Interop/DiscoServices/Upload/UploadOnlineService.cs

@@ -0,0 +1,27 @@
+using Disco.Models.Repository;
+using System;
+using System.IO;
+using System.Threading.Tasks;
+
+namespace Disco.Services.Interop.DiscoServices.Upload
+{
+    public static class UploadOnlineService
+    {
+        private static readonly UploadOnlineClient client;
+
+        static UploadOnlineService()
+        {
+            client = new UploadOnlineClient();
+        }
+
+        public static Task<(Uri sessionUri, DateTime sessionExpiration)> CreateSession(User techUser, IAttachmentTarget attachmentTarget)
+        {
+            return client.CreateSession(techUser, attachmentTarget);
+        }
+
+        internal static MemoryStream SyncUploads(string lastFileId, string hintFileId)
+        {
+            return client.SyncUploads(lastFileId, hintFileId);
+        }
+    }
+}

Disco.Services/Interop/DiscoServices/Upload/UploadOnlineSyncTask.cs

@@ -0,0 +1,209 @@
+using Disco.Data.Repository;
+using Disco.Models.Repository;
+using Disco.Models.Services.Interop.DiscoServices;
+using Disco.Services.Tasks;
+using Disco.Services.Users;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Serialization;
+using Quartz;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+
+namespace Disco.Services.Interop.DiscoServices.Upload
+{
+    public class UploadOnlineSyncTask : ScheduledTask
+    {
+        private readonly static object runSerialLock = new object();
+        private const int connectNotificationType = 1628986937;
+        private const string AttachmentHandlerId = "Upload";
+
+        public override string TaskName { get { return "Upload Online - Sync"; } }
+        public override bool SingleInstanceTask { get; } = false;
+
+        public static ScheduledTaskStatus RunningStatus
+            => ScheduledTasks.GetTaskStatuses(typeof(UploadOnlineSyncTask)).Where(ts => ts.IsRunning).FirstOrDefault();
+
+        public override void InitalizeScheduledTask(DiscoDataContext Database)
+        {
+
+            OnlineServicesConnect.SubscribeToNotifications(HandleConnectNotification, connectNotificationType);
+        }
+
+        public static ScheduledTaskStatus ScheduleInOneHour()
+        {
+            var instance = new UploadOnlineSyncTask();
+            var trigger = TriggerBuilder.Create()
+                .StartAt(DateTimeOffset.Now.AddHours(1));
+            return instance.ScheduleTask(trigger);
+        }
+
+        private static ScheduledTaskStatus ScheduleNow(string hintFileId)
+        {
+            var taskState = new JobDataMap() { { "hintFileId", hintFileId } };
+            var instance = new UploadOnlineSyncTask();
+            return instance.ScheduleTask(taskState);
+        }
+
+        private static void HandleConnectNotification(IConnectNotification notification)
+        {
+            if (notification.Version == 1 && notification.Type == connectNotificationType && Guid.TryParse(notification.Content, out _))
+            {
+                ScheduleNow(notification.Content);
+            }
+        }
+
+        protected override void ExecuteTask()
+        {
+            var hintFileId = (string)(ExecutionContext.JobDetail?.JobDataMap?["hintFileId"]);
+
+            using (var database = new DiscoDataContext())
+            {
+                if (hintFileId != null && UploadAttachmentExists(database, hintFileId))
+                {
+                    Status.Finished("Hinted attachment has already been downloaded");
+                    return;
+                }
+
+                lock (runSerialLock)
+                {
+                    if (hintFileId != null && UploadAttachmentExists(database, hintFileId))
+                    {
+                        Status.Finished("Hinted attachment has already been downloaded");
+                        return;
+                    }
+
+                    var lastFileId = LastUploadFileId(database);
+
+                    Status.UpdateStatus(10, "Fetching attachments from Online Services");
+                    var archiveStream = UploadOnlineService.SyncUploads(lastFileId, hintFileId);
+
+                    if (archiveStream == null)
+                    {
+                        Status.Finished("No new uploads found");
+                        return;
+                    }
+
+                    using (var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read))
+                    {
+                        Status.UpdateStatus(45, "Reading manifest");
+
+                        List<SyncUploadModel> manifest;
+                        var manifestEntry = archive.GetEntry("manifest.json");
+                        using (var manifestStream = manifestEntry.Open())
+                        {
+                            using (var manifestReader = new StreamReader(manifestStream))
+                            {
+                                using (var jsonReader = new JsonTextReader(manifestReader))
+                                {
+                                    var serializerSettings = new JsonSerializerSettings()
+                                    {
+                                        ContractResolver = new CamelCasePropertyNamesContractResolver(),
+                                    };
+                                    var serializer = JsonSerializer.Create(serializerSettings);
+                                    manifest = serializer.Deserialize<List<SyncUploadModel>>(jsonReader);
+                                }
+                            }
+                        }
+
+                        if (manifest == null || manifest.Count == 0)
+                        {
+                            Status.Finished("No uploads found in the archive manifest");
+                            return;
+                        }
+
+                        Status.UpdateStatus(50, $"Importing {manifest.Count} attachments");
+                        var attachmentStream = new MemoryStream();
+                        foreach (var upload in manifest)
+                        {
+                            var archiveEntry = archive.GetEntry(upload.Id);
+                            if (archiveEntry == null)
+                                continue;
+
+                            if (!UserService.TryGetUser(upload.CreatedBy, database, false, out var createdBy))
+                                continue;
+
+                            var createdOn = DateTimeOffset.FromUnixTimeMilliseconds(upload.CreatedOn).ToLocalTime().DateTime;
+
+                            using (var uploadStream = archiveEntry.Open())
+                            {
+                                uploadStream.CopyTo(attachmentStream);
+                                attachmentStream.Position = 0;
+                                switch (upload.TargetType)
+                                {
+                                    case AttachmentTypes.Device:
+                                        var device = database.Devices.Find(upload.TargetId);
+                                        if (device == null)
+                                            continue;
+                                        if (database.DeviceAttachments.Any(da => da.DeviceSerialNumber == device.SerialNumber && da.HandlerId == AttachmentHandlerId && da.HandlerReferenceId == upload.Id))
+                                            continue;
+                                        device.CreateAttachment(database, createdBy, upload.FileName, createdOn, upload.MimeType, upload.Comments, attachmentStream, DocumentTemplate: null, PdfThumbnail: null, HandlerId: AttachmentHandlerId, HandlerReferenceId: upload.Id, HandlerData: null);
+                                        break;
+                                    case AttachmentTypes.Job:
+                                        var jobId = int.Parse(upload.TargetId);
+                                        var job = database.Jobs.Find(jobId);
+                                        if (job == null)
+                                            continue;
+                                        if (database.JobAttachments.Any(ja => ja.JobId == jobId && ja.HandlerId == AttachmentHandlerId && ja.HandlerReferenceId == upload.Id))
+                                            continue;
+                                        job.CreateAttachment(database, createdBy, upload.FileName, createdOn, upload.MimeType, upload.Comments, attachmentStream, DocumentTemplate: null, PdfThumbnail: null, HandlerId: AttachmentHandlerId, HandlerReferenceId: upload.Id, HandlerData: null);
+                                        break;
+                                    case AttachmentTypes.User:
+                                        if (UserService.TryGetUser(upload.TargetId, database, true, out var targetUser))
+                                        {
+                                            if (database.UserAttachments.Any(ua => ua.UserId == targetUser.UserId && ua.HandlerId == AttachmentHandlerId && ua.HandlerReferenceId == upload.Id))
+                                                continue;
+                                            targetUser.CreateAttachment(database, createdBy, upload.FileName, createdOn, upload.MimeType, upload.Comments, attachmentStream, DocumentTemplate: null, PdfThumbnail: null, HandlerId: AttachmentHandlerId, HandlerReferenceId: upload.Id, HandlerData: null);
+                                        }
+                                        break;
+                                }
+                                attachmentStream.SetLength(0);
+                            }
+                        }
+
+                        Status.Finished("Sync completed successfully");
+                    }
+                }
+            }
+        }
+
+        private static bool UploadAttachmentExists(DiscoDataContext database, string fileId)
+        {
+            return database.JobAttachments
+                    .Any(ja => ja.HandlerId == AttachmentHandlerId && ja.HandlerReferenceId == fileId) ||
+                database.DeviceAttachments
+                    .Any(da => da.HandlerId == AttachmentHandlerId && da.HandlerReferenceId == fileId) ||
+                database.UserAttachments
+                    .Any(ua => ua.HandlerId == AttachmentHandlerId && ua.HandlerReferenceId == fileId);
+        }
+
+        private static string LastUploadFileId(DiscoDataContext database)
+        {
+            var ids = new List<string>(3)
+            {
+                database.JobAttachments.Where(ja => ja.HandlerId == AttachmentHandlerId).Max(ja => ja.HandlerReferenceId),
+                database.DeviceAttachments.Where(ja => ja.HandlerId == AttachmentHandlerId).Max(ja => ja.HandlerReferenceId),
+                database.UserAttachments.Where(ja => ja.HandlerId == AttachmentHandlerId).Max(ja => ja.HandlerReferenceId),
+            };
+            ids.Sort(StringComparer.Ordinal);
+            if (ids[2] == null)
+                return null;
+            else
+                return ids[2];
+        }
+
+        private class SyncUploadModel
+        {
+            public string Id { get; set; }
+            public string CreatedBy { get; set; }
+            public long CreatedOn { get; set; }
+            public AttachmentTypes TargetType { get; set; }
+            public string TargetId { get; set; }
+            public string FileName { get; set; }
+            public string MimeType { get; set; }
+            public string Comments { get; set; }
+        }
+    }
+}