qol: make Online Services Connect more reliable

Disco.Services/Interop/DiscoServices/OnlineServicesAuthentication.cs

@@ -133,7 +133,7 @@ namespace Disco.Services.Interop.DiscoServices
                     token = null;
                     tokenExpires = null;
 
-                    ThreadPool.QueueUserWorkItem(async _ => await OnlineServicesConnect.StartAsync());
+                    OnlineServicesConnect.QueueStart();
                 }
                 else
                 {

Disco.Services/Interop/DiscoServices/OnlineServicesConnect.cs

@@ -2,6 +2,7 @@
 using Disco.Services.Logging;
 using Microsoft.AspNetCore.SignalR.Client;
 using System;
+using System.Threading;
 using System.Threading.Tasks;
 
 namespace Disco.Services.Interop.DiscoServices
@@ -11,7 +12,23 @@ namespace Disco.Services.Interop.DiscoServices
 
         private static readonly HubConnection connection;
 
-        public static string State => connection.State.ToString();
+        public static ConnectionState State
+        {
+            get
+            {
+                switch (connection.State)
+                {
+                    case HubConnectionState.Connected:
+                        return ConnectionState.Connected;
+                    case HubConnectionState.Connecting:
+                        return ConnectionState.Connecting;
+                    case HubConnectionState.Reconnecting:
+                        return ConnectionState.Reconnecting;
+                    default:
+                        return ConnectionState.Disconnected;
+                }
+            }
+        }
 
         static OnlineServicesConnect()
         {
@@ -23,13 +40,30 @@ namespace Disco.Services.Interop.DiscoServices
                 .WithAutomaticReconnect()
                 .WithStatefulReconnect()
                 .Build();
+
+            connection.Closed += ex =>
+            {
+                SystemLog.LogException("Online Services: Connection Closed", ex);
+                return Task.CompletedTask;
+            };
+            connection.Reconnected += connectionId =>
+            {
+                SystemLog.LogInformation("Online Services: Connection Reconnected");
+                return Task.CompletedTask;
+            };
+            connection.Reconnecting += ex =>
+            {
+                SystemLog.LogInformation("Online Services: Connection Reconnecting");
+                return Task.CompletedTask;
+            };
         }
 
         public static async Task StartAsync()
         {
             try
             {
-                await connection.StartAsync();
+                if (connection.State == HubConnectionState.Disconnected)
+                    await connection.StartAsync();
             }
             catch (Exception ex)
             {
@@ -37,6 +71,12 @@ namespace Disco.Services.Interop.DiscoServices
             }
         }
 
+        public static void QueueStart()
+        {
+            if (connection.State == HubConnectionState.Disconnected)
+                ThreadPool.QueueUserWorkItem(async _ => await StartAsync());
+        }
+
         public static async Task StopAsync()
         {
             await connection.StopAsync();
@@ -88,5 +128,12 @@ namespace Disco.Services.Interop.DiscoServices
             public string Content { get; set; }
         }
 
+        public enum ConnectionState
+        {
+            Disconnected,
+            Connected,
+            Connecting,
+            Reconnecting
+        }
     }
 }

Disco.Services/Interop/DiscoServices/OnlineServicesConnectStartTask.cs

@@ -0,0 +1,39 @@
+using Disco.Data.Repository;
+using Disco.Services.Tasks;
+using Quartz;
+using System;
+
+namespace Disco.Services.Interop.DiscoServices
+{
+    public class OnlineServicesConnectStartTask : ScheduledTask
+    {
+        public override string TaskName => "Online Services Connect - Start";
+        public override bool LogExceptionsOnly => true;
+
+        public override void InitalizeScheduledTask(DiscoDataContext Database)
+        {
+            // Trigger in 1 + 0-14 minutes
+            var rng = new Random();
+            var delay = rng.Next(15) + 1;
+
+            TriggerBuilder triggerBuilder = TriggerBuilder.Create()
+                .StartAt(DateTimeOffset.Now.AddMinutes(delay))
+                .WithSchedule(SimpleScheduleBuilder.RepeatHourlyForever());
+
+            ScheduleTask(triggerBuilder);
+        }
+
+        protected override void ExecuteTask()
+        {
+            using (var database = new DiscoDataContext())
+            {
+                if (database.DiscoConfiguration.IsActivated)
+                {
+                    if (OnlineServicesConnect.State == OnlineServicesConnect.ConnectionState.Disconnected)
+                        OnlineServicesConnect.QueueStart();
+                }
+            }
+        }
+
+    }
+}