Feature #33: Enhanced Device Importing
Dynamic device importing. better input parsing and 5 additional import fields.
This commit is contained in:
Gary Sharp
@@ -1,272 +0,0 @@
|
||||
using Disco.BI.Extensions;
|
||||
using Disco.Data.Repository;
|
||||
using Disco.Models.BI.Device;
|
||||
using Disco.Models.Repository;
|
||||
using Disco.Services.Interop.ActiveDirectory;
|
||||
using Disco.Services.Users;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using System.Web;
|
||||
using PopulateRecordReferences = System.Tuple<System.Collections.Generic.Dictionary<int, Disco.Models.Repository.DeviceModel>, System.Collections.Generic.Dictionary<int, Disco.Models.Repository.DeviceProfile>, System.Collections.Generic.Dictionary<int, Disco.Models.Repository.DeviceBatch>>;
|
||||
|
||||
namespace Disco.BI.DeviceBI.Importing
|
||||
{
|
||||
public static class Import
|
||||
{
|
||||
internal const string ImportParseCacheKey = "ImportParseResults_{0}";
|
||||
|
||||
public static ImportDeviceSession GetSession(string ImportParseTaskId)
|
||||
{
|
||||
string parseKey = string.Format(ImportParseCacheKey, ImportParseTaskId);
|
||||
|
||||
return (ImportDeviceSession)HttpRuntime.Cache.Get(parseKey);
|
||||
}
|
||||
|
||||
internal static bool ImportRecord(this ImportDevice device, DiscoDataContext Database, PopulateRecordReferences references)
|
||||
{
|
||||
// Skips If Errors
|
||||
if (device.Errors == null || device.Errors.Count == 0)
|
||||
{
|
||||
// Re-Populate & Skip If Errors
|
||||
device.PopulateRecord(Database, references);
|
||||
if (device.Errors == null || device.Errors.Count == 0)
|
||||
{
|
||||
Device discoDevice = device.Device;
|
||||
|
||||
if (discoDevice == null)
|
||||
{
|
||||
// New Device
|
||||
discoDevice = new Device()
|
||||
{
|
||||
SerialNumber = device.SerialNumber.ToUpper(),
|
||||
CreatedDate = DateTime.Now,
|
||||
AllowUnauthenticatedEnrol = true,
|
||||
};
|
||||
Database.Devices.Add(discoDevice);
|
||||
}
|
||||
|
||||
if (discoDevice.DeviceModelId != device.DeviceModelId)
|
||||
discoDevice.DeviceModelId = device.DeviceModelId;
|
||||
if (discoDevice.DeviceProfileId != device.DeviceProfileId)
|
||||
discoDevice.DeviceProfileId = device.DeviceProfileId;
|
||||
if (discoDevice.DeviceBatchId != device.DeviceBatchId)
|
||||
discoDevice.DeviceBatchId = device.DeviceBatchId;
|
||||
if (discoDevice.Location != device.Location)
|
||||
discoDevice.Location = device.Location;
|
||||
if (discoDevice.AssetNumber != device.AssetNumber)
|
||||
discoDevice.AssetNumber = device.AssetNumber;
|
||||
|
||||
if (discoDevice.AssignedUserId != device.AssignedUserId)
|
||||
{
|
||||
discoDevice.AssignDevice(Database, device.AssignedUser);
|
||||
}
|
||||
|
||||
Database.SaveChanges();
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
internal static PopulateRecordReferences GetPopulateRecordReferences(DiscoDataContext Database)
|
||||
{
|
||||
return new PopulateRecordReferences(
|
||||
Database.DeviceModels.ToDictionary(dm => dm.Id),
|
||||
Database.DeviceProfiles.ToDictionary(dp => dp.Id),
|
||||
Database.DeviceBatches.ToDictionary(db => db.Id)
|
||||
);
|
||||
}
|
||||
|
||||
internal static void PopulateRecord(this ImportDevice device, DiscoDataContext Database, PopulateRecordReferences references)
|
||||
{
|
||||
|
||||
var deviceModels = references.Item1;
|
||||
var deviceProfiles = references.Item2;
|
||||
var deviceBatches = references.Item3;
|
||||
|
||||
// SERIAL NUMBER - Existing Device
|
||||
if (!device.Errors.ContainsKey("SerialNumber"))
|
||||
{
|
||||
device.Device = Database.Devices.Find(device.SerialNumber);
|
||||
if (device.Device != null && device.Device.DecommissionedDate.HasValue)
|
||||
device.Errors.Add("SerialNumber", "The device is decommissioned");
|
||||
}
|
||||
|
||||
|
||||
// DEVICE MODEL
|
||||
if (!device.Errors.ContainsKey("DeviceModelId"))
|
||||
{
|
||||
DeviceModel deviceModel;
|
||||
|
||||
if (!device.DeviceModelId.HasValue)
|
||||
device.DeviceModelId = 1; // Default 'Unknown Device Model'
|
||||
|
||||
if (!deviceModels.TryGetValue(device.DeviceModelId.Value, out deviceModel))
|
||||
device.Errors.Add("DeviceModelId", string.Format("Unknown device model id: {0}", device.DeviceModelId));
|
||||
else
|
||||
device.DeviceModel = deviceModel;
|
||||
}
|
||||
|
||||
// DEVICE PROFILE
|
||||
if (!device.Errors.ContainsKey("DeviceProfileId"))
|
||||
{
|
||||
DeviceProfile deviceProfile;
|
||||
if (!deviceProfiles.TryGetValue(device.DeviceProfileId, out deviceProfile))
|
||||
device.Errors.Add("DeviceProfileId", string.Format("Unknown device profile id: {0}", device.DeviceProfileId));
|
||||
else
|
||||
device.DeviceProfile = deviceProfile;
|
||||
}
|
||||
|
||||
// DEVICE BATCH
|
||||
if (!device.Errors.ContainsKey("DeviceBatchId") && device.DeviceBatchId.HasValue)
|
||||
{
|
||||
DeviceBatch deviceBatch;
|
||||
if (!deviceBatches.TryGetValue(device.DeviceBatchId.Value, out deviceBatch))
|
||||
device.Errors.Add("DeviceBatchId", string.Format("Unknown device Batch id: {0}", device.DeviceBatchId));
|
||||
else
|
||||
device.DeviceBatch = deviceBatch;
|
||||
}
|
||||
|
||||
// ASSIGNED USER
|
||||
if (!device.Errors.ContainsKey("AssignedUserId") && device.AssignedUserId != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
device.AssignedUser = UserService.GetUser(device.AssignedUserId, Database, true);
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
device.Errors.Add("AssignedUserId", string.Format("Unknown user id: {0}", device.AssignedUserId));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
internal static ImportDevice ParseRecord(this string[] record)
|
||||
{
|
||||
int csvFieldCount = record.Length;
|
||||
if (csvFieldCount < 1)
|
||||
throw new ArgumentException("At least one CSV field is required (Serial Number)");
|
||||
|
||||
string csvSerialNumber;
|
||||
string csvDeviceModelId;
|
||||
int deviceModelId = 1; // Default 'Unknown Device Model'
|
||||
string csvDeviceProfileId;
|
||||
int deviceProfileId = 1; // 'Default' Profile
|
||||
string csvDeviceBatchId;
|
||||
int deviceBatchId = 0; // No Batch
|
||||
string csvAssignedUserId = null;
|
||||
string csvLocation = null;
|
||||
string csvAssetNumber = null;
|
||||
Dictionary<string, string> errors = new Dictionary<string, string>();
|
||||
|
||||
// SERIAL NUMBER
|
||||
csvSerialNumber = record[0];
|
||||
if (string.IsNullOrWhiteSpace(csvSerialNumber))
|
||||
errors.Add("SerialNumber", "The serial number is required");
|
||||
else if (csvSerialNumber.Trim().Length > 60)
|
||||
errors.Add("SerialNumber", "The serial number must be less than or equal to 60 characters");
|
||||
|
||||
if (csvFieldCount > 1)
|
||||
{
|
||||
// DEVICE MODEL
|
||||
csvDeviceModelId = record[1];
|
||||
if (!string.IsNullOrWhiteSpace(csvDeviceModelId))
|
||||
if (!int.TryParse(csvDeviceModelId, out deviceModelId))
|
||||
errors.Add("DeviceModelId", "The device model is optional, but when supplied must be a number");
|
||||
else if (deviceModelId < 1)
|
||||
errors.Add("DeviceModelId", "The device model is optional, but when supplied must be greater than 0");
|
||||
|
||||
if (csvFieldCount > 2)
|
||||
{
|
||||
// DEVICE PROFILE
|
||||
csvDeviceProfileId = record[2];
|
||||
if (!string.IsNullOrWhiteSpace(csvDeviceProfileId))
|
||||
if (!int.TryParse(csvDeviceProfileId, out deviceProfileId))
|
||||
errors.Add("DeviceProfileId", "The device profile is optional, but when supplied must be a number");
|
||||
else if (deviceProfileId < 1)
|
||||
errors.Add("DeviceProfileId", "The device profile is optional, but when supplied must be greater than 0");
|
||||
|
||||
if (csvFieldCount > 3)
|
||||
{
|
||||
// DEVICE BATCH
|
||||
csvDeviceBatchId = record[3];
|
||||
if (!string.IsNullOrWhiteSpace(csvDeviceBatchId))
|
||||
if (!int.TryParse(csvDeviceBatchId, out deviceBatchId))
|
||||
errors.Add("DeviceBatchId", "The device batch is optional, but when supplied must be a number");
|
||||
else if (deviceBatchId < 1)
|
||||
errors.Add("DeviceBatchId", "The device batch is optional, but when supplied must be greater than 0");
|
||||
|
||||
if (csvFieldCount > 4)
|
||||
{
|
||||
// ASSIGNED USER
|
||||
csvAssignedUserId = record[4];
|
||||
if (string.IsNullOrWhiteSpace(csvAssignedUserId))
|
||||
csvAssignedUserId = null; // Not Assigned
|
||||
else
|
||||
{
|
||||
if (csvAssignedUserId.Length > 50)
|
||||
errors.Add("AssignedUserId", "The assigned user must be less than or equal to 50 characters");
|
||||
else if (!csvAssignedUserId.Contains('\\')) // Assume Primary Domain
|
||||
csvAssignedUserId = string.Format(@"{0}\{1}", ActiveDirectory.Context.PrimaryDomain.NetBiosName, csvAssignedUserId);
|
||||
}
|
||||
|
||||
if (csvFieldCount > 5)
|
||||
{
|
||||
// LOCATION
|
||||
csvLocation = record[5];
|
||||
if (string.IsNullOrWhiteSpace(csvLocation))
|
||||
csvLocation = null; // No Location Specified
|
||||
else if (csvLocation.Length > 250)
|
||||
errors.Add("Location", "The location must be less than or equal to 250 characters");
|
||||
|
||||
if (csvFieldCount > 6)
|
||||
{
|
||||
// ASSET NUMBER
|
||||
csvAssetNumber = record[6];
|
||||
if (string.IsNullOrWhiteSpace(csvAssetNumber))
|
||||
csvAssetNumber = null; // No Location Specified
|
||||
else if (csvAssetNumber.Length > 40)
|
||||
errors.Add("AssetNumber", "The asset number must be less than or equal to 40 characters");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return new ImportDevice()
|
||||
{
|
||||
SerialNumber = csvSerialNumber.Trim(),
|
||||
DeviceModelId = deviceModelId,
|
||||
DeviceProfileId = deviceProfileId,
|
||||
DeviceBatchId = deviceBatchId == 0 ? (int?)null : deviceBatchId,
|
||||
AssignedUserId = csvAssignedUserId,
|
||||
Location = csvLocation,
|
||||
AssetNumber = csvAssetNumber,
|
||||
Errors = errors
|
||||
};
|
||||
}
|
||||
|
||||
#region ImportDevice Extensions
|
||||
|
||||
public static string ImportStatus(this ImportDevice device)
|
||||
{
|
||||
if (device.Errors.Count > 0)
|
||||
return "Error";
|
||||
|
||||
if (device.Device != null)
|
||||
return "Update";
|
||||
|
||||
return "New";
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,91 +0,0 @@
|
||||
using Disco.Data.Repository;
|
||||
using Disco.Models.BI.Device;
|
||||
using Disco.Models.Repository;
|
||||
using Disco.Services.Tasks;
|
||||
using LumenWorks.Framework.IO.Csv;
|
||||
using Quartz;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using System.Web;
|
||||
using System.Web.Caching;
|
||||
|
||||
namespace Disco.BI.DeviceBI.Importing
|
||||
{
|
||||
public class ImportParseTask : ScheduledTask
|
||||
{
|
||||
public override string TaskName { get { return "Import Devices - Parsing"; } }
|
||||
|
||||
public override bool SingleInstanceTask { get { return false; } }
|
||||
public override bool CancelInitiallySupported { get { return false; } }
|
||||
|
||||
protected override void ExecuteTask()
|
||||
{
|
||||
string csvFilename = (string)this.ExecutionContext.JobDetail.JobDataMap["CsvFilename"];
|
||||
MemoryStream csvStream = (MemoryStream)this.ExecutionContext.JobDetail.JobDataMap["CsvImport"];
|
||||
|
||||
this.Status.UpdateStatus(0, "Parsing CSV File", "Loading Records");
|
||||
|
||||
List<ImportDevice> records;
|
||||
|
||||
using (TextReader csvTextReader = new StreamReader(csvStream))
|
||||
{
|
||||
using (CsvReader csvReader = new CsvReader(csvTextReader, true))
|
||||
{
|
||||
csvReader.DefaultParseErrorAction = ParseErrorAction.ThrowException;
|
||||
csvReader.MissingFieldAction = MissingFieldAction.ReplaceByNull;
|
||||
|
||||
records = csvReader.Select(record => record.ParseRecord()).ToList();
|
||||
}
|
||||
}
|
||||
csvStream.Dispose();
|
||||
|
||||
this.Status.UpdateStatus(20, "Parsing CSV File", string.Format("Linking {0} Records", records.Count));
|
||||
|
||||
using (DiscoDataContext database = new DiscoDataContext())
|
||||
{
|
||||
var populateReferences = Import.GetPopulateRecordReferences(database);
|
||||
|
||||
DateTime lastUpdate = DateTime.Now;
|
||||
foreach (var record in records)
|
||||
{
|
||||
record.PopulateRecord(database, populateReferences);
|
||||
|
||||
if (DateTime.Now.Subtract(lastUpdate).TotalSeconds > 1)
|
||||
{
|
||||
// Update every second
|
||||
this.Status.UpdateStatus((int)Math.Floor((((double)(records.IndexOf(record) + 1) / records.Count) * 80)));
|
||||
lastUpdate = DateTime.Now;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create Session Result
|
||||
ImportDeviceSession session = new ImportDeviceSession()
|
||||
{
|
||||
ImportParseTaskId = this.Status.SessionId,
|
||||
ImportFilename = csvFilename,
|
||||
ImportDevices = records
|
||||
};
|
||||
|
||||
// Set Results to Cache
|
||||
string key = string.Format(Import.ImportParseCacheKey, this.Status.SessionId);
|
||||
HttpRuntime.Cache.Insert(key, session, null, Cache.NoAbsoluteExpiration, TimeSpan.FromMinutes(60), CacheItemPriority.NotRemovable, null);
|
||||
}
|
||||
|
||||
public static ScheduledTaskStatus Run(Stream CsvImport, String CsvFilename)
|
||||
{
|
||||
|
||||
MemoryStream csvStream = new MemoryStream();
|
||||
CsvImport.CopyTo(csvStream);
|
||||
csvStream.Position = 0;
|
||||
|
||||
var task = new ImportParseTask();
|
||||
JobDataMap taskData = new JobDataMap() { { "CsvImport", csvStream }, { "CsvFilename", CsvFilename } };
|
||||
return task.ScheduleTask(taskData);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
using Disco.Data.Repository;
|
||||
using Disco.Models.BI.Device;
|
||||
using Disco.Services.Tasks;
|
||||
using Quartz;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using System.Web;
|
||||
|
||||
namespace Disco.BI.DeviceBI.Importing
|
||||
{
|
||||
public class ImportProcessTask : ScheduledTask
|
||||
{
|
||||
public override string TaskName { get { return "Import Devices - Processing Changes"; } }
|
||||
|
||||
public override bool SingleInstanceTask { get { return false; } }
|
||||
public override bool CancelInitiallySupported { get { return false; } }
|
||||
|
||||
protected override void ExecuteTask()
|
||||
{
|
||||
string importParseTaskId = (string)this.ExecutionContext.JobDetail.JobDataMap["ImportParseTaskId"];
|
||||
|
||||
if (string.IsNullOrWhiteSpace(importParseTaskId))
|
||||
throw new ArgumentNullException("ImportParseTaskId");
|
||||
|
||||
ImportDeviceSession session = Import.GetSession(importParseTaskId);
|
||||
|
||||
if (session == null)
|
||||
throw new InvalidOperationException("The session timed out (60 minutes), try importing again");
|
||||
|
||||
List<ImportDevice> records = session.ImportDevices;
|
||||
int recordsImported = 0;
|
||||
|
||||
this.Status.UpdateStatus(0, "Processing Device Import", "Importing Devices");
|
||||
|
||||
using (DiscoDataContext database = new DiscoDataContext())
|
||||
{
|
||||
var populateReferences = Import.GetPopulateRecordReferences(database);
|
||||
|
||||
DateTime lastUpdate = DateTime.Now;
|
||||
foreach (var record in records)
|
||||
{
|
||||
if (record.ImportRecord(database, populateReferences))
|
||||
recordsImported++;
|
||||
|
||||
if (DateTime.Now.Subtract(lastUpdate).TotalSeconds > 1)
|
||||
{
|
||||
// Update every second
|
||||
this.Status.UpdateStatus((int)Math.Floor((((double)(records.IndexOf(record) + 1) / records.Count) * 100)), string.Format("Importing: {0} ({1} of {2})", record.SerialNumber, records.IndexOf(record) + 1, records.Count));
|
||||
lastUpdate = DateTime.Now;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.Status.SetFinishedMessage(string.Format("Imported {0} of {1} Devices", recordsImported, records.Count));
|
||||
}
|
||||
|
||||
public static ScheduledTaskStatus Run(string ImportParseTaskId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(ImportParseTaskId))
|
||||
throw new ArgumentNullException("ImportParseTaskId");
|
||||
|
||||
var task = new ImportProcessTask();
|
||||
JobDataMap taskData = new JobDataMap() { { "ImportParseTaskId", ImportParseTaskId } };
|
||||
return task.ScheduleTask(taskData);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user