Feature: MS Excel (xlsx) Import/Export

Microsoft Excel files can be used to import/export devices. Several
import bugs were also fixed in the process.

Disco.Services/Devices/Exporting/DeviceExport.cs

@@ -1,4 +1,5 @@
-using Disco.Data.Repository;
+using ClosedXML.Excel;
+using Disco.Data.Repository;
 using Disco.Models.Repository;
 using Disco.Models.Services.Devices.Exporting;
 using Disco.Services.Tasks;
@@ -6,10 +7,10 @@ using Disco.Services.Users;
 using System;
 using System.Collections.Generic;
 using System.ComponentModel.DataAnnotations;
+using System.Data;
 using System.IO;
 using System.Linq;
 using System.Text;
-using System.Threading.Tasks;
 
 namespace Disco.Services.Devices.Exporting
 {
@@ -52,7 +53,7 @@ namespace Disco.Services.Devices.Exporting
                 try
                 {
                     TaskStatus.IgnoreCurrentProcessChanges = true;
-                    TaskStatus.ProgressMultiplier = 20 / 100;
+                    TaskStatus.ProgressMultiplier = (double)20 / 100;
                     TaskStatus.ProgressOffset = 40;
 
                     Interop.ActiveDirectory.ADNetworkLogonDatesUpdateTask.UpdateLastNetworkLogonDates(Database, TaskStatus);
@@ -73,41 +74,23 @@ namespace Disco.Services.Devices.Exporting
 
             TaskStatus.UpdateStatus(80, string.Format("Formatting {0} records for export", records.Count));
 
-            using (StreamWriter writer = new StreamWriter(stream, Encoding.Default, 0x400, true))
+            if (Options.ExcelFormat)
             {
-                // Header
-                writer.Write('"');
-                writer.Write(string.Join("\",\"", metadata.Select(m => m.Item2)));
-                writer.Write('"');
-
-                // Records
-                foreach (var record in records)
-                {
-                    writer.WriteLine();
-                    writer.Write(string.Join(",", metadata.Select(m =>
-                    {
-                        var value = m.Item3(record);
-                        var isString = m.Item4;
-
-                        if (value == null)
-                            return null;
-                        else if (!isString)
-                            return value;
-                        else if (Options.ExcelCsvFormat)
-                            return string.Concat("=\"", value, "\"");
-                        else
-                            return string.Concat("\"", value, "\"");
-                    })));
-                }
+                WriteXlsx(stream, metadata, records);
+            }
+            else
+            {
+                WriteCSV(stream, metadata, records);
             }
 
             stream.Position = 0;
             return new DeviceExportResult()
             {
-                CsvResult = stream,
+                Result = stream,
                 RecordCount = records.Count
             };
         }
+
         public static DeviceExportResult GenerateExport(DiscoDataContext Database, IQueryable<Device> Devices, DeviceExportOptions Options)
         {
             return GenerateExport(Database, Devices, Options, ScheduledTaskMockStatus.Create("Device Export"));
@@ -137,6 +120,57 @@ namespace Disco.Services.Devices.Exporting
             return GenerateExport(Database, Options, ScheduledTaskMockStatus.Create("Device Export"));
         }
 
+        private static void WriteCSV(Stream OutputStream, List<DeviceExportFieldMetadata> Metadata, List<DeviceExportRecord> Records)
+        {
+            using (StreamWriter writer = new StreamWriter(OutputStream, Encoding.Default, 0x400, true))
+            {
+                // Header
+                writer.Write('"');
+                writer.Write(string.Join("\",\"", Metadata.Select(m => m.ColumnName)));
+                writer.Write('"');
+
+                // Records
+                foreach (var record in Records)
+                {
+                    writer.WriteLine();
+                    for (int i = 0; i < Metadata.Count; i++)
+                    {
+                        if (i != 0)
+                        {
+                            writer.Write(',');
+                        }
+                        var value = Metadata[i].Accessor(record);
+                        writer.Write(Metadata[i].CsvEncoder(value));
+                    }
+                }
+            }
+        }
+
+        private static void WriteXlsx(Stream OutputStream, List<DeviceExportFieldMetadata> Metadata, List<DeviceExportRecord> Records)
+        {
+            // Create DataTable
+            var dataTable = new DataTable();
+            foreach (var field in Metadata)
+            {
+                dataTable.Columns.Add(field.ColumnName, field.ValueType);
+            }
+            foreach (var record in Records)
+            {
+                dataTable.Rows.Add(Metadata.Select(m => m.Accessor(record)).ToArray());
+            }
+
+            using (var xlWorkbook = new XLWorkbook())
+            {
+                var sheet = xlWorkbook.AddWorksheet("DeviceExport");
+                var table = sheet.Cell(1, 1).InsertTable(dataTable, "Devices");
+                table.Theme = XLTableTheme.TableStyleMedium2;
+
+                table.Columns().ForEach(c => c.WorksheetColumn().AdjustToContents(2, 15, 30));
+
+                xlWorkbook.SaveAs(OutputStream);
+            }
+        }
+
         private static IEnumerable<DeviceExportRecord> BuildRecords(IQueryable<Device> Devices)
         {
             var deviceDetailHardwareKeys = new List<string> {
@@ -185,93 +219,103 @@ namespace Disco.Services.Devices.Exporting
             });
         }
 
-        /// <returns>Tuple Format: Property Name, Column Name, Property Access, Escape CSV Value?</returns>
-        private static List<Tuple<string, string, Func<DeviceExportRecord, string>, bool>> BuildMetadata(this DeviceExportOptions Options)
+        private static List<DeviceExportFieldMetadata> BuildMetadata(this DeviceExportOptions Options)
         {
-            var allAssessors = BuildRecordAssessors().ToList();
+            var allAssessors = BuildRecordAccessors().ToList();
 
             return typeof(DeviceExportOptions).GetProperties()
                 .Where(p => p.PropertyType == typeof(bool))
-                .Select(p => Tuple.Create(p, (DisplayAttribute)p.GetCustomAttributes(typeof(DisplayAttribute), false).FirstOrDefault()))
-                .Where(p => p.Item2 != null && (bool)p.Item1.GetValue(Options))
+                .Select(p => new
+                {
+                    property = p,
+                    details = (DisplayAttribute)p.GetCustomAttributes(typeof(DisplayAttribute), false).FirstOrDefault()
+                })
+                .Where(p => p.details != null && (bool)p.property.GetValue(Options))
                 .Select(p =>
                 {
-                    var accessor = allAssessors.First(i => i.Item1 == p.Item1.Name);
-                    var columnName = (p.Item2.ShortName == "Device" || p.Item2.ShortName == "Details") ? p.Item2.Name : string.Format("{0} {1}", p.Item2.ShortName, p.Item2.Name);
-                    return Tuple.Create(p.Item1.Name, columnName, accessor.Item2, accessor.Item3);
+                    var fieldMetadata = allAssessors.First(i => i.Name == p.property.Name);
+                    fieldMetadata.ColumnName = (p.details.ShortName == "Device" || p.details.ShortName == "Details") ? p.details.Name : $"{p.details.ShortName} {p.details.Name}";
+                    return fieldMetadata;
                 }).ToList();
         }
 
-        /// <returns>Tuple Format: Property Name, Property Access, Escape CSV Value?</returns>
-        private static IEnumerable<Tuple<string, Func<DeviceExportRecord, string>, bool>> BuildRecordAssessors()
+        private static IEnumerable<DeviceExportFieldMetadata> BuildRecordAccessors()
         {
             const string DateFormat = "yyyy-MM-dd";
             const string DateTimeFormat = DateFormat + " HH:mm:ss";
 
+            Func<object, string> csvStringEncoded = (o) => o == null ? null : $"\"{((string)o).Replace("\"", "\"\"")}\"";
+            Func<object, string> csvToStringEncoded = (o) => o == null ? null : o.ToString();
+            Func<object, string> csvCurrencyEncoded = (o) => ((decimal?)o).HasValue ? ((decimal?)o).Value.ToString("C") : null;
+            Func<object, string> csvDateEncoded = (o) => ((DateTime)o).ToString(DateFormat);
+            Func<object, string> csvDateTimeEncoded = (o) => ((DateTime)o).ToString(DateTimeFormat);
+            Func<object, string> csvNullableDateEncoded = (o) => ((DateTime?)o).HasValue ? csvDateEncoded(o) : null;
+            Func<object, string> csvNullableDateTimeEncoded = (o) => ((DateTime?)o).HasValue ? csvDateTimeEncoded(o) : null;
+
             // Device
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("DeviceSerialNumber", r => r.Device.SerialNumber, true);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("DeviceAssetNumber", r => r.Device.AssetNumber, true);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("DeviceLocation", r => r.Device.Location, true);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("DeviceComputerName", r => r.Device.DeviceDomainId, true);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("DeviceLastNetworkLogon", r => r.Device.LastNetworkLogonDate.HasValue ? r.Device.LastNetworkLogonDate.Value.ToString(DateTimeFormat) : null, false);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("DeviceCreatedDate", r => r.Device.CreatedDate.ToString(DateTimeFormat), false);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("DeviceFirstEnrolledDate", r => r.Device.EnrolledDate.HasValue ? r.Device.EnrolledDate.Value.ToString(DateTimeFormat) : null, false);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("DeviceLastEnrolledDate", r => r.Device.LastEnrolDate.HasValue ? r.Device.LastEnrolDate.Value.ToString(DateTimeFormat) : null, false);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("DeviceAllowUnauthenticatedEnrol", r => r.Device.AllowUnauthenticatedEnrol.ToString(), false);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("DeviceDecommissionedDate", r => r.Device.DecommissionedDate.HasValue ? r.Device.DecommissionedDate.Value.ToString(DateTimeFormat) : null, false);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("DeviceDecommissionedReason", r => r.Device.DecommissionReason.HasValue ? r.Device.DecommissionReason.Value.ToString() : null, true);
+            yield return new DeviceExportFieldMetadata("DeviceSerialNumber", typeof(string), r => r.Device.SerialNumber, csvStringEncoded);
+            yield return new DeviceExportFieldMetadata("DeviceAssetNumber", typeof(string), r => r.Device.AssetNumber, csvStringEncoded);
+            yield return new DeviceExportFieldMetadata("DeviceLocation", typeof(string), r => r.Device.Location, csvStringEncoded);
+            yield return new DeviceExportFieldMetadata("DeviceComputerName", typeof(string), r => r.Device.DeviceDomainId, csvStringEncoded);
+            yield return new DeviceExportFieldMetadata("DeviceLastNetworkLogon", typeof(DateTime), r => r.Device.LastNetworkLogonDate, csvNullableDateTimeEncoded);
+            yield return new DeviceExportFieldMetadata("DeviceCreatedDate", typeof(DateTime), r => r.Device.CreatedDate, csvDateTimeEncoded);
+            yield return new DeviceExportFieldMetadata("DeviceFirstEnrolledDate", typeof(DateTime), r => r.Device.EnrolledDate, csvNullableDateTimeEncoded);
+            yield return new DeviceExportFieldMetadata("DeviceLastEnrolledDate", typeof(DateTime), r => r.Device.LastEnrolDate, csvNullableDateTimeEncoded);
+            yield return new DeviceExportFieldMetadata("DeviceAllowUnauthenticatedEnrol", typeof(bool), r => r.Device.AllowUnauthenticatedEnrol, csvToStringEncoded);
+            yield return new DeviceExportFieldMetadata("DeviceDecommissionedDate", typeof(DateTime), r => r.Device.DecommissionedDate, csvNullableDateTimeEncoded);
+            yield return new DeviceExportFieldMetadata("DeviceDecommissionedReason", typeof(string), r => r.Device.DecommissionReason, csvToStringEncoded);
 
             // Details
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("DetailLanMacAddress", r => r.DeviceDetails.Where(dd => dd.Key == DeviceDetail.HardwareKeyLanMacAddress).Select(dd => dd.Value).FirstOrDefault(), true);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("DetailWLanMacAddress", r => r.DeviceDetails.Where(dd => dd.Key == DeviceDetail.HardwareKeyWLanMacAddress).Select(dd => dd.Value).FirstOrDefault(), true);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("DetailACAdapter", r => r.DeviceDetails.Where(dd => dd.Key == DeviceDetail.HardwareKeyACAdapter).Select(dd => dd.Value).FirstOrDefault(), true);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("DetailBattery", r => r.DeviceDetails.Where(dd => dd.Key == DeviceDetail.HardwareKeyBattery).Select(dd => dd.Value).FirstOrDefault(), true);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("DetailKeyboard", r => r.DeviceDetails.Where(dd => dd.Key == DeviceDetail.HardwareKeyKeyboard).Select(dd => dd.Value).FirstOrDefault(), true);
+            yield return new DeviceExportFieldMetadata("DetailLanMacAddress", typeof(string), r => r.DeviceDetails.Where(dd => dd.Key == DeviceDetail.HardwareKeyLanMacAddress).Select(dd => dd.Value).FirstOrDefault(), csvStringEncoded);
+            yield return new DeviceExportFieldMetadata("DetailWLanMacAddress", typeof(string), r => r.DeviceDetails.Where(dd => dd.Key == DeviceDetail.HardwareKeyWLanMacAddress).Select(dd => dd.Value).FirstOrDefault(), csvStringEncoded);
+            yield return new DeviceExportFieldMetadata("DetailACAdapter", typeof(string), r => r.DeviceDetails.Where(dd => dd.Key == DeviceDetail.HardwareKeyACAdapter).Select(dd => dd.Value).FirstOrDefault(), csvStringEncoded);
+            yield return new DeviceExportFieldMetadata("DetailBattery", typeof(string), r => r.DeviceDetails.Where(dd => dd.Key == DeviceDetail.HardwareKeyBattery).Select(dd => dd.Value).FirstOrDefault(), csvStringEncoded);
+            yield return new DeviceExportFieldMetadata("DetailKeyboard", typeof(string), r => r.DeviceDetails.Where(dd => dd.Key == DeviceDetail.HardwareKeyKeyboard).Select(dd => dd.Value).FirstOrDefault(), csvStringEncoded);
 
             // Model
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("ModelId", r => r.ModelId.HasValue ? r.ModelId.Value.ToString() : null, false);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("ModelDescription", r => r.ModelDescription, true);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("ModelManufacturer", r => r.ModelManufacturer, true);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("ModelModel", r => r.ModelModel, true);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("ModelType", r => r.ModelType, true);
+            yield return new DeviceExportFieldMetadata("ModelId", typeof(int), r => r.ModelId, csvToStringEncoded);
+            yield return new DeviceExportFieldMetadata("ModelDescription", typeof(string), r => r.ModelDescription, csvStringEncoded);
+            yield return new DeviceExportFieldMetadata("ModelManufacturer", typeof(string), r => r.ModelManufacturer, csvStringEncoded);
+            yield return new DeviceExportFieldMetadata("ModelModel", typeof(string), r => r.ModelModel, csvStringEncoded);
+            yield return new DeviceExportFieldMetadata("ModelType", typeof(string), r => r.ModelType, csvStringEncoded);
 
             // Batch
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("BatchId", r => r.BatchId.HasValue ? r.BatchId.Value.ToString() : null, false);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("BatchName", r => r.BatchName, true);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("BatchPurchaseDate", r => r.BatchPurchaseDate.HasValue ? r.BatchPurchaseDate.Value.ToString(DateFormat) : null, false);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("BatchSupplier", r => r.BatchSupplier, true);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("BatchUnitCost", r => r.BatchUnitCost.HasValue ? r.BatchUnitCost.ToString() : null, false);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("BatchWarrantyValidUntilDate", r => r.BatchWarrantyValidUntilDate.HasValue ? r.BatchWarrantyValidUntilDate.Value.ToString(DateFormat) : null, false);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("BatchInsuredDate", r => r.BatchInsuredDate.HasValue ? r.BatchInsuredDate.Value.ToString(DateFormat) : null, false);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("BatchInsuranceSupplier", r => r.BatchInsuranceSupplier, true);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("BatchInsuredUntilDate", r => r.BatchInsuredUntilDate.HasValue ? r.BatchInsuredUntilDate.Value.ToString(DateFormat) : null, false);
+            yield return new DeviceExportFieldMetadata("BatchId", typeof(int), r => r.BatchId, csvToStringEncoded);
+            yield return new DeviceExportFieldMetadata("BatchName", typeof(string), r => r.BatchName, csvStringEncoded);
+            yield return new DeviceExportFieldMetadata("BatchPurchaseDate", typeof(DateTime), r => r.BatchPurchaseDate, csvNullableDateEncoded);
+            yield return new DeviceExportFieldMetadata("BatchSupplier", typeof(string), r => r.BatchSupplier, csvStringEncoded);
+            yield return new DeviceExportFieldMetadata("BatchUnitCost", typeof(decimal), r => r.BatchUnitCost, csvCurrencyEncoded);
+            yield return new DeviceExportFieldMetadata("BatchWarrantyValidUntilDate", typeof(DateTime), r => r.BatchWarrantyValidUntilDate, csvNullableDateEncoded);
+            yield return new DeviceExportFieldMetadata("BatchInsuredDate", typeof(DateTime), r => r.BatchInsuredDate, csvNullableDateEncoded);
+            yield return new DeviceExportFieldMetadata("BatchInsuranceSupplier", typeof(string), r => r.BatchInsuranceSupplier, csvStringEncoded);
+            yield return new DeviceExportFieldMetadata("BatchInsuredUntilDate", typeof(DateTime), r => r.BatchInsuredUntilDate, csvNullableDateEncoded);
 
             // Profile
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("ProfileId", r => r.ProfileId.ToString(), false);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("ProfileName", r => r.ProfileName, true);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("ProfileShortName", r => r.ProfileShortName, true);
+            yield return new DeviceExportFieldMetadata("ProfileId", typeof(int), r => r.ProfileId, csvToStringEncoded);
+            yield return new DeviceExportFieldMetadata("ProfileName", typeof(string), r => r.ProfileName, csvStringEncoded);
+            yield return new DeviceExportFieldMetadata("ProfileShortName", typeof(string), r => r.ProfileShortName, csvStringEncoded);
 
             // User
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("AssignedUserId", r => r.AssignedUser != null ? r.AssignedUser.UserId : null, true);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("AssignedUserDate", r => r.DeviceUserAssignment != null ? r.DeviceUserAssignment.AssignedDate.ToString(DateTimeFormat) : null, false);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("AssignedUserDisplayName", r => r.AssignedUser != null ? r.AssignedUser.DisplayName : null, true);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("AssignedUserSurname", r => r.AssignedUser != null ? r.AssignedUser.Surname : null, true);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("AssignedUserGivenName", r => r.AssignedUser != null ? r.AssignedUser.GivenName : null, true);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("AssignedUserPhoneNumber", r => r.AssignedUser != null ? r.AssignedUser.PhoneNumber : null, true);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("AssignedUserEmailAddress", r => r.AssignedUser != null ? r.AssignedUser.EmailAddress : null, true);
+            yield return new DeviceExportFieldMetadata("AssignedUserId", typeof(string), r => r.AssignedUser?.UserId, csvStringEncoded);
+            yield return new DeviceExportFieldMetadata("AssignedUserDate", typeof(DateTime), r => r.DeviceUserAssignment?.AssignedDate, csvNullableDateTimeEncoded);
+            yield return new DeviceExportFieldMetadata("AssignedUserDisplayName", typeof(string), r => r.AssignedUser?.DisplayName, csvStringEncoded);
+            yield return new DeviceExportFieldMetadata("AssignedUserSurname", typeof(string), r => r.AssignedUser?.Surname, csvStringEncoded);
+            yield return new DeviceExportFieldMetadata("AssignedUserGivenName", typeof(string), r => r.AssignedUser?.GivenName, csvStringEncoded);
+            yield return new DeviceExportFieldMetadata("AssignedUserPhoneNumber", typeof(string), r => r.AssignedUser?.PhoneNumber, csvStringEncoded);
+            yield return new DeviceExportFieldMetadata("AssignedUserEmailAddress", typeof(string), r => r.AssignedUser?.EmailAddress, csvStringEncoded);
 
             // Jobs
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("JobsTotalCount", r => r.JobsTotalCount.ToString(), false);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("JobsOpenCount", r => r.JobsOpenCount.ToString(), false);
+            yield return new DeviceExportFieldMetadata("JobsTotalCount", typeof(int), r => r.JobsTotalCount, csvToStringEncoded);
+            yield return new DeviceExportFieldMetadata("JobsOpenCount", typeof(int), r => r.JobsOpenCount, csvToStringEncoded);
 
             // Attachments
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("AttachmentsCount", r => r.AttachmentsCount.ToString(), false);
+            yield return new DeviceExportFieldMetadata("AttachmentsCount", typeof(int), r => r.AttachmentsCount, csvToStringEncoded);
 
             // Certificates
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("CertificateName", r => r.DeviceCertificate != null ? r.DeviceCertificate.Name : null, true);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("CertificateAllocatedDate", r => r.DeviceCertificate != null && r.DeviceCertificate.AllocatedDate.HasValue ? r.DeviceCertificate.AllocatedDate.Value.ToString(DateTimeFormat) : null, false);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("CertificateExpirationDate", r => r.DeviceCertificate != null && r.DeviceCertificate.ExpirationDate.HasValue ? r.DeviceCertificate.ExpirationDate.Value.ToString(DateTimeFormat) : null, false);
-            yield return new Tuple<string, Func<DeviceExportRecord, string>, bool>("CertificateProviderId", r => r.DeviceCertificate != null ? r.DeviceCertificate.ProviderId : null, true);
+            yield return new DeviceExportFieldMetadata("CertificateName", typeof(string), r => r.DeviceCertificate?.Name, csvStringEncoded);
+            yield return new DeviceExportFieldMetadata("CertificateAllocatedDate", typeof(DateTime), r => r.DeviceCertificate?.AllocatedDate, csvNullableDateTimeEncoded);
+            yield return new DeviceExportFieldMetadata("CertificateExpirationDate", typeof(DateTime), r => r.DeviceCertificate?.ExpirationDate, csvNullableDateTimeEncoded);
+            yield return new DeviceExportFieldMetadata("CertificateProviderId", typeof(string), r => r.DeviceCertificate?.ProviderId, csvStringEncoded);
         }
 
     }

Disco.Services/Devices/Importing/BaseDeviceImportContext.cs

@@ -0,0 +1,142 @@
+using Disco.Models.Services.Devices.Importing;
+using Disco.Services.Devices.Importing.Fields;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Disco.Services.Devices.Importing
+{
+    public abstract class BaseDeviceImportContext : IDeviceImportContext
+    {
+        private static Lazy<Dictionary<DeviceImportFieldTypes, Type>> FieldHandlers = new Lazy<Dictionary<DeviceImportFieldTypes, Type>>(() =>
+        {
+            return new Dictionary<DeviceImportFieldTypes, Type>()
+            {
+                { DeviceImportFieldTypes.DeviceSerialNumber, typeof(DeviceSerialNumberImportField) },
+                { DeviceImportFieldTypes.DeviceAssetNumber, typeof(DeviceAssetNumberImportField) },
+                { DeviceImportFieldTypes.DeviceLocation, typeof(DeviceLocationImportField) },
+                { DeviceImportFieldTypes.DeviceAllowUnauthenticatedEnrol, typeof(DeviceAllowUnauthenticatedEnrolImportField) },
+                { DeviceImportFieldTypes.DeviceDecommissionedDate, typeof(DeviceDecommissionedDateImportField) },
+                { DeviceImportFieldTypes.DeviceDecommissionedReason, typeof(DeviceDecommissionedReasonImportField) },
+
+                { DeviceImportFieldTypes.DetailLanMacAddress, typeof(DetailLanMacAddressImportField) },
+                { DeviceImportFieldTypes.DetailWLanMacAddress, typeof(DetailWLanMacAddressImportField) },
+                { DeviceImportFieldTypes.DetailACAdapter, typeof(DetailACAdapterImportField) },
+                { DeviceImportFieldTypes.DetailBattery, typeof(DetailBatteryImportField) },
+                { DeviceImportFieldTypes.DetailKeyboard, typeof(DetailKeyboardImportField) },
+
+                { DeviceImportFieldTypes.ModelId, typeof(ModelIdImportField) },
+
+                { DeviceImportFieldTypes.BatchId, typeof(BatchIdImportField) },
+
+                { DeviceImportFieldTypes.ProfileId, typeof(ProfileIdImportField) },
+
+                { DeviceImportFieldTypes.AssignedUserId, typeof(AssignedUserIdImportField) }
+            };
+        });
+
+        private List<DeviceImportColumn> columns;
+        private Dictionary<DeviceImportFieldTypes, DeviceImportColumn> columnsByType;
+
+        public string SessionId { get; }
+        public string Filename { get; }
+        public string DatasetName { get; private set; }
+
+        public abstract int RecordCount { get; }
+
+        public List<IDeviceImportRecord> Records { get; set; }
+        public int AffectedRecords { get; set; }
+
+        public int ColumnCount { get { return columns.Count; } }
+        public IEnumerable<IDeviceImportColumn> Columns
+        {
+            get
+            {
+                if (columns == null)
+                    throw new ArgumentNullException(nameof(columns));
+
+                return columns;
+            }
+        }
+
+        public BaseDeviceImportContext(string Filename)
+        {
+            SessionId = Guid.NewGuid().ToString("D");
+            this.Filename = DatasetName = string.IsNullOrWhiteSpace(Filename) ? "<File Name Not Specified>" : Filename;
+        }
+
+        public abstract IDeviceImportDataReader GetDataReader();
+
+        public IDeviceImportColumn GetColumn(int Index)
+        {
+            if (columns == null)
+                throw new ArgumentNullException(nameof(columns));
+
+            return columns[Index];
+        }
+
+        public void SetColumnType(int Index, DeviceImportFieldTypes Type)
+        {
+            if (columns == null)
+                throw new ArgumentNullException(nameof(columns));
+
+            var column = columns[Index];
+
+            if (column.Type == Type)
+            {
+                return; // No change
+            }
+
+            if (column.Type != DeviceImportFieldTypes.IgnoreColumn)
+            {
+                columnsByType.Remove(column.Type);
+            }
+
+            column.Type = Type;
+
+            if (Type == DeviceImportFieldTypes.IgnoreColumn)
+            {
+                column.Handler = null;
+            }
+            else
+            {
+                columnsByType[Type] = column;
+                column.Handler = FieldHandlers.Value[Type];
+            }
+        }
+
+        public int? GetColumnByType(DeviceImportFieldTypes FieldType)
+        {
+            if (columnsByType == null)
+                throw new ArgumentNullException(nameof(columnsByType));
+
+            DeviceImportColumn column;
+            if (columnsByType.TryGetValue(FieldType, out column))
+            {
+                return column.Index;
+            }
+            else
+            {
+                return null;
+            }
+        }
+
+        protected void SetDatasetName(string DatasetName)
+        {
+            this.DatasetName = DatasetName;
+        }
+
+        protected void SetColumns(IEnumerable<DeviceImportColumn> Columns)
+        {
+            columns = Columns.ToList();
+            columnsByType = columns
+                .Where(c => c.Type != DeviceImportFieldTypes.IgnoreColumn)
+                .ToDictionary(c => c.Type);
+        }
+
+        public IEnumerable<KeyValuePair<DeviceImportFieldTypes, Type>> GetFieldHandlers()
+        {
+            return FieldHandlers.Value;
+        }
+    }
+}

Disco.Services/Devices/Importing/CsvDeviceImportContext.cs

@@ -0,0 +1,56 @@
+using Disco.Models.Services.Devices.Importing;
+using LumenWorks.Framework.IO.Csv;
+using System.Collections.Generic;
+using System.Data;
+using System.IO;
+using System.Linq;
+
+namespace Disco.Services.Devices.Importing
+{
+    public class CsvDeviceImportContext : BaseDeviceImportContext
+    {
+        private List<string[]> rawData;
+        private bool hasHeaderRow;
+
+        public CsvDeviceImportContext(string Filename, bool HasHeader, Stream CsvStream)
+            : base(Filename)
+        {
+            hasHeaderRow = HasHeader;
+
+            ParseCsv(CsvStream);
+        }
+
+        public override int RecordCount
+        {
+            get
+            {
+                return rawData.Count;
+            }
+        }
+
+        public override IDeviceImportDataReader GetDataReader()
+        {
+            return new CsvDeviceImportDataReader(this, rawData, hasHeaderRow);
+        }
+
+        private void ParseCsv(Stream CsvStream)
+        {
+            using (TextReader csvTextReader = new StreamReader(CsvStream))
+            {
+                using (CsvReader csvReader = new CsvReader(csvTextReader, hasHeaderRow))
+                {
+                    csvReader.DefaultParseErrorAction = ParseErrorAction.ThrowException;
+                    csvReader.MissingFieldAction = MissingFieldAction.ReplaceByNull;
+
+                    rawData = csvReader.ToList();
+                    SetColumns(csvReader.GetFieldHeaders().Select((h, i) => new DeviceImportColumn()
+                    {
+                        Index = i,
+                        Name = h,
+                        Type = DeviceImportFieldTypes.IgnoreColumn
+                    }));
+                }
+            }
+        }
+    }
+}

Disco.Services/Devices/Importing/CsvDeviceImportDataReader.cs

@@ -0,0 +1,222 @@
+using Disco.Models.Services.Devices.Importing;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+
+namespace Disco.Services.Devices.Importing
+{
+    public class CsvDeviceImportDataReader : IDeviceImportDataReader
+    {
+        private static string[] TrueValues = { "true", "1", "yes", "-1", "on" };
+        private static string[] FalseValues = { "false", "0", "no", "off" };
+
+        private CsvDeviceImportContext context;
+        private List<string[]> rawData;
+        private int currentRowIndex;
+        private int rowOffset;
+        private string[] currentRow;
+
+        public int Index { get { return currentRowIndex; } }
+
+        public CsvDeviceImportDataReader(CsvDeviceImportContext Context, List<string[]> RawData, bool HasHeaderRow)
+        {
+            context = Context;
+            rawData = RawData;
+            currentRowIndex = 0;
+            rowOffset = HasHeaderRow ? 1 : 0;
+        }
+
+        public void Reset()
+        {
+            currentRowIndex = 0;
+            currentRow = null;
+        }
+
+        public bool Read()
+        {
+            if (++currentRowIndex >= rawData.Count)
+            {
+                currentRowIndex--;
+                return false;
+            }
+
+            currentRow = rawData[currentRowIndex];
+            return true;
+        }
+
+        public int GetRowNumber(int Index)
+        {
+            return Index + rowOffset;
+        }
+
+        public string GetString(int ColumnIndex)
+        {
+            if (currentRow == null)
+                throw new InvalidOperationException($"{nameof(CsvDeviceImportDataReader.Read)} must be called before retrieving values");
+
+            var value = currentRow[ColumnIndex];
+
+            if (value.Length == 0)
+                return null;
+
+            return value;
+        }
+
+        public IEnumerable<string> GetStrings(int ColumnIndex)
+        {
+            return rawData.Select(r => r[ColumnIndex]);
+        }
+
+        public bool TryGetNullableInt(int ColumnIndex, out int? value)
+        {
+            if (currentRow == null)
+                throw new InvalidOperationException($"{nameof(CsvDeviceImportDataReader.Read)} must be called before retrieving values");
+
+            return TryGetNullableInt(currentRow[ColumnIndex], out value);
+        }
+
+        public bool TryGetNullableBool(int ColumnIndex, out bool? value)
+        {
+            if (currentRow == null)
+                throw new InvalidOperationException($"{nameof(CsvDeviceImportDataReader.Read)} must be called before retrieving values");
+
+            return TryGetNullableBool(currentRow[ColumnIndex], out value);
+        }
+
+        public bool TryGetNullableDateTime(int ColumnIndex, out DateTime? value)
+        {
+            if (currentRow == null)
+                throw new InvalidOperationException($"{nameof(CsvDeviceImportDataReader.Read)} must be called before retrieving values");
+
+            return TryGetNullableDateTime(currentRow[ColumnIndex], out value);
+        }
+
+        public bool TestAllNotEmpty(int ColumnIndex)
+        {
+            return GetStrings(ColumnIndex).All(s => !string.IsNullOrWhiteSpace(s));
+        }
+
+        public bool TestAllNullableInt(int ColumnIndex)
+        {
+            return rawData.Select(r => r[ColumnIndex])
+                .All(c =>
+                {
+                    int? value;
+                    return TryGetNullableInt(c, out value);
+                });
+        }
+
+        public bool TestAllInt(int ColumnIndex)
+        {
+            return rawData.Select(r => r[ColumnIndex])
+                .All(c =>
+                {
+                    int? value;
+                    return TryGetNullableInt(c, out value) && value.HasValue;
+                });
+        }
+
+        public bool TestAllNullableBool(int ColumnIndex)
+        {
+            return rawData.Select(r => r[ColumnIndex])
+                .All(c =>
+                {
+                    bool? value;
+                    return TryGetNullableBool(c, out value);
+                });
+        }
+
+        public bool TestAllNullableDateTime(int ColumnIndex)
+        {
+            return rawData.Select(r => r[ColumnIndex])
+                .All(c =>
+                {
+                    DateTime? value;
+                    return TryGetNullableDateTime(c, out value);
+                });
+        }
+
+        public void Dispose()
+        {
+            // Nothing to dispose
+        }
+
+        private bool TryGetNullableDateTime(string content, out DateTime? value)
+        {
+            if (string.IsNullOrWhiteSpace(content))
+            {
+                value = null;
+                return true;
+            }
+            else
+            {
+                content = content.Trim();
+
+                DateTime valueDateTime;
+                if (DateTime.TryParse(content, CultureInfo.CurrentCulture, DateTimeStyles.AssumeLocal, out valueDateTime))
+                {
+                    value = valueDateTime;
+                    return true;
+                }
+                else
+                {
+                    value = null;
+                    return false;
+                }
+            }
+        }
+
+        private bool TryGetNullableBool(string content, out bool? value)
+        {
+            if (string.IsNullOrWhiteSpace(content))
+            {
+                value = null;
+                return true;
+            }
+            else
+            {
+                content = content.Trim();
+
+                if (TrueValues.Contains(content, StringComparer.OrdinalIgnoreCase))
+                {
+                    value = true;
+                    return true;
+                }
+                else if (FalseValues.Contains(content, StringComparer.OrdinalIgnoreCase))
+                {
+                    value = false;
+                    return true;
+                }
+                else
+                {
+                    value = null;
+                    return false;
+                }
+            }
+        }
+
+        private bool TryGetNullableInt(string content, out int? value)
+        {
+            if (string.IsNullOrWhiteSpace(content))
+            {
+                value = null;
+                return true;
+            }
+            else
+            {
+                int intValue;
+                if (int.TryParse(content, out intValue))
+                {
+                    value = intValue;
+                    return true;
+                }
+                else
+                {
+                    value = null;
+                    return false;
+                }
+            }
+        }
+    }
+}

Disco.Services/Devices/Importing/DeviceImport.cs

@@ -3,170 +3,143 @@ using Disco.Models.Repository;
 using Disco.Models.Services.Devices.Importing;
 using Disco.Services.Devices.Importing.Fields;
 using Disco.Services.Tasks;
-using LumenWorks.Framework.IO.Csv;
 using System;
 using System.Collections.Generic;
 using System.Data;
 using System.IO;
 using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace Disco.Services.Devices.Importing
 {
     public static class DeviceImport
     {
 
-        internal static Lazy<Dictionary<DeviceImportFieldTypes, Type>> FieldHandlers = new Lazy<Dictionary<DeviceImportFieldTypes, Type>>(() =>
-        {
-            return new Dictionary<DeviceImportFieldTypes, Type>()
-            {
-                { DeviceImportFieldTypes.DeviceSerialNumber, typeof(DeviceSerialNumberImportField) },
-                { DeviceImportFieldTypes.DeviceAssetNumber, typeof(DeviceAssetNumberImportField) },
-                { DeviceImportFieldTypes.DeviceLocation, typeof(DeviceLocationImportField) },
-                { DeviceImportFieldTypes.DeviceAllowUnauthenticatedEnrol, typeof(DeviceAllowUnauthenticatedEnrolImportField) },
-                { DeviceImportFieldTypes.DeviceDecommissionedDate, typeof(DeviceDecommissionedDateImportField) },
-                { DeviceImportFieldTypes.DeviceDecommissionedReason, typeof(DeviceDecommissionedReasonImportField) },
-
-                { DeviceImportFieldTypes.DetailLanMacAddress, typeof(DetailLanMacAddressImportField) },
-                { DeviceImportFieldTypes.DetailWLanMacAddress, typeof(DetailWLanMacAddressImportField) },
-                { DeviceImportFieldTypes.DetailACAdapter, typeof(DetailACAdapterImportField) },
-                { DeviceImportFieldTypes.DetailBattery, typeof(DetailBatteryImportField) },
-                { DeviceImportFieldTypes.DetailKeyboard, typeof(DetailKeyboardImportField) },
-
-                { DeviceImportFieldTypes.ModelId, typeof(ModelIdImportField) },
-
-                { DeviceImportFieldTypes.BatchId, typeof(BatchIdImportField) },
-
-                { DeviceImportFieldTypes.ProfileId, typeof(ProfileIdImportField) },
-
-                { DeviceImportFieldTypes.AssignedUserId, typeof(AssignedUserIdImportField) }
-            };
-        });
-
-        public static DeviceImportContext BeginImport(DiscoDataContext Database, string Filename, bool HasHeader, Stream FileContent)
+        public static IDeviceImportContext BeginImport(DiscoDataContext Database, string Filename, bool HasHeader, Stream FileContent)
         {
             if (FileContent == null)
-                throw new ArgumentNullException("FileContent");
+                throw new ArgumentNullException(nameof(FileContent));
 
-            if (string.IsNullOrWhiteSpace(Filename))
-                Filename = "<None Specified>";
+            if (FileContent.Length == 0)
+                throw new ArgumentNullException(nameof(FileContent));
 
-            DeviceImportContext context;
-            List<Tuple<string, DeviceImportFieldTypes>> header;
-            List<string[]> rawData;
+            IDeviceImportContext context;
 
-            using (TextReader csvTextReader = new StreamReader(FileContent))
+            if (Filename?.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase) ?? false)
             {
-                using (CsvReader csvReader = new CsvReader(csvTextReader, HasHeader))
-                {
-                    csvReader.DefaultParseErrorAction = ParseErrorAction.ThrowException;
-                    csvReader.MissingFieldAction = MissingFieldAction.ReplaceByNull;
-
-                    rawData = csvReader.ToList();
-                    header = csvReader.GetFieldHeaders().Select(h => Tuple.Create(h, DeviceImportFieldTypes.IgnoreColumn)).ToList();
-                }
+                // Use Xlsx Context
+                context = new XlsxDeviceImportContext(Filename, HasHeader, FileContent);
+            }
+            else
+            {
+                // Use/Default Csv Context
+                context = new CsvDeviceImportContext(Filename, HasHeader, FileContent);
             }
-
-            context = new DeviceImportContext(Filename, header, rawData);
 
             context.GuessHeaderTypes(Database);
 
             return context;
         }
 
-        private static void GuessHeaderTypes(this DeviceImportContext Context, DiscoDataContext Database)
+        private static void GuessHeaderTypes(this IDeviceImportContext Context, DiscoDataContext Database)
         {
-            FieldHandlers.Value.ToList().ForEach(h =>
+            using (var dataReader = Context.GetDataReader())
             {
-                var instance = (DeviceImportFieldBase)Activator.CreateInstance(h.Value);
-                var column = instance.GuessHeader(Database, Context);
-                if (column.HasValue)
-                    Context.Header[column.Value] = Tuple.Create(Context.Header[column.Value].Item1, instance.FieldType);
-            });
+                foreach (var fieldHandler in Context.GetFieldHandlers())
+                {
+                    dataReader.Reset();
+
+                    var instance = (DeviceImportFieldBase)Activator.CreateInstance(fieldHandler.Value);
+                    var column = instance.GuessColumn(Database, Context, dataReader);
+
+                    if (column.HasValue)
+                        Context.SetColumnType(column.Value, instance.FieldType);
+                }
+            }
         }
 
-        public static void UpdateHeaderTypes(this DeviceImportContext Context, List<DeviceImportFieldTypes> HeaderTypes)
+        public static void UpdateColumnTypes(this IDeviceImportContext Context, List<DeviceImportFieldTypes> ColumnTypes)
         {
-            if (HeaderTypes == null)
-                throw new ArgumentNullException("HeaderTypes");
+            if (ColumnTypes == null)
+                throw new ArgumentNullException(nameof(ColumnTypes));
 
-            if (HeaderTypes.Count != Context.Header.Count)
-                throw new ArgumentException("The number of Header Types supplied does not match the number of Headers", "HeaderTypes");
+            if (ColumnTypes.Count != Context.ColumnCount)
+                throw new ArgumentException("The number of Column Types supplied does not match the number of Headers", nameof(ColumnTypes));
 
-            if (!HeaderTypes.Any(h => h == DeviceImportFieldTypes.DeviceSerialNumber))
-                throw new ArgumentException("At least one column must be the Device Serial Number", "HeaderTypes");
+            if (!ColumnTypes.Any(h => h == DeviceImportFieldTypes.DeviceSerialNumber))
+                throw new ArgumentException("At least one column must be the Device Serial Number", nameof(ColumnTypes));
 
-            if (HeaderTypes.Where(h => h != DeviceImportFieldTypes.IgnoreColumn).GroupBy(h => h, (k, i) => Tuple.Create(k, i.Count())).Any(g => g.Item2 > 1))
-                throw new ArgumentException("Column types can only be specified once for each type", "HeaderTypes");
+            if (ColumnTypes.Where(h => h != DeviceImportFieldTypes.IgnoreColumn).GroupBy(h => h, (k, i) => Tuple.Create(k, i.Count())).Any(g => g.Item2 > 1))
+                throw new ArgumentException("Column types can only be specified once for each type", nameof(ColumnTypes));
 
-            Context.Header = Context.Header.Zip(HeaderTypes, (h, ht) => Tuple.Create(h.Item1, ht)).ToList();
+            for (int columnIndex = 0; columnIndex < Context.ColumnCount; columnIndex++)
+            {
+                Context.SetColumnType(columnIndex, ColumnTypes[columnIndex]);
+            }
         }
 
-        public static void ParseRecords(this DeviceImportContext Context, DiscoDataContext Database, IScheduledTaskStatus Status)
+        public static void ParseRecords(this IDeviceImportContext Context, DiscoDataContext Database, IScheduledTaskStatus Status)
         {
-            if (Context.Header == null)
-                throw new InvalidOperationException("The Import Context has not been initialized");
+            if (Context.ColumnCount == 0)
+                throw new InvalidOperationException("No columns were found");
 
-            if (Context.Header.Count == 0)
-                throw new InvalidOperationException("No Headers were found");
+            if (!Context.GetColumnByType(DeviceImportFieldTypes.DeviceSerialNumber).HasValue)
+                throw new ArgumentException("At least one column must be the Device Serial Number", nameof(Context.Columns));
 
-            if (!Context.Header.Any(h => h.Item2 == DeviceImportFieldTypes.DeviceSerialNumber))
-                throw new ArgumentException("At least one column must be the Device Serial Number", "Header");
-
-            if (Context.RawData == null || Context.RawData.Count == 0)
-                throw new ArgumentException("No data was found in the import file", "RawData");
+            if (Context.RecordCount == 0)
+                throw new ArgumentException("No data was found in the import file", nameof(Context.RecordCount));
 
             IDeviceImportCache cache;
-            if (Context.RawData.Count > 20)
+            if (Context.RecordCount > 20)
                 cache = new DeviceImportInMemoryCache(Database);
             else
                 cache = new DeviceImportDatabaseCache(Database);
 
-            Context.HeaderDeviceSerialNumberIndex = Context.Header.IndexOf(Context.Header.First(h => h.Item2 == DeviceImportFieldTypes.DeviceSerialNumber));
-            Context.ParsedHeaders = Context.Header
-                .Select((h, i) => Tuple.Create(h.Item1, h.Item2, i))
-                .Where(h => h.Item2 != DeviceImportFieldTypes.IgnoreColumn)
-                .Select(h => new Tuple<string, DeviceImportFieldTypes, Func<string[], string>, Type>(h.Item1, h.Item2, (f) => f[h.Item3], DeviceImport.FieldHandlers.Value[h.Item2]))
+            var deviceSerialNumberIndex = Context.GetColumnByType(DeviceImportFieldTypes.DeviceSerialNumber).Value;
+            var columns = Context.Columns
+                .Where(h => h.Type != DeviceImportFieldTypes.IgnoreColumn)
                 .ToList();
 
             Status.UpdateStatus(0, "Parsing Import Records", "Starting...");
 
-            Context.Records = Context.RawData.Select((d, recordIndex) =>
+            var records = new List<IDeviceImportRecord>();
+
+            using (var dataReader = Context.GetDataReader())
             {
-                string deviceSerialNumber = Fields.DeviceSerialNumberImportField.ParseRawDeviceSerialNumber(d[Context.HeaderDeviceSerialNumberIndex]);
-
-                Status.UpdateStatus(((double)recordIndex / Context.RawData.Count) * 100, string.Format("Parsing: {0}", deviceSerialNumber));
-
-                Device existingDevice = null;
-                if (Fields.DeviceSerialNumberImportField.IsDeviceSerialNumberValid(deviceSerialNumber))
-                    existingDevice = cache.Devices.FirstOrDefault(device => device.SerialNumber == deviceSerialNumber);
-
-                var values = Context.ParsedHeaders
-                    .ToDictionary(k => k.Item2, k => k.Item3(d));
-
-                var fields = Context.ParsedHeaders.Select(h =>
+                while (dataReader.Read())
                 {
-                    var f = (DeviceImportFieldBase)Activator.CreateInstance(h.Item4);
-                    f.Parse(Database, cache, Context, recordIndex, deviceSerialNumber, existingDevice, values, h.Item3(d));
-                    return f;
-                }).ToList();
+                    string deviceSerialNumber = DeviceSerialNumberImportField.ParseRawDeviceSerialNumber(dataReader.GetString(deviceSerialNumberIndex));
 
-                EntityState recordAction;
-                if (fields.Any(f => !f.FieldAction.HasValue))
-                    recordAction = EntityState.Detached;
-                else if (existingDevice == null)
-                    recordAction = EntityState.Added;
-                else if (fields.Any(f => f.FieldAction == EntityState.Modified))
-                    recordAction = EntityState.Modified;
-                else
-                    recordAction = EntityState.Unchanged;
+                    Status.UpdateStatus(((double)dataReader.Index / Context.RecordCount) * 100, string.Format("Parsing: {0}", deviceSerialNumber));
 
-                return new DeviceImportRecord(deviceSerialNumber, fields, recordAction);
-            }).Cast<IDeviceImportRecord>().ToList();
+                    Device existingDevice = null;
+                    if (DeviceSerialNumberImportField.IsDeviceSerialNumberValid(deviceSerialNumber))
+                        existingDevice = cache.Devices.FirstOrDefault(device => device.SerialNumber == deviceSerialNumber);
+
+                    var fields = columns.Select(h =>
+                    {
+                        var f = (DeviceImportFieldBase)h.GetHandlerInstance();
+                        f.Parse(Database, cache, Context, deviceSerialNumber, existingDevice, records, dataReader, h.Index);
+                        return f;
+                    }).ToList();
+
+                    EntityState recordAction;
+                    if (fields.Any(f => !f.FieldAction.HasValue))
+                        recordAction = EntityState.Detached;
+                    else if (existingDevice == null)
+                        recordAction = EntityState.Added;
+                    else if (fields.Any(f => f.FieldAction == EntityState.Modified))
+                        recordAction = EntityState.Modified;
+                    else
+                        recordAction = EntityState.Unchanged;
+
+                    records.Add(new DeviceImportRecord(dataReader.Index, deviceSerialNumber, fields, recordAction));
+                }
+            }
+
+            Context.Records = records;
         }
 
-        public static int ApplyRecords(this DeviceImportContext Context, DiscoDataContext Database, IScheduledTaskStatus Status)
+        public static int ApplyRecords(this IDeviceImportContext Context, DiscoDataContext Database, IScheduledTaskStatus Status)
         {
             if (Context.Records == null)
                 throw new InvalidOperationException("Import Records have not been parsed");

Disco.Services/Devices/Importing/DeviceImportContext.cs

@@ -1,35 +0,0 @@
-using Disco.Data.Repository;
-using Disco.Models.Repository;
-using Disco.Models.Services.Devices.Importing;
-using Disco.Services.Tasks;
-using System;
-using System.Collections.Generic;
-using System.Data;
-using System.Linq;
-
-namespace Disco.Services.Devices.Importing
-{
-    public class DeviceImportContext : IDeviceImportContext
-    {
-        public string SessionId { get; private set; }
-        public string Filename { get; private set; }
-        
-        public List<Tuple<string, DeviceImportFieldTypes>> Header { get; internal set; }
-        public List<Tuple<string, DeviceImportFieldTypes, Func<string[], string>, Type>> ParsedHeaders { get; internal set; }
-        internal int HeaderDeviceSerialNumberIndex { get; set; }
-        
-        public List<string[]> RawData { get; private set; }
-
-        public List<IDeviceImportRecord> Records { get; internal set; }
-        public int AffectedRecords { get; internal set; }
-
-        internal DeviceImportContext(string Filename, List<Tuple<string, DeviceImportFieldTypes>> Header, List<string[]> RawData)
-        {
-            this.SessionId = Guid.NewGuid().ToString("D");
-
-            this.Filename = Filename;
-            this.Header = Header;
-            this.RawData = RawData;
-        }
-    }
-}

Disco.Services/Devices/Importing/DeviceImportFieldBase.cs

@@ -1,12 +1,8 @@
 using Disco.Data.Repository;
 using Disco.Models.Repository;
 using Disco.Models.Services.Devices.Importing;
-using System;
 using System.Collections.Generic;
 using System.Data;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace Disco.Services.Devices.Importing
 {
@@ -22,21 +18,22 @@ namespace Disco.Services.Devices.Importing
         public abstract string FriendlyValue { get; }
         public abstract string FriendlyPreviousValue { get; }
 
-        public abstract bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, DeviceImportContext Context, int RecordIndex, string DeviceSerialNumber, Device ExistingDevice, Dictionary<DeviceImportFieldTypes, string> Values, string Value);
+        public abstract bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, IDeviceImportContext Context, string DeviceSerialNumber, Device ExistingDevice, List<IDeviceImportRecord> PreviousRecords, IDeviceImportDataReader DataReader, int ColumnIndex);
         public abstract bool Apply(DiscoDataContext Database, Device Device);
+        public virtual void Applied(DiscoDataContext Database, Device Device, ref bool DeviceADDescriptionSet) { return; }
 
-        public abstract int? GuessHeader(DiscoDataContext Database, DeviceImportContext Context);
+        public abstract int? GuessColumn(DiscoDataContext Database, IDeviceImportContext Context, IDeviceImportDataReader DataReader);
 
         #region Helpers
         protected bool Error(string Message)
         {
-            this.ErrorMessage = Message;
-            this.FieldAction = null;
+            ErrorMessage = Message;
+            FieldAction = null;
             return false;
         }
         protected bool Success(EntityState Action)
         {
-            this.FieldAction = Action;
+            FieldAction = Action;
             return true;
         }
         #endregion

Disco.Services/Devices/Importing/DeviceImportParseTask.cs

@@ -1,4 +1,5 @@
 using Disco.Data.Repository;
+using Disco.Models.Services.Devices.Importing;
 using Disco.Services.Tasks;
 using Quartz;
 using System;
@@ -13,7 +14,7 @@ namespace Disco.Services.Devices.Importing
         public override bool SingleInstanceTask { get { return false; } }
         public override bool CancelInitiallySupported { get { return false; } }
 
-        public static ScheduledTaskStatus ScheduleNow(DeviceImportContext Context)
+        public static ScheduledTaskStatus ScheduleNow(IDeviceImportContext Context)
         {
             if (Context == null)
                 throw new ArgumentNullException("Context");
@@ -28,11 +29,11 @@ namespace Disco.Services.Devices.Importing
 
         protected override void ExecuteTask()
         {
-            var context = (DeviceImportContext)this.ExecutionContext.JobDetail.JobDataMap[JobDataMapContext];
+            var context = (IDeviceImportContext)ExecutionContext.JobDetail.JobDataMap[JobDataMapContext];
 
             using (DiscoDataContext Database = new DiscoDataContext())
             {
-                context.ParseRecords(Database, this.Status);
+                context.ParseRecords(Database, Status);
             }
         }
     }

Disco.Services/Devices/Importing/DeviceImportRecord.cs

@@ -5,13 +5,12 @@ using System;
 using System.Collections.Generic;
 using System.Data;
 using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace Disco.Services.Devices.Importing
 {
     internal class DeviceImportRecord : IDeviceImportRecord
     {
+        public int Index { get; private set; }
         public string DeviceSerialNumber { get; private set; }
 
         public IEnumerable<IDeviceImportField> Fields { get; private set; }
@@ -23,8 +22,9 @@ namespace Disco.Services.Devices.Importing
             get { return Fields.Any(f => !f.FieldAction.HasValue); }
         }
 
-        internal DeviceImportRecord(string DeviceSerialNumber, IEnumerable<IDeviceImportField> Fields, EntityState RecordAction)
+        internal DeviceImportRecord(int Index, string DeviceSerialNumber, IEnumerable<IDeviceImportField> Fields, EntityState RecordAction)
         {
+            this.Index = Index;
             this.DeviceSerialNumber = DeviceSerialNumber;
             this.Fields = Fields;
             this.RecordAction = RecordAction;
@@ -43,7 +43,7 @@ namespace Disco.Services.Devices.Importing
                 }
                 else if (RecordAction == EntityState.Modified)
                 {
-                    device = Database.Devices.Find(this.DeviceSerialNumber);
+                    device = Database.Devices.Find(DeviceSerialNumber);
                 }
                 else if (RecordAction == EntityState.Added)
                 {
@@ -82,6 +82,13 @@ namespace Disco.Services.Devices.Importing
                 if (changesMade)
                     Database.SaveChanges();
 
+                bool adDescriptionSet = false;
+
+                foreach (var field in Fields.Cast<DeviceImportFieldBase>())
+                {
+                    field.Applied(Database, device, ref adDescriptionSet);
+                }
+
                 return changesMade;
             }
 

Disco.Services/Devices/Importing/Fields/AssignedUserIdImportField.cs

@@ -22,18 +22,18 @@ namespace Disco.Services.Devices.Importing.Fields
         public override string FriendlyValue { get { return friendlyValue; } }
         public override string FriendlyPreviousValue { get { return friendlyPreviousValue; } }
 
-        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, DeviceImportContext Context, int RecordIndex, string DeviceSerialNumber, Device ExistingDevice, Dictionary<DeviceImportFieldTypes, string> Values, string Value)
+        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, IDeviceImportContext Context, string DeviceSerialNumber, Device ExistingDevice, List<IDeviceImportRecord> PreviousRecords, IDeviceImportDataReader DataReader, int ColumnIndex)
         {
-            friendlyValue = Value;
+            var value = friendlyValue = DataReader.GetString(ColumnIndex);
 
-            if (string.IsNullOrWhiteSpace(Value))
+            if (string.IsNullOrWhiteSpace(value))
             {
                 friendlyValue = null;
                 parsedValue = null;
             }
             else
             {
-                parsedValue = Value.Trim();
+                parsedValue = value.Trim();
 
                 parsedValue = ActiveDirectory.ParseDomainAccountId(parsedValue);
 
@@ -48,25 +48,32 @@ namespace Disco.Services.Devices.Importing.Fields
                 // Check User Exists
 
                 // Try Database
-                User user = Database.Users.FirstOrDefault(u => u.UserId == parsedValue);
-                try
+                using (var database = new DiscoDataContext())
                 {
-                    // Try Updating from AD
-                    user = UserService.GetUser(parsedValue, Database);
+                    User user = database.Users.FirstOrDefault(u => u.UserId == parsedValue);
+                    try
+                    {
+                        // Try Updating from AD
+                        user = UserService.GetUser(parsedValue, database);
+                    }
+                    catch (Exception ex)
+                    {
+                        if (user == null)
+                            return Error(ex.Message);
+                    }
+                    parsedValue = user.UserId;
+                    friendlyValue = $"{user.DisplayName} [{user.UserId}]";
                 }
-                catch (Exception ex)
-                {
-                    if (user == null)
-                        return Error(ex.Message);
-                }
-                parsedValue = user.UserId;
-                friendlyValue = string.Format("{0} [{1}]", user.DisplayName, user.UserId);
 
                 // Check Decommissioned
                 bool? importDecommissioning = null;
-                if (Values.ContainsKey(DeviceImportFieldTypes.DeviceDecommissionedDate) || Values.ContainsKey(DeviceImportFieldTypes.DeviceDecommissionedReason))
-                    importDecommissioning = Values.ContainsKey(DeviceImportFieldTypes.DeviceDecommissionedDate) && !string.IsNullOrWhiteSpace(Values[DeviceImportFieldTypes.DeviceDecommissionedDate]) ||
-                        Values.ContainsKey(DeviceImportFieldTypes.DeviceDecommissionedReason) && !string.IsNullOrWhiteSpace(Values[DeviceImportFieldTypes.DeviceDecommissionedReason]);
+                int? decommissionedDateIndex = Context.GetColumnByType(DeviceImportFieldTypes.DeviceDecommissionedDate);
+                int? decommissionedReasonIndex = Context.GetColumnByType(DeviceImportFieldTypes.DeviceDecommissionedReason);
+                if (decommissionedDateIndex.HasValue || decommissionedReasonIndex.HasValue)
+                {
+                    importDecommissioning = (decommissionedDateIndex.HasValue && !string.IsNullOrWhiteSpace(DataReader.GetString(decommissionedDateIndex.Value))) ||
+                        (decommissionedReasonIndex.HasValue && !string.IsNullOrWhiteSpace(DataReader.GetString(decommissionedReasonIndex.Value)));
+                }
 
                 if (importDecommissioning.HasValue && importDecommissioning.Value)
                     return Error("Cannot assign a user to a device being decommissioned");
@@ -84,7 +91,7 @@ namespace Disco.Services.Devices.Importing.Fields
             else if (ExistingDevice != null && ExistingDevice.AssignedUserId != parsedValue)
             {
                 if (ExistingDevice.AssignedUserId != null)
-                    friendlyPreviousValue = string.Format("{0} [{1}]", ExistingDevice.AssignedUser.DisplayName, ExistingDevice.AssignedUser.UserId);
+                    friendlyPreviousValue = $"{ExistingDevice.AssignedUser.DisplayName} [{ExistingDevice.AssignedUser.UserId}]";
                 else
                     friendlyPreviousValue = null;
 
@@ -96,7 +103,7 @@ namespace Disco.Services.Devices.Importing.Fields
 
         public override bool Apply(DiscoDataContext Database, Device Device)
         {
-            if (this.FieldAction == EntityState.Modified)
+            if (FieldAction == EntityState.Modified)
             {
                 // Remove Current Assignments
                 var currentAssignments = Device.DeviceUserAssignments.Where(dua => !dua.UnassignedDate.HasValue);
@@ -130,16 +137,31 @@ namespace Disco.Services.Devices.Importing.Fields
             }
         }
 
-        public override int? GuessHeader(DiscoDataContext Database, DeviceImportContext Context)
+        public override void Applied(DiscoDataContext Database, Device Device, ref bool DeviceADDescriptionSet)
+        {
+            if (!DeviceADDescriptionSet)
+            {
+                if (ActiveDirectory.IsValidDomainAccountId(Device.DeviceDomainId))
+                {
+                    var adAccount = Device.ActiveDirectoryAccount();
+
+                    if (adAccount != null && !adAccount.IsCriticalSystemObject)
+                    {
+                        adAccount.SetDescription(Device);
+                        DeviceADDescriptionSet = true;
+                    }
+                }
+            }
+        }
+
+        public override int? GuessColumn(DiscoDataContext Database, IDeviceImportContext Context, IDeviceImportDataReader DataReader)
         {
             // column name
-            var possibleColumns = Context.Header
-                .Select((h, i) => Tuple.Create(h, i))
-                .Where(h => h.Item1.Item2 == DeviceImportFieldTypes.IgnoreColumn &&
-                    h.Item1.Item1.IndexOf("user", System.StringComparison.OrdinalIgnoreCase) >= 0
-                    );
+            var possibleColumns = Context.Columns
+                .Where(h => h.Type == DeviceImportFieldTypes.IgnoreColumn &&
+                    h.Name.IndexOf("user", StringComparison.OrdinalIgnoreCase) >= 0);
 
-            return possibleColumns.Select(h => (int?)h.Item2).FirstOrDefault();
+            return possibleColumns.Select(h => (int?)h.Index).FirstOrDefault();
         }
     }
 }

Disco.Services/Devices/Importing/Fields/BatchIdImportField.cs

@@ -20,28 +20,23 @@ namespace Disco.Services.Devices.Importing.Fields
         public override string FriendlyValue { get { return friendlyValue; } }
         public override string FriendlyPreviousValue { get { return friendlyPreviousValue; } }
 
-        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, DeviceImportContext Context, int RecordIndex, string DeviceSerialNumber, Device ExistingDevice, Dictionary<DeviceImportFieldTypes, string> Values, string Value)
+        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, IDeviceImportContext Context, string DeviceSerialNumber, Device ExistingDevice, List<IDeviceImportRecord> PreviousRecords, IDeviceImportDataReader DataReader, int ColumnIndex)
         {
-            friendlyValue = Value;
-
-            // Validate
-            if (string.IsNullOrWhiteSpace(Value))
-                this.parsedValue = null; // Default = null
+            if (DataReader.TryGetNullableInt(ColumnIndex, out parsedValue))
+            {
+                friendlyValue = parsedValue.ToString();
+            }
             else
             {
-                int valueInt;
-                if (int.TryParse(Value, out valueInt))
-                    this.parsedValue = valueInt;
-                else
-                    return Error("The Batch Identifier must be a number");
+                return Error("The Batch Identifier must be a number");
             }
 
-            if (this.parsedValue.HasValue)
+            if (parsedValue.HasValue)
             {
                 var b = Cache.DeviceBatches.FirstOrDefault(db => db.Id == parsedValue);
                 if (b == null)
-                    return Error(string.Format("The identifier ({0}) does not match any Device Batch", Value));
-                friendlyValue = string.Format("{0} [{1}]", b.Name, b.Id);
+                    return Error($"The identifier ({friendlyValue}) does not match any Device Batch");
+                friendlyValue = $"{b.Name} [{b.Id}]";
             }
             else
                 friendlyValue = null;
@@ -55,7 +50,7 @@ namespace Disco.Services.Devices.Importing.Fields
                     previousBatch = Cache.DeviceBatches.FirstOrDefault(db => db.Id == ExistingDevice.DeviceBatchId.Value);
 
                 if (previousBatch != null)
-                    friendlyPreviousValue = string.Format("{0} [{1}]", previousBatch.Name, previousBatch.Id);
+                    friendlyPreviousValue = $"{previousBatch.Name} [{previousBatch.Id}]";
 
                 return Success(EntityState.Modified);
             }
@@ -65,10 +60,10 @@ namespace Disco.Services.Devices.Importing.Fields
 
         public override bool Apply(DiscoDataContext Database, Device Device)
         {
-            if (this.FieldAction == EntityState.Added ||
-                this.FieldAction == EntityState.Modified)
+            if (FieldAction == EntityState.Added ||
+                FieldAction == EntityState.Modified)
             {
-                Device.DeviceBatchId = this.parsedValue;
+                Device.DeviceBatchId = parsedValue;
                 return true;
             }
             else
@@ -77,30 +72,27 @@ namespace Disco.Services.Devices.Importing.Fields
             }
         }
 
-        public override int? GuessHeader(DiscoDataContext Database, DeviceImportContext Context)
+        public override int? GuessColumn(DiscoDataContext Database, IDeviceImportContext Context, IDeviceImportDataReader DataReader)
         {
             // column name
-            var possibleColumns = Context.Header
-                .Select((h, i) => Tuple.Create(h, i))
-                .Where(h => h.Item1.Item2 == DeviceImportFieldTypes.IgnoreColumn && h.Item1.Item1.IndexOf("batch", System.StringComparison.OrdinalIgnoreCase) >= 0);
+            var possibleColumns = Context.Columns
+                .Where(h => h.Type == DeviceImportFieldTypes.IgnoreColumn &&
+                    h.Name.IndexOf("batch", StringComparison.OrdinalIgnoreCase) >= 0);
 
-            // All Integers Numbers
-            possibleColumns = possibleColumns.Where(h =>
-            {
-                int lastValue;
-                return Context.RawData.Select(v => v[h.Item2]).Take(100).Where(v => !string.IsNullOrWhiteSpace(v)).All(v => int.TryParse(v, out lastValue));
-            }).ToList();
+            // All Nullable<int> Values
+            possibleColumns = possibleColumns
+                .Where(h => DataReader.TestAllNullableInt(h.Index)).ToList();
 
             // Multiple Columns, tighten column definition
             if (possibleColumns.Count() > 1)
             {
                 possibleColumns = possibleColumns
                     .Where(h =>
-                        h.Item1.Item1.IndexOf("batchid", StringComparison.OrdinalIgnoreCase) >= 0 ||
-                        h.Item1.Item1.IndexOf("batch id", StringComparison.OrdinalIgnoreCase) >= 0);
+                        h.Name.IndexOf("batchid", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                        h.Name.IndexOf("batch id", StringComparison.OrdinalIgnoreCase) >= 0);
             }
 
-            return possibleColumns.Select(h => (int?)h.Item2).FirstOrDefault();
+            return possibleColumns.Select(h => (int?)h.Index).FirstOrDefault();
         }
     }
 }

Disco.Services/Devices/Importing/Fields/DetailACAdapterImportField.cs

@@ -19,13 +19,15 @@ namespace Disco.Services.Devices.Importing.Fields
         public override string FriendlyValue { get { return parsedValue; } }
         public override string FriendlyPreviousValue { get { return previousValue; } }
 
-        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, DeviceImportContext Context, int RecordIndex, string DeviceSerialNumber, Device ExistingDevice, Dictionary<DeviceImportFieldTypes, string> Values, string Value)
+        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, IDeviceImportContext Context, string DeviceSerialNumber, Device ExistingDevice, List<IDeviceImportRecord> PreviousRecords, IDeviceImportDataReader DataReader, int ColumnIndex)
         {
-            if (string.IsNullOrWhiteSpace(Value))
+            var value = DataReader.GetString(ColumnIndex);
+
+            if (string.IsNullOrWhiteSpace(value))
                 parsedValue = null;
             else
             {
-                parsedValue = Value.Trim();
+                parsedValue = value.Trim();
             }
 
             if (ExistingDevice == null && parsedValue != null)
@@ -54,8 +56,8 @@ namespace Disco.Services.Devices.Importing.Fields
 
         public override bool Apply(DiscoDataContext Database, Device Device)
         {
-            if (this.FieldAction == EntityState.Added ||
-                this.FieldAction == EntityState.Modified)
+            if (FieldAction == EntityState.Added ||
+                FieldAction == EntityState.Modified)
             {
 
                 DeviceDetail detail = Database.DeviceDetails.FirstOrDefault(dd =>
@@ -84,16 +86,15 @@ namespace Disco.Services.Devices.Importing.Fields
             }
         }
 
-        public override int? GuessHeader(DiscoDataContext Database, DeviceImportContext Context)
+        public override int? GuessColumn(DiscoDataContext Database, IDeviceImportContext Context, IDeviceImportDataReader DataReader)
         {
             // column name
-            var possibleColumns = Context.Header
-                .Select((h, i) => Tuple.Create(h, i))
-                .Where(h => h.Item1.Item2 == DeviceImportFieldTypes.IgnoreColumn &&
-                    (h.Item1.Item1.IndexOf("ac adapter", System.StringComparison.OrdinalIgnoreCase) >= 0 ||
-                    h.Item1.Item1.IndexOf("acadapter", System.StringComparison.OrdinalIgnoreCase) >= 0));
+            var possibleColumns = Context.Columns
+                .Where(h => h.Type == DeviceImportFieldTypes.IgnoreColumn &&
+                    (h.Name.IndexOf("ac adapter", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                    h.Name.IndexOf("acadapter", StringComparison.OrdinalIgnoreCase) >= 0));
 
-            return possibleColumns.Select(h => (int?)h.Item2).FirstOrDefault();
+            return possibleColumns.Select(h => (int?)h.Index).FirstOrDefault();
         }
     }
 }

Disco.Services/Devices/Importing/Fields/DetailBatteryImportField.cs

@@ -19,13 +19,15 @@ namespace Disco.Services.Devices.Importing.Fields
         public override string FriendlyValue { get { return parsedValue; } }
         public override string FriendlyPreviousValue { get { return previousValue; } }
 
-        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, DeviceImportContext Context, int RecordIndex, string DeviceSerialNumber, Device ExistingDevice, Dictionary<DeviceImportFieldTypes, string> Values, string Value)
+        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, IDeviceImportContext Context, string DeviceSerialNumber, Device ExistingDevice, List<IDeviceImportRecord> PreviousRecords, IDeviceImportDataReader DataReader, int ColumnIndex)
         {
-            if (string.IsNullOrWhiteSpace(Value))
+            var value = DataReader.GetString(ColumnIndex);
+
+            if (string.IsNullOrWhiteSpace(value))
                 parsedValue = null;
             else
             {
-                parsedValue = Value.Trim();
+                parsedValue = value.Trim();
             }
 
             if (ExistingDevice == null && parsedValue != null)
@@ -54,8 +56,8 @@ namespace Disco.Services.Devices.Importing.Fields
 
         public override bool Apply(DiscoDataContext Database, Device Device)
         {
-            if (this.FieldAction == EntityState.Added ||
-                this.FieldAction == EntityState.Modified)
+            if (FieldAction == EntityState.Added ||
+                FieldAction == EntityState.Modified)
             {
 
                 DeviceDetail detail = Database.DeviceDetails.FirstOrDefault(dd =>
@@ -84,15 +86,14 @@ namespace Disco.Services.Devices.Importing.Fields
             }
         }
 
-        public override int? GuessHeader(DiscoDataContext Database, DeviceImportContext Context)
+        public override int? GuessColumn(DiscoDataContext Database, IDeviceImportContext Context, IDeviceImportDataReader DataReader)
         {
             // column name
-            var possibleColumns = Context.Header
-                .Select((h, i) => Tuple.Create(h, i))
-                .Where(h => h.Item1.Item2 == DeviceImportFieldTypes.IgnoreColumn &&
-                    h.Item1.Item1.IndexOf("battery", System.StringComparison.OrdinalIgnoreCase) >= 0);
+            var possibleColumns = Context.Columns
+                .Where(h => h.Type == DeviceImportFieldTypes.IgnoreColumn &&
+                    h.Name.IndexOf("battery", StringComparison.OrdinalIgnoreCase) >= 0);
 
-            return possibleColumns.Select(h => (int?)h.Item2).FirstOrDefault();
+            return possibleColumns.Select(h => (int?)h.Index).FirstOrDefault();
         }
     }
 }

Disco.Services/Devices/Importing/Fields/DetailKeyboardImportField.cs

@@ -19,13 +19,15 @@ namespace Disco.Services.Devices.Importing.Fields
         public override string FriendlyValue { get { return parsedValue; } }
         public override string FriendlyPreviousValue { get { return previousValue; } }
 
-        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, DeviceImportContext Context, int RecordIndex, string DeviceSerialNumber, Device ExistingDevice, Dictionary<DeviceImportFieldTypes, string> Values, string Value)
+        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, IDeviceImportContext Context, string DeviceSerialNumber, Device ExistingDevice, List<IDeviceImportRecord> PreviousRecords, IDeviceImportDataReader DataReader, int ColumnIndex)
         {
-            if (string.IsNullOrWhiteSpace(Value))
+            var value = DataReader.GetString(ColumnIndex);
+
+            if (string.IsNullOrWhiteSpace(value))
                 parsedValue = null;
             else
             {
-                parsedValue = Value.Trim();
+                parsedValue = value.Trim();
             }
 
             if (ExistingDevice == null && parsedValue != null)
@@ -54,8 +56,8 @@ namespace Disco.Services.Devices.Importing.Fields
 
         public override bool Apply(DiscoDataContext Database, Device Device)
         {
-            if (this.FieldAction == EntityState.Added ||
-                this.FieldAction == EntityState.Modified)
+            if (FieldAction == EntityState.Added ||
+                FieldAction == EntityState.Modified)
             {
 
                 DeviceDetail detail = Database.DeviceDetails.FirstOrDefault(dd =>
@@ -84,15 +86,14 @@ namespace Disco.Services.Devices.Importing.Fields
             }
         }
 
-        public override int? GuessHeader(DiscoDataContext Database, DeviceImportContext Context)
+        public override int? GuessColumn(DiscoDataContext Database, IDeviceImportContext Context, IDeviceImportDataReader DataReader)
         {
             // column name
-            var possibleColumns = Context.Header
-                .Select((h, i) => Tuple.Create(h, i))
-                .Where(h => h.Item1.Item2 == DeviceImportFieldTypes.IgnoreColumn &&
-                    h.Item1.Item1.IndexOf("keyboard", System.StringComparison.OrdinalIgnoreCase) >= 0);
+            var possibleColumns = Context.Columns
+                .Where(h => h.Type == DeviceImportFieldTypes.IgnoreColumn &&
+                    h.Name.IndexOf("keyboard", StringComparison.OrdinalIgnoreCase) >= 0);
 
-            return possibleColumns.Select(h => (int?)h.Item2).FirstOrDefault();
+            return possibleColumns.Select(h => (int?)h.Index).FirstOrDefault();
         }
     }
 }

Disco.Services/Devices/Importing/Fields/DetailLanMacAddressImportField.cs

@@ -19,13 +19,15 @@ namespace Disco.Services.Devices.Importing.Fields
         public override string FriendlyValue { get { return parsedValue; } }
         public override string FriendlyPreviousValue { get { return previousValue; } }
 
-        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, DeviceImportContext Context, int RecordIndex, string DeviceSerialNumber, Device ExistingDevice, Dictionary<DeviceImportFieldTypes, string> Values, string Value)
+        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, IDeviceImportContext Context, string DeviceSerialNumber, Device ExistingDevice, List<IDeviceImportRecord> PreviousRecords, IDeviceImportDataReader DataReader, int ColumnIndex)
         {
-            if (string.IsNullOrWhiteSpace(Value))
+            var value = DataReader.GetString(ColumnIndex);
+
+            if (string.IsNullOrWhiteSpace(value))
                 parsedValue = null;
             else
             {
-                parsedValue = Value.Trim();
+                parsedValue = value.Trim();
             }
 
             if (ExistingDevice == null && parsedValue != null)
@@ -54,8 +56,8 @@ namespace Disco.Services.Devices.Importing.Fields
 
         public override bool Apply(DiscoDataContext Database, Device Device)
         {
-            if (this.FieldAction == EntityState.Added ||
-                this.FieldAction == EntityState.Modified)
+            if (FieldAction == EntityState.Added ||
+                FieldAction == EntityState.Modified)
             {
 
                 DeviceDetail detail = Database.DeviceDetails.FirstOrDefault(dd =>
@@ -84,23 +86,22 @@ namespace Disco.Services.Devices.Importing.Fields
             }
         }
 
-        public override int? GuessHeader(DiscoDataContext Database, DeviceImportContext Context)
+        public override int? GuessColumn(DiscoDataContext Database, IDeviceImportContext Context, IDeviceImportDataReader DataReader)
         {
             // column name
-            var possibleColumns = Context.Header
-                .Select((h, i) => Tuple.Create(h, i))
-                .Where(h => h.Item1.Item2 == DeviceImportFieldTypes.IgnoreColumn && (
-                    h.Item1.Item1.IndexOf("wlan", System.StringComparison.OrdinalIgnoreCase) < 0 &&
-                    h.Item1.Item1.IndexOf("wireless", System.StringComparison.OrdinalIgnoreCase) < 0 && (
-                    h.Item1.Item1.IndexOf("lan address", System.StringComparison.OrdinalIgnoreCase) >= 0 ||
-                    h.Item1.Item1.IndexOf("lan mac", System.StringComparison.OrdinalIgnoreCase) >= 0 ||
-                    h.Item1.Item1.IndexOf("lan mac address", System.StringComparison.OrdinalIgnoreCase) >= 0 ||
-                    h.Item1.Item1.IndexOf("lanaddress", System.StringComparison.OrdinalIgnoreCase) >= 0 ||
-                    h.Item1.Item1.IndexOf("lanmac", System.StringComparison.OrdinalIgnoreCase) >= 0 ||
-                    h.Item1.Item1.IndexOf("lanmacaddress", System.StringComparison.OrdinalIgnoreCase) >= 0
+            var possibleColumns = Context.Columns
+                .Where(h => h.Type == DeviceImportFieldTypes.IgnoreColumn && (
+                    h.Name.IndexOf("wlan", StringComparison.OrdinalIgnoreCase) < 0 &&
+                    h.Name.IndexOf("wireless", StringComparison.OrdinalIgnoreCase) < 0 && (
+                    h.Name.IndexOf("lan address", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                    h.Name.IndexOf("lan mac", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                    h.Name.IndexOf("lan mac address", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                    h.Name.IndexOf("lanaddress", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                    h.Name.IndexOf("lanmac", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                    h.Name.IndexOf("lanmacaddress", StringComparison.OrdinalIgnoreCase) >= 0
                     )));
 
-            return possibleColumns.Select(h => (int?)h.Item2).FirstOrDefault();
+            return possibleColumns.Select(h => (int?)h.Index).FirstOrDefault();
         }
     }
 }

Disco.Services/Devices/Importing/Fields/DetailWLanMacAddressImportField.cs

@@ -19,13 +19,15 @@ namespace Disco.Services.Devices.Importing.Fields
         public override string FriendlyValue { get { return parsedValue; } }
         public override string FriendlyPreviousValue { get { return previousValue; } }
 
-        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, DeviceImportContext Context, int RecordIndex, string DeviceSerialNumber, Device ExistingDevice, Dictionary<DeviceImportFieldTypes, string> Values, string Value)
+        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, IDeviceImportContext Context, string DeviceSerialNumber, Device ExistingDevice, List<IDeviceImportRecord> PreviousRecords, IDeviceImportDataReader DataReader, int ColumnIndex)
         {
-            if (string.IsNullOrWhiteSpace(Value))
+            var value = DataReader.GetString(ColumnIndex);
+
+            if (string.IsNullOrWhiteSpace(value))
                 parsedValue = null;
             else
             {
-                parsedValue = Value.Trim();
+                parsedValue = value.Trim();
             }
 
             if (ExistingDevice == null && parsedValue != null)
@@ -54,8 +56,8 @@ namespace Disco.Services.Devices.Importing.Fields
 
         public override bool Apply(DiscoDataContext Database, Device Device)
         {
-            if (this.FieldAction == EntityState.Added ||
-                this.FieldAction == EntityState.Modified)
+            if (FieldAction == EntityState.Added ||
+                FieldAction == EntityState.Modified)
             {
 
                 DeviceDetail detail = Database.DeviceDetails.FirstOrDefault(dd =>
@@ -84,25 +86,24 @@ namespace Disco.Services.Devices.Importing.Fields
             }
         }
 
-        public override int? GuessHeader(DiscoDataContext Database, DeviceImportContext Context)
+        public override int? GuessColumn(DiscoDataContext Database, IDeviceImportContext Context, IDeviceImportDataReader DataReader)
         {
             // column name
-            var possibleColumns = Context.Header
-                .Select((h, i) => Tuple.Create(h, i))
-                .Where(h => h.Item1.Item2 == DeviceImportFieldTypes.IgnoreColumn && (
-                    h.Item1.Item1.IndexOf("wireless lan mac address", System.StringComparison.OrdinalIgnoreCase) >= 0 ||
-                    h.Item1.Item1.IndexOf("wireless lan address", System.StringComparison.OrdinalIgnoreCase) >= 0 ||
-                    h.Item1.Item1.IndexOf("wireless mac address", System.StringComparison.OrdinalIgnoreCase) >= 0 ||
-                    h.Item1.Item1.IndexOf("wireless mac", System.StringComparison.OrdinalIgnoreCase) >= 0 ||
-                    h.Item1.Item1.IndexOf("wlan address", System.StringComparison.OrdinalIgnoreCase) >= 0 ||
-                    h.Item1.Item1.IndexOf("wlan mac", System.StringComparison.OrdinalIgnoreCase) >= 0 ||
-                    h.Item1.Item1.IndexOf("wlan mac address", System.StringComparison.OrdinalIgnoreCase) >= 0 ||
-                    h.Item1.Item1.IndexOf("wlanaddress", System.StringComparison.OrdinalIgnoreCase) >= 0 ||
-                    h.Item1.Item1.IndexOf("wlanmac", System.StringComparison.OrdinalIgnoreCase) >= 0 ||
-                    h.Item1.Item1.IndexOf("wlanmacaddress", System.StringComparison.OrdinalIgnoreCase) >= 0
+            var possibleColumns = Context.Columns
+                .Where(h => h.Type == DeviceImportFieldTypes.IgnoreColumn && (
+                    h.Name.IndexOf("wireless lan mac address", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                    h.Name.IndexOf("wireless lan address", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                    h.Name.IndexOf("wireless mac address", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                    h.Name.IndexOf("wireless mac", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                    h.Name.IndexOf("wlan address", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                    h.Name.IndexOf("wlan mac", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                    h.Name.IndexOf("wlan mac address", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                    h.Name.IndexOf("wlanaddress", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                    h.Name.IndexOf("wlanmac", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                    h.Name.IndexOf("wlanmacaddress", StringComparison.OrdinalIgnoreCase) >= 0
                     ));
 
-            return possibleColumns.Select(h => (int?)h.Item2).FirstOrDefault();
+            return possibleColumns.Select(h => (int?)h.Index).FirstOrDefault();
         }
     }
 }

Disco.Services/Devices/Importing/Fields/DeviceAllowUnauthenticatedEnrolImportField.cs

@@ -1,7 +1,6 @@
 using Disco.Data.Repository;
 using Disco.Models.Repository;
 using Disco.Models.Services.Devices.Importing;
-using Disco.Services.Users;
 using System;
 using System.Collections.Generic;
 using System.Data;
@@ -11,9 +10,7 @@ namespace Disco.Services.Devices.Importing.Fields
 {
     internal class DeviceAllowUnauthenticatedEnrolImportField : DeviceImportFieldBase
     {
-        private static string[] TrueValues = { "true", "1", "yes", "-1", "on" };
-        private static string[] FalseValues = { "false", "0", "no", "off" };
-        private bool parsedValue;
+        private bool? parsedValue;
         private string friendlyValue;
         private string friendlyPreviousValue;
 
@@ -23,22 +20,30 @@ namespace Disco.Services.Devices.Importing.Fields
         public override string FriendlyValue { get { return friendlyValue; } }
         public override string FriendlyPreviousValue { get { return friendlyPreviousValue; } }
 
-        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, DeviceImportContext Context, int RecordIndex, string DeviceSerialNumber, Device ExistingDevice, Dictionary<DeviceImportFieldTypes, string> Values, string Value)
+        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, IDeviceImportContext Context, string DeviceSerialNumber, Device ExistingDevice, List<IDeviceImportRecord> PreviousRecords, IDeviceImportDataReader DataReader, int ColumnIndex)
         {
-            friendlyValue = Value;
-
-            if (!ParseBoolean(Value, out parsedValue))
+            if (DataReader.TryGetNullableBool(ColumnIndex, out parsedValue))
+            {
+                friendlyValue = parsedValue.ToString();
+            }
+            else
+            {
                 return Error("Expected a Boolean expression (True, 1, Yes, On, False, 0, No, Off)");
+            }
 
-            friendlyValue = parsedValue.ToString();
+            friendlyValue = parsedValue?.ToString() ?? "Not Set";
 
-            if (parsedValue == true)
+            if (parsedValue.HasValue && parsedValue.Value == true)
             {
                 // Check Decommissioned
                 bool? importDecommissioning = null;
-                if (Values.ContainsKey(DeviceImportFieldTypes.DeviceDecommissionedDate) || Values.ContainsKey(DeviceImportFieldTypes.DeviceDecommissionedReason))
-                    importDecommissioning = Values.ContainsKey(DeviceImportFieldTypes.DeviceDecommissionedDate) && !string.IsNullOrWhiteSpace(Values[DeviceImportFieldTypes.DeviceDecommissionedDate]) ||
-                        Values.ContainsKey(DeviceImportFieldTypes.DeviceDecommissionedReason) && !string.IsNullOrWhiteSpace(Values[DeviceImportFieldTypes.DeviceDecommissionedReason]);
+                int? decommissionedDateIndex = Context.GetColumnByType(DeviceImportFieldTypes.DeviceDecommissionedDate);
+                int? decommissionedReasonIndex = Context.GetColumnByType(DeviceImportFieldTypes.DeviceDecommissionedReason);
+                if (decommissionedDateIndex.HasValue || decommissionedReasonIndex.HasValue)
+                {
+                    importDecommissioning = (decommissionedDateIndex.HasValue && !string.IsNullOrWhiteSpace(DataReader.GetString(decommissionedDateIndex.Value))) ||
+                        (decommissionedReasonIndex.HasValue && !string.IsNullOrWhiteSpace(DataReader.GetString(decommissionedReasonIndex.Value)));
+                }
 
                 if (importDecommissioning.HasValue && importDecommissioning.Value)
                     return Error("Cannot enrol a device being decommissioned");
@@ -49,7 +54,11 @@ namespace Disco.Services.Devices.Importing.Fields
                 }
             }
 
-            if (ExistingDevice == null && parsedValue != false) // Default: True
+            if (!parsedValue.HasValue)
+            {
+                return Success(EntityState.Unchanged);
+            }
+            else if (ExistingDevice == null && parsedValue.Value != false) // Default: True
             {
                 return Success(EntityState.Added);
             }
@@ -65,10 +74,11 @@ namespace Disco.Services.Devices.Importing.Fields
 
         public override bool Apply(DiscoDataContext Database, Device Device)
         {
-            if (this.FieldAction == EntityState.Modified ||
-                this.FieldAction == EntityState.Added)
+            if (parsedValue.HasValue &&
+                (FieldAction == EntityState.Modified ||
+                FieldAction == EntityState.Added))
             {
-                Device.AllowUnauthenticatedEnrol = parsedValue;
+                Device.AllowUnauthenticatedEnrol = parsedValue.Value;
 
                 return true;
             }
@@ -78,52 +88,20 @@ namespace Disco.Services.Devices.Importing.Fields
             }
         }
 
-        public override int? GuessHeader(DiscoDataContext Database, DeviceImportContext Context)
+        public override int? GuessColumn(DiscoDataContext Database, IDeviceImportContext Context, IDeviceImportDataReader DataReader)
         {
             // column name
-            var possibleColumns = Context.Header
-                .Select((h, i) => Tuple.Create(h, i))
-                .Where(h => h.Item1.Item2 == DeviceImportFieldTypes.IgnoreColumn &&
-                    h.Item1.Item1.IndexOf("trust enrol", System.StringComparison.OrdinalIgnoreCase) >= 0 ||
-                    h.Item1.Item1.IndexOf("enrolment trusted", System.StringComparison.OrdinalIgnoreCase) >= 0 ||
-                    h.Item1.Item1.IndexOf("trust", System.StringComparison.OrdinalIgnoreCase) >= 0
+            var possibleColumns = Context.Columns
+                .Where(h => h.Type == DeviceImportFieldTypes.IgnoreColumn &&
+                    h.Name.IndexOf("trust enrol", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                    h.Name.IndexOf("enrolment trusted", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                    h.Name.IndexOf("trust", StringComparison.OrdinalIgnoreCase) >= 0
                     );
 
             // All Boolean
-            possibleColumns = possibleColumns.Where(h =>
-            {
-                bool lastValue;
-                return Context.RawData.Select(v => v[h.Item2]).Take(100).Where(v => !string.IsNullOrWhiteSpace(v)).All(v => ParseBoolean(v, out lastValue));
-            }).ToList();
+            possibleColumns = possibleColumns.Where(h => DataReader.TestAllNullableBool(h.Index)).ToList();
 
-            return possibleColumns.Select(h => (int?)h.Item2).FirstOrDefault();
-        }
-
-        private static bool ParseBoolean(string value, out bool result)
-        {
-            if (string.IsNullOrWhiteSpace(value))
-            {
-                result = false;
-                return false;
-            }
-
-            value = value.Trim();
-
-            if (TrueValues.Contains(value, StringComparer.OrdinalIgnoreCase))
-            {
-                result = true;
-                return true;
-            }
-            else if (FalseValues.Contains(value, StringComparer.OrdinalIgnoreCase))
-            {
-                result = false;
-                return true;
-            }
-            else
-            {
-                result = false;
-                return false;
-            }
+            return possibleColumns.Select(h => (int?)h.Index).FirstOrDefault();
         }
     }
 }

Disco.Services/Devices/Importing/Fields/DeviceAssetNumberImportField.cs

@@ -19,13 +19,15 @@ namespace Disco.Services.Devices.Importing.Fields
         public override string FriendlyValue { get { return parsedValue; } }
         public override string FriendlyPreviousValue { get { return previousValue; } }
 
-        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, DeviceImportContext Context, int RecordIndex, string DeviceSerialNumber, Device ExistingDevice, Dictionary<DeviceImportFieldTypes, string> Values, string Value)
+        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, IDeviceImportContext Context, string DeviceSerialNumber, Device ExistingDevice, List<IDeviceImportRecord> PreviousRecords, IDeviceImportDataReader DataReader, int ColumnIndex)
         {
-            if (string.IsNullOrWhiteSpace(Value))
+            var value = DataReader.GetString(ColumnIndex);
+
+            if (string.IsNullOrWhiteSpace(value))
                 parsedValue = null;
             else
             {
-                parsedValue = Value.Trim();
+                parsedValue = value.Trim();
                 if (parsedValue.Length > 40)
                     return Error("Cannot be more than 40 characters");
             }
@@ -43,10 +45,10 @@ namespace Disco.Services.Devices.Importing.Fields
 
         public override bool Apply(DiscoDataContext Database, Device Device)
         {
-            if (this.FieldAction == EntityState.Added ||
-                this.FieldAction == EntityState.Modified)
+            if (FieldAction == EntityState.Added ||
+                FieldAction == EntityState.Modified)
             {
-                Device.AssetNumber = this.parsedValue;
+                Device.AssetNumber = parsedValue;
                 return true;
             }
             else
@@ -55,14 +57,14 @@ namespace Disco.Services.Devices.Importing.Fields
             }
         }
 
-        public override int? GuessHeader(DiscoDataContext Database, DeviceImportContext Context)
+        public override int? GuessColumn(DiscoDataContext Database, IDeviceImportContext Context, IDeviceImportDataReader DataReader)
         {
-            // 'asset' in column name
-            var possibleColumns = Context.Header
-                .Select((h, i) => Tuple.Create(h, i))
-                .Where(h => h.Item1.Item2 == DeviceImportFieldTypes.IgnoreColumn && h.Item1.Item1.IndexOf("asset", System.StringComparison.OrdinalIgnoreCase) >= 0);
+            // column name
+            var possibleColumns = Context.Columns
+                .Where(h => h.Type == DeviceImportFieldTypes.IgnoreColumn &&
+                    h.Name.IndexOf("asset", StringComparison.OrdinalIgnoreCase) >= 0);
 
-            return possibleColumns.Select(h => (int?)h.Item2).FirstOrDefault();
+            return possibleColumns.Select(h => (int?)h.Index).FirstOrDefault();
         }
     }
 }

Disco.Services/Devices/Importing/Fields/DeviceDecommissionedDateImportField.cs

@@ -1,6 +1,7 @@
 using Disco.Data.Repository;
 using Disco.Models.Repository;
 using Disco.Models.Services.Devices.Importing;
+using Disco.Services.Interop.ActiveDirectory;
 using System;
 using System.Collections.Generic;
 using System.Data;
@@ -24,53 +25,67 @@ namespace Disco.Services.Devices.Importing.Fields
         public override string FriendlyValue { get { return parsedValue.HasValue ? parsedValue.Value.ToString(DateFormat) : rawValue; } }
         public override string FriendlyPreviousValue { get { return previousValue.HasValue ? previousValue.Value.ToString(DateFormat) : null; } }
 
-        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, DeviceImportContext Context, int RecordIndex, string DeviceSerialNumber, Device ExistingDevice, Dictionary<DeviceImportFieldTypes, string> Values, string Value)
+        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, IDeviceImportContext Context, string DeviceSerialNumber, Device ExistingDevice, List<IDeviceImportRecord> PreviousRecords, IDeviceImportDataReader DataReader, int ColumnIndex)
         {
-            if (string.IsNullOrWhiteSpace(Value))
+            if (!DataReader.TryGetNullableDateTime(ColumnIndex, out parsedValue))
             {
-                rawValue = null;
-                parsedValue = null;
+                rawValue = DataReader.GetString(ColumnIndex);
+                return Error($"Cannot parse the value as a Date/Time using {CultureInfo.CurrentCulture.Name} culture (system default).");
             }
-            else
+
+            if (parsedValue.HasValue)
             {
-                DateTime valueDateTime;
-                if (!DateTime.TryParse(Value.Trim(), CultureInfo.CurrentCulture, System.Globalization.DateTimeStyles.AssumeLocal, out valueDateTime))
-                {
-                    rawValue = Value.Trim();
-                    return Error(string.Format("Cannot parse the value as a Date/Time using {0} culture (system default).", CultureInfo.CurrentCulture.Name));
-                }
-                else
-                {
-                    // Accuracy to the second (remove any milliseconds)
-                    parsedValue = new DateTime((valueDateTime.Ticks / 10000000L) * 10000000L);
-                }
+                // Accuracy to the second (remove any milliseconds)
+                parsedValue = new DateTime((parsedValue.Value.Ticks / 10000000L) * 10000000L);
             }
 
             string errorMessage;
-            if (parsedValue.HasValue && !CanDecommissionDevice(ExistingDevice, Values, out errorMessage))
+            if (parsedValue.HasValue && !CanDecommissionDevice(ExistingDevice, Context, DataReader, out errorMessage))
                 return Error(errorMessage);
 
-            setReason = !Values.ContainsKey(DeviceImportFieldTypes.DeviceDecommissionedReason) ||
-                (parsedValue.HasValue && string.IsNullOrWhiteSpace(Values[DeviceImportFieldTypes.DeviceDecommissionedReason]));
+            var decommissionReasonIndex = Context.GetColumnByType(DeviceImportFieldTypes.DeviceDecommissionedReason);
+            setReason = !decommissionReasonIndex.HasValue ||
+                (parsedValue.HasValue && string.IsNullOrWhiteSpace(DataReader.GetString(decommissionReasonIndex.Value)));
 
-            if (ExistingDevice != null && ExistingDevice.DecommissionedDate != parsedValue)
+            if (ExistingDevice != null && ExistingDevice.DecommissionedDate.HasValue)
+            {
+                // Accuracy to the second (remove any milliseconds)
+                previousValue = new DateTime((ExistingDevice.DecommissionedDate.Value.Ticks / 10000000L) * 10000000L);
+            }
+            else
+            {
+                previousValue = null;
+            }
+
+            if (previousValue != parsedValue)
             {
-                previousValue = ExistingDevice.DecommissionedDate;
                 return Success(EntityState.Modified);
             }
             else
+            {
+                previousValue = null;
                 return Success(EntityState.Unchanged);
+            }
         }
 
         public override bool Apply(DiscoDataContext Database, Device Device)
         {
-            if (this.FieldAction == EntityState.Modified)
+            if (FieldAction == EntityState.Modified)
             {
                 // Decommission or Recommission Device
-                Device.DecommissionedDate = this.parsedValue;
+                Device.DecommissionedDate = parsedValue;
 
                 if (setReason)
-                    Device.DecommissionReason = this.parsedValue.HasValue ? (DecommissionReasons?)DecommissionReasons.EndOfLife : null;
+                {
+                    if (parsedValue.HasValue && !Device.DecommissionReason.HasValue)
+                    {
+                        Device.DecommissionReason = DecommissionReasons.EndOfLife;
+                    }
+                    else if (!parsedValue.HasValue && Device.DecommissionReason.HasValue)
+                    {
+                        Device.DecommissionReason = null;
+                    }
+                }
 
                 return true;
             }
@@ -80,22 +95,52 @@ namespace Disco.Services.Devices.Importing.Fields
             }
         }
 
-        public override int? GuessHeader(DiscoDataContext Database, DeviceImportContext Context)
+        public override void Applied(DiscoDataContext Database, Device Device, ref bool DeviceADDescriptionSet)
         {
-            // column name
-            var possibleColumns = Context.Header
-                .Select((h, i) => Tuple.Create(h, i))
-                .Where(h => h.Item1.Item2 == DeviceImportFieldTypes.IgnoreColumn &&
-                    h.Item1.Item1.IndexOf("decommission date", System.StringComparison.OrdinalIgnoreCase) >= 0 ||
-                    h.Item1.Item1.IndexOf("decommissiondate", System.StringComparison.OrdinalIgnoreCase) >= 0 ||
-                    h.Item1.Item1.IndexOf("decommissioned date", System.StringComparison.OrdinalIgnoreCase) >= 0 ||
-                    h.Item1.Item1.IndexOf("decommissioneddate", System.StringComparison.OrdinalIgnoreCase) >= 0
-                    );
+            if (ActiveDirectory.IsValidDomainAccountId(Device.DeviceDomainId))
+            {
+                var adAccount = Device.ActiveDirectoryAccount();
 
-            return possibleColumns.Select(h => (int?)h.Item2).FirstOrDefault();
+                if (adAccount != null && !adAccount.IsCriticalSystemObject)
+                {
+                    if (Device.DecommissionedDate.HasValue)
+                    {
+                        // Disable AD Account
+                        adAccount.DisableAccount();
+                    }
+                    else
+                    {
+                        // Enable AD Account
+                        adAccount.EnableAccount();
+                    }
+
+                    if (!DeviceADDescriptionSet)
+                    {
+                        adAccount.SetDescription(Device);
+                        DeviceADDescriptionSet = true;
+                    }
+                }
+            }
         }
 
-        public static bool CanDecommissionDevice(Device Device, Dictionary<DeviceImportFieldTypes, string> Values, out string ErrorMessage)
+        public override int? GuessColumn(DiscoDataContext Database, IDeviceImportContext Context, IDeviceImportDataReader DataReader)
+        {
+            // column name
+            var possibleColumns = Context.Columns
+                .Where(h => h.Type == DeviceImportFieldTypes.IgnoreColumn &&
+                    h.Name.IndexOf("decommission date", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                    h.Name.IndexOf("decommissiondate", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                    h.Name.IndexOf("decommissioned date", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                    h.Name.IndexOf("decommissioneddate", StringComparison.OrdinalIgnoreCase) >= 0
+                    );
+
+            possibleColumns = possibleColumns
+                .Where(h => DataReader.TestAllNullableDateTime(h.Index)).ToList();
+
+            return possibleColumns.Select(h => (int?)h.Index).FirstOrDefault();
+        }
+
+        public static bool CanDecommissionDevice(Device Device, IDeviceImportContext Context, IDeviceImportDataReader DataReader, out string ErrorMessage)
         {
             if (Device == null)
             {
@@ -105,13 +150,14 @@ namespace Disco.Services.Devices.Importing.Fields
 
             // Check device is assigned (or being removed in this import)
 
-            if ((!Values.ContainsKey(DeviceImportFieldTypes.AssignedUserId) && Device.AssignedUserId != null) ||
-                (Values.ContainsKey(DeviceImportFieldTypes.AssignedUserId) && !string.IsNullOrWhiteSpace(Values[DeviceImportFieldTypes.AssignedUserId])))
+            var assignedUserIndex = Context.GetColumnByType(DeviceImportFieldTypes.AssignedUserId);
+            if ((!assignedUserIndex.HasValue && Device.AssignedUserId != null) ||
+                (assignedUserIndex.HasValue && !string.IsNullOrWhiteSpace(DataReader.GetString(assignedUserIndex.Value))))
             {
                 if (Device.AssignedUserId != null)
-                    ErrorMessage = string.Format("The device is assigned to a user ({0} [{1}]) and cannot be decommissioned", Device.AssignedUser.DisplayName, Device.AssignedUser.UserId);
+                    ErrorMessage = $"The device is assigned to a user ({Device.AssignedUser.DisplayName} [{Device.AssignedUser.UserId}]) and cannot be decommissioned";
                 else
-                    ErrorMessage = string.Format("The device is being assigned to a user ({0}) and cannot be decommissioned", Values[DeviceImportFieldTypes.AssignedUserId]);
+                    ErrorMessage = $"The device is being assigned to a user ({DataReader.GetString(assignedUserIndex.Value)}) and cannot be decommissioned";
                 return false;
             }
 
@@ -119,7 +165,7 @@ namespace Disco.Services.Devices.Importing.Fields
             var openJobCount = Device.Jobs.Count(j => !j.ClosedDate.HasValue);
             if (openJobCount > 0)
             {
-                ErrorMessage = string.Format("The device is associated with {0} open job{1} and cannot be decommissioned", openJobCount, openJobCount == 1 ? null : "s");
+                ErrorMessage = $"The device is associated with {openJobCount} open job{(openJobCount == 1 ? null : "s")} and cannot be decommissioned";
                 return false;
             }
 

Disco.Services/Devices/Importing/Fields/DeviceDecommissionedReasonImportField.cs

@@ -1,6 +1,7 @@
 using Disco.Data.Repository;
 using Disco.Models.Repository;
 using Disco.Models.Services.Devices.Importing;
+using Disco.Services.Interop.ActiveDirectory;
 using System;
 using System.Collections.Generic;
 using System.Data;
@@ -22,9 +23,11 @@ namespace Disco.Services.Devices.Importing.Fields
         public override string FriendlyValue { get { return parsedValue.HasValue ? parsedValue.Value.ToString() : rawValue; } }
         public override string FriendlyPreviousValue { get { return previousValue.HasValue ? previousValue.Value.ToString() : null; } }
 
-        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, DeviceImportContext Context, int RecordIndex, string DeviceSerialNumber, Device ExistingDevice, Dictionary<DeviceImportFieldTypes, string> Values, string Value)
+        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, IDeviceImportContext Context, string DeviceSerialNumber, Device ExistingDevice, List<IDeviceImportRecord> PreviousRecords, IDeviceImportDataReader DataReader, int ColumnIndex)
         {
-            if (string.IsNullOrWhiteSpace(Value))
+            var value = DataReader.GetString(ColumnIndex);
+
+            if (string.IsNullOrWhiteSpace(value))
             {
                 rawValue = null;
                 parsedValue = null;
@@ -32,9 +35,9 @@ namespace Disco.Services.Devices.Importing.Fields
             else
             {
                 DecommissionReasons valueReason;
-                if (!decommissionReasonsMap.Value.TryGetValue(Value.Trim(), out valueReason))
+                if (!decommissionReasonsMap.Value.TryGetValue(value.Trim(), out valueReason))
                 {
-                    rawValue = Value.Trim();
+                    rawValue = value.Trim();
                     return Error("Cannot parse the value as a Decommission Reason");
                 }
                 else
@@ -43,15 +46,16 @@ namespace Disco.Services.Devices.Importing.Fields
                 }
             }
 
-            if (parsedValue.HasValue && !Values.ContainsKey(DeviceImportFieldTypes.DeviceDecommissionedDate))
+            var decommissionedDateIndex = Context.GetColumnByType(DeviceImportFieldTypes.DeviceDecommissionedDate);
+            if (parsedValue.HasValue && !decommissionedDateIndex.HasValue)
             {
                 string errorMessage;
-                if (!DeviceDecommissionedDateImportField.CanDecommissionDevice(ExistingDevice, Values, out errorMessage))
+                if (!DeviceDecommissionedDateImportField.CanDecommissionDevice(ExistingDevice, Context, DataReader, out errorMessage))
                     return Error(errorMessage);
 
                 setDate = true;
             }
-            else if (parsedValue.HasValue && string.IsNullOrWhiteSpace(Values[DeviceImportFieldTypes.DeviceDecommissionedDate]))
+            else if (parsedValue.HasValue && string.IsNullOrWhiteSpace(DataReader.GetString(decommissionedDateIndex.Value)))
             {
                 setDate = true;
             }
@@ -67,13 +71,22 @@ namespace Disco.Services.Devices.Importing.Fields
 
         public override bool Apply(DiscoDataContext Database, Device Device)
         {
-            if (this.FieldAction == EntityState.Modified)
+            if (FieldAction == EntityState.Modified)
             {
                 // Decommission or Recommission Device
-                Device.DecommissionReason = this.parsedValue;
+                Device.DecommissionReason = parsedValue;
 
                 if (setDate)
-                    Device.DecommissionedDate = this.parsedValue.HasValue ? (DateTime?)DateTime.Now : null;
+                {
+                    if (parsedValue.HasValue && !Device.DecommissionedDate.HasValue)
+                    {
+                        Device.DecommissionedDate = DateTime.Now;
+                    }
+                    else if (!parsedValue.HasValue && Device.DecommissionedDate.HasValue)
+                    {
+                        Device.DecommissionedDate = null;
+                    }
+                }
 
                 return true;
             }
@@ -83,19 +96,50 @@ namespace Disco.Services.Devices.Importing.Fields
             }
         }
 
-        public override int? GuessHeader(DiscoDataContext Database, DeviceImportContext Context)
+        public override void Applied(DiscoDataContext Database, Device Device, ref bool DeviceADDescriptionSet)
+        {
+            // Only Enable/Disable if DeviceDecommissionedDate field is not present
+            if (setDate)
+            {
+                if (ActiveDirectory.IsValidDomainAccountId(Device.DeviceDomainId))
+                {
+                    var adAccount = Device.ActiveDirectoryAccount();
+
+                    if (adAccount != null && !adAccount.IsCriticalSystemObject)
+                    {
+                        if (Device.DecommissionedDate.HasValue)
+                        {
+                            // Disable AD Account
+                            adAccount.DisableAccount();
+                        }
+                        else
+                        {
+                            // Enable AD Account
+                            adAccount.EnableAccount();
+                        }
+
+                        if (!DeviceADDescriptionSet)
+                        {
+                            adAccount.SetDescription(Device);
+                            DeviceADDescriptionSet = true;
+                        }
+                    }
+                }
+            }
+        }
+
+        public override int? GuessColumn(DiscoDataContext Database, IDeviceImportContext Context, IDeviceImportDataReader DataReader)
         {
             // column name
-            var possibleColumns = Context.Header
-                .Select((h, i) => Tuple.Create(h, i))
-                .Where(h => h.Item1.Item2 == DeviceImportFieldTypes.IgnoreColumn &&
-                    h.Item1.Item1.IndexOf("decommission reason", System.StringComparison.OrdinalIgnoreCase) >= 0 ||
-                    h.Item1.Item1.IndexOf("decommissionreason", System.StringComparison.OrdinalIgnoreCase) >= 0 ||
-                    h.Item1.Item1.IndexOf("decommissioned reason", System.StringComparison.OrdinalIgnoreCase) >= 0 ||
-                    h.Item1.Item1.IndexOf("decommissionedreason", System.StringComparison.OrdinalIgnoreCase) >= 0
+            var possibleColumns = Context.Columns
+                .Where(h => h.Type == DeviceImportFieldTypes.IgnoreColumn &&
+                    h.Name.IndexOf("decommission reason", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                    h.Name.IndexOf("decommissionreason", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                    h.Name.IndexOf("decommissioned reason", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                    h.Name.IndexOf("decommissionedreason", StringComparison.OrdinalIgnoreCase) >= 0
                     );
 
-            return possibleColumns.Select(h => (int?)h.Item2).FirstOrDefault();
+            return possibleColumns.Select(h => (int?)h.Index).FirstOrDefault();
         }
 
         public static Dictionary<string, DecommissionReasons> BuildDecommissionReasonsMap()

Disco.Services/Devices/Importing/Fields/DeviceLocationImportField.cs

@@ -19,13 +19,15 @@ namespace Disco.Services.Devices.Importing.Fields
         public override string FriendlyValue { get { return parsedValue; } }
         public override string FriendlyPreviousValue { get { return previousValue; } }
 
-        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, DeviceImportContext Context, int RecordIndex, string DeviceSerialNumber, Device ExistingDevice, Dictionary<DeviceImportFieldTypes, string> Values, string Value)
+        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, IDeviceImportContext Context, string DeviceSerialNumber, Device ExistingDevice, List<IDeviceImportRecord> PreviousRecords, IDeviceImportDataReader DataReader, int ColumnIndex)
         {
-            if (string.IsNullOrWhiteSpace(Value))
+            var value = DataReader.GetString(ColumnIndex);
+
+            if (string.IsNullOrWhiteSpace(value))
                 parsedValue = null;
             else
             {
-                parsedValue = Value.Trim();
+                parsedValue = value.Trim();
                 if (parsedValue.Length > 250)
                     return Error("Cannot be more than 250 characters");
             }
@@ -43,10 +45,10 @@ namespace Disco.Services.Devices.Importing.Fields
 
         public override bool Apply(DiscoDataContext Database, Device Device)
         {
-            if (this.FieldAction == EntityState.Added ||
-                this.FieldAction == EntityState.Modified)
+            if (FieldAction == EntityState.Added ||
+                FieldAction == EntityState.Modified)
             {
-                Device.Location = this.parsedValue;
+                Device.Location = parsedValue;
                 return true;
             }
             else
@@ -55,14 +57,14 @@ namespace Disco.Services.Devices.Importing.Fields
             }
         }
 
-        public override int? GuessHeader(DiscoDataContext Database, DeviceImportContext Context)
+        public override int? GuessColumn(DiscoDataContext Database, IDeviceImportContext Context, IDeviceImportDataReader DataReader)
         {
             // column name
-            var possibleColumns = Context.Header
-                .Select((h, i) => Tuple.Create(h, i))
-                .Where(h => h.Item1.Item2 == DeviceImportFieldTypes.IgnoreColumn && h.Item1.Item1.IndexOf("location", System.StringComparison.OrdinalIgnoreCase) >= 0);
+            var possibleColumns = Context.Columns
+                .Where(h => h.Type == DeviceImportFieldTypes.IgnoreColumn &&
+                    h.Name.IndexOf("location", StringComparison.OrdinalIgnoreCase) >= 0);
 
-            return possibleColumns.Select(h => (int?)h.Item2).FirstOrDefault();
+            return possibleColumns.Select(h => (int?)h.Index).FirstOrDefault();
         }
     }
 }

Disco.Services/Devices/Importing/Fields/DeviceSerialNumberImportField.cs

@@ -19,14 +19,16 @@ namespace Disco.Services.Devices.Importing.Fields
         public override string FriendlyValue { get { return parsedValue; } }
         public override string FriendlyPreviousValue { get { return parsedValue; } }
 
-        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, DeviceImportContext Context, int RecordIndex, string DeviceSerialNumber, Device ExistingDevice, Dictionary<DeviceImportFieldTypes, string> Values, string Value)
+        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, IDeviceImportContext Context, string DeviceSerialNumber, Device ExistingDevice, List<IDeviceImportRecord> PreviousRecords, IDeviceImportDataReader DataReader, int ColumnIndex)
         {
+            var value = DataReader.GetString(ColumnIndex);
+
             // Validate
-            if (string.IsNullOrWhiteSpace(Value))
+            if (string.IsNullOrWhiteSpace(value))
                 return Error("The Device Serial Number is required");
             else
             {
-                parsedValue = Value.Trim();
+                parsedValue = value.Trim();
                 if (parsedValue.Length > maxLength)
                     return Error($"Cannot be more than {maxLength} characters");
                 if (parsedValue.Contains(@"/"))
@@ -36,13 +38,11 @@ namespace Disco.Services.Devices.Importing.Fields
             }
 
             // Duplicate
-            var duplicate = Context.RawData
-                .Take(RecordIndex)
-                .Select((r, i) => Tuple.Create(i, ParseRawDeviceSerialNumber(r[Context.HeaderDeviceSerialNumberIndex])))
-                .Where(r => IsDeviceSerialNumberValid(r.Item2))
-                .FirstOrDefault(r => r.Item2.Equals(parsedValue, StringComparison.OrdinalIgnoreCase));
+            var duplicate = PreviousRecords
+                .Where(r => IsDeviceSerialNumberValid(r.DeviceSerialNumber))
+                .FirstOrDefault(r => r.DeviceSerialNumber.Equals(parsedValue, StringComparison.OrdinalIgnoreCase));
             if (duplicate != null)
-                return Error($"This Device Serial Number was already present on Row {duplicate.Item1 + 1}");
+                return Error($"This Device Serial Number was already present on Row {DataReader.GetRowNumber(duplicate.Index)}");
 
             // No action required
             return Success(EntityState.Unchanged);
@@ -54,18 +54,14 @@ namespace Disco.Services.Devices.Importing.Fields
             return false;
         }
 
-        public override int? GuessHeader(DiscoDataContext Database, DeviceImportContext Context)
+        public override int? GuessColumn(DiscoDataContext Database, IDeviceImportContext Context, IDeviceImportDataReader DataReader)
         {
             // 'serial' in column name
-            var possibleColumns = Context.Header
-                .Select((h, i) => Tuple.Create(h, i))
-                .Where(h => h.Item1.Item2 == DeviceImportFieldTypes.IgnoreColumn && h.Item1.Item1.IndexOf("serial", System.StringComparison.OrdinalIgnoreCase) >= 0);
+            var possibleColumns = Context.Columns
+                .Where(h => h.Type == DeviceImportFieldTypes.IgnoreColumn && h.Name.IndexOf("serial", StringComparison.OrdinalIgnoreCase) >= 0);
 
             // All Values
-            possibleColumns = possibleColumns.Where(h =>
-            {
-                return Context.RawData.Select(v => v[h.Item2]).All(v => !string.IsNullOrWhiteSpace(v));
-            }).ToList();
+            possibleColumns = possibleColumns.Where(h => DataReader.TestAllNotEmpty(h.Index)).ToList();
 
             if (possibleColumns.Count() > 1)
             {
@@ -74,17 +70,17 @@ namespace Disco.Services.Devices.Importing.Fields
                 {
                     try
                     {
-                        var top50SerialNumbers = Context.RawData.Select(v => v[h.Item2]).Take(50).ToArray();
+                        var top50SerialNumbers = DataReader.GetStrings(h.Index).Take(50).ToList();
                         return Database.Devices.Count(d => top50SerialNumbers.Contains(d.SerialNumber)) > 0;
                     }
                     catch (Exception) { return false; }
-                }).Select(h => (int?)h.Item2).FirstOrDefault();
+                }).Select(h => (int?)h.Index).FirstOrDefault();
 
                 if (possibleColumnIndex.HasValue)
                     return possibleColumnIndex;
             }
 
-            return possibleColumns.Select(h => (int?)h.Item2).FirstOrDefault();
+            return possibleColumns.Select(h => (int?)h.Index).FirstOrDefault();
         }
 
         public static string ParseRawDeviceSerialNumber(string DeviceSerialNumber)
@@ -98,5 +94,6 @@ namespace Disco.Services.Devices.Importing.Fields
         {
             return DeviceSerialNumber != null && DeviceSerialNumber.Length <= maxLength;
         }
+
     }
 }

Disco.Services/Devices/Importing/Fields/ModelIdImportField.cs

@@ -1,6 +1,7 @@
 using Disco.Data.Repository;
 using Disco.Models.Repository;
 using Disco.Models.Services.Devices.Importing;
+using Disco.Services.Interop.ActiveDirectory;
 using System;
 using System.Collections.Generic;
 using System.Data;
@@ -20,23 +21,37 @@ namespace Disco.Services.Devices.Importing.Fields
         public override string FriendlyValue { get { return friendlyValue; } }
         public override string FriendlyPreviousValue { get { return friendlyPreviousValue; } }
 
-        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, DeviceImportContext Context, int RecordIndex, string DeviceSerialNumber, Device ExistingDevice, Dictionary<DeviceImportFieldTypes, string> Values, string Value)
+        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, IDeviceImportContext Context, string DeviceSerialNumber, Device ExistingDevice, List<IDeviceImportRecord> PreviousRecords, IDeviceImportDataReader DataReader, int ColumnIndex)
         {
-            friendlyValue = Value;
+            int? intValue;
+            if (DataReader.TryGetNullableInt(ColumnIndex, out intValue))
+            {
+                if (!intValue.HasValue)
+                {
+                    if (ExistingDevice == null)
+                    {
+                        intValue = 1; // Default Model for new devices
+                    }
+                    else
+                    {
+                        return Error("The Model Identifier cannot be blank");
+                    }
+                }
 
-            // Validate
-            if (string.IsNullOrWhiteSpace(Value))
-                this.parsedValue = 1; // Default Model
+                parsedValue = intValue.Value;
+                friendlyValue = parsedValue.ToString();
+            }
             else
-                if (!int.TryParse(Value, out parsedValue))
-                    return Error("The Model Identifier must be a number");
+            {
+                return Error("The Model Identifier must be a number");
+            }
 
             var m = Cache.DeviceModels.FirstOrDefault(dm => dm.Id == parsedValue);
 
             if (m == null)
-                return Error(string.Format("The identifier ({0}) does not match any Device Model", Value));
+                return Error($"The identifier ({parsedValue}) does not match any Device Model");
 
-            friendlyValue = string.Format("{0} [{1}]", m.Description, m.Id);
+            friendlyValue = $"{m.Description} [{m.Id}]";
 
             if (ExistingDevice == null)
                 return Success(EntityState.Added);
@@ -46,7 +61,7 @@ namespace Disco.Services.Devices.Importing.Fields
                 if (ExistingDevice.DeviceModelId.HasValue)
                 {
                     var previousModel = Cache.DeviceModels.FirstOrDefault(dm => dm.Id == ExistingDevice.DeviceModelId.Value);
-                    friendlyPreviousValue = string.Format("{0} [{1}]", previousModel.Description, previousModel.Id);
+                    friendlyPreviousValue = $"{previousModel.Description} [{previousModel.Id}]";
                 }
 
                 return Success(EntityState.Modified);
@@ -57,10 +72,10 @@ namespace Disco.Services.Devices.Importing.Fields
 
         public override bool Apply(DiscoDataContext Database, Device Device)
         {
-            if (this.FieldAction == EntityState.Added ||
-                this.FieldAction == EntityState.Modified)
+            if (FieldAction == EntityState.Added ||
+                FieldAction == EntityState.Modified)
             {
-                Device.DeviceModelId = this.parsedValue;
+                Device.DeviceModelId = parsedValue;
                 return true;
             }
             else
@@ -69,30 +84,43 @@ namespace Disco.Services.Devices.Importing.Fields
             }
         }
 
-        public override int? GuessHeader(DiscoDataContext Database, DeviceImportContext Context)
+        public override void Applied(DiscoDataContext Database, Device Device, ref bool DeviceADDescriptionSet)
         {
-            // 'model' in column name
-            var possibleColumns = Context.Header
-                .Select((h, i) => Tuple.Create(h, i))
-                .Where(h => h.Item1.Item2 == DeviceImportFieldTypes.IgnoreColumn && h.Item1.Item1.IndexOf("model", System.StringComparison.OrdinalIgnoreCase) >= 0);
+            if (!DeviceADDescriptionSet)
+            {
+                if (ActiveDirectory.IsValidDomainAccountId(Device.DeviceDomainId))
+                {
+                    var adAccount = Device.ActiveDirectoryAccount();
+
+                    if (adAccount != null && !adAccount.IsCriticalSystemObject)
+                    {
+                        adAccount.SetDescription(Device);
+                        DeviceADDescriptionSet = true;
+                    }
+                }
+            }
+        }
+
+        public override int? GuessColumn(DiscoDataContext Database, IDeviceImportContext Context, IDeviceImportDataReader DataReader)
+        {
+            // column name
+            var possibleColumns = Context.Columns
+                .Where(h => h.Type == DeviceImportFieldTypes.IgnoreColumn &&
+                    h.Name.IndexOf("model", StringComparison.OrdinalIgnoreCase) >= 0);
 
             // All Integers Numbers
-            possibleColumns = possibleColumns.Where(h =>
-            {
-                int lastValue;
-                return Context.RawData.Select(v => v[h.Item2]).Take(100).Where(v => !string.IsNullOrWhiteSpace(v)).All(v => int.TryParse(v, out lastValue));
-            }).ToList();
+            possibleColumns = possibleColumns.Where(h => DataReader.TestAllInt(h.Index)).ToList();
 
             // Multiple Columns, tighten column definition
             if (possibleColumns.Count() > 1)
             {
                 possibleColumns = possibleColumns
                     .Where(h =>
-                        h.Item1.Item1.IndexOf("modelid", StringComparison.OrdinalIgnoreCase) >= 0 ||
-                        h.Item1.Item1.IndexOf("model id", StringComparison.OrdinalIgnoreCase) >= 0);
+                        h.Name.IndexOf("modelid", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                        h.Name.IndexOf("model id", StringComparison.OrdinalIgnoreCase) >= 0);
             }
 
-            return possibleColumns.Select(h => (int?)h.Item2).FirstOrDefault();
+            return possibleColumns.Select(h => (int?)h.Index).FirstOrDefault();
         }
     }
 }

Disco.Services/Devices/Importing/Fields/ProfileIdImportField.cs

@@ -1,6 +1,7 @@
 using Disco.Data.Repository;
 using Disco.Models.Repository;
 using Disco.Models.Services.Devices.Importing;
+using Disco.Services.Interop.ActiveDirectory;
 using System;
 using System.Collections.Generic;
 using System.Data;
@@ -20,30 +21,44 @@ namespace Disco.Services.Devices.Importing.Fields
         public override string FriendlyValue { get { return friendlyValue; } }
         public override string FriendlyPreviousValue { get { return friendlyPreviousValue; } }
 
-        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, DeviceImportContext Context, int RecordIndex, string DeviceSerialNumber, Device ExistingDevice, Dictionary<DeviceImportFieldTypes, string> Values, string Value)
+        public override bool Parse(DiscoDataContext Database, IDeviceImportCache Cache, IDeviceImportContext Context, string DeviceSerialNumber, Device ExistingDevice, List<IDeviceImportRecord> PreviousRecords, IDeviceImportDataReader DataReader, int ColumnIndex)
         {
-            friendlyValue = Value;
+            int? intValue;
+            if (DataReader.TryGetNullableInt(ColumnIndex, out intValue))
+            {
+                if (!intValue.HasValue)
+                {
+                    if (ExistingDevice == null)
+                    {
+                        intValue = Database.DiscoConfiguration.DeviceProfiles.DefaultAddDeviceOfflineDeviceProfileId; // Default Model for new devices
+                    }
+                    else
+                    {
+                        return Error("The Profile Identifier cannot be blank");
+                    }
+                }
 
-            // Validate
-            if (string.IsNullOrWhiteSpace(Value))
-                this.parsedValue = Database.DiscoConfiguration.DeviceProfiles.DefaultAddDeviceOfflineDeviceProfileId;
+                parsedValue = intValue.Value;
+                friendlyValue = parsedValue.ToString();
+            }
             else
-                if (!int.TryParse(Value, out parsedValue))
-                    return Error("The Profile Identifier must be a number");
+            {
+                return Error("The Profile Identifier must be a number");
+            }
 
             var p = Cache.DeviceProfiles.FirstOrDefault(dp => dp.Id == parsedValue);
 
             if (p == null)
-                return Error(string.Format("The identifier ({0}) does not match any Device Profile", Value));
+                return Error($"The identifier ({parsedValue}) does not match any Device Profile");
 
-            friendlyValue = string.Format("{0} [{1}]", p.Description, p.Id);
+            friendlyValue = $"{p.Description} [{p.Id}]";
 
             if (ExistingDevice == null)
                 return Success(EntityState.Added);
             else if (ExistingDevice != null && ExistingDevice.DeviceProfileId != parsedValue)
             {
                 var previousProfile = Cache.DeviceProfiles.FirstOrDefault(dp => dp.Id == ExistingDevice.DeviceProfileId);
-                friendlyPreviousValue = string.Format("{0} [{1}]", previousProfile.Description, previousProfile.Id);
+                friendlyPreviousValue = $"{previousProfile.Description} [{previousProfile.Id}]";
 
                 return Success(EntityState.Modified);
             }
@@ -53,10 +68,10 @@ namespace Disco.Services.Devices.Importing.Fields
 
         public override bool Apply(DiscoDataContext Database, Device Device)
         {
-            if (this.FieldAction == EntityState.Added ||
-                this.FieldAction == EntityState.Modified)
+            if (FieldAction == EntityState.Added ||
+                FieldAction == EntityState.Modified)
             {
-                Device.DeviceProfileId = this.parsedValue;
+                Device.DeviceProfileId = parsedValue;
                 return true;
             }
             else
@@ -65,30 +80,43 @@ namespace Disco.Services.Devices.Importing.Fields
             }
         }
 
-        public override int? GuessHeader(DiscoDataContext Database, DeviceImportContext Context)
+        public override void Applied(DiscoDataContext Database, Device Device, ref bool DeviceADDescriptionSet)
+        {
+            if (!DeviceADDescriptionSet)
+            {
+                if (ActiveDirectory.IsValidDomainAccountId(Device.DeviceDomainId))
+                {
+                    var adAccount = Device.ActiveDirectoryAccount();
+
+                    if (adAccount != null && !adAccount.IsCriticalSystemObject)
+                    {
+                        adAccount.SetDescription(Device);
+                        DeviceADDescriptionSet = true;
+                    }
+                }
+            }
+        }
+
+        public override int? GuessColumn(DiscoDataContext Database, IDeviceImportContext Context, IDeviceImportDataReader DataReader)
         {
             // column name
-            var possibleColumns = Context.Header
-                .Select((h, i) => Tuple.Create(h, i))
-                .Where(h => h.Item1.Item2 == DeviceImportFieldTypes.IgnoreColumn && h.Item1.Item1.IndexOf("profile", System.StringComparison.OrdinalIgnoreCase) >= 0);
+            var possibleColumns = Context.Columns
+                .Where(h => h.Type == DeviceImportFieldTypes.IgnoreColumn &&
+                    h.Name.IndexOf("profile", StringComparison.OrdinalIgnoreCase) >= 0);
 
             // All Integers Numbers
-            possibleColumns = possibleColumns.Where(h =>
-            {
-                int lastValue;
-                return Context.RawData.Select(v => v[h.Item2]).Take(100).Where(v => !string.IsNullOrWhiteSpace(v)).All(v => int.TryParse(v, out lastValue));
-            }).ToList();
+            possibleColumns = possibleColumns.Where(h => DataReader.TestAllInt(h.Index)).ToList();
 
             // Multiple Columns, tighten column definition
             if (possibleColumns.Count() > 1)
             {
                 possibleColumns = possibleColumns
                     .Where(h =>
-                        h.Item1.Item1.IndexOf("profileid", StringComparison.OrdinalIgnoreCase) >= 0 ||
-                        h.Item1.Item1.IndexOf("profile id", StringComparison.OrdinalIgnoreCase) >= 0);
+                        h.Name.IndexOf("profileid", StringComparison.OrdinalIgnoreCase) >= 0 ||
+                        h.Name.IndexOf("profile id", StringComparison.OrdinalIgnoreCase) >= 0);
             }
 
-            return possibleColumns.Select(h => (int?)h.Item2).FirstOrDefault();
+            return possibleColumns.Select(h => (int?)h.Index).FirstOrDefault();
         }
     }
 }

Disco.Services/Devices/Importing/XlsxDeviceImportContext.cs

@@ -0,0 +1,133 @@
+using ClosedXML.Excel;
+using Disco.Models.Services.Devices.Importing;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+
+namespace Disco.Services.Devices.Importing
+{
+    public class XlsxDeviceImportContext : BaseDeviceImportContext
+    {
+        private bool hasHeaderRow;
+        private List<object[]> rawData;
+
+        public XlsxDeviceImportContext(string Filename, bool HasHeaderRow, Stream XlsxStream)
+            : base(Filename)
+        {
+            hasHeaderRow = HasHeaderRow;
+
+            using (var stream = new MemoryStream((int)XlsxStream.Length))
+            {
+                XlsxStream.CopyTo(stream);
+
+                ParseXlsx(stream);
+            }
+        }
+
+        public override int RecordCount
+        {
+            get
+            {
+                return rawData.Count;
+            }
+        }
+
+        public override IDeviceImportDataReader GetDataReader()
+        {
+            return new XlsxDeviceImportDataReader(this, rawData, hasHeaderRow);
+        }
+
+        private void ParseXlsx(Stream XlsxStream)
+        {
+            using (var xlWorkbook = new XLWorkbook(XlsxStream, XLEventTracking.Disabled))
+            {
+                if (xlWorkbook.Worksheets.Count == 0)
+                    throw new IndexOutOfRangeException("Workbook contains no worksheets");
+
+                // Use first worksheet
+                var worksheet = xlWorkbook.Worksheets.Worksheet(1);
+
+                SetDatasetName($"{Filename} [Sheet: {worksheet.Name}]");
+
+                var columnCount = worksheet.LastColumnUsed().ColumnNumber();
+
+                if (hasHeaderRow)
+                {
+                    var headerRow = worksheet.FirstRow();
+                    SetColumns(Enumerable.Range(1, columnCount)
+                        .Select(i =>
+                        {
+                            var cell = headerRow.Cell(i);
+                            var headerName = cell.GetString();
+                            if (string.IsNullOrWhiteSpace(headerName))
+                            {
+                                headerName = $"Column {cell.WorksheetColumn().ColumnLetter()}";
+                            }
+                            return new DeviceImportColumn()
+                            {
+                                Index = i - 1,
+                                Name = headerName,
+                                Type = DeviceImportFieldTypes.IgnoreColumn
+                            };
+                        }));
+                }
+                else
+                {
+                    SetColumns(Enumerable.Range(1, columnCount)
+                        .Select(i => new DeviceImportColumn()
+                        {
+                            Index = i - 1,
+                            Name = $"Column {worksheet.Column(i).ColumnLetter()}",
+                            Type = DeviceImportFieldTypes.IgnoreColumn
+                        }));
+                }
+
+                // Import Data
+                var rawData = new List<object[]>();
+                foreach (var row in worksheet.RowsUsed().Skip(hasHeaderRow ? 1 : 0))
+                {
+                    var record = new object[columnCount];
+                    rawData.Add(record);
+
+                    for (int columnIndex = 0; columnIndex < columnCount; columnIndex++)
+                    {
+                        var cell = row.Cell(columnIndex + 1);
+                        var cellValue = cell.Value;
+
+                        switch (cell.DataType)
+                        {
+                            case XLCellValues.Number:
+                                if (cellValue is double)
+                                {
+                                    record[columnIndex] = (double)cellValue;
+                                }
+                                continue;
+                            case XLCellValues.Boolean:
+                                if (cellValue is bool)
+                                {
+                                    record[columnIndex] = (bool)cellValue;
+                                }
+                                continue;
+                            case XLCellValues.DateTime:
+                                if (cellValue is DateTime)
+                                {
+                                    record[columnIndex] = (DateTime)cellValue;
+                                }
+                                continue;
+                        }
+
+                        var stringValue = cellValue == null ? null : cellValue.ToString();
+                        if (stringValue != null && stringValue.Length == 0)
+                        {
+                            stringValue = null;
+                        }
+                        record[columnIndex] = stringValue;
+                    }
+                }
+
+                this.rawData = rawData;
+            }
+        }
+    }
+}

Disco.Services/Devices/Importing/XlsxDeviceImportDataReader.cs

@@ -0,0 +1,264 @@
+using Disco.Models.Services.Devices.Importing;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+
+namespace Disco.Services.Devices.Importing
+{
+    public class XlsxDeviceImportDataReader : IDeviceImportDataReader
+    {
+        private static string[] TrueValues = { "true", "1", "yes", "-1", "on" };
+        private static string[] FalseValues = { "false", "0", "no", "off" };
+
+        private XlsxDeviceImportContext context;
+        private List<object[]> rawData;
+        private int currentRowIndex;
+        private int rowOffset;
+        private object[] currentRow;
+
+        public int Index { get { return currentRowIndex; } }
+
+        public XlsxDeviceImportDataReader(XlsxDeviceImportContext Context, List<object[]> RawData, bool HasHeaderRow)
+        {
+            context = Context;
+            rawData = RawData;
+            currentRowIndex = 0;
+            rowOffset = currentRowIndex = HasHeaderRow ? 2 : 1;
+        }
+
+        public void Reset()
+        {
+            currentRowIndex = 0;
+            currentRow = null;
+        }
+
+        public bool Read()
+        {
+            if (++currentRowIndex >= rawData.Count)
+            {
+                currentRowIndex--;
+                return false;
+            }
+
+            currentRow = rawData[currentRowIndex];
+            return true;
+        }
+
+        public int GetRowNumber(int Index)
+        {
+            return Index + rowOffset;
+        }
+
+        public string GetString(int ColumnIndex)
+        {
+            if (currentRow == null)
+                throw new InvalidOperationException($"{nameof(XlsxDeviceImportDataReader.Read)} must be called before retrieving values");
+
+            var cell = currentRow[ColumnIndex];
+
+            if (cell == null)
+                return null;
+            else
+                return cell.ToString();
+        }
+
+        public IEnumerable<string> GetStrings(int ColumnIndex)
+        {
+            return rawData.Select(r => r[ColumnIndex]?.ToString());
+        }
+
+        public bool TryGetNullableInt(int ColumnIndex, out int? value)
+        {
+            if (currentRow == null)
+                throw new InvalidOperationException($"{nameof(XlsxDeviceImportDataReader.Read)} must be called before retrieving values");
+
+            return TryGetNullableInt(currentRow[ColumnIndex], out value);
+        }
+
+        public bool TryGetNullableBool(int ColumnIndex, out bool? value)
+        {
+            if (currentRow == null)
+                throw new InvalidOperationException($"{nameof(XlsxDeviceImportDataReader.Read)} must be called before retrieving values");
+
+            return TryGetNullableBool(currentRow[ColumnIndex], out value);
+        }
+
+        public bool TryGetNullableDateTime(int ColumnIndex, out DateTime? value)
+        {
+            if (currentRow == null)
+                throw new InvalidOperationException($"{nameof(XlsxDeviceImportDataReader.Read)} must be called before retrieving values");
+
+            return TryGetNullableDateTime(currentRow[ColumnIndex], out value);
+        }
+
+        public bool TestAllNotEmpty(int ColumnIndex)
+        {
+            return GetStrings(ColumnIndex).All(s => !string.IsNullOrWhiteSpace(s));
+        }
+
+        public bool TestAllNullableInt(int ColumnIndex)
+        {
+            return rawData.Select(r => r[ColumnIndex])
+                .All(c =>
+                {
+                    int? value;
+                    return TryGetNullableInt(c, out value);
+                });
+        }
+
+        public bool TestAllInt(int ColumnIndex)
+        {
+            return rawData.Select(r => r[ColumnIndex])
+                .All(c =>
+                {
+                    int? value;
+                    return TryGetNullableInt(c, out value) && value.HasValue;
+                });
+        }
+
+        public bool TestAllNullableBool(int ColumnIndex)
+        {
+            return rawData.Select(r => r[ColumnIndex])
+                .All(c =>
+                {
+                    bool? value;
+                    return TryGetNullableBool(c, out value);
+                });
+        }
+
+        public bool TestAllNullableDateTime(int ColumnIndex)
+        {
+            return rawData.Select(r => r[ColumnIndex])
+                .All(c =>
+                {
+                    DateTime? value;
+                    return TryGetNullableDateTime(c, out value);
+                });
+        }
+
+        public void Dispose()
+        {
+            // Nothing to dispose
+        }
+
+        private bool TryGetNullableDateTime(object Content, out DateTime? value)
+        {
+            if (Content == null)
+            {
+                value = null;
+                return true;
+            }
+
+            if (Content is DateTime)
+            {
+                value = (DateTime)Content;
+                return true;
+            }
+
+            var stringValue = Content.ToString();
+
+            if (string.IsNullOrWhiteSpace(stringValue))
+            {
+                value = null;
+                return true;
+            }
+            else
+            {
+                stringValue = stringValue.Trim();
+
+                DateTime valueDateTime;
+                if (DateTime.TryParse(stringValue, CultureInfo.CurrentCulture, DateTimeStyles.AssumeLocal, out valueDateTime))
+                {
+                    value = valueDateTime;
+                    return true;
+                }
+                else
+                {
+                    value = null;
+                    return false;
+                }
+            }
+        }
+
+        private bool TryGetNullableBool(object Content, out bool? value)
+        {
+            if (Content == null)
+            {
+                value = null;
+                return true;
+            }
+
+            if (Content is bool)
+            {
+                value = (bool)Content;
+                return true;
+            }
+
+            var stringValue = Content.ToString();
+
+            if (string.IsNullOrWhiteSpace(stringValue))
+            {
+                value = null;
+                return true;
+            }
+            else
+            {
+                stringValue = stringValue.Trim();
+
+                if (TrueValues.Contains(stringValue, StringComparer.OrdinalIgnoreCase))
+                {
+                    value = true;
+                    return true;
+                }
+                else if (FalseValues.Contains(stringValue, StringComparer.OrdinalIgnoreCase))
+                {
+                    value = false;
+                    return true;
+                }
+                else
+                {
+                    value = null;
+                    return false;
+                }
+            }
+        }
+
+        private bool TryGetNullableInt(object Content, out int? value)
+        {
+            if (Content == null)
+            {
+                value = null;
+                return true;
+            }
+
+            if (Content is double)
+            {
+                value = (int)((double)Content);
+                return true;
+            }
+
+            var stringValue = Content.ToString();
+
+            if (string.IsNullOrWhiteSpace(stringValue))
+            {
+                value = null;
+                return true;
+            }
+            else
+            {
+                int intValue;
+                if (int.TryParse(stringValue, out intValue))
+                {
+                    value = intValue;
+                    return true;
+                }
+                else
+                {
+                    value = null;
+                    return false;
+                }
+            }
+        }
+    }
+}