Improve QR binary encoding compression

Disco.Services/Documents/AttachmentImport/Importer.cs

@@ -65,7 +65,7 @@ namespace Disco.Services.Documents.AttachmentImport
                                 page.WriteThumbnailSessionCache();
                                 DocumentsLog.LogImportPageImageUpdate(SessionId, pageNumber);
                                 var identifier = page.Identifier;
-                                DocumentsLog.LogImportPageDetected(SessionId, pageNumber, identifier.DocumentTemplateId, identifier.DocumentTemplate.Description, identifier.DocumentTemplate.Scope, identifier.TargetId, identifier.Target.ToString());
+                                DocumentsLog.LogImportPageDetected(SessionId, pageNumber, identifier.DocumentTemplate.Id, identifier.DocumentTemplate.Description, identifier.DocumentTemplate.Scope, identifier.Target.AttachmentReferenceId, identifier.Target.ToString());
                             }
                             else
                             {

Disco.Services/Documents/DocumentUniqueIdentifier.cs

@@ -37,6 +37,7 @@ namespace Disco.Services.Documents
                     if (documentTemplate != null)
                     {
                         attachmentType = documentTemplate.AttachmentType;
+                        DocumentTemplateId = documentTemplate.Id;
                     }
                 }
 
@@ -108,6 +109,10 @@ namespace Disco.Services.Documents
                         default:
                             throw new ArgumentException("Unexpected Attachment Type", nameof(AttachmentType));
                     }
+                    if (target != null)
+                    {
+                        TargetId = target.AttachmentReferenceId;
+                    }
                 }
 
                 return target;
@@ -121,6 +126,10 @@ namespace Disco.Services.Documents
                 if (creator == null)
                 {
                     creator = database.Users.Find(ActiveDirectory.ParseDomainAccountId(CreatorId));
+                    if (creator != null)
+                    {
+                        CreatorId = creator.UserId;
+                    }
                 }
                 return creator;
             }
@@ -176,85 +185,95 @@ namespace Disco.Services.Documents
         public byte[] ToQRCodeBytes()
         {
             // Byte | Meaning
-            // 0    | magic number = 0x0D
+            // 0    | magic number = 0xC4
-            // 1    | bits 0-3 = version; 4-7 = flags (1 = has document template id, 2 = device attachment,
+            // 1    | bits 0-3 = version;
-            //      |                                  4 = job attachment, 8 = user attachment)
+            //      |      4   = flag: has document template id
+            //      |      5-6 = (01 = device attachment, 10 = job attachment, 11 = user attachment)
+            //      |      7   = not used
             // 2-3  | deployment checksum (int16)
             // 4-7  | timestamp (uint32 unix epoch)
             // 8-9  | page index (uint16)
-            // 10   | creator id length
+            // 10   | creator id encoded
-            // 11-? | creator id (UTF8)
+            // ?    | target id encoded
-            // ?    | target id length
+            // ?    | document template id encoded (optional based on flag)
-            // ?-?  | target id
+            const int version = 2;
-            // ?    | document template id length (optional based on flag)
-            // ?-?  | document template id (UTF8, optional based on flag)
 
-            var encoding = Encoding.UTF8;
+            // encode variable-length parameters first
-            byte flags = 0;
 
+            // encode creator id (strip default domain)
+            var creatorIdBytes = this.BinaryEncode(ActiveDirectory.FriendlyAccountId(CreatorId));
+
+            // encode target id
+            byte[] targetIdBytes;
+            if (AttachmentType.HasValue && AttachmentType.Value == AttachmentTypes.User)
+            {
+                // strip default domain from user targetted attachments
+                targetIdBytes = this.BinaryEncode(ActiveDirectory.FriendlyAccountId(TargetId));
+            }
+            else
+            {
+                targetIdBytes = this.BinaryEncode(TargetId);
+            }
+
+            byte[] documentTemplateIdBytes = null;
+            if (DocumentTemplateId != null)
+            {
+                documentTemplateIdBytes = this.BinaryEncode(DocumentTemplateId);
+            }
+
+            var result = new byte[10 + creatorIdBytes.Length + targetIdBytes.Length + (documentTemplateIdBytes?.Length ?? 0)];
+
+            // write magic number
+            result[0] = MagicNumber;
+
+            // write version
+            result[1] = (version << 4);
+
+            // write 'has document template id' flag
+            if (documentTemplateIdBytes != null)
+            {
+                result[1] |= 0x8; // 0000 1000
+            }
+
+            // write attachment type
             switch (AttachmentType)
             {
                 case AttachmentTypes.Device:
-                    flags = 2;
+                    result[1] |= 0x2; // 0000 0010 - 01
                     break;
                 case AttachmentTypes.Job:
-                    flags = 4;
+                    result[1] |= 0x4; // 0000 0100 - 10
                     break;
                 case AttachmentTypes.User:
-                    flags = 8;
+                    result[1] |= 0x6; // 0000 0110 - 11
                     break;
             }
 
-            var deploymentChecksumBytes = BitConverter.GetBytes(DeploymentChecksum);
+            // write deployment checksum
-            var timeStampEpochBytes = BitConverter.GetBytes((uint)(TimeStamp.ToUniversalTime().Subtract(DateTime.FromFileTimeUtc(116444736000000000L)).Ticks / TimeSpan.TicksPerSecond));
+            result[2] = (byte)(DeploymentChecksum >> 8);
-            var pageIndexBytes = BitConverter.GetBytes((ushort)PageIndex);
+            result[3] = (byte)DeploymentChecksum;
 
+            // write timestamp
+            var timestamp = (uint)(TimeStamp.ToUniversalTime().Subtract(DateTime.FromFileTimeUtc(116444736000000000L)).Ticks / TimeSpan.TicksPerSecond);
+            result[4] = (byte)(timestamp >> 24);
+            result[5] = (byte)(timestamp >> 16);
+            result[6] = (byte)(timestamp >> 8);
+            result[7] = (byte)timestamp;
 
-            // magic number (1) + version/flags (1) + deployment checksum (2) + timestamp (4) +
+            // write page index
-            //  page index (2) + creator id length (1) + target id length (1)
+            result[8] = (byte)(PageIndex >> 8);
-            var creatorIdLength = encoding.GetByteCount(CreatorId);
+            result[9] = (byte)PageIndex;
-            var targetIdLength = encoding.GetByteCount(TargetId);
-            var requiredBytes = 12 + creatorIdLength + targetIdLength;
-            var documentTemplateIdLength = 0;
 
-            if (DocumentTemplateId != null)
+            // write creator id
+            creatorIdBytes.CopyTo(result, 10);
+
+            // write target id
+            targetIdBytes.CopyTo(result, 10 + creatorIdBytes.Length);
+
+            // write document template id
+            if (documentTemplateIdBytes != null)
             {
-                flags |= 1;
+                documentTemplateIdBytes.CopyTo(result, 10 + creatorIdBytes.Length + targetIdBytes.Length);
-                requiredBytes++;
-                documentTemplateIdLength = encoding.GetByteCount(DocumentTemplateId);
-                requiredBytes += documentTemplateIdLength;
-            }
-
-            int position = 0;
-            var result = new byte[requiredBytes];
-
-            // magic number
-            result[position++] = MagicNumber;
-            // version & flags
-            result[position++] = (byte)(flags | (2 << 4));
-            // deployment checksum
-            deploymentChecksumBytes.CopyTo(result, position);
-            position += 2;
-            // timestamp
-            timeStampEpochBytes.CopyTo(result, position);
-            position += 4;
-            // page index
-            pageIndexBytes.CopyTo(result, position);
-            position += 2;
-            // creator id length
-            result[position++] = (byte)creatorIdLength;
-            // creator id
-            position += encoding.GetBytes(CreatorId, 0, CreatorId.Length, result, position);
-            // target id length
-            result[position++] = (byte)targetIdLength;
-            // target id
-            position += encoding.GetBytes(TargetId, 0, TargetId.Length, result, position);
-            if (documentTemplateIdLength > 0)
-            {
-                // document template id length
-                result[position++] = (byte)documentTemplateIdLength;
-                // document template id
-                position += encoding.GetBytes(DocumentTemplateId, 0, DocumentTemplateId.Length, result, position);
             }
 
             return result;
@@ -339,68 +358,74 @@ namespace Disco.Services.Documents
             return false;
         }
 
-        public static bool TryParse(DiscoDataContext Database, byte[] UniqueIdentifier, out DocumentUniqueIdentifier Identifier)
+        public static bool TryParse(DiscoDataContext Database, byte[] Data, out DocumentUniqueIdentifier Identifier)
         {
-            if (IsDocumentUniqueIdentifier(UniqueIdentifier))
+            if (IsDocumentUniqueIdentifier(Data))
             {
                 // first 4 bit indicate version
-                var version = UniqueIdentifier[1] >> 4;
+                var version = Data[1] >> 4;
 
                 // Version 2
                 if (version == 2)
                 {
                     // Byte | Meaning
-                    // 0    | magic number = 0x0D
+                    // 0    | magic number = 0xC4
-                    // 1    | bits 0-3 = version; 4-7 = flags (1 = has document template id, 2 = device attachment,
+                    // 1    | bits 0-3 = version;
-                    //      |                                  4 = job attachment, 8 = user attachment)
+                    //      |      4   = flag: has document template id
+                    //      |      5-6 = (01 = device attachment, 10 = job attachment, 11 = user attachment)
+                    //      |      7   = not used
                     // 2-3  | deployment checksum (int16)
                     // 4-7  | timestamp (uint32 unix epoch)
                     // 8-9  | page index (uint16)
-                    // 10   | creator id length
+                    // 10   | creator id encoded
-                    // 11-? | creator id (UTF8)
+                    // ?    | target id encoded
-                    // ?    | target id length
+                    // ?    | document template id encoded (optional based on flag)
-                    // ?-?  | target id
-                    // ?    | document template id length (optional based on flag)
-                    // ?-?  | document template id (UTF8, optional based on flag)
-                    var encoding = Encoding.UTF8;
-                    var position = 1;
 
-                    // next 4 bits are flags
+                    // read flags
-                    var flags = UniqueIdentifier[position++] & 0x0F;
+                    var flags = Data[1] & 0x0F;
 
-                    var deploymentChecksum = BitConverter.ToInt16(UniqueIdentifier, position);
+                    // read deployment checksum
-                    position += 2;
+                    short deploymentChecksum = (short)((Data[2] << 8) | Data[3]);
 
-                    var timeStampEpoch = BitConverter.ToUInt32(UniqueIdentifier, position);
+                    // read timestamp
-                    position += 4;
+                    var timeStampEpoch = ((long)Data[4] << 24) |
+                        ((long)Data[5] << 16) |
+                        ((long)Data[6] << 8) |
+                        Data[7];
 
-                    var pageIndex = BitConverter.ToUInt16(UniqueIdentifier, position);
+                    // write page index
-                    position += 2;
+                    var pageIndex = (Data[8] << 8) | Data[9];
 
-                    var creatorIdLength = UniqueIdentifier[position++];
+                    var position = 10;
-                    var creatorId = encoding.GetString(UniqueIdentifier, position, creatorIdLength);
-                    position += creatorIdLength;
 
-                    var targetIdLength = UniqueIdentifier[position++];
+                    // write creator id
-                    var targetId = encoding.GetString(UniqueIdentifier, position, targetIdLength);
+                    var creatorId = DocumentUniqueIdentifierExtensions.BinaryDecode(Data, position, out position);
-                    position += targetIdLength;
 
+                    // write target id
+                    var targetId = DocumentUniqueIdentifierExtensions.BinaryDecode(Data, position, out position);
+
+                    // write document template id
                     string documentTemplateId = null;
 
                     // Has document template id flag
-                    if ((flags & 1) == 1)
+                    if ((flags & 0x8) == 0x8)
                     {
-                        var documentTemplateIdLength = UniqueIdentifier[position++];
+                        documentTemplateId = DocumentUniqueIdentifierExtensions.BinaryDecode(Data, position, out position);
-                        documentTemplateId = encoding.GetString(UniqueIdentifier, position, documentTemplateIdLength);
                     }
 
                     AttachmentTypes? attachmentType = null;
-                    if ((flags & 2) == 2)
+                    switch (flags & 0x6)
+                    {
+                        case 0x2:
                             attachmentType = AttachmentTypes.Device;
-                    else if ((flags & 4) == 4)
+                            break;
+                        case 0x4:
                             attachmentType = AttachmentTypes.Job;
-                    else if ((flags & 8) == 8)
+                            break;
+                        case 0x6:
                             attachmentType = AttachmentTypes.User;
+                            break;
+                    }
 
                     var timeStamp = DateTime.FromFileTimeUtc(116444736000000000L).AddTicks(TimeSpan.TicksPerSecond * timeStampEpoch).ToLocalTime();
 

Disco.Services/Documents/DocumentUniqueIdentifierExtensions.cs

@@ -17,5 +17,305 @@ namespace Disco.Services
             return DocumentUniqueIdentifier.Create(Database, Template, Target, Creator, Timestamp, PageIndex);
         }
 
+        public static byte[] BinaryEncode(this DocumentUniqueIdentifier Identifier, string Data)
+        {
+            return BinaryEncode(Data);
+        }
+
+        public static byte[] BinaryEncode(string Data)
+        {
+            if (Data == null)
+                throw new ArgumentNullException(nameof(Data));
+
+            byte[] result;
+
+            if (Data.Length == 0)
+            {
+                // Zero-length Alpha Encode (1 byte)
+                return new byte[] { 0x40 };
+            }
+
+            // Try Numeric Encode
+            if (TryBinaryNumericEncode(Data, out result))
+            {
+                return result;
+            }
+
+            // Try CASES21 ST/DF Key Encode
+            if (TryBinaryC21Encode(Data, out result))
+            {
+                return result;
+            }
+
+            // Try Alpha Encode
+            if (TryBinaryAlphaEncode(Data, out result))
+            {
+                return result;
+            }
+
+            // Use UTF8 Encoding
+            return BinaryUTF8Encode(Data);
+        }
+
+        public static string BinaryDecode(this DocumentUniqueIdentifier Identifier, byte[] Data, int Offset, out int NewOffset)
+        {
+            return BinaryDecode(Data, Offset, out NewOffset);
+        }
+
+        public static string BinaryDecode(byte[] Data, int Offset, out int NewOffset)
+        {
+            int encoding = Data[Offset] >> 6;
+
+            switch (encoding)
+            {
+                case 0: // 00 - numeric encoding
+                    return BinaryNumericDecode(Data, Offset, out NewOffset);
+                case 1: // 01 - alpha encoding
+                    return BinaryAlphaDecode(Data, Offset, out NewOffset);
+                case 2: // 10 - C21 encoding
+                    return BinaryC21Decode(Data, Offset, out NewOffset);
+                case 3: // 11 - UTF8 encoding
+                    return BinaryUTF8Decode(Data, Offset, out NewOffset);
+                default:
+                    throw new InvalidOperationException();
+            }
+        }
+
+        public static bool TryBinaryNumericEncode(string Data, out byte[] Result)
+        {
+            // byte[0] = XXZZ YYYY
+            // byte[1] = YYYY YYYY
+            // byte[2] = YYYY YYYY
+            // byte[3] = YYYY YYYY
+            //              X = 00 = numeric encoded
+            //              Z = number of leading zeros
+            //              Y = number component < 0x0FFFFFFF (268,435,455)
+
+            uint number;
+            if (uint.TryParse(Data, out number) && number <= 0x0FFFFFFF)
+            {
+                Result = new byte[4];
+                int leadingZeros = 0;
+
+                for (leadingZeros = 0; leadingZeros <= 4; leadingZeros++)
+                {
+                    if (Data[leadingZeros] != '0')
+                    {
+                        break;
+                    }
+                }
+
+                if (leadingZeros <= 3)
+                {
+                    Result[0] = (byte)((byte)(leadingZeros << 4) | (number >> 24));
+                    Result[1] = (byte)(number >> 16);
+                    Result[2] = (byte)(number >> 8);
+                    Result[3] = (byte)(number);
+                    return true;
+                }
+            }
+
+            Result = null;
+            return false;
+        }
+
+        public static string BinaryNumericDecode(byte[] Data, int Offset, out int NewOffset)
+        {
+            int leadingZeros = (Data[Offset] & 0x30) >> 4;
+            int number = ((Data[Offset] & 0x0F) << 24) |
+                (Data[Offset + 1] << 16) |
+                (Data[Offset + 2] << 8) |
+                (Data[Offset + 3]);
+
+            NewOffset = Offset + 4;
+
+            if (leadingZeros == 0)
+            {
+                return number.ToString();
+            }
+            else
+            {
+                var builder = new StringBuilder(12); // 12 = max number length
+                builder.Append('0', leadingZeros);
+                builder.Append(number);
+                return builder.ToString();
+            }
+        }
+
+        private const string AlphaEncodeMap = @"ABCDEFGHIJKLMNOPQRSTUVWXYZ-.01\";
+
+        public static bool TryBinaryAlphaEncode(string Data, out byte[] Result)
+        {
+            // byte[0] = XXYY YYYY
+            //              X = 01 = alpha encoded
+            //              Y = data length <= 63
+            // byte[1.] = AAAA ABBB
+            // byte[2.] = BBCD DDDD
+            //              A = first character index
+            //              B = second character index
+            //              C = not used
+            //              D = third character index
+
+            if (Data.Length == 0)
+            {
+                // Zero-length Alpha Encode (1 byte)
+                Result = new byte[] { 0x40 };
+                return true;
+            }
+
+            if (Data.Length > 0 && Data.Length <= 63)
+            {
+                Data = Data.ToUpperInvariant();
+                var position = 1;
+                var requiredBytes = ((Data.Length / 3) * 2);
+                switch (Data.Length % 3)
+                {
+                    case 2:
+                        requiredBytes += 3;
+                        break;
+                    case 1:
+                        requiredBytes += 2;
+                        break;
+                    default:
+                        requiredBytes++;
+                        break;
+                }
+
+                Result = new byte[requiredBytes];
+                Result[0] = (byte)(0x40 | Data.Length);
+                for (int i = 0; i < Data.Length; i++)
+                {
+                    var charIndex = AlphaEncodeMap.IndexOf(Data[i]);
+                    if (charIndex == -1)
+                    {
+                        Result = null;
+                        return false;
+                    }
+                    switch (i % 3)
+                    {
+                        case 0:
+                            Result[position] = (byte)(charIndex << 3);
+                            break;
+                        case 1:
+                            Result[position] = (byte)(Result[position] | (charIndex >> 2));
+                            Result[++position] = (byte)(charIndex << 6);
+                            break;
+                        case 2:
+                            Result[position] = (byte)(Result[position] | charIndex);
+                            position++;
+                            break;
+                    }
+                }
+
+                return true;
+            }
+            Result = null;
+            return false;
+        }
+
+        public static string BinaryAlphaDecode(byte[] Data, int Offset, out int NewOffset)
+        {
+            var length = Data[Offset++] & 0x3F;
+            return BinaryAlphaDecode(Data, Offset, length, out NewOffset);
+        }
+
+        private static string BinaryAlphaDecode(byte[] Data, int Offset, int Length, out int NewOffset)
+        {
+            var builder = new StringBuilder(Length);
+
+            for (int i = 0; i < Length; i++)
+            {
+                switch (i % 3)
+                {
+                    case 0:
+                        builder.Append(AlphaEncodeMap[Data[Offset++] >> 3]);
+                        break;
+                    case 1:
+                        builder.Append(AlphaEncodeMap[((Data[Offset - 1] & 0x7) << 2) | ((Data[Offset++] >> 6) & 0x3)]);
+                        break;
+                    case 2:
+                        builder.Append(AlphaEncodeMap[Data[Offset - 1] & 0x1F]);
+                        break;
+                }
+            }
+
+            NewOffset = Offset;
+            return builder.ToString();
+        }
+
+        public static bool TryBinaryC21Encode(string Data, out byte[] Result)
+        {
+            // byte[0] = XXYY YYYY
+            // byte[1] = YYYY YYYY
+            //              X = 10 = C21 encoded
+            //              Y = number component
+            // byte[2] = AAAA ABBB
+            // byte[3] = BBCC CCCD
+            //              A,B,C = character component in
+            //                      alpha encoded format
+
+            short number;
+            byte[] chars;
+            if (Data.Length == 7 &&
+                short.TryParse(Data.Substring(3), out number) &&
+                number <= 9999 &&
+                TryBinaryAlphaEncode(Data.Substring(0, 3), out chars))
+            {
+                Result = new byte[4];
+                Result[0] = (byte)(0x80 | (number >> 8));
+                Result[1] = (byte)number;
+                Result[2] = chars[1];
+                Result[3] = chars[2];
+                return true;
+            }
+
+            Result = null;
+            return false;
+        }
+
+        public static string BinaryC21Decode(byte[] Data, int Offset, out int NewOffset)
+        {
+            var number = ((Data[Offset++] & 0x3F) << 8) |
+                (Data[Offset++]);
+            var chars = BinaryAlphaDecode(Data, Offset, 3, out NewOffset);
+
+            return $"{chars}{number:0000}";
+        }
+
+        public static byte[] BinaryUTF8Encode(string Data)
+        {
+            // byte[0] = XXYY YYYY
+            //              X = 11 = UTF8 encoded
+            //              Y = data length <= 63
+            // byte[.] = AAAA AAAA
+            //              A = UTF8 encoded string
+
+            if (Data.Length == 0)
+            {
+                // Zero-length Alpha Encode (1 byte)
+                return new byte[] { 0xC0 };
+            }
+
+            if (Data.Length <= 63)
+            {
+                var utf8Bytes = Encoding.UTF8.GetBytes(Data);
+                if (utf8Bytes.Length <= 63)
+                {
+                    var result = new byte[1 + utf8Bytes.Length];
+                    result[0] = (byte)(0xC0 | utf8Bytes.Length);
+                    utf8Bytes.CopyTo(result, 1);
+                    return result;
+                }
+            }
+
+            throw new ArgumentException("Unable to encode the data. The input data is to long.");
+        }
+
+        public static string BinaryUTF8Decode(byte[] Data, int Offset, out int NewOffset)
+        {
+            var length = Data[Offset] & 0x3F;
+            NewOffset = Offset + length + 1;
+            return Encoding.UTF8.GetString(Data, Offset + 1, length);
+        }
     }
 }

Disco.Services/Documents/QRCodeBinaryEncoder.cs

@@ -19,7 +19,7 @@ namespace Disco.Services.Documents
                 mode.getCharacterCountBits(ZXing.QrCode.Internal.Version.getVersionForNumber(1)) +
                 (Content.Length * 8);
 
-            var version = ChooseVersion(bitsNeeded, ecLevel);
+            var version = ChooseVersion(bitsNeeded, out ecLevel);
             var ecBlocks = version.getECBlocksForLevel(ecLevel);
             var totalByteCapacity = version.TotalCodewords - ecBlocks.TotalECCodewords;
             var totalBitCapacity = totalByteCapacity << 3;
@@ -75,8 +75,11 @@ namespace Disco.Services.Documents
             return scaleMatrix(matrix.Array, Width, Height);
         }
 
-        private static ZXing.QrCode.Internal.Version ChooseVersion(int RequiredBits, ErrorCorrectionLevel ECLevel)
+        private static ZXing.QrCode.Internal.Version ChooseVersion(int RequiredBits, out ErrorCorrectionLevel ECLevel)
         {
+            var ecls = new ErrorCorrectionLevel[] { ErrorCorrectionLevel.H, ErrorCorrectionLevel.Q, ErrorCorrectionLevel.M, ErrorCorrectionLevel.L };
+            int totalInputBytes = (RequiredBits + 7) / 8;
+
             // In the following comments, we use numbers of Version 7-H.
             for (int versionNum = 1; versionNum <= 40; versionNum++)
             {
@@ -84,16 +87,23 @@ namespace Disco.Services.Documents
                 // numBytes = 196
                 int numBytes = version.TotalCodewords;
                 // getNumECBytes = 130
-                var ecBlocks = version.getECBlocksForLevel(ECLevel);
+
+                if (numBytes >= totalInputBytes)
+                {
+                    for (int ecl = 0; ecl < ecls.Length; ecl++)
+                    {
+                        var ecBlocks = version.getECBlocksForLevel(ecls[ecl]);
                         int numEcBytes = ecBlocks.TotalECCodewords;
                         // getNumDataBytes = 196 - 130 = 66
                         int numDataBytes = numBytes - numEcBytes;
-                int totalInputBytes = (RequiredBits + 7) / 8;
                         if (numDataBytes >= totalInputBytes)
                         {
+                            ECLevel = ecls[ecl];
                             return version;
                         }
                     }
+                }
+            }
             throw new ArgumentException("Data too big", nameof(RequiredBits));
         }