From 85425d2a1f018123f86e6f9fa77773bab811c271 Mon Sep 17 00:00:00 2001 From: Gary Sharp Date: Wed, 21 Sep 2016 19:55:00 +1000 Subject: [PATCH] Improve QR binary encoding compression --- .../Documents/AttachmentImport/Importer.cs | 2 +- .../Documents/DocumentUniqueIdentifier.cs | 219 +++++++------ .../DocumentUniqueIdentifierExtensions.cs | 300 ++++++++++++++++++ .../Documents/QRCodeBinaryEncoder.cs | 28 +- 4 files changed, 442 insertions(+), 107 deletions(-) diff --git a/Disco.Services/Documents/AttachmentImport/Importer.cs b/Disco.Services/Documents/AttachmentImport/Importer.cs index 933dcc1b..1c48320e 100644 --- a/Disco.Services/Documents/AttachmentImport/Importer.cs +++ b/Disco.Services/Documents/AttachmentImport/Importer.cs @@ -65,7 +65,7 @@ namespace Disco.Services.Documents.AttachmentImport page.WriteThumbnailSessionCache(); DocumentsLog.LogImportPageImageUpdate(SessionId, pageNumber); var identifier = page.Identifier; - DocumentsLog.LogImportPageDetected(SessionId, pageNumber, identifier.DocumentTemplateId, identifier.DocumentTemplate.Description, identifier.DocumentTemplate.Scope, identifier.TargetId, identifier.Target.ToString()); + DocumentsLog.LogImportPageDetected(SessionId, pageNumber, identifier.DocumentTemplate.Id, identifier.DocumentTemplate.Description, identifier.DocumentTemplate.Scope, identifier.Target.AttachmentReferenceId, identifier.Target.ToString()); } else { diff --git a/Disco.Services/Documents/DocumentUniqueIdentifier.cs b/Disco.Services/Documents/DocumentUniqueIdentifier.cs index f09507a2..8bc70d39 100644 --- a/Disco.Services/Documents/DocumentUniqueIdentifier.cs +++ b/Disco.Services/Documents/DocumentUniqueIdentifier.cs @@ -37,6 +37,7 @@ namespace Disco.Services.Documents if (documentTemplate != null) { attachmentType = documentTemplate.AttachmentType; + DocumentTemplateId = documentTemplate.Id; } } @@ -108,6 +109,10 @@ namespace Disco.Services.Documents default: throw new ArgumentException("Unexpected Attachment Type", nameof(AttachmentType)); } + if (target != null) + { + TargetId = target.AttachmentReferenceId; + } } return target; @@ -121,6 +126,10 @@ namespace Disco.Services.Documents if (creator == null) { creator = database.Users.Find(ActiveDirectory.ParseDomainAccountId(CreatorId)); + if (creator != null) + { + CreatorId = creator.UserId; + } } return creator; } @@ -176,85 +185,95 @@ namespace Disco.Services.Documents public byte[] ToQRCodeBytes() { // Byte | Meaning - // 0 | magic number = 0x0D - // 1 | bits 0-3 = version; 4-7 = flags (1 = has document template id, 2 = device attachment, - // | 4 = job attachment, 8 = user attachment) + // 0 | magic number = 0xC4 + // 1 | bits 0-3 = version; + // | 4 = flag: has document template id + // | 5-6 = (01 = device attachment, 10 = job attachment, 11 = user attachment) + // | 7 = not used // 2-3 | deployment checksum (int16) // 4-7 | timestamp (uint32 unix epoch) // 8-9 | page index (uint16) - // 10 | creator id length - // 11-? | creator id (UTF8) - // ? | target id length - // ?-? | target id - // ? | document template id length (optional based on flag) - // ?-? | document template id (UTF8, optional based on flag) + // 10 | creator id encoded + // ? | target id encoded + // ? | document template id encoded (optional based on flag) + const int version = 2; - var encoding = Encoding.UTF8; - byte flags = 0; + // encode variable-length parameters first + // encode creator id (strip default domain) + var creatorIdBytes = this.BinaryEncode(ActiveDirectory.FriendlyAccountId(CreatorId)); + + // encode target id + byte[] targetIdBytes; + if (AttachmentType.HasValue && AttachmentType.Value == AttachmentTypes.User) + { + // strip default domain from user targetted attachments + targetIdBytes = this.BinaryEncode(ActiveDirectory.FriendlyAccountId(TargetId)); + } + else + { + targetIdBytes = this.BinaryEncode(TargetId); + } + + byte[] documentTemplateIdBytes = null; + if (DocumentTemplateId != null) + { + documentTemplateIdBytes = this.BinaryEncode(DocumentTemplateId); + } + + var result = new byte[10 + creatorIdBytes.Length + targetIdBytes.Length + (documentTemplateIdBytes?.Length ?? 0)]; + + // write magic number + result[0] = MagicNumber; + + // write version + result[1] = (version << 4); + + // write 'has document template id' flag + if (documentTemplateIdBytes != null) + { + result[1] |= 0x8; // 0000 1000 + } + + // write attachment type switch (AttachmentType) { case AttachmentTypes.Device: - flags = 2; + result[1] |= 0x2; // 0000 0010 - 01 break; case AttachmentTypes.Job: - flags = 4; + result[1] |= 0x4; // 0000 0100 - 10 break; case AttachmentTypes.User: - flags = 8; + result[1] |= 0x6; // 0000 0110 - 11 break; } - var deploymentChecksumBytes = BitConverter.GetBytes(DeploymentChecksum); - var timeStampEpochBytes = BitConverter.GetBytes((uint)(TimeStamp.ToUniversalTime().Subtract(DateTime.FromFileTimeUtc(116444736000000000L)).Ticks / TimeSpan.TicksPerSecond)); - var pageIndexBytes = BitConverter.GetBytes((ushort)PageIndex); + // write deployment checksum + result[2] = (byte)(DeploymentChecksum >> 8); + result[3] = (byte)DeploymentChecksum; + // write timestamp + var timestamp = (uint)(TimeStamp.ToUniversalTime().Subtract(DateTime.FromFileTimeUtc(116444736000000000L)).Ticks / TimeSpan.TicksPerSecond); + result[4] = (byte)(timestamp >> 24); + result[5] = (byte)(timestamp >> 16); + result[6] = (byte)(timestamp >> 8); + result[7] = (byte)timestamp; - // magic number (1) + version/flags (1) + deployment checksum (2) + timestamp (4) + - // page index (2) + creator id length (1) + target id length (1) - var creatorIdLength = encoding.GetByteCount(CreatorId); - var targetIdLength = encoding.GetByteCount(TargetId); - var requiredBytes = 12 + creatorIdLength + targetIdLength; - var documentTemplateIdLength = 0; + // write page index + result[8] = (byte)(PageIndex >> 8); + result[9] = (byte)PageIndex; - if (DocumentTemplateId != null) + // write creator id + creatorIdBytes.CopyTo(result, 10); + + // write target id + targetIdBytes.CopyTo(result, 10 + creatorIdBytes.Length); + + // write document template id + if (documentTemplateIdBytes != null) { - flags |= 1; - requiredBytes++; - documentTemplateIdLength = encoding.GetByteCount(DocumentTemplateId); - requiredBytes += documentTemplateIdLength; - } - - int position = 0; - var result = new byte[requiredBytes]; - - // magic number - result[position++] = MagicNumber; - // version & flags - result[position++] = (byte)(flags | (2 << 4)); - // deployment checksum - deploymentChecksumBytes.CopyTo(result, position); - position += 2; - // timestamp - timeStampEpochBytes.CopyTo(result, position); - position += 4; - // page index - pageIndexBytes.CopyTo(result, position); - position += 2; - // creator id length - result[position++] = (byte)creatorIdLength; - // creator id - position += encoding.GetBytes(CreatorId, 0, CreatorId.Length, result, position); - // target id length - result[position++] = (byte)targetIdLength; - // target id - position += encoding.GetBytes(TargetId, 0, TargetId.Length, result, position); - if (documentTemplateIdLength > 0) - { - // document template id length - result[position++] = (byte)documentTemplateIdLength; - // document template id - position += encoding.GetBytes(DocumentTemplateId, 0, DocumentTemplateId.Length, result, position); + documentTemplateIdBytes.CopyTo(result, 10 + creatorIdBytes.Length + targetIdBytes.Length); } return result; @@ -339,68 +358,74 @@ namespace Disco.Services.Documents return false; } - public static bool TryParse(DiscoDataContext Database, byte[] UniqueIdentifier, out DocumentUniqueIdentifier Identifier) + public static bool TryParse(DiscoDataContext Database, byte[] Data, out DocumentUniqueIdentifier Identifier) { - if (IsDocumentUniqueIdentifier(UniqueIdentifier)) + if (IsDocumentUniqueIdentifier(Data)) { // first 4 bit indicate version - var version = UniqueIdentifier[1] >> 4; + var version = Data[1] >> 4; // Version 2 if (version == 2) { // Byte | Meaning - // 0 | magic number = 0x0D - // 1 | bits 0-3 = version; 4-7 = flags (1 = has document template id, 2 = device attachment, - // | 4 = job attachment, 8 = user attachment) + // 0 | magic number = 0xC4 + // 1 | bits 0-3 = version; + // | 4 = flag: has document template id + // | 5-6 = (01 = device attachment, 10 = job attachment, 11 = user attachment) + // | 7 = not used // 2-3 | deployment checksum (int16) // 4-7 | timestamp (uint32 unix epoch) // 8-9 | page index (uint16) - // 10 | creator id length - // 11-? | creator id (UTF8) - // ? | target id length - // ?-? | target id - // ? | document template id length (optional based on flag) - // ?-? | document template id (UTF8, optional based on flag) - var encoding = Encoding.UTF8; - var position = 1; + // 10 | creator id encoded + // ? | target id encoded + // ? | document template id encoded (optional based on flag) - // next 4 bits are flags - var flags = UniqueIdentifier[position++] & 0x0F; + // read flags + var flags = Data[1] & 0x0F; - var deploymentChecksum = BitConverter.ToInt16(UniqueIdentifier, position); - position += 2; + // read deployment checksum + short deploymentChecksum = (short)((Data[2] << 8) | Data[3]); - var timeStampEpoch = BitConverter.ToUInt32(UniqueIdentifier, position); - position += 4; + // read timestamp + var timeStampEpoch = ((long)Data[4] << 24) | + ((long)Data[5] << 16) | + ((long)Data[6] << 8) | + Data[7]; - var pageIndex = BitConverter.ToUInt16(UniqueIdentifier, position); - position += 2; + // write page index + var pageIndex = (Data[8] << 8) | Data[9]; - var creatorIdLength = UniqueIdentifier[position++]; - var creatorId = encoding.GetString(UniqueIdentifier, position, creatorIdLength); - position += creatorIdLength; + var position = 10; - var targetIdLength = UniqueIdentifier[position++]; - var targetId = encoding.GetString(UniqueIdentifier, position, targetIdLength); - position += targetIdLength; + // write creator id + var creatorId = DocumentUniqueIdentifierExtensions.BinaryDecode(Data, position, out position); + // write target id + var targetId = DocumentUniqueIdentifierExtensions.BinaryDecode(Data, position, out position); + + // write document template id string documentTemplateId = null; // Has document template id flag - if ((flags & 1) == 1) + if ((flags & 0x8) == 0x8) { - var documentTemplateIdLength = UniqueIdentifier[position++]; - documentTemplateId = encoding.GetString(UniqueIdentifier, position, documentTemplateIdLength); + documentTemplateId = DocumentUniqueIdentifierExtensions.BinaryDecode(Data, position, out position); } AttachmentTypes? attachmentType = null; - if ((flags & 2) == 2) - attachmentType = AttachmentTypes.Device; - else if ((flags & 4) == 4) - attachmentType = AttachmentTypes.Job; - else if ((flags & 8) == 8) - attachmentType = AttachmentTypes.User; + switch (flags & 0x6) + { + case 0x2: + attachmentType = AttachmentTypes.Device; + break; + case 0x4: + attachmentType = AttachmentTypes.Job; + break; + case 0x6: + attachmentType = AttachmentTypes.User; + break; + } var timeStamp = DateTime.FromFileTimeUtc(116444736000000000L).AddTicks(TimeSpan.TicksPerSecond * timeStampEpoch).ToLocalTime(); diff --git a/Disco.Services/Documents/DocumentUniqueIdentifierExtensions.cs b/Disco.Services/Documents/DocumentUniqueIdentifierExtensions.cs index 76682e09..9fcee845 100644 --- a/Disco.Services/Documents/DocumentUniqueIdentifierExtensions.cs +++ b/Disco.Services/Documents/DocumentUniqueIdentifierExtensions.cs @@ -17,5 +17,305 @@ namespace Disco.Services return DocumentUniqueIdentifier.Create(Database, Template, Target, Creator, Timestamp, PageIndex); } + public static byte[] BinaryEncode(this DocumentUniqueIdentifier Identifier, string Data) + { + return BinaryEncode(Data); + } + + public static byte[] BinaryEncode(string Data) + { + if (Data == null) + throw new ArgumentNullException(nameof(Data)); + + byte[] result; + + if (Data.Length == 0) + { + // Zero-length Alpha Encode (1 byte) + return new byte[] { 0x40 }; + } + + // Try Numeric Encode + if (TryBinaryNumericEncode(Data, out result)) + { + return result; + } + + // Try CASES21 ST/DF Key Encode + if (TryBinaryC21Encode(Data, out result)) + { + return result; + } + + // Try Alpha Encode + if (TryBinaryAlphaEncode(Data, out result)) + { + return result; + } + + // Use UTF8 Encoding + return BinaryUTF8Encode(Data); + } + + public static string BinaryDecode(this DocumentUniqueIdentifier Identifier, byte[] Data, int Offset, out int NewOffset) + { + return BinaryDecode(Data, Offset, out NewOffset); + } + + public static string BinaryDecode(byte[] Data, int Offset, out int NewOffset) + { + int encoding = Data[Offset] >> 6; + + switch (encoding) + { + case 0: // 00 - numeric encoding + return BinaryNumericDecode(Data, Offset, out NewOffset); + case 1: // 01 - alpha encoding + return BinaryAlphaDecode(Data, Offset, out NewOffset); + case 2: // 10 - C21 encoding + return BinaryC21Decode(Data, Offset, out NewOffset); + case 3: // 11 - UTF8 encoding + return BinaryUTF8Decode(Data, Offset, out NewOffset); + default: + throw new InvalidOperationException(); + } + } + + public static bool TryBinaryNumericEncode(string Data, out byte[] Result) + { + // byte[0] = XXZZ YYYY + // byte[1] = YYYY YYYY + // byte[2] = YYYY YYYY + // byte[3] = YYYY YYYY + // X = 00 = numeric encoded + // Z = number of leading zeros + // Y = number component < 0x0FFFFFFF (268,435,455) + + uint number; + if (uint.TryParse(Data, out number) && number <= 0x0FFFFFFF) + { + Result = new byte[4]; + int leadingZeros = 0; + + for (leadingZeros = 0; leadingZeros <= 4; leadingZeros++) + { + if (Data[leadingZeros] != '0') + { + break; + } + } + + if (leadingZeros <= 3) + { + Result[0] = (byte)((byte)(leadingZeros << 4) | (number >> 24)); + Result[1] = (byte)(number >> 16); + Result[2] = (byte)(number >> 8); + Result[3] = (byte)(number); + return true; + } + } + + Result = null; + return false; + } + + public static string BinaryNumericDecode(byte[] Data, int Offset, out int NewOffset) + { + int leadingZeros = (Data[Offset] & 0x30) >> 4; + int number = ((Data[Offset] & 0x0F) << 24) | + (Data[Offset + 1] << 16) | + (Data[Offset + 2] << 8) | + (Data[Offset + 3]); + + NewOffset = Offset + 4; + + if (leadingZeros == 0) + { + return number.ToString(); + } + else + { + var builder = new StringBuilder(12); // 12 = max number length + builder.Append('0', leadingZeros); + builder.Append(number); + return builder.ToString(); + } + } + + private const string AlphaEncodeMap = @"ABCDEFGHIJKLMNOPQRSTUVWXYZ-.01\"; + + public static bool TryBinaryAlphaEncode(string Data, out byte[] Result) + { + // byte[0] = XXYY YYYY + // X = 01 = alpha encoded + // Y = data length <= 63 + // byte[1.] = AAAA ABBB + // byte[2.] = BBCD DDDD + // A = first character index + // B = second character index + // C = not used + // D = third character index + + if (Data.Length == 0) + { + // Zero-length Alpha Encode (1 byte) + Result = new byte[] { 0x40 }; + return true; + } + + if (Data.Length > 0 && Data.Length <= 63) + { + Data = Data.ToUpperInvariant(); + var position = 1; + var requiredBytes = ((Data.Length / 3) * 2); + switch (Data.Length % 3) + { + case 2: + requiredBytes += 3; + break; + case 1: + requiredBytes += 2; + break; + default: + requiredBytes++; + break; + } + + Result = new byte[requiredBytes]; + Result[0] = (byte)(0x40 | Data.Length); + for (int i = 0; i < Data.Length; i++) + { + var charIndex = AlphaEncodeMap.IndexOf(Data[i]); + if (charIndex == -1) + { + Result = null; + return false; + } + switch (i % 3) + { + case 0: + Result[position] = (byte)(charIndex << 3); + break; + case 1: + Result[position] = (byte)(Result[position] | (charIndex >> 2)); + Result[++position] = (byte)(charIndex << 6); + break; + case 2: + Result[position] = (byte)(Result[position] | charIndex); + position++; + break; + } + } + + return true; + } + Result = null; + return false; + } + + public static string BinaryAlphaDecode(byte[] Data, int Offset, out int NewOffset) + { + var length = Data[Offset++] & 0x3F; + return BinaryAlphaDecode(Data, Offset, length, out NewOffset); + } + + private static string BinaryAlphaDecode(byte[] Data, int Offset, int Length, out int NewOffset) + { + var builder = new StringBuilder(Length); + + for (int i = 0; i < Length; i++) + { + switch (i % 3) + { + case 0: + builder.Append(AlphaEncodeMap[Data[Offset++] >> 3]); + break; + case 1: + builder.Append(AlphaEncodeMap[((Data[Offset - 1] & 0x7) << 2) | ((Data[Offset++] >> 6) & 0x3)]); + break; + case 2: + builder.Append(AlphaEncodeMap[Data[Offset - 1] & 0x1F]); + break; + } + } + + NewOffset = Offset; + return builder.ToString(); + } + + public static bool TryBinaryC21Encode(string Data, out byte[] Result) + { + // byte[0] = XXYY YYYY + // byte[1] = YYYY YYYY + // X = 10 = C21 encoded + // Y = number component + // byte[2] = AAAA ABBB + // byte[3] = BBCC CCCD + // A,B,C = character component in + // alpha encoded format + + short number; + byte[] chars; + if (Data.Length == 7 && + short.TryParse(Data.Substring(3), out number) && + number <= 9999 && + TryBinaryAlphaEncode(Data.Substring(0, 3), out chars)) + { + Result = new byte[4]; + Result[0] = (byte)(0x80 | (number >> 8)); + Result[1] = (byte)number; + Result[2] = chars[1]; + Result[3] = chars[2]; + return true; + } + + Result = null; + return false; + } + + public static string BinaryC21Decode(byte[] Data, int Offset, out int NewOffset) + { + var number = ((Data[Offset++] & 0x3F) << 8) | + (Data[Offset++]); + var chars = BinaryAlphaDecode(Data, Offset, 3, out NewOffset); + + return $"{chars}{number:0000}"; + } + + public static byte[] BinaryUTF8Encode(string Data) + { + // byte[0] = XXYY YYYY + // X = 11 = UTF8 encoded + // Y = data length <= 63 + // byte[.] = AAAA AAAA + // A = UTF8 encoded string + + if (Data.Length == 0) + { + // Zero-length Alpha Encode (1 byte) + return new byte[] { 0xC0 }; + } + + if (Data.Length <= 63) + { + var utf8Bytes = Encoding.UTF8.GetBytes(Data); + if (utf8Bytes.Length <= 63) + { + var result = new byte[1 + utf8Bytes.Length]; + result[0] = (byte)(0xC0 | utf8Bytes.Length); + utf8Bytes.CopyTo(result, 1); + return result; + } + } + + throw new ArgumentException("Unable to encode the data. The input data is to long."); + } + + public static string BinaryUTF8Decode(byte[] Data, int Offset, out int NewOffset) + { + var length = Data[Offset] & 0x3F; + NewOffset = Offset + length + 1; + return Encoding.UTF8.GetString(Data, Offset + 1, length); + } } } diff --git a/Disco.Services/Documents/QRCodeBinaryEncoder.cs b/Disco.Services/Documents/QRCodeBinaryEncoder.cs index fd57e1e3..74b663a6 100644 --- a/Disco.Services/Documents/QRCodeBinaryEncoder.cs +++ b/Disco.Services/Documents/QRCodeBinaryEncoder.cs @@ -19,7 +19,7 @@ namespace Disco.Services.Documents mode.getCharacterCountBits(ZXing.QrCode.Internal.Version.getVersionForNumber(1)) + (Content.Length * 8); - var version = ChooseVersion(bitsNeeded, ecLevel); + var version = ChooseVersion(bitsNeeded, out ecLevel); var ecBlocks = version.getECBlocksForLevel(ecLevel); var totalByteCapacity = version.TotalCodewords - ecBlocks.TotalECCodewords; var totalBitCapacity = totalByteCapacity << 3; @@ -75,8 +75,11 @@ namespace Disco.Services.Documents return scaleMatrix(matrix.Array, Width, Height); } - private static ZXing.QrCode.Internal.Version ChooseVersion(int RequiredBits, ErrorCorrectionLevel ECLevel) + private static ZXing.QrCode.Internal.Version ChooseVersion(int RequiredBits, out ErrorCorrectionLevel ECLevel) { + var ecls = new ErrorCorrectionLevel[] { ErrorCorrectionLevel.H, ErrorCorrectionLevel.Q, ErrorCorrectionLevel.M, ErrorCorrectionLevel.L }; + int totalInputBytes = (RequiredBits + 7) / 8; + // In the following comments, we use numbers of Version 7-H. for (int versionNum = 1; versionNum <= 40; versionNum++) { @@ -84,14 +87,21 @@ namespace Disco.Services.Documents // numBytes = 196 int numBytes = version.TotalCodewords; // getNumECBytes = 130 - var ecBlocks = version.getECBlocksForLevel(ECLevel); - int numEcBytes = ecBlocks.TotalECCodewords; - // getNumDataBytes = 196 - 130 = 66 - int numDataBytes = numBytes - numEcBytes; - int totalInputBytes = (RequiredBits + 7) / 8; - if (numDataBytes >= totalInputBytes) + + if (numBytes >= totalInputBytes) { - return version; + for (int ecl = 0; ecl < ecls.Length; ecl++) + { + var ecBlocks = version.getECBlocksForLevel(ecls[ecl]); + int numEcBytes = ecBlocks.TotalECCodewords; + // getNumDataBytes = 196 - 130 = 66 + int numDataBytes = numBytes - numEcBytes; + if (numDataBytes >= totalInputBytes) + { + ECLevel = ecls[ecl]; + return version; + } + } } } throw new ArgumentException("Data too big", nameof(RequiredBits));