Improve QR binary encoding compression

This commit is contained in:
Gary Sharp
2016-09-21 19:55:00 +10:00
parent 4b866c9486
commit 85425d2a1f
4 changed files with 442 additions and 107 deletions
@@ -65,7 +65,7 @@ namespace Disco.Services.Documents.AttachmentImport
page.WriteThumbnailSessionCache();
DocumentsLog.LogImportPageImageUpdate(SessionId, pageNumber);
var identifier = page.Identifier;
DocumentsLog.LogImportPageDetected(SessionId, pageNumber, identifier.DocumentTemplateId, identifier.DocumentTemplate.Description, identifier.DocumentTemplate.Scope, identifier.TargetId, identifier.Target.ToString());
DocumentsLog.LogImportPageDetected(SessionId, pageNumber, identifier.DocumentTemplate.Id, identifier.DocumentTemplate.Description, identifier.DocumentTemplate.Scope, identifier.Target.AttachmentReferenceId, identifier.Target.ToString());
}
else
{
@@ -37,6 +37,7 @@ namespace Disco.Services.Documents
if (documentTemplate != null)
{
attachmentType = documentTemplate.AttachmentType;
DocumentTemplateId = documentTemplate.Id;
}
}
@@ -108,6 +109,10 @@ namespace Disco.Services.Documents
default:
throw new ArgumentException("Unexpected Attachment Type", nameof(AttachmentType));
}
if (target != null)
{
TargetId = target.AttachmentReferenceId;
}
}
return target;
@@ -121,6 +126,10 @@ namespace Disco.Services.Documents
if (creator == null)
{
creator = database.Users.Find(ActiveDirectory.ParseDomainAccountId(CreatorId));
if (creator != null)
{
CreatorId = creator.UserId;
}
}
return creator;
}
@@ -176,85 +185,95 @@ namespace Disco.Services.Documents
public byte[] ToQRCodeBytes()
{
// Byte | Meaning
// 0 | magic number = 0x0D
// 1 | bits 0-3 = version; 4-7 = flags (1 = has document template id, 2 = device attachment,
// | 4 = job attachment, 8 = user attachment)
// 0 | magic number = 0xC4
// 1 | bits 0-3 = version;
// | 4 = flag: has document template id
// | 5-6 = (01 = device attachment, 10 = job attachment, 11 = user attachment)
// | 7 = not used
// 2-3 | deployment checksum (int16)
// 4-7 | timestamp (uint32 unix epoch)
// 8-9 | page index (uint16)
// 10 | creator id length
// 11-? | creator id (UTF8)
// ? | target id length
// ?-? | target id
// ? | document template id length (optional based on flag)
// ?-? | document template id (UTF8, optional based on flag)
// 10 | creator id encoded
// ? | target id encoded
// ? | document template id encoded (optional based on flag)
const int version = 2;
var encoding = Encoding.UTF8;
byte flags = 0;
// encode variable-length parameters first
// encode creator id (strip default domain)
var creatorIdBytes = this.BinaryEncode(ActiveDirectory.FriendlyAccountId(CreatorId));
// encode target id
byte[] targetIdBytes;
if (AttachmentType.HasValue && AttachmentType.Value == AttachmentTypes.User)
{
// strip default domain from user targetted attachments
targetIdBytes = this.BinaryEncode(ActiveDirectory.FriendlyAccountId(TargetId));
}
else
{
targetIdBytes = this.BinaryEncode(TargetId);
}
byte[] documentTemplateIdBytes = null;
if (DocumentTemplateId != null)
{
documentTemplateIdBytes = this.BinaryEncode(DocumentTemplateId);
}
var result = new byte[10 + creatorIdBytes.Length + targetIdBytes.Length + (documentTemplateIdBytes?.Length ?? 0)];
// write magic number
result[0] = MagicNumber;
// write version
result[1] = (version << 4);
// write 'has document template id' flag
if (documentTemplateIdBytes != null)
{
result[1] |= 0x8; // 0000 1000
}
// write attachment type
switch (AttachmentType)
{
case AttachmentTypes.Device:
flags = 2;
result[1] |= 0x2; // 0000 0010 - 01
break;
case AttachmentTypes.Job:
flags = 4;
result[1] |= 0x4; // 0000 0100 - 10
break;
case AttachmentTypes.User:
flags = 8;
result[1] |= 0x6; // 0000 0110 - 11
break;
}
var deploymentChecksumBytes = BitConverter.GetBytes(DeploymentChecksum);
var timeStampEpochBytes = BitConverter.GetBytes((uint)(TimeStamp.ToUniversalTime().Subtract(DateTime.FromFileTimeUtc(116444736000000000L)).Ticks / TimeSpan.TicksPerSecond));
var pageIndexBytes = BitConverter.GetBytes((ushort)PageIndex);
// write deployment checksum
result[2] = (byte)(DeploymentChecksum >> 8);
result[3] = (byte)DeploymentChecksum;
// write timestamp
var timestamp = (uint)(TimeStamp.ToUniversalTime().Subtract(DateTime.FromFileTimeUtc(116444736000000000L)).Ticks / TimeSpan.TicksPerSecond);
result[4] = (byte)(timestamp >> 24);
result[5] = (byte)(timestamp >> 16);
result[6] = (byte)(timestamp >> 8);
result[7] = (byte)timestamp;
// magic number (1) + version/flags (1) + deployment checksum (2) + timestamp (4) +
// page index (2) + creator id length (1) + target id length (1)
var creatorIdLength = encoding.GetByteCount(CreatorId);
var targetIdLength = encoding.GetByteCount(TargetId);
var requiredBytes = 12 + creatorIdLength + targetIdLength;
var documentTemplateIdLength = 0;
// write page index
result[8] = (byte)(PageIndex >> 8);
result[9] = (byte)PageIndex;
if (DocumentTemplateId != null)
// write creator id
creatorIdBytes.CopyTo(result, 10);
// write target id
targetIdBytes.CopyTo(result, 10 + creatorIdBytes.Length);
// write document template id
if (documentTemplateIdBytes != null)
{
flags |= 1;
requiredBytes++;
documentTemplateIdLength = encoding.GetByteCount(DocumentTemplateId);
requiredBytes += documentTemplateIdLength;
}
int position = 0;
var result = new byte[requiredBytes];
// magic number
result[position++] = MagicNumber;
// version & flags
result[position++] = (byte)(flags | (2 << 4));
// deployment checksum
deploymentChecksumBytes.CopyTo(result, position);
position += 2;
// timestamp
timeStampEpochBytes.CopyTo(result, position);
position += 4;
// page index
pageIndexBytes.CopyTo(result, position);
position += 2;
// creator id length
result[position++] = (byte)creatorIdLength;
// creator id
position += encoding.GetBytes(CreatorId, 0, CreatorId.Length, result, position);
// target id length
result[position++] = (byte)targetIdLength;
// target id
position += encoding.GetBytes(TargetId, 0, TargetId.Length, result, position);
if (documentTemplateIdLength > 0)
{
// document template id length
result[position++] = (byte)documentTemplateIdLength;
// document template id
position += encoding.GetBytes(DocumentTemplateId, 0, DocumentTemplateId.Length, result, position);
documentTemplateIdBytes.CopyTo(result, 10 + creatorIdBytes.Length + targetIdBytes.Length);
}
return result;
@@ -339,68 +358,74 @@ namespace Disco.Services.Documents
return false;
}
public static bool TryParse(DiscoDataContext Database, byte[] UniqueIdentifier, out DocumentUniqueIdentifier Identifier)
public static bool TryParse(DiscoDataContext Database, byte[] Data, out DocumentUniqueIdentifier Identifier)
{
if (IsDocumentUniqueIdentifier(UniqueIdentifier))
if (IsDocumentUniqueIdentifier(Data))
{
// first 4 bit indicate version
var version = UniqueIdentifier[1] >> 4;
var version = Data[1] >> 4;
// Version 2
if (version == 2)
{
// Byte | Meaning
// 0 | magic number = 0x0D
// 1 | bits 0-3 = version; 4-7 = flags (1 = has document template id, 2 = device attachment,
// | 4 = job attachment, 8 = user attachment)
// 0 | magic number = 0xC4
// 1 | bits 0-3 = version;
// | 4 = flag: has document template id
// | 5-6 = (01 = device attachment, 10 = job attachment, 11 = user attachment)
// | 7 = not used
// 2-3 | deployment checksum (int16)
// 4-7 | timestamp (uint32 unix epoch)
// 8-9 | page index (uint16)
// 10 | creator id length
// 11-? | creator id (UTF8)
// ? | target id length
// ?-? | target id
// ? | document template id length (optional based on flag)
// ?-? | document template id (UTF8, optional based on flag)
var encoding = Encoding.UTF8;
var position = 1;
// 10 | creator id encoded
// ? | target id encoded
// ? | document template id encoded (optional based on flag)
// next 4 bits are flags
var flags = UniqueIdentifier[position++] & 0x0F;
// read flags
var flags = Data[1] & 0x0F;
var deploymentChecksum = BitConverter.ToInt16(UniqueIdentifier, position);
position += 2;
// read deployment checksum
short deploymentChecksum = (short)((Data[2] << 8) | Data[3]);
var timeStampEpoch = BitConverter.ToUInt32(UniqueIdentifier, position);
position += 4;
// read timestamp
var timeStampEpoch = ((long)Data[4] << 24) |
((long)Data[5] << 16) |
((long)Data[6] << 8) |
Data[7];
var pageIndex = BitConverter.ToUInt16(UniqueIdentifier, position);
position += 2;
// write page index
var pageIndex = (Data[8] << 8) | Data[9];
var creatorIdLength = UniqueIdentifier[position++];
var creatorId = encoding.GetString(UniqueIdentifier, position, creatorIdLength);
position += creatorIdLength;
var position = 10;
var targetIdLength = UniqueIdentifier[position++];
var targetId = encoding.GetString(UniqueIdentifier, position, targetIdLength);
position += targetIdLength;
// write creator id
var creatorId = DocumentUniqueIdentifierExtensions.BinaryDecode(Data, position, out position);
// write target id
var targetId = DocumentUniqueIdentifierExtensions.BinaryDecode(Data, position, out position);
// write document template id
string documentTemplateId = null;
// Has document template id flag
if ((flags & 1) == 1)
if ((flags & 0x8) == 0x8)
{
var documentTemplateIdLength = UniqueIdentifier[position++];
documentTemplateId = encoding.GetString(UniqueIdentifier, position, documentTemplateIdLength);
documentTemplateId = DocumentUniqueIdentifierExtensions.BinaryDecode(Data, position, out position);
}
AttachmentTypes? attachmentType = null;
if ((flags & 2) == 2)
attachmentType = AttachmentTypes.Device;
else if ((flags & 4) == 4)
attachmentType = AttachmentTypes.Job;
else if ((flags & 8) == 8)
attachmentType = AttachmentTypes.User;
switch (flags & 0x6)
{
case 0x2:
attachmentType = AttachmentTypes.Device;
break;
case 0x4:
attachmentType = AttachmentTypes.Job;
break;
case 0x6:
attachmentType = AttachmentTypes.User;
break;
}
var timeStamp = DateTime.FromFileTimeUtc(116444736000000000L).AddTicks(TimeSpan.TicksPerSecond * timeStampEpoch).ToLocalTime();
@@ -17,5 +17,305 @@ namespace Disco.Services
return DocumentUniqueIdentifier.Create(Database, Template, Target, Creator, Timestamp, PageIndex);
}
public static byte[] BinaryEncode(this DocumentUniqueIdentifier Identifier, string Data)
{
return BinaryEncode(Data);
}
public static byte[] BinaryEncode(string Data)
{
if (Data == null)
throw new ArgumentNullException(nameof(Data));
byte[] result;
if (Data.Length == 0)
{
// Zero-length Alpha Encode (1 byte)
return new byte[] { 0x40 };
}
// Try Numeric Encode
if (TryBinaryNumericEncode(Data, out result))
{
return result;
}
// Try CASES21 ST/DF Key Encode
if (TryBinaryC21Encode(Data, out result))
{
return result;
}
// Try Alpha Encode
if (TryBinaryAlphaEncode(Data, out result))
{
return result;
}
// Use UTF8 Encoding
return BinaryUTF8Encode(Data);
}
public static string BinaryDecode(this DocumentUniqueIdentifier Identifier, byte[] Data, int Offset, out int NewOffset)
{
return BinaryDecode(Data, Offset, out NewOffset);
}
public static string BinaryDecode(byte[] Data, int Offset, out int NewOffset)
{
int encoding = Data[Offset] >> 6;
switch (encoding)
{
case 0: // 00 - numeric encoding
return BinaryNumericDecode(Data, Offset, out NewOffset);
case 1: // 01 - alpha encoding
return BinaryAlphaDecode(Data, Offset, out NewOffset);
case 2: // 10 - C21 encoding
return BinaryC21Decode(Data, Offset, out NewOffset);
case 3: // 11 - UTF8 encoding
return BinaryUTF8Decode(Data, Offset, out NewOffset);
default:
throw new InvalidOperationException();
}
}
public static bool TryBinaryNumericEncode(string Data, out byte[] Result)
{
// byte[0] = XXZZ YYYY
// byte[1] = YYYY YYYY
// byte[2] = YYYY YYYY
// byte[3] = YYYY YYYY
// X = 00 = numeric encoded
// Z = number of leading zeros
// Y = number component < 0x0FFFFFFF (268,435,455)
uint number;
if (uint.TryParse(Data, out number) && number <= 0x0FFFFFFF)
{
Result = new byte[4];
int leadingZeros = 0;
for (leadingZeros = 0; leadingZeros <= 4; leadingZeros++)
{
if (Data[leadingZeros] != '0')
{
break;
}
}
if (leadingZeros <= 3)
{
Result[0] = (byte)((byte)(leadingZeros << 4) | (number >> 24));
Result[1] = (byte)(number >> 16);
Result[2] = (byte)(number >> 8);
Result[3] = (byte)(number);
return true;
}
}
Result = null;
return false;
}
public static string BinaryNumericDecode(byte[] Data, int Offset, out int NewOffset)
{
int leadingZeros = (Data[Offset] & 0x30) >> 4;
int number = ((Data[Offset] & 0x0F) << 24) |
(Data[Offset + 1] << 16) |
(Data[Offset + 2] << 8) |
(Data[Offset + 3]);
NewOffset = Offset + 4;
if (leadingZeros == 0)
{
return number.ToString();
}
else
{
var builder = new StringBuilder(12); // 12 = max number length
builder.Append('0', leadingZeros);
builder.Append(number);
return builder.ToString();
}
}
private const string AlphaEncodeMap = @"ABCDEFGHIJKLMNOPQRSTUVWXYZ-.01\";
public static bool TryBinaryAlphaEncode(string Data, out byte[] Result)
{
// byte[0] = XXYY YYYY
// X = 01 = alpha encoded
// Y = data length <= 63
// byte[1.] = AAAA ABBB
// byte[2.] = BBCD DDDD
// A = first character index
// B = second character index
// C = not used
// D = third character index
if (Data.Length == 0)
{
// Zero-length Alpha Encode (1 byte)
Result = new byte[] { 0x40 };
return true;
}
if (Data.Length > 0 && Data.Length <= 63)
{
Data = Data.ToUpperInvariant();
var position = 1;
var requiredBytes = ((Data.Length / 3) * 2);
switch (Data.Length % 3)
{
case 2:
requiredBytes += 3;
break;
case 1:
requiredBytes += 2;
break;
default:
requiredBytes++;
break;
}
Result = new byte[requiredBytes];
Result[0] = (byte)(0x40 | Data.Length);
for (int i = 0; i < Data.Length; i++)
{
var charIndex = AlphaEncodeMap.IndexOf(Data[i]);
if (charIndex == -1)
{
Result = null;
return false;
}
switch (i % 3)
{
case 0:
Result[position] = (byte)(charIndex << 3);
break;
case 1:
Result[position] = (byte)(Result[position] | (charIndex >> 2));
Result[++position] = (byte)(charIndex << 6);
break;
case 2:
Result[position] = (byte)(Result[position] | charIndex);
position++;
break;
}
}
return true;
}
Result = null;
return false;
}
public static string BinaryAlphaDecode(byte[] Data, int Offset, out int NewOffset)
{
var length = Data[Offset++] & 0x3F;
return BinaryAlphaDecode(Data, Offset, length, out NewOffset);
}
private static string BinaryAlphaDecode(byte[] Data, int Offset, int Length, out int NewOffset)
{
var builder = new StringBuilder(Length);
for (int i = 0; i < Length; i++)
{
switch (i % 3)
{
case 0:
builder.Append(AlphaEncodeMap[Data[Offset++] >> 3]);
break;
case 1:
builder.Append(AlphaEncodeMap[((Data[Offset - 1] & 0x7) << 2) | ((Data[Offset++] >> 6) & 0x3)]);
break;
case 2:
builder.Append(AlphaEncodeMap[Data[Offset - 1] & 0x1F]);
break;
}
}
NewOffset = Offset;
return builder.ToString();
}
public static bool TryBinaryC21Encode(string Data, out byte[] Result)
{
// byte[0] = XXYY YYYY
// byte[1] = YYYY YYYY
// X = 10 = C21 encoded
// Y = number component
// byte[2] = AAAA ABBB
// byte[3] = BBCC CCCD
// A,B,C = character component in
// alpha encoded format
short number;
byte[] chars;
if (Data.Length == 7 &&
short.TryParse(Data.Substring(3), out number) &&
number <= 9999 &&
TryBinaryAlphaEncode(Data.Substring(0, 3), out chars))
{
Result = new byte[4];
Result[0] = (byte)(0x80 | (number >> 8));
Result[1] = (byte)number;
Result[2] = chars[1];
Result[3] = chars[2];
return true;
}
Result = null;
return false;
}
public static string BinaryC21Decode(byte[] Data, int Offset, out int NewOffset)
{
var number = ((Data[Offset++] & 0x3F) << 8) |
(Data[Offset++]);
var chars = BinaryAlphaDecode(Data, Offset, 3, out NewOffset);
return $"{chars}{number:0000}";
}
public static byte[] BinaryUTF8Encode(string Data)
{
// byte[0] = XXYY YYYY
// X = 11 = UTF8 encoded
// Y = data length <= 63
// byte[.] = AAAA AAAA
// A = UTF8 encoded string
if (Data.Length == 0)
{
// Zero-length Alpha Encode (1 byte)
return new byte[] { 0xC0 };
}
if (Data.Length <= 63)
{
var utf8Bytes = Encoding.UTF8.GetBytes(Data);
if (utf8Bytes.Length <= 63)
{
var result = new byte[1 + utf8Bytes.Length];
result[0] = (byte)(0xC0 | utf8Bytes.Length);
utf8Bytes.CopyTo(result, 1);
return result;
}
}
throw new ArgumentException("Unable to encode the data. The input data is to long.");
}
public static string BinaryUTF8Decode(byte[] Data, int Offset, out int NewOffset)
{
var length = Data[Offset] & 0x3F;
NewOffset = Offset + length + 1;
return Encoding.UTF8.GetString(Data, Offset + 1, length);
}
}
}
@@ -19,7 +19,7 @@ namespace Disco.Services.Documents
mode.getCharacterCountBits(ZXing.QrCode.Internal.Version.getVersionForNumber(1)) +
(Content.Length * 8);
var version = ChooseVersion(bitsNeeded, ecLevel);
var version = ChooseVersion(bitsNeeded, out ecLevel);
var ecBlocks = version.getECBlocksForLevel(ecLevel);
var totalByteCapacity = version.TotalCodewords - ecBlocks.TotalECCodewords;
var totalBitCapacity = totalByteCapacity << 3;
@@ -75,8 +75,11 @@ namespace Disco.Services.Documents
return scaleMatrix(matrix.Array, Width, Height);
}
private static ZXing.QrCode.Internal.Version ChooseVersion(int RequiredBits, ErrorCorrectionLevel ECLevel)
private static ZXing.QrCode.Internal.Version ChooseVersion(int RequiredBits, out ErrorCorrectionLevel ECLevel)
{
var ecls = new ErrorCorrectionLevel[] { ErrorCorrectionLevel.H, ErrorCorrectionLevel.Q, ErrorCorrectionLevel.M, ErrorCorrectionLevel.L };
int totalInputBytes = (RequiredBits + 7) / 8;
// In the following comments, we use numbers of Version 7-H.
for (int versionNum = 1; versionNum <= 40; versionNum++)
{
@@ -84,14 +87,21 @@ namespace Disco.Services.Documents
// numBytes = 196
int numBytes = version.TotalCodewords;
// getNumECBytes = 130
var ecBlocks = version.getECBlocksForLevel(ECLevel);
int numEcBytes = ecBlocks.TotalECCodewords;
// getNumDataBytes = 196 - 130 = 66
int numDataBytes = numBytes - numEcBytes;
int totalInputBytes = (RequiredBits + 7) / 8;
if (numDataBytes >= totalInputBytes)
if (numBytes >= totalInputBytes)
{
return version;
for (int ecl = 0; ecl < ecls.Length; ecl++)
{
var ecBlocks = version.getECBlocksForLevel(ecls[ecl]);
int numEcBytes = ecBlocks.TotalECCodewords;
// getNumDataBytes = 196 - 130 = 66
int numDataBytes = numBytes - numEcBytes;
if (numDataBytes >= totalInputBytes)
{
ECLevel = ecls[ecl];
return version;
}
}
}
}
throw new ArgumentException("Data too big", nameof(RequiredBits));