From acfa0e5094c313ab5ce1d4f6a5fcffa4374d2cd9 Mon Sep 17 00:00:00 2001 From: Gary Sharp Date: Thu, 15 Sep 2016 19:27:52 +1000 Subject: [PATCH] Document Template Binary QR Codes The QR Code now stores the data using a custom binary encoding format. This reduces the amount of data stored by ~30%. This increases the chance of a lower-version QR Code being used and the likelyhood of even small QR Codes being detected. Compatibility with previously generated documents is maintained. --- Disco.BI/BI/Interop/Pdf/PdfGenerator.cs | 24 +- Disco.Services/Disco.Services.csproj | 9 +- .../Documents/DocumentUniqueIdentifier.cs | 12 +- .../Documents/QRCodeBinaryEncoder.cs | 285 ++++++++++++++++++ Disco.Services/packages.config | 2 +- 5 files changed, 317 insertions(+), 15 deletions(-) create mode 100644 Disco.Services/Documents/QRCodeBinaryEncoder.cs diff --git a/Disco.BI/BI/Interop/Pdf/PdfGenerator.cs b/Disco.BI/BI/Interop/Pdf/PdfGenerator.cs index 6373ee77..9d58cbd0 100644 --- a/Disco.BI/BI/Interop/Pdf/PdfGenerator.cs +++ b/Disco.BI/BI/Interop/Pdf/PdfGenerator.cs @@ -4,10 +4,12 @@ using Disco.Models.BI.Expressions; using Disco.Models.Repository; using Disco.Models.Services.Documents; using Disco.Services; +using Disco.Services.Documents; using Disco.Services.Expressions; using Disco.Services.Interop.ActiveDirectory; using Disco.Services.Users; using iTextSharp.text.pdf; +using iTextSharp.text.pdf.codec; using System; using System.Collections; using System.Collections.Concurrent; @@ -135,11 +137,23 @@ namespace Disco.BI.Interop.Pdf for (int pdfFieldOrdinal = 0; pdfFieldOrdinal < fields.Size; pdfFieldOrdinal++) { AcroFields.FieldPosition pdfFieldPosition = pdfFieldPositions[pdfFieldOrdinal]; - string pdfBarcodeContent = dt.CreateUniqueIdentifier(Database, Data, CreatorUser, TimeStamp, pdfFieldPosition.page).ToQRCodeString(); - BarcodeQRCode pdfBarcode = new BarcodeQRCode(pdfBarcodeContent, (int)pdfFieldPosition.position.Width, (int)pdfFieldPosition.position.Height, null); - iTextSharp.text.Image pdfBarcodeImage = pdfBarcode.GetImage(); - pdfBarcodeImage.SetAbsolutePosition(pdfFieldPosition.position.Left, pdfFieldPosition.position.Bottom); - pdfStamper.GetOverContent(pdfFieldPosition.page).AddImage(pdfBarcodeImage); + + // Create Binary Unique Identifier + var pageUniqueId = dt.CreateUniqueIdentifier(Database, Data, CreatorUser, TimeStamp, pdfFieldPosition.page); + var pageUniqueIdBytes = pageUniqueId.ToQRCodeBytes(); + + // Encode to QRCode byte array + var pageUniqueIdWidth = (int)pdfFieldPosition.position.Width; + var pageUniqueIdHeight = (int)pdfFieldPosition.position.Height; + var pageUniqueIdEncoded = QRCodeBinaryEncoder.Encode(pageUniqueIdBytes, pageUniqueIdWidth, pageUniqueIdHeight); + + // Encode byte array to Image + var pageUniqueIdImageData = CCITTG4Encoder.Compress(pageUniqueIdEncoded, pageUniqueIdWidth, pageUniqueIdHeight); + var pageUniqueIdImage = iTextSharp.text.Image.GetInstance(pageUniqueIdWidth, pageUniqueIdHeight, false, 256, 1, pageUniqueIdImageData, null); + + // Add to the pdf page + pageUniqueIdImage.SetAbsolutePosition(pdfFieldPosition.position.Left, pdfFieldPosition.position.Bottom); + pdfStamper.GetOverContent(pdfFieldPosition.page).AddImage(pageUniqueIdImage); } // Hide Fields PdfDictionary field = fields.GetValue(0); diff --git a/Disco.Services/Disco.Services.csproj b/Disco.Services/Disco.Services.csproj index 82110de3..d25fe952 100644 --- a/Disco.Services/Disco.Services.csproj +++ b/Disco.Services/Disco.Services.csproj @@ -74,12 +74,12 @@ ..\packages\PdfiumViewer.2.10.0.0\lib\net20\PdfiumViewer.dll True - - ..\packages\PDFsharp.1.32.3057.0\lib\net20\PdfSharp.dll + + ..\packages\PDFsharp.1.50.4000-beta3b\lib\net20\PdfSharp.dll True - - ..\packages\PDFsharp.1.32.3057.0\lib\net20\PdfSharp.Charting.dll + + ..\packages\PDFsharp.1.50.4000-beta3b\lib\net20\PdfSharp.Charting.dll True @@ -266,6 +266,7 @@ + diff --git a/Disco.Services/Documents/DocumentUniqueIdentifier.cs b/Disco.Services/Documents/DocumentUniqueIdentifier.cs index 91c72ac2..f09507a2 100644 --- a/Disco.Services/Documents/DocumentUniqueIdentifier.cs +++ b/Disco.Services/Documents/DocumentUniqueIdentifier.cs @@ -212,9 +212,9 @@ namespace Disco.Services.Documents // magic number (1) + version/flags (1) + deployment checksum (2) + timestamp (4) + // page index (2) + creator id length (1) + target id length (1) - var requiredBytes = 12; var creatorIdLength = encoding.GetByteCount(CreatorId); var targetIdLength = encoding.GetByteCount(TargetId); + var requiredBytes = 12 + creatorIdLength + targetIdLength; var documentTemplateIdLength = 0; if (DocumentTemplateId != null) @@ -222,6 +222,7 @@ namespace Disco.Services.Documents flags |= 1; requiredBytes++; documentTemplateIdLength = encoding.GetByteCount(DocumentTemplateId); + requiredBytes += documentTemplateIdLength; } int position = 0; @@ -230,7 +231,7 @@ namespace Disco.Services.Documents // magic number result[position++] = MagicNumber; // version & flags - result[position++] = (byte)(2 | (flags << 4)); + result[position++] = (byte)(flags | (2 << 4)); // deployment checksum deploymentChecksumBytes.CopyTo(result, position); position += 2; @@ -343,7 +344,7 @@ namespace Disco.Services.Documents if (IsDocumentUniqueIdentifier(UniqueIdentifier)) { // first 4 bit indicate version - var version = UniqueIdentifier[2] & 0x0F; + var version = UniqueIdentifier[1] >> 4; // Version 2 if (version == 2) @@ -362,10 +363,10 @@ namespace Disco.Services.Documents // ? | document template id length (optional based on flag) // ?-? | document template id (UTF8, optional based on flag) var encoding = Encoding.UTF8; - var position = 2; + var position = 1; // next 4 bits are flags - var flags = UniqueIdentifier[position++] >> 4; + var flags = UniqueIdentifier[position++] & 0x0F; var deploymentChecksum = BitConverter.ToInt16(UniqueIdentifier, position); position += 2; @@ -404,6 +405,7 @@ namespace Disco.Services.Documents var timeStamp = DateTime.FromFileTimeUtc(116444736000000000L).AddTicks(TimeSpan.TicksPerSecond * timeStampEpoch).ToLocalTime(); Identifier = new DocumentUniqueIdentifier(Database, version, deploymentChecksum, documentTemplateId, targetId, creatorId, timeStamp, pageIndex, attachmentType); + return true; } } diff --git a/Disco.Services/Documents/QRCodeBinaryEncoder.cs b/Disco.Services/Documents/QRCodeBinaryEncoder.cs new file mode 100644 index 00000000..fd57e1e3 --- /dev/null +++ b/Disco.Services/Documents/QRCodeBinaryEncoder.cs @@ -0,0 +1,285 @@ +using System; +using System.Collections.Generic; +using ZXing.Common; +using ZXing.Common.ReedSolomon; +using ZXing.QrCode.Internal; + +namespace Disco.Services.Documents +{ + public static class QRCodeBinaryEncoder + { + + public static byte[] Encode(byte[] Content, int Width, int Height) + { + var ecLevel = ErrorCorrectionLevel.L; + var bits = new BitArray(); + + var mode = Mode.BYTE; + var bitsNeeded = 4 + + mode.getCharacterCountBits(ZXing.QrCode.Internal.Version.getVersionForNumber(1)) + + (Content.Length * 8); + + var version = ChooseVersion(bitsNeeded, ecLevel); + var ecBlocks = version.getECBlocksForLevel(ecLevel); + var totalByteCapacity = version.TotalCodewords - ecBlocks.TotalECCodewords; + var totalBitCapacity = totalByteCapacity << 3; + + // Write the mode marker (BYTE) + bits.appendBits(mode.Bits, 4); + + // Write the number of bytes + bits.appendBits(Content.Length, mode.getCharacterCountBits(version)); + + // Write the bytes + for (int i = 0; i < Content.Length; i++) + { + bits.appendBits(Content[i], 8); + } + + // Terminate the bit stream + // - Write Termination Mode if space + for (int i = 0; i < 4 && bits.Size < totalBitCapacity; ++i) + { + bits.appendBit(false); + } + + // Add 8-bit alignment padding + var bitsInLastByte = bits.Size & 0x07; + if (bitsInLastByte > 0) + { + for (int i = bitsInLastByte; i < 8; i++) + { + bits.appendBit(false); + } + } + + // Fill remain space with padding patterns + var paddingBytes = totalByteCapacity - bits.SizeInBytes; + for (int i = 0; i < paddingBytes; ++i) + { + bits.appendBits((i & 0x01) == 0 ? 0xEC : 0x11, 8); + } + + // Interleave data bits with error correction code. + var finalBits = interleaveWithECBytes(bits, version.TotalCodewords, totalByteCapacity, ecBlocks.NumBlocks); + + // Choose the mask pattern and set to "qrCode". + int dimension = version.DimensionForVersion; + ByteMatrix matrix = new ByteMatrix(dimension, dimension); + int maskPattern = chooseMaskPattern(finalBits, ecLevel, version, matrix); + + // Build the matrix and set it to "qrCode". + MatrixUtil.buildMatrix(finalBits, ecLevel, version, maskPattern, matrix); + + // Render matrix to bytes + return scaleMatrix(matrix.Array, Width, Height); + } + + private static ZXing.QrCode.Internal.Version ChooseVersion(int RequiredBits, ErrorCorrectionLevel ECLevel) + { + // In the following comments, we use numbers of Version 7-H. + for (int versionNum = 1; versionNum <= 40; versionNum++) + { + var version = ZXing.QrCode.Internal.Version.getVersionForNumber(versionNum); + // numBytes = 196 + int numBytes = version.TotalCodewords; + // getNumECBytes = 130 + var ecBlocks = version.getECBlocksForLevel(ECLevel); + int numEcBytes = ecBlocks.TotalECCodewords; + // getNumDataBytes = 196 - 130 = 66 + int numDataBytes = numBytes - numEcBytes; + int totalInputBytes = (RequiredBits + 7) / 8; + if (numDataBytes >= totalInputBytes) + { + return version; + } + } + throw new ArgumentException("Data too big", nameof(RequiredBits)); + } + + private static BitArray interleaveWithECBytes(BitArray bits, int numTotalBytes, int numDataBytes, int numRSBlocks) + { + // Step 1. Divide data bytes into blocks and generate error correction bytes for them. We'll + // store the divided data bytes blocks and error correction bytes blocks into "blocks". + int dataBytesOffset = 0; + int maxNumDataBytes = 0; + int maxNumEcBytes = 0; + + // Since, we know the number of reedsolmon blocks, we can initialize the vector with the number. + var blocks = new List>(numRSBlocks); + + for (int i = 0; i < numRSBlocks; ++i) + { + + int numDataBytesInBlock; + int numEcBytesInBlock; + getNumDataBytesAndNumECBytesForBlockID( + numTotalBytes, numDataBytes, numRSBlocks, i, + out numDataBytesInBlock, out numEcBytesInBlock); + + byte[] dataBytes = new byte[numDataBytesInBlock]; + bits.toBytes(8 * dataBytesOffset, dataBytes, 0, numDataBytesInBlock); + byte[] ecBytes = generateECBytes(dataBytes, numEcBytesInBlock); + blocks.Add(new Tuple(dataBytes, ecBytes)); + + maxNumDataBytes = Math.Max(maxNumDataBytes, numDataBytesInBlock); + maxNumEcBytes = Math.Max(maxNumEcBytes, ecBytes.Length); + dataBytesOffset += numEcBytesInBlock; + } + + BitArray result = new BitArray(); + + // First, place data blocks. + for (int i = 0; i < maxNumDataBytes; ++i) + { + foreach (Tuple block in blocks) + { + byte[] dataBytes = block.Item1; + if (i < dataBytes.Length) + { + result.appendBits(dataBytes[i], 8); + } + } + } + // Then, place error correction blocks. + for (int i = 0; i < maxNumEcBytes; ++i) + { + foreach (Tuple block in blocks) + { + byte[] ecBytes = block.Item2; + if (i < ecBytes.Length) + { + result.appendBits(ecBytes[i], 8); + } + } + } + + return result; + } + + private static void getNumDataBytesAndNumECBytesForBlockID(int numTotalBytes, int numDataBytes, int numRSBlocks, int blockID, out int numDataBytesInBlock, out int numECBytesInBlock) + { + // numRsBlocksInGroup2 = 196 % 5 = 1 + int numRsBlocksInGroup2 = numTotalBytes % numRSBlocks; + // numRsBlocksInGroup1 = 5 - 1 = 4 + int numRsBlocksInGroup1 = numRSBlocks - numRsBlocksInGroup2; + // numTotalBytesInGroup1 = 196 / 5 = 39 + int numTotalBytesInGroup1 = numTotalBytes / numRSBlocks; + // numTotalBytesInGroup2 = 39 + 1 = 40 + int numTotalBytesInGroup2 = numTotalBytesInGroup1 + 1; + // numDataBytesInGroup1 = 66 / 5 = 13 + int numDataBytesInGroup1 = numDataBytes / numRSBlocks; + // numDataBytesInGroup2 = 13 + 1 = 14 + int numDataBytesInGroup2 = numDataBytesInGroup1 + 1; + // numEcBytesInGroup1 = 39 - 13 = 26 + int numEcBytesInGroup1 = numTotalBytesInGroup1 - numDataBytesInGroup1; + // numEcBytesInGroup2 = 40 - 14 = 26 + int numEcBytesInGroup2 = numTotalBytesInGroup2 - numDataBytesInGroup2; + + if (blockID < numRsBlocksInGroup1) + { + + numDataBytesInBlock = numDataBytesInGroup1; + numECBytesInBlock = numEcBytesInGroup1; + } + else + { + numDataBytesInBlock = numDataBytesInGroup2; + numECBytesInBlock = numEcBytesInGroup2; + } + } + + private static byte[] generateECBytes(byte[] dataBytes, int numEcBytesInBlock) + { + int numDataBytes = dataBytes.Length; + int[] toEncode = new int[numDataBytes + numEcBytesInBlock]; + for (int i = 0; i < numDataBytes; i++) + { + toEncode[i] = dataBytes[i] & 0xFF; + + } + new ReedSolomonEncoder(GenericGF.QR_CODE_FIELD_256).encode(toEncode, numEcBytesInBlock); + + byte[] ecBytes = new byte[numEcBytesInBlock]; + for (int i = 0; i < numEcBytesInBlock; i++) + { + ecBytes[i] = (byte)toEncode[numDataBytes + i]; + + } + return ecBytes; + } + + private static int chooseMaskPattern(BitArray bits, ErrorCorrectionLevel ecLevel, ZXing.QrCode.Internal.Version version, ByteMatrix matrix) + { + int minPenalty = Int32.MaxValue; // Lower penalty is better. + int bestMaskPattern = -1; + // We try all mask patterns to choose the best one. + for (int maskPattern = 0; maskPattern < QRCode.NUM_MASK_PATTERNS; maskPattern++) + { + + MatrixUtil.buildMatrix(bits, ecLevel, version, maskPattern, matrix); + int penalty = MaskUtil.applyMaskPenaltyRule1(matrix) + + MaskUtil.applyMaskPenaltyRule2(matrix) + + MaskUtil.applyMaskPenaltyRule3(matrix) + + MaskUtil.applyMaskPenaltyRule4(matrix); + if (penalty < minPenalty) + { + + minPenalty = penalty; + bestMaskPattern = maskPattern; + } + } + return bestMaskPattern; + } + + private static byte[] scaleMatrix(byte[][] matrix, int Width, int Height) + { + var matrixWidth = matrix[0].Length; + var matrixHeight = matrix.Length; + Width = Math.Max(Width, matrixWidth); + Height = Math.Max(Height, matrixHeight); + var byteColumns = (Width + 7) / 8; + var outputBytes = new byte[byteColumns * Height]; + var scale = Math.Min(Width / (matrixWidth + 1), Height / (matrixHeight + 1)); + var offsetX = (Width - (matrixWidth * scale)) / 2; + var offsetY = (Height - (matrixHeight * scale)) / 2; + // initialize output bytes + for (int i = 0; i < outputBytes.Length; i++) + { + outputBytes[i] = 0xFF; + } + // render row + for (int rowIndex = 0; rowIndex < matrixHeight; rowIndex++) + { + var rowMatrix = matrix[rowIndex]; + var rowLocation = ((rowIndex * scale) + offsetY) * byteColumns; + var bitOffset = offsetX; + for (int c = 0; c < matrixWidth; c++) + { + if (rowMatrix[c] == 1) + { + for (int cS = 0; cS < scale; cS++) + { + int index = rowLocation + (bitOffset / 8); + outputBytes[index] = (byte)(outputBytes[index] ^ ((byte)(0x80 >> (bitOffset % 8)))); + bitOffset++; + } + } + else + { + bitOffset += scale; + } + } + // Write row for scale + for (int i = 1; i < scale; i++) + { + var offsetLocation = rowLocation + (i * byteColumns); + Array.Copy(outputBytes, rowLocation, outputBytes, offsetLocation, byteColumns); + } + } + + return outputBytes; + } + + } +} diff --git a/Disco.Services/packages.config b/Disco.Services/packages.config index e0d81d99..05b75e64 100644 --- a/Disco.Services/packages.config +++ b/Disco.Services/packages.config @@ -17,7 +17,7 @@ - +