Document Template Binary QR Codes

The QR Code now stores the data using a custom binary encoding format. This reduces the amount of data stored by ~30%. This increases the chance of a lower-version QR Code being used and the likelyhood of even small QR Codes being detected. Compatibility with previously generated documents is maintained.
2016-09-15 19:27:52 +10:00
parent 5ea9a814d6
commit acfa0e5094
5 changed files with 317 additions and 15 deletions
@@ -4,10 +4,12 @@ using Disco.Models.BI.Expressions;
 using Disco.Models.Repository;
 using Disco.Models.Services.Documents;
 using Disco.Services;
+using Disco.Services.Documents;
 using Disco.Services.Expressions;
 using Disco.Services.Interop.ActiveDirectory;
 using Disco.Services.Users;
 using iTextSharp.text.pdf;
+using iTextSharp.text.pdf.codec;
 using System;
 using System.Collections;
 using System.Collections.Concurrent;
@@ -135,11 +137,23 @@ namespace Disco.BI.Interop.Pdf
                    for (int pdfFieldOrdinal = 0; pdfFieldOrdinal < fields.Size; pdfFieldOrdinal++)
                    {
                        AcroFields.FieldPosition pdfFieldPosition = pdfFieldPositions[pdfFieldOrdinal];
-                        string pdfBarcodeContent = dt.CreateUniqueIdentifier(Database, Data, CreatorUser, TimeStamp, pdfFieldPosition.page).ToQRCodeString();
-                        BarcodeQRCode pdfBarcode = new BarcodeQRCode(pdfBarcodeContent, (int)pdfFieldPosition.position.Width, (int)pdfFieldPosition.position.Height, null);
-                        iTextSharp.text.Image pdfBarcodeImage = pdfBarcode.GetImage();
-                        pdfBarcodeImage.SetAbsolutePosition(pdfFieldPosition.position.Left, pdfFieldPosition.position.Bottom);
-                        pdfStamper.GetOverContent(pdfFieldPosition.page).AddImage(pdfBarcodeImage);
+
+                        // Create Binary Unique Identifier
+                        var pageUniqueId = dt.CreateUniqueIdentifier(Database, Data, CreatorUser, TimeStamp, pdfFieldPosition.page);
+                        var pageUniqueIdBytes = pageUniqueId.ToQRCodeBytes();
+
+                        // Encode to QRCode byte array
+                        var pageUniqueIdWidth = (int)pdfFieldPosition.position.Width;
+                        var pageUniqueIdHeight = (int)pdfFieldPosition.position.Height;
+                        var pageUniqueIdEncoded = QRCodeBinaryEncoder.Encode(pageUniqueIdBytes, pageUniqueIdWidth, pageUniqueIdHeight);
+
+                        // Encode byte array to Image
+                        var pageUniqueIdImageData = CCITTG4Encoder.Compress(pageUniqueIdEncoded, pageUniqueIdWidth, pageUniqueIdHeight);
+                        var pageUniqueIdImage = iTextSharp.text.Image.GetInstance(pageUniqueIdWidth, pageUniqueIdHeight, false, 256, 1, pageUniqueIdImageData, null);
+
+                        // Add to the pdf page
+                        pageUniqueIdImage.SetAbsolutePosition(pdfFieldPosition.position.Left, pdfFieldPosition.position.Bottom);
+                        pdfStamper.GetOverContent(pdfFieldPosition.page).AddImage(pageUniqueIdImage);
                    }
                    // Hide Fields
                    PdfDictionary field = fields.GetValue(0);
@@ -74,12 +74,12 @@
      <HintPath>..\packages\PdfiumViewer.2.10.0.0\lib\net20\PdfiumViewer.dll</HintPath>
      <Private>True</Private>
    </Reference>
-    <Reference Include="PdfSharp, Version=1.32.3057.0, Culture=neutral, PublicKeyToken=f94615aa0424f9eb, processorArchitecture=MSIL">
-      <HintPath>..\packages\PDFsharp.1.32.3057.0\lib\net20\PdfSharp.dll</HintPath>
+    <Reference Include="PdfSharp, Version=1.50.4000.0, Culture=neutral, PublicKeyToken=f94615aa0424f9eb, processorArchitecture=MSIL">
+      <HintPath>..\packages\PDFsharp.1.50.4000-beta3b\lib\net20\PdfSharp.dll</HintPath>
      <Private>True</Private>
    </Reference>
-    <Reference Include="PdfSharp.Charting, Version=1.32.3057.0, Culture=neutral, PublicKeyToken=f94615aa0424f9eb, processorArchitecture=MSIL">
-      <HintPath>..\packages\PDFsharp.1.32.3057.0\lib\net20\PdfSharp.Charting.dll</HintPath>
+    <Reference Include="PdfSharp.Charting, Version=1.50.4000.0, Culture=neutral, PublicKeyToken=f94615aa0424f9eb, processorArchitecture=MSIL">
+      <HintPath>..\packages\PDFsharp.1.50.4000-beta3b\lib\net20\PdfSharp.Charting.dll</HintPath>
      <Private>True</Private>
    </Reference>
    <Reference Include="Quartz">
@@ -266,6 +266,7 @@
    <Compile Include="Documents\DocumentTemplateExpressionExtensions.cs" />
    <Compile Include="Documents\DocumentUniqueIdentifier.cs" />
    <Compile Include="Documents\DocumentUniqueIdentifierExtensions.cs" />
+    <Compile Include="Documents\QRCodeBinaryEncoder.cs" />
    <Compile Include="Expressions\EvaluateExpressionParseException.cs" />
    <Compile Include="Expressions\EvaluateExpressionPart.cs" />
    <Compile Include="Expressions\Expression.cs" />
@@ -212,9 +212,9 @@ namespace Disco.Services.Documents

            // magic number (1) + version/flags (1) + deployment checksum (2) + timestamp (4) +
            //  page index (2) + creator id length (1) + target id length (1)
-            var requiredBytes = 12;
            var creatorIdLength = encoding.GetByteCount(CreatorId);
            var targetIdLength = encoding.GetByteCount(TargetId);
+            var requiredBytes = 12 + creatorIdLength + targetIdLength;
            var documentTemplateIdLength = 0;

            if (DocumentTemplateId != null)
@@ -222,6 +222,7 @@ namespace Disco.Services.Documents
                flags |= 1;
                requiredBytes++;
                documentTemplateIdLength = encoding.GetByteCount(DocumentTemplateId);
+                requiredBytes += documentTemplateIdLength;
            }

            int position = 0;
@@ -230,7 +231,7 @@ namespace Disco.Services.Documents
            // magic number
            result[position++] = MagicNumber;
            // version & flags
-            result[position++] = (byte)(2 | (flags << 4));
+            result[position++] = (byte)(flags | (2 << 4));
            // deployment checksum
            deploymentChecksumBytes.CopyTo(result, position);
            position += 2;
@@ -343,7 +344,7 @@ namespace Disco.Services.Documents
            if (IsDocumentUniqueIdentifier(UniqueIdentifier))
            {
                // first 4 bit indicate version
-                var version = UniqueIdentifier[2] & 0x0F;
+                var version = UniqueIdentifier[1] >> 4;

                // Version 2
                if (version == 2)
@@ -362,10 +363,10 @@ namespace Disco.Services.Documents
                    // ?    | document template id length (optional based on flag)
                    // ?-?  | document template id (UTF8, optional based on flag)
                    var encoding = Encoding.UTF8;
-                    var position = 2;
+                    var position = 1;

                    // next 4 bits are flags
-                    var flags = UniqueIdentifier[position++] >> 4;
+                    var flags = UniqueIdentifier[position++] & 0x0F;

                    var deploymentChecksum = BitConverter.ToInt16(UniqueIdentifier, position);
                    position += 2;
@@ -404,6 +405,7 @@ namespace Disco.Services.Documents
                    var timeStamp = DateTime.FromFileTimeUtc(116444736000000000L).AddTicks(TimeSpan.TicksPerSecond * timeStampEpoch).ToLocalTime();

                    Identifier = new DocumentUniqueIdentifier(Database, version, deploymentChecksum, documentTemplateId, targetId, creatorId, timeStamp, pageIndex, attachmentType);
+                    return true;
                }
            }

@@ -0,0 +1,285 @@
+using System;
+using System.Collections.Generic;
+using ZXing.Common;
+using ZXing.Common.ReedSolomon;
+using ZXing.QrCode.Internal;
+
+namespace Disco.Services.Documents
+{
+    public static class QRCodeBinaryEncoder
+    {
+
+        public static byte[] Encode(byte[] Content, int Width, int Height)
+        {
+            var ecLevel = ErrorCorrectionLevel.L;
+            var bits = new BitArray();
+
+            var mode = Mode.BYTE;
+            var bitsNeeded = 4 +
+                mode.getCharacterCountBits(ZXing.QrCode.Internal.Version.getVersionForNumber(1)) +
+                (Content.Length * 8);
+
+            var version = ChooseVersion(bitsNeeded, ecLevel);
+            var ecBlocks = version.getECBlocksForLevel(ecLevel);
+            var totalByteCapacity = version.TotalCodewords - ecBlocks.TotalECCodewords;
+            var totalBitCapacity = totalByteCapacity << 3;
+
+            // Write the mode marker (BYTE)
+            bits.appendBits(mode.Bits, 4);
+
+            // Write the number of bytes
+            bits.appendBits(Content.Length, mode.getCharacterCountBits(version));
+
+            // Write the bytes
+            for (int i = 0; i < Content.Length; i++)
+            {
+                bits.appendBits(Content[i], 8);
+            }
+
+            // Terminate the bit stream
+            //  - Write Termination Mode if space
+            for (int i = 0; i < 4 && bits.Size < totalBitCapacity; ++i)
+            {
+                bits.appendBit(false);
+            }
+
+            // Add 8-bit alignment padding
+            var bitsInLastByte = bits.Size & 0x07;
+            if (bitsInLastByte > 0)
+            {
+                for (int i = bitsInLastByte; i < 8; i++)
+                {
+                    bits.appendBit(false);
+                }
+            }
+
+            // Fill remain space with padding patterns
+            var paddingBytes = totalByteCapacity - bits.SizeInBytes;
+            for (int i = 0; i < paddingBytes; ++i)
+            {
+                bits.appendBits((i & 0x01) == 0 ? 0xEC : 0x11, 8);
+            }
+
+            // Interleave data bits with error correction code.
+            var finalBits = interleaveWithECBytes(bits, version.TotalCodewords, totalByteCapacity, ecBlocks.NumBlocks);
+
+            //  Choose the mask pattern and set to "qrCode".
+            int dimension = version.DimensionForVersion;
+            ByteMatrix matrix = new ByteMatrix(dimension, dimension);
+            int maskPattern = chooseMaskPattern(finalBits, ecLevel, version, matrix);
+
+            // Build the matrix and set it to "qrCode".
+            MatrixUtil.buildMatrix(finalBits, ecLevel, version, maskPattern, matrix);
+
+            // Render matrix to bytes
+            return scaleMatrix(matrix.Array, Width, Height);
+        }
+
+        private static ZXing.QrCode.Internal.Version ChooseVersion(int RequiredBits, ErrorCorrectionLevel ECLevel)
+        {
+            // In the following comments, we use numbers of Version 7-H.
+            for (int versionNum = 1; versionNum <= 40; versionNum++)
+            {
+                var version = ZXing.QrCode.Internal.Version.getVersionForNumber(versionNum);
+                // numBytes = 196
+                int numBytes = version.TotalCodewords;
+                // getNumECBytes = 130
+                var ecBlocks = version.getECBlocksForLevel(ECLevel);
+                int numEcBytes = ecBlocks.TotalECCodewords;
+                // getNumDataBytes = 196 - 130 = 66
+                int numDataBytes = numBytes - numEcBytes;
+                int totalInputBytes = (RequiredBits + 7) / 8;
+                if (numDataBytes >= totalInputBytes)
+                {
+                    return version;
+                }
+            }
+            throw new ArgumentException("Data too big", nameof(RequiredBits));
+        }
+
+        private static BitArray interleaveWithECBytes(BitArray bits, int numTotalBytes, int numDataBytes, int numRSBlocks)
+        {
+            // Step 1.  Divide data bytes into blocks and generate error correction bytes for them. We'll
+            // store the divided data bytes blocks and error correction bytes blocks into "blocks".
+            int dataBytesOffset = 0;
+            int maxNumDataBytes = 0;
+            int maxNumEcBytes = 0;
+
+            // Since, we know the number of reedsolmon blocks, we can initialize the vector with the number.
+            var blocks = new List<Tuple<byte[], byte[]>>(numRSBlocks);
+
+            for (int i = 0; i < numRSBlocks; ++i)
+            {
+
+                int numDataBytesInBlock;
+                int numEcBytesInBlock;
+                getNumDataBytesAndNumECBytesForBlockID(
+                    numTotalBytes, numDataBytes, numRSBlocks, i,
+                    out numDataBytesInBlock, out numEcBytesInBlock);
+
+                byte[] dataBytes = new byte[numDataBytesInBlock];
+                bits.toBytes(8 * dataBytesOffset, dataBytes, 0, numDataBytesInBlock);
+                byte[] ecBytes = generateECBytes(dataBytes, numEcBytesInBlock);
+                blocks.Add(new Tuple<byte[], byte[]>(dataBytes, ecBytes));
+
+                maxNumDataBytes = Math.Max(maxNumDataBytes, numDataBytesInBlock);
+                maxNumEcBytes = Math.Max(maxNumEcBytes, ecBytes.Length);
+                dataBytesOffset += numEcBytesInBlock;
+            }
+
+            BitArray result = new BitArray();
+
+            // First, place data blocks.
+            for (int i = 0; i < maxNumDataBytes; ++i)
+            {
+                foreach (Tuple<byte[], byte[]> block in blocks)
+                {
+                    byte[] dataBytes = block.Item1;
+                    if (i < dataBytes.Length)
+                    {
+                        result.appendBits(dataBytes[i], 8);
+                    }
+                }
+            }
+            // Then, place error correction blocks.
+            for (int i = 0; i < maxNumEcBytes; ++i)
+            {
+                foreach (Tuple<byte[], byte[]> block in blocks)
+                {
+                    byte[] ecBytes = block.Item2;
+                    if (i < ecBytes.Length)
+                    {
+                        result.appendBits(ecBytes[i], 8);
+                    }
+                }
+            }
+
+            return result;
+        }
+
+        private static void getNumDataBytesAndNumECBytesForBlockID(int numTotalBytes, int numDataBytes, int numRSBlocks, int blockID, out int numDataBytesInBlock, out int numECBytesInBlock)
+        {
+            // numRsBlocksInGroup2 = 196 % 5 = 1
+            int numRsBlocksInGroup2 = numTotalBytes % numRSBlocks;
+            // numRsBlocksInGroup1 = 5 - 1 = 4
+            int numRsBlocksInGroup1 = numRSBlocks - numRsBlocksInGroup2;
+            // numTotalBytesInGroup1 = 196 / 5 = 39
+            int numTotalBytesInGroup1 = numTotalBytes / numRSBlocks;
+            // numTotalBytesInGroup2 = 39 + 1 = 40
+            int numTotalBytesInGroup2 = numTotalBytesInGroup1 + 1;
+            // numDataBytesInGroup1 = 66 / 5 = 13
+            int numDataBytesInGroup1 = numDataBytes / numRSBlocks;
+            // numDataBytesInGroup2 = 13 + 1 = 14
+            int numDataBytesInGroup2 = numDataBytesInGroup1 + 1;
+            // numEcBytesInGroup1 = 39 - 13 = 26
+            int numEcBytesInGroup1 = numTotalBytesInGroup1 - numDataBytesInGroup1;
+            // numEcBytesInGroup2 = 40 - 14 = 26
+            int numEcBytesInGroup2 = numTotalBytesInGroup2 - numDataBytesInGroup2;
+
+            if (blockID < numRsBlocksInGroup1)
+            {
+
+                numDataBytesInBlock = numDataBytesInGroup1;
+                numECBytesInBlock = numEcBytesInGroup1;
+            }
+            else
+            {
+                numDataBytesInBlock = numDataBytesInGroup2;
+                numECBytesInBlock = numEcBytesInGroup2;
+            }
+        }
+
+        private static byte[] generateECBytes(byte[] dataBytes, int numEcBytesInBlock)
+        {
+            int numDataBytes = dataBytes.Length;
+            int[] toEncode = new int[numDataBytes + numEcBytesInBlock];
+            for (int i = 0; i < numDataBytes; i++)
+            {
+                toEncode[i] = dataBytes[i] & 0xFF;
+
+            }
+            new ReedSolomonEncoder(GenericGF.QR_CODE_FIELD_256).encode(toEncode, numEcBytesInBlock);
+
+            byte[] ecBytes = new byte[numEcBytesInBlock];
+            for (int i = 0; i < numEcBytesInBlock; i++)
+            {
+                ecBytes[i] = (byte)toEncode[numDataBytes + i];
+
+            }
+            return ecBytes;
+        }
+
+        private static int chooseMaskPattern(BitArray bits, ErrorCorrectionLevel ecLevel, ZXing.QrCode.Internal.Version version, ByteMatrix matrix)
+        {
+            int minPenalty = Int32.MaxValue;  // Lower penalty is better.
+            int bestMaskPattern = -1;
+            // We try all mask patterns to choose the best one.
+            for (int maskPattern = 0; maskPattern < QRCode.NUM_MASK_PATTERNS; maskPattern++)
+            {
+
+                MatrixUtil.buildMatrix(bits, ecLevel, version, maskPattern, matrix);
+                int penalty = MaskUtil.applyMaskPenaltyRule1(matrix)
+                    + MaskUtil.applyMaskPenaltyRule2(matrix)
+                    + MaskUtil.applyMaskPenaltyRule3(matrix)
+                    + MaskUtil.applyMaskPenaltyRule4(matrix);
+                if (penalty < minPenalty)
+                {
+
+                    minPenalty = penalty;
+                    bestMaskPattern = maskPattern;
+                }
+            }
+            return bestMaskPattern;
+        }
+
+        private static byte[] scaleMatrix(byte[][] matrix, int Width, int Height)
+        {
+            var matrixWidth = matrix[0].Length;
+            var matrixHeight = matrix.Length;
+            Width = Math.Max(Width, matrixWidth);
+            Height = Math.Max(Height, matrixHeight);
+            var byteColumns = (Width + 7) / 8;
+            var outputBytes = new byte[byteColumns * Height];
+            var scale = Math.Min(Width / (matrixWidth + 1), Height / (matrixHeight + 1));
+            var offsetX = (Width - (matrixWidth * scale)) / 2;
+            var offsetY = (Height - (matrixHeight * scale)) / 2;
+            // initialize output bytes
+            for (int i = 0; i < outputBytes.Length; i++)
+            {
+                outputBytes[i] = 0xFF;
+            }
+            // render row
+            for (int rowIndex = 0; rowIndex < matrixHeight; rowIndex++)
+            {
+                var rowMatrix = matrix[rowIndex];
+                var rowLocation = ((rowIndex * scale) + offsetY) * byteColumns;
+                var bitOffset = offsetX;
+                for (int c = 0; c < matrixWidth; c++)
+                {
+                    if (rowMatrix[c] == 1)
+                    {
+                        for (int cS = 0; cS < scale; cS++)
+                        {
+                            int index = rowLocation + (bitOffset / 8);
+                            outputBytes[index] = (byte)(outputBytes[index] ^ ((byte)(0x80 >> (bitOffset % 8))));
+                            bitOffset++;
+                        }
+                    }
+                    else
+                    {
+                        bitOffset += scale;
+                    }
+                }
+                // Write row for scale
+                for (int i = 1; i < scale; i++)
+                {
+                    var offsetLocation = rowLocation + (i * byteColumns);
+                    Array.Copy(outputBytes, rowLocation, outputBytes, offsetLocation, byteColumns);
+                }
+            }
+
+            return outputBytes;
+        }
+
+    }
+}
@@ -17,7 +17,7 @@
  <package id="Newtonsoft.Json" version="6.0.3" targetFramework="net45" />
  <package id="Owin" version="1.0" targetFramework="net45" />
  <package id="PdfiumViewer" version="2.10.0.0" targetFramework="net45" />
-  <package id="PDFsharp" version="1.32.3057.0" targetFramework="net45" />
+  <package id="PDFsharp" version="1.50.4000-beta3b" targetFramework="net45" />
  <package id="RazorGenerator.Mvc" version="2.2.3" targetFramework="net45" />
  <package id="Rx-Core" version="2.2.5" targetFramework="net45" />
  <package id="Rx-Interfaces" version="2.2.5" targetFramework="net45" />