update

qrCameraAI/Program.cs

@@ -19,9 +19,11 @@ namespace BarcodeReaderExample
         // ---- tunables -------------------------------------------------
         const bool EnableOverlay = true;          // master draw switch (debug)
         const bool TryHarderMultiDecode = true;    // more reliable multi-QR detection, costs some speed
-        const float QuadExpansion = 1.60f;         // how much bigger than the QR the overlay is drawn
+        const float QuadExpansion = 1.50f;         // how much bigger than the QR the overlay is drawn
         const int MaxMissedFrames = 8;             // keep last known position this many frames if briefly lost
-        const int TargetFps = 60;
+        const int TargetFps = 24;
+        const float LocalRedetectPadding = 0.6f;   // extra search margin around last known box, as a fraction of its size
+        const int LocalRedetectUpscale = 3;        // how much to enlarge the crop before decoding it
 
         static readonly HttpClient Http = new HttpClient();
         static readonly ConcurrentDictionary<string, Mat> ImageCache = new();   // text -> decoded image (null = failed)
@@ -34,13 +36,23 @@ namespace BarcodeReaderExample
             { DecodeHintType.POSSIBLE_FORMATS, new List<BarcodeFormat> { BarcodeFormat.QR_CODE } },
             { DecodeHintType.TRY_HARDER, TryHarderMultiDecode },
         };
+        // Used only for the small, targeted local-redetect crop, so the extra cost is negligible there.
+        static readonly Dictionary<DecodeHintType, object> LocalHints = new()
+        {
+            { DecodeHintType.POSSIBLE_FORMATS, new List<BarcodeFormat> { BarcodeFormat.QR_CODE } },
+            { DecodeHintType.TRY_HARDER, true },
+            { DecodeHintType.ALSO_INVERTED, true },
+        };
+
+        static readonly CLAHE Clahe = Cv2.CreateCLAHE(clipLimit: 3.0, tileGridSize: new Size(8, 8));
 
         static byte[] _grayBuffer;
         static readonly Dictionary<string, TrackedQr> Tracked = new();
 
         class TrackedQr
         {
-            public Point2f[] Quad;   // TL, TR, BR, BL (frame coordinates)
+            public Point2f[] RawQuad;    // TL, TR, BR, BL as detected, unexpanded - used to seed the next search
+            public Point2f[] Quad;       // TL, TR, BR, BL, expanded - used for drawing
             public int MissedFrames;
         }
 
@@ -117,42 +129,40 @@ namespace BarcodeReaderExample
         static void ProcessFrame(Mat frame, Mat gray)
         {
             Cv2.CvtColor(frame, gray, ColorConversionCodes.BGR2GRAY);
-            Result[] results = ReadQrCodes(gray);
+
+            // Contrast-normalize before decoding - this is the single biggest lever for detection
+            // reliability under uneven lighting (glare, shadow across part of the code, dim rooms).
+            using var enhanced = new Mat();
+            Clahe.Apply(gray, enhanced);
+
+            Result[] results = ReadQrCodes(enhanced);
 
             var seenThisFrame = new HashSet<string>();
 
             foreach (var result in results)
             {
                 if (result?.Text == null) continue;
-                var points = result.ResultPoints;
+                var (pTL, pTR, pBR, pBL) = CornersFromResult(result);
-                if (points == null || points.Length < 3) continue;
+                if (pTL == null) continue;
-
-                // ZXing QR point order: [0]=bottom-left, [1]=top-left, [2]=top-right,
-                // optional [3]=alignment pattern. We deliberately do NOT use points[3] for the
-                // bottom-right corner: the alignment pattern sits several modules inset from the
-                // actual edge of the code, so it's always noticeably closer to the center than the
-                // true corner is - using it directly shrinks that whole side of the overlay.
-                // Instead derive the true bottom-right corner from the 3 finder-pattern corners,
-                // which ZXing locates precisely, assuming the QR is (near) square: BR = TR + BL - TL.
-                Point2f pBL = new Point2f(points[0].X, points[0].Y);
-                Point2f pTL = new Point2f(points[1].X, points[1].Y);
-                Point2f pTR = new Point2f(points[2].X, points[2].Y);
-                Point2f pBR = new Point2f(pTR.X + pBL.X - pTL.X, pTR.Y + pBL.Y - pTL.Y);
-
-                Point2f[] quad = ExpandQuad(pTL, pTR, pBR, pBL, QuadExpansion);
 
+                UpdateTracked(result.Text, pTL.Value, pTR.Value, pBR.Value, pBL.Value);
                 seenThisFrame.Add(result.Text);
-                if (Tracked.TryGetValue(result.Text, out var t))
+                EnsureImageLoaded(result.Text);
-                {
-                    t.Quad = quad;
-                    t.MissedFrames = 0;
-                }
-                else
-                {
-                    Tracked[result.Text] = new TrackedQr { Quad = quad, MissedFrames = 0 };
             }
 
-                EnsureImageLoaded(result.Text);
+            // For codes we're already tracking but the global pass didn't find this frame, actively
+            // re-search a small, upscaled crop around their last known position instead of just
+            // freezing them in place. This recovers far more often for codes that are small,
+            // partially blurred, or at a steep angle - the global full-frame pass is much less
+            // likely to catch those, but a zoomed-in targeted decode often still can.
+            foreach (var kv in Tracked)
+            {
+                if (seenThisFrame.Contains(kv.Key)) continue;
+                if (TryLocalRedetect(enhanced, kv.Value.RawQuad, out var pTL, out var pTR, out var pBR, out var pBL))
+                {
+                    UpdateTracked(kv.Key, pTL, pTR, pBR, pBL);
+                    seenThisFrame.Add(kv.Key);
+                }
             }
 
             // Age out codes that weren't seen this frame; keep them briefly to avoid flicker.
@@ -178,6 +188,44 @@ namespace BarcodeReaderExample
             }
         }
 
+        static void UpdateTracked(string text, Point2f tl, Point2f tr, Point2f br, Point2f bl)
+        {
+            Point2f[] raw = { tl, tr, br, bl };
+            Point2f[] quad = ExpandQuad(tl, tr, br, bl, QuadExpansion);
+
+            if (Tracked.TryGetValue(text, out var t))
+            {
+                t.RawQuad = raw;
+                t.Quad = quad;
+                t.MissedFrames = 0;
+            }
+            else
+            {
+                Tracked[text] = new TrackedQr { RawQuad = raw, Quad = quad, MissedFrames = 0 };
+            }
+        }
+
+        /// <summary>
+        /// ZXing QR point order: [0]=bottom-left, [1]=top-left, [2]=top-right, optional [3]=alignment
+        /// pattern. We deliberately do NOT use points[3] for the bottom-right corner: the alignment
+        /// pattern sits several modules inset from the actual edge of the code, so it's always
+        /// noticeably closer to the center than the true corner is - using it directly shrinks that
+        /// whole side of the overlay. Instead derive the true bottom-right corner from the 3
+        /// finder-pattern corners, which ZXing locates precisely, assuming the QR is (near) square:
+        /// BR = TR + BL - TL.
+        /// </summary>
+        static (Point2f? tl, Point2f? tr, Point2f? br, Point2f? bl) CornersFromResult(Result result)
+        {
+            var points = result?.ResultPoints;
+            if (points == null || points.Length < 3) return (null, null, null, null);
+
+            Point2f pBL = new Point2f(points[0].X, points[0].Y);
+            Point2f pTL = new Point2f(points[1].X, points[1].Y);
+            Point2f pTR = new Point2f(points[2].X, points[2].Y);
+            Point2f pBR = new Point2f(pTR.X + pBL.X - pTL.X, pTR.Y + pBL.Y - pTL.Y);
+            return (pTL, pTR, pBR, pBL);
+        }
+
         /// <summary>Expand a quad (TL,TR,BR,BL) outward from its centroid by a scale factor.</summary>
         static Point2f[] ExpandQuad(Point2f tl, Point2f tr, Point2f br, Point2f bl, float scale)
         {
@@ -222,17 +270,7 @@ namespace BarcodeReaderExample
 
         static Result[] ReadQrCodes(Mat gray)
         {
-            if (!gray.IsContinuous())
+            var luminanceSource = ToLuminanceSource(gray, ref _grayBuffer);
-                gray = gray.Clone();
-
-            int size = (int)(gray.Total() * gray.ElemSize());
-            if (_grayBuffer == null || _grayBuffer.Length != size)
-                _grayBuffer = new byte[size];
-            Marshal.Copy(gray.Data, _grayBuffer, 0, size);
-
-            // Gray8 skips ZXing's internal RGB->luminance conversion entirely - the fastest
-            // format to hand it, and matches what BGR2GRAY already computed for us in OpenCV.
-            var luminanceSource = new RGBLuminanceSource(_grayBuffer, gray.Width, gray.Height, RGBLuminanceSource.BitmapFormat.Gray8);
             var bitmap = new BinaryBitmap(new HybridBinarizer(luminanceSource));
 
             try
@@ -245,6 +283,69 @@ namespace BarcodeReaderExample
             }
         }
 
+        /// <summary>
+        /// Crop tightly around a code's last known position (with padding for movement), upscale
+        /// that crop, and try to decode just it. Works much better than a full-frame scan for a
+        /// code that's already small, tilted, or motion-blurred, since the decoder effectively gets
+        /// a bigger, cleaner version of just the region that matters.
+        /// </summary>
+        static bool TryLocalRedetect(Mat enhanced, Point2f[] lastRaw, out Point2f tl, out Point2f tr, out Point2f br, out Point2f bl)
+        {
+            tl = tr = br = bl = default;
+            if (lastRaw == null) return false;
+
+            var intPoints = lastRaw.Select(p => new Point((int)p.X, (int)p.Y)).ToArray();
+            Rect box = Cv2.BoundingRect(intPoints);
+            int pad = (int)(Math.Max(box.Width, box.Height) * LocalRedetectPadding);
+            Rect search = new Rect(box.X - pad, box.Y - pad, box.Width + pad * 2, box.Height + pad * 2);
+            search = search.Intersect(new Rect(0, 0, enhanced.Width, enhanced.Height));
+            if (search.Width < 15 || search.Height < 15) return false;
+
+            using var crop = new Mat(enhanced, search);
+            using var scaled = new Mat();
+            Cv2.Resize(crop, scaled, new Size(crop.Width * LocalRedetectUpscale, crop.Height * LocalRedetectUpscale),
+                0, 0, InterpolationFlags.Cubic);
+
+            byte[] localBuffer = null;
+            var luminanceSource = ToLuminanceSource(scaled, ref localBuffer);
+            var bitmap = new BinaryBitmap(new HybridBinarizer(luminanceSource));
+
+            Result result;
+            try { result = QrReader.decode(bitmap, LocalHints); }
+            catch { return false; }
+            if (result == null) return false;
+
+            var (cTl, cTr, cBr, cBl) = CornersFromResult(result);
+            if (cTl == null) return false;
+
+            // Map corners from the upscaled crop back into full-frame coordinates.
+            Point2f ToFrame(Point2f p) => new Point2f(
+                search.X + p.X / LocalRedetectUpscale,
+                search.Y + p.Y / LocalRedetectUpscale);
+
+            tl = ToFrame(cTl.Value);
+            tr = ToFrame(cTr.Value);
+            br = ToFrame(cBr.Value);
+            bl = ToFrame(cBl.Value);
+            return true;
+        }
+
+        static RGBLuminanceSource ToLuminanceSource(Mat gray, ref byte[] buffer)
+        {
+            Mat continuous = gray.IsContinuous() ? gray : gray.Clone();
+            int size = (int)(continuous.Total() * continuous.ElemSize());
+            if (buffer == null || buffer.Length != size)
+                buffer = new byte[size];
+            Marshal.Copy(continuous.Data, buffer, 0, size);
+
+            // Gray8 skips ZXing's internal RGB->luminance conversion entirely - the fastest
+            // format to hand it, and matches what BGR2GRAY/CLAHE already produced for us.
+            var source = new RGBLuminanceSource(buffer, continuous.Width, continuous.Height, RGBLuminanceSource.BitmapFormat.Gray8);
+            if (!ReferenceEquals(continuous, gray))
+                continuous.Dispose();
+            return source;
+        }
+
         // ---- async image download/caching --------------------------------
 
         static void EnsureImageLoaded(string url)