diff --git a/qrCameraAI/Program.cs b/qrCameraAI/Program.cs index e6418e8..29cea91 100644 --- a/qrCameraAI/Program.cs +++ b/qrCameraAI/Program.cs @@ -19,9 +19,11 @@ namespace BarcodeReaderExample // ---- tunables ------------------------------------------------- const bool EnableOverlay = true; // master draw switch (debug) const bool TryHarderMultiDecode = true; // more reliable multi-QR detection, costs some speed - const float QuadExpansion = 1.60f; // how much bigger than the QR the overlay is drawn + const float QuadExpansion = 1.50f; // how much bigger than the QR the overlay is drawn const int MaxMissedFrames = 8; // keep last known position this many frames if briefly lost - const int TargetFps = 60; + const int TargetFps = 24; + const float LocalRedetectPadding = 0.6f; // extra search margin around last known box, as a fraction of its size + const int LocalRedetectUpscale = 3; // how much to enlarge the crop before decoding it static readonly HttpClient Http = new HttpClient(); static readonly ConcurrentDictionary ImageCache = new(); // text -> decoded image (null = failed) @@ -34,13 +36,23 @@ namespace BarcodeReaderExample { DecodeHintType.POSSIBLE_FORMATS, new List { BarcodeFormat.QR_CODE } }, { DecodeHintType.TRY_HARDER, TryHarderMultiDecode }, }; + // Used only for the small, targeted local-redetect crop, so the extra cost is negligible there. + static readonly Dictionary LocalHints = new() + { + { DecodeHintType.POSSIBLE_FORMATS, new List { BarcodeFormat.QR_CODE } }, + { DecodeHintType.TRY_HARDER, true }, + { DecodeHintType.ALSO_INVERTED, true }, + }; + + static readonly CLAHE Clahe = Cv2.CreateCLAHE(clipLimit: 3.0, tileGridSize: new Size(8, 8)); static byte[] _grayBuffer; static readonly Dictionary Tracked = new(); class TrackedQr { - public Point2f[] Quad; // TL, TR, BR, BL (frame coordinates) + public Point2f[] RawQuad; // TL, TR, BR, BL as detected, unexpanded - used to seed the next search + public Point2f[] Quad; // TL, TR, BR, BL, expanded - used for drawing public int MissedFrames; } @@ -117,44 +129,42 @@ namespace BarcodeReaderExample static void ProcessFrame(Mat frame, Mat gray) { Cv2.CvtColor(frame, gray, ColorConversionCodes.BGR2GRAY); - Result[] results = ReadQrCodes(gray); + + // Contrast-normalize before decoding - this is the single biggest lever for detection + // reliability under uneven lighting (glare, shadow across part of the code, dim rooms). + using var enhanced = new Mat(); + Clahe.Apply(gray, enhanced); + + Result[] results = ReadQrCodes(enhanced); var seenThisFrame = new HashSet(); foreach (var result in results) { if (result?.Text == null) continue; - var points = result.ResultPoints; - if (points == null || points.Length < 3) continue; - - // ZXing QR point order: [0]=bottom-left, [1]=top-left, [2]=top-right, - // optional [3]=alignment pattern. We deliberately do NOT use points[3] for the - // bottom-right corner: the alignment pattern sits several modules inset from the - // actual edge of the code, so it's always noticeably closer to the center than the - // true corner is - using it directly shrinks that whole side of the overlay. - // Instead derive the true bottom-right corner from the 3 finder-pattern corners, - // which ZXing locates precisely, assuming the QR is (near) square: BR = TR + BL - TL. - Point2f pBL = new Point2f(points[0].X, points[0].Y); - Point2f pTL = new Point2f(points[1].X, points[1].Y); - Point2f pTR = new Point2f(points[2].X, points[2].Y); - Point2f pBR = new Point2f(pTR.X + pBL.X - pTL.X, pTR.Y + pBL.Y - pTL.Y); - - Point2f[] quad = ExpandQuad(pTL, pTR, pBR, pBL, QuadExpansion); + var (pTL, pTR, pBR, pBL) = CornersFromResult(result); + if (pTL == null) continue; + UpdateTracked(result.Text, pTL.Value, pTR.Value, pBR.Value, pBL.Value); seenThisFrame.Add(result.Text); - if (Tracked.TryGetValue(result.Text, out var t)) - { - t.Quad = quad; - t.MissedFrames = 0; - } - else - { - Tracked[result.Text] = new TrackedQr { Quad = quad, MissedFrames = 0 }; - } - EnsureImageLoaded(result.Text); } + // For codes we're already tracking but the global pass didn't find this frame, actively + // re-search a small, upscaled crop around their last known position instead of just + // freezing them in place. This recovers far more often for codes that are small, + // partially blurred, or at a steep angle - the global full-frame pass is much less + // likely to catch those, but a zoomed-in targeted decode often still can. + foreach (var kv in Tracked) + { + if (seenThisFrame.Contains(kv.Key)) continue; + if (TryLocalRedetect(enhanced, kv.Value.RawQuad, out var pTL, out var pTR, out var pBR, out var pBL)) + { + UpdateTracked(kv.Key, pTL, pTR, pBR, pBL); + seenThisFrame.Add(kv.Key); + } + } + // Age out codes that weren't seen this frame; keep them briefly to avoid flicker. List toRemove = null; foreach (var kv in Tracked) @@ -178,6 +188,44 @@ namespace BarcodeReaderExample } } + static void UpdateTracked(string text, Point2f tl, Point2f tr, Point2f br, Point2f bl) + { + Point2f[] raw = { tl, tr, br, bl }; + Point2f[] quad = ExpandQuad(tl, tr, br, bl, QuadExpansion); + + if (Tracked.TryGetValue(text, out var t)) + { + t.RawQuad = raw; + t.Quad = quad; + t.MissedFrames = 0; + } + else + { + Tracked[text] = new TrackedQr { RawQuad = raw, Quad = quad, MissedFrames = 0 }; + } + } + + /// + /// ZXing QR point order: [0]=bottom-left, [1]=top-left, [2]=top-right, optional [3]=alignment + /// pattern. We deliberately do NOT use points[3] for the bottom-right corner: the alignment + /// pattern sits several modules inset from the actual edge of the code, so it's always + /// noticeably closer to the center than the true corner is - using it directly shrinks that + /// whole side of the overlay. Instead derive the true bottom-right corner from the 3 + /// finder-pattern corners, which ZXing locates precisely, assuming the QR is (near) square: + /// BR = TR + BL - TL. + /// + static (Point2f? tl, Point2f? tr, Point2f? br, Point2f? bl) CornersFromResult(Result result) + { + var points = result?.ResultPoints; + if (points == null || points.Length < 3) return (null, null, null, null); + + Point2f pBL = new Point2f(points[0].X, points[0].Y); + Point2f pTL = new Point2f(points[1].X, points[1].Y); + Point2f pTR = new Point2f(points[2].X, points[2].Y); + Point2f pBR = new Point2f(pTR.X + pBL.X - pTL.X, pTR.Y + pBL.Y - pTL.Y); + return (pTL, pTR, pBR, pBL); + } + /// Expand a quad (TL,TR,BR,BL) outward from its centroid by a scale factor. static Point2f[] ExpandQuad(Point2f tl, Point2f tr, Point2f br, Point2f bl, float scale) { @@ -222,17 +270,7 @@ namespace BarcodeReaderExample static Result[] ReadQrCodes(Mat gray) { - if (!gray.IsContinuous()) - gray = gray.Clone(); - - int size = (int)(gray.Total() * gray.ElemSize()); - if (_grayBuffer == null || _grayBuffer.Length != size) - _grayBuffer = new byte[size]; - Marshal.Copy(gray.Data, _grayBuffer, 0, size); - - // Gray8 skips ZXing's internal RGB->luminance conversion entirely - the fastest - // format to hand it, and matches what BGR2GRAY already computed for us in OpenCV. - var luminanceSource = new RGBLuminanceSource(_grayBuffer, gray.Width, gray.Height, RGBLuminanceSource.BitmapFormat.Gray8); + var luminanceSource = ToLuminanceSource(gray, ref _grayBuffer); var bitmap = new BinaryBitmap(new HybridBinarizer(luminanceSource)); try @@ -245,6 +283,69 @@ namespace BarcodeReaderExample } } + /// + /// Crop tightly around a code's last known position (with padding for movement), upscale + /// that crop, and try to decode just it. Works much better than a full-frame scan for a + /// code that's already small, tilted, or motion-blurred, since the decoder effectively gets + /// a bigger, cleaner version of just the region that matters. + /// + static bool TryLocalRedetect(Mat enhanced, Point2f[] lastRaw, out Point2f tl, out Point2f tr, out Point2f br, out Point2f bl) + { + tl = tr = br = bl = default; + if (lastRaw == null) return false; + + var intPoints = lastRaw.Select(p => new Point((int)p.X, (int)p.Y)).ToArray(); + Rect box = Cv2.BoundingRect(intPoints); + int pad = (int)(Math.Max(box.Width, box.Height) * LocalRedetectPadding); + Rect search = new Rect(box.X - pad, box.Y - pad, box.Width + pad * 2, box.Height + pad * 2); + search = search.Intersect(new Rect(0, 0, enhanced.Width, enhanced.Height)); + if (search.Width < 15 || search.Height < 15) return false; + + using var crop = new Mat(enhanced, search); + using var scaled = new Mat(); + Cv2.Resize(crop, scaled, new Size(crop.Width * LocalRedetectUpscale, crop.Height * LocalRedetectUpscale), + 0, 0, InterpolationFlags.Cubic); + + byte[] localBuffer = null; + var luminanceSource = ToLuminanceSource(scaled, ref localBuffer); + var bitmap = new BinaryBitmap(new HybridBinarizer(luminanceSource)); + + Result result; + try { result = QrReader.decode(bitmap, LocalHints); } + catch { return false; } + if (result == null) return false; + + var (cTl, cTr, cBr, cBl) = CornersFromResult(result); + if (cTl == null) return false; + + // Map corners from the upscaled crop back into full-frame coordinates. + Point2f ToFrame(Point2f p) => new Point2f( + search.X + p.X / LocalRedetectUpscale, + search.Y + p.Y / LocalRedetectUpscale); + + tl = ToFrame(cTl.Value); + tr = ToFrame(cTr.Value); + br = ToFrame(cBr.Value); + bl = ToFrame(cBl.Value); + return true; + } + + static RGBLuminanceSource ToLuminanceSource(Mat gray, ref byte[] buffer) + { + Mat continuous = gray.IsContinuous() ? gray : gray.Clone(); + int size = (int)(continuous.Total() * continuous.ElemSize()); + if (buffer == null || buffer.Length != size) + buffer = new byte[size]; + Marshal.Copy(continuous.Data, buffer, 0, size); + + // Gray8 skips ZXing's internal RGB->luminance conversion entirely - the fastest + // format to hand it, and matches what BGR2GRAY/CLAHE already produced for us. + var source = new RGBLuminanceSource(buffer, continuous.Width, continuous.Height, RGBLuminanceSource.BitmapFormat.Gray8); + if (!ReferenceEquals(continuous, gray)) + continuous.Dispose(); + return source; + } + // ---- async image download/caching -------------------------------- static void EnsureImageLoaded(string url)