using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Net.Http; using System.Runtime.InteropServices; using System.Threading.Tasks; using OpenCvSharp; using ZXing; using ZXing.Common; using ZXing.Multi; using ZXing.QrCode; namespace BarcodeReaderExample { class Program { // ---- tunables ------------------------------------------------- const bool EnableOverlay = true; // master draw switch (debug) const bool TryHarderMultiDecode = true; // more reliable multi-QR detection, costs some speed const float QuadExpansion = 1.50f; // how much bigger than the QR the overlay is drawn const int MaxMissedFrames = 8; // keep last known position this many frames if briefly lost const int TargetFps = 24; const float LocalRedetectPadding = 0.6f; // extra search margin around last known box, as a fraction of its size const int LocalRedetectUpscale = 3; // how much to enlarge the crop before decoding it static readonly HttpClient Http = new HttpClient(); static readonly ConcurrentDictionary ImageCache = new(); // text -> decoded image (null = failed) static readonly ConcurrentDictionary Pending = new(); // text currently downloading static readonly QRCodeReader QrReader = new QRCodeReader(); static readonly GenericMultipleBarcodeReader MultiReader = new GenericMultipleBarcodeReader(QrReader); static readonly Dictionary Hints = new() { { DecodeHintType.POSSIBLE_FORMATS, new List { BarcodeFormat.QR_CODE } }, { DecodeHintType.TRY_HARDER, TryHarderMultiDecode }, }; // Used only for the small, targeted local-redetect crop, so the extra cost is negligible there. static readonly Dictionary LocalHints = new() { { DecodeHintType.POSSIBLE_FORMATS, new List { BarcodeFormat.QR_CODE } }, { DecodeHintType.TRY_HARDER, true }, { DecodeHintType.ALSO_INVERTED, true }, }; static readonly CLAHE Clahe = Cv2.CreateCLAHE(clipLimit: 3.0, tileGridSize: new Size(8, 8)); static byte[] _grayBuffer; static readonly Dictionary Tracked = new(); class TrackedQr { public Point2f[] RawQuad; // TL, TR, BR, BL as detected, unexpanded - used to seed the next search public Point2f[] Quad; // TL, TR, BR, BL, expanded - used for drawing public int MissedFrames; } static void Main(string[] args) { using VideoCapture camera = new VideoCapture("/dev/video0", VideoCaptureAPIs.V4L2); if (!camera.IsOpened()) throw new Exception("Nie można otworzyć kamery /dev/video0"); camera.Set(VideoCaptureProperties.FrameWidth, 1280); camera.Set(VideoCaptureProperties.FrameHeight, 720); camera.Set(VideoCaptureProperties.Fps, TargetFps); Cv2.NamedWindow("Camera preview", WindowFlags.AutoSize); using var frame = new Mat(); using var gray = new Mat(); var sw = Stopwatch.StartNew(); var frameTimer = Stopwatch.StartNew(); int frameCount = 0; double fps = 0; try { while (true) { frameTimer.Restart(); if (!camera.Read(frame) || frame.Empty()) continue; try { ProcessFrame(frame, gray); } catch (Exception ex) { // Never let a single bad frame kill the capture loop. Console.WriteLine("Frame processing error: " + ex.Message); } frameCount++; if (sw.ElapsedMilliseconds >= 500) { fps = frameCount / (sw.ElapsedMilliseconds / 1000.0); frameCount = 0; sw.Restart(); } Cv2.PutText(frame, $"{fps:0.0} fps", new Point(10, frame.Height - 15), HersheyFonts.HersheySimplex, 0.6, Scalar.Lime, 2); Cv2.ImShow("Camera preview", frame); // Only sleep for whatever time is left in the frame budget instead of // always sleeping the full 1000/fps on top of WaitKey (which halved // the effective frame rate in the original code). int budgetMs = 1000 / TargetFps; int elapsedMs = (int)frameTimer.ElapsedMilliseconds; int waitMs = Math.Max(1, budgetMs - elapsedMs); int key = Cv2.WaitKey(waitMs); if (key == 27 || key == 'q') break; } } finally { foreach (var mat in ImageCache.Values) mat?.Dispose(); Cv2.DestroyAllWindows(); } } static void ProcessFrame(Mat frame, Mat gray) { Cv2.CvtColor(frame, gray, ColorConversionCodes.BGR2GRAY); // Contrast-normalize before decoding - this is the single biggest lever for detection // reliability under uneven lighting (glare, shadow across part of the code, dim rooms). using var enhanced = new Mat(); Clahe.Apply(gray, enhanced); Result[] results = ReadQrCodes(enhanced); var seenThisFrame = new HashSet(); foreach (var result in results) { if (result?.Text == null) continue; var (pTL, pTR, pBR, pBL) = CornersFromResult(result); if (pTL == null) continue; UpdateTracked(result.Text, pTL.Value, pTR.Value, pBR.Value, pBL.Value); seenThisFrame.Add(result.Text); EnsureImageLoaded(result.Text); } // For codes we're already tracking but the global pass didn't find this frame, actively // re-search a small, upscaled crop around their last known position instead of just // freezing them in place. This recovers far more often for codes that are small, // partially blurred, or at a steep angle - the global full-frame pass is much less // likely to catch those, but a zoomed-in targeted decode often still can. foreach (var kv in Tracked) { if (seenThisFrame.Contains(kv.Key)) continue; if (TryLocalRedetect(enhanced, kv.Value.RawQuad, out var pTL, out var pTR, out var pBR, out var pBL)) { UpdateTracked(kv.Key, pTL, pTR, pBR, pBL); seenThisFrame.Add(kv.Key); } } // Age out codes that weren't seen this frame; keep them briefly to avoid flicker. List toRemove = null; foreach (var kv in Tracked) { if (seenThisFrame.Contains(kv.Key)) continue; kv.Value.MissedFrames++; if (kv.Value.MissedFrames > MaxMissedFrames) (toRemove ??= new List()).Add(kv.Key); } if (toRemove != null) foreach (var key in toRemove) Tracked.Remove(key); if (!EnableOverlay) return; foreach (var kv in Tracked) { if (ImageCache.TryGetValue(kv.Key, out Mat img) && img != null && !img.Empty()) { OverlayImagePerspective(frame, img, kv.Value.Quad); } } } static void UpdateTracked(string text, Point2f tl, Point2f tr, Point2f br, Point2f bl) { Point2f[] raw = { tl, tr, br, bl }; Point2f[] quad = ExpandQuad(tl, tr, br, bl, QuadExpansion); if (Tracked.TryGetValue(text, out var t)) { t.RawQuad = raw; t.Quad = quad; t.MissedFrames = 0; } else { Tracked[text] = new TrackedQr { RawQuad = raw, Quad = quad, MissedFrames = 0 }; } } /// /// ZXing QR point order: [0]=bottom-left, [1]=top-left, [2]=top-right, optional [3]=alignment /// pattern. We deliberately do NOT use points[3] for the bottom-right corner: the alignment /// pattern sits several modules inset from the actual edge of the code, so it's always /// noticeably closer to the center than the true corner is - using it directly shrinks that /// whole side of the overlay. Instead derive the true bottom-right corner from the 3 /// finder-pattern corners, which ZXing locates precisely, assuming the QR is (near) square: /// BR = TR + BL - TL. /// static (Point2f? tl, Point2f? tr, Point2f? br, Point2f? bl) CornersFromResult(Result result) { var points = result?.ResultPoints; if (points == null || points.Length < 3) return (null, null, null, null); Point2f pBL = new Point2f(points[0].X, points[0].Y); Point2f pTL = new Point2f(points[1].X, points[1].Y); Point2f pTR = new Point2f(points[2].X, points[2].Y); Point2f pBR = new Point2f(pTR.X + pBL.X - pTL.X, pTR.Y + pBL.Y - pTL.Y); return (pTL, pTR, pBR, pBL); } /// Expand a quad (TL,TR,BR,BL) outward from its centroid by a scale factor. static Point2f[] ExpandQuad(Point2f tl, Point2f tr, Point2f br, Point2f bl, float scale) { float cx = (tl.X + tr.X + br.X + bl.X) / 4f; float cy = (tl.Y + tr.Y + br.Y + bl.Y) / 4f; Point2f Expand(Point2f p) => new Point2f(cx + (p.X - cx) * scale, cy + (p.Y - cy) * scale); return new[] { Expand(tl), Expand(tr), Expand(br), Expand(bl) }; } /// /// Warp `overlayImage` onto the quadrilateral `quad` (TL,TR,BR,BL) in `frame`, so the /// pasted image follows the QR code's rotation/perspective instead of a plain axis-aligned resize. /// Only the quad's bounding box is warped, not the whole frame, to keep this fast with /// several codes on screen at once. /// static void OverlayImagePerspective(Mat frame, Mat overlayImage, Point2f[] quad) { var intPoints = quad.Select(p => new Point((int)p.X, (int)p.Y)).ToArray(); Rect rect = Cv2.BoundingRect(intPoints); rect = rect.Intersect(new Rect(0, 0, frame.Width, frame.Height)); if (rect.Width <= 1 || rect.Height <= 1) return; int w = overlayImage.Width, h = overlayImage.Height; Point2f[] src = { new(0, 0), new(w, 0), new(w, h), new(0, h) }; Point2f[] dstLocal = quad.Select(p => new Point2f(p.X - rect.X, p.Y - rect.Y)).ToArray(); using Mat homography = Cv2.GetPerspectiveTransform(src, dstLocal); using var warped = new Mat(); Cv2.WarpPerspective(overlayImage, warped, homography, rect.Size); using var mask = new Mat(rect.Size, MatType.CV_8UC1, Scalar.Black); var localPoly = dstLocal.Select(p => new Point((int)p.X, (int)p.Y)).ToArray(); Cv2.FillPoly(mask, new[] { localPoly }, Scalar.White); using var roi = new Mat(frame, rect); warped.CopyTo(roi, mask); } // ---- decoding --------------------------------------------------- static Result[] ReadQrCodes(Mat gray) { var luminanceSource = ToLuminanceSource(gray, ref _grayBuffer); var bitmap = new BinaryBitmap(new HybridBinarizer(luminanceSource)); try { return MultiReader.decodeMultiple(bitmap, Hints) ?? Array.Empty(); } catch { return Array.Empty(); } } /// /// Crop tightly around a code's last known position (with padding for movement), upscale /// that crop, and try to decode just it. Works much better than a full-frame scan for a /// code that's already small, tilted, or motion-blurred, since the decoder effectively gets /// a bigger, cleaner version of just the region that matters. /// static bool TryLocalRedetect(Mat enhanced, Point2f[] lastRaw, out Point2f tl, out Point2f tr, out Point2f br, out Point2f bl) { tl = tr = br = bl = default; if (lastRaw == null) return false; var intPoints = lastRaw.Select(p => new Point((int)p.X, (int)p.Y)).ToArray(); Rect box = Cv2.BoundingRect(intPoints); int pad = (int)(Math.Max(box.Width, box.Height) * LocalRedetectPadding); Rect search = new Rect(box.X - pad, box.Y - pad, box.Width + pad * 2, box.Height + pad * 2); search = search.Intersect(new Rect(0, 0, enhanced.Width, enhanced.Height)); if (search.Width < 15 || search.Height < 15) return false; using var crop = new Mat(enhanced, search); using var scaled = new Mat(); Cv2.Resize(crop, scaled, new Size(crop.Width * LocalRedetectUpscale, crop.Height * LocalRedetectUpscale), 0, 0, InterpolationFlags.Cubic); byte[] localBuffer = null; var luminanceSource = ToLuminanceSource(scaled, ref localBuffer); var bitmap = new BinaryBitmap(new HybridBinarizer(luminanceSource)); Result result; try { result = QrReader.decode(bitmap, LocalHints); } catch { return false; } if (result == null) return false; var (cTl, cTr, cBr, cBl) = CornersFromResult(result); if (cTl == null) return false; // Map corners from the upscaled crop back into full-frame coordinates. Point2f ToFrame(Point2f p) => new Point2f( search.X + p.X / LocalRedetectUpscale, search.Y + p.Y / LocalRedetectUpscale); tl = ToFrame(cTl.Value); tr = ToFrame(cTr.Value); br = ToFrame(cBr.Value); bl = ToFrame(cBl.Value); return true; } static RGBLuminanceSource ToLuminanceSource(Mat gray, ref byte[] buffer) { Mat continuous = gray.IsContinuous() ? gray : gray.Clone(); int size = (int)(continuous.Total() * continuous.ElemSize()); if (buffer == null || buffer.Length != size) buffer = new byte[size]; Marshal.Copy(continuous.Data, buffer, 0, size); // Gray8 skips ZXing's internal RGB->luminance conversion entirely - the fastest // format to hand it, and matches what BGR2GRAY/CLAHE already produced for us. var source = new RGBLuminanceSource(buffer, continuous.Width, continuous.Height, RGBLuminanceSource.BitmapFormat.Gray8); if (!ReferenceEquals(continuous, gray)) continuous.Dispose(); return source; } // ---- async image download/caching -------------------------------- static void EnsureImageLoaded(string url) { if (ImageCache.ContainsKey(url) || Pending.ContainsKey(url)) return; if (!Uri.TryCreate(url, UriKind.Absolute, out var uri) || (uri.Scheme != Uri.UriSchemeHttp && uri.Scheme != Uri.UriSchemeHttps)) { // Not a downloadable URL - remember that so we don't keep retrying every frame. ImageCache[url] = null; return; } Pending[url] = 0; _ = Task.Run(async () => { try { byte[] bytes = await Http.GetByteArrayAsync(uri); Mat decoded = Mat.FromImageData(bytes, ImreadModes.Color); ImageCache[url] = decoded.Empty() ? null : decoded; } catch (Exception ex) { Console.WriteLine($"Failed to download {url}: {ex.Message}"); ImageCache[url] = null; // negative-cache so we stop retrying a bad URL } finally { Pending.TryRemove(url, out _); } }); } } }