From 0b3f31d21b53c3d6a1eb6b5d66958a45aabb8fda Mon Sep 17 00:00:00 2001 From: Jake Humphrey Date: Sun, 15 Mar 2020 18:24:38 -0500 Subject: [PATCH 1/2] Look for largest rectangle not just first rectangle --- .../helpers/ImageProcessor.java | 86 ++++++++++++------- 1 file changed, 56 insertions(+), 30 deletions(-) diff --git a/android/src/main/java/com/rectanglescanner/helpers/ImageProcessor.java b/android/src/main/java/com/rectanglescanner/helpers/ImageProcessor.java index 5f1b816..37abdc1 100755 --- a/android/src/main/java/com/rectanglescanner/helpers/ImageProcessor.java +++ b/android/src/main/java/com/rectanglescanner/helpers/ImageProcessor.java @@ -22,6 +22,7 @@ import org.opencv.core.MatOfPoint2f; import org.opencv.core.Point; import org.opencv.core.Size; +import org.opencv.core.Rect; import org.opencv.imgcodecs.Imgcodecs; import org.opencv.imgproc.Imgproc; @@ -108,8 +109,7 @@ private void detectRectangleInFrame(Mat inputRgba) { Size srcSize = inputRgba.size(); this.lastDetectedRectangle = getQuadrilateral(contours, srcSize); Bundle data = new Bundle(); - boolean focused = mMainActivity.isFocused(); - if (focused && this.lastDetectedRectangle != null) { + if (this.lastDetectedRectangle != null) { Bundle quadMap = this.lastDetectedRectangle.toBundle(); data.putBundle("detectedRectangle", quadMap); } else { @@ -151,26 +151,41 @@ private Quadrilateral getQuadrilateral(ArrayList contours, Size srcS int width = Double.valueOf(srcSize.width).intValue(); Size size = new Size(width, height); + double areaOfPreview = height * width; + double largestContourArea = 0; + Point[] bestFittingPoints = null; + MatOfPoint bestFittingContour = null; Log.i(TAG, "Size----->" + size); for (MatOfPoint c : contours) { + double contourArea = Imgproc.contourArea(c); + double minArea = size.area() * 0.01; + + // Only allow shapes that have an area of at least 1% of the preview size + if (contourArea < minArea) { + continue; + } + MatOfPoint2f c2f = new MatOfPoint2f(c.toArray()); double peri = Imgproc.arcLength(c2f, true); MatOfPoint2f approx = new MatOfPoint2f(); Imgproc.approxPolyDP(c2f, approx, 0.02 * peri, true); + Point[] foundPoints = sortPoints(approx.toArray()); + if (isValidRectangle(foundPoints) && contourArea > largestContourArea) { + largestContourArea = contourArea; + bestFittingPoints = foundPoints; + bestFittingContour = c; + } - Point[] points = approx.toArray(); - - // select biggest 4 angles polygon - // if (points.length == 4) { - Point[] foundPoints = sortPoints(points); - - if (insideArea(foundPoints, size)) { - - return new Quadrilateral(c, foundPoints, new Size(srcSize.width, srcSize.height)); + if (largestContourArea > areaOfPreview * 0.7) { + break; } - // } } + if (bestFittingContour != null) { + return new Quadrilateral(bestFittingContour, bestFittingPoints, new Size(srcSize.width, srcSize.height)); + } + + return null; } @@ -210,28 +225,39 @@ public int compare(Point lhs, Point rhs) { return result; } - private boolean insideArea(Point[] rp, Size size) { - - int width = Double.valueOf(size.width).intValue(); - int height = Double.valueOf(size.height).intValue(); - - int minimumSize = width / 10; - + private boolean isValidRectangle(Point[] rp) { boolean isANormalShape = rp[0].x != rp[1].x && rp[1].y != rp[0].y && rp[2].y != rp[3].y && rp[3].x != rp[2].x; - boolean isBigEnough = ((rp[1].x - rp[0].x >= minimumSize) && (rp[2].x - rp[3].x >= minimumSize) - && (rp[3].y - rp[0].y >= minimumSize) && (rp[2].y - rp[1].y >= minimumSize)); + if (!isANormalShape) { + return false; + } - double leftOffset = rp[0].x - rp[3].x; - double rightOffset = rp[1].x - rp[2].x; - double bottomOffset = rp[0].y - rp[1].y; - double topOffset = rp[2].y - rp[3].y; + double leftOffset = Math.abs(rp[0].x - rp[3].x); + double rightOffset = Math.abs(rp[1].x - rp[2].x); + double bottomOffset = Math.abs(rp[0].y - rp[1].y); + double topOffset = Math.abs(rp[2].y - rp[3].y); + + double largestVertical = Math.max(leftOffset, rightOffset); + double verticalOffset = Math.abs(leftOffset - rightOffset); + double largestHorizontal = Math.max(topOffset, bottomOffset); + double horizontalOffset = Math.abs(topOffset - bottomOffset); + double largestSide = Math.max(largestHorizontal, largestVertical); + double sideOffset = Math.abs(largestHorizontal - largestVertical); + + // Ensure vertical sides are within 90% of each other + if (verticalOffset > (largestVertical * 0.9)) { + return false; + } - boolean isAnActualRectangle = ((leftOffset <= minimumSize && leftOffset >= -minimumSize) - && (rightOffset <= minimumSize && rightOffset >= -minimumSize) - && (bottomOffset <= minimumSize && bottomOffset >= -minimumSize) - && (topOffset <= minimumSize && topOffset >= -minimumSize)); + // Ensure horizontal sides are within 90% of each other + if (horizontalOffset > (largestHorizontal * 0.9)) { + return false; + } - return isANormalShape && isAnActualRectangle && isBigEnough; + // Ensure all sides are within 99.5% of each other +// if (sideOffset > (largestSide * 0.995)) { +// return false; +// } + return true; } private Mat fourPointTransform(Mat src, Point[] pts) { From 784ba3967efae0361bd7b453af15280c5131516f Mon Sep 17 00:00:00 2001 From: Jake Humphrey Date: Sun, 22 Mar 2020 19:52:28 -0500 Subject: [PATCH 2/2] messing with image before checking for rectangles --- .../helpers/ImageProcessor.java | 87 ++++++++++--------- 1 file changed, 44 insertions(+), 43 deletions(-) diff --git a/android/src/main/java/com/rectanglescanner/helpers/ImageProcessor.java b/android/src/main/java/com/rectanglescanner/helpers/ImageProcessor.java index 37abdc1..0b6e623 100755 --- a/android/src/main/java/com/rectanglescanner/helpers/ImageProcessor.java +++ b/android/src/main/java/com/rectanglescanner/helpers/ImageProcessor.java @@ -18,9 +18,12 @@ import org.opencv.core.Core; import org.opencv.core.CvType; import org.opencv.core.Mat; +import org.opencv.core.MatOfDouble; import org.opencv.core.MatOfPoint; import org.opencv.core.MatOfPoint2f; import org.opencv.core.Point; +import org.opencv.core.Range; +import org.opencv.core.Scalar; import org.opencv.core.Size; import org.opencv.core.Rect; import org.opencv.imgcodecs.Imgcodecs; @@ -152,15 +155,13 @@ private Quadrilateral getQuadrilateral(ArrayList contours, Size srcS Size size = new Size(width, height); double areaOfPreview = height * width; - double largestContourArea = 0; - Point[] bestFittingPoints = null; + double minArea = size.area() * 0.01; MatOfPoint bestFittingContour = null; + Log.i(TAG, "Size----->" + size); for (MatOfPoint c : contours) { double contourArea = Imgproc.contourArea(c); - double minArea = size.area() * 0.01; - // Only allow shapes that have an area of at least 1% of the preview size if (contourArea < minArea) { continue; } @@ -170,22 +171,10 @@ private Quadrilateral getQuadrilateral(ArrayList contours, Size srcS MatOfPoint2f approx = new MatOfPoint2f(); Imgproc.approxPolyDP(c2f, approx, 0.02 * peri, true); Point[] foundPoints = sortPoints(approx.toArray()); - if (isValidRectangle(foundPoints) && contourArea > largestContourArea) { - largestContourArea = contourArea; - bestFittingPoints = foundPoints; - bestFittingContour = c; - } - - if (largestContourArea > areaOfPreview * 0.7) { - break; + if (isValidRectangle(foundPoints)) { + return new Quadrilateral(c, foundPoints, new Size(srcSize.width, srcSize.height)); } } - - if (bestFittingContour != null) { - return new Quadrilateral(bestFittingContour, bestFittingPoints, new Size(srcSize.width, srcSize.height)); - } - - return null; } @@ -243,17 +232,14 @@ private boolean isValidRectangle(Point[] rp) { double largestSide = Math.max(largestHorizontal, largestVertical); double sideOffset = Math.abs(largestHorizontal - largestVertical); - // Ensure vertical sides are within 90% of each other if (verticalOffset > (largestVertical * 0.9)) { return false; } - // Ensure horizontal sides are within 90% of each other if (horizontalOffset > (largestHorizontal * 0.9)) { return false; } - // Ensure all sides are within 99.5% of each other // if (sideOffset > (largestSide * 0.995)) { // return false; // } @@ -295,28 +281,12 @@ private Mat fourPointTransform(Mat src, Point[] pts) { } private ArrayList findContours(Mat src) { - - Mat grayImage; - Mat cannedImage; - Mat resizedImage; - - int height = Double.valueOf(src.size().height).intValue(); - int width = Double.valueOf(src.size().width).intValue(); - Size size = new Size(width, height); - - resizedImage = new Mat(size, CvType.CV_8UC4); - grayImage = new Mat(size, CvType.CV_8UC4); - cannedImage = new Mat(size, CvType.CV_8UC1); - - Imgproc.resize(src, resizedImage, size); - Imgproc.cvtColor(resizedImage, grayImage, Imgproc.COLOR_RGBA2GRAY, 4); - Imgproc.GaussianBlur(grayImage, grayImage, new Size(5, 5), 0); - Imgproc.Canny(grayImage, cannedImage, 80, 100, 3, false); + Mat cannedImage = applyCannedFilterToImage(src); ArrayList contours = new ArrayList<>(); Mat hierarchy = new Mat(); - Imgproc.findContours(cannedImage, contours, hierarchy, Imgproc.RETR_TREE, Imgproc.CHAIN_APPROX_SIMPLE); + Imgproc.findContours(cannedImage, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE); hierarchy.release(); @@ -328,8 +298,6 @@ public int compare(MatOfPoint lhs, MatOfPoint rhs) { } }); - resizedImage.release(); - grayImage.release(); cannedImage.release(); return contours; @@ -362,6 +330,20 @@ public void applyFilters(Mat image) { } } + /** + * Returns the mean brightness level of the provided image + * @param image + * @return brightness level + */ + public double brightnessOfImage(Mat image) + { + MatOfDouble meansrc= new MatOfDouble(); + MatOfDouble stdsrc= new MatOfDouble(); + + Core.meanStdDev(image, meansrc, stdsrc); + return meansrc.get(0,0)[0]; + } + /*! Slightly enhances the black and white image */ @@ -376,8 +358,9 @@ public Mat applyGreyscaleFilterToImage(Mat image) */ public Mat applyBlackAndWhiteFilterToImage(Mat image) { + double imageBrightnessLevel = brightnessOfImage(image); + image.convertTo(image, -1, 2.8 - (imageBrightnessLevel * 0.0045), imageBrightnessLevel * -1.15); Imgproc.cvtColor(image, image, Imgproc.COLOR_RGBA2GRAY); - image.convertTo(image, -1, 1, 10); return image; } @@ -386,10 +369,28 @@ public Mat applyBlackAndWhiteFilterToImage(Mat image) */ public Mat applyColorFilterToImage(Mat image) { - image.convertTo(image, -1, 1.2, 0); + double imageBrightnessLevel = brightnessOfImage(image); + image.convertTo(image, -1, 2.8 - (imageBrightnessLevel * 0.0045), imageBrightnessLevel * -1.15); return image; } + public Mat applyCannedFilterToImage(Mat image) + { + int blurSize = 5; + int thresh = 100; + double imageBrightnessLevel = brightnessOfImage(image); + Mat colorAdjustedImage = new Mat(); + Mat cannedImage = new Mat(); + + image.convertTo(colorAdjustedImage, -1, 2.8 - (imageBrightnessLevel * 0.0045), imageBrightnessLevel * -1.15); + Imgproc.cvtColor(colorAdjustedImage, colorAdjustedImage, Imgproc.COLOR_RGBA2GRAY, 4); + // Imgproc.equalizeHist(colorAdjustedImage, colorAdjustedImage); + Imgproc.bilateralFilter(colorAdjustedImage, cannedImage, blurSize, blurSize * 2, blurSize / 2); + Imgproc.Canny(cannedImage, cannedImage, thresh, thresh * 3, 3, true); + colorAdjustedImage.release(); + return cannedImage; + } + public void rotateImageForScreen(Mat image) { switch (this.mMainActivity.lastDetectedRotation) {