001/* 002 * Copyright (C) Photon Vision. 003 * 004 * This program is free software: you can redistribute it and/or modify 005 * it under the terms of the GNU General Public License as published by 006 * the Free Software Foundation, either version 3 of the License, or 007 * (at your option) any later version. 008 * 009 * This program is distributed in the hope that it will be useful, 010 * but WITHOUT ANY WARRANTY; without even the implied warranty of 011 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 012 * GNU General Public License for more details. 013 * 014 * You should have received a copy of the GNU General Public License 015 * along with this program. If not, see <https://www.gnu.org/licenses/>. 016 */ 017 018package org.photonvision.vision.pipe.impl; 019 020import java.util.*; 021import org.opencv.core.MatOfPoint2f; 022import org.opencv.core.Point; 023import org.opencv.imgproc.Imgproc; 024import org.photonvision.vision.pipe.CVPipe; 025import org.photonvision.vision.target.TrackedTarget; 026 027/** 028 * Determines the target corners of the {@link TrackedTarget}. The {@link 029 * CornerDetectionPipeParameters} affect how these corners are calculated. 030 */ 031public class CornerDetectionPipe 032 extends CVPipe< 033 List<TrackedTarget>, 034 List<TrackedTarget>, 035 CornerDetectionPipe.CornerDetectionPipeParameters> { 036 Comparator<Point> leftRightComparator = Comparator.comparingDouble(point -> point.x); 037 Comparator<Point> verticalComparator = Comparator.comparingDouble(point -> point.y); 038 MatOfPoint2f polyOutput = new MatOfPoint2f(); 039 040 @Override 041 protected List<TrackedTarget> process(List<TrackedTarget> targetList) { 042 for (var target : targetList) { 043 // detect corners. Might implement more algorithms later but 044 // APPROX_POLY_DP_AND_EXTREME_CORNERS should be year agnostic 045 if (Objects.requireNonNull(params.cornerDetectionStrategy) 046 == DetectionStrategy.APPROX_POLY_DP_AND_EXTREME_CORNERS) { 047 var targetCorners = detectExtremeCornersByApproxPolyDp(target, params.calculateConvexHulls); 048 target.setTargetCorners(targetCorners); 049 } 050 } 051 return targetList; 052 } 053 054 /** 055 * @param target The target to find the corners of. 056 * @return Corners: (bottom-left, bottom-right, top-right, top-left) 057 */ 058 private List<Point> findBoundingBoxCorners(TrackedTarget target) { 059 // extract the corners 060 var points = new Point[4]; 061 target.m_mainContour.getMinAreaRect().points(points); 062 063 // find the bl/br/tr/tl corners 064 // first, min by left/right 065 var list_ = Arrays.asList(points); 066 list_.sort(leftRightComparator); 067 // of this, we now have left and right 068 // sort to get top and bottom 069 var left = new ArrayList<>(List.of(list_.get(0), list_.get(1))); 070 left.sort(verticalComparator); 071 var right = new ArrayList<>(List.of(list_.get(2), list_.get(3))); 072 right.sort(verticalComparator); 073 074 var tl = left.get(0); 075 var bl = left.get(1); 076 var tr = right.get(0); 077 var br = right.get(1); 078 079 return List.of(bl, br, tr, tl); 080 } 081 082 /** 083 * @param a First point. 084 * @param b Second point. 085 * @return The straight line distance between them. 086 */ 087 private static double distanceBetween(Point a, Point b) { 088 double xDelta = a.x - b.x; 089 double yDelta = a.y - b.y; 090 return Math.sqrt(xDelta * xDelta + yDelta * yDelta); 091 } 092 093 /** 094 * Find the 4 most extreme corners of the target's contour. 095 * 096 * @param target The target to track. 097 * @param convexHull Whether to use the convex hull of the contour instead. 098 * @return The 4 extreme corners of the contour: (bottom-left, bottom-right, top-right, top-left) 099 */ 100 private List<Point> detectExtremeCornersByApproxPolyDp(TrackedTarget target, boolean convexHull) { 101 var centroid = target.getMinAreaRect().center; 102 Comparator<Point> compareCenterDist = 103 Comparator.comparingDouble((Point point) -> distanceBetween(centroid, point)); 104 105 MatOfPoint2f targetContour; 106 if (convexHull) { 107 targetContour = target.m_mainContour.getConvexHull(); 108 } else { 109 targetContour = target.m_mainContour.getMat2f(); 110 } 111 112 /* 113 approximating a shape around the contours 114 Can be tuned to allow/disallow hulls 115 we want a number between 0 and 0.16 out of a percentage from 0 to 100 116 so take accuracy and divide by 600 117 118 Furthermore, we know that the contour is open if we haven't done convex hulls, 119 and it has subcontours. 120 */ 121 var isOpen = !convexHull && target.hasSubContours(); 122 var peri = Imgproc.arcLength(targetContour, true); 123 Imgproc.approxPolyDP( 124 targetContour, polyOutput, params.accuracyPercentage / 600.0 * peri, !isOpen); 125 126 // we must have at least 4 corners for this strategy to work. 127 // If we are looking for an exact side count that is handled here too. 128 var pointList = new ArrayList<>(polyOutput.toList()); 129 if (pointList.size() < 4 || (params.exactSideCount && params.sideCount != pointList.size())) 130 return null; 131 132 target.setApproximateBoundingPolygon(polyOutput); 133 134 // left top, left bottom, right bottom, right top 135 var boundingBoxCorners = findBoundingBoxCorners(target); 136 137 var compareDistToTl = 138 Comparator.comparingDouble((Point p) -> distanceBetween(p, boundingBoxCorners.get(3))); 139 140 var compareDistToTr = 141 Comparator.comparingDouble((Point p) -> distanceBetween(p, boundingBoxCorners.get(2))); 142 143 // top left and top right are the poly corners closest to the bounding box tl and tr 144 pointList.sort(compareDistToTl); 145 var tl = pointList.get(0); 146 pointList.remove(tl); 147 pointList.sort(compareDistToTr); 148 var tr = pointList.get(0); 149 pointList.remove(tr); 150 151 // at this point we look for points on the left/right of the center of the remaining points 152 // and maximize their distance from the center of the min area rectangle 153 var leftList = new ArrayList<Point>(); 154 var rightList = new ArrayList<Point>(); 155 double averageXCoordinate = 0.0; 156 for (var p : pointList) { 157 averageXCoordinate += p.x; 158 } 159 averageXCoordinate /= pointList.size(); 160 161 // add points that are below the center of the min area rectangle of the target 162 for (var p : pointList) { 163 if (p.y 164 > target.m_mainContour.getBoundingRect().y 165 + target.m_mainContour.getBoundingRect().height / 2.0) 166 if (p.x < averageXCoordinate) { 167 leftList.add(p); 168 } else { 169 rightList.add(p); 170 } 171 } 172 if (leftList.isEmpty() || rightList.isEmpty()) return null; 173 leftList.sort(compareCenterDist); 174 rightList.sort(compareCenterDist); 175 var bl = leftList.get(leftList.size() - 1); 176 var br = rightList.get(rightList.size() - 1); 177 return List.of(bl, br, tr, tl); 178 } 179 180 public static class CornerDetectionPipeParameters { 181 private final DetectionStrategy cornerDetectionStrategy; 182 183 private final boolean calculateConvexHulls; 184 private final boolean exactSideCount; 185 private final int sideCount; 186 187 /** This number can be changed to change how "accurate" our approximate polygon must be. */ 188 private final double accuracyPercentage; 189 190 public CornerDetectionPipeParameters( 191 DetectionStrategy cornerDetectionStrategy, 192 boolean calculateConvexHulls, 193 boolean exactSideCount, 194 int sideCount, 195 double accuracyPercentage) { 196 this.cornerDetectionStrategy = cornerDetectionStrategy; 197 this.calculateConvexHulls = calculateConvexHulls; 198 this.exactSideCount = exactSideCount; 199 this.sideCount = sideCount; 200 this.accuracyPercentage = accuracyPercentage; 201 } 202 } 203 204 public enum DetectionStrategy { 205 APPROX_POLY_DP_AND_EXTREME_CORNERS 206 } 207}