001/*
002 * Copyright (C) Photon Vision.
003 *
004 * This program is free software: you can redistribute it and/or modify
005 * it under the terms of the GNU General Public License as published by
006 * the Free Software Foundation, either version 3 of the License, or
007 * (at your option) any later version.
008 *
009 * This program is distributed in the hope that it will be useful,
010 * but WITHOUT ANY WARRANTY; without even the implied warranty of
011 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
012 * GNU General Public License for more details.
013 *
014 * You should have received a copy of the GNU General Public License
015 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
016 */
017
018package org.photonvision.vision.pipe.impl;
019
020import java.util.*;
021import org.opencv.core.MatOfPoint2f;
022import org.opencv.core.Point;
023import org.opencv.imgproc.Imgproc;
024import org.photonvision.vision.pipe.CVPipe;
025import org.photonvision.vision.target.TrackedTarget;
026
027/**
028 * Determines the target corners of the {@link TrackedTarget}. The {@link
029 * CornerDetectionPipeParameters} affect how these corners are calculated.
030 */
031public class CornerDetectionPipe
032        extends CVPipe<
033                List<TrackedTarget>,
034                List<TrackedTarget>,
035                CornerDetectionPipe.CornerDetectionPipeParameters> {
036    Comparator<Point> leftRightComparator = Comparator.comparingDouble(point -> point.x);
037    Comparator<Point> verticalComparator = Comparator.comparingDouble(point -> point.y);
038    MatOfPoint2f polyOutput = new MatOfPoint2f();
039
040    @Override
041    protected List<TrackedTarget> process(List<TrackedTarget> targetList) {
042        for (var target : targetList) {
043            // detect corners. Might implement more algorithms later but
044            // APPROX_POLY_DP_AND_EXTREME_CORNERS should be year agnostic
045            if (Objects.requireNonNull(params.cornerDetectionStrategy)
046                    == DetectionStrategy.APPROX_POLY_DP_AND_EXTREME_CORNERS) {
047                var targetCorners = detectExtremeCornersByApproxPolyDp(target, params.calculateConvexHulls);
048                target.setTargetCorners(targetCorners);
049            }
050        }
051        return targetList;
052    }
053
054    /**
055     * @param target The target to find the corners of.
056     * @return Corners: (bottom-left, bottom-right, top-right, top-left)
057     */
058    private List<Point> findBoundingBoxCorners(TrackedTarget target) {
059        // extract the corners
060        var points = new Point[4];
061        target.m_mainContour.getMinAreaRect().points(points);
062
063        // find the bl/br/tr/tl corners
064        // first, min by left/right
065        var list_ = Arrays.asList(points);
066        list_.sort(leftRightComparator);
067        // of this, we now have left and right
068        // sort to get top and bottom
069        var left = new ArrayList<>(List.of(list_.get(0), list_.get(1)));
070        left.sort(verticalComparator);
071        var right = new ArrayList<>(List.of(list_.get(2), list_.get(3)));
072        right.sort(verticalComparator);
073
074        var tl = left.get(0);
075        var bl = left.get(1);
076        var tr = right.get(0);
077        var br = right.get(1);
078
079        return List.of(bl, br, tr, tl);
080    }
081
082    /**
083     * @param a First point.
084     * @param b Second point.
085     * @return The straight line distance between them.
086     */
087    private static double distanceBetween(Point a, Point b) {
088        double xDelta = a.x - b.x;
089        double yDelta = a.y - b.y;
090        return Math.sqrt(xDelta * xDelta + yDelta * yDelta);
091    }
092
093    /**
094     * Find the 4 most extreme corners of the target's contour.
095     *
096     * @param target The target to track.
097     * @param convexHull Whether to use the convex hull of the contour instead.
098     * @return The 4 extreme corners of the contour: (bottom-left, bottom-right, top-right, top-left)
099     */
100    private List<Point> detectExtremeCornersByApproxPolyDp(TrackedTarget target, boolean convexHull) {
101        var centroid = target.getMinAreaRect().center;
102        Comparator<Point> compareCenterDist =
103                Comparator.comparingDouble((Point point) -> distanceBetween(centroid, point));
104
105        MatOfPoint2f targetContour;
106        if (convexHull) {
107            targetContour = target.m_mainContour.getConvexHull();
108        } else {
109            targetContour = target.m_mainContour.getMat2f();
110        }
111
112        /*
113        approximating a shape around the contours
114        Can be tuned to allow/disallow hulls
115        we want a number between 0 and 0.16 out of a percentage from 0 to 100
116        so take accuracy and divide by 600
117
118        Furthermore, we know that the contour is open if we haven't done convex hulls,
119        and it has subcontours.
120        */
121        var isOpen = !convexHull && target.hasSubContours();
122        var peri = Imgproc.arcLength(targetContour, true);
123        Imgproc.approxPolyDP(
124                targetContour, polyOutput, params.accuracyPercentage / 600.0 * peri, !isOpen);
125
126        // we must have at least 4 corners for this strategy to work.
127        // If we are looking for an exact side count that is handled here too.
128        var pointList = new ArrayList<>(polyOutput.toList());
129        if (pointList.size() < 4 || (params.exactSideCount && params.sideCount != pointList.size()))
130            return null;
131
132        target.setApproximateBoundingPolygon(polyOutput);
133
134        // left top, left bottom, right bottom, right top
135        var boundingBoxCorners = findBoundingBoxCorners(target);
136
137        var compareDistToTl =
138                Comparator.comparingDouble((Point p) -> distanceBetween(p, boundingBoxCorners.get(3)));
139
140        var compareDistToTr =
141                Comparator.comparingDouble((Point p) -> distanceBetween(p, boundingBoxCorners.get(2)));
142
143        // top left and top right are the poly corners closest to the bounding box tl and tr
144        pointList.sort(compareDistToTl);
145        var tl = pointList.get(0);
146        pointList.remove(tl);
147        pointList.sort(compareDistToTr);
148        var tr = pointList.get(0);
149        pointList.remove(tr);
150
151        // at this point we look for points on the left/right of the center of the remaining points
152        // and maximize their distance from the center of the min area rectangle
153        var leftList = new ArrayList<Point>();
154        var rightList = new ArrayList<Point>();
155        double averageXCoordinate = 0.0;
156        for (var p : pointList) {
157            averageXCoordinate += p.x;
158        }
159        averageXCoordinate /= pointList.size();
160
161        // add points that are below the center of the min area rectangle of the target
162        for (var p : pointList) {
163            if (p.y
164                    > target.m_mainContour.getBoundingRect().y
165                            + target.m_mainContour.getBoundingRect().height / 2.0)
166                if (p.x < averageXCoordinate) {
167                    leftList.add(p);
168                } else {
169                    rightList.add(p);
170                }
171        }
172        if (leftList.isEmpty() || rightList.isEmpty()) return null;
173        leftList.sort(compareCenterDist);
174        rightList.sort(compareCenterDist);
175        var bl = leftList.get(leftList.size() - 1);
176        var br = rightList.get(rightList.size() - 1);
177        return List.of(bl, br, tr, tl);
178    }
179
180    public static class CornerDetectionPipeParameters {
181        private final DetectionStrategy cornerDetectionStrategy;
182
183        private final boolean calculateConvexHulls;
184        private final boolean exactSideCount;
185        private final int sideCount;
186
187        /** This number can be changed to change how "accurate" our approximate polygon must be. */
188        private final double accuracyPercentage;
189
190        public CornerDetectionPipeParameters(
191                DetectionStrategy cornerDetectionStrategy,
192                boolean calculateConvexHulls,
193                boolean exactSideCount,
194                int sideCount,
195                double accuracyPercentage) {
196            this.cornerDetectionStrategy = cornerDetectionStrategy;
197            this.calculateConvexHulls = calculateConvexHulls;
198            this.exactSideCount = exactSideCount;
199            this.sideCount = sideCount;
200            this.accuracyPercentage = accuracyPercentage;
201        }
202    }
203
204    public enum DetectionStrategy {
205        APPROX_POLY_DP_AND_EXTREME_CORNERS
206    }
207}