001/*
002 * Copyright (C) Photon Vision.
003 *
004 * This program is free software: you can redistribute it and/or modify
005 * it under the terms of the GNU General Public License as published by
006 * the Free Software Foundation, either version 3 of the License, or
007 * (at your option) any later version.
008 *
009 * This program is distributed in the hope that it will be useful,
010 * but WITHOUT ANY WARRANTY; without even the implied warranty of
011 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
012 * GNU General Public License for more details.
013 *
014 * You should have received a copy of the GNU General Public License
015 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
016 */
017
018package org.photonvision.vision.target;
019
020import edu.wpi.first.apriltag.AprilTagDetection;
021import edu.wpi.first.apriltag.AprilTagPoseEstimate;
022import edu.wpi.first.math.geometry.Transform3d;
023import java.util.ArrayList;
024import java.util.HashMap;
025import java.util.List;
026import org.opencv.core.CvType;
027import org.opencv.core.Mat;
028import org.opencv.core.MatOfPoint;
029import org.opencv.core.MatOfPoint2f;
030import org.opencv.core.Point;
031import org.opencv.core.RotatedRect;
032import org.photonvision.common.util.SerializationUtils;
033import org.photonvision.common.util.math.MathUtils;
034import org.photonvision.targeting.PhotonTrackedTarget;
035import org.photonvision.targeting.TargetCorner;
036import org.photonvision.vision.aruco.ArucoDetectionResult;
037import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
038import org.photonvision.vision.frame.FrameStaticProperties;
039import org.photonvision.vision.opencv.CVShape;
040import org.photonvision.vision.opencv.Contour;
041import org.photonvision.vision.opencv.DualOffsetValues;
042import org.photonvision.vision.opencv.Releasable;
043
044public class TrackedTarget implements Releasable {
045    public final Contour m_mainContour;
046    public List<Contour> m_subContours; // can be empty
047
048    private MatOfPoint2f m_approximateBoundingPolygon;
049
050    private List<Point> m_targetCorners = List.of();
051
052    private Point m_targetOffsetPoint;
053    private Point m_robotOffsetPoint;
054
055    private double m_pitch;
056    private double m_yaw;
057    private double m_area;
058    private double m_skew;
059
060    private Transform3d m_bestCameraToTarget3d = new Transform3d();
061    private Transform3d m_altCameraToTarget3d = new Transform3d();
062
063    private CVShape m_shape;
064
065    private int m_fiducialId = -1;
066    private double m_poseAmbiguity = -1;
067
068    private Mat m_cameraRelativeTvec, m_cameraRelativeRvec;
069
070    private int m_classId = -1;
071    private double m_confidence = -1;
072
073    public TrackedTarget(
074            PotentialTarget origTarget, TargetCalculationParameters params, CVShape shape) {
075        this.m_mainContour = origTarget.m_mainContour;
076        this.m_subContours = origTarget.m_subContours;
077        this.m_shape = shape;
078        calculateValues(params);
079
080        this.m_classId = origTarget.clsId;
081        this.m_confidence = origTarget.confidence;
082    }
083
084    public TrackedTarget(
085            AprilTagDetection tagDetection,
086            AprilTagPoseEstimate tagPose,
087            TargetCalculationParameters params) {
088        m_targetOffsetPoint = new Point(tagDetection.getCenterX(), tagDetection.getCenterY());
089        m_robotOffsetPoint = new Point();
090        var yawPitch =
091                TargetCalculations.calculateYawPitch(
092                        params.cameraCenterPoint.x,
093                        tagDetection.getCenterX(),
094                        params.horizontalFocalLength,
095                        params.cameraCenterPoint.y,
096                        tagDetection.getCenterY(),
097                        params.verticalFocalLength,
098                        params.cameraCal);
099        m_yaw = yawPitch.getFirst();
100        m_pitch = yawPitch.getSecond();
101        var bestPose = new Transform3d();
102        var altPose = new Transform3d();
103
104        if (tagPose != null) {
105            if (tagPose.error1 <= tagPose.error2) {
106                bestPose = tagPose.pose1;
107                altPose = tagPose.pose2;
108            } else {
109                bestPose = tagPose.pose2;
110                altPose = tagPose.pose1;
111            }
112
113            bestPose = MathUtils.convertApriltagtoOpenCV(bestPose);
114            altPose = MathUtils.convertApriltagtoOpenCV(altPose);
115
116            m_bestCameraToTarget3d = bestPose;
117            m_altCameraToTarget3d = altPose;
118
119            m_poseAmbiguity = tagPose.getAmbiguity();
120
121            var tvec = new Mat(3, 1, CvType.CV_64FC1);
122            tvec.put(
123                    0,
124                    0,
125                    new double[] {
126                        bestPose.getTranslation().getX(),
127                        bestPose.getTranslation().getY(),
128                        bestPose.getTranslation().getZ()
129                    });
130            setCameraRelativeTvec(tvec);
131
132            // Opencv expects a 3d vector with norm = angle and direction = axis
133            var rvec = new Mat(3, 1, CvType.CV_64FC1);
134            MathUtils.rotationToOpencvRvec(bestPose.getRotation(), rvec);
135            setCameraRelativeRvec(rvec);
136        }
137
138        double[] corners = tagDetection.getCorners();
139        Point[] cornerPoints =
140                new Point[] {
141                    new Point(corners[0], corners[1]),
142                    new Point(corners[2], corners[3]),
143                    new Point(corners[4], corners[5]),
144                    new Point(corners[6], corners[7])
145                };
146        m_targetCorners = List.of(cornerPoints);
147        MatOfPoint contourMat = new MatOfPoint(cornerPoints);
148        m_approximateBoundingPolygon = new MatOfPoint2f(cornerPoints);
149        m_mainContour = new Contour(contourMat);
150        m_area = m_mainContour.getArea() / params.imageArea * 100;
151        m_fiducialId = tagDetection.getId();
152        m_shape = null;
153
154        // TODO implement skew? or just yeet
155        m_skew = 0;
156    }
157
158    public TrackedTarget(List<Point> corners) {
159        m_mainContour = new Contour(new MatOfPoint());
160        m_mainContour.mat.fromList(List.of(new Point(0, 0), new Point(0, 1), new Point(1, 0)));
161        this.setTargetCorners(corners);
162        m_targetOffsetPoint = new Point();
163        m_robotOffsetPoint = new Point();
164    }
165
166    /**
167     * @return Returns the confidence of the detection ranging from 0 - 1.
168     */
169    public double getConfidence() {
170        return m_confidence;
171    }
172
173    /**
174     * @return O-indexed class index for the detected object.
175     */
176    public int getClassID() {
177        return m_classId;
178    }
179
180    public TrackedTarget(
181            ArucoDetectionResult result,
182            AprilTagPoseEstimate tagPose,
183            TargetCalculationParameters params) {
184        m_targetOffsetPoint = new Point(result.getCenterX(), result.getCenterY());
185        m_robotOffsetPoint = new Point();
186        var yawPitch =
187                TargetCalculations.calculateYawPitch(
188                        params.cameraCenterPoint.x,
189                        result.getCenterX(),
190                        params.horizontalFocalLength,
191                        params.cameraCenterPoint.y,
192                        result.getCenterY(),
193                        params.verticalFocalLength,
194                        params.cameraCal);
195        m_yaw = yawPitch.getFirst();
196        m_pitch = yawPitch.getSecond();
197
198        double[] xCorners = result.getXCorners();
199        double[] yCorners = result.getYCorners();
200
201        Point[] cornerPoints =
202                new Point[] {
203                    new Point(xCorners[0], yCorners[0]),
204                    new Point(xCorners[1], yCorners[1]),
205                    new Point(xCorners[2], yCorners[2]),
206                    new Point(xCorners[3], yCorners[3])
207                };
208        m_targetCorners = List.of(cornerPoints);
209        MatOfPoint contourMat = new MatOfPoint(cornerPoints);
210        m_approximateBoundingPolygon = new MatOfPoint2f(cornerPoints);
211        m_mainContour = new Contour(contourMat);
212        m_area = m_mainContour.getArea() / params.imageArea * 100;
213        m_fiducialId = result.getId();
214        m_shape = null;
215
216        // TODO implement skew? or just yeet
217        m_skew = 0;
218
219        var bestPose = new Transform3d();
220        var altPose = new Transform3d();
221        if (tagPose != null) {
222            if (tagPose.error1 <= tagPose.error2) {
223                bestPose = tagPose.pose1;
224                altPose = tagPose.pose2;
225            } else {
226                bestPose = tagPose.pose2;
227                altPose = tagPose.pose1;
228            }
229
230            m_bestCameraToTarget3d = bestPose;
231            m_altCameraToTarget3d = altPose;
232
233            m_poseAmbiguity = tagPose.getAmbiguity();
234
235            var tvec = new Mat(3, 1, CvType.CV_64FC1);
236            tvec.put(
237                    0,
238                    0,
239                    new double[] {
240                        bestPose.getTranslation().getX(),
241                        bestPose.getTranslation().getY(),
242                        bestPose.getTranslation().getZ()
243                    });
244            setCameraRelativeTvec(tvec);
245
246            var rvec = new Mat(3, 1, CvType.CV_64FC1);
247            MathUtils.rotationToOpencvRvec(bestPose.getRotation(), rvec);
248            setCameraRelativeRvec(rvec);
249        }
250    }
251
252    public void setFiducialId(int id) {
253        m_fiducialId = id;
254    }
255
256    public int getFiducialId() {
257        return m_fiducialId;
258    }
259
260    public void setPoseAmbiguity(double ambiguity) {
261        m_poseAmbiguity = ambiguity;
262    }
263
264    public double getPoseAmbiguity() {
265        return m_poseAmbiguity;
266    }
267
268    /**
269     * Set the approximate bounding polygon.
270     *
271     * @param boundingPolygon List of points to copy. Not modified.
272     */
273    public void setApproximateBoundingPolygon(MatOfPoint2f boundingPolygon) {
274        if (m_approximateBoundingPolygon == null) m_approximateBoundingPolygon = new MatOfPoint2f();
275        boundingPolygon.copyTo(m_approximateBoundingPolygon);
276    }
277
278    public Point getTargetOffsetPoint() {
279        return m_targetOffsetPoint;
280    }
281
282    public Point getRobotOffsetPoint() {
283        return m_robotOffsetPoint;
284    }
285
286    public double getPitch() {
287        return m_pitch;
288    }
289
290    public double getYaw() {
291        return m_yaw;
292    }
293
294    public double getSkew() {
295        return m_skew;
296    }
297
298    public double getArea() {
299        return m_area;
300    }
301
302    public RotatedRect getMinAreaRect() {
303        return m_mainContour.getMinAreaRect();
304    }
305
306    public MatOfPoint2f getApproximateBoundingPolygon() {
307        return m_approximateBoundingPolygon;
308    }
309
310    public void calculateValues(TargetCalculationParameters params) {
311        // this MUST happen in this exact order! (TODO: document why)
312        m_targetOffsetPoint =
313                TargetCalculations.calculateTargetOffsetPoint(
314                        params.isLandscape, params.targetOffsetPointEdge, getMinAreaRect());
315        m_robotOffsetPoint =
316                TargetCalculations.calculateRobotOffsetPoint(
317                        params.robotOffsetSinglePoint,
318                        params.cameraCenterPoint,
319                        params.dualOffsetValues,
320                        params.robotOffsetPointMode);
321
322        // order of this stuff doesnt matter though
323        var yawPitch =
324                TargetCalculations.calculateYawPitch(
325                        m_robotOffsetPoint.x,
326                        m_targetOffsetPoint.x,
327                        params.horizontalFocalLength,
328                        m_robotOffsetPoint.y,
329                        m_targetOffsetPoint.y,
330                        params.verticalFocalLength,
331                        params.cameraCal);
332        m_yaw = yawPitch.getFirst();
333        m_pitch = yawPitch.getSecond();
334
335        m_area = m_mainContour.getMinAreaRect().size.area() / params.imageArea * 100;
336
337        m_skew = TargetCalculations.calculateSkew(params.isLandscape, getMinAreaRect());
338    }
339
340    @Override
341    public void release() {
342        m_mainContour.release();
343
344        // TODO how can this check fail?
345        if (m_subContours != null) {
346            for (var sc : m_subContours) {
347                sc.release();
348            }
349        }
350
351        if (m_cameraRelativeTvec != null) m_cameraRelativeTvec.release();
352        if (m_cameraRelativeRvec != null) m_cameraRelativeRvec.release();
353    }
354
355    public void setTargetCorners(List<Point> targetCorners) {
356        this.m_targetCorners = targetCorners;
357    }
358
359    public List<Point> getTargetCorners() {
360        return m_targetCorners;
361    }
362
363    public boolean hasSubContours() {
364        return !m_subContours.isEmpty();
365    }
366
367    public Transform3d getBestCameraToTarget3d() {
368        return m_bestCameraToTarget3d;
369    }
370
371    public Transform3d getAltCameraToTarget3d() {
372        return m_altCameraToTarget3d;
373    }
374
375    public void setBestCameraToTarget3d(Transform3d pose) {
376        this.m_bestCameraToTarget3d = pose;
377    }
378
379    public void setAltCameraToTarget3d(Transform3d pose) {
380        this.m_altCameraToTarget3d = pose;
381    }
382
383    public Mat getCameraRelativeTvec() {
384        return m_cameraRelativeTvec;
385    }
386
387    public void setCameraRelativeTvec(Mat cameraRelativeTvec) {
388        if (this.m_cameraRelativeTvec == null) m_cameraRelativeTvec = new Mat();
389        cameraRelativeTvec.copyTo(this.m_cameraRelativeTvec);
390    }
391
392    public Mat getCameraRelativeRvec() {
393        return m_cameraRelativeRvec;
394    }
395
396    public void setCameraRelativeRvec(Mat cameraRelativeRvec) {
397        if (this.m_cameraRelativeRvec == null) m_cameraRelativeRvec = new Mat();
398        cameraRelativeRvec.copyTo(this.m_cameraRelativeRvec);
399    }
400
401    public CVShape getShape() {
402        return m_shape;
403    }
404
405    public void setShape(CVShape shape) {
406        this.m_shape = shape;
407    }
408
409    public HashMap<String, Object> toHashMap() {
410        var ret = new HashMap<String, Object>();
411        ret.put("pitch", getPitch());
412        ret.put("yaw", getYaw());
413        ret.put("skew", getSkew());
414        ret.put("area", getArea());
415        ret.put("ambiguity", getPoseAmbiguity());
416        ret.put("confidence", m_confidence);
417        ret.put("classId", m_classId);
418
419        var bestCameraToTarget3d = getBestCameraToTarget3d();
420        if (bestCameraToTarget3d != null) {
421            ret.put("pose", SerializationUtils.transformToHashMap(bestCameraToTarget3d));
422        }
423        ret.put("fiducialId", getFiducialId());
424        return ret;
425    }
426
427    public boolean isFiducial() {
428        return this.m_fiducialId >= 0;
429    }
430
431    public static List<PhotonTrackedTarget> simpleFromTrackedTargets(List<TrackedTarget> targets) {
432        var ret = new ArrayList<PhotonTrackedTarget>();
433        for (var t : targets) {
434            var minAreaRectCorners = new ArrayList<TargetCorner>();
435            var detectedCorners = new ArrayList<TargetCorner>();
436            {
437                var points = new Point[4];
438                t.getMinAreaRect().points(points);
439                for (int i = 0; i < 4; i++) {
440                    minAreaRectCorners.add(new TargetCorner(points[i].x, points[i].y));
441                }
442            }
443            {
444                var points = t.getTargetCorners();
445                if (points != null) {
446                    for (Point point : points) {
447                        detectedCorners.add(new TargetCorner(point.x, point.y));
448                    }
449                }
450            }
451
452            ret.add(
453                    new PhotonTrackedTarget(
454                            t.getYaw(),
455                            t.getPitch(),
456                            t.getArea(),
457                            t.getSkew(),
458                            t.getFiducialId(),
459                            t.getClassID(),
460                            (float) t.getConfidence(),
461                            t.getBestCameraToTarget3d(),
462                            t.getAltCameraToTarget3d(),
463                            t.getPoseAmbiguity(),
464                            minAreaRectCorners,
465                            detectedCorners));
466        }
467        return ret;
468    }
469
470    public static class TargetCalculationParameters {
471        // TargetOffset calculation values
472        final boolean isLandscape;
473        final TargetOffsetPointEdge targetOffsetPointEdge;
474
475        // RobotOffset calculation values
476        final RobotOffsetPointMode robotOffsetPointMode;
477        final Point robotOffsetSinglePoint;
478        final DualOffsetValues dualOffsetValues;
479
480        // center point of image
481        final Point cameraCenterPoint;
482
483        // yaw calculation values
484        final double horizontalFocalLength;
485
486        // pitch calculation values
487        final double verticalFocalLength;
488
489        // area calculation values
490        final double imageArea;
491
492        // Camera calibration, null if not calibrated
493        final CameraCalibrationCoefficients cameraCal;
494
495        public TargetCalculationParameters(
496                boolean isLandscape,
497                TargetOffsetPointEdge targetOffsetPointEdge,
498                RobotOffsetPointMode robotOffsetPointMode,
499                Point robotOffsetSinglePoint,
500                DualOffsetValues dualOffsetValues,
501                Point cameraCenterPoint,
502                double horizontalFocalLength,
503                double verticalFocalLength,
504                double imageArea,
505                CameraCalibrationCoefficients cal) {
506            this.isLandscape = isLandscape;
507            this.targetOffsetPointEdge = targetOffsetPointEdge;
508            this.robotOffsetPointMode = robotOffsetPointMode;
509            this.robotOffsetSinglePoint = robotOffsetSinglePoint;
510            this.dualOffsetValues = dualOffsetValues;
511            this.cameraCenterPoint = cameraCenterPoint;
512            this.horizontalFocalLength = horizontalFocalLength;
513            this.verticalFocalLength = verticalFocalLength;
514            this.imageArea = imageArea;
515            this.cameraCal = cal;
516        }
517
518        public TargetCalculationParameters(
519                boolean isLandscape,
520                TargetOffsetPointEdge targetOffsetPointEdge,
521                RobotOffsetPointMode robotOffsetPointMode,
522                Point robotOffsetSinglePoint,
523                DualOffsetValues dualOffsetValues,
524                FrameStaticProperties frameStaticProperties) {
525            this.isLandscape = isLandscape;
526            this.targetOffsetPointEdge = targetOffsetPointEdge;
527            this.robotOffsetPointMode = robotOffsetPointMode;
528            this.robotOffsetSinglePoint = robotOffsetSinglePoint;
529            this.dualOffsetValues = dualOffsetValues;
530
531            this.cameraCenterPoint = frameStaticProperties.centerPoint;
532            this.horizontalFocalLength = frameStaticProperties.horizontalFocalLength;
533            this.verticalFocalLength = frameStaticProperties.verticalFocalLength;
534            this.imageArea = frameStaticProperties.imageArea;
535            this.cameraCal = frameStaticProperties.cameraCalibration;
536        }
537    }
538}