001/* 002 * Copyright (C) Photon Vision. 003 * 004 * This program is free software: you can redistribute it and/or modify 005 * it under the terms of the GNU General Public License as published by 006 * the Free Software Foundation, either version 3 of the License, or 007 * (at your option) any later version. 008 * 009 * This program is distributed in the hope that it will be useful, 010 * but WITHOUT ANY WARRANTY; without even the implied warranty of 011 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 012 * GNU General Public License for more details. 013 * 014 * You should have received a copy of the GNU General Public License 015 * along with this program. If not, see <https://www.gnu.org/licenses/>. 016 */ 017 018package org.photonvision.vision.target; 019 020import edu.wpi.first.apriltag.AprilTagDetection; 021import edu.wpi.first.apriltag.AprilTagPoseEstimate; 022import edu.wpi.first.math.geometry.Transform3d; 023import java.util.ArrayList; 024import java.util.HashMap; 025import java.util.List; 026import org.opencv.core.CvType; 027import org.opencv.core.Mat; 028import org.opencv.core.MatOfPoint; 029import org.opencv.core.MatOfPoint2f; 030import org.opencv.core.Point; 031import org.opencv.core.RotatedRect; 032import org.photonvision.common.util.SerializationUtils; 033import org.photonvision.common.util.math.MathUtils; 034import org.photonvision.targeting.PhotonTrackedTarget; 035import org.photonvision.targeting.TargetCorner; 036import org.photonvision.vision.aruco.ArucoDetectionResult; 037import org.photonvision.vision.calibration.CameraCalibrationCoefficients; 038import org.photonvision.vision.frame.FrameStaticProperties; 039import org.photonvision.vision.opencv.CVShape; 040import org.photonvision.vision.opencv.Contour; 041import org.photonvision.vision.opencv.DualOffsetValues; 042import org.photonvision.vision.opencv.Releasable; 043 044public class TrackedTarget implements Releasable { 045 public final Contour m_mainContour; 046 public List<Contour> m_subContours; // can be empty 047 048 private MatOfPoint2f m_approximateBoundingPolygon; 049 050 private List<Point> m_targetCorners = List.of(); 051 052 private Point m_targetOffsetPoint; 053 private Point m_robotOffsetPoint; 054 055 private double m_pitch; 056 private double m_yaw; 057 private double m_area; 058 private double m_skew; 059 060 private Transform3d m_bestCameraToTarget3d = new Transform3d(); 061 private Transform3d m_altCameraToTarget3d = new Transform3d(); 062 063 private CVShape m_shape; 064 065 private int m_fiducialId = -1; 066 private double m_poseAmbiguity = -1; 067 068 private Mat m_cameraRelativeTvec, m_cameraRelativeRvec; 069 070 private int m_classId = -1; 071 private double m_confidence = -1; 072 073 public TrackedTarget( 074 PotentialTarget origTarget, TargetCalculationParameters params, CVShape shape) { 075 this.m_mainContour = origTarget.m_mainContour; 076 this.m_subContours = origTarget.m_subContours; 077 this.m_shape = shape; 078 calculateValues(params); 079 080 this.m_classId = origTarget.clsId; 081 this.m_confidence = origTarget.confidence; 082 } 083 084 public TrackedTarget( 085 AprilTagDetection tagDetection, 086 AprilTagPoseEstimate tagPose, 087 TargetCalculationParameters params) { 088 m_targetOffsetPoint = new Point(tagDetection.getCenterX(), tagDetection.getCenterY()); 089 m_robotOffsetPoint = new Point(); 090 var yawPitch = 091 TargetCalculations.calculateYawPitch( 092 params.cameraCenterPoint.x, 093 tagDetection.getCenterX(), 094 params.horizontalFocalLength, 095 params.cameraCenterPoint.y, 096 tagDetection.getCenterY(), 097 params.verticalFocalLength, 098 params.cameraCal); 099 m_yaw = yawPitch.getFirst(); 100 m_pitch = yawPitch.getSecond(); 101 var bestPose = new Transform3d(); 102 var altPose = new Transform3d(); 103 104 if (tagPose != null) { 105 if (tagPose.error1 <= tagPose.error2) { 106 bestPose = tagPose.pose1; 107 altPose = tagPose.pose2; 108 } else { 109 bestPose = tagPose.pose2; 110 altPose = tagPose.pose1; 111 } 112 113 bestPose = MathUtils.convertApriltagtoOpenCV(bestPose); 114 altPose = MathUtils.convertApriltagtoOpenCV(altPose); 115 116 m_bestCameraToTarget3d = bestPose; 117 m_altCameraToTarget3d = altPose; 118 119 m_poseAmbiguity = tagPose.getAmbiguity(); 120 121 var tvec = new Mat(3, 1, CvType.CV_64FC1); 122 tvec.put( 123 0, 124 0, 125 new double[] { 126 bestPose.getTranslation().getX(), 127 bestPose.getTranslation().getY(), 128 bestPose.getTranslation().getZ() 129 }); 130 setCameraRelativeTvec(tvec); 131 132 // Opencv expects a 3d vector with norm = angle and direction = axis 133 var rvec = new Mat(3, 1, CvType.CV_64FC1); 134 MathUtils.rotationToOpencvRvec(bestPose.getRotation(), rvec); 135 setCameraRelativeRvec(rvec); 136 } 137 138 double[] corners = tagDetection.getCorners(); 139 Point[] cornerPoints = 140 new Point[] { 141 new Point(corners[0], corners[1]), 142 new Point(corners[2], corners[3]), 143 new Point(corners[4], corners[5]), 144 new Point(corners[6], corners[7]) 145 }; 146 m_targetCorners = List.of(cornerPoints); 147 MatOfPoint contourMat = new MatOfPoint(cornerPoints); 148 m_approximateBoundingPolygon = new MatOfPoint2f(cornerPoints); 149 m_mainContour = new Contour(contourMat); 150 m_area = m_mainContour.getArea() / params.imageArea * 100; 151 m_fiducialId = tagDetection.getId(); 152 m_shape = null; 153 154 // TODO implement skew? or just yeet 155 m_skew = 0; 156 } 157 158 public TrackedTarget(List<Point> corners) { 159 m_mainContour = new Contour(new MatOfPoint()); 160 m_mainContour.mat.fromList(List.of(new Point(0, 0), new Point(0, 1), new Point(1, 0))); 161 this.setTargetCorners(corners); 162 m_targetOffsetPoint = new Point(); 163 m_robotOffsetPoint = new Point(); 164 } 165 166 /** 167 * @return Returns the confidence of the detection ranging from 0 - 1. 168 */ 169 public double getConfidence() { 170 return m_confidence; 171 } 172 173 /** 174 * @return O-indexed class index for the detected object. 175 */ 176 public int getClassID() { 177 return m_classId; 178 } 179 180 public TrackedTarget( 181 ArucoDetectionResult result, 182 AprilTagPoseEstimate tagPose, 183 TargetCalculationParameters params) { 184 m_targetOffsetPoint = new Point(result.getCenterX(), result.getCenterY()); 185 m_robotOffsetPoint = new Point(); 186 var yawPitch = 187 TargetCalculations.calculateYawPitch( 188 params.cameraCenterPoint.x, 189 result.getCenterX(), 190 params.horizontalFocalLength, 191 params.cameraCenterPoint.y, 192 result.getCenterY(), 193 params.verticalFocalLength, 194 params.cameraCal); 195 m_yaw = yawPitch.getFirst(); 196 m_pitch = yawPitch.getSecond(); 197 198 double[] xCorners = result.getXCorners(); 199 double[] yCorners = result.getYCorners(); 200 201 Point[] cornerPoints = 202 new Point[] { 203 new Point(xCorners[0], yCorners[0]), 204 new Point(xCorners[1], yCorners[1]), 205 new Point(xCorners[2], yCorners[2]), 206 new Point(xCorners[3], yCorners[3]) 207 }; 208 m_targetCorners = List.of(cornerPoints); 209 MatOfPoint contourMat = new MatOfPoint(cornerPoints); 210 m_approximateBoundingPolygon = new MatOfPoint2f(cornerPoints); 211 m_mainContour = new Contour(contourMat); 212 m_area = m_mainContour.getArea() / params.imageArea * 100; 213 m_fiducialId = result.getId(); 214 m_shape = null; 215 216 // TODO implement skew? or just yeet 217 m_skew = 0; 218 219 var bestPose = new Transform3d(); 220 var altPose = new Transform3d(); 221 if (tagPose != null) { 222 if (tagPose.error1 <= tagPose.error2) { 223 bestPose = tagPose.pose1; 224 altPose = tagPose.pose2; 225 } else { 226 bestPose = tagPose.pose2; 227 altPose = tagPose.pose1; 228 } 229 230 m_bestCameraToTarget3d = bestPose; 231 m_altCameraToTarget3d = altPose; 232 233 m_poseAmbiguity = tagPose.getAmbiguity(); 234 235 var tvec = new Mat(3, 1, CvType.CV_64FC1); 236 tvec.put( 237 0, 238 0, 239 new double[] { 240 bestPose.getTranslation().getX(), 241 bestPose.getTranslation().getY(), 242 bestPose.getTranslation().getZ() 243 }); 244 setCameraRelativeTvec(tvec); 245 246 var rvec = new Mat(3, 1, CvType.CV_64FC1); 247 MathUtils.rotationToOpencvRvec(bestPose.getRotation(), rvec); 248 setCameraRelativeRvec(rvec); 249 } 250 } 251 252 public void setFiducialId(int id) { 253 m_fiducialId = id; 254 } 255 256 public int getFiducialId() { 257 return m_fiducialId; 258 } 259 260 public void setPoseAmbiguity(double ambiguity) { 261 m_poseAmbiguity = ambiguity; 262 } 263 264 public double getPoseAmbiguity() { 265 return m_poseAmbiguity; 266 } 267 268 /** 269 * Set the approximate bounding polygon. 270 * 271 * @param boundingPolygon List of points to copy. Not modified. 272 */ 273 public void setApproximateBoundingPolygon(MatOfPoint2f boundingPolygon) { 274 if (m_approximateBoundingPolygon == null) m_approximateBoundingPolygon = new MatOfPoint2f(); 275 boundingPolygon.copyTo(m_approximateBoundingPolygon); 276 } 277 278 public Point getTargetOffsetPoint() { 279 return m_targetOffsetPoint; 280 } 281 282 public Point getRobotOffsetPoint() { 283 return m_robotOffsetPoint; 284 } 285 286 public double getPitch() { 287 return m_pitch; 288 } 289 290 public double getYaw() { 291 return m_yaw; 292 } 293 294 public double getSkew() { 295 return m_skew; 296 } 297 298 public double getArea() { 299 return m_area; 300 } 301 302 public RotatedRect getMinAreaRect() { 303 return m_mainContour.getMinAreaRect(); 304 } 305 306 public MatOfPoint2f getApproximateBoundingPolygon() { 307 return m_approximateBoundingPolygon; 308 } 309 310 public void calculateValues(TargetCalculationParameters params) { 311 // this MUST happen in this exact order! (TODO: document why) 312 m_targetOffsetPoint = 313 TargetCalculations.calculateTargetOffsetPoint( 314 params.isLandscape, params.targetOffsetPointEdge, getMinAreaRect()); 315 m_robotOffsetPoint = 316 TargetCalculations.calculateRobotOffsetPoint( 317 params.robotOffsetSinglePoint, 318 params.cameraCenterPoint, 319 params.dualOffsetValues, 320 params.robotOffsetPointMode); 321 322 // order of this stuff doesnt matter though 323 var yawPitch = 324 TargetCalculations.calculateYawPitch( 325 m_robotOffsetPoint.x, 326 m_targetOffsetPoint.x, 327 params.horizontalFocalLength, 328 m_robotOffsetPoint.y, 329 m_targetOffsetPoint.y, 330 params.verticalFocalLength, 331 params.cameraCal); 332 m_yaw = yawPitch.getFirst(); 333 m_pitch = yawPitch.getSecond(); 334 335 m_area = m_mainContour.getMinAreaRect().size.area() / params.imageArea * 100; 336 337 m_skew = TargetCalculations.calculateSkew(params.isLandscape, getMinAreaRect()); 338 } 339 340 @Override 341 public void release() { 342 m_mainContour.release(); 343 344 // TODO how can this check fail? 345 if (m_subContours != null) { 346 for (var sc : m_subContours) { 347 sc.release(); 348 } 349 } 350 351 if (m_cameraRelativeTvec != null) m_cameraRelativeTvec.release(); 352 if (m_cameraRelativeRvec != null) m_cameraRelativeRvec.release(); 353 } 354 355 public void setTargetCorners(List<Point> targetCorners) { 356 this.m_targetCorners = targetCorners; 357 } 358 359 public List<Point> getTargetCorners() { 360 return m_targetCorners; 361 } 362 363 public boolean hasSubContours() { 364 return !m_subContours.isEmpty(); 365 } 366 367 public Transform3d getBestCameraToTarget3d() { 368 return m_bestCameraToTarget3d; 369 } 370 371 public Transform3d getAltCameraToTarget3d() { 372 return m_altCameraToTarget3d; 373 } 374 375 public void setBestCameraToTarget3d(Transform3d pose) { 376 this.m_bestCameraToTarget3d = pose; 377 } 378 379 public void setAltCameraToTarget3d(Transform3d pose) { 380 this.m_altCameraToTarget3d = pose; 381 } 382 383 public Mat getCameraRelativeTvec() { 384 return m_cameraRelativeTvec; 385 } 386 387 public void setCameraRelativeTvec(Mat cameraRelativeTvec) { 388 if (this.m_cameraRelativeTvec == null) m_cameraRelativeTvec = new Mat(); 389 cameraRelativeTvec.copyTo(this.m_cameraRelativeTvec); 390 } 391 392 public Mat getCameraRelativeRvec() { 393 return m_cameraRelativeRvec; 394 } 395 396 public void setCameraRelativeRvec(Mat cameraRelativeRvec) { 397 if (this.m_cameraRelativeRvec == null) m_cameraRelativeRvec = new Mat(); 398 cameraRelativeRvec.copyTo(this.m_cameraRelativeRvec); 399 } 400 401 public CVShape getShape() { 402 return m_shape; 403 } 404 405 public void setShape(CVShape shape) { 406 this.m_shape = shape; 407 } 408 409 public HashMap<String, Object> toHashMap() { 410 var ret = new HashMap<String, Object>(); 411 ret.put("pitch", getPitch()); 412 ret.put("yaw", getYaw()); 413 ret.put("skew", getSkew()); 414 ret.put("area", getArea()); 415 ret.put("ambiguity", getPoseAmbiguity()); 416 ret.put("confidence", m_confidence); 417 ret.put("classId", m_classId); 418 419 var bestCameraToTarget3d = getBestCameraToTarget3d(); 420 if (bestCameraToTarget3d != null) { 421 ret.put("pose", SerializationUtils.transformToHashMap(bestCameraToTarget3d)); 422 } 423 ret.put("fiducialId", getFiducialId()); 424 return ret; 425 } 426 427 public boolean isFiducial() { 428 return this.m_fiducialId >= 0; 429 } 430 431 public static List<PhotonTrackedTarget> simpleFromTrackedTargets(List<TrackedTarget> targets) { 432 var ret = new ArrayList<PhotonTrackedTarget>(); 433 for (var t : targets) { 434 var minAreaRectCorners = new ArrayList<TargetCorner>(); 435 var detectedCorners = new ArrayList<TargetCorner>(); 436 { 437 var points = new Point[4]; 438 t.getMinAreaRect().points(points); 439 for (int i = 0; i < 4; i++) { 440 minAreaRectCorners.add(new TargetCorner(points[i].x, points[i].y)); 441 } 442 } 443 { 444 var points = t.getTargetCorners(); 445 if (points != null) { 446 for (Point point : points) { 447 detectedCorners.add(new TargetCorner(point.x, point.y)); 448 } 449 } 450 } 451 452 ret.add( 453 new PhotonTrackedTarget( 454 t.getYaw(), 455 t.getPitch(), 456 t.getArea(), 457 t.getSkew(), 458 t.getFiducialId(), 459 t.getClassID(), 460 (float) t.getConfidence(), 461 t.getBestCameraToTarget3d(), 462 t.getAltCameraToTarget3d(), 463 t.getPoseAmbiguity(), 464 minAreaRectCorners, 465 detectedCorners)); 466 } 467 return ret; 468 } 469 470 public static class TargetCalculationParameters { 471 // TargetOffset calculation values 472 final boolean isLandscape; 473 final TargetOffsetPointEdge targetOffsetPointEdge; 474 475 // RobotOffset calculation values 476 final RobotOffsetPointMode robotOffsetPointMode; 477 final Point robotOffsetSinglePoint; 478 final DualOffsetValues dualOffsetValues; 479 480 // center point of image 481 final Point cameraCenterPoint; 482 483 // yaw calculation values 484 final double horizontalFocalLength; 485 486 // pitch calculation values 487 final double verticalFocalLength; 488 489 // area calculation values 490 final double imageArea; 491 492 // Camera calibration, null if not calibrated 493 final CameraCalibrationCoefficients cameraCal; 494 495 public TargetCalculationParameters( 496 boolean isLandscape, 497 TargetOffsetPointEdge targetOffsetPointEdge, 498 RobotOffsetPointMode robotOffsetPointMode, 499 Point robotOffsetSinglePoint, 500 DualOffsetValues dualOffsetValues, 501 Point cameraCenterPoint, 502 double horizontalFocalLength, 503 double verticalFocalLength, 504 double imageArea, 505 CameraCalibrationCoefficients cal) { 506 this.isLandscape = isLandscape; 507 this.targetOffsetPointEdge = targetOffsetPointEdge; 508 this.robotOffsetPointMode = robotOffsetPointMode; 509 this.robotOffsetSinglePoint = robotOffsetSinglePoint; 510 this.dualOffsetValues = dualOffsetValues; 511 this.cameraCenterPoint = cameraCenterPoint; 512 this.horizontalFocalLength = horizontalFocalLength; 513 this.verticalFocalLength = verticalFocalLength; 514 this.imageArea = imageArea; 515 this.cameraCal = cal; 516 } 517 518 public TargetCalculationParameters( 519 boolean isLandscape, 520 TargetOffsetPointEdge targetOffsetPointEdge, 521 RobotOffsetPointMode robotOffsetPointMode, 522 Point robotOffsetSinglePoint, 523 DualOffsetValues dualOffsetValues, 524 FrameStaticProperties frameStaticProperties) { 525 this.isLandscape = isLandscape; 526 this.targetOffsetPointEdge = targetOffsetPointEdge; 527 this.robotOffsetPointMode = robotOffsetPointMode; 528 this.robotOffsetSinglePoint = robotOffsetSinglePoint; 529 this.dualOffsetValues = dualOffsetValues; 530 531 this.cameraCenterPoint = frameStaticProperties.centerPoint; 532 this.horizontalFocalLength = frameStaticProperties.horizontalFocalLength; 533 this.verticalFocalLength = frameStaticProperties.verticalFocalLength; 534 this.imageArea = frameStaticProperties.imageArea; 535 this.cameraCal = frameStaticProperties.cameraCalibration; 536 } 537 } 538}