001/*
002 * Copyright (C) Photon Vision.
003 *
004 * This program is free software: you can redistribute it and/or modify
005 * it under the terms of the GNU General Public License as published by
006 * the Free Software Foundation, either version 3 of the License, or
007 * (at your option) any later version.
008 *
009 * This program is distributed in the hope that it will be useful,
010 * but WITHOUT ANY WARRANTY; without even the implied warranty of
011 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
012 * GNU General Public License for more details.
013 *
014 * You should have received a copy of the GNU General Public License
015 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
016 */
017
018package org.photonvision.vision.frame.provider;
019
020import edu.wpi.first.cameraserver.CameraServer;
021import edu.wpi.first.cscore.CvSink;
022import edu.wpi.first.cscore.UsbCamera;
023import edu.wpi.first.networktables.BooleanSubscriber;
024import edu.wpi.first.util.PixelFormat;
025import edu.wpi.first.util.RawFrame;
026import org.opencv.core.Mat;
027import org.photonvision.common.dataflow.networktables.NetworkTablesManager;
028import org.photonvision.common.logging.LogGroup;
029import org.photonvision.common.logging.Logger;
030import org.photonvision.jni.CscoreExtras;
031import org.photonvision.vision.opencv.CVMat;
032import org.photonvision.vision.processes.VisionSourceSettables;
033
034public class USBFrameProvider extends CpuImageProcessor {
035    private final Logger logger;
036
037    private UsbCamera camera = null;
038    private CvSink cvSink = null;
039
040    @SuppressWarnings("SpellCheckingInspection")
041    private VisionSourceSettables settables;
042
043    private Runnable connectedCallback;
044
045    private long lastTime = 0;
046
047    // subscribers are lightweight, and I'm lazy
048    private final BooleanSubscriber useNewBehaviorSub;
049
050    @SuppressWarnings("SpellCheckingInspection")
051    public USBFrameProvider(
052            UsbCamera camera, VisionSourceSettables visionSettables, Runnable connectedCallback) {
053        this.camera = camera;
054        this.cvSink = CameraServer.getVideo(this.camera);
055        this.logger =
056                new Logger(
057                        USBFrameProvider.class, visionSettables.getConfiguration().nickname, LogGroup.Camera);
058        this.cvSink.setEnabled(true);
059
060        this.settables = visionSettables;
061
062        var useNewBehaviorTopic =
063                NetworkTablesManager.getInstance().kRootTable.getBooleanTopic("use_new_cscore_frametime");
064
065        useNewBehaviorSub = useNewBehaviorTopic.subscribe(false);
066        this.connectedCallback = connectedCallback;
067    }
068
069    @Override
070    public boolean checkCameraConnected() {
071        boolean connected = camera.isConnected();
072
073        if (!cameraPropertiesCached && connected) {
074            logger.info("Camera connected! running callback");
075            onCameraConnected();
076        }
077
078        return connected;
079    }
080
081    final double CSCORE_DEFAULT_FRAME_TIMEOUT = 1.0 / 4.0;
082
083    @Override
084    public CapturedFrame getInputMat() {
085        if (!cameraPropertiesCached && camera.isConnected()) {
086            onCameraConnected();
087        }
088
089        if (!useNewBehaviorSub.get()) {
090            // We allocate memory so we don't fill a Mat in use by another thread (memory model is easier)
091            var mat = new CVMat();
092            // This is from wpi::Now, or WPIUtilJNI.now(). The epoch from grabFrame is uS since
093            // Hal::initialize was called
094            // TODO - under the hood, this incurs an extra copy. We should avoid this, if we
095            // can.
096            long captureTimeNs = cvSink.grabFrame(mat.getMat(), CSCORE_DEFAULT_FRAME_TIMEOUT) * 1000;
097
098            if (captureTimeNs == 0) {
099                var error = cvSink.getError();
100                logger.error("Error grabbing image: " + error);
101            }
102
103            return new CapturedFrame(mat, settables.getFrameStaticProperties(), captureTimeNs);
104        } else {
105            // We allocate memory so we don't fill a Mat in use by another thread (memory model is easier)
106            // TODO - consider a frame pool
107            // TODO - getCurrentVideoMode is a JNI call for us, but profiling indicates it's fast
108            var cameraMode = settables.getCurrentVideoMode();
109            var frame = new RawFrame();
110            frame.setInfo(
111                    cameraMode.width,
112                    cameraMode.height,
113                    // hard-coded 3 channel
114                    cameraMode.width * 3,
115                    PixelFormat.kBGR);
116
117            // This is from wpi::Now, or WPIUtilJNI.now(). The epoch from grabFrame is uS since
118            // Hal::initialize was called
119            long captureTimeUs =
120                    CscoreExtras.grabRawSinkFrameTimeoutLastTime(
121                            cvSink.getHandle(), frame.getNativeObj(), CSCORE_DEFAULT_FRAME_TIMEOUT, lastTime);
122            lastTime = captureTimeUs;
123
124            CVMat ret;
125
126            if (captureTimeUs == 0) {
127                var error = cvSink.getError();
128                logger.error("Error grabbing image: " + error);
129
130                frame.close();
131                ret = new CVMat();
132            } else {
133                // No error! yay
134                var mat = new Mat(CscoreExtras.wrapRawFrame(frame.getNativeObj()));
135
136                ret = new CVMat(mat, frame);
137            }
138
139            return new CapturedFrame(ret, settables.getFrameStaticProperties(), captureTimeUs * 1000);
140        }
141    }
142
143    @Override
144    public String getName() {
145        return "USBFrameProvider - " + cvSink.getName();
146    }
147
148    @Override
149    public void release() {
150        CameraServer.removeServer(cvSink.getName());
151        cvSink.close();
152        cvSink = null;
153    }
154
155    @Override
156    public void onCameraConnected() {
157        super.onCameraConnected();
158
159        this.connectedCallback.run();
160    }
161
162    @Override
163    public boolean isConnected() {
164        return camera.isConnected();
165    }
166
167    public void updateSettables(VisionSourceSettables settables) {
168        this.settables = settables;
169    }
170}