001/* 002 * Copyright (C) Photon Vision. 003 * 004 * This program is free software: you can redistribute it and/or modify 005 * it under the terms of the GNU General Public License as published by 006 * the Free Software Foundation, either version 3 of the License, or 007 * (at your option) any later version. 008 * 009 * This program is distributed in the hope that it will be useful, 010 * but WITHOUT ANY WARRANTY; without even the implied warranty of 011 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 012 * GNU General Public License for more details. 013 * 014 * You should have received a copy of the GNU General Public License 015 * along with this program. If not, see <https://www.gnu.org/licenses/>. 016 */ 017 018package org.photonvision.vision.processes; 019 020import edu.wpi.first.cscore.UsbCamera; 021import java.util.ArrayList; 022import java.util.Arrays; 023import java.util.Collection; 024import java.util.HashMap; 025import java.util.List; 026import java.util.Optional; 027import java.util.function.Predicate; 028import java.util.regex.Pattern; 029import java.util.stream.Stream; 030import org.photonvision.common.configuration.CameraConfiguration; 031import org.photonvision.common.configuration.ConfigManager; 032import org.photonvision.common.dataflow.DataChangeService; 033import org.photonvision.common.dataflow.events.OutgoingUIEvent; 034import org.photonvision.common.dataflow.websocket.UICameraConfiguration; 035import org.photonvision.common.dataflow.websocket.UIPhotonConfiguration; 036import org.photonvision.common.hardware.Platform; 037import org.photonvision.common.hardware.Platform.OSType; 038import org.photonvision.common.logging.LogGroup; 039import org.photonvision.common.logging.Logger; 040import org.photonvision.common.util.TimedTaskManager; 041import org.photonvision.raspi.LibCameraJNI; 042import org.photonvision.raspi.LibCameraJNILoader; 043import org.photonvision.vision.camera.CameraType; 044import org.photonvision.vision.camera.FileVisionSource; 045import org.photonvision.vision.camera.PVCameraInfo; 046import org.photonvision.vision.camera.USBCameras.USBCameraSource; 047import org.photonvision.vision.camera.csi.LibcameraGpuSource; 048 049/** 050 * This class manages starting up VisionModules for serialized devices ({@link 051 * VisionSourceManager#loadVisionSourceFromCamConfig}), as well as handling requests from users to 052 * disable (release the camera device, but keep the configuration around) ({@link 053 * VisionSourceManager#deactivateVisionSource}), reactivate (recreate a VisionModule from a saved 054 * and currently disabled configuration) ({@link 055 * VisionSourceManager#reactivateDisabledCameraConfig}), and create a new VisionModule from a {@link 056 * PVCameraInfo} ({@link VisionSourceManager#assignUnmatchedCamera}). 057 * 058 * <p>We now require user interaction for pretty much every operation this undertakes. 059 */ 060public class VisionSourceManager { 061 private static final Logger logger = new Logger(VisionSourceManager.class, LogGroup.Camera); 062 063 private static final List<String> deviceBlacklist = List.of("bcm2835-isp"); 064 065 private static class SingletonHolder { 066 private static final VisionSourceManager INSTANCE = new VisionSourceManager(); 067 } 068 069 public static VisionSourceManager getInstance() { 070 return SingletonHolder.INSTANCE; 071 } 072 073 // Jackson does use these members even if your IDE claims otherwise 074 public static class VisionSourceManagerState { 075 public List<UICameraConfiguration> disabledConfigs; 076 public List<PVCameraInfo> allConnectedCameras; 077 } 078 079 // Map of (unique name) -> (all CameraConfigurations) that have been registered 080 protected final HashMap<String, CameraConfiguration> disabledCameraConfigs = new HashMap<>(); 081 082 // The subset of cameras that are "active", converted to VisionModules 083 public VisionModuleManager vmm = new VisionModuleManager(); 084 085 public void registerTimedTasks() { 086 TimedTaskManager.getInstance().addTask("CameraDeviceExplorer", this::pushUiUpdate, 1000); 087 } 088 089 /** 090 * Register new camera configs loaded from disk. This will create vision modules for each camera 091 * config and start them. 092 * 093 * @param configs The loaded camera configs. 094 */ 095 public synchronized void registerLoadedConfigs(Collection<CameraConfiguration> configs) { 096 logger.info("Registering loaded camera configs"); 097 098 final HashMap<String, CameraConfiguration> deserializedConfigs = new HashMap<>(); 099 100 // 1. Verify all camera unique names are unique and paths/types are unique for paranoia. This 101 // seems redundant, consider deleting 102 for (var config : configs) { 103 Predicate<PVCameraInfo> checkDuplicateCamera = 104 (other) -> 105 (other.type().equals(config.matchedCameraInfo.type()) 106 && other.uniquePath().equals(config.matchedCameraInfo.uniquePath())); 107 108 if (deserializedConfigs.containsKey(config.uniqueName)) { 109 logger.error( 110 "Duplicate unique name for config " + config.uniqueName + " -- not overwriting"); 111 } else if (deserializedConfigs.values().stream() 112 .map(it -> it.matchedCameraInfo) 113 .anyMatch(checkDuplicateCamera)) { 114 logger.error( 115 "Duplicate camera type & path for config " + config.uniqueName + " -- not overwriting"); 116 } else { 117 deserializedConfigs.put(config.uniqueName, config); 118 } 119 } 120 121 // 2. create sources -> VMMs for all active cameras and add to our VMM. We don't care about if 122 // the underlying device is currently connected or not. 123 deserializedConfigs.values().stream() 124 .filter(it -> !it.deactivated) 125 .map(this::loadVisionSourceFromCamConfig) 126 .map(vmm::addSource) 127 .forEach(VisionModule::start); 128 129 // 3. write down all disabled sources for later 130 deserializedConfigs.entrySet().stream() 131 .filter(it -> it.getValue().deactivated) 132 .forEach(it -> this.disabledCameraConfigs.put(it.getKey(), it.getValue())); 133 134 logger.info( 135 "Finished registering loaded camera configs! Started " 136 + vmm.getModules().size() 137 + " active VisionModules, with " 138 + deserializedConfigs.size() 139 + " disabled VisionModules"); 140 } 141 142 /** 143 * Reactivate a previously created vision source 144 * 145 * @param uniqueName 146 */ 147 public synchronized boolean reactivateDisabledCameraConfig(String uniqueName) { 148 // Make sure we have an old, currently -inactive- camera around 149 var deactivatedConfig = Optional.ofNullable(this.disabledCameraConfigs.remove(uniqueName)); 150 if (deactivatedConfig.isEmpty() || !deactivatedConfig.get().deactivated) { 151 // Not in map, give up 152 return false; 153 } 154 155 // Check if the camera is already in use by another module 156 if (vmm.getModules().stream() 157 .anyMatch( 158 module -> 159 module 160 .getCameraConfiguration() 161 .matchedCameraInfo 162 .uniquePath() 163 .equals(deactivatedConfig.get().matchedCameraInfo.uniquePath()))) { 164 logger.error( 165 "Camera unique-path already in use by active VisionModule! Cannot reactivate " 166 + deactivatedConfig.get().nickname); 167 } 168 169 // transform the camera info all the way to a VisionModule and then start it 170 var created = 171 deactivatedConfig 172 .map(this::loadVisionSourceFromCamConfig) 173 .map(vmm::addSource) 174 .map( 175 it -> { 176 it.start(); 177 it.saveAndBroadcastAll(); 178 return it; 179 }) 180 .isPresent(); 181 182 if (!created) { 183 // Couldn't create a VM for this config - restore state 184 this.disabledCameraConfigs.put(uniqueName, deactivatedConfig.get()); 185 } 186 187 // We have a new camera! Tell the world about it 188 DataChangeService.getInstance() 189 .publishEvent( 190 new OutgoingUIEvent<>( 191 "fullsettings", 192 UIPhotonConfiguration.programStateToUi(ConfigManager.getInstance().getConfig()))); 193 194 pushUiUpdate(); 195 196 return created; 197 } 198 199 /** 200 * Assign a camera that currently has no associated CameraConfiguration loaded. 201 * 202 * @param cameraInfo 203 */ 204 public synchronized boolean assignUnmatchedCamera(PVCameraInfo cameraInfo) { 205 // Check if the camera is already in use by another module 206 if (vmm.getModules().stream() 207 .anyMatch( 208 module -> 209 module 210 .getCameraConfiguration() 211 .matchedCameraInfo 212 .uniquePath() 213 .equals(cameraInfo.uniquePath()))) { 214 logger.error( 215 "Camera unique-path already in use by active VisionModule! Cannot add " + cameraInfo); 216 return false; 217 } 218 219 var source = loadVisionSourceFromCamConfig(new CameraConfiguration(cameraInfo)); 220 var module = vmm.addSource(source); 221 222 module.start(); 223 224 // We have a new camera! Tell the world about it 225 DataChangeService.getInstance() 226 .publishEvent( 227 new OutgoingUIEvent<>( 228 "fullsettings", 229 UIPhotonConfiguration.programStateToUi(ConfigManager.getInstance().getConfig()))); 230 231 pushUiUpdate(); 232 233 return true; 234 } 235 236 public synchronized boolean deleteVisionSource(String uniqueName) { 237 deactivateVisionSource(uniqueName); 238 var config = disabledCameraConfigs.remove(uniqueName); 239 ConfigManager.getInstance().getConfig().removeCameraConfig(uniqueName); 240 ConfigManager.getInstance().saveToDisk(); 241 242 DataChangeService.getInstance() 243 .publishEvent( 244 new OutgoingUIEvent<>( 245 "fullsettings", 246 UIPhotonConfiguration.programStateToUi(ConfigManager.getInstance().getConfig()))); 247 pushUiUpdate(); 248 249 return config != null; 250 } 251 252 public synchronized boolean deactivateVisionSource(String uniqueName) { 253 // try to find the module. If we find it, remove it from the VMM 254 var removedConfig = 255 vmm.getModules().stream() 256 .filter(module -> module.uniqueName().equals(uniqueName)) 257 .findFirst() 258 .map( 259 it -> { 260 vmm.removeModule(it); 261 return it.getCameraConfiguration(); 262 }); 263 264 if (removedConfig.isEmpty()) { 265 logger.error("Could not find module " + uniqueName); 266 return false; 267 } 268 269 // And stuff it into our list of disabled camera configs 270 disabledCameraConfigs.put(removedConfig.get().uniqueName, removedConfig.get()); 271 272 logger.info("Disabled the VisionModule for " + removedConfig.get().nickname); 273 274 pushUiUpdate(); 275 276 return true; 277 } 278 279 protected synchronized VisionSourceManagerState getVsmState() { 280 var ret = new VisionSourceManagerState(); 281 282 ret.allConnectedCameras = filterAllowedDevices(getConnectedCameras()); 283 ret.disabledConfigs = 284 disabledCameraConfigs.values().stream().map(it -> it.toUiConfig()).toList(); 285 286 return ret; 287 } 288 289 protected void pushUiUpdate() { 290 DataChangeService.getInstance() 291 .publishEvent(OutgoingUIEvent.wrappedOf("visionSourceManager", getVsmState())); 292 } 293 294 protected List<PVCameraInfo> getConnectedCameras() { 295 List<PVCameraInfo> cameraInfos = new ArrayList<>(); 296 // find all connected cameras 297 // cscore can return usb and csi cameras but csi are filtered out 298 Stream.of(UsbCamera.enumerateUsbCameras()) 299 .map(c -> PVCameraInfo.fromUsbCameraInfo(c)) 300 .filter(c -> !(String.join("", c.otherPaths()).contains("csi-video"))) 301 .filter(c -> !c.name().equals("unicam")) 302 .forEach(cameraInfos::add); 303 if (LibCameraJNILoader.isSupported()) { 304 // find all CSI cameras (Raspberry Pi cameras) 305 Stream.of(LibCameraJNI.getCameraNames()) 306 .map( 307 path -> { 308 String name = LibCameraJNI.getSensorModel(path).getFriendlyName(); 309 return PVCameraInfo.fromCSICameraInfo(path, name); 310 }) 311 .forEach(cameraInfos::add); 312 } 313 314 // FileVisionSources are a bit quirky. They aren't enumerated by the above, but i still want my 315 // UI to look like it ought to work 316 vmm.getModules().stream() 317 .map(it -> it.getCameraConfiguration().matchedCameraInfo) 318 .filter(info -> info instanceof PVCameraInfo.PVFileCameraInfo) 319 .forEach(cameraInfos::add); 320 321 return cameraInfos; 322 } 323 324 private static List<PVCameraInfo> filterAllowedDevices(List<PVCameraInfo> allDevices) { 325 Platform platform = Platform.getCurrentPlatform(); 326 ArrayList<PVCameraInfo> filteredDevices = new ArrayList<>(); 327 for (var device : allDevices) { 328 boolean valid = false; 329 if (deviceBlacklist.contains(device.name())) { 330 logger.trace( 331 "Skipping blacklisted device: \"" + device.name() + "\" at \"" + device.path() + "\""); 332 } else if (device instanceof PVCameraInfo.PVUsbCameraInfo usbDevice) { 333 if (usbDevice.otherPaths.length == 0 334 && platform.osType == OSType.LINUX 335 && device.type() == CameraType.UsbCamera) { 336 logger.trace( 337 "Skipping device with no other paths: \"" 338 + device.name() 339 + "\" at \"" 340 + device.path()); 341 } else if (Arrays.stream(usbDevice.otherPaths).anyMatch(it -> it.contains("csi-video")) 342 || usbDevice.name().equals("unicam")) { 343 logger.trace( 344 "Skipping CSI device from CSCore: \"" 345 + device.name() 346 + "\" at \"" 347 + device.path() 348 + "\""); 349 } else { 350 valid = true; 351 } 352 } else { 353 valid = true; 354 } 355 if (valid) { 356 filteredDevices.add(device); 357 logger.trace( 358 "Adding local video device - \"" + device.name() + "\" at \"" + device.path() + "\""); 359 } 360 } 361 return filteredDevices; 362 } 363 364 /** 365 * Convert a configuration into a VisionSource. The VisionSource type is pulled from the {@link 366 * CameraConfiguration}'s matchedCameraInfo. We depend on the underlying {@link VisionSource} to 367 * be robust to disconnected sources at boot 368 * 369 * <p>Verify that nickname is unique within the set of deserialized camera configurations, adding 370 * random characters if this isn't the case 371 */ 372 protected VisionSource loadVisionSourceFromCamConfig(CameraConfiguration configuration) { 373 logger.debug("Creating VisionSource for " + configuration.toShortString()); 374 375 // First, make sure that nickname is globally unique since we use the nickname in NetworkTables. 376 // "Just one more source of truth bro it'll real this time I promise" 377 var currentNicknames = new ArrayList<String>(); 378 this.disabledCameraConfigs.values().stream() 379 .map(it -> it.nickname) 380 .forEach(currentNicknames::add); 381 this.vmm.getModules().stream() 382 .map(it -> it.getCameraConfiguration().nickname) 383 .forEach(currentNicknames::add); 384 // while it's a duplicate 385 while (currentNicknames.contains(configuration.nickname)) { 386 // if we already have a number, extract 387 var pattern = Pattern.compile("(^.*) \\(([0-9]+)\\)$"); 388 var matcher = pattern.matcher(configuration.nickname); 389 if (matcher.find()) { 390 int oldNumber = Integer.parseInt(matcher.group(2)); 391 int newNumber = oldNumber + 1; 392 configuration.nickname = matcher.group(1) + " (" + newNumber + ")"; 393 } else { 394 configuration.nickname += " (1)"; 395 } 396 } 397 398 VisionSource source = 399 switch (configuration.matchedCameraInfo.type()) { 400 case UsbCamera -> new USBCameraSource(configuration); 401 case ZeroCopyPicam -> new LibcameraGpuSource(configuration); 402 case FileCamera -> new FileVisionSource(configuration); 403 }; 404 405 if (source.getFrameProvider() == null) { 406 logger.error("Frame provider is null?"); 407 } 408 if (source.getSettables() == null) { 409 logger.error("Settables are null?"); 410 } 411 412 return source; 413 } 414 415 public List<VisionModule> getVisionModules() { 416 return vmm.getModules(); 417 } 418}