ci2_sim/
lib.rs

1// Copyright (C) The Strand-Braid Authors
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4//! A [ci2] camera backend that renders synthetic images of simulated insects.
5//!
6//! This backend lets the *entire* live Braid pipeline (acquisition, background
7//! subtraction, feature detection, UDP transport, the mainbrain, the Kalman
8//! filter, data association, `.braidz` saving) run against a known ground truth
9//! with no camera hardware. It is the image-level injection path of the
10//! simulation test harness; see the `braid-sim` crate for the shared core.
11//!
12//! The scenario (a `sim.toml` parsed by [`braid_sim::Scenario`]) is provided via
13//! the `STRAND_CAM_SIM_SPEC` environment variable; the camera to render is
14//! selected by the `--camera-name` Strand Camera already receives (one of the
15//! [`braid_sim::Scenario::camera_name`] values). Each camera independently
16//! evaluates the deterministic world for its current frame and projects it with
17//! its own calibration, so multiple sim cameras need no coordination.
18
19extern crate machine_vision_formats as formats;
20
21use std::time::{Duration, Instant};
22
23use ci2::{
24    AcquisitionMode, AutoMode, DynamicFrameWithInfo, HostTimingInfo, TriggerMode, TriggerSelector,
25};
26use flydra_mvg::FlydraMultiCameraSystem;
27use formats::PixFmt;
28use strand_dynamic_frame::DynamicFrameOwned;
29
30use braid_sim::Scenario;
31use braid_sim::scenario::BlobParams;
32use braid_sim::world::World;
33
34/// The environment variable naming the `sim.toml` scenario file.
35pub const SIM_SPEC_ENV: &str = "STRAND_CAM_SIM_SPEC";
36
37/// Pixel formats the sim backend can render. The single deterministic gray
38/// scene is carried in each: RGB8 replicates the gray into three channels,
39/// YUV422 carries it as luma with neutral chroma, and the Bayer variants label
40/// the mono mosaic (1 byte/pixel) with the requested color-filter array. This
41/// lets the color recording paths be exercised without camera hardware.
42const SUPPORTED_PIXEL_FORMATS: [PixFmt; 7] = [
43    PixFmt::Mono8,
44    PixFmt::RGB8,
45    PixFmt::YUV422,
46    PixFmt::BayerRG8,
47    PixFmt::BayerGR8,
48    PixFmt::BayerGB8,
49    PixFmt::BayerBG8,
50];
51
52/// Load the scenario named by [`SIM_SPEC_ENV`].
53fn load_scenario() -> ci2::Result<Scenario> {
54    let path = std::env::var_os(SIM_SPEC_ENV).ok_or_else(|| {
55        ci2::Error::from(format!(
56            "the sim camera backend requires the {SIM_SPEC_ENV} environment variable \
57             to point at a sim.toml scenario file"
58        ))
59    })?;
60    let text = std::fs::read_to_string(&path)
61        .map_err(|e| ci2::Error::from(format!("reading {SIM_SPEC_ENV} ({path:?}): {e}")))?;
62    Scenario::from_toml_str(&text)
63        .map_err(|e| ci2::Error::from(format!("parsing {SIM_SPEC_ENV} ({path:?}): {e}")))
64}
65
66pub struct WrappedModule {}
67
68pub fn new_module() -> ci2::Result<WrappedModule> {
69    Ok(WrappedModule {})
70}
71
72/// The sim backend keeps no global state that needs tearing down; the guard is a
73/// no-op that exists to match the shape of the other ci2 backends.
74pub struct SimTerminateGuard {}
75
76pub fn make_singleton_guard(
77    _module: &dyn ci2::CameraModule<CameraType = WrappedCamera, Guard = SimTerminateGuard>,
78) -> ci2::Result<SimTerminateGuard> {
79    Ok(SimTerminateGuard {})
80}
81
82impl<'a> ci2::CameraModule for &'a WrappedModule {
83    type CameraType = WrappedCamera;
84    type Guard = SimTerminateGuard;
85
86    fn name(self: &&'a WrappedModule) -> &'static str {
87        "sim"
88    }
89
90    fn camera_infos(self: &&'a WrappedModule) -> ci2::Result<Vec<Box<dyn ci2::CameraInfo>>> {
91        let scenario = load_scenario()?;
92        let infos = (0..scenario.cameras.count)
93            .map(|k| {
94                let ci: Box<dyn ci2::CameraInfo> = Box::new(SimCameraInfo::new(k));
95                ci
96            })
97            .collect();
98        Ok(infos)
99    }
100
101    fn camera(self: &mut &'a WrappedModule, name: &str) -> ci2::Result<Self::CameraType> {
102        WrappedCamera::new(name)
103    }
104
105    fn settings_file_extension(&self) -> &str {
106        // Sim cameras have no node map, but a value is required by the trait.
107        "toml"
108    }
109}
110
111#[derive(Debug, Clone)]
112struct SimCameraInfo {
113    name: String,
114    serial: String,
115}
116
117impl SimCameraInfo {
118    fn new(k: usize) -> Self {
119        Self {
120            name: Scenario::camera_name(k),
121            serial: format!("{k}"),
122        }
123    }
124}
125
126impl ci2::CameraInfo for SimCameraInfo {
127    fn name(&self) -> &str {
128        &self.name
129    }
130    fn serial(&self) -> &str {
131        &self.serial
132    }
133    fn model(&self) -> &str {
134        "sim"
135    }
136    fn vendor(&self) -> &str {
137        "braid-sim"
138    }
139}
140
141pub struct WrappedCamera {
142    info: SimCameraInfo,
143    /// This camera's name, used to project with its own calibration.
144    cam_name: String,
145    /// This camera's index (the `k` in `simcam{k}`), for timing perturbation.
146    cam_index: usize,
147    /// Scenario RNG seed, for deterministic timing jitter.
148    seed: u64,
149    /// Per-camera frame-arrival timing perturbation.
150    timing: braid_sim::scenario::TimingModel,
151    /// The full multi-camera calibration (this camera projects with its entry).
152    system: FlydraMultiCameraSystem<f64>,
153    /// The deterministic ground-truth world.
154    world: World,
155    image_width: usize,
156    image_height: usize,
157    blob: BlobParams,
158    /// If set, report host timestamps as if frames arrived at this rate (instead
159    /// of wall-clock `now()`), to reproduce the wrong-measured-fps bug. The
160    /// reference instant for frame 0 is `start`.
161    reported_fps: Option<f64>,
162    /// Number of insect-free frames rendered first so the background model
163    /// settles before insects appear.
164    bg_warmup_frames: u32,
165    /// Frame rate, frames per second. Used both to evaluate the world at the
166    /// right logical time and to pace acquisition. Defaults to the scenario
167    /// `fps`; Braid overrides it via the software frame-rate-limit path (under
168    /// FakeSync it sends the scenario frame rate, so this is consistent).
169    fps: f64,
170    /// Whether acquisition is paced to `fps`. Braid's software frame-rate-limit
171    /// enables this; it is on by default.
172    frame_rate_enabled: bool,
173    /// The pixel format frames are rendered in. Defaults to Mono8; can be set
174    /// to RGB8 (e.g. via `--pixel-format RGB8`) to exercise the color
175    /// recording path.
176    pixel_format: PixFmt,
177    /// When acquisition started (for pacing); `None` until `acquisition_start`.
178    start: Option<Instant>,
179    /// Wall-clock datetime captured at `acquisition_start`, used as the time of
180    /// frame 0 when `reported_fps` synthesizes timestamps.
181    start_datetime: Option<chrono::DateTime<chrono::Utc>>,
182    /// Next frame number to emit.
183    next_fno: usize,
184}
185
186fn _test_camera_is_send() {
187    // Compile-time check: ci2-async drives the camera from a worker thread.
188    fn implements<T: Send>() {}
189    implements::<WrappedCamera>();
190}
191
192impl WrappedCamera {
193    fn new(name: &str) -> ci2::Result<Self> {
194        let scenario = load_scenario()?;
195
196        // The requested name must be one of the scenario's cameras.
197        let valid = (0..scenario.cameras.count).any(|k| Scenario::camera_name(k) == name);
198        if !valid {
199            return Err(ci2::Error::from(format!(
200                "unknown sim camera \"{name}\"; expected one of simcam0..simcam{}",
201                scenario.cameras.count.saturating_sub(1)
202            )));
203        }
204
205        let system = braid_sim::calibration::build_calibration(&scenario)
206            .map_err(|e| ci2::Error::from(format!("building sim calibration: {e}")))?;
207
208        let cam_index = Scenario::camera_index(name).ok_or_else(|| {
209            ci2::Error::from(format!("cannot parse camera index from \"{name}\""))
210        })?;
211        let info = SimCameraInfo {
212            name: name.to_string(),
213            serial: name.trim_start_matches("simcam").to_string(),
214        };
215
216        Ok(Self {
217            info,
218            cam_name: name.to_string(),
219            cam_index,
220            seed: scenario.seed,
221            timing: scenario.timing.clone(),
222            image_width: scenario.cameras.image_width,
223            image_height: scenario.cameras.image_height,
224            blob: scenario.blob.clone(),
225            reported_fps: scenario.reported_fps,
226            bg_warmup_frames: scenario.bg_warmup_frames,
227            fps: scenario.fps,
228            frame_rate_enabled: true,
229            pixel_format: PixFmt::Mono8,
230            start: None,
231            start_datetime: None,
232            next_fno: 0,
233            world: World::new(scenario),
234            system,
235        })
236    }
237
238    /// Wall-clock interval between frames.
239    fn frame_period(&self) -> Duration {
240        Duration::from_secs_f64(1.0 / self.fps)
241    }
242
243    /// Pixel centers of all insects visible to this camera at frame `fno`,
244    /// after applying the scenario's observation-model imperfections (detection
245    /// noise, dropout, clutter). Empty during the background-warmup phase.
246    fn blobs_for_frame(&self, fno: usize) -> Vec<(f64, f64)> {
247        if (fno as u32) < self.bg_warmup_frames {
248            return Vec::new();
249        }
250        // Logical world time: t = 0 at the first post-warmup frame.
251        let t = (fno as u32 - self.bg_warmup_frames) as f64 / self.fps;
252        let obs = &self.world.scenario().observation;
253        let mut blobs: Vec<(f64, f64)> = self
254            .world
255            .state_at(t)
256            .iter()
257            .filter(|insect| !obs.is_suppressed(self.seed, self.cam_index, fno, insect.id))
258            .filter_map(|insect| {
259                braid_sim::projection::project_pixel(
260                    &self.system,
261                    &self.cam_name,
262                    self.image_width,
263                    self.image_height,
264                    &insect.pos,
265                )
266                .map(|(x, y)| obs.jitter_pixel(self.seed, self.cam_index, fno, insect.id, x, y))
267            })
268            .collect();
269        // Spurious clutter detections (false positives).
270        blobs.extend(obs.clutter(
271            self.seed,
272            self.cam_index,
273            fno,
274            self.image_width,
275            self.image_height,
276        ));
277        blobs
278    }
279}
280
281impl ci2::CameraInfo for WrappedCamera {
282    fn name(&self) -> &str {
283        &self.info.name
284    }
285    fn serial(&self) -> &str {
286        &self.info.serial
287    }
288    fn model(&self) -> &str {
289        "sim"
290    }
291    fn vendor(&self) -> &str {
292        "braid-sim"
293    }
294}
295
296impl ci2::Camera for WrappedCamera {
297    // ----- Sim cameras have no GenICam feature tree. -----
298    fn command_execute(&self, _name: &str, _verify: bool) -> ci2::Result<()> {
299        Err(ci2::Error::FeatureNotPresent())
300    }
301    fn feature_bool(&self, _name: &str) -> ci2::Result<bool> {
302        Err(ci2::Error::FeatureNotPresent())
303    }
304    fn feature_bool_set(&self, _name: &str, _value: bool) -> ci2::Result<()> {
305        Err(ci2::Error::FeatureNotPresent())
306    }
307    fn feature_enum(&self, _name: &str) -> ci2::Result<String> {
308        Err(ci2::Error::FeatureNotPresent())
309    }
310    fn feature_enum_set(&self, _name: &str, _value: &str) -> ci2::Result<()> {
311        Err(ci2::Error::FeatureNotPresent())
312    }
313    fn feature_float(&self, _name: &str) -> ci2::Result<f64> {
314        Err(ci2::Error::FeatureNotPresent())
315    }
316    fn feature_float_set(&self, _name: &str, _value: f64) -> ci2::Result<()> {
317        Err(ci2::Error::FeatureNotPresent())
318    }
319    fn feature_int(&self, _name: &str) -> ci2::Result<i64> {
320        Err(ci2::Error::FeatureNotPresent())
321    }
322    fn feature_int_set(&self, _name: &str, _value: i64) -> ci2::Result<()> {
323        Err(ci2::Error::FeatureNotPresent())
324    }
325
326    fn node_map_load(&self, _settings: &str) -> ci2::Result<()> {
327        Err(ci2::Error::FeatureNotPresent())
328    }
329    fn node_map_save(&self) -> ci2::Result<String> {
330        Err(ci2::Error::FeatureNotPresent())
331    }
332
333    fn width(&self) -> ci2::Result<u32> {
334        Ok(self.image_width as u32)
335    }
336    fn height(&self) -> ci2::Result<u32> {
337        Ok(self.image_height as u32)
338    }
339
340    fn pixel_format(&self) -> ci2::Result<PixFmt> {
341        Ok(self.pixel_format)
342    }
343    fn possible_pixel_formats(&self) -> ci2::Result<Vec<PixFmt>> {
344        Ok(SUPPORTED_PIXEL_FORMATS.to_vec())
345    }
346    fn set_pixel_format(&mut self, pixel_format: PixFmt) -> ci2::Result<()> {
347        if SUPPORTED_PIXEL_FORMATS.contains(&pixel_format) {
348            self.pixel_format = pixel_format;
349            Ok(())
350        } else {
351            Err(ci2::Error::from(format!(
352                "sim backend does not support pixel format {pixel_format}; \
353                 supported: {SUPPORTED_PIXEL_FORMATS:?}"
354            )))
355        }
356    }
357
358    fn exposure_time(&self) -> ci2::Result<f64> {
359        Err(ci2::Error::FeatureNotPresent())
360    }
361    fn exposure_time_range(&self) -> ci2::Result<(f64, f64)> {
362        Err(ci2::Error::FeatureNotPresent())
363    }
364    fn set_exposure_time(&mut self, _: f64) -> ci2::Result<()> {
365        Err(ci2::Error::FeatureNotPresent())
366    }
367    fn exposure_auto(&self) -> ci2::Result<AutoMode> {
368        Err(ci2::Error::FeatureNotPresent())
369    }
370    fn set_exposure_auto(&mut self, _: AutoMode) -> ci2::Result<()> {
371        Err(ci2::Error::FeatureNotPresent())
372    }
373
374    fn gain(&self) -> ci2::Result<f64> {
375        Err(ci2::Error::FeatureNotPresent())
376    }
377    fn gain_range(&self) -> ci2::Result<(f64, f64)> {
378        Err(ci2::Error::FeatureNotPresent())
379    }
380    fn set_gain(&mut self, _: f64) -> ci2::Result<()> {
381        Err(ci2::Error::FeatureNotPresent())
382    }
383    fn gain_auto(&self) -> ci2::Result<AutoMode> {
384        Err(ci2::Error::FeatureNotPresent())
385    }
386    fn set_gain_auto(&mut self, _: AutoMode) -> ci2::Result<()> {
387        Err(ci2::Error::FeatureNotPresent())
388    }
389
390    fn trigger_mode(&self) -> ci2::Result<TriggerMode> {
391        Err(ci2::Error::FeatureNotPresent())
392    }
393    fn set_trigger_mode(&mut self, _: TriggerMode) -> ci2::Result<()> {
394        Err(ci2::Error::FeatureNotPresent())
395    }
396
397    // Frame-rate control is supported: the sim camera paces its frames to this
398    // rate. Under FakeSync, Braid drives this via the software frame-rate-limit
399    // path, sending the scenario frame rate (consistent with the sim's own fps).
400    fn acquisition_frame_rate_enable(&self) -> ci2::Result<bool> {
401        Ok(self.frame_rate_enabled)
402    }
403    fn set_acquisition_frame_rate_enable(&mut self, value: bool) -> ci2::Result<()> {
404        self.frame_rate_enabled = value;
405        Ok(())
406    }
407    fn acquisition_frame_rate(&self) -> ci2::Result<f64> {
408        Ok(self.fps)
409    }
410    fn acquisition_frame_rate_range(&self) -> ci2::Result<(f64, f64)> {
411        Ok((1.0, 1000.0))
412    }
413    fn set_acquisition_frame_rate(&mut self, value: f64) -> ci2::Result<()> {
414        if value <= 0.0 {
415            return Err(ci2::Error::from("frame rate must be positive"));
416        }
417        self.fps = value;
418        Ok(())
419    }
420
421    fn trigger_selector(&self) -> ci2::Result<TriggerSelector> {
422        Err(ci2::Error::FeatureNotPresent())
423    }
424    fn set_trigger_selector(&mut self, _: TriggerSelector) -> ci2::Result<()> {
425        Err(ci2::Error::FeatureNotPresent())
426    }
427
428    fn acquisition_mode(&self) -> ci2::Result<AcquisitionMode> {
429        Err(ci2::Error::FeatureNotPresent())
430    }
431    fn set_acquisition_mode(&mut self, _: AcquisitionMode) -> ci2::Result<()> {
432        Err(ci2::Error::FeatureNotPresent())
433    }
434
435    fn acquisition_start(&mut self) -> ci2::Result<()> {
436        self.next_fno = 0;
437        self.start = Some(Instant::now());
438        self.start_datetime = Some(chrono::Utc::now());
439        Ok(())
440    }
441    fn acquisition_stop(&mut self) -> ci2::Result<()> {
442        self.start = None;
443        Ok(())
444    }
445
446    fn next_frame(&mut self) -> ci2::Result<DynamicFrameWithInfo> {
447        let fno = self.next_fno;
448        self.next_fno += 1;
449
450        // Pace to the frame rate (when enabled) so the pipeline runs at a
451        // realistic rate. The optional per-camera timing perturbation
452        // delays delivery of this frame so its 2D detections reach the mainbrain
453        // late (and may be dropped from live bundling). Only delivery is delayed;
454        // the frame content is unchanged.
455        if let (Some(start), true) = (self.start, self.frame_rate_enabled) {
456            let extra = self.timing.extra_delay_sec(self.seed, self.cam_index, fno);
457            let target = start + self.frame_period() * fno as u32 + Duration::from_secs_f64(extra);
458            let now = Instant::now();
459            if target > now {
460                std::thread::sleep(target - now);
461            }
462        }
463
464        let blobs = self.blobs_for_frame(fno);
465        // Render in the selected pixel format, all carrying the same gray scene
466        // to exercise the various recording paths. RGB8 replicates the gray into
467        // three channels; YUV422 carries it as luma with neutral chroma; the
468        // Bayer variants use the mono mosaic (1 byte/pixel) labeled with the CFA.
469        let bg = self.blob.background;
470        let peak = self.blob.peak as f64;
471        let sigma = self.blob.sigma;
472        let (w, h) = (self.image_width, self.image_height);
473        let (buf, stride) = match self.pixel_format {
474            PixFmt::RGB8 => (
475                braid_sim::render::render_rgb8(w, h, bg, &blobs, peak, sigma),
476                w * 3,
477            ),
478            PixFmt::YUV422 => (
479                braid_sim::render::render_yuv422_uyvy(w, h, bg, &blobs, peak, sigma),
480                w * 2,
481            ),
482            // Mono8 and all Bayer variants are a single byte per pixel; the
483            // Bayer mosaic is simply the mono scene labeled with a CFA.
484            _ => (
485                braid_sim::render::render_mono8(w, h, bg, &blobs, peak, sigma),
486                w,
487            ),
488        };
489
490        let image = DynamicFrameOwned::from_buf(
491            self.image_width as u32,
492            self.image_height as u32,
493            stride,
494            buf,
495            self.pixel_format,
496        )
497        .ok_or_else(|| ci2::Error::SingleFrameError("sim frame had invalid layout".into()))?;
498
499        // Host grab time. Normally the true wall-clock now(); with `reported_fps`
500        // set, a synthetic time advancing at that rate, modeling a host clock
501        // that is *bunched* relative to the true frame cadence (as happens under
502        // load when the driver delivers buffered frames in bursts). The fps
503        // estimator must not be fooled by this when a hardware timestamp exists.
504        let datetime = match (self.reported_fps, self.start_datetime) {
505            (Some(rfps), Some(base)) if rfps > 0.0 => {
506                base + chrono::Duration::nanoseconds((fno as f64 / rfps * 1e9) as i64)
507            }
508            _ => chrono::Utc::now(),
509        };
510
511        // Hardware (device) timestamp at the TRUE frame cadence, in nanoseconds.
512        // The sim emulates a camera that provides a reliable hardware clock; this
513        // is what a correct fps estimator should use, and it is unaffected by the
514        // `reported_fps` host-clock bunching above.
515        let device_timestamp = (fno as f64 / self.fps * 1e9) as u64;
516        let backend_data: Option<Box<dyn ci2::BackendData>> =
517            Some(Box::new(ci2_pylon_types::PylonExtra {
518                block_id: fno as u64,
519                device_timestamp,
520            }));
521
522        Ok(DynamicFrameWithInfo {
523            image: std::sync::Arc::new(image),
524            host_timing: HostTimingInfo { fno, datetime },
525            backend_data,
526        })
527    }
528}
ci2_sim/lib.rs

ci2_sim/
lib.rs
