Video.

Cargo.toml

@@ -1,7 +1,9 @@
 [workspace]
 members = [
     "bevy_nannou",
-    "bevy_nannou_draw", "bevy_nannou_isf",
+    "bevy_nannou_draw",
+    "bevy_nannou_isf",
+    "bevy_nannou_video",
     "examples",
     "generative_design",
     "guide/book_tests",
@@ -23,4 +25,5 @@ resolver = "2"
 [workspace.dependencies]
 bevy = "0.14.0"
 bevy_egui = "0.28.0"
-bevy-inspector-egui = "0.25.0"
+bevy-inspector-egui = "0.25.0"
+rayon = "1.10"

bevy_nannou/Cargo.toml

@@ -7,7 +7,9 @@ edition = "2021"
 bevy = { workspace = true }
 bevy_nannou_draw = { path = "../bevy_nannou_draw" }
 bevy_nannou_isf = { path = "../bevy_nannou_isf", optional = true }
+bevy_nannou_video = { path = "../bevy_nannou_video", optional = true }
 
 [features]
 nightly = ["bevy_nannou_draw/nightly"]
-isf = ["bevy_nannou_isf"]
+isf = ["bevy_nannou_isf"]
+video = ["bevy_nannou_video"]

bevy_nannou/src/lib.rs

@@ -19,6 +19,8 @@ pub mod prelude {
 
     #[cfg(feature = "isf")]
     pub use bevy_nannou_isf::prelude::*;
+    #[cfg(feature = "video")]
+    pub use bevy_nannou_video::prelude::*;
 }
 
 pub struct NannouPlugin;
@@ -30,5 +32,10 @@ impl Plugin for NannouPlugin {
         {
             app.add_plugins(bevy_nannou_isf::NannouIsfPlugin);
         }
+        #[cfg(feature = "video")]
+        {
+            info!("Adding video plugin");
+            app.add_plugins(bevy_nannou_video::NannouVideoPlugin);
+        }
     }
 }

bevy_nannou_draw/Cargo.toml

@@ -10,7 +10,7 @@ nannou_core = { path = "../nannou_core" }
 rusttype = { version = "0.8", features = ["gpu_cache"] }
 num-traits = "0.2"
 bytemuck = "1.15.0"
-rayon = "1.10"
+rayon = { workspace = true }
 uuid = "1.8"
 
 [features]

bevy_nannou_video/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "bevy_nannou_video"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+bevy = { workspace = true }
+video-rs = "0.8.1"
+thiserror = "1"
+rayon = { workspace = true }
+serde = { version = "1.0.204", features = ["derive"] }

bevy_nannou_video/src/asset.rs

@@ -0,0 +1,159 @@
+use bevy::asset::io::{AssetReader, AssetReaderError, AssetSource, PathStream, Reader};
+use bevy::asset::{AssetLoader, LoadContext};
+use bevy::prelude::*;
+use bevy::render::render_asset::RenderAssetUsages;
+use bevy::render::render_resource::{Extent3d, TextureFormat};
+use rayon::prelude::*;
+use std::ops::{Deref, DerefMut};
+use std::path::Path;
+use bevy::asset::io::file::FileAssetReader;
+use bevy::utils::HashMap;
+use thiserror::Error;
+use video_rs::{Decoder, DecoderBuilder};
+use video_rs::hwaccel::HardwareAccelerationDeviceType;
+use serde::{Deserialize, Serialize};
+
+pub struct VideoAssetPlugin;
+
+impl Plugin for VideoAssetPlugin {
+    fn build(&self, app: &mut App) {
+        info!("Adding video asset plugin");
+        app.init_asset::<Video>().init_asset_loader::<VideoLoader>()
+            .add_systems(Update, load_next_frame);
+    }
+}
+
+fn load_next_frame(
+    video_q: Query<&Handle<Video>>,
+    mut videos: ResMut<Assets<Video>>,
+    mut images: ResMut<Assets<Image>>,
+    time: Res<Time>,
+) {
+    for handle in video_q.iter() {
+        let video = videos.get_mut(handle).unwrap();
+        let current_time = time.elapsed_seconds_f64();
+        if current_time - video.last_update < video.frame_duration {
+            continue; // Not time for next frame yet
+        }
+        video.last_update = current_time;
+        if let Some(next_texture) = video.next_texture() {
+            let texture = images.get_mut(&video.texture).unwrap();
+            *texture = next_texture;
+        }
+    }
+}
+
+#[derive(Asset, TypePath)]
+pub struct Video {
+    last_update: f64,
+    frame_duration: f64,
+    pub decoder: Decoder,
+    pub texture: Handle<Image>,
+}
+
+impl Video {
+    fn next_texture(&mut self) -> Option<Image> {
+        let (width, height) = self.decoder.size();
+        if let Some(Ok(frame)) = self.decoder.decode_raw_iter().next() {
+            // TODO: Handle other formats
+            let rgba = frame
+                .data(0)
+                .par_iter()
+                .chunks(3)
+                .flat_map(|chunk| {
+                    let r = chunk[0];
+                    let g = chunk[1];
+                    let b = chunk[2];
+                    [*r, *g, *b, 255]
+                })
+                .collect::<Vec<u8>>();
+
+            let mut image = Image::default();
+            image.texture_descriptor.format = TextureFormat::Rgba8UnormSrgb;
+            image.resize(Extent3d {
+                width,
+                height,
+                ..default()
+            });
+            image.data = rgba;
+            image.asset_usage = RenderAssetUsages::RENDER_WORLD;
+            return Some(image);
+        }
+        None
+    }
+}
+
+impl Deref for Video {
+    type Target = Decoder;
+
+    fn deref(&self) -> &Self::Target {
+        &self.decoder
+    }
+}
+
+impl DerefMut for Video {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.decoder
+    }
+}
+
+#[derive(Default)]
+struct VideoLoader;
+
+#[derive(Default, Serialize, Deserialize)]
+pub struct VideoLoaderSettings {
+    pub options: std::collections::HashMap<String, String>,
+}
+
+impl AssetLoader for VideoLoader {
+    type Asset = Video;
+    type Settings = VideoLoaderSettings;
+    type Error = VideoAssetLoaderError;
+
+    async fn load<'a>(
+        &'a self,
+        _reader: &'a mut Reader<'_>,
+        settings: &'a Self::Settings,
+        load_context: &'a mut LoadContext<'_>,
+    ) -> Result<Self::Asset, Self::Error> {
+        let path = load_context.asset_path().path();
+        // TODO: support web loading
+        let base_path = FileAssetReader::get_base_path().join("assets");
+        let path = base_path.join(path);
+        let options = settings.options.clone();
+        let options = options.into();
+        let decoder = DecoderBuilder::new(path)
+            .with_options(&options)
+            .build()
+            .map_err(|e| VideoAssetLoaderError::Decoder(e))?;
+        let (width, height) = decoder.size();
+        let mut image = Image::default();
+        image.texture_descriptor.format = TextureFormat::Rgba8UnormSrgb;
+        image.resize(Extent3d {
+            width,
+            height,
+            ..default()
+        });
+        image.asset_usage = RenderAssetUsages::RENDER_WORLD;
+        let texture = load_context.add_labeled_asset(String::from("video_texture"), image);
+        let fps = decoder.frame_rate() as f64;
+        Ok(Video {
+            last_update: 0.0,
+            frame_duration: 1.0 / fps,
+            decoder,
+            texture,
+        })
+    }
+
+    fn extensions(&self) -> &[&str] {
+        &["mp4"]
+    }
+}
+
+#[derive(Debug, Error)]
+pub enum VideoAssetLoaderError {
+    #[error("Failed to construct video decoder {0}")]
+    Decoder(#[from] video_rs::Error),
+    #[error("Failed to load video file")]
+    Io(#[from] std::io::Error),
+}

bevy_nannou_video/src/lib.rs

@@ -0,0 +1,18 @@
+use bevy::app::{App, Plugin};
+use crate::asset::VideoAssetPlugin;
+
+mod asset;
+
+pub mod prelude {
+    pub use crate::asset::{Video, VideoLoaderSettings};
+}
+
+pub struct NannouVideoPlugin;
+
+impl Plugin for NannouVideoPlugin {
+    fn build(&self, app: &mut App) {
+        app.add_plugins(
+            VideoAssetPlugin
+        );
+    }
+}

examples/Cargo.toml

@@ -36,6 +36,7 @@ tokio = { version = "1", features = ["rt"]}
 [features]
 egui = ["nannou/egui"]
 isf = ["nannou/isf"]
+video = ["nannou/video"]
 
 # Audio
 [[example]]
@@ -219,4 +220,10 @@ required-features = ["egui"]
 [[example]]
 name = "inspector_ui"
 path = "ui/egui/inspector_ui.rs"
-required-features = ["egui"]
+required-features = ["egui"]
+
+# Video
+[[example]]
+name = "simple_video"
+path = "video/simple_video.rs"
+required-features = ["video"]

examples/isf/simple.rs

@@ -23,7 +23,7 @@ fn model(app: &App) -> Model {
         .view(view)
         .build();
 
-    let isf = app.asset_server().load("isf/Test-Color.fs");
+    let isf = app.asset_server().load("isf/Test-MultiPassRendering.fs");
     let texture_1 = app.asset_server().load("images/nature/nature_1.jpg");
     let texture_2 = app.asset_server().load("images/nature/nature_2.jpg");
     Model {