From cfe9699997a940f6a4583b632e5a231a95fcb646 Mon Sep 17 00:00:00 2001 From: Vinzenz Schroeter Date: Sat, 8 Mar 2025 12:06:13 +0100 Subject: [PATCH] stream video file --- .github/workflows/rust.yml | 4 +- Cargo.lock | 64 ++++++++++++++++++++++++++++++- Cargo.toml | 1 + README.md | 20 ++++++++++ flake.nix | 9 +++-- src/cli.rs | 10 +++++ src/image_processing.rs | 1 + src/pixels.rs | 77 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 179 insertions(+), 7 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e944336..b84ada5 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -26,10 +26,10 @@ jobs: - name: Install rust toolchain run: sudo apt-get install -qy cargo-1.80 rust-1.80-clippy - name: Install system dependencies - run: sudo apt-get install -qy liblzma-dev libpipewire-0.3-dev libclang-dev libdbus-1-dev + run: sudo apt-get install -qy liblzma-dev libpipewire-0.3-dev libclang-dev libdbus-1-dev ffmpeg libavutil-dev libavformat-dev libavfilter-dev libavdevice-dev - name: Run Clippy run: cargo clippy --all-targets --all-features - name: Build - run: cargo build --release --verbose + run: cargo build --release diff --git a/Cargo.lock b/Cargo.lock index 143f29b..0f6ccc2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -162,6 +162,24 @@ dependencies = [ "syn", ] +[[package]] +name = "bindgen" +version = "0.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" +dependencies = [ + "bitflags 2.9.0", + "cexpr", + "clang-sys", + "itertools", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", +] + [[package]] name = "bit_field" version = "0.10.2" @@ -633,6 +651,31 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "ffmpeg-next" +version = "7.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da02698288e0275e442a47fc12ca26d50daf0d48b15398ba5906f20ac2e2a9f9" +dependencies = [ + "bitflags 2.9.0", + "ffmpeg-sys-next", + "libc", +] + +[[package]] +name = "ffmpeg-sys-next" +version = "7.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bc3234d0a4b2f7d083699d0860c6c9dd83713908771b60f94a96f8704adfe45" +dependencies = [ + "bindgen 0.70.1", + "cc", + "libc", + "num_cpus", + "pkg-config", + "vcpkg", +] + [[package]] name = "flate2" version = "1.1.1" @@ -842,6 +885,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + [[package]] name = "http" version = "1.3.1" @@ -1056,7 +1105,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf0d9716420364790e85cbb9d3ac2c950bde16a7dd36f3209b7dfdfc4a24d01f" dependencies = [ - "bindgen", + "bindgen 0.69.5", "cc", "system-deps", ] @@ -1237,6 +1286,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "objc" version = "0.2.7" @@ -1346,7 +1405,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "849e188f90b1dda88fe2bfe1ad31fe5f158af2c98f80fb5d13726c44f3f01112" dependencies = [ - "bindgen", + "bindgen 0.69.5", "libspa-sys", "system-deps", ] @@ -1751,6 +1810,7 @@ dependencies = [ "clap", "env_logger", "fast_image_resize", + "ffmpeg-next", "image", "log", "scap", diff --git a/Cargo.toml b/Cargo.toml index 7c5315b..ca24e69 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ scap = "0.0.8" image = "0.25.5" fast_image_resize = { version = "5.1", features = ["image"] } tungstenite = "0.26" +ffmpeg-next = "7.1.0" [dependencies.servicepoint] package = "servicepoint" diff --git a/README.md b/README.md index 37f94a5..3876566 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ Commands: flip Invert the state of all pixels [aliases: f] on Set all pixels to the on state image Send an image file (e.g. jpeg or png) to the display. [aliases: i] + video Stream a video file (e.g. mp4) to the display. [aliases: v] screen Stream the default screen capture source to the display. On Linux Wayland, this pops up a screen or window chooser, but it also may directly start streaming your main screen. [aliases: s] ``` @@ -91,6 +92,25 @@ Options: --no-aspect Do not keep aspect ratio when resizing. ``` +#### Video file + +```text +Stream a video file (e.g. mp4) to the display. + +Usage: servicepoint-cli pixels video [OPTIONS] + +Arguments: + + +Options: + --no-hist Disable histogram correction + --no-blur Disable blur + --no-sharp Disable sharpening + --no-dither Disable dithering. Brightness will be adjusted so that around half of the pixels are on. + --no-spacers Do not remove the spacers from the image. + --no-aspect Do not keep aspect ratio when resizing. +``` + #### Screen ```text diff --git a/flake.nix b/flake.nix index 405734a..e8230c9 100644 --- a/flake.nix +++ b/flake.nix @@ -90,8 +90,7 @@ { default = pkgs.mkShell rec { inputsFrom = [ self.packages.${system}.default ]; - packages = [ - pkgs.gdb + packages = with pkgs; [ (pkgs.symlinkJoin { name = "rust-toolchain"; paths = with pkgs; [ @@ -103,7 +102,11 @@ cargo-expand ]; }) - pkgs.cargo-flamegraph + + cargo-flamegraph + gdb + + ffmpeg-headless ]; LD_LIBRARY_PATH = "${pkgs.lib.makeLibraryPath (builtins.concatMap (d: d.buildInputs) inputsFrom)}"; RUST_SRC_PATH = "${pkgs.rust.packages.stable.rustPlatform.rustLibSrc}"; diff --git a/src/cli.rs b/src/cli.rs index 543ba36..c7a4a45 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -74,6 +74,16 @@ pub enum PixelCommand { #[command(flatten)] image_processing_options: ImageProcessingOptions, }, + #[command( + visible_alias = "v", + about = "Stream a video file (e.g. mp4) to the display." + )] + Video { + #[command(flatten)] + send_image_options: SendImageOptions, + #[command(flatten)] + image_processing_options: ImageProcessingOptions, + }, #[command( visible_alias = "s", about = "Stream the default screen capture source to the display. \ diff --git a/src/image_processing.rs b/src/image_processing.rs index 9048dd3..6e27cde 100644 --- a/src/image_processing.rs +++ b/src/image_processing.rs @@ -35,6 +35,7 @@ impl ImageProcessingPipeline { } } + #[must_use] pub fn process(&mut self, frame: DynamicImage) -> Bitmap { let start_time = Instant::now(); diff --git a/src/pixels.rs b/src/pixels.rs index a7aad66..3f9665d 100644 --- a/src/pixels.rs +++ b/src/pixels.rs @@ -4,6 +4,8 @@ use crate::{ stream_window::stream_window, transport::Transport, }; +use ffmpeg_next as ffmpeg; +use image::{DynamicImage, RgbImage}; use log::info; use servicepoint::{ BinaryOperation, BitVecCommand, BitmapCommand, ClearCommand, CompressionCode, DisplayBitVec, @@ -23,6 +25,10 @@ pub(crate) fn pixels(connection: &Transport, pixel_command: PixelCommand) { stream_options, image_processing, } => stream_window(connection, stream_options, image_processing), + PixelCommand::Video { + image_processing_options: processing_options, + send_image_options: image_options, + } => pixels_video(connection, image_options, processing_options), } } @@ -78,3 +84,74 @@ fn pixels_image( .expect("failed to send image command"); info!("sent image to display"); } + +fn pixels_video( + connection: &Transport, + options: SendImageOptions, + processing_options: ImageProcessingOptions, +) { + ffmpeg::init().unwrap(); + + let mut ictx = ffmpeg::format::input(&options.file_name).expect("failed to open video input file"); + + let input = ictx + .streams() + .best(ffmpeg::media::Type::Video) + .ok_or(ffmpeg::Error::StreamNotFound) + .expect("could not get video stream from input file"); + let video_stream_index = input.index(); + + let context_decoder = ffmpeg::codec::context::Context::from_parameters(input.parameters()) + .expect("could not extract video context from parameters"); + let mut decoder = context_decoder.decoder().video() + .expect("failed to create decoder for video stream"); + + let src_width = decoder.width(); + let src_height = decoder.height(); + + let mut scaler = ffmpeg::software::scaling::Context::get( + decoder.format(), + src_width, + src_height, + ffmpeg::format::Pixel::RGB24, + src_width, + src_height, + ffmpeg::software::scaling::Flags::BILINEAR, + ).expect("failed to create scaling context"); + + let mut frame_index = 0; + + let mut processing_pipeline = ImageProcessingPipeline::new(processing_options); + + let mut receive_and_process_decoded_frames = + |decoder: &mut ffmpeg::decoder::Video| -> Result<(), ffmpeg::Error> { + let mut decoded = ffmpeg::util::frame::video::Video::empty(); + let mut rgb_frame = ffmpeg::util::frame::video::Video::empty(); + while decoder.receive_frame(&mut decoded).is_ok() { + scaler.run(&decoded, &mut rgb_frame) + .expect("failed to scale frame"); + + let image = RgbImage::from_raw(src_width, src_height, rgb_frame.data(0).to_owned()) + .expect("could not read rgb data to image"); + let image = DynamicImage::from(image); + let bitmap= processing_pipeline.process(image); + connection.send_command(BitmapCommand::from(bitmap)) + .expect("failed to send image command"); + + frame_index += 1; + } + Ok(()) + }; + + for (stream, packet) in ictx.packets() { + if stream.index() == video_stream_index { + decoder.send_packet(&packet) + .expect("failed to send video packet"); + receive_and_process_decoded_frames(&mut decoder) + .expect("failed to process video packet"); + } + } + decoder.send_eof().expect("failed to send eof"); + receive_and_process_decoded_frames(&mut decoder) + .expect("failed to eof packet"); +}