From a1fa13b6e557590c9c66e2f78f4a6f214f174ce9 Mon Sep 17 00:00:00 2001 From: Vinzenz Schroeter Date: Sun, 2 Mar 2025 01:26:09 +0100 Subject: [PATCH] fast resize, now higher quality; keep aspect ratio --- Cargo.lock | 30 ++++++++++ Cargo.toml | 1 + src/image_processing.rs | 126 ++++++++++++++++++++++++++++------------ src/ledwand_dither.rs | 30 +++++----- src/pixels.rs | 2 +- src/stream_window.rs | 31 +++++++--- 6 files changed, 160 insertions(+), 60 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 19ef7a5..b9ad7ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -557,6 +557,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b" +[[package]] +name = "document-features" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95249b50c6c185bee49034bcb378a49dc2b5dff0be90ff6616d31d64febab05d" +dependencies = [ + "litrs", +] + [[package]] name = "either" version = "1.14.0" @@ -607,6 +616,20 @@ dependencies = [ "zune-inflate", ] +[[package]] +name = "fast_image_resize" +version = "5.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b55264ccc579fc127eebf6c6c1841d0c160d79a44c8f6f97047b7bc4a9c0d1a5" +dependencies = [ + "bytemuck", + "cfg-if", + "document-features", + "image", + "num-traits", + "thiserror 1.0.69", +] + [[package]] name = "fdeflate" version = "0.3.7" @@ -1025,6 +1048,12 @@ dependencies = [ "system-deps", ] +[[package]] +name = "litrs" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" + [[package]] name = "lock_api" version = "0.4.12" @@ -1690,6 +1719,7 @@ version = "0.3.0" dependencies = [ "clap", "env_logger", + "fast_image_resize", "image", "log", "scap", diff --git a/Cargo.toml b/Cargo.toml index 54edfe6..a8cd60f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,3 +19,4 @@ env_logger = "0.11" log = "0.4" scap = "0.0.8" image = "0.25.5" +fast_image_resize = { version = "5.1.2", features = ["image"] } diff --git a/src/image_processing.rs b/src/image_processing.rs index 4c06e68..bcc3104 100644 --- a/src/image_processing.rs +++ b/src/image_processing.rs @@ -2,29 +2,39 @@ use crate::{ cli::ImageProcessingOptions, ledwand_dither::{blur, histogram_correction, median_brightness, ostromoukhov_dither, sharpen}, }; -use image::{ - imageops::{resize, FilterType}, - DynamicImage, ImageBuffer, Luma, -}; +use fast_image_resize::{ResizeOptions, Resizer}; +use image::{DynamicImage, GrayImage}; use log::{debug, trace}; use servicepoint::{Bitmap, Grid, PIXEL_HEIGHT, PIXEL_WIDTH, TILE_HEIGHT, TILE_SIZE}; -use std::time::Instant; +use std::{default::Default, time::Instant}; #[derive(Debug)] pub struct ImageProcessingPipeline { options: ImageProcessingOptions, + resizer: Resizer, + render_size: (usize, usize), } const SPACER_HEIGHT: usize = TILE_SIZE / 2; -const PIXEL_HEIGHT_INCLUDING_SPACERS: usize = SPACER_HEIGHT * (TILE_HEIGHT - 1) + PIXEL_HEIGHT; impl ImageProcessingPipeline { pub fn new(options: ImageProcessingOptions) -> Self { debug!("Creating image pipeline: {:?}", options); - Self { options } + + let spacers_height = if options.no_spacers { + 0 + } else { + SPACER_HEIGHT * (TILE_HEIGHT - 1) + }; + + Self { + options, + resizer: Resizer::new(), + render_size: (PIXEL_WIDTH, PIXEL_HEIGHT + spacers_height), + } } - pub fn process(&self, frame: DynamicImage) -> Bitmap { + pub fn process(&mut self, frame: DynamicImage) -> Bitmap { let start_time = Instant::now(); let frame = self.resize_grayscale(frame); @@ -35,34 +45,31 @@ impl ImageProcessingPipeline { result = Self::remove_spacers(result); } - trace!("image processing took {:?}", start_time.elapsed()); + trace!("pipeline took {:?}", start_time.elapsed()); result } - fn resize_grayscale(&self, frame: DynamicImage) -> ImageBuffer, Vec> { - // TODO: keep aspect ratio - // TODO: make it work for non-maximum sizes + fn resize_grayscale(&mut self, frame: DynamicImage) -> GrayImage { + let start_time = Instant::now(); - let frame = frame.grayscale().to_luma8(); + let (scaled_width, scaled_height) = self.fit_size((frame.width(), frame.height())); + let mut dst_image = DynamicImage::new(scaled_width, scaled_height, frame.color()); - let target_height = if self.options.no_spacers { - PIXEL_HEIGHT - } else { - PIXEL_HEIGHT_INCLUDING_SPACERS - }; + self.resizer + .resize(&frame, &mut dst_image, &ResizeOptions::default()) + .expect("image resize failed"); - resize( - &frame, - PIXEL_WIDTH as u32, - target_height as u32, - FilterType::Nearest, - ) + trace!("resizing took {:?}", start_time.elapsed()); + + let start_time = Instant::now(); + let result = dst_image.into_luma8(); + trace!("grayscale took {:?}", start_time.elapsed()); + + result } - fn grayscale_processing( - &self, - mut frame: ImageBuffer, Vec>, - ) -> ImageBuffer, Vec> { + fn grayscale_processing(&self, mut frame: GrayImage) -> GrayImage { + let start_time = Instant::now(); if !self.options.no_hist { histogram_correction(&mut frame); } @@ -78,35 +85,78 @@ impl ImageProcessingPipeline { sharpen(&orig, &mut frame); std::mem::swap(&mut frame, &mut orig); } + + trace!("image processing took {:?}", start_time.elapsed()); orig } - fn grayscale_to_bitmap(&self, orig: ImageBuffer, Vec>) -> Bitmap { - if self.options.no_dither { + fn grayscale_to_bitmap(&self, orig: GrayImage) -> Bitmap { + let start_time = Instant::now(); + let result = if self.options.no_dither { let cutoff = median_brightness(&orig); let bits = orig.iter().map(move |x| x > &cutoff).collect(); Bitmap::from_bitvec(orig.width() as usize, bits) } else { ostromoukhov_dither(orig, u8::MAX / 2) - } + }; + trace!("bitmap conversion took {:?}", start_time.elapsed()); + result } - fn remove_spacers(bitmap: Bitmap) -> Bitmap { - let mut result = Bitmap::max_sized(); + fn remove_spacers(source: Bitmap) -> Bitmap { + let start_time = Instant::now(); + + let full_tile_rows_with_spacers = source.height() / (TILE_SIZE + SPACER_HEIGHT); + let remaining_pixel_rows = source.height() % (TILE_SIZE + SPACER_HEIGHT); + let total_spacer_height = full_tile_rows_with_spacers * SPACER_HEIGHT + + remaining_pixel_rows.saturating_sub(TILE_SIZE); + let height_without_spacers = source.height() - total_spacer_height; + trace!( + "spacers take up {total_spacer_height}, resulting in height {height_without_spacers}" + ); + + let mut result = Bitmap::new(source.width(), height_without_spacers); let mut source_y = 0; for result_y in 0..result.height() { - if result_y != 0 && result_y % TILE_SIZE == 0 { - source_y += 4; - } - for x in 0..result.width() { - result.set(x, result_y, bitmap.get(x, source_y)); + result.set(x, result_y, source.get(x, source_y)); } + if result_y != 0 && result_y % TILE_SIZE == 0 { + source_y += SPACER_HEIGHT; + } source_y += 1; } + trace!("removing spacers took {:?}", start_time.elapsed()); + result + } + + fn fit_size(&self, source: (u32, u32)) -> (u32, u32) { + let (source_width, source_height) = source; + let (target_width, target_height) = self.render_size; + debug_assert_eq!(target_width % TILE_SIZE, 0); + + let width_scale = target_width as f32 / source_width as f32; + let height_scale = target_height as f32 / source_height as f32; + let scale = f32::min(width_scale, height_scale); + + let height = (source_height as f32 * scale) as u32; + let mut width = (source_width as f32 * scale) as u32; + + if width % TILE_SIZE as u32 != 0 { + // because we do not have many pixels, round up even if it is a worse fit + width += 8 - width % 8; + } + + let result = (width, height); + trace!( + "scaling {:?} to {:?} to fit {:?}", + source, + result, + self.render_size + ); result } } diff --git a/src/ledwand_dither.rs b/src/ledwand_dither.rs index be1091e..6dbf370 100644 --- a/src/ledwand_dither.rs +++ b/src/ledwand_dither.rs @@ -174,11 +174,11 @@ pub(crate) fn ostromoukhov_dither(source: GrayImage, bias: u8) -> Bitmap { for y in 0..height as usize { let start = y * width as usize; if y % 2 == 0 { - for x in 0..width as usize { + for x in start..start + width as usize { ostromoukhov_dither_pixel( &mut source, &mut destination, - start + x, + x, width as usize, y == (height - 1) as usize, 1, @@ -186,11 +186,11 @@ pub(crate) fn ostromoukhov_dither(source: GrayImage, bias: u8) -> Bitmap { ); } } else { - for x in (0..width as usize).rev() { + for x in (start..start + width as usize).rev() { ostromoukhov_dither_pixel( &mut source, &mut destination, - start + x, + x, width as usize, y == (height - 1) as usize, -1, @@ -213,17 +213,9 @@ fn ostromoukhov_dither_pixel( direction: isize, bias: u8, ) { - let old_pixel = source[position]; - - let destination_value = old_pixel > bias; + let (destination_value, error) = gray_to_bit(source[position], bias); destination.set(position, destination_value); - - let error = if destination_value { - 255 - old_pixel - } else { - old_pixel - }; - + let mut diffuse = |to: usize, mat: i16| { let diffuse_value = source[to] as i16 + mat; source[to] = diffuse_value.clamp(u8::MIN.into(), u8::MAX.into()) as u8; @@ -245,6 +237,16 @@ fn ostromoukhov_dither_pixel( } } +fn gray_to_bit(old_pixel: u8, bias: u8) -> (bool, u8) { + let destination_value = old_pixel > bias; + let error = if destination_value { + 255 - old_pixel + } else { + old_pixel + }; + (destination_value, error) +} + const ERROR_DIFFUSION_MATRIX: [[i16; 3]; 256] = [ [0, 1, 0], [1, 0, 0], diff --git a/src/pixels.rs b/src/pixels.rs index a4fece0..c92be07 100644 --- a/src/pixels.rs +++ b/src/pixels.rs @@ -44,7 +44,7 @@ fn pixels_image( processing_options: ImageProcessingOptions, ) { let image = image::open(&options.file_name).expect("failed to open image file"); - let pipeline = ImageProcessingPipeline::new(processing_options); + let mut pipeline = ImageProcessingPipeline::new(processing_options); let bitmap = pipeline.process(image); connection .send(Command::BitmapLinearWin( diff --git a/src/stream_window.rs b/src/stream_window.rs index e66cf26..f8e41ec 100644 --- a/src/stream_window.rs +++ b/src/stream_window.rs @@ -3,14 +3,14 @@ use crate::{ image_processing::ImageProcessingPipeline, }; use image::{DynamicImage, ImageBuffer, Rgb, Rgba}; -use log::{error, info, warn}; +use log::{debug, error, info, trace, warn}; use scap::{ capturer::{Capturer, Options}, frame::convert_bgra_to_rgb, frame::Frame, }; use servicepoint::{Command, CompressionCode, Connection, Origin, FRAME_PACING}; -use std::time::Duration; +use std::time::{Duration, Instant}; pub fn stream_window( connection: &Connection, @@ -23,20 +23,27 @@ pub fn stream_window( None => return, }; - let pipeline = ImageProcessingPipeline::new(processing_options); + let mut pipeline = ImageProcessingPipeline::new(processing_options); info!("now starting to stream images"); loop { - let frame = capturer.get_next_frame().expect("failed to capture frame"); + let start = Instant::now(); + + let frame = capture_frame(&capturer); let frame = frame_to_image(frame); let bitmap = pipeline.process(frame); + + trace!("bitmap ready to send in: {:?}", start.elapsed()); + connection .send(Command::BitmapLinearWin( Origin::ZERO, bitmap.clone(), - CompressionCode::Uncompressed, + CompressionCode::default(), )) .expect("failed to send frame to display"); + + debug!("frame time: {:?}", start.elapsed()); } } @@ -66,8 +73,16 @@ fn start_capture(options: &StreamScreenOptions) -> Option { Some(capturer) } +fn capture_frame(capturer: &Capturer) -> Frame { + let start_time = Instant::now(); + let result = capturer.get_next_frame().expect("failed to capture frame"); + trace!("capture took: {:?}", start_time.elapsed()); + result +} + fn frame_to_image(frame: Frame) -> DynamicImage { - match frame { + let start_time = Instant::now(); + let result = match frame { Frame::BGRx(frame) => bgrx_to_rgb(frame.width, frame.height, frame.data), Frame::RGBx(frame) => DynamicImage::from( ImageBuffer::, _>::from_raw( @@ -84,7 +99,9 @@ fn frame_to_image(frame: Frame) -> DynamicImage { ), Frame::BGRA(frame) => bgrx_to_rgb(frame.width, frame.height, frame.data), Frame::YUVFrame(_) | Frame::XBGR(_) => panic!("unsupported frame format"), - } + }; + trace!("conversion to image took: {:?}", start_time.elapsed()); + result } fn bgrx_to_rgb(width: i32, height: i32, data: Vec) -> DynamicImage {