fast resize, now higher quality; keep aspect ratio

Cargo.lock

@@ -557,6 +557,15 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b"
 
+[[package]]
+name = "document-features"
+version = "0.2.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95249b50c6c185bee49034bcb378a49dc2b5dff0be90ff6616d31d64febab05d"
+dependencies = [
+ "litrs",
+]
+
 [[package]]
 name = "either"
 version = "1.14.0"
@@ -607,6 +616,20 @@ dependencies = [
  "zune-inflate",
 ]
 
+[[package]]
+name = "fast_image_resize"
+version = "5.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b55264ccc579fc127eebf6c6c1841d0c160d79a44c8f6f97047b7bc4a9c0d1a5"
+dependencies = [
+ "bytemuck",
+ "cfg-if",
+ "document-features",
+ "image",
+ "num-traits",
+ "thiserror 1.0.69",
+]
+
 [[package]]
 name = "fdeflate"
 version = "0.3.7"
@@ -1025,6 +1048,12 @@ dependencies = [
  "system-deps",
 ]
 
+[[package]]
+name = "litrs"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5"
+
 [[package]]
 name = "lock_api"
 version = "0.4.12"
@@ -1690,6 +1719,7 @@ version = "0.3.0"
 dependencies = [
  "clap",
  "env_logger",
+ "fast_image_resize",
  "image",
  "log",
  "scap",

Cargo.toml

@@ -19,3 +19,4 @@ env_logger = "0.11"
 log = "0.4"
 scap = "0.0.8"
 image = "0.25.5"
+fast_image_resize = { version = "5.1.2", features = ["image"] }

src/image_processing.rs

@@ -2,17 +2,16 @@ use crate::{
     cli::ImageProcessingOptions,
     ledwand_dither::{blur, histogram_correction, median_brightness, ostromoukhov_dither, sharpen},
 };
-use image::{
+use fast_image_resize::{ResizeOptions, Resizer};
-    imageops::{resize, FilterType},
+use image::{DynamicImage, GrayImage};
-    DynamicImage, ImageBuffer, Luma,
-};
 use log::{debug, trace};
 use servicepoint::{Bitmap, Grid, PIXEL_HEIGHT, PIXEL_WIDTH, TILE_HEIGHT, TILE_SIZE};
-use std::time::Instant;
+use std::{default::Default, time::Instant};
 
 #[derive(Debug)]
 pub struct ImageProcessingPipeline {
     options: ImageProcessingOptions,
+    resizer: Resizer,
 }
 
 const SPACER_HEIGHT: usize = TILE_SIZE / 2;
@@ -21,12 +20,14 @@ const PIXEL_HEIGHT_INCLUDING_SPACERS: usize = SPACER_HEIGHT * (TILE_HEIGHT - 1)
 impl ImageProcessingPipeline {
     pub fn new(options: ImageProcessingOptions) -> Self {
         debug!("Creating image pipeline: {:?}", options);
-        Self { options }
+        Self {
+            options,
+            resizer: Resizer::new(),
+        }
     }
 
-    pub fn process(&self, frame: DynamicImage) -> Bitmap {
+    pub fn process(&mut self, frame: DynamicImage) -> Bitmap {
         let start_time = Instant::now();
-
         let frame = self.resize_grayscale(frame);
         let frame = self.grayscale_processing(frame);
         let mut result = self.grayscale_to_bitmap(frame);
@@ -35,34 +36,42 @@ impl ImageProcessingPipeline {
             result = Self::remove_spacers(result);
         }
 
-        trace!("image processing took {:?}", start_time.elapsed());
+        trace!("pipeline took {:?}", start_time.elapsed());
         result
     }
 
-    fn resize_grayscale(&self, frame: DynamicImage) -> ImageBuffer<Luma<u8>, Vec<u8>> {
+    fn resize_grayscale(&mut self, frame: DynamicImage) -> GrayImage {
-        // TODO: keep aspect ratio
+        let start_time = Instant::now();
         // TODO: make it work for non-maximum sizes
 
-        let frame = frame.grayscale().to_luma8();
-
         let target_height = if self.options.no_spacers {
             PIXEL_HEIGHT
         } else {
             PIXEL_HEIGHT_INCLUDING_SPACERS
         };
 
-        resize(
+        let (target_width, target_height) = Self::fit_size(
-            &frame,
+            (frame.width(), frame.height()),
-            PIXEL_WIDTH as u32,
+            (target_height as u32, PIXEL_WIDTH as u32),
-            target_height as u32,
+        );
-            FilterType::Nearest,
+
-        )
+        let mut dst_image = DynamicImage::new(target_width, target_height, frame.color());
+
+        self.resizer
+            .resize(&frame, &mut dst_image, &ResizeOptions::default())
+            .expect("image resize failed");
+
+        trace!("resizing took {:?}", start_time.elapsed());
+
+        let start_time = Instant::now();
+        let result = dst_image.into_luma8();
+        trace!("grayscale took {:?}", start_time.elapsed());
+
+        result
     }
 
-    fn grayscale_processing(
+    fn grayscale_processing(&self, mut frame: GrayImage) -> GrayImage {
-        &self,
+        let start_time = Instant::now();
-        mut frame: ImageBuffer<Luma<u8>, Vec<u8>>,
-    ) -> ImageBuffer<Luma<u8>, Vec<u8>> {
         if !self.options.no_hist {
             histogram_correction(&mut frame);
         }
@@ -78,26 +87,36 @@ impl ImageProcessingPipeline {
             sharpen(&orig, &mut frame);
             std::mem::swap(&mut frame, &mut orig);
         }
+
+        trace!("image processing took {:?}", start_time.elapsed());
         orig
     }
 
-    fn grayscale_to_bitmap(&self, orig: ImageBuffer<Luma<u8>, Vec<u8>>) -> Bitmap {
+    fn grayscale_to_bitmap(&self, orig: GrayImage) -> Bitmap {
-        if self.options.no_dither {
+        let start_time = Instant::now();
+        let result = if self.options.no_dither {
             let cutoff = median_brightness(&orig);
             let bits = orig.iter().map(move |x| x > &cutoff).collect();
             Bitmap::from_bitvec(orig.width() as usize, bits)
         } else {
             ostromoukhov_dither(orig, u8::MAX / 2)
-        }
+        };
+        trace!("bitmap conversion took {:?}", start_time.elapsed());
+        result
     }
 
     fn remove_spacers(bitmap: Bitmap) -> Bitmap {
-        let mut result = Bitmap::max_sized();
+        let start_time = Instant::now();
+        let height = bitmap.height() - SPACER_HEIGHT * (bitmap.height() / TILE_SIZE - 1);
+        let mut result = Bitmap::new(bitmap.width(), height);
 
         let mut source_y = 0;
         for result_y in 0..result.height() {
             if result_y != 0 && result_y % TILE_SIZE == 0 {
                 source_y += 4;
+                if source_y >= bitmap.height() {
+                    break;
+                }
             }
 
             for x in 0..result.width() {
@@ -105,8 +124,59 @@ impl ImageProcessingPipeline {
             }
 
             source_y += 1;
+            if source_y >= bitmap.height() {
+                break;
+            }
         }
 
+        trace!("removing spacers took {:?}", start_time.elapsed());
         result
     }
+
+    fn fit_size(source: (u32, u32), target: (u32, u32)) -> (u32, u32) {
+        let (source_width, source_height) = source;
+        let (target_width, target_height) = target;
+
+        let source_aspect = source_width as f32 / source_height as f32;
+        let target_aspect = target_width as f32 / target_height as f32;
+
+        if source_aspect > target_aspect {
+            // more x per y in source --> limit width
+            let scale = target_width as f32 / source_width as f32;
+            let height = (source_height as f32 * scale) as u32;
+            (target_width, height)
+        } else {
+            // less x per y in source --> limit height
+            let scale = target_height as f32 / source_height as f32;
+            let mut width = (source_width as f32 * scale) as u32;
+            width += 8 - width % 8;
+            (width, target_height)
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn fit_size_no_change() {
+        assert_eq!(ImageProcessingPipeline::fit_size((8, 7), (8, 7)), (8, 7));
+    }
+
+    #[test]
+    fn fit_size_no_aspect_change() {
+        assert_eq!(ImageProcessingPipeline::fit_size((16, 8), (8, 4)), (8, 4));
+    }
+
+    #[test]
+    fn fit_size_aspect_change() {
+        assert_eq!(ImageProcessingPipeline::fit_size((16, 8), (8, 5)), (8, 4));
+        assert_eq!(ImageProcessingPipeline::fit_size((16, 8), (16, 4)), (8, 4));
+    }
+
+    #[test]
+    fn fit_size_width_div8() {
+        assert_eq!(ImageProcessingPipeline::fit_size((12, 12), (16, 15)), (16,15));
+    }
 }

src/ledwand_dither.rs

@@ -174,11 +174,11 @@ pub(crate) fn ostromoukhov_dither(source: GrayImage, bias: u8) -> Bitmap {
     for y in 0..height as usize {
         let start = y * width as usize;
         if y % 2 == 0 {
-            for x in 0..width as usize {
+            for x in start..start + width as usize {
                 ostromoukhov_dither_pixel(
                     &mut source,
                     &mut destination,
-                    start + x,
+                    x,
                     width as usize,
                     y == (height - 1) as usize,
                     1,
@@ -186,11 +186,11 @@ pub(crate) fn ostromoukhov_dither(source: GrayImage, bias: u8) -> Bitmap {
                 );
             }
         } else {
-            for x in (0..width as usize).rev() {
+            for x in (start..start + width as usize).rev() {
                 ostromoukhov_dither_pixel(
                     &mut source,
                     &mut destination,
-                    start + x,
+                    x,
                     width as usize,
                     y == (height - 1) as usize,
                     -1,
@@ -213,17 +213,9 @@ fn ostromoukhov_dither_pixel(
     direction: isize,
     bias: u8,
 ) {
-    let old_pixel = source[position];
+    let (destination_value, error) = gray_to_bit(source[position], bias);
-
-    let destination_value = old_pixel > bias;
     destination.set(position, destination_value);
-
+    
-    let error = if destination_value {
-        255 - old_pixel
-    } else {
-        old_pixel
-    };
-
     let mut diffuse = |to: usize, mat: i16| {
         let diffuse_value = source[to] as i16 + mat;
         source[to] = diffuse_value.clamp(u8::MIN.into(), u8::MAX.into()) as u8;
@@ -245,6 +237,16 @@ fn ostromoukhov_dither_pixel(
     }
 }
 
+fn gray_to_bit(old_pixel: u8, bias: u8) -> (bool, u8) {
+    let destination_value = old_pixel > bias;
+    let error = if destination_value {
+        255 - old_pixel
+    } else {
+        old_pixel
+    };
+    (destination_value, error)
+}
+
 const ERROR_DIFFUSION_MATRIX: [[i16; 3]; 256] = [
     [0, 1, 0],
     [1, 0, 0],

src/pixels.rs

@@ -44,7 +44,7 @@ fn pixels_image(
     processing_options: ImageProcessingOptions,
 ) {
     let image = image::open(&options.file_name).expect("failed to open image file");
-    let pipeline = ImageProcessingPipeline::new(processing_options);
+    let mut pipeline = ImageProcessingPipeline::new(processing_options);
     let bitmap = pipeline.process(image);
     connection
         .send(Command::BitmapLinearWin(

src/stream_window.rs

@@ -3,14 +3,14 @@ use crate::{
     image_processing::ImageProcessingPipeline,
 };
 use image::{DynamicImage, ImageBuffer, Rgb, Rgba};
-use log::{error, info, warn};
+use log::{debug, error, info, trace, warn};
 use scap::{
     capturer::{Capturer, Options},
     frame::convert_bgra_to_rgb,
     frame::Frame,
 };
 use servicepoint::{Command, CompressionCode, Connection, Origin, FRAME_PACING};
-use std::time::Duration;
+use std::time::{Duration, Instant};
 
 pub fn stream_window(
     connection: &Connection,
@@ -23,20 +23,27 @@ pub fn stream_window(
         None => return,
     };
 
-    let pipeline = ImageProcessingPipeline::new(processing_options);
+    let mut pipeline = ImageProcessingPipeline::new(processing_options);
 
     info!("now starting to stream images");
     loop {
-        let frame = capturer.get_next_frame().expect("failed to capture frame");
+        let start = Instant::now();
+
+        let frame = capture_frame(&capturer);
         let frame = frame_to_image(frame);
         let bitmap = pipeline.process(frame);
+
+        trace!("bitmap ready to send in: {:?}", start.elapsed());
+
         connection
             .send(Command::BitmapLinearWin(
                 Origin::ZERO,
                 bitmap.clone(),
-                CompressionCode::Uncompressed,
+                CompressionCode::default(),
             ))
             .expect("failed to send frame to display");
+
+        debug!("frame time: {:?}", start.elapsed());
     }
 }
 
@@ -66,8 +73,16 @@ fn start_capture(options: &StreamScreenOptions) -> Option<Capturer> {
     Some(capturer)
 }
 
+fn capture_frame(capturer: &Capturer) -> Frame {
+    let start_time = Instant::now();
+    let result = capturer.get_next_frame().expect("failed to capture frame");
+    trace!("capture took: {:?}", start_time.elapsed());
+    result
+}
+
 fn frame_to_image(frame: Frame) -> DynamicImage {
-    match frame {
+    let start_time = Instant::now();
+    let result = match frame {
         Frame::BGRx(frame) => bgrx_to_rgb(frame.width, frame.height, frame.data),
         Frame::RGBx(frame) => DynamicImage::from(
             ImageBuffer::<Rgba<_>, _>::from_raw(
@@ -84,7 +99,9 @@ fn frame_to_image(frame: Frame) -> DynamicImage {
         ),
         Frame::BGRA(frame) => bgrx_to_rgb(frame.width, frame.height, frame.data),
         Frame::YUVFrame(_) | Frame::XBGR(_) => panic!("unsupported frame format"),
-    }
+    };
+    trace!("conversion to image took: {:?}", start_time.elapsed());
+    result
 }
 
 fn bgrx_to_rgb(width: i32, height: i32, data: Vec<u8>) -> DynamicImage {