first CMD_UTF8_DATA implementation

UTF8 now works

.gitignore

@@ -4,3 +4,4 @@ out
 .direnv
 .envrc
 result
+mutants.*

crates/servicepoint/examples/announce.rs

@@ -2,7 +2,7 @@
 
 use clap::Parser;
 
-use servicepoint::{CharGrid, Command, Connection, Cp437Grid, Origin};
+use servicepoint::{CharGrid, Command, Connection, Origin, TILE_WIDTH};
 
 #[derive(Parser, Debug)]
 struct Cli {
@@ -41,11 +41,21 @@ fn main() {
             .expect("sending clear failed");
     }
 
-    let text = cli.text.join("\n");
+    let text = cli
-    let grid = CharGrid::from(text);
+        .text
-    let grid = Cp437Grid::from(grid);
+        .iter()
+        .flat_map(move |x| {
+            x.chars()
+                .collect::<Vec<_>>()
+                .chunks(TILE_WIDTH)
+                .map(|c| String::from_iter(c))
+                .collect::<Vec<_>>()
+        })
+        .collect::<Vec<_>>()
+        .join("\n");
 
+    let grid = CharGrid::from(text);
     connection
-        .send(Command::Cp437Data(Origin::ZERO, grid))
+        .send(Command::Utf8Data(Origin::ZERO, grid))
         .expect("sending text failed");
 }

crates/servicepoint/examples/random_brightness.rs

@@ -31,11 +31,8 @@ fn main() {
         let mut filled_grid = Bitmap::max_sized();
         filled_grid.fill(true);
 
-        let command = BitmapLinearWin(
+        let command =
-            Origin::ZERO,
+            BitmapLinearWin(Origin::ZERO, filled_grid, CompressionCode::Lzma);
-            filled_grid,
-            CompressionCode::Lzma,
-        );
         connection.send(command).expect("send failed");
     }
 

crates/servicepoint/examples/wiping_clear.rs

@@ -34,7 +34,11 @@ fn main() {
         }
 
         connection
-            .send(Command::BitmapLinearWin(Origin::ZERO, enabled_pixels.clone(), CompressionCode::Lzma))
+            .send(Command::BitmapLinearWin(
+                Origin::ZERO,
+                enabled_pixels.clone(),
+                CompressionCode::Lzma,
+            ))
             .expect("could not send command to display");
         thread::sleep(sleep_duration);
     }

crates/servicepoint/src/bitmap.rs

@@ -203,7 +203,7 @@ impl<'t> Iterator for IterRows<'t> {
 
 #[cfg(test)]
 mod tests {
-    use crate::{Bitmap, DataRef, Grid};
+    use crate::{BitVec, Bitmap, DataRef, Grid};
 
     #[test]
     fn fill() {
@@ -295,4 +295,12 @@ mod tests {
         data[1] = 0x0F;
         assert!(grid.get(7, 1));
     }
+
+    #[test]
+    fn to_bitvec() {
+        let mut grid = Bitmap::new(8, 2);
+        grid.set(0, 0, true);
+        let bitvec: BitVec = grid.into();
+        assert_eq!(bitvec.as_raw_slice(), [0x80, 0x00]);
+    }
 }

crates/servicepoint/src/brightness.rs

@@ -1,5 +1,5 @@
-use crate::{Grid, PrimitiveGrid};
+use crate::primitive_grid::PrimitiveGrid;
-
+use crate::{ByteGrid, Grid};
 #[cfg(feature = "rand")]
 use rand::{
     distributions::{Distribution, Standard},
@@ -40,7 +40,8 @@ pub type BrightnessGrid = PrimitiveGrid<Brightness>;
 impl BrightnessGrid {
     /// Like [Self::load], but ignoring any out-of-range brightness values
     pub fn saturating_load(width: usize, height: usize, data: &[u8]) -> Self {
-        PrimitiveGrid::load(width, height, data).map(Brightness::saturating_from)
+        PrimitiveGrid::load(width, height, data)
+            .map(Brightness::saturating_from)
     }
 }
 
@@ -101,7 +102,7 @@ impl From<BrightnessGrid> for Vec<u8> {
     }
 }
 
-impl From<&BrightnessGrid> for PrimitiveGrid<u8> {
+impl From<&BrightnessGrid> for ByteGrid {
     fn from(value: &PrimitiveGrid<Brightness>) -> Self {
         let u8s = value
             .iter()
@@ -111,10 +112,10 @@ impl From<&BrightnessGrid> for PrimitiveGrid<u8> {
     }
 }
 
-impl TryFrom<PrimitiveGrid<u8>> for BrightnessGrid {
+impl TryFrom<ByteGrid> for BrightnessGrid {
     type Error = u8;
 
-    fn try_from(value: PrimitiveGrid<u8>) -> Result<Self, Self::Error> {
+    fn try_from(value: ByteGrid) -> Result<Self, Self::Error> {
         let brightnesses = value
             .iter()
             .map(|b| Brightness::try_from(*b))
@@ -171,7 +172,18 @@ mod tests {
 
     #[test]
     fn saturating_load() {
-        assert_eq!(BrightnessGrid::load(2,2, &[Brightness::MAX, Brightness::MAX, Brightness::MIN, Brightness::MAX]),
+        assert_eq!(
-            BrightnessGrid::saturating_load(2,2, &[255u8, 23, 0, 42]));
+            BrightnessGrid::load(
+                2,
+                2,
+                &[
+                    Brightness::MAX,
+                    Brightness::MAX,
+                    Brightness::MIN,
+                    Brightness::MAX
+                ]
+            ),
+            BrightnessGrid::saturating_load(2, 2, &[255u8, 23, 0, 42])
+        );
     }
 }

crates/servicepoint/src/char_grid.rs

@@ -1,5 +1,8 @@
-use crate::primitive_grid::SeriesError;
+use crate::primitive_grid::{
-use crate::{Grid, PrimitiveGrid};
+    PrimitiveGrid, SeriesError, TryLoadPrimitiveGridError,
+};
+use crate::Grid;
+use std::string::FromUtf8Error;
 
 /// A grid containing UTF-8 characters.
 pub type CharGrid = PrimitiveGrid<char>;
@@ -40,17 +43,39 @@ impl CharGrid {
     ) -> Result<(), SeriesError> {
         self.set_col(x, value.chars().collect::<Vec<_>>().as_ref())
     }
+
+    /// Loads a [CharGrid] with the specified dimensions from the provided UTF-8 bytes.
+    ///
+    /// returns: [CharGrid] that contains the provided data, or [FromUtf8Error] if the data is invalid.
+    ///
+    /// # Panics
+    ///
+    /// - when the dimensions and data size do not match exactly.
+    pub fn load_utf8(
+        width: usize,
+        height: usize,
+        bytes: Vec<u8>,
+    ) -> Result<CharGrid, LoadUtf8Error> {
+        let s: Vec<char> = String::from_utf8(bytes)?.chars().collect();
+        Ok(CharGrid::try_load(width, height, s)?)
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum LoadUtf8Error {
+    #[error(transparent)]
+    FromUtf8Error(#[from] FromUtf8Error),
+    #[error(transparent)]
+    TryLoadError(#[from] TryLoadPrimitiveGridError),
 }
 
 impl From<&str> for CharGrid {
     fn from(value: &str) -> Self {
         let value = value.replace("\r\n", "\n");
-        let mut lines = value
+        let mut lines = value.split('\n').collect::<Vec<_>>();
-            .split('\n')
+        let width = lines
-            .map(move |line| line.trim_end())
+            .iter()
-            .collect::<Vec<_>>();
+            .fold(0, move |a, x| std::cmp::max(a, x.chars().count()));
-        let width =
-            lines.iter().fold(0, move |a, x| std::cmp::max(a, x.len()));
 
         while lines.last().is_some_and(move |line| line.is_empty()) {
             _ = lines.pop();
@@ -73,22 +98,34 @@ impl From<String> for CharGrid {
     }
 }
 
+impl From<CharGrid> for String {
+    fn from(grid: CharGrid) -> Self {
+        String::from(&grid)
+    }
+}
+
 impl From<&CharGrid> for String {
     fn from(value: &CharGrid) -> Self {
         value
             .iter_rows()
-            .map(move |chars| {
+            .map(String::from_iter)
-                chars
+            .collect::<Vec<String>>()
-                    .collect::<String>()
-                    .replace('\0', " ")
-                    .trim_end()
-                    .to_string()
-            })
-            .collect::<Vec<_>>()
             .join("\n")
     }
 }
 
+impl From<&CharGrid> for Vec<u8> {
+    fn from(value: &CharGrid) -> Self {
+        String::from_iter(value.iter()).into_bytes()
+    }
+}
+
+impl From<CharGrid> for Vec<u8> {
+    fn from(value: CharGrid) -> Self {
+        Self::from(&value)
+    }
+}
+
 #[cfg(test)]
 mod test {
     use super::*;
@@ -120,10 +157,28 @@ mod test {
 
     #[test]
     fn str_to_char_grid() {
-        let original = "Hello\r\nWorld!\n...\n";
+        // conversion with .to_string() covers one more line
+        let original = "Hello\r\nWorld!\n...\n".to_string();
+
         let grid = CharGrid::from(original);
         assert_eq!(3, grid.height());
-        let actual = String::from(&grid);
+        assert_eq!("Hello\0\nWorld!\n...\0\0\0", String::from(grid));
-        assert_eq!("Hello\nWorld!\n...", actual);
+    }
+
+    #[test]
+    fn round_trip_bytes() {
+        let grid = CharGrid::from("Hello\0\nWorld!\n...\0\0\0");
+        let bytes: Vec<u8> = grid.clone().into();
+        let copy =
+            CharGrid::load_utf8(grid.width(), grid.height(), bytes).unwrap();
+        assert_eq!(grid, copy);
+    }
+
+    #[test]
+    fn round_trip_string() {
+        let grid = CharGrid::from("Hello\0\nWorld!\n...\0\0\0");
+        let str: String = grid.clone().into();
+        let copy = CharGrid::from(str);
+        assert_eq!(grid, copy);
     }
 }

crates/servicepoint/src/command.rs

@@ -1,9 +1,10 @@
+use crate::primitive_grid::PrimitiveGrid;
 use crate::{
     command_code::CommandCode,
     compression::into_decompressed,
     packet::{Header, Packet},
-    Bitmap, Brightness, BrightnessGrid, CompressionCode, Cp437Grid, Origin,
+    BitVec, Bitmap, Brightness, BrightnessGrid, CharGrid, CompressionCode,
-    Pixels, PrimitiveGrid, BitVec, Tiles, TILE_SIZE,
+    Cp437Grid, Origin, Pixels, Tiles, TILE_SIZE,
 };
 
 /// Type alias for documenting the meaning of the u16 in enum values
@@ -72,10 +73,27 @@ pub enum Command {
     /// ```
     Clear,
 
+    /// Show text on the screen.
+    ///
+    /// The text is sent in the form of a 2D grid of UTF-8 encoded characters (the default encoding in rust).
+    ///
+    /// # Examples
+    ///
+    /// ```rust
+    /// # use servicepoint::{Command, Connection, Origin};
+    /// # let connection = Connection::Fake;
+    /// use servicepoint::{CharGrid};
+    /// let grid = CharGrid::from("Hello,\nWorld!");
+    /// connection.send(Command::Utf8Data(Origin::ZERO, grid)).expect("send failed");
+    /// ```
+    Utf8Data(Origin<Tiles>, CharGrid),
+
     /// Show text on the screen.
     ///
     /// The text is sent in the form of a 2D grid of [CP-437] encoded characters.
     ///
+    /// <div class="warning">You probably want to use [Command::Utf8Data] instead</div>
+    ///
     /// # Examples
     ///
     /// ```rust
@@ -234,6 +252,8 @@ pub enum TryFromPacketError {
     /// The given brightness value is out of bounds
     #[error("The given brightness value {0} is out of bounds.")]
     InvalidBrightness(u8),
+    #[error(transparent)]
+    InvalidUtf8(#[from] std::string::FromUtf8Error),
 }
 
 impl TryFrom<Packet> for Command {
@@ -269,6 +289,7 @@ impl TryFrom<Packet> for Command {
             CommandCode::CharBrightness => {
                 Self::packet_into_char_brightness(&packet)
             }
+            CommandCode::Utf8Data => Self::packet_into_utf8(&packet),
             #[allow(deprecated)]
             CommandCode::BitmapLegacy => Ok(Command::BitmapLegacy),
             CommandCode::BitmapLinear => {
@@ -489,6 +510,28 @@ impl Command {
             Cp437Grid::load(*c as usize, *d as usize, payload),
         ))
     }
+
+    fn packet_into_utf8(
+        packet: &Packet,
+    ) -> Result<Command, TryFromPacketError> {
+        let Packet {
+            header:
+                Header {
+                    command_code: _,
+                    a,
+                    b,
+                    c,
+                    d,
+                },
+            payload,
+        } = packet;
+        let payload: Vec<_> =
+            String::from_utf8(payload.clone())?.chars().collect();
+        Ok(Command::Utf8Data(
+            Origin::new(*a as usize, *b as usize),
+            CharGrid::load(*c as usize, *d as usize, &*payload),
+        ))
+    }
 }
 
 #[cfg(test)]
@@ -499,8 +542,8 @@ mod tests {
         command_code::CommandCode,
         origin::Pixels,
         packet::{Header, Packet},
-        Bitmap, Brightness, BrightnessGrid, Command, CompressionCode, Origin,
+        Bitmap, Brightness, BrightnessGrid, CharGrid, Command, CompressionCode,
-        PrimitiveGrid,
+        Cp437Grid, Origin,
     };
 
     fn round_trip(original: Command) {
@@ -556,16 +599,18 @@ mod tests {
     fn round_trip_char_brightness() {
         round_trip(Command::CharBrightness(
             Origin::new(5, 2),
-            PrimitiveGrid::new(7, 5),
+            BrightnessGrid::new(7, 5),
         ));
     }
 
     #[test]
     fn round_trip_cp437_data() {
-        round_trip(Command::Cp437Data(
+        round_trip(Command::Cp437Data(Origin::new(5, 2), Cp437Grid::new(7, 5)));
-            Origin::new(5, 2),
+    }
-            PrimitiveGrid::new(7, 5),
+
-        ));
+    #[test]
+    fn round_trip_utf8_data() {
+        round_trip(Command::Utf8Data(Origin::new(5, 2), CharGrid::new(7, 5)));
     }
 
     #[test]

crates/servicepoint/src/command_code.rs

@@ -21,6 +21,7 @@ pub(crate) enum CommandCode {
     BitmapLinearWinBzip2 = 0x0018,
     #[cfg(feature = "compression_lzma")]
     BitmapLinearWinLzma = 0x0019,
+    Utf8Data = 0x0020,
     #[cfg(feature = "compression_zstd")]
     BitmapLinearWinZstd = 0x001A,
 }
@@ -93,6 +94,9 @@ impl TryFrom<u16> for CommandCode {
             value if value == CommandCode::BitmapLinearWinBzip2 as u16 => {
                 Ok(CommandCode::BitmapLinearWinBzip2)
             }
+            value if value == CommandCode::Utf8Data as u16 => {
+                Ok(CommandCode::Utf8Data)
+            }
             _ => Err(()),
         }
     }

crates/servicepoint/src/connection.rs

@@ -107,9 +107,7 @@ impl Connection {
 
         let request = ClientRequestBuilder::new(uri).into_client_request()?;
         let (sock, _) = connect(request)?;
-        Ok(Self::WebSocket(std::sync::Mutex::new(
+        Ok(Self::WebSocket(std::sync::Mutex::new(sock)))
-            sock,
-        )))
     }
 
     /// Send something packet-like to the display. Usually this is in the form of a Command.
@@ -159,9 +157,7 @@ impl Drop for Connection {
     fn drop(&mut self) {
         #[cfg(feature = "protocol_websocket")]
         if let Connection::WebSocket(sock) = self {
-            _ = sock
+            _ = sock.try_lock().map(move |mut sock| sock.close(None));
-                .try_lock()
-                .map(move |mut sock| sock.close(None));
         }
     }
 }

crates/servicepoint/src/cp437.rs

@@ -2,7 +2,7 @@
 //!
 //! Most of the functionality is only available with feature "cp437" enabled.
 
-use crate::{Grid, PrimitiveGrid};
+use crate::{Grid, primitive_grid::PrimitiveGrid};
 use std::collections::HashMap;
 
 /// A grid containing codepage 437 characters.
@@ -12,7 +12,9 @@ pub type Cp437Grid = PrimitiveGrid<u8>;
 
 /// The error occurring when loading an invalid character
 #[derive(Debug, PartialEq, thiserror::Error)]
-#[error("The character {char:?} at position {index} is not a valid CP437 character")]
+#[error(
+    "The character {char:?} at position {index} is not a valid CP437 character"
+)]
 pub struct InvalidCharError {
     /// invalid character is at this position in input
     index: usize,

crates/servicepoint/src/lib.rs

@@ -49,11 +49,13 @@ pub use crate::cp437::Cp437Grid;
 pub use crate::data_ref::DataRef;
 pub use crate::grid::Grid;
 pub use crate::origin::{Origin, Pixels, Tiles};
-pub use crate::primitive_grid::{PrimitiveGrid, SeriesError};
 
 /// An alias for the specific type of [bitvec::prelude::BitVec] used.
 pub type BitVec = bitvec::prelude::BitVec<u8, bitvec::prelude::Msb0>;
 
+/// A simple grid of bytes - see [primitive_grid::PrimitiveGrid].
+pub type ByteGrid = primitive_grid::PrimitiveGrid<u8>;
+
 mod bitmap;
 mod brightness;
 mod char_grid;
@@ -67,7 +69,7 @@ mod data_ref;
 mod grid;
 mod origin;
 pub mod packet;
-mod primitive_grid;
+pub mod primitive_grid;
 
 /// size of a single tile in one dimension
 pub const TILE_SIZE: usize = 8;

crates/servicepoint/src/packet.rs

@@ -209,6 +209,9 @@ impl From<Command> for Packet {
                 grid,
                 CommandCode::Cp437Data,
             ),
+            Command::Utf8Data(origin, grid) => {
+                Self::origin_grid_to_packet(origin, grid, CommandCode::Utf8Data)
+            }
         }
     }
 }

crates/servicepoint/src/primitive_grid.rs

@@ -1,9 +1,13 @@
+//! This module contains the implementation of the [PrimitiveGrid].
+
+use std::fmt::Debug;
 use std::slice::{Iter, IterMut};
 
 use crate::{DataRef, Grid};
 
-pub trait PrimitiveGridType: Sized + Default + Copy + Clone {}
+/// A type that can be stored in a [PrimitiveGrid], e.g. [char], [u8].
-impl<T: Sized + Default + Copy + Clone> PrimitiveGridType for T {}
+pub trait PrimitiveGridType: Sized + Default + Copy + Clone + Debug {}
+impl<T: Sized + Default + Copy + Clone + Debug> PrimitiveGridType for T {}
 
 /// A 2D grid of bytes
 #[derive(Debug, Clone, PartialEq)]
@@ -60,7 +64,11 @@ impl<T: PrimitiveGridType> PrimitiveGrid<T> {
     /// - when the dimensions and data size do not match exactly.
     #[must_use]
     pub fn load(width: usize, height: usize, data: &[T]) -> Self {
-        assert_eq!(width * height, data.len());
+        assert_eq!(
+            width * height,
+            data.len(),
+            "dimension mismatch for data {data:?}"
+        );
         Self {
             data: Vec::from(data),
             width,
@@ -68,12 +76,31 @@ impl<T: PrimitiveGridType> PrimitiveGrid<T> {
         }
     }
 
+    /// Loads a [PrimitiveGrid] with the specified dimensions from the provided data.
+    ///
+    /// returns: [PrimitiveGrid] that contains a copy of the provided data or [TryLoadPrimitiveGridError].
+    pub fn try_load(
+        width: usize,
+        height: usize,
+        data: Vec<T>,
+    ) -> Result<Self, TryLoadPrimitiveGridError> {
+        if width * height != data.len() {
+            return Err(TryLoadPrimitiveGridError::InvalidDimensions);
+        }
+
+        Ok(Self {
+            data,
+            width,
+            height,
+        })
+    }
+
     /// Iterate over all cells in [PrimitiveGrid].
     ///
     /// Order is equivalent to the following loop:
     /// ```
-    /// # use servicepoint::{PrimitiveGrid, Grid};
+    /// # use servicepoint::{ByteGrid, Grid};
-    /// # let grid = PrimitiveGrid::<u8>::new(2,2);
+    /// # let grid = ByteGrid::new(2,2);
     /// for y in 0..grid.height() {
     ///     for x in 0..grid.width() {
     ///         grid.get(x, y);
@@ -140,9 +167,9 @@ impl<T: PrimitiveGridType> PrimitiveGrid<T> {
     ///
     /// Use logic written for u8s and then convert to [Brightness] values for sending in a [Command].
     /// ```
-    /// # fn foo(grid: &mut PrimitiveGrid<u8>) {}
+    /// # fn foo(grid: &mut ByteGrid) {}
-    /// # use servicepoint::{Brightness, BrightnessGrid, Command, Origin, PrimitiveGrid, TILE_HEIGHT, TILE_WIDTH};
+    /// # use servicepoint::{Brightness, BrightnessGrid, ByteGrid, Command, Origin, TILE_HEIGHT, TILE_WIDTH};
-    /// let mut grid: PrimitiveGrid<u8> = PrimitiveGrid::new(TILE_WIDTH, TILE_HEIGHT);
+    /// let mut grid: ByteGrid = ByteGrid::new(TILE_WIDTH, TILE_HEIGHT);
     /// foo(&mut grid);
     /// let grid: BrightnessGrid = grid.map(Brightness::saturating_from);
     /// let command = Command::CharBrightness(Origin::ZERO, grid);
@@ -238,6 +265,12 @@ impl<T: PrimitiveGridType> PrimitiveGrid<T> {
     }
 }
 
+#[derive(Debug, thiserror::Error)]
+pub enum TryLoadPrimitiveGridError {
+    #[error("The provided dimensions do not match with the data size")]
+    InvalidDimensions,
+}
+
 impl<T: PrimitiveGridType> Grid<T> for PrimitiveGrid<T> {
     /// Sets the value of the cell at the specified position in the `PrimitiveGrid.
     ///
@@ -300,6 +333,7 @@ impl<T: PrimitiveGridType> From<PrimitiveGrid<T>> for Vec<T> {
     }
 }
 
+/// An iterator iver the rows in a [PrimitiveGrid]
 pub struct IterRows<'t, T: PrimitiveGridType> {
     byte_grid: &'t PrimitiveGrid<T>,
     row: usize,
@@ -323,7 +357,8 @@ impl<'t, T: PrimitiveGridType> Iterator for IterRows<'t, T> {
 
 #[cfg(test)]
 mod tests {
-    use crate::{DataRef, Grid, PrimitiveGrid, SeriesError};
+    use crate::primitive_grid::{PrimitiveGrid, SeriesError};
+    use crate::{DataRef, Grid};
 
     #[test]
     fn fill() {

crates/servicepoint_binding_c/src/brightness_grid.rs

@@ -3,7 +3,7 @@
 //! prefix `sp_brightness_grid_`
 
 use crate::SPByteSlice;
-use servicepoint::{Brightness, DataRef, Grid, PrimitiveGrid};
+use servicepoint::{Brightness, ByteGrid, DataRef, Grid};
 use std::convert::Into;
 use std::intrinsics::transmute;
 use std::ptr::NonNull;
@@ -80,7 +80,7 @@ pub unsafe extern "C" fn sp_brightness_grid_load(
 ) -> NonNull<SPBrightnessGrid> {
     assert!(!data.is_null());
     let data = std::slice::from_raw_parts(data, data_length);
-    let grid = PrimitiveGrid::load(width, height, data);
+    let grid = ByteGrid::load(width, height, data);
     let grid = servicepoint::BrightnessGrid::try_from(grid)
         .expect("invalid brightness value");
     let result = Box::new(SPBrightnessGrid(grid));

crates/servicepoint_binding_uniffi/src/char_grid.rs

@@ -1,7 +1,7 @@
-use servicepoint::{Grid, SeriesError};
+use crate::cp437_grid::Cp437Grid;
+use servicepoint::{Grid, primitive_grid::SeriesError};
 use std::convert::Into;
 use std::sync::{Arc, RwLock};
-use crate::cp437_grid::Cp437Grid;
 
 #[derive(uniffi::Object)]
 pub struct CharGrid {