From 39b72329b424f818ea3d6b9c6c1f1525416bd72c Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Sun, 26 Jan 2025 16:52:29 -0500 Subject: [PATCH] stdx: Add floor/ceil/is grapheme boundary functions to RopeSliceExt These functions are the equivalent of 23b424a46 for grapheme clusters. In order to add the `is_grapheme_boundary` function we also need to query whether a byte index lies on a character boundary, so this change also adds `is_char_boundary`. --- Cargo.lock | 1 + Cargo.toml | 1 + helix-core/Cargo.toml | 2 +- helix-stdx/Cargo.toml | 1 + helix-stdx/src/rope.rs | 210 ++++++++++++++++++++++++++++++++++++++++- helix-tui/Cargo.toml | 2 +- 6 files changed, 213 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2419fa9ba..3c73da803 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1427,6 +1427,7 @@ dependencies = [ "ropey", "rustix", "tempfile", + "unicode-segmentation", "which", "windows-sys 0.59.0", ] diff --git a/Cargo.toml b/Cargo.toml index b117fe80d..acca8732f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,6 +43,7 @@ slotmap = "1.0.7" thiserror = "2.0" tempfile = "3.15.0" bitflags = "2.8" +unicode-segmentation = "1.2" [workspace.package] version = "25.1.1" diff --git a/helix-core/Cargo.toml b/helix-core/Cargo.toml index b8821c411..eff794e73 100644 --- a/helix-core/Cargo.toml +++ b/helix-core/Cargo.toml @@ -23,7 +23,7 @@ helix-parsec = { path = "../helix-parsec" } ropey = { version = "1.6.1", default-features = false, features = ["simd"] } smallvec = "1.13" smartstring = "1.0.1" -unicode-segmentation = "1.12" +unicode-segmentation.workspace = true # unicode-width is changing width definitions # that both break our logic and disagree with common # width definitions in terminals, we need to replace it. diff --git a/helix-stdx/Cargo.toml b/helix-stdx/Cargo.toml index d575a28fe..708971140 100644 --- a/helix-stdx/Cargo.toml +++ b/helix-stdx/Cargo.toml @@ -20,6 +20,7 @@ regex-cursor = "0.1.4" bitflags.workspace = true once_cell = "1.19" regex-automata = "0.4.9" +unicode-segmentation.workspace = true [target.'cfg(windows)'.dependencies] windows-sys = { version = "0.59", features = ["Win32_Foundation", "Win32_Security", "Win32_Security_Authorization", "Win32_Storage_FileSystem", "Win32_System_Threading"] } diff --git a/helix-stdx/src/rope.rs b/helix-stdx/src/rope.rs index 4a1bc59c2..eac1450bf 100644 --- a/helix-stdx/src/rope.rs +++ b/helix-stdx/src/rope.rs @@ -4,6 +4,7 @@ pub use regex_cursor::engines::meta::{Builder as RegexBuilder, Regex}; pub use regex_cursor::regex_automata::util::syntax::Config; use regex_cursor::{Input as RegexInput, RopeyCursor}; use ropey::RopeSlice; +use unicode_segmentation::{GraphemeCursor, GraphemeIncomplete}; pub trait RopeSliceExt<'a>: Sized { fn ends_with(self, text: &str) -> bool; @@ -52,6 +53,75 @@ pub trait RopeSliceExt<'a>: Sized { /// assert_eq!(text.ceil_char_boundary(3), 3); /// ``` fn ceil_char_boundary(self, byte_idx: usize) -> usize; + /// Checks whether the given `byte_idx` lies on a character boundary. + /// + /// # Example + /// + /// ``` + /// # use ropey::RopeSlice; + /// # use helix_stdx::rope::RopeSliceExt; + /// let text = RopeSlice::from("⌚"); // three bytes: e2 8c 9a + /// assert!(text.is_char_boundary(0)); + /// assert!(!text.is_char_boundary(1)); + /// assert!(!text.is_char_boundary(2)); + /// assert!(text.is_char_boundary(3)); + /// ``` + #[allow(clippy::wrong_self_convention)] + fn is_char_boundary(self, byte_idx: usize) -> bool; + /// Finds the closest byte index not exceeding `byte_idx` which lies on a grapheme cluster + /// boundary. + /// + /// If `byte_idx` already lies on a grapheme cluster boundary then it is returned as-is. When + /// `byte_idx` lies between two grapheme cluster boundaries, this function returns the byte + /// index of the lesser / earlier / left-hand-side boundary. + /// + /// `byte_idx` does not need to be aligned to a character boundary. + /// + /// # Example + /// + /// ``` + /// # use ropey::RopeSlice; + /// # use helix_stdx::rope::RopeSliceExt; + /// let text = RopeSlice::from("\r\n"); // U+000D U+000A, hex: 0d 0a + /// assert_eq!(text.floor_grapheme_boundary(0), 0); + /// assert_eq!(text.floor_grapheme_boundary(1), 0); + /// assert_eq!(text.floor_grapheme_boundary(2), 2); + /// ``` + fn floor_grapheme_boundary(self, byte_idx: usize) -> usize; + /// Finds the closest byte index not exceeding `byte_idx` which lies on a grapheme cluster + /// boundary. + /// + /// If `byte_idx` already lies on a grapheme cluster boundary then it is returned as-is. When + /// `byte_idx` lies between two grapheme cluster boundaries, this function returns the byte + /// index of the greater / later / right-hand-side boundary. + /// + /// `byte_idx` does not need to be aligned to a character boundary. + /// + /// # Example + /// + /// ``` + /// # use ropey::RopeSlice; + /// # use helix_stdx::rope::RopeSliceExt; + /// let text = RopeSlice::from("\r\n"); // U+000D U+000A, hex: 0d 0a + /// assert_eq!(text.ceil_grapheme_boundary(0), 0); + /// assert_eq!(text.ceil_grapheme_boundary(1), 2); + /// assert_eq!(text.ceil_grapheme_boundary(2), 2); + /// ``` + fn ceil_grapheme_boundary(self, byte_idx: usize) -> usize; + /// Checks whether the `byte_idx` lies on a grapheme cluster boundary. + /// + /// # Example + /// + /// ``` + /// # use ropey::RopeSlice; + /// # use helix_stdx::rope::RopeSliceExt; + /// let text = RopeSlice::from("\r\n"); // U+000D U+000A, hex: 0d 0a + /// assert!(text.is_grapheme_boundary(0)); + /// assert!(!text.is_grapheme_boundary(1)); + /// assert!(text.is_grapheme_boundary(2)); + /// ``` + #[allow(clippy::wrong_self_convention)] + fn is_grapheme_boundary(self, byte_idx: usize) -> bool; } impl<'a> RopeSliceExt<'a> for RopeSlice<'a> { @@ -112,7 +182,7 @@ impl<'a> RopeSliceExt<'a> for RopeSlice<'a> { .map(|pos| self.len_chars() - pos - 1) } - // These two are adapted from std's `round_char_boundary` functions: + // These three are adapted from std: fn floor_char_boundary(self, byte_idx: usize) -> usize { if byte_idx >= self.len_bytes() { @@ -140,6 +210,101 @@ impl<'a> RopeSliceExt<'a> for RopeSlice<'a> { .map_or(upper_bound, |pos| pos + byte_idx) } } + + fn is_char_boundary(self, byte_idx: usize) -> bool { + if byte_idx == 0 { + return true; + } + + if byte_idx >= self.len_bytes() { + byte_idx == self.len_bytes() + } else { + is_utf8_char_boundary(self.bytes_at(byte_idx).next().unwrap()) + } + } + + fn floor_grapheme_boundary(self, mut byte_idx: usize) -> usize { + if byte_idx >= self.len_bytes() { + return self.len_bytes(); + } + + byte_idx = self.ceil_char_boundary(byte_idx + 1); + + let (mut chunk, mut chunk_byte_idx, _, _) = self.chunk_at_byte(byte_idx); + + let mut cursor = GraphemeCursor::new(byte_idx, self.len_bytes(), true); + + loop { + match cursor.prev_boundary(chunk, chunk_byte_idx) { + Ok(None) => return 0, + Ok(Some(boundary)) => return boundary, + Err(GraphemeIncomplete::PrevChunk) => { + let (ch, ch_byte_idx, _, _) = self.chunk_at_byte(chunk_byte_idx - 1); + chunk = ch; + chunk_byte_idx = ch_byte_idx; + } + Err(GraphemeIncomplete::PreContext(n)) => { + let ctx_chunk = self.chunk_at_byte(n - 1).0; + cursor.provide_context(ctx_chunk, n - ctx_chunk.len()); + } + _ => unreachable!(), + } + } + } + + fn ceil_grapheme_boundary(self, mut byte_idx: usize) -> usize { + if byte_idx >= self.len_bytes() { + return self.len_bytes(); + } + + if byte_idx == 0 { + return 0; + } + + byte_idx = self.floor_char_boundary(byte_idx - 1); + + let (mut chunk, mut chunk_byte_idx, _, _) = self.chunk_at_byte(byte_idx); + + let mut cursor = GraphemeCursor::new(byte_idx, self.len_bytes(), true); + + loop { + match cursor.next_boundary(chunk, chunk_byte_idx) { + Ok(None) => return self.len_bytes(), + Ok(Some(boundary)) => return boundary, + Err(GraphemeIncomplete::NextChunk) => { + chunk_byte_idx += chunk.len(); + chunk = self.chunk_at_byte(chunk_byte_idx).0; + } + Err(GraphemeIncomplete::PreContext(n)) => { + let ctx_chunk = self.chunk_at_byte(n - 1).0; + cursor.provide_context(ctx_chunk, n - ctx_chunk.len()); + } + _ => unreachable!(), + } + } + } + + fn is_grapheme_boundary(self, byte_idx: usize) -> bool { + // The byte must lie on a character boundary to lie on a grapheme cluster boundary. + if !self.is_char_boundary(byte_idx) { + return false; + } + + let (chunk, chunk_byte_idx, _, _) = self.chunk_at_byte(byte_idx); + + let mut cursor = GraphemeCursor::new(byte_idx, self.len_bytes(), true); + + loop { + match cursor.is_boundary(chunk, chunk_byte_idx) { + Ok(n) => return n, + Err(GraphemeIncomplete::PreContext(n)) => { + let (ctx_chunk, ctx_byte_start, _, _) = self.chunk_at_byte(n - 1); + cursor.provide_context(ctx_chunk, ctx_byte_start); + } + Err(_) => unreachable!(), + } + } + } } // copied from std @@ -166,12 +331,13 @@ mod tests { } #[test] - fn floor_ceil_char_boundary() { + fn char_boundaries() { let ascii = RopeSlice::from("ascii"); // When the given index lies on a character boundary, the index should not change. for byte_idx in 0..=ascii.len_bytes() { assert_eq!(ascii.floor_char_boundary(byte_idx), byte_idx); assert_eq!(ascii.ceil_char_boundary(byte_idx), byte_idx); + assert!(ascii.is_char_boundary(byte_idx)); } // This is a polyfill of a method of this trait which was replaced by ceil_char_boundary. @@ -198,4 +364,44 @@ mod tests { } } } + + #[test] + fn grapheme_boundaries() { + let ascii = RopeSlice::from("ascii"); + // When the given index lies on a grapheme boundary, the index should not change. + for byte_idx in 0..=ascii.len_bytes() { + assert_eq!(ascii.floor_char_boundary(byte_idx), byte_idx); + assert_eq!(ascii.ceil_char_boundary(byte_idx), byte_idx); + assert!(ascii.is_grapheme_boundary(byte_idx)); + } + + // 🏴‍☠️: U+1F3F4 U+200D U+2620 U+FE0F + // 13 bytes, hex: f0 9f 8f b4 + e2 80 8d + e2 98 a0 + ef b8 8f + let g = RopeSlice::from("🏴‍☠️\r\n"); + let emoji_len = "🏴‍☠️".len(); + let end = g.len_bytes(); + + for byte_idx in 0..emoji_len { + assert_eq!(g.floor_grapheme_boundary(byte_idx), 0); + } + for byte_idx in emoji_len..end { + assert_eq!(g.floor_grapheme_boundary(byte_idx), emoji_len); + } + assert_eq!(g.floor_grapheme_boundary(end), end); + + assert_eq!(g.ceil_grapheme_boundary(0), 0); + for byte_idx in 1..=emoji_len { + assert_eq!(g.ceil_grapheme_boundary(byte_idx), emoji_len); + } + for byte_idx in emoji_len + 1..=end { + assert_eq!(g.ceil_grapheme_boundary(byte_idx), end); + } + + assert!(g.is_grapheme_boundary(0)); + assert!(g.is_grapheme_boundary(emoji_len)); + assert!(g.is_grapheme_boundary(end)); + for byte_idx in (1..emoji_len).chain(emoji_len + 1..end) { + assert!(!g.is_grapheme_boundary(byte_idx)); + } + } } diff --git a/helix-tui/Cargo.toml b/helix-tui/Cargo.toml index 97765800a..0a3a35534 100644 --- a/helix-tui/Cargo.toml +++ b/helix-tui/Cargo.toml @@ -20,7 +20,7 @@ helix-core = { path = "../helix-core" } bitflags.workspace = true cassowary = "0.3" -unicode-segmentation = "1.12" +unicode-segmentation.workspace = true crossterm = { version = "0.28", optional = true } termini = "1.0" serde = { version = "1", "optional" = true, features = ["derive"]}