Replace tree-sitter with tree-house

Cargo.lock

@@ -30,7 +30,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
 dependencies = [
  "cfg-if",
- "getrandom 0.2.15",
  "once_cell",
  "version_check",
  "zerocopy",
@@ -1319,14 +1318,13 @@ dependencies = [
 name = "helix-core"
 version = "25.1.1"
 dependencies = [
- "ahash",
  "anyhow",
  "arc-swap",
  "bitflags",
  "chrono",
  "encoding_rs",
+ "foldhash",
  "globset",
- "hashbrown 0.14.5",
  "helix-loader",
  "helix-parsec",
  "helix-stdx",
@@ -1347,7 +1345,7 @@ dependencies = [
  "smartstring",
  "textwrap",
  "toml",
- "tree-sitter",
+ "tree-house",
  "unicode-general-category",
  "unicode-segmentation",
  "unicode-width 0.1.12",
@@ -1391,14 +1389,13 @@ dependencies = [
  "cc",
  "etcetera",
  "helix-stdx",
- "libloading",
  "log",
  "once_cell",
  "serde",
  "tempfile",
  "threadpool",
  "toml",
- "tree-sitter",
+ "tree-house",
 ]
 
 [[package]]
@@ -2665,13 +2662,31 @@ dependencies = [
 ]
 
 [[package]]
-name = "tree-sitter"
+name = "tree-house"
-version = "0.22.6"
+version = "0.1.0-beta.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://github.com/helix-editor/tree-house#1fa65eca36fdbb2837e0655bfda53ed627fc25c0"
-checksum = "df7cc499ceadd4dcdf7ec6d4cbc34ece92c3fa07821e287aedecd4416c516dca"
+dependencies = [
+ "arc-swap",
+ "hashbrown 0.15.2",
+ "kstring",
+ "once_cell",
+ "regex",
+ "regex-cursor",
+ "ropey",
+ "slab",
+ "tree-house-bindings",
+]
+
+[[package]]
+name = "tree-house-bindings"
+version = "0.1.0-beta.2"
+source = "git+https://github.com/helix-editor/tree-house#1fa65eca36fdbb2837e0655bfda53ed627fc25c0"
 dependencies = [
  "cc",
- "regex",
+ "libloading",
+ "regex-cursor",
+ "ropey",
+ "thiserror 2.0.12",
 ]
 
 [[package]]

Cargo.toml

@@ -37,7 +37,7 @@ package.helix-tui.opt-level = 2
 package.helix-term.opt-level = 2
 
 [workspace.dependencies]
-tree-sitter = { version = "0.22" }
+tree-house = { git = "https://github.com/helix-editor/tree-house", default-features = false }
 nucleo = "0.5.0"
 slotmap = "1.0.7"
 thiserror = "2.0"

helix-core/Cargo.toml

@@ -32,13 +32,12 @@ unicode-segmentation.workspace = true
 unicode-width = "=0.1.12"
 unicode-general-category = "1.0"
 slotmap.workspace = true
-tree-sitter.workspace = true
+tree-house.workspace = true
 once_cell = "1.21"
 arc-swap = "1"
 regex = "1"
 bitflags.workspace = true
-ahash = "0.8.11"
+foldhash.workspace = true
-hashbrown = { version = "0.14.5", features = ["raw"] }
 url = "2.5.4"
 
 log = "0.4"

helix-core/src/indent.rs

@@ -1,17 +1,17 @@
 use std::{borrow::Cow, collections::HashMap, iter};
 
 use helix_stdx::rope::RopeSliceExt;
-use tree_sitter::{Query, QueryCursor, QueryPredicateArg};
 
 use crate::{
     chars::{char_is_line_ending, char_is_whitespace},
     graphemes::{grapheme_width, tab_width_at},
-    syntax::{
+    syntax::{self, config::IndentationHeuristic},
-        config::{IndentationHeuristic, LanguageConfiguration},
+    tree_sitter::{
-        RopeProvider, Syntax,
+        self,
+        query::{InvalidPredicateError, UserPredicate},
+        Capture, Grammar, InactiveQueryCursor, Node, Pattern, Query, QueryMatch, RopeInput,
     },
-    tree_sitter::Node,
+    Position, Rope, RopeSlice, Syntax, Tendril,
-    Position, Rope, RopeSlice, Tendril,
 };
 
 /// Enum representing indentation style.
@@ -282,18 +282,164 @@ fn add_indent_level(
 
 /// Return true if only whitespace comes before the node on its line.
 /// If given, new_line_byte_pos is treated the same way as any existing newline.
-fn is_first_in_line(node: Node, text: RopeSlice, new_line_byte_pos: Option<usize>) -> bool {
+fn is_first_in_line(node: &Node, text: RopeSlice, new_line_byte_pos: Option<u32>) -> bool {
-    let mut line_start_byte_pos = text.line_to_byte(node.start_position().row);
+    let line = text.byte_to_line(node.start_byte() as usize);
+    let mut line_start_byte_pos = text.line_to_byte(line) as u32;
     if let Some(pos) = new_line_byte_pos {
         if line_start_byte_pos < pos && pos <= node.start_byte() {
             line_start_byte_pos = pos;
         }
     }
-    text.byte_slice(line_start_byte_pos..node.start_byte())
+    text.byte_slice(line_start_byte_pos as usize..node.start_byte() as usize)
         .chars()
         .all(|c| c.is_whitespace())
 }
 
+#[derive(Debug, Default)]
+pub struct IndentQueryPredicates {
+    not_kind_eq: Option<(Capture, Box<str>)>,
+    same_line: Option<(Capture, Capture, bool)>,
+    one_line: Option<(Capture, bool)>,
+}
+
+impl IndentQueryPredicates {
+    fn are_satisfied(
+        &self,
+        match_: &QueryMatch,
+        text: RopeSlice,
+        new_line_byte_pos: Option<u32>,
+    ) -> bool {
+        if let Some((capture, not_expected_kind)) = self.not_kind_eq.as_ref() {
+            if !match_
+                .nodes_for_capture(*capture)
+                .next()
+                .is_some_and(|node| node.kind() != not_expected_kind.as_ref())
+            {
+                return false;
+            }
+        }
+
+        if let Some((capture1, capture2, negated)) = self.same_line {
+            let n1 = match_.nodes_for_capture(capture1).next();
+            let n2 = match_.nodes_for_capture(capture2).next();
+            let satisfied = n1.zip(n2).is_some_and(|(n1, n2)| {
+                let n1_line = get_node_start_line(text, n1, new_line_byte_pos);
+                let n2_line = get_node_start_line(text, n2, new_line_byte_pos);
+                let same_line = n1_line == n2_line;
+                same_line != negated
+            });
+
+            if !satisfied {
+                return false;
+            }
+        }
+
+        if let Some((capture, negated)) = self.one_line {
+            let node = match_.nodes_for_capture(capture).next();
+            let satisfied = node.is_some_and(|node| {
+                let start_line = get_node_start_line(text, node, new_line_byte_pos);
+                let end_line = get_node_end_line(text, node, new_line_byte_pos);
+                let one_line = end_line == start_line;
+                one_line != negated
+            });
+
+            if !satisfied {
+                return false;
+            }
+        }
+
+        true
+    }
+}
+
+#[derive(Debug)]
+pub struct IndentQuery {
+    query: Query,
+    properties: HashMap<Pattern, IndentScope>,
+    predicates: HashMap<Pattern, IndentQueryPredicates>,
+    indent_capture: Option<Capture>,
+    indent_always_capture: Option<Capture>,
+    outdent_capture: Option<Capture>,
+    outdent_always_capture: Option<Capture>,
+    align_capture: Option<Capture>,
+    anchor_capture: Option<Capture>,
+    extend_capture: Option<Capture>,
+    extend_prevent_once_capture: Option<Capture>,
+}
+
+impl IndentQuery {
+    pub fn new(grammar: Grammar, source: &str) -> Result<Self, tree_sitter::query::ParseError> {
+        let mut properties = HashMap::new();
+        let mut predicates: HashMap<Pattern, IndentQueryPredicates> = HashMap::new();
+        let query = Query::new(grammar, source, |pattern, predicate| match predicate {
+            UserPredicate::SetProperty { key: "scope", val } => {
+                let scope = match val {
+                    Some("all") => IndentScope::All,
+                    Some("tail") => IndentScope::Tail,
+                    Some(other) => {
+                        return Err(format!("unknown scope (#set! scope \"{other}\")").into())
+                    }
+                    None => return Err("missing scope value (#set! scope ...)".into()),
+                };
+
+                properties.insert(pattern, scope);
+
+                Ok(())
+            }
+            UserPredicate::Other(predicate) => {
+                let name = predicate.name();
+                match name {
+                    "not-kind-eq?" => {
+                        predicate.check_arg_count(2)?;
+                        let capture = predicate.capture_arg(0)?;
+                        let not_expected_kind = predicate.str_arg(1)?;
+
+                        predicates.entry(pattern).or_default().not_kind_eq =
+                            Some((capture, not_expected_kind.to_string().into_boxed_str()));
+                        Ok(())
+                    }
+                    "same-line?" | "not-same-line?" => {
+                        predicate.check_arg_count(2)?;
+                        let capture1 = predicate.capture_arg(0)?;
+                        let capture2 = predicate.capture_arg(1)?;
+                        let negated = name == "not-same-line?";
+
+                        predicates.entry(pattern).or_default().same_line =
+                            Some((capture1, capture2, negated));
+                        Ok(())
+                    }
+                    "one-line?" | "not-one-line?" => {
+                        predicate.check_arg_count(1)?;
+                        let capture = predicate.capture_arg(0)?;
+                        let negated = name == "not-one-line?";
+
+                        predicates.entry(pattern).or_default().one_line = Some((capture, negated));
+                        Ok(())
+                    }
+                    _ => Err(InvalidPredicateError::unknown(UserPredicate::Other(
+                        predicate,
+                    ))),
+                }
+            }
+            _ => Err(InvalidPredicateError::unknown(predicate)),
+        })?;
+
+        Ok(Self {
+            properties,
+            predicates,
+            indent_capture: query.get_capture("indent"),
+            indent_always_capture: query.get_capture("indent.always"),
+            outdent_capture: query.get_capture("outdent"),
+            outdent_always_capture: query.get_capture("outdent.always"),
+            align_capture: query.get_capture("align"),
+            anchor_capture: query.get_capture("anchor"),
+            extend_capture: query.get_capture("extend"),
+            extend_prevent_once_capture: query.get_capture("extend.prevent-once"),
+            query,
+        })
+    }
+}
+
 /// The total indent for some line of code.
 /// This is usually constructed in one of 2 ways:
 /// - Successively add indent captures to get the (added) indent from a single line
@@ -456,16 +602,16 @@ struct IndentQueryResult<'a> {
     extend_captures: HashMap<usize, Vec<ExtendCapture>>,
 }
 
-fn get_node_start_line(node: Node, new_line_byte_pos: Option<usize>) -> usize {
+fn get_node_start_line(text: RopeSlice, node: &Node, new_line_byte_pos: Option<u32>) -> usize {
-    let mut node_line = node.start_position().row;
+    let mut node_line = text.byte_to_line(node.start_byte() as usize);
     // Adjust for the new line that will be inserted
     if new_line_byte_pos.is_some_and(|pos| node.start_byte() >= pos) {
         node_line += 1;
     }
     node_line
 }
-fn get_node_end_line(node: Node, new_line_byte_pos: Option<usize>) -> usize {
+fn get_node_end_line(text: RopeSlice, node: &Node, new_line_byte_pos: Option<u32>) -> usize {
-    let mut node_line = node.end_position().row;
+    let mut node_line = text.byte_to_line(node.end_byte() as usize);
     // Adjust for the new line that will be inserted (with a strict inequality since end_byte is exclusive)
     if new_line_byte_pos.is_some_and(|pos| node.end_byte() > pos) {
         node_line += 1;
@@ -474,175 +620,98 @@ fn get_node_end_line(node: Node, new_line_byte_pos: Option<usize>) -> usize {
 }
 
 fn query_indents<'a>(
-    query: &Query,
+    query: &IndentQuery,
     syntax: &Syntax,
-    cursor: &mut QueryCursor,
     text: RopeSlice<'a>,
-    range: std::ops::Range<usize>,
+    range: std::ops::Range<u32>,
-    new_line_byte_pos: Option<usize>,
+    new_line_byte_pos: Option<u32>,
 ) -> IndentQueryResult<'a> {
     let mut indent_captures: HashMap<usize, Vec<IndentCapture>> = HashMap::new();
     let mut extend_captures: HashMap<usize, Vec<ExtendCapture>> = HashMap::new();
+
+    let mut cursor = InactiveQueryCursor::new();
     cursor.set_byte_range(range);
+    let mut cursor = cursor.execute_query(
+        &query.query,
+        &syntax.tree().root_node(),
+        RopeInput::new(text),
+    );
 
     // Iterate over all captures from the query
-    for m in cursor.matches(query, syntax.tree().root_node(), RopeProvider(text)) {
+    while let Some(m) = cursor.next_match() {
         // Skip matches where not all custom predicates are fulfilled
-        if !query.general_predicates(m.pattern_index).iter().all(|pred| {
+        if query
-            match pred.operator.as_ref() {
+            .predicates
-                "not-kind-eq?" => match (pred.args.first(), pred.args.get(1)) {
+            .get(&m.pattern())
-                    (
+            .is_some_and(|preds| !preds.are_satisfied(&m, text, new_line_byte_pos))
-                        Some(QueryPredicateArg::Capture(capture_idx)),
+        {
-                        Some(QueryPredicateArg::String(kind)),
-                    ) => {
-                        let node = m.nodes_for_capture_index(*capture_idx).next();
-                        match node {
-                            Some(node) => node.kind()!=kind.as_ref(),
-                            _ => true,
-                        }
-                    }
-                    _ => {
-                        panic!("Invalid indent query: Arguments to \"not-kind-eq?\" must be a capture and a string");
-                    }
-                },
-                "same-line?" | "not-same-line?" => {
-                    match (pred.args.first(), pred.args.get(1)) {
-                        (
-                            Some(QueryPredicateArg::Capture(capt1)),
-                            Some(QueryPredicateArg::Capture(capt2))
-                        ) => {
-                            let n1 = m.nodes_for_capture_index(*capt1).next();
-                            let n2 = m.nodes_for_capture_index(*capt2).next();
-                            match (n1, n2) {
-                                (Some(n1), Some(n2)) => {
-                                    let n1_line = get_node_start_line(n1, new_line_byte_pos);
-                                    let n2_line = get_node_start_line(n2, new_line_byte_pos);
-                                    let same_line = n1_line == n2_line;
-                                    same_line==(pred.operator.as_ref()=="same-line?")
-                                }
-                                _ => true,
-                            }
-                        }
-                        _ => {
-                            panic!("Invalid indent query: Arguments to \"{}\" must be 2 captures", pred.operator);
-                        }
-                    }
-                }
-                "one-line?" | "not-one-line?" => match pred.args.first() {
-                    Some(QueryPredicateArg::Capture(capture_idx)) => {
-                        let node = m.nodes_for_capture_index(*capture_idx).next();
-
-                        match node {
-                            Some(node) => {
-                                let (start_line, end_line) = (get_node_start_line(node,new_line_byte_pos), get_node_end_line(node, new_line_byte_pos));
-                                let one_line = end_line == start_line;
-                                one_line != (pred.operator.as_ref() == "not-one-line?")
-                            },
-                            _ => true,
-                        }
-                    }
-                    _ => {
-                        panic!("Invalid indent query: Arguments to \"not-kind-eq?\" must be a capture and a string");
-                    }
-                },
-                _ => {
-                    panic!(
-                        "Invalid indent query: Unknown predicate (\"{}\")",
-                        pred.operator
-                    );
-                }
-            }
-        }) {
             continue;
         }
         // A list of pairs (node_id, indent_capture) that are added by this match.
         // They cannot be added to indent_captures immediately since they may depend on other captures (such as an @anchor).
         let mut added_indent_captures: Vec<(usize, IndentCapture)> = Vec::new();
         // The row/column position of the optional anchor in this query
-        let mut anchor: Option<tree_sitter::Node> = None;
+        let mut anchor: Option<&Node> = None;
-        for capture in m.captures {
+        for matched_node in m.matched_nodes() {
-            let capture_name = query.capture_names()[capture.index as usize];
+            let node_id = matched_node.node.id();
-            let capture_type = match capture_name {
+            let capture = Some(matched_node.capture);
-                "indent" => IndentCaptureType::Indent,
+            let capture_type = if capture == query.indent_capture {
-                "indent.always" => IndentCaptureType::IndentAlways,
+                IndentCaptureType::Indent
-                "outdent" => IndentCaptureType::Outdent,
+            } else if capture == query.indent_always_capture {
-                "outdent.always" => IndentCaptureType::OutdentAlways,
+                IndentCaptureType::IndentAlways
-                // The alignment will be updated to the correct value at the end, when the anchor is known.
+            } else if capture == query.outdent_capture {
-                "align" => IndentCaptureType::Align(RopeSlice::from("")),
+                IndentCaptureType::Outdent
-                "anchor" => {
+            } else if capture == query.outdent_always_capture {
-                    if anchor.is_some() {
+                IndentCaptureType::OutdentAlways
-                        log::error!("Invalid indent query: Encountered more than one @anchor in the same match.")
+            } else if capture == query.align_capture {
-                    } else {
+                IndentCaptureType::Align(RopeSlice::from(""))
-                        anchor = Some(capture.node);
+            } else if capture == query.anchor_capture {
-                    }
+                if anchor.is_some() {
-                    continue;
+                    log::error!("Invalid indent query: Encountered more than one @anchor in the same match.")
-                }
+                } else {
-                "extend" => {
+                    anchor = Some(&matched_node.node);
-                    extend_captures
-                        .entry(capture.node.id())
-                        .or_insert_with(|| Vec::with_capacity(1))
-                        .push(ExtendCapture::Extend);
-                    continue;
-                }
-                "extend.prevent-once" => {
-                    extend_captures
-                        .entry(capture.node.id())
-                        .or_insert_with(|| Vec::with_capacity(1))
-                        .push(ExtendCapture::PreventOnce);
-                    continue;
-                }
-                _ => {
-                    // Ignore any unknown captures (these may be needed for predicates such as #match?)
-                    continue;
                 }
+                continue;
+            } else if capture == query.extend_capture {
+                extend_captures
+                    .entry(node_id)
+                    .or_insert_with(|| Vec::with_capacity(1))
+                    .push(ExtendCapture::Extend);
+                continue;
+            } else if capture == query.extend_prevent_once_capture {
+                extend_captures
+                    .entry(node_id)
+                    .or_insert_with(|| Vec::with_capacity(1))
+                    .push(ExtendCapture::PreventOnce);
+                continue;
+            } else {
+                // Ignore any unknown captures (these may be needed for predicates such as #match?)
+                continue;
             };
-            let scope = capture_type.default_scope();
+
-            let mut indent_capture = IndentCapture {
+            // Apply additional settings for this capture
+            let scope = query
+                .properties
+                .get(&m.pattern())
+                .copied()
+                .unwrap_or_else(|| capture_type.default_scope());
+            let indent_capture = IndentCapture {
                 capture_type,
                 scope,
             };
-            // Apply additional settings for this capture
+            added_indent_captures.push((node_id, indent_capture))
-            for property in query.property_settings(m.pattern_index) {
-                match property.key.as_ref() {
-                    "scope" => {
-                        indent_capture.scope = match property.value.as_deref() {
-                            Some("all") => IndentScope::All,
-                            Some("tail") => IndentScope::Tail,
-                            Some(s) => {
-                                panic!("Invalid indent query: Unknown value for \"scope\" property (\"{}\")", s);
-                            }
-                            None => {
-                                panic!(
-                                    "Invalid indent query: Missing value for \"scope\" property"
-                                );
-                            }
-                        }
-                    }
-                    _ => {
-                        panic!(
-                            "Invalid indent query: Unknown property \"{}\"",
-                            property.key
-                        );
-                    }
-                }
-            }
-            added_indent_captures.push((capture.node.id(), indent_capture))
         }
         for (node_id, mut capture) in added_indent_captures {
             // Set the anchor for all align queries.
             if let IndentCaptureType::Align(_) = capture.capture_type {
-                let anchor = match anchor {
+                let Some(anchor) = anchor else {
-                    None => {
+                    log::error!("Invalid indent query: @align requires an accompanying @anchor.");
-                        log::error!(
+                    continue;
-                            "Invalid indent query: @align requires an accompanying @anchor."
-                        );
-                        continue;
-                    }
-                    Some(anchor) => anchor,
                 };
+                let line = text.byte_to_line(anchor.start_byte() as usize);
+                let line_start = text.line_to_byte(line);
                 capture.capture_type = IndentCaptureType::Align(
-                    text.line(anchor.start_position().row)
+                    text.byte_slice(line_start..anchor.start_byte() as usize),
-                        .byte_slice(0..anchor.start_position().column),
                 );
             }
             indent_captures
@@ -694,13 +763,15 @@ fn extend_nodes<'a>(
                         // - the cursor is on the same line as the end of the node OR
                         // - the line that the cursor is on is more indented than the
                         //   first line of the node
-                        if deepest_preceding.end_position().row == line {
+                        if text.byte_to_line(deepest_preceding.end_byte() as usize) == line {
                             extend_node = true;
                         } else {
                             let cursor_indent =
                                 indent_level_for_line(text.line(line), tab_width, indent_width);
                             let node_indent = indent_level_for_line(
-                                text.line(deepest_preceding.start_position().row),
+                                text.line(
+                                    text.byte_to_line(deepest_preceding.start_byte() as usize),
+                                ),
                                 tab_width,
                                 indent_width,
                             );
@@ -717,7 +788,7 @@ fn extend_nodes<'a>(
         if node_captured && stop_extend {
             stop_extend = false;
         } else if extend_node && !stop_extend {
-            *node = deepest_preceding;
+            *node = deepest_preceding.clone();
             break;
         }
         // If the tree contains a syntax error, `deepest_preceding` may not
@@ -734,17 +805,17 @@ fn extend_nodes<'a>(
 /// - The indent captures for all relevant nodes.
 #[allow(clippy::too_many_arguments)]
 fn init_indent_query<'a, 'b>(
-    query: &Query,
+    query: &IndentQuery,
     syntax: &'a Syntax,
     text: RopeSlice<'b>,
     tab_width: usize,
     indent_width: usize,
     line: usize,
-    byte_pos: usize,
+    byte_pos: u32,
-    new_line_byte_pos: Option<usize>,
+    new_line_byte_pos: Option<u32>,
 ) -> Option<(Node<'a>, HashMap<usize, Vec<IndentCapture<'b>>>)> {
     // The innermost tree-sitter node which is considered for the indent
-    // computation. It may change if some predeceding node is extended
+    // computation. It may change if some preceding node is extended
     let mut node = syntax
         .tree()
         .root_node()
@@ -754,37 +825,25 @@ fn init_indent_query<'a, 'b>(
         // The query range should intersect with all nodes directly preceding
         // the position of the indent query in case one of them is extended.
         let mut deepest_preceding = None; // The deepest node preceding the indent query position
-        let mut tree_cursor = node.walk();
+        for child in node.children() {
-        for child in node.children(&mut tree_cursor) {
             if child.byte_range().end <= byte_pos {
-                deepest_preceding = Some(child);
+                deepest_preceding = Some(child.clone());
             }
         }
         deepest_preceding = deepest_preceding.map(|mut prec| {
             // Get the deepest directly preceding node
             while prec.child_count() > 0 {
-                prec = prec.child(prec.child_count() - 1).unwrap();
+                prec = prec.child(prec.child_count() - 1).unwrap().clone();
             }
             prec
         });
         let query_range = deepest_preceding
+            .as_ref()
             .map(|prec| prec.byte_range().end - 1..byte_pos + 1)
             .unwrap_or(byte_pos..byte_pos + 1);
 
-        crate::syntax::PARSER.with(|ts_parser| {
+        let query_result = query_indents(query, syntax, text, query_range, new_line_byte_pos);
-            let mut ts_parser = ts_parser.borrow_mut();
+        (query_result, deepest_preceding)
-            let mut cursor = ts_parser.cursors.pop().unwrap_or_default();
-            let query_result = query_indents(
-                query,
-                syntax,
-                &mut cursor,
-                text,
-                query_range,
-                new_line_byte_pos,
-            );
-            ts_parser.cursors.push(cursor);
-            (query_result, deepest_preceding)
-        })
     };
     let extend_captures = query_result.extend_captures;
 
@@ -842,7 +901,7 @@ fn init_indent_query<'a, 'b>(
 /// ```
 #[allow(clippy::too_many_arguments)]
 pub fn treesitter_indent_for_pos<'a>(
-    query: &Query,
+    query: &IndentQuery,
     syntax: &Syntax,
     tab_width: usize,
     indent_width: usize,
@@ -851,7 +910,7 @@ pub fn treesitter_indent_for_pos<'a>(
     pos: usize,
     new_line: bool,
 ) -> Option<Indentation<'a>> {
-    let byte_pos = text.char_to_byte(pos);
+    let byte_pos = text.char_to_byte(pos) as u32;
     let new_line_byte_pos = new_line.then_some(byte_pos);
     let (mut node, mut indent_captures) = init_indent_query(
         query,
@@ -871,7 +930,7 @@ pub fn treesitter_indent_for_pos<'a>(
     let mut indent_for_line_below = Indentation::default();
 
     loop {
-        let is_first = is_first_in_line(node, text, new_line_byte_pos);
+        let is_first = is_first_in_line(&node, text, new_line_byte_pos);
 
         // Apply all indent definitions for this node.
         // Since we only iterate over each node once, we can remove the
@@ -894,8 +953,8 @@ pub fn treesitter_indent_for_pos<'a>(
         }
 
         if let Some(parent) = node.parent() {
-            let node_line = get_node_start_line(node, new_line_byte_pos);
+            let node_line = get_node_start_line(text, &node, new_line_byte_pos);
-            let parent_line = get_node_start_line(parent, new_line_byte_pos);
+            let parent_line = get_node_start_line(text, &parent, new_line_byte_pos);
 
             if node_line != parent_line {
                 // Don't add indent for the line below the line of the query
@@ -917,8 +976,9 @@ pub fn treesitter_indent_for_pos<'a>(
         } else {
             // Only add the indentation for the line below if that line
             // is not after the line that the indentation is calculated for.
-            if (node.start_position().row < line)
+            let node_start_line = text.byte_to_line(node.start_byte() as usize);
-                || (new_line && node.start_position().row == line && node.start_byte() < byte_pos)
+            if node_start_line < line
+                || (new_line && node_start_line == line && node.start_byte() < byte_pos)
             {
                 result.add_line(indent_for_line_below);
             }
@@ -933,7 +993,7 @@ pub fn treesitter_indent_for_pos<'a>(
 /// This is done either using treesitter, or if that's not available by copying the indentation from the current line
 #[allow(clippy::too_many_arguments)]
 pub fn indent_for_newline(
-    language_config: Option<&LanguageConfiguration>,
+    loader: &syntax::Loader,
     syntax: Option<&Syntax>,
     indent_heuristic: &IndentationHeuristic,
     indent_style: &IndentStyle,
@@ -950,7 +1010,7 @@ pub fn indent_for_newline(
         Some(syntax),
     ) = (
         indent_heuristic,
-        language_config.and_then(|config| config.indent_query()),
+        syntax.and_then(|syntax| loader.indent_query(syntax.root_language())),
         syntax,
     ) {
         if let Some(indent) = treesitter_indent_for_pos(
@@ -1018,10 +1078,10 @@ pub fn indent_for_newline(
     indent_style.as_str().repeat(indent_level)
 }
 
-pub fn get_scopes(syntax: Option<&Syntax>, text: RopeSlice, pos: usize) -> Vec<&'static str> {
+pub fn get_scopes<'a>(syntax: Option<&'a Syntax>, text: RopeSlice, pos: usize) -> Vec<&'a str> {
     let mut scopes = Vec::new();
     if let Some(syntax) = syntax {
-        let pos = text.char_to_byte(pos);
+        let pos = text.char_to_byte(pos) as u32;
         let mut node = match syntax
             .tree()
             .root_node()

helix-core/src/lib.rs

@@ -53,7 +53,7 @@ pub use smartstring::SmartString;
 pub type Tendril = SmartString<smartstring::LazyCompact>;
 
 #[doc(inline)]
-pub use {regex, tree_sitter};
+pub use {regex, tree_house::tree_sitter};
 
 pub use position::{
     char_idx_at_visual_offset, coords_at_pos, pos_at_coords, softwrapped_dimensions,
@@ -73,3 +73,5 @@ pub use line_ending::{LineEnding, NATIVE_LINE_ENDING};
 pub use transaction::{Assoc, Change, ChangeSet, Deletion, Operation, Transaction};
 
 pub use uri::Uri;
+
+pub use tree_house::Language;

helix-core/src/match_brackets.rs

@@ -1,7 +1,7 @@
 use std::iter;
 
+use crate::tree_sitter::Node;
 use ropey::RopeSlice;
-use tree_sitter::Node;
 
 use crate::movement::Direction::{self, Backward, Forward};
 use crate::Syntax;
@@ -75,7 +75,7 @@ fn find_pair(
     pos_: usize,
     traverse_parents: bool,
 ) -> Option<usize> {
-    let pos = doc.char_to_byte(pos_);
+    let pos = doc.char_to_byte(pos_) as u32;
 
     let root = syntax.tree_for_byte_range(pos, pos).root_node();
     let mut node = root.descendant_for_byte_range(pos, pos)?;
@@ -128,7 +128,7 @@ fn find_pair(
                 if find_pair_end(doc, sibling.prev_sibling(), start_char, end_char, Backward)
                     .is_some()
                 {
-                    return doc.try_byte_to_char(sibling.start_byte()).ok();
+                    return doc.try_byte_to_char(sibling.start_byte() as usize).ok();
                 }
             }
         } else if node.is_named() {
@@ -144,9 +144,9 @@ fn find_pair(
     if node.child_count() != 0 {
         return None;
     }
-    let node_start = doc.byte_to_char(node.start_byte());
+    let node_start = doc.byte_to_char(node.start_byte() as usize);
-    find_matching_bracket_plaintext(doc.byte_slice(node.byte_range()), pos_ - node_start)
+    let node_text = doc.byte_slice(node.start_byte() as usize..node.end_byte() as usize);
-        .map(|pos| pos + node_start)
+    find_matching_bracket_plaintext(node_text, pos_ - node_start).map(|pos| pos + node_start)
 }
 
 /// Returns the position of the matching bracket under cursor.
@@ -304,7 +304,7 @@ fn as_char(doc: RopeSlice, node: &Node) -> Option<(usize, char)> {
     if node.byte_range().len() != 1 {
         return None;
     }
-    let pos = doc.try_byte_to_char(node.start_byte()).ok()?;
+    let pos = doc.try_byte_to_char(node.start_byte() as usize).ok()?;
     Some((pos, doc.char(pos)))
 }
 

helix-core/src/movement.rs

@@ -1,7 +1,6 @@
-use std::{cmp::Reverse, iter};
+use std::{borrow::Cow, cmp::Reverse, iter};
 
 use ropey::iter::Chars;
-use tree_sitter::{Node, QueryCursor};
 
 use crate::{
     char_idx_at_visual_offset,
@@ -13,9 +12,10 @@ use crate::{
     },
     line_ending::rope_is_line_ending,
     position::char_idx_at_visual_block_offset,
-    syntax::config::LanguageConfiguration,
+    syntax,
     text_annotations::TextAnnotations,
     textobject::TextObject,
+    tree_sitter::Node,
     visual_offset_from_block, Range, RopeSlice, Selection, Syntax,
 };
 
@@ -560,21 +560,23 @@ fn reached_target(target: WordMotionTarget, prev_ch: char, next_ch: char) -> boo
 
 /// Finds the range of the next or previous textobject in the syntax sub-tree of `node`.
 /// Returns the range in the forwards direction.
+#[allow(clippy::too_many_arguments)]
 pub fn goto_treesitter_object(
     slice: RopeSlice,
     range: Range,
     object_name: &str,
     dir: Direction,
-    slice_tree: Node,
+    slice_tree: &Node,
-    lang_config: &LanguageConfiguration,
+    syntax: &Syntax,
+    loader: &syntax::Loader,
     count: usize,
 ) -> Range {
+    let textobject_query = loader.textobject_query(syntax.root_language());
     let get_range = move |range: Range| -> Option<Range> {
         let byte_pos = slice.char_to_byte(range.cursor(slice));
 
         let cap_name = |t: TextObject| format!("{}.{}", object_name, t);
-        let mut cursor = QueryCursor::new();
+        let nodes = textobject_query?.capture_nodes_any(
-        let nodes = lang_config.textobject_query()?.capture_nodes_any(
             &[
                 &cap_name(TextObject::Movement),
                 &cap_name(TextObject::Around),
@@ -582,7 +584,6 @@ pub fn goto_treesitter_object(
             ],
             slice_tree,
             slice,
-            &mut cursor,
         )?;
 
         let node = match dir {
@@ -617,14 +618,15 @@ pub fn goto_treesitter_object(
     last_range
 }
 
-fn find_parent_start(mut node: Node) -> Option<Node> {
+fn find_parent_start<'tree>(node: &Node<'tree>) -> Option<Node<'tree>> {
     let start = node.start_byte();
+    let mut node = Cow::Borrowed(node);
 
     while node.start_byte() >= start || !node.is_named() {
-        node = node.parent()?;
+        node = Cow::Owned(node.parent()?);
     }
 
-    Some(node)
+    Some(node.into_owned())
 }
 
 pub fn move_parent_node_end(
@@ -635,8 +637,8 @@ pub fn move_parent_node_end(
     movement: Movement,
 ) -> Selection {
     selection.transform(|range| {
-        let start_from = text.char_to_byte(range.from());
+        let start_from = text.char_to_byte(range.from()) as u32;
-        let start_to = text.char_to_byte(range.to());
+        let start_to = text.char_to_byte(range.to()) as u32;
 
         let mut node = match syntax.named_descendant_for_byte_range(start_from, start_to) {
             Some(node) => node,
@@ -654,18 +656,18 @@ pub fn move_parent_node_end(
             // moving forward, we always want to move one past the end of the
             // current node, so use the end byte of the current node, which is an exclusive
             // end of the range
-            Direction::Forward => text.byte_to_char(node.end_byte()),
+            Direction::Forward => text.byte_to_char(node.end_byte() as usize),
 
             // moving backward, we want the cursor to land on the start char of
             // the current node, or if it is already at the start of a node, to traverse up to
             // the parent
             Direction::Backward => {
-                let end_head = text.byte_to_char(node.start_byte());
+                let end_head = text.byte_to_char(node.start_byte() as usize);
 
                 // if we're already on the beginning, look up to the parent
                 if end_head == range.cursor(text) {
-                    node = find_parent_start(node).unwrap_or(node);
+                    node = find_parent_start(&node).unwrap_or(node);
-                    text.byte_to_char(node.start_byte())
+                    text.byte_to_char(node.start_byte() as usize)
                 } else {
                     end_head
                 }

helix-core/src/object.rs

@@ -4,8 +4,8 @@ pub fn expand_selection(syntax: &Syntax, text: RopeSlice, selection: Selection)
     let cursor = &mut syntax.walk();
 
     selection.transform(|range| {
-        let from = text.char_to_byte(range.from());
+        let from = text.char_to_byte(range.from()) as u32;
-        let to = text.char_to_byte(range.to());
+        let to = text.char_to_byte(range.to()) as u32;
 
         let byte_range = from..to;
         cursor.reset_to_byte_range(from, to);
@@ -17,8 +17,8 @@ pub fn expand_selection(syntax: &Syntax, text: RopeSlice, selection: Selection)
         }
 
         let node = cursor.node();
-        let from = text.byte_to_char(node.start_byte());
+        let from = text.byte_to_char(node.start_byte() as usize);
-        let to = text.byte_to_char(node.end_byte());
+        let to = text.byte_to_char(node.end_byte() as usize);
 
         Range::new(to, from).with_direction(range.direction())
     })
@@ -53,10 +53,10 @@ pub fn select_next_sibling(syntax: &Syntax, text: RopeSlice, selection: Selectio
 }
 
 pub fn select_all_siblings(syntax: &Syntax, text: RopeSlice, selection: Selection) -> Selection {
-    selection.transform_iter(|range| {
+    let mut cursor = syntax.walk();
-        let mut cursor = syntax.walk();
+    selection.transform_iter(move |range| {
         let (from, to) = range.into_byte_range(text);
-        cursor.reset_to_byte_range(from, to);
+        cursor.reset_to_byte_range(from as u32, to as u32);
 
         if !cursor.goto_parent_with(|parent| parent.child_count() > 1) {
             return vec![range].into_iter();
@@ -67,21 +67,18 @@ pub fn select_all_siblings(syntax: &Syntax, text: RopeSlice, selection: Selectio
 }
 
 pub fn select_all_children(syntax: &Syntax, text: RopeSlice, selection: Selection) -> Selection {
-    selection.transform_iter(|range| {
+    let mut cursor = syntax.walk();
-        let mut cursor = syntax.walk();
+    selection.transform_iter(move |range| {
         let (from, to) = range.into_byte_range(text);
-        cursor.reset_to_byte_range(from, to);
+        cursor.reset_to_byte_range(from as u32, to as u32);
         select_children(&mut cursor, text, range).into_iter()
     })
 }
 
-fn select_children<'n>(
+fn select_children(cursor: &mut TreeCursor, text: RopeSlice, range: Range) -> Vec<Range> {
-    cursor: &'n mut TreeCursor<'n>,
-    text: RopeSlice,
-    range: Range,
-) -> Vec<Range> {
     let children = cursor
-        .named_children()
+        .children()
+        .filter(|child| child.is_named())
         .map(|child| Range::from_node(child, text, range.direction()))
         .collect::<Vec<_>>();
 
@@ -98,7 +95,7 @@ pub fn select_prev_sibling(syntax: &Syntax, text: RopeSlice, selection: Selectio
         text,
         selection,
         |cursor| {
-            while !cursor.goto_prev_sibling() {
+            while !cursor.goto_previous_sibling() {
                 if !cursor.goto_parent() {
                     break;
                 }
@@ -121,16 +118,16 @@ where
     let cursor = &mut syntax.walk();
 
     selection.transform(|range| {
-        let from = text.char_to_byte(range.from());
+        let from = text.char_to_byte(range.from()) as u32;
-        let to = text.char_to_byte(range.to());
+        let to = text.char_to_byte(range.to()) as u32;
 
         cursor.reset_to_byte_range(from, to);
 
         motion(cursor);
 
         let node = cursor.node();
-        let from = text.byte_to_char(node.start_byte());
+        let from = text.byte_to_char(node.start_byte() as usize);
-        let to = text.byte_to_char(node.end_byte());
+        let to = text.byte_to_char(node.end_byte() as usize);
 
         Range::new(from, to).with_direction(direction.unwrap_or_else(|| range.direction()))
     })

helix-core/src/position.rs

@@ -89,11 +89,6 @@ impl From<(usize, usize)> for Position {
     }
 }
 
-impl From<Position> for tree_sitter::Point {
-    fn from(pos: Position) -> Self {
-        Self::new(pos.row, pos.col)
-    }
-}
 /// Convert a character index to (line, column) coordinates.
 ///
 /// column in `char` count which can be used for row:column display in

helix-core/src/selection.rs

@@ -9,13 +9,13 @@ use crate::{
     },
     line_ending::get_line_ending,
     movement::Direction,
+    tree_sitter::Node,
     Assoc, ChangeSet, RopeSlice,
 };
 use helix_stdx::range::is_subset;
 use helix_stdx::rope::{self, RopeSliceExt};
 use smallvec::{smallvec, SmallVec};
 use std::{borrow::Cow, iter, slice};
-use tree_sitter::Node;
 
 /// A single selection range.
 ///
@@ -76,8 +76,8 @@ impl Range {
     }
 
     pub fn from_node(node: Node, text: RopeSlice, direction: Direction) -> Self {
-        let from = text.byte_to_char(node.start_byte());
+        let from = text.byte_to_char(node.start_byte() as usize);
-        let to = text.byte_to_char(node.end_byte());
+        let to = text.byte_to_char(node.end_byte() as usize);
         Range::new(from, to).with_direction(direction)
     }
 

helix-core/src/snippets/active.rs

@@ -1,6 +1,6 @@
 use std::ops::{Index, IndexMut};
 
-use hashbrown::HashSet;
+use foldhash::HashSet;
 use helix_stdx::range::{is_exact_subset, is_subset};
 use helix_stdx::Range;
 use ropey::Rope;
@@ -35,7 +35,7 @@ impl ActiveSnippet {
         let snippet = Self {
             ranges: snippet.ranges,
             tabstops: snippet.tabstops,
-            active_tabstops: HashSet::new(),
+            active_tabstops: HashSet::default(),
             current_tabstop: TabstopIdx(0),
         };
         (snippet.tabstops.len() != 1).then_some(snippet)

helix-core/src/syntax/config.rs

@@ -1,8 +1,7 @@
-use crate::{auto_pairs::AutoPairs, diagnostic::Severity};
+use crate::{auto_pairs::AutoPairs, diagnostic::Severity, Language};
 
 use globset::GlobSet;
 use helix_stdx::rope;
-use once_cell::sync::OnceCell;
 use serde::{ser::SerializeSeq as _, Deserialize, Serialize};
 
 use std::{
@@ -10,7 +9,6 @@ use std::{
     fmt::{self, Display},
     path::PathBuf,
     str::FromStr,
-    sync::Arc,
 };
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -24,6 +22,9 @@ pub struct Configuration {
 #[derive(Debug, Serialize, Deserialize)]
 #[serde(rename_all = "kebab-case", deny_unknown_fields)]
 pub struct LanguageConfiguration {
+    #[serde(skip)]
+    pub(super) language: Option<Language>,
+
     #[serde(rename = "name")]
     pub language_id: String, // c-sharp, rust, tsx
     #[serde(rename = "language-id")]
@@ -70,9 +71,6 @@ pub struct LanguageConfiguration {
     pub injection_regex: Option<rope::Regex>,
     // first_line_regex
     //
-    #[serde(skip)]
-    pub(crate) highlight_config: OnceCell<Option<Arc<super::HighlightConfiguration>>>,
-    // tags_config OnceCell<> https://github.com/tree-sitter/tree-sitter/pull/583
     #[serde(
         default,
         skip_serializing_if = "Vec::is_empty",
@@ -83,10 +81,6 @@ pub struct LanguageConfiguration {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub indent: Option<IndentationConfiguration>,
 
-    #[serde(skip)]
-    pub(crate) indent_query: OnceCell<Option<tree_sitter::Query>>,
-    #[serde(skip)]
-    pub(crate) textobject_query: OnceCell<Option<super::TextObjectQuery>>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub debugger: Option<DebugAdapterConfig>,
 
@@ -106,6 +100,13 @@ pub struct LanguageConfiguration {
     pub persistent_diagnostic_sources: Vec<String>,
 }
 
+impl LanguageConfiguration {
+    pub fn language(&self) -> Language {
+        // This value must be set by `super::Loader::new`.
+        self.language.unwrap()
+    }
+}
+
 #[derive(Debug, PartialEq, Eq, Hash)]
 pub enum FileType {
     /// The extension of the file, either the `Path::extension` or the full

helix-core/src/syntax/tree_cursor.rs

@@ -1,264 +0,0 @@
-use std::{cmp::Reverse, ops::Range};
-
-use super::{LanguageLayer, LayerId};
-
-use slotmap::HopSlotMap;
-use tree_sitter::Node;
-
-/// The byte range of an injection layer.
-///
-/// Injection ranges may overlap, but all overlapping parts are subsets of their parent ranges.
-/// This allows us to sort the ranges ahead of time in order to efficiently find a range that
-/// contains a point with maximum depth.
-#[derive(Debug)]
-struct InjectionRange {
-    start: usize,
-    end: usize,
-    layer_id: LayerId,
-    depth: u32,
-}
-
-pub struct TreeCursor<'a> {
-    layers: &'a HopSlotMap<LayerId, LanguageLayer>,
-    root: LayerId,
-    current: LayerId,
-    injection_ranges: Vec<InjectionRange>,
-    // TODO: Ideally this would be a `tree_sitter::TreeCursor<'a>` but
-    // that returns very surprising results in testing.
-    cursor: Node<'a>,
-}
-
-impl<'a> TreeCursor<'a> {
-    pub(super) fn new(layers: &'a HopSlotMap<LayerId, LanguageLayer>, root: LayerId) -> Self {
-        let mut injection_ranges = Vec::new();
-
-        for (layer_id, layer) in layers.iter() {
-            // Skip the root layer
-            if layer.parent.is_none() {
-                continue;
-            }
-            for byte_range in layer.ranges.iter() {
-                let range = InjectionRange {
-                    start: byte_range.start_byte,
-                    end: byte_range.end_byte,
-                    layer_id,
-                    depth: layer.depth,
-                };
-                injection_ranges.push(range);
-            }
-        }
-
-        injection_ranges.sort_unstable_by_key(|range| (range.end, Reverse(range.depth)));
-
-        let cursor = layers[root].tree().root_node();
-
-        Self {
-            layers,
-            root,
-            current: root,
-            injection_ranges,
-            cursor,
-        }
-    }
-
-    pub fn node(&self) -> Node<'a> {
-        self.cursor
-    }
-
-    pub fn goto_parent(&mut self) -> bool {
-        if let Some(parent) = self.node().parent() {
-            self.cursor = parent;
-            return true;
-        }
-
-        // If we are already on the root layer, we cannot ascend.
-        if self.current == self.root {
-            return false;
-        }
-
-        // Ascend to the parent layer.
-        let range = self.node().byte_range();
-        let parent_id = self.layers[self.current]
-            .parent
-            .expect("non-root layers have a parent");
-        self.current = parent_id;
-        let root = self.layers[self.current].tree().root_node();
-        self.cursor = root
-            .descendant_for_byte_range(range.start, range.end)
-            .unwrap_or(root);
-
-        true
-    }
-
-    pub fn goto_parent_with<P>(&mut self, predicate: P) -> bool
-    where
-        P: Fn(&Node) -> bool,
-    {
-        while self.goto_parent() {
-            if predicate(&self.node()) {
-                return true;
-            }
-        }
-
-        false
-    }
-
-    /// Finds the injection layer that has exactly the same range as the given `range`.
-    fn layer_id_of_byte_range(&self, search_range: Range<usize>) -> Option<LayerId> {
-        let start_idx = self
-            .injection_ranges
-            .partition_point(|range| range.end < search_range.end);
-
-        self.injection_ranges[start_idx..]
-            .iter()
-            .take_while(|range| range.end == search_range.end)
-            .find_map(|range| (range.start == search_range.start).then_some(range.layer_id))
-    }
-
-    fn goto_first_child_impl(&mut self, named: bool) -> bool {
-        // Check if the current node's range is an exact injection layer range.
-        if let Some(layer_id) = self
-            .layer_id_of_byte_range(self.node().byte_range())
-            .filter(|&layer_id| layer_id != self.current)
-        {
-            // Switch to the child layer.
-            self.current = layer_id;
-            self.cursor = self.layers[self.current].tree().root_node();
-            return true;
-        }
-
-        let child = if named {
-            self.cursor.named_child(0)
-        } else {
-            self.cursor.child(0)
-        };
-
-        if let Some(child) = child {
-            // Otherwise descend in the current tree.
-            self.cursor = child;
-            true
-        } else {
-            false
-        }
-    }
-
-    pub fn goto_first_child(&mut self) -> bool {
-        self.goto_first_child_impl(false)
-    }
-
-    pub fn goto_first_named_child(&mut self) -> bool {
-        self.goto_first_child_impl(true)
-    }
-
-    fn goto_next_sibling_impl(&mut self, named: bool) -> bool {
-        let sibling = if named {
-            self.cursor.next_named_sibling()
-        } else {
-            self.cursor.next_sibling()
-        };
-
-        if let Some(sibling) = sibling {
-            self.cursor = sibling;
-            true
-        } else {
-            false
-        }
-    }
-
-    pub fn goto_next_sibling(&mut self) -> bool {
-        self.goto_next_sibling_impl(false)
-    }
-
-    pub fn goto_next_named_sibling(&mut self) -> bool {
-        self.goto_next_sibling_impl(true)
-    }
-
-    fn goto_prev_sibling_impl(&mut self, named: bool) -> bool {
-        let sibling = if named {
-            self.cursor.prev_named_sibling()
-        } else {
-            self.cursor.prev_sibling()
-        };
-
-        if let Some(sibling) = sibling {
-            self.cursor = sibling;
-            true
-        } else {
-            false
-        }
-    }
-
-    pub fn goto_prev_sibling(&mut self) -> bool {
-        self.goto_prev_sibling_impl(false)
-    }
-
-    pub fn goto_prev_named_sibling(&mut self) -> bool {
-        self.goto_prev_sibling_impl(true)
-    }
-
-    /// Finds the injection layer that contains the given start-end range.
-    fn layer_id_containing_byte_range(&self, start: usize, end: usize) -> LayerId {
-        let start_idx = self
-            .injection_ranges
-            .partition_point(|range| range.end < end);
-
-        self.injection_ranges[start_idx..]
-            .iter()
-            .take_while(|range| range.start < end || range.depth > 1)
-            .find_map(|range| (range.start <= start).then_some(range.layer_id))
-            .unwrap_or(self.root)
-    }
-
-    pub fn reset_to_byte_range(&mut self, start: usize, end: usize) {
-        self.current = self.layer_id_containing_byte_range(start, end);
-        let root = self.layers[self.current].tree().root_node();
-        self.cursor = root.descendant_for_byte_range(start, end).unwrap_or(root);
-    }
-
-    /// Returns an iterator over the children of the node the TreeCursor is on
-    /// at the time this is called.
-    pub fn children(&'a mut self) -> ChildIter<'a> {
-        let parent = self.node();
-
-        ChildIter {
-            cursor: self,
-            parent,
-            named: false,
-        }
-    }
-
-    /// Returns an iterator over the named children of the node the TreeCursor is on
-    /// at the time this is called.
-    pub fn named_children(&'a mut self) -> ChildIter<'a> {
-        let parent = self.node();
-
-        ChildIter {
-            cursor: self,
-            parent,
-            named: true,
-        }
-    }
-}
-
-pub struct ChildIter<'n> {
-    cursor: &'n mut TreeCursor<'n>,
-    parent: Node<'n>,
-    named: bool,
-}
-
-impl<'n> Iterator for ChildIter<'n> {
-    type Item = Node<'n>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        // first iteration, just visit the first child
-        if self.cursor.node() == self.parent {
-            self.cursor
-                .goto_first_child_impl(self.named)
-                .then(|| self.cursor.node())
-        } else {
-            self.cursor
-                .goto_next_sibling_impl(self.named)
-                .then(|| self.cursor.node())
-        }
-    }
-}

helix-core/src/text_annotations.rs

@@ -5,7 +5,7 @@ use std::ops::Range;
 use std::ptr::NonNull;
 
 use crate::doc_formatter::FormattedGrapheme;
-use crate::syntax::Highlight;
+use crate::syntax::{Highlight, OverlayHighlights};
 use crate::{Position, Tendril};
 
 /// An inline annotation is continuous text shown
@@ -300,10 +300,7 @@ impl<'a> TextAnnotations<'a> {
         }
     }
 
-    pub fn collect_overlay_highlights(
+    pub fn collect_overlay_highlights(&self, char_range: Range<usize>) -> OverlayHighlights {
-        &self,
-        char_range: Range<usize>,
-    ) -> Vec<(usize, Range<usize>)> {
         let mut highlights = Vec::new();
         self.reset_pos(char_range.start);
         for char_idx in char_range {
@@ -311,11 +308,11 @@ impl<'a> TextAnnotations<'a> {
                 // we don't know the number of chars the original grapheme takes
                 // however it doesn't matter as highlight boundaries are automatically
                 // aligned to grapheme boundaries in the rendering code
-                highlights.push((highlight.0, char_idx..char_idx + 1))
+                highlights.push((highlight, char_idx..char_idx + 1));
             }
         }
 
-        highlights
+        OverlayHighlights::Heterogenous { highlights }
     }
 
     /// Add new inline annotations.

helix-core/src/textobject.rs

@@ -1,13 +1,12 @@
 use std::fmt::Display;
 
 use ropey::RopeSlice;
-use tree_sitter::{Node, QueryCursor};
 
 use crate::chars::{categorize_char, char_is_whitespace, CharCategory};
 use crate::graphemes::{next_grapheme_boundary, prev_grapheme_boundary};
 use crate::line_ending::rope_is_line_ending;
 use crate::movement::Direction;
-use crate::syntax::config::LanguageConfiguration;
+use crate::syntax;
 use crate::Range;
 use crate::{surround, Syntax};
 
@@ -260,18 +259,18 @@ pub fn textobject_treesitter(
     range: Range,
     textobject: TextObject,
     object_name: &str,
-    slice_tree: Node,
+    syntax: &Syntax,
-    lang_config: &LanguageConfiguration,
+    loader: &syntax::Loader,
     _count: usize,
 ) -> Range {
+    let root = syntax.tree().root_node();
+    let textobject_query = loader.textobject_query(syntax.root_language());
     let get_range = move || -> Option<Range> {
         let byte_pos = slice.char_to_byte(range.cursor(slice));
 
         let capture_name = format!("{}.{}", object_name, textobject); // eg. function.inner
-        let mut cursor = QueryCursor::new();
+        let node = textobject_query?
-        let node = lang_config
+            .capture_nodes(&capture_name, &root, slice)?
-            .textobject_query()?
-            .capture_nodes(&capture_name, slice_tree, slice, &mut cursor)?
             .filter(|node| node.byte_range().contains(&byte_pos))
             .min_by_key(|node| node.byte_range().len())?;
 

helix-core/tests/indent.rs

@@ -1,4 +1,3 @@
-use arc_swap::ArcSwap;
 use helix_core::{
     indent::{indent_level_for_line, treesitter_indent_for_pos, IndentStyle},
     syntax::{config::Configuration, Loader},
@@ -6,7 +5,7 @@ use helix_core::{
 };
 use helix_stdx::rope::RopeSliceExt;
 use ropey::Rope;
-use std::{ops::Range, path::PathBuf, process::Command, sync::Arc};
+use std::{ops::Range, path::PathBuf, process::Command};
 
 #[test]
 fn test_treesitter_indent_rust() {
@@ -196,17 +195,12 @@ fn test_treesitter_indent(
     runtime.push("../runtime");
     std::env::set_var("HELIX_RUNTIME", runtime.to_str().unwrap());
 
-    let language_config = loader.language_config_for_scope(lang_scope).unwrap();
+    let language = loader.language_for_scope(lang_scope).unwrap();
+    let language_config = loader.language(language).config();
     let indent_style = IndentStyle::from_str(&language_config.indent.as_ref().unwrap().unit);
-    let highlight_config = language_config.highlight_config(&[]).unwrap();
     let text = doc.slice(..);
-    let syntax = Syntax::new(
+    let syntax = Syntax::new(text, language, &loader).unwrap();
-        text,
+    let indent_query = loader.indent_query(language).unwrap();
-        highlight_config,
-        Arc::new(ArcSwap::from_pointee(loader)),
-    )
-    .unwrap();
-    let indent_query = language_config.indent_query().unwrap();
 
     for i in 0..doc.len_lines() {
         let line = text.line(i);

helix-loader/Cargo.toml

@@ -21,7 +21,6 @@ anyhow = "1"
 serde = { version = "1.0", features = ["derive"] }
 toml = "0.8"
 etcetera = "0.10"
-tree-sitter.workspace = true
 once_cell = "1.21"
 log = "0.4"
 
@@ -32,5 +31,4 @@ cc = { version = "1" }
 threadpool = { version = "1.0" }
 tempfile.workspace = true
 
-[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+tree-house.workspace = true
-libloading = "0.8"

helix-loader/src/grammar.rs

@@ -9,7 +9,7 @@ use std::{
     sync::mpsc::channel,
 };
 use tempfile::TempPath;
-use tree_sitter::Language;
+use tree_house::tree_sitter::Grammar;
 
 #[cfg(unix)]
 const DYLIB_EXTENSION: &str = "so";
@@ -61,28 +61,21 @@ const BUILD_TARGET: &str = env!("BUILD_TARGET");
 const REMOTE_NAME: &str = "origin";
 
 #[cfg(target_arch = "wasm32")]
-pub fn get_language(name: &str) -> Result<Language> {
+pub fn get_language(name: &str) -> Result<Option<Grammar>> {
     unimplemented!()
 }
 
 #[cfg(not(target_arch = "wasm32"))]
-pub fn get_language(name: &str) -> Result<Language> {
+pub fn get_language(name: &str) -> Result<Option<Grammar>> {
-    use libloading::{Library, Symbol};
     let mut rel_library_path = PathBuf::new().join("grammars").join(name);
     rel_library_path.set_extension(DYLIB_EXTENSION);
     let library_path = crate::runtime_file(&rel_library_path);
+    if !library_path.exists() {
+        return Ok(None);
+    }
 
-    let library = unsafe { Library::new(&library_path) }
+    let grammar = unsafe { Grammar::new(name, &library_path) }?;
-        .with_context(|| format!("Error opening dynamic library {:?}", library_path))?;
+    Ok(Some(grammar))
-    let language_fn_name = format!("tree_sitter_{}", name.replace('-', "_"));
-    let language = unsafe {
-        let language_fn: Symbol<unsafe extern "C" fn() -> Language> = library
-            .get(language_fn_name.as_bytes())
-            .with_context(|| format!("Failed to load symbol {}", language_fn_name))?;
-        language_fn()
-    };
-    std::mem::forget(library);
-    Ok(language)
 }
 
 fn ensure_git_is_available() -> Result<()> {

helix-term/src/commands.rs

@@ -3482,12 +3482,12 @@ fn insert_with_indent(cx: &mut Context, cursor_fallback: IndentFallbackPos) {
     enter_insert_mode(cx);
 
     let (view, doc) = current!(cx.editor);
+    let loader = cx.editor.syn_loader.load();
 
     let text = doc.text().slice(..);
     let contents = doc.text();
     let selection = doc.selection(view.id);
 
-    let language_config = doc.language_config();
     let syntax = doc.syntax();
     let tab_width = doc.tab_width();
 
@@ -3503,7 +3503,7 @@ fn insert_with_indent(cx: &mut Context, cursor_fallback: IndentFallbackPos) {
             let line_end_index = cursor_line_start;
 
             let indent = indent::indent_for_newline(
-                language_config,
+                &loader,
                 syntax,
                 &doc.config.load().indent_heuristic,
                 &doc.indent_style,
@@ -3613,6 +3613,7 @@ fn open(cx: &mut Context, open: Open, comment_continuation: CommentContinuation)
     enter_insert_mode(cx);
     let config = cx.editor.config();
     let (view, doc) = current!(cx.editor);
+    let loader = cx.editor.syn_loader.load();
 
     let text = doc.text().slice(..);
     let contents = doc.text();
@@ -3662,7 +3663,7 @@ fn open(cx: &mut Context, open: Open, comment_continuation: CommentContinuation)
         let indent = match line.first_non_whitespace_char() {
             Some(pos) if continue_comment_token.is_some() => line.slice(..pos).to_string(),
             _ => indent::indent_for_newline(
-                doc.language_config(),
+                &loader,
                 doc.syntax(),
                 &config.indent_heuristic,
                 &doc.indent_style,
@@ -4126,6 +4127,7 @@ pub mod insert {
     pub fn insert_newline(cx: &mut Context) {
         let config = cx.editor.config();
         let (view, doc) = current_ref!(cx.editor);
+        let loader = cx.editor.syn_loader.load();
         let text = doc.text().slice(..);
         let line_ending = doc.line_ending.as_str();
 
@@ -4171,7 +4173,7 @@ pub mod insert {
                 let indent = match line.first_non_whitespace_char() {
                     Some(pos) if continue_comment_token.is_some() => line.slice(..pos).to_string(),
                     _ => indent::indent_for_newline(
-                        doc.language_config(),
+                        &loader,
                         doc.syntax(),
                         &config.indent_heuristic,
                         &doc.indent_style,
@@ -5728,19 +5730,14 @@ fn goto_ts_object_impl(cx: &mut Context, object: &'static str, direction: Direct
     let count = cx.count();
     let motion = move |editor: &mut Editor| {
         let (view, doc) = current!(editor);
-        if let Some((lang_config, syntax)) = doc.language_config().zip(doc.syntax()) {
+        let loader = editor.syn_loader.load();
+        if let Some(syntax) = doc.syntax() {
             let text = doc.text().slice(..);
             let root = syntax.tree().root_node();
 
             let selection = doc.selection(view.id).clone().transform(|range| {
                 let new_range = movement::goto_treesitter_object(
-                    text,
+                    text, range, object, direction, &root, syntax, &loader, count,
-                    range,
-                    object,
-                    direction,
-                    root,
-                    lang_config,
-                    count,
                 );
 
                 if editor.mode == Mode::Select {
@@ -5828,21 +5825,15 @@ fn select_textobject(cx: &mut Context, objtype: textobject::TextObject) {
         if let Some(ch) = event.char() {
             let textobject = move |editor: &mut Editor| {
                 let (view, doc) = current!(editor);
+                let loader = editor.syn_loader.load();
                 let text = doc.text().slice(..);
 
                 let textobject_treesitter = |obj_name: &str, range: Range| -> Range {
-                    let (lang_config, syntax) = match doc.language_config().zip(doc.syntax()) {
+                    let Some(syntax) = doc.syntax() else {
-                        Some(t) => t,
+                        return range;
-                        None => return range,
                     };
                     textobject::textobject_treesitter(
-                        text,
+                        text, range, objtype, obj_name, syntax, &loader, count,
-                        range,
-                        objtype,
-                        obj_name,
-                        syntax.tree().root_node(),
-                        lang_config,
-                        count,
                     )
                 };
 

helix-term/src/commands/typed.rs

@@ -1670,16 +1670,14 @@ fn tree_sitter_highlight_name(
     _args: Args,
     event: PromptEvent,
 ) -> anyhow::Result<()> {
-    fn find_highlight_at_cursor(
+    use helix_core::syntax::Highlight;
-        cx: &mut compositor::Context<'_>,
-    ) -> Option<helix_core::syntax::Highlight> {
-        use helix_core::syntax::HighlightEvent;
 
-        let (view, doc) = current!(cx.editor);
+    fn find_highlight_at_cursor(editor: &Editor) -> Option<Highlight> {
+        let (view, doc) = current_ref!(editor);
         let syntax = doc.syntax()?;
         let text = doc.text().slice(..);
         let cursor = doc.selection(view.id).primary().cursor(text);
-        let byte = text.char_to_byte(cursor);
+        let byte = text.char_to_byte(cursor) as u32;
         let node = syntax.descendant_for_byte_range(byte, byte)?;
         // Query the same range as the one used in syntax highlighting.
         let range = {
@@ -1689,25 +1687,22 @@ fn tree_sitter_highlight_name(
             let last_line = text.len_lines().saturating_sub(1);
             let height = view.inner_area(doc).height;
             let last_visible_line = (row + height as usize).saturating_sub(1).min(last_line);
-            let start = text.line_to_byte(row.min(last_line));
+            let start = text.line_to_byte(row.min(last_line)) as u32;
-            let end = text.line_to_byte(last_visible_line + 1);
+            let end = text.line_to_byte(last_visible_line + 1) as u32;
 
             start..end
         };
 
-        let mut highlight = None;
+        let loader = editor.syn_loader.load();
+        let mut highlighter = syntax.highlighter(text, &loader, range);
 
-        for event in syntax.highlight_iter(text, Some(range), None) {
+        while highlighter.next_event_offset() != u32::MAX {
-            match event.unwrap() {
+            let start = highlighter.next_event_offset();
-                HighlightEvent::Source { start, end }
+            highlighter.advance();
-                    if start == node.start_byte() && end == node.end_byte() =>
+            let end = highlighter.next_event_offset();
-                {
+
-                    return highlight;
+            if start <= node.start_byte() && end >= node.end_byte() {
-                }
+                return highlighter.active_highlights().next_back();
-                HighlightEvent::HighlightStart(hl) => {
-                    highlight = Some(hl);
-                }
-                _ => (),
             }
         }
 
@@ -1718,11 +1713,11 @@ fn tree_sitter_highlight_name(
         return Ok(());
     }
 
-    let Some(highlight) = find_highlight_at_cursor(cx) else {
+    let Some(highlight) = find_highlight_at_cursor(cx.editor) else {
         return Ok(());
     };
 
-    let content = cx.editor.theme.scope(highlight.0).to_string();
+    let content = cx.editor.theme.scope(highlight).to_string();
 
     let callback = async move {
         let call: job::Callback = Callback::EditorCompositor(Box::new(
@@ -2190,8 +2185,8 @@ fn tree_sitter_subtree(
     if let Some(syntax) = doc.syntax() {
         let primary_selection = doc.selection(view.id).primary();
         let text = doc.text();
-        let from = text.char_to_byte(primary_selection.from());
+        let from = text.char_to_byte(primary_selection.from()) as u32;
-        let to = text.char_to_byte(primary_selection.to());
+        let to = text.char_to_byte(primary_selection.to()) as u32;
         if let Some(selected_node) = syntax.descendant_for_byte_range(from, to) {
             let mut contents = String::from("```tsq\n");
             helix_core::syntax::pretty_print_tree(&mut contents, selected_node)?;

helix-term/src/ui/document.rs

@@ -3,8 +3,7 @@ use std::cmp::min;
 use helix_core::doc_formatter::{DocumentFormatter, GraphemeSource, TextFormat};
 use helix_core::graphemes::Grapheme;
 use helix_core::str_utils::char_to_byte_idx;
-use helix_core::syntax::Highlight;
+use helix_core::syntax::{self, HighlightEvent, Highlighter, OverlayHighlights};
-use helix_core::syntax::HighlightEvent;
 use helix_core::text_annotations::TextAnnotations;
 use helix_core::{visual_offset_from_block, Position, RopeSlice};
 use helix_stdx::rope::RopeSliceExt;
@@ -17,61 +16,6 @@ use tui::buffer::Buffer as Surface;
 
 use crate::ui::text_decorations::DecorationManager;
 
-#[derive(Debug, PartialEq, Eq, Clone, Copy)]
-enum StyleIterKind {
-    /// base highlights (usually emitted by TS), byte indices (potentially not codepoint aligned)
-    BaseHighlights,
-    /// overlay highlights (emitted by custom code from selections), char indices
-    Overlay,
-}
-
-/// A wrapper around a HighlightIterator
-/// that merges the layered highlights to create the final text style
-/// and yields the active text style and the char_idx where the active
-/// style will have to be recomputed.
-///
-/// TODO(ropey2): hopefully one day helix and ropey will operate entirely
-/// on byte ranges and we can remove this
-struct StyleIter<'a, H: Iterator<Item = HighlightEvent>> {
-    text_style: Style,
-    active_highlights: Vec<Highlight>,
-    highlight_iter: H,
-    kind: StyleIterKind,
-    text: RopeSlice<'a>,
-    theme: &'a Theme,
-}
-
-impl<H: Iterator<Item = HighlightEvent>> Iterator for StyleIter<'_, H> {
-    type Item = (Style, usize);
-    fn next(&mut self) -> Option<(Style, usize)> {
-        while let Some(event) = self.highlight_iter.next() {
-            match event {
-                HighlightEvent::HighlightStart(highlights) => {
-                    self.active_highlights.push(highlights)
-                }
-                HighlightEvent::HighlightEnd => {
-                    self.active_highlights.pop();
-                }
-                HighlightEvent::Source { mut end, .. } => {
-                    let style = self
-                        .active_highlights
-                        .iter()
-                        .fold(self.text_style, |acc, span| {
-                            acc.patch(self.theme.highlight(span.0))
-                        });
-                    if self.kind == StyleIterKind::BaseHighlights {
-                        // Move the end byte index to the nearest character boundary (rounding up)
-                        // and convert it to a character index.
-                        end = self.text.byte_to_char(self.text.ceil_char_boundary(end));
-                    }
-                    return Some((style, end));
-                }
-            }
-        }
-        None
-    }
-}
-
 #[derive(Debug, PartialEq, Eq, Copy, Clone)]
 pub struct LinePos {
     /// Indicates whether the given visual line
@@ -90,8 +34,8 @@ pub fn render_document(
     doc: &Document,
     offset: ViewPosition,
     doc_annotations: &TextAnnotations,
-    syntax_highlight_iter: impl Iterator<Item = HighlightEvent>,
+    syntax_highlighter: Option<Highlighter<'_>>,
-    overlay_highlight_iter: impl Iterator<Item = HighlightEvent>,
+    overlay_highlights: Vec<syntax::OverlayHighlights>,
     theme: &Theme,
     decorations: DecorationManager,
 ) {
@@ -108,8 +52,8 @@ pub fn render_document(
         offset.anchor,
         &doc.text_format(viewport.width, Some(theme)),
         doc_annotations,
-        syntax_highlight_iter,
+        syntax_highlighter,
-        overlay_highlight_iter,
+        overlay_highlights,
         theme,
         decorations,
     )
@@ -122,8 +66,8 @@ pub fn render_text(
     anchor: usize,
     text_fmt: &TextFormat,
     text_annotations: &TextAnnotations,
-    syntax_highlight_iter: impl Iterator<Item = HighlightEvent>,
+    syntax_highlighter: Option<Highlighter<'_>>,
-    overlay_highlight_iter: impl Iterator<Item = HighlightEvent>,
+    overlay_highlights: Vec<syntax::OverlayHighlights>,
     theme: &Theme,
     mut decorations: DecorationManager,
 ) {
@@ -133,22 +77,8 @@ pub fn render_text(
 
     let mut formatter =
         DocumentFormatter::new_at_prev_checkpoint(text, text_fmt, text_annotations, anchor);
-    let mut syntax_styles = StyleIter {
+    let mut syntax_highlighter = SyntaxHighlighter::new(syntax_highlighter, text, theme);
-        text_style: renderer.text_style,
+    let mut overlay_highlighter = OverlayHighlighter::new(overlay_highlights, theme);
-        active_highlights: Vec::with_capacity(64),
-        highlight_iter: syntax_highlight_iter,
-        kind: StyleIterKind::BaseHighlights,
-        theme,
-        text,
-    };
-    let mut overlay_styles = StyleIter {
-        text_style: Style::default(),
-        active_highlights: Vec::with_capacity(64),
-        highlight_iter: overlay_highlight_iter,
-        kind: StyleIterKind::Overlay,
-        theme,
-        text,
-    };
 
     let mut last_line_pos = LinePos {
         first_visual_line: false,
@@ -158,12 +88,6 @@ pub fn render_text(
     let mut last_line_end = 0;
     let mut is_in_indent_area = true;
     let mut last_line_indent_level = 0;
-    let mut syntax_style_span = syntax_styles
-        .next()
-        .unwrap_or_else(|| (Style::default(), usize::MAX));
-    let mut overlay_style_span = overlay_styles
-        .next()
-        .unwrap_or_else(|| (Style::default(), usize::MAX));
     let mut reached_view_top = false;
 
     loop {
@@ -207,21 +131,17 @@ pub fn render_text(
         }
 
         // acquire the correct grapheme style
-        while grapheme.char_idx >= syntax_style_span.1 {
+        while grapheme.char_idx >= syntax_highlighter.pos {
-            syntax_style_span = syntax_styles
+            syntax_highlighter.advance();
-                .next()
-                .unwrap_or((Style::default(), usize::MAX));
         }
-        while grapheme.char_idx >= overlay_style_span.1 {
+        while grapheme.char_idx >= overlay_highlighter.pos {
-            overlay_style_span = overlay_styles
+            overlay_highlighter.advance();
-                .next()
-                .unwrap_or((Style::default(), usize::MAX));
         }
 
         let grapheme_style = if let GraphemeSource::VirtualText { highlight } = grapheme.source {
             let mut style = renderer.text_style;
             if let Some(highlight) = highlight {
-                style = style.patch(theme.highlight(highlight.0));
+                style = style.patch(theme.highlight(highlight));
             }
             GraphemeStyle {
                 syntax_style: style,
@@ -229,8 +149,8 @@ pub fn render_text(
             }
         } else {
             GraphemeStyle {
-                syntax_style: syntax_style_span.0,
+                syntax_style: syntax_highlighter.style,
-                overlay_style: overlay_style_span.0,
+                overlay_style: overlay_highlighter.style,
             }
         };
         decorations.decorate_grapheme(renderer, &grapheme);
@@ -549,3 +469,98 @@ impl<'a> TextRenderer<'a> {
         )
     }
 }
+
+struct SyntaxHighlighter<'h, 'r, 't> {
+    inner: Option<Highlighter<'h>>,
+    text: RopeSlice<'r>,
+    /// The character index of the next highlight event, or `usize::MAX` if the highlighter is
+    /// finished.
+    pos: usize,
+    theme: &'t Theme,
+    style: Style,
+}
+
+impl<'h, 'r, 't> SyntaxHighlighter<'h, 'r, 't> {
+    fn new(inner: Option<Highlighter<'h>>, text: RopeSlice<'r>, theme: &'t Theme) -> Self {
+        let mut highlighter = Self {
+            inner,
+            text,
+            pos: 0,
+            theme,
+            style: Style::default(),
+        };
+        highlighter.update_pos();
+        highlighter
+    }
+
+    fn update_pos(&mut self) {
+        self.pos = self
+            .inner
+            .as_ref()
+            .and_then(|highlighter| {
+                let next_byte_idx = highlighter.next_event_offset();
+                (next_byte_idx != u32::MAX).then(|| {
+                    // Move the byte index to the nearest character boundary (rounding up) and
+                    // convert it to a character index.
+                    self.text
+                        .byte_to_char(self.text.ceil_char_boundary(next_byte_idx as usize))
+                })
+            })
+            .unwrap_or(usize::MAX);
+    }
+
+    fn advance(&mut self) {
+        let Some(highlighter) = self.inner.as_mut() else {
+            return;
+        };
+
+        let (event, highlights) = highlighter.advance();
+        let base = match event {
+            HighlightEvent::Refresh => Style::default(),
+            HighlightEvent::Push => self.style,
+        };
+
+        self.style = highlights.fold(base, |acc, highlight| {
+            acc.patch(self.theme.highlight(highlight))
+        });
+        self.update_pos();
+    }
+}
+
+struct OverlayHighlighter<'t> {
+    inner: syntax::OverlayHighlighter,
+    pos: usize,
+    theme: &'t Theme,
+    style: Style,
+}
+
+impl<'t> OverlayHighlighter<'t> {
+    fn new(overlays: Vec<OverlayHighlights>, theme: &'t Theme) -> Self {
+        let inner = syntax::OverlayHighlighter::new(overlays);
+        let mut highlighter = Self {
+            inner,
+            pos: 0,
+            theme,
+            style: Style::default(),
+        };
+        highlighter.update_pos();
+        highlighter
+    }
+
+    fn update_pos(&mut self) {
+        self.pos = self.inner.next_event_offset();
+    }
+
+    fn advance(&mut self) {
+        let (event, highlights) = self.inner.advance();
+        let base = match event {
+            HighlightEvent::Refresh => Style::default(),
+            HighlightEvent::Push => self.style,
+        };
+
+        self.style = highlights.fold(base, |acc, highlight| {
+            acc.patch(self.theme.highlight(highlight))
+        });
+        self.update_pos();
+    }
+}

helix-term/src/ui/editor.rs

@@ -17,7 +17,7 @@ use helix_core::{
     diagnostic::NumberOrString,
     graphemes::{next_grapheme_boundary, prev_grapheme_boundary},
     movement::Direction,
-    syntax::{self, HighlightEvent},
+    syntax::{self, OverlayHighlights},
     text_annotations::TextAnnotations,
     unicode::width::UnicodeWidthStr,
     visual_offset_from_block, Change, Position, Range, Selection, Transaction,
@@ -31,7 +31,7 @@ use helix_view::{
     keyboard::{KeyCode, KeyModifiers},
     Document, Editor, Theme, View,
 };
-use std::{mem::take, num::NonZeroUsize, path::PathBuf, rc::Rc};
+use std::{mem::take, num::NonZeroUsize, ops, path::PathBuf, rc::Rc};
 
 use tui::{buffer::Buffer as Surface, text::Span};
 
@@ -87,6 +87,7 @@ impl EditorView {
         let area = view.area;
         let theme = &editor.theme;
         let config = editor.config();
+        let loader = editor.syn_loader.load();
 
         let view_offset = doc.view_offset(view.id);
 
@@ -115,51 +116,33 @@ impl EditorView {
             decorations.add_decoration(line_decoration);
         }
 
-        let syntax_highlights =
+        let syntax_highlighter =
-            Self::doc_syntax_highlights(doc, view_offset.anchor, inner.height, theme);
+            Self::doc_syntax_highlighter(doc, view_offset.anchor, inner.height, &loader);
+        let mut overlays = Vec::new();
 
-        let mut overlay_highlights =
+        overlays.push(Self::overlay_syntax_highlights(
-            Self::empty_highlight_iter(doc, view_offset.anchor, inner.height);
-        let overlay_syntax_highlights = Self::overlay_syntax_highlights(
             doc,
             view_offset.anchor,
             inner.height,
             &text_annotations,
-        );
+        ));
-        if !overlay_syntax_highlights.is_empty() {
-            overlay_highlights =
-                Box::new(syntax::merge(overlay_highlights, overlay_syntax_highlights));
-        }
 
-        for diagnostic in Self::doc_diagnostics_highlights(doc, theme) {
+        Self::doc_diagnostics_highlights_into(doc, theme, &mut overlays);
-            // Most of the `diagnostic` Vecs are empty most of the time. Skipping
-            // a merge for any empty Vec saves a significant amount of work.
-            if diagnostic.is_empty() {
-                continue;
-            }
-            overlay_highlights = Box::new(syntax::merge(overlay_highlights, diagnostic));
-        }
 
         if is_focused {
             if let Some(tabstops) = Self::tabstop_highlights(doc, theme) {
-                overlay_highlights = Box::new(syntax::merge(overlay_highlights, tabstops));
+                overlays.push(tabstops);
             }
-            let highlights = syntax::merge(
+            overlays.push(Self::doc_selection_highlights(
-                overlay_highlights,
+                editor.mode(),
-                Self::doc_selection_highlights(
+                doc,
-                    editor.mode(),
+                view,
-                    doc,
+                theme,
-                    view,
+                &config.cursor_shape,
-                    theme,
+                self.terminal_focused,
-                    &config.cursor_shape,
+            ));
-                    self.terminal_focused,
+            if let Some(overlay) = Self::highlight_focused_view_elements(view, doc, theme) {
-                ),
+                overlays.push(overlay);
-            );
-            let focused_view_elements = Self::highlight_focused_view_elements(view, doc, theme);
-            if focused_view_elements.is_empty() {
-                overlay_highlights = Box::new(highlights)
-            } else {
-                overlay_highlights = Box::new(syntax::merge(highlights, focused_view_elements))
             }
         }
 
@@ -207,8 +190,8 @@ impl EditorView {
             doc,
             view_offset,
             &text_annotations,
-            syntax_highlights,
+            syntax_highlighter,
-            overlay_highlights,
+            overlays,
             theme,
             decorations,
         );
@@ -287,57 +270,23 @@ impl EditorView {
         start..end
     }
 
-    pub fn empty_highlight_iter(
+    /// Get the syntax highlighter for a document in a view represented by the first line
-        doc: &Document,
-        anchor: usize,
-        height: u16,
-    ) -> Box<dyn Iterator<Item = HighlightEvent>> {
-        let text = doc.text().slice(..);
-        let row = text.char_to_line(anchor.min(text.len_chars()));
-
-        // Calculate viewport byte ranges:
-        // Saturating subs to make it inclusive zero indexing.
-        let range = Self::viewport_byte_range(text, row, height);
-        Box::new(
-            [HighlightEvent::Source {
-                start: text.byte_to_char(range.start),
-                end: text.byte_to_char(range.end),
-            }]
-            .into_iter(),
-        )
-    }
-
-    /// Get syntax highlights for a document in a view represented by the first line
     /// and column (`offset`) and the last line. This is done instead of using a view
     /// directly to enable rendering syntax highlighted docs anywhere (eg. picker preview)
-    pub fn doc_syntax_highlights<'doc>(
+    pub fn doc_syntax_highlighter<'editor>(
-        doc: &'doc Document,
+        doc: &'editor Document,
         anchor: usize,
         height: u16,
-        _theme: &Theme,
+        loader: &'editor syntax::Loader,
-    ) -> Box<dyn Iterator<Item = HighlightEvent> + 'doc> {
+    ) -> Option<syntax::Highlighter<'editor>> {
+        let syntax = doc.syntax()?;
         let text = doc.text().slice(..);
         let row = text.char_to_line(anchor.min(text.len_chars()));
-
         let range = Self::viewport_byte_range(text, row, height);
+        let range = range.start as u32..range.end as u32;
 
-        match doc.syntax() {
+        let highlighter = syntax.highlighter(text, loader, range);
-            Some(syntax) => {
+        Some(highlighter)
-                let iter = syntax
-                    // TODO: range doesn't actually restrict source, just highlight range
-                    .highlight_iter(text.slice(..), Some(range), None)
-                    .map(|event| event.unwrap());
-
-                Box::new(iter)
-            }
-            None => Box::new(
-                [HighlightEvent::Source {
-                    start: range.start,
-                    end: range.end,
-                }]
-                .into_iter(),
-            ),
-        }
     }
 
     pub fn overlay_syntax_highlights(
@@ -345,7 +294,7 @@ impl EditorView {
         anchor: usize,
         height: u16,
         text_annotations: &TextAnnotations,
-    ) -> Vec<(usize, std::ops::Range<usize>)> {
+    ) -> OverlayHighlights {
         let text = doc.text().slice(..);
         let row = text.char_to_line(anchor.min(text.len_chars()));
 
@@ -356,35 +305,29 @@ impl EditorView {
     }
 
     /// Get highlight spans for document diagnostics
-    pub fn doc_diagnostics_highlights(
+    pub fn doc_diagnostics_highlights_into(
         doc: &Document,
         theme: &Theme,
-    ) -> [Vec<(usize, std::ops::Range<usize>)>; 7] {
+        overlay_highlights: &mut Vec<OverlayHighlights>,
+    ) {
         use helix_core::diagnostic::{DiagnosticTag, Range, Severity};
         let get_scope_of = |scope| {
             theme
-            .find_scope_index_exact(scope)
+                .find_highlight_exact(scope)
-            // get one of the themes below as fallback values
+                // get one of the themes below as fallback values
-            .or_else(|| theme.find_scope_index_exact("diagnostic"))
+                .or_else(|| theme.find_highlight_exact("diagnostic"))
-            .or_else(|| theme.find_scope_index_exact("ui.cursor"))
+                .or_else(|| theme.find_highlight_exact("ui.cursor"))
-            .or_else(|| theme.find_scope_index_exact("ui.selection"))
+                .or_else(|| theme.find_highlight_exact("ui.selection"))
-            .expect(
+                .expect(
-                "at least one of the following scopes must be defined in the theme: `diagnostic`, `ui.cursor`, or `ui.selection`",
+                    "at least one of the following scopes must be defined in the theme: `diagnostic`, `ui.cursor`, or `ui.selection`",
-            )
+                )
         };
 
-        // basically just queries the theme color defined in the config
-        let hint = get_scope_of("diagnostic.hint");
-        let info = get_scope_of("diagnostic.info");
-        let warning = get_scope_of("diagnostic.warning");
-        let error = get_scope_of("diagnostic.error");
-        let r#default = get_scope_of("diagnostic"); // this is a bit redundant but should be fine
-
         // Diagnostic tags
-        let unnecessary = theme.find_scope_index_exact("diagnostic.unnecessary");
+        let unnecessary = theme.find_highlight_exact("diagnostic.unnecessary");
-        let deprecated = theme.find_scope_index_exact("diagnostic.deprecated");
+        let deprecated = theme.find_highlight_exact("diagnostic.deprecated");
 
-        let mut default_vec: Vec<(usize, std::ops::Range<usize>)> = Vec::new();
+        let mut default_vec = Vec::new();
         let mut info_vec = Vec::new();
         let mut hint_vec = Vec::new();
         let mut warning_vec = Vec::new();
@@ -392,31 +335,30 @@ impl EditorView {
         let mut unnecessary_vec = Vec::new();
         let mut deprecated_vec = Vec::new();
 
-        let push_diagnostic =
+        let push_diagnostic = |vec: &mut Vec<ops::Range<usize>>, range: Range| {
-            |vec: &mut Vec<(usize, std::ops::Range<usize>)>, scope, range: Range| {
+            // If any diagnostic overlaps ranges with the prior diagnostic,
-                // If any diagnostic overlaps ranges with the prior diagnostic,
+            // merge the two together. Otherwise push a new span.
-                // merge the two together. Otherwise push a new span.
+            match vec.last_mut() {
-                match vec.last_mut() {
+                Some(existing_range) if range.start <= existing_range.end => {
-                    Some((_, existing_range)) if range.start <= existing_range.end => {
+                    // This branch merges overlapping diagnostics, assuming that the current
-                        // This branch merges overlapping diagnostics, assuming that the current
+                    // diagnostic starts on range.start or later. If this assertion fails,
-                        // diagnostic starts on range.start or later. If this assertion fails,
+                    // we will discard some part of `diagnostic`. This implies that
-                        // we will discard some part of `diagnostic`. This implies that
+                    // `doc.diagnostics()` is not sorted by `diagnostic.range`.
-                        // `doc.diagnostics()` is not sorted by `diagnostic.range`.
+                    debug_assert!(existing_range.start <= range.start);
-                        debug_assert!(existing_range.start <= range.start);
+                    existing_range.end = range.end.max(existing_range.end)
-                        existing_range.end = range.end.max(existing_range.end)
-                    }
-                    _ => vec.push((scope, range.start..range.end)),
                 }
-            };
+                _ => vec.push(range.start..range.end),
+            }
+        };
 
         for diagnostic in doc.diagnostics() {
             // Separate diagnostics into different Vecs by severity.
-            let (vec, scope) = match diagnostic.severity {
+            let vec = match diagnostic.severity {
-                Some(Severity::Info) => (&mut info_vec, info),
+                Some(Severity::Info) => &mut info_vec,
-                Some(Severity::Hint) => (&mut hint_vec, hint),
+                Some(Severity::Hint) => &mut hint_vec,
-                Some(Severity::Warning) => (&mut warning_vec, warning),
+                Some(Severity::Warning) => &mut warning_vec,
-                Some(Severity::Error) => (&mut error_vec, error),
+                Some(Severity::Error) => &mut error_vec,
-                _ => (&mut default_vec, r#default),
+                _ => &mut default_vec,
             };
 
             // If the diagnostic has tags and a non-warning/error severity, skip rendering
@@ -429,34 +371,59 @@ impl EditorView {
                     Some(Severity::Warning | Severity::Error)
                 )
             {
-                push_diagnostic(vec, scope, diagnostic.range);
+                push_diagnostic(vec, diagnostic.range);
             }
 
             for tag in &diagnostic.tags {
                 match tag {
                     DiagnosticTag::Unnecessary => {
-                        if let Some(scope) = unnecessary {
+                        if unnecessary.is_some() {
-                            push_diagnostic(&mut unnecessary_vec, scope, diagnostic.range)
+                            push_diagnostic(&mut unnecessary_vec, diagnostic.range)
                         }
                     }
                     DiagnosticTag::Deprecated => {
-                        if let Some(scope) = deprecated {
+                        if deprecated.is_some() {
-                            push_diagnostic(&mut deprecated_vec, scope, diagnostic.range)
+                            push_diagnostic(&mut deprecated_vec, diagnostic.range)
                         }
                     }
                 }
             }
         }
 
-        [
+        overlay_highlights.push(OverlayHighlights::Homogeneous {
-            default_vec,
+            highlight: get_scope_of("diagnostic"),
-            unnecessary_vec,
+            ranges: default_vec,
-            deprecated_vec,
+        });
-            info_vec,
+        if let Some(highlight) = unnecessary {
-            hint_vec,
+            overlay_highlights.push(OverlayHighlights::Homogeneous {
-            warning_vec,
+                highlight,
-            error_vec,
+                ranges: unnecessary_vec,
-        ]
+            });
+        }
+        if let Some(highlight) = deprecated {
+            overlay_highlights.push(OverlayHighlights::Homogeneous {
+                highlight,
+                ranges: deprecated_vec,
+            });
+        }
+        overlay_highlights.extend([
+            OverlayHighlights::Homogeneous {
+                highlight: get_scope_of("diagnostic.info"),
+                ranges: info_vec,
+            },
+            OverlayHighlights::Homogeneous {
+                highlight: get_scope_of("diagnostic.hint"),
+                ranges: hint_vec,
+            },
+            OverlayHighlights::Homogeneous {
+                highlight: get_scope_of("diagnostic.warning"),
+                ranges: warning_vec,
+            },
+            OverlayHighlights::Homogeneous {
+                highlight: get_scope_of("diagnostic.error"),
+                ranges: error_vec,
+            },
+        ]);
     }
 
     /// Get highlight spans for selections in a document view.
@@ -467,7 +434,7 @@ impl EditorView {
         theme: &Theme,
         cursor_shape_config: &CursorShapeConfig,
         is_terminal_focused: bool,
-    ) -> Vec<(usize, std::ops::Range<usize>)> {
+    ) -> OverlayHighlights {
         let text = doc.text().slice(..);
         let selection = doc.selection(view.id);
         let primary_idx = selection.primary_index();
@@ -476,34 +443,34 @@ impl EditorView {
         let cursor_is_block = cursorkind == CursorKind::Block;
 
         let selection_scope = theme
-            .find_scope_index_exact("ui.selection")
+            .find_highlight_exact("ui.selection")
             .expect("could not find `ui.selection` scope in the theme!");
         let primary_selection_scope = theme
-            .find_scope_index_exact("ui.selection.primary")
+            .find_highlight_exact("ui.selection.primary")
             .unwrap_or(selection_scope);
 
         let base_cursor_scope = theme
-            .find_scope_index_exact("ui.cursor")
+            .find_highlight_exact("ui.cursor")
             .unwrap_or(selection_scope);
         let base_primary_cursor_scope = theme
-            .find_scope_index("ui.cursor.primary")
+            .find_highlight("ui.cursor.primary")
             .unwrap_or(base_cursor_scope);
 
         let cursor_scope = match mode {
-            Mode::Insert => theme.find_scope_index_exact("ui.cursor.insert"),
+            Mode::Insert => theme.find_highlight_exact("ui.cursor.insert"),
-            Mode::Select => theme.find_scope_index_exact("ui.cursor.select"),
+            Mode::Select => theme.find_highlight_exact("ui.cursor.select"),
-            Mode::Normal => theme.find_scope_index_exact("ui.cursor.normal"),
+            Mode::Normal => theme.find_highlight_exact("ui.cursor.normal"),
         }
         .unwrap_or(base_cursor_scope);
 
         let primary_cursor_scope = match mode {
-            Mode::Insert => theme.find_scope_index_exact("ui.cursor.primary.insert"),
+            Mode::Insert => theme.find_highlight_exact("ui.cursor.primary.insert"),
-            Mode::Select => theme.find_scope_index_exact("ui.cursor.primary.select"),
+            Mode::Select => theme.find_highlight_exact("ui.cursor.primary.select"),
-            Mode::Normal => theme.find_scope_index_exact("ui.cursor.primary.normal"),
+            Mode::Normal => theme.find_highlight_exact("ui.cursor.primary.normal"),
         }
         .unwrap_or(base_primary_cursor_scope);
 
-        let mut spans: Vec<(usize, std::ops::Range<usize>)> = Vec::new();
+        let mut spans = Vec::new();
         for (i, range) in selection.iter().enumerate() {
             let selection_is_primary = i == primary_idx;
             let (cursor_scope, selection_scope) = if selection_is_primary {
@@ -563,7 +530,7 @@ impl EditorView {
             }
         }
 
-        spans
+        OverlayHighlights::Heterogenous { highlights: spans }
     }
 
     /// Render brace match, etc (meant for the focused view only)
@@ -571,41 +538,24 @@ impl EditorView {
         view: &View,
         doc: &Document,
         theme: &Theme,
-    ) -> Vec<(usize, std::ops::Range<usize>)> {
+    ) -> Option<OverlayHighlights> {
         // Highlight matching braces
-        if let Some(syntax) = doc.syntax() {
+        let syntax = doc.syntax()?;
-            let text = doc.text().slice(..);
+        let highlight = theme.find_highlight_exact("ui.cursor.match")?;
-            use helix_core::match_brackets;
+        let text = doc.text().slice(..);
-            let pos = doc.selection(view.id).primary().cursor(text);
+        let pos = doc.selection(view.id).primary().cursor(text);
-
+        let pos = helix_core::match_brackets::find_matching_bracket(syntax, text, pos)?;
-            if let Some(pos) =
+        Some(OverlayHighlights::single(highlight, pos..pos + 1))
-                match_brackets::find_matching_bracket(syntax, doc.text().slice(..), pos)
-            {
-                // ensure col is on screen
-                if let Some(highlight) = theme.find_scope_index_exact("ui.cursor.match") {
-                    return vec![(highlight, pos..pos + 1)];
-                }
-            }
-        }
-        Vec::new()
     }
 
-    pub fn tabstop_highlights(
+    pub fn tabstop_highlights(doc: &Document, theme: &Theme) -> Option<OverlayHighlights> {
-        doc: &Document,
-        theme: &Theme,
-    ) -> Option<Vec<(usize, std::ops::Range<usize>)>> {
         let snippet = doc.active_snippet.as_ref()?;
-        let highlight = theme.find_scope_index_exact("tabstop")?;
+        let highlight = theme.find_highlight_exact("tabstop")?;
-        let mut highlights = Vec::new();
+        let mut ranges = Vec::new();
         for tabstop in snippet.tabstops() {
-            highlights.extend(
+            ranges.extend(tabstop.ranges.iter().map(|range| range.start..range.end));
-                tabstop
-                    .ranges
-                    .iter()
-                    .map(|range| (highlight, range.start..range.end)),
-            );
         }
-        (!highlights.is_empty()).then_some(highlights)
+        Some(OverlayHighlights::Homogeneous { highlight, ranges })
     }
 
     /// Render bufferline at the top

helix-term/src/ui/lsp/signature_help.rs

@@ -1,7 +1,7 @@
 use std::sync::Arc;
 
 use arc_swap::ArcSwap;
-use helix_core::syntax;
+use helix_core::syntax::{self, OverlayHighlights};
 use helix_view::graphics::{Margin, Rect, Style};
 use helix_view::input::Event;
 use tui::buffer::Buffer;
@@ -102,13 +102,12 @@ impl Component for SignatureHelp {
             .unwrap_or_else(|| &self.signatures[0]);
 
         let active_param_span = signature.active_param_range.map(|(start, end)| {
-            vec![(
+            let highlight = cx
-                cx.editor
+                .editor
-                    .theme
+                .theme
-                    .find_scope_index_exact("ui.selection")
+                .find_highlight_exact("ui.selection")
-                    .unwrap(),
+                .unwrap();
-                start..end,
+            OverlayHighlights::single(highlight, start..end)
-            )]
         });
 
         let signature = self
@@ -120,7 +119,7 @@ impl Component for SignatureHelp {
             signature.signature.as_str(),
             &self.language,
             Some(&cx.editor.theme),
-            Arc::clone(&self.config_loader),
+            &self.config_loader.load(),
             active_param_span,
         );
 
@@ -178,7 +177,7 @@ impl Component for SignatureHelp {
             signature.signature.as_str(),
             &self.language,
             None,
-            Arc::clone(&self.config_loader),
+            &self.config_loader.load(),
             None,
         );
         let (sig_width, sig_height) =

helix-term/src/ui/markdown.rs

@@ -10,8 +10,8 @@ use std::sync::Arc;
 use pulldown_cmark::{CodeBlockKind, Event, HeadingLevel, Options, Parser, Tag, TagEnd};
 
 use helix_core::{
-    syntax::{self, HighlightEvent, InjectionLanguageMarker, Syntax},
+    syntax::{self, HighlightEvent, OverlayHighlights},
-    RopeSlice,
+    RopeSlice, Syntax,
 };
 use helix_view::{
     graphics::{Margin, Rect, Style},
@@ -32,8 +32,12 @@ pub fn highlighted_code_block<'a>(
     text: &str,
     language: &str,
     theme: Option<&Theme>,
-    config_loader: Arc<ArcSwap<syntax::Loader>>,
+    loader: &syntax::Loader,
-    additional_highlight_spans: Option<Vec<(usize, std::ops::Range<usize>)>>,
+    // Optional overlay highlights to mix in with the syntax highlights.
+    //
+    // Note that `OverlayHighlights` is typically used with char indexing but the only caller
+    // which passes this parameter currently passes **byte indices** instead.
+    additional_highlight_spans: Option<OverlayHighlights>,
 ) -> Text<'a> {
     let mut spans = Vec::new();
     let mut lines = Vec::new();
@@ -48,67 +52,74 @@ pub fn highlighted_code_block<'a>(
     };
 
     let ropeslice = RopeSlice::from(text);
-    let syntax = config_loader
+    let Some(syntax) = loader
-        .load()
+        .language_for_match(RopeSlice::from(language))
-        .language_configuration_for_injection_string(&InjectionLanguageMarker::Name(
+        .and_then(|lang| Syntax::new(ropeslice, lang, loader).ok())
-            language.into(),
+    else {
-        ))
+        return styled_multiline_text(text, code_style);
-        .and_then(|config| config.highlight_config(theme.scopes()))
-        .and_then(|config| Syntax::new(ropeslice, config, Arc::clone(&config_loader)));
-
-    let syntax = match syntax {
-        Some(s) => s,
-        None => return styled_multiline_text(text, code_style),
     };
 
-    let highlight_iter = syntax
+    let mut syntax_highlighter = syntax.highlighter(ropeslice, loader, ..);
-        .highlight_iter(ropeslice, None, None)
+    let mut syntax_highlight_stack = Vec::new();
-        .map(|e| e.unwrap());
+    let mut overlay_highlight_stack = Vec::new();
-    let highlight_iter: Box<dyn Iterator<Item = HighlightEvent>> =
+    let mut overlay_highlighter = syntax::OverlayHighlighter::new(additional_highlight_spans);
-        if let Some(spans) = additional_highlight_spans {
+    let mut pos = 0;
-            Box::new(helix_core::syntax::merge(highlight_iter, spans))
-        } else {
-            Box::new(highlight_iter)
-        };
 
-    let mut highlights = Vec::new();
+    while pos < ropeslice.len_bytes() as u32 {
-    for event in highlight_iter {
+        if pos == syntax_highlighter.next_event_offset() {
-        match event {
+            let (event, new_highlights) = syntax_highlighter.advance();
-            HighlightEvent::HighlightStart(span) => {
+            if event == HighlightEvent::Refresh {
-                highlights.push(span);
+                syntax_highlight_stack.clear();
             }
-            HighlightEvent::HighlightEnd => {
+            syntax_highlight_stack.extend(new_highlights);
-                highlights.pop();
+        } else if pos == overlay_highlighter.next_event_offset() as u32 {
+            let (event, new_highlights) = overlay_highlighter.advance();
+            if event == HighlightEvent::Refresh {
+                overlay_highlight_stack.clear();
             }
-            HighlightEvent::Source { start, end } => {
+            overlay_highlight_stack.extend(new_highlights)
-                let style = highlights
+        }
-                    .iter()
-                    .fold(text_style, |acc, span| acc.patch(theme.highlight(span.0)));
 
-                let mut slice = &text[start..end];
+        let start = pos;
-                // TODO: do we need to handle all unicode line endings
+        pos = syntax_highlighter
-                // here, or is just '\n' okay?
+            .next_event_offset()
-                while let Some(end) = slice.find('\n') {
+            .min(overlay_highlighter.next_event_offset() as u32);
-                    // emit span up to newline
+        if pos == u32::MAX {
-                    let text = &slice[..end];
+            pos = ropeslice.len_bytes() as u32;
-                    let text = text.replace('\t', "    "); // replace tabs
+        }
-                    let span = Span::styled(text, style);
+        if pos == start {
-                    spans.push(span);
+            continue;
+        }
+        assert!(pos > start);
 
-                    // truncate slice to after newline
+        let style = syntax_highlight_stack
-                    slice = &slice[end + 1..];
+            .iter()
+            .chain(overlay_highlight_stack.iter())
+            .fold(text_style, |acc, highlight| {
+                acc.patch(theme.highlight(*highlight))
+            });
 
-                    // make a new line
+        let mut slice = &text[start as usize..pos as usize];
-                    let spans = std::mem::take(&mut spans);
+        // TODO: do we need to handle all unicode line endings
-                    lines.push(Spans::from(spans));
+        // here, or is just '\n' okay?
-                }
+        while let Some(end) = slice.find('\n') {
+            // emit span up to newline
+            let text = &slice[..end];
+            let text = text.replace('\t', "    "); // replace tabs
+            let span = Span::styled(text, style);
+            spans.push(span);
 
-                // if there's anything left, emit it too
+            // truncate slice to after newline
-                if !slice.is_empty() {
+            slice = &slice[end + 1..];
-                    let span = Span::styled(slice.replace('\t', "    "), style);
+
-                    spans.push(span);
+            // make a new line
-                }
+            let spans = std::mem::take(&mut spans);
-            }
+            lines.push(Spans::from(spans));
+        }
+
+        if !slice.is_empty() {
+            let span = Span::styled(slice.replace('\t', "    "), style);
+            spans.push(span);
         }
     }
 
@@ -286,7 +297,7 @@ impl Markdown {
                             &text,
                             language,
                             theme,
-                            Arc::clone(&self.config_loader),
+                            &self.config_loader.load(),
                             None,
                         );
                         lines.extend(tui_text.lines.into_iter());

helix-term/src/ui/picker.rs

@@ -940,21 +940,18 @@ impl<T: 'static + Send + Sync, D: 'static + Send + Sync> Picker<T, D> {
                 }
             }
 
-            let syntax_highlights = EditorView::doc_syntax_highlights(
+            let loader = cx.editor.syn_loader.load();
+
+            let syntax_highlighter =
+                EditorView::doc_syntax_highlighter(doc, offset.anchor, area.height, &loader);
+            let mut overlay_highlights = Vec::new();
+
+            EditorView::doc_diagnostics_highlights_into(
                 doc,
-                offset.anchor,
-                area.height,
                 &cx.editor.theme,
+                &mut overlay_highlights,
             );
 
-            let mut overlay_highlights =
-                EditorView::empty_highlight_iter(doc, offset.anchor, area.height);
-            for spans in EditorView::doc_diagnostics_highlights(doc, &cx.editor.theme) {
-                if spans.is_empty() {
-                    continue;
-                }
-                overlay_highlights = Box::new(helix_core::syntax::merge(overlay_highlights, spans));
-            }
             let mut decorations = DecorationManager::default();
 
             if let Some((start, end)) = range {
@@ -984,7 +981,7 @@ impl<T: 'static + Send + Sync, D: 'static + Send + Sync> Picker<T, D> {
                 offset,
                 // TODO: compute text annotations asynchronously here (like inlay hints)
                 &TextAnnotations::default(),
-                syntax_highlights,
+                syntax_highlighter,
                 overlay_highlights,
                 &cx.editor.theme,
                 decorations,

helix-term/src/ui/picker/handlers.rs

@@ -70,23 +70,21 @@ impl<T: 'static + Send + Sync, D: 'static + Send + Sync> AsyncHook
                 return;
             }
 
-            let Some(language_config) = doc.detect_language_config(&editor.syn_loader.load())
+            let loader = editor.syn_loader.load();
-            else {
+            let Some(language_config) = doc.detect_language_config(&loader) else {
                 return;
             };
-            doc.language = Some(language_config.clone());
+            let language = language_config.language();
+            doc.language = Some(language_config);
             let text = doc.text().clone();
-            let loader = editor.syn_loader.clone();
 
             tokio::task::spawn_blocking(move || {
-                let Some(syntax) = language_config
+                let syntax = match helix_core::Syntax::new(text.slice(..), language, &loader) {
-                    .highlight_config(&loader.load().scopes())
+                    Ok(syntax) => syntax,
-                    .and_then(|highlight_config| {
+                    Err(err) => {
-                        helix_core::Syntax::new(text.slice(..), highlight_config, loader)
+                        log::info!("highlighting picker preview failed: {err}");
-                    })
+                        return;
-                else {
+                    }
-                    log::info!("highlighting picker item failed");
-                    return;
                 };
 
                 job::dispatch_blocking(move |editor, compositor| {

helix-term/src/ui/prompt.rs

@@ -529,7 +529,7 @@ impl Prompt {
                 &self.line,
                 language,
                 Some(&cx.editor.theme),
-                loader.clone(),
+                &loader.load(),
                 None,
             )
             .into();

helix-view/src/document.rs

@@ -9,7 +9,7 @@ use helix_core::diagnostic::DiagnosticProvider;
 use helix_core::doc_formatter::TextFormat;
 use helix_core::encoding::Encoding;
 use helix_core::snippets::{ActiveSnippet, SnippetRenderCtx};
-use helix_core::syntax::{config::LanguageServerFeature, Highlight};
+use helix_core::syntax::config::LanguageServerFeature;
 use helix_core::text_annotations::{InlineAnnotation, Overlay};
 use helix_event::TaskController;
 use helix_lsp::util::lsp_pos_to_pos;
@@ -217,7 +217,7 @@ pub struct Document {
 #[derive(Debug, Clone, Default)]
 pub struct DocumentColorSwatches {
     pub color_swatches: Vec<InlineAnnotation>,
-    pub colors: Vec<Highlight>,
+    pub colors: Vec<syntax::Highlight>,
     pub color_swatches_padding: Vec<InlineAnnotation>,
 }
 
@@ -1121,11 +1121,13 @@ impl Document {
     /// Detect the programming language based on the file type.
     pub fn detect_language_config(
         &self,
-        config_loader: &syntax::Loader,
+        loader: &syntax::Loader,
     ) -> Option<Arc<syntax::config::LanguageConfiguration>> {
-        config_loader
+        let language = loader
-            .language_config_for_file_name(self.path.as_ref()?)
+            .language_for_filename(self.path.as_ref()?)
-            .or_else(|| config_loader.language_config_for_shebang(self.text().slice(..)))
+            .or_else(|| loader.language_for_shebang(self.text().slice(..)))?;
+
+        Some(loader.language(language).config().clone())
     }
 
     /// Detect the indentation used in the file, or otherwise defaults to the language indentation
@@ -1268,17 +1270,18 @@ impl Document {
         loader: &syntax::Loader,
     ) {
         self.language = language_config;
-        self.syntax = self
+        self.syntax = self.language.as_ref().and_then(|config| {
-            .language
+            Syntax::new(self.text.slice(..), config.language(), loader)
-            .as_ref()
+                .map_err(|err| {
-            .and_then(|config| config.highlight_config(&loader.scopes()))
+                    // `NoRootConfig` means that there was an issue loading the language/syntax
-            .and_then(|highlight_config| {
+                    // config for the root language of the document. An error must have already
-                Syntax::new(
+                    // been logged by `LanguageData::syntax_config`.
-                    self.text.slice(..),
+                    if err != syntax::HighlighterError::NoRootConfig {
-                    highlight_config,
+                        log::warn!("Error building syntax for '{}': {err}", self.display_name());
-                    self.syn_loader.clone(),
+                    }
-                )
+                })
-            });
+                .ok()
+        });
     }
 
     /// Set the programming language for the file if you know the language but don't have the
@@ -1288,10 +1291,11 @@ impl Document {
         language_id: &str,
         loader: &syntax::Loader,
     ) -> anyhow::Result<()> {
-        let language_config = loader
+        let language = loader
-            .language_config_for_language_id(language_id)
+            .language_for_name(language_id)
             .ok_or_else(|| anyhow!("invalid language id: {}", language_id))?;
-        self.set_language(Some(language_config), loader);
+        let config = loader.language(language).config().clone();
+        self.set_language(Some(config), loader);
         Ok(())
     }
 
@@ -1410,14 +1414,14 @@ impl Document {
 
         // update tree-sitter syntax tree
         if let Some(syntax) = &mut self.syntax {
-            // TODO: no unwrap
+            let loader = self.syn_loader.load();
-            let res = syntax.update(
+            if let Err(err) = syntax.update(
                 old_doc.slice(..),
                 self.text.slice(..),
                 transaction.changes(),
-            );
+                &loader,
-            if res.is_err() {
+            ) {
-                log::error!("TS parser failed, disabling TS for the current buffer: {res:?}");
+                log::error!("TS parser failed, disabling TS for the current buffer: {err}");
                 self.syntax = None;
             }
         }
@@ -2225,8 +2229,7 @@ impl Document {
             viewport_width,
             wrap_indicator: wrap_indicator.into_boxed_str(),
             wrap_indicator_highlight: theme
-                .and_then(|theme| theme.find_scope_index("ui.virtual.wrap"))
+                .and_then(|theme| theme.find_highlight("ui.virtual.wrap")),
-                .map(Highlight),
             soft_wrap_at_text_width,
         }
     }

helix-view/src/editor.rs

@@ -1358,7 +1358,7 @@ impl Editor {
 
     fn set_theme_impl(&mut self, theme: Theme, preview: ThemeAction) {
         // `ui.selection` is the only scope required to be able to render a theme.
-        if theme.find_scope_index_exact("ui.selection").is_none() {
+        if theme.find_highlight_exact("ui.selection").is_none() {
             self.set_error("Invalid theme: `ui.selection` required");
             return;
         }
@@ -1512,12 +1512,12 @@ impl Editor {
                         if let helix_lsp::Error::ExecutableNotFound(err) = err {
                             // Silence by default since some language servers might just not be installed
                             log::debug!(
-                                "Language server not found for `{}` {} {}", language.scope(), lang, err,
+                                "Language server not found for `{}` {} {}", language.scope, lang, err,
                             );
                         } else {
                             log::error!(
                                 "Failed to initialize the language servers for `{}` - `{}` {{ {} }}",
-                                language.scope(),
+                                language.scope,
                                 lang,
                                 err
                             );

helix-view/src/theme.rs

@@ -294,43 +294,36 @@ fn build_theme_values(
 
 impl Theme {
     /// To allow `Highlight` to represent arbitrary RGB colors without turning it into an enum,
-    /// we interpret the last 3 bytes of a `Highlight` as RGB colors.
+    /// we interpret the last 256^3 numbers as RGB.
-    const RGB_START: usize = (usize::MAX << (8 + 8 + 8)) - 1;
+    const RGB_START: u32 = (u32::MAX << (8 + 8 + 8)) - 1 - (u32::MAX - Highlight::MAX);
 
     /// Interpret a Highlight with the RGB foreground
-    fn decode_rgb_highlight(rgb: usize) -> Option<(u8, u8, u8)> {
+    fn decode_rgb_highlight(highlight: Highlight) -> Option<(u8, u8, u8)> {
-        (rgb > Self::RGB_START).then(|| {
+        (highlight.get() > Self::RGB_START).then(|| {
-            let [b, g, r, ..] = rgb.to_ne_bytes();
+            let [b, g, r, ..] = (highlight.get() + 1).to_ne_bytes();
             (r, g, b)
         })
     }
 
     /// Create a Highlight that represents an RGB color
     pub fn rgb_highlight(r: u8, g: u8, b: u8) -> Highlight {
-        Highlight(usize::from_ne_bytes([
+        // -1 because highlight is "non-max": u32::MAX is reserved for the null pointer
-            b,
+        // optimization.
-            g,
+        Highlight::new(u32::from_ne_bytes([b, g, r, u8::MAX]) - 1)
-            r,
-            u8::MAX,
-            u8::MAX,
-            u8::MAX,
-            u8::MAX,
-            u8::MAX,
-        ]))
     }
 
     #[inline]
-    pub fn highlight(&self, index: usize) -> Style {
+    pub fn highlight(&self, highlight: Highlight) -> Style {
-        if let Some((red, green, blue)) = Self::decode_rgb_highlight(index) {
+        if let Some((red, green, blue)) = Self::decode_rgb_highlight(highlight) {
             Style::new().fg(Color::Rgb(red, green, blue))
         } else {
-            self.highlights[index]
+            self.highlights[highlight.idx()]
         }
     }
 
     #[inline]
-    pub fn scope(&self, index: usize) -> &str {
+    pub fn scope(&self, highlight: Highlight) -> &str {
-        &self.scopes[index]
+        &self.scopes[highlight.idx()]
     }
 
     pub fn name(&self) -> &str {
@@ -361,13 +354,16 @@ impl Theme {
         &self.scopes
     }
 
-    pub fn find_scope_index_exact(&self, scope: &str) -> Option<usize> {
+    pub fn find_highlight_exact(&self, scope: &str) -> Option<Highlight> {
-        self.scopes().iter().position(|s| s == scope)
+        self.scopes()
+            .iter()
+            .position(|s| s == scope)
+            .map(|idx| Highlight::new(idx as u32))
     }
 
-    pub fn find_scope_index(&self, mut scope: &str) -> Option<usize> {
+    pub fn find_highlight(&self, mut scope: &str) -> Option<Highlight> {
         loop {
-            if let Some(highlight) = self.find_scope_index_exact(scope) {
+            if let Some(highlight) = self.find_highlight_exact(scope) {
                 return Some(highlight);
             }
             if let Some(new_end) = scope.rfind('.') {
@@ -626,23 +622,13 @@ mod tests {
     fn convert_to_and_from() {
         let (r, g, b) = (0xFF, 0xFE, 0xFA);
         let highlight = Theme::rgb_highlight(r, g, b);
-        assert_eq!(Theme::decode_rgb_highlight(highlight.0), Some((r, g, b)));
+        assert_eq!(Theme::decode_rgb_highlight(highlight), Some((r, g, b)));
     }
 
     /// make sure we can store all the colors at the end
-    /// ```
-    /// FF FF FF FF FF FF FF FF
-    ///          xor
-    /// FF FF FF FF FF 00 00 00
-    ///           =
-    /// 00 00 00 00 00 FF FF FF
-    /// ```
-    ///
-    /// where the ending `(FF, FF, FF)` represents `(r, g, b)`
     #[test]
     fn full_numeric_range() {
-        assert_eq!(usize::MAX ^ Theme::RGB_START, 256_usize.pow(3));
+        assert_eq!(Highlight::MAX - Theme::RGB_START, 256_u32.pow(3));
-        assert_eq!(Theme::RGB_START + 256_usize.pow(3), usize::MAX);
     }
 
     #[test]
@@ -650,30 +636,27 @@ mod tests {
         // color in the middle
         let (r, g, b) = (0x14, 0xAA, 0xF7);
         assert_eq!(
-            Theme::default().highlight(Theme::rgb_highlight(r, g, b).0),
+            Theme::default().highlight(Theme::rgb_highlight(r, g, b)),
             Style::new().fg(Color::Rgb(r, g, b))
         );
         // pure black
         let (r, g, b) = (0x00, 0x00, 0x00);
         assert_eq!(
-            Theme::default().highlight(Theme::rgb_highlight(r, g, b).0),
+            Theme::default().highlight(Theme::rgb_highlight(r, g, b)),
             Style::new().fg(Color::Rgb(r, g, b))
         );
         // pure white
         let (r, g, b) = (0xff, 0xff, 0xff);
         assert_eq!(
-            Theme::default().highlight(Theme::rgb_highlight(r, g, b).0),
+            Theme::default().highlight(Theme::rgb_highlight(r, g, b)),
             Style::new().fg(Color::Rgb(r, g, b))
         );
     }
 
     #[test]
-    #[should_panic(
+    #[should_panic(expected = "index out of bounds: the len is 0 but the index is 4278190078")]
-        expected = "index out of bounds: the len is 0 but the index is 18446744073692774399"
-    )]
     fn out_of_bounds() {
-        let (r, g, b) = (0x00, 0x00, 0x00);
+        let highlight = Highlight::new(Theme::rgb_highlight(0, 0, 0).get() - 1);
-
+        Theme::default().highlight(highlight);
-        Theme::default().highlight(Theme::rgb_highlight(r, g, b).0 - 1);
     }
 }

helix-view/src/view.rs

@@ -11,7 +11,6 @@ use crate::{
 use helix_core::{
     char_idx_at_visual_offset,
     doc_formatter::TextFormat,
-    syntax::Highlight,
     text_annotations::TextAnnotations,
     visual_offset_from_anchor, visual_offset_from_block, Position, RopeSlice, Selection,
     Transaction,
@@ -446,9 +445,7 @@ impl View {
         let mut text_annotations = TextAnnotations::default();
 
         if let Some(labels) = doc.jump_labels.get(&self.id) {
-            let style = theme
+            let style = theme.and_then(|t| t.find_highlight("ui.virtual.jump-label"));
-                .and_then(|t| t.find_scope_index("ui.virtual.jump-label"))
-                .map(Highlight);
             text_annotations.add_overlay(labels, style);
         }
 
@@ -461,15 +458,10 @@ impl View {
             padding_after_inlay_hints,
         }) = doc.inlay_hints.get(&self.id)
         {
-            let type_style = theme
+            let type_style = theme.and_then(|t| t.find_highlight("ui.virtual.inlay-hint.type"));
-                .and_then(|t| t.find_scope_index("ui.virtual.inlay-hint.type"))
+            let parameter_style =
-                .map(Highlight);
+                theme.and_then(|t| t.find_highlight("ui.virtual.inlay-hint.parameter"));
-            let parameter_style = theme
+            let other_style = theme.and_then(|t| t.find_highlight("ui.virtual.inlay-hint"));
-                .and_then(|t| t.find_scope_index("ui.virtual.inlay-hint.parameter"))
-                .map(Highlight);
-            let other_style = theme
-                .and_then(|t| t.find_scope_index("ui.virtual.inlay-hint"))
-                .map(Highlight);
 
             // Overlapping annotations are ignored apart from the first so the order here is not random:
             // types -> parameters -> others should hopefully be the "correct" order for most use cases,

xtask/src/helpers.rs

@@ -1,7 +1,7 @@
 use std::path::{Path, PathBuf};
 
 use crate::path;
-use helix_core::syntax::config::Configuration as LangConfig;
+use helix_core::syntax::{self, config::Configuration as LangConfig};
 use helix_term::health::TsFeature;
 
 /// Get the list of languages that support a particular tree-sitter
@@ -42,3 +42,7 @@ pub fn lang_config() -> LangConfig {
     let text = std::fs::read_to_string(path::lang_config()).unwrap();
     toml::from_str(&text).unwrap()
 }
+
+pub fn syn_loader() -> syntax::Loader {
+    syntax::Loader::new(lang_config()).unwrap()
+}

xtask/src/main.rs

@@ -18,36 +18,18 @@ pub mod tasks {
     }
 
     pub fn querycheck() -> Result<(), DynError> {
-        use crate::helpers::lang_config;
+        use helix_core::syntax::LanguageData;
-        use helix_core::{syntax::read_query, tree_sitter::Query};
-        use helix_loader::grammar::get_language;
 
-        let query_files = [
+        let loader = crate::helpers::syn_loader();
-            "highlights.scm",
-            "locals.scm",
-            "injections.scm",
-            "textobjects.scm",
-            "indents.scm",
-        ];
 
-        for language in lang_config().language {
+        for (_language, lang_data) in loader.languages() {
-            let language_name = &language.language_id;
+            let config = lang_data.config();
-            let grammar_name = language.grammar.as_ref().unwrap_or(language_name);
+            let Some(syntax_config) = LanguageData::compile_syntax_config(config, &loader)? else {
-            for query_file in query_files {
+                continue;
-                let language = get_language(grammar_name);
+            };
-                let query_text = read_query(language_name, query_file);
+            let grammar = syntax_config.grammar;
-                if let Ok(lang) = language {
+            LanguageData::compile_indent_query(grammar, config)?;
-                    if !query_text.is_empty() {
+            LanguageData::compile_textobject_query(grammar, config)?;
-                        if let Err(reason) = Query::new(&lang, &query_text) {
-                            return Err(format!(
-                                "Failed to parse {} queries for {}: {}",
-                                query_file, language_name, reason
-                            )
-                            .into());
-                        }
-                    }
-                }
-            }
         }
 
         println!("Query check succeeded");