1use std::{
2 borrow::Cow,
3 ffi::OsStr,
4 fmt::Write,
5 fs,
6 path::{Path, PathBuf},
7 process::Command,
8 sync::LazyLock,
9};
10
11use anyhow::{Context, bail};
12use heck::{ToSnakeCase, ToUpperCamelCase};
13use quote::{format_ident, quote};
14use serde::Deserialize;
15use threadpool::ThreadPool;
16
17const GRAMMAR_REPOSITORY_URL: &str = "https://github.com/JordanForks/helix";
18const GRAMMAR_REPOSITORY_REF: &str = "1603715cc91bf6fdffb4aedfb5b76fb69fd10e28";
19const GRAMMAR_REPOSITORY_CONFIG_PATH: &str = "languages.toml";
20
21static BLACKLISTED_MODULES: &[&str] = &[
22 // these languages all don't have corresponding grammars
23 "cabal",
24 "idris",
25 "llvm-mir-yaml",
26 "prolog",
27 "mint",
28 "hare",
29 "wren",
30 // doesn't compile on macos
31 "gemini",
32];
33
34static BLACKLISTED_FOR_STATIC_LINKING: &[&str] = &[
35 // all of these have some sort of conflict with higher value highlighters
36 // when statically linking due to variations of:
37 // https://github.com/ikatyang/tree-sitter-vue/issues/27
38 "vue",
39 "purescript",
40 "rst",
41 "comment",
42 "v",
43 "unison",
44 "llvm",
45];
46
47fn main() -> anyhow::Result<()> {
48 let out_dir = PathBuf::from(std::env::var("OUT_DIR").context("OUT_DIR not set by rustc")?);
49
50 let root = std::env::var("TREE_SITTER_GRAMMAR_LIB_DIR").ok();
51 println!("cargo::rerun-if-env-changed=TREE_SITTER_GRAMMAR_LIB_DIR");
52
53 let (root, dylib) = if let Some(root) = root.as_deref() {
54 (Path::new(root), true)
55 } else {
56 (out_dir.as_path(), false)
57 };
58
59 let (config, query_path) = if dylib {
60 let config = fs::read_to_string(root.join("languages.toml"))
61 .context("failed to read helix languages.toml")?;
62
63 // find the start of the language arrays & skip parsing everything before them (primarily because
64 // `basic_toml` can't handle some of the new syntax used in this file, and `toml` pulls in a lot
65 // of dependencies)
66 let language_defs_start = memchr::memmem::find(config.as_bytes(), b"[[language]]")
67 .context("languages.toml is missing languages")?;
68
69 let config = &config[language_defs_start..];
70
71 let config: HelixLanguages =
72 basic_toml::from_str(config).context("failed to parse helix languages.toml")?;
73
74 println!("cargo::rustc-link-search=native={}", root.display());
75
76 for grammar in &config.grammar {
77 if BLACKLISTED_MODULES.contains(&grammar.name.as_str()) {
78 continue;
79 }
80
81 println!("cargo::rustc-link-lib=dylib=tree-sitter-{}", grammar.name);
82 }
83
84 (config, root.join("queries"))
85 } else {
86 let sources = out_dir.join("sources");
87 fs::create_dir_all(&sources)?;
88
89 let helix_root = sources.join("helix");
90
91 fetch_git_repository(GRAMMAR_REPOSITORY_URL, GRAMMAR_REPOSITORY_REF, &helix_root)
92 .context(GRAMMAR_REPOSITORY_URL)?;
93
94 let config = fs::read_to_string(helix_root.join(GRAMMAR_REPOSITORY_CONFIG_PATH))
95 .context("failed to read helix languages.toml")?;
96
97 // find the start of the language arrays & skip parsing everything before them (primarily because
98 // `basic_toml` can't handle some of the new syntax used in this file, and `toml` pulls in a lot
99 // of dependencies)
100 let language_defs_start = memchr::memmem::find(config.as_bytes(), b"[[language]]")
101 .context("languages.toml is missing languages")?;
102
103 let config = &config[language_defs_start..];
104
105 let config: HelixLanguages =
106 basic_toml::from_str(config).context("failed to parse helix languages.toml")?;
107
108 fetch_and_build_grammar(config.grammar.clone(), &sources)?;
109
110 (config, helix_root.join("runtime/queries"))
111 };
112
113 let mut grammar_defs = Vec::new();
114 for grammar in &config.grammar {
115 let name = &grammar.name;
116 if let Some(tokens) = build_language_module(name, query_path.as_path(), dylib)
117 .with_context(|| name.to_string())?
118 {
119 grammar_defs.push(tokens);
120 }
121 }
122 fs::write(
123 out_dir.join("grammar.defs.rs"),
124 prettyplease::unparse(
125 &syn::parse2(quote!(#(#grammar_defs)*)).context("failed to parse grammar defs")?,
126 ),
127 )
128 .context("failed to write grammar defs")?;
129
130 let registry = build_grammar_registry(config.grammar.iter().map(|v| v.name.clone()), dylib);
131 fs::write(
132 out_dir.join("grammar.registry.rs"),
133 prettyplease::unparse(&syn::parse2(registry).context("failed to parse grammar registry")?),
134 )
135 .context("failed to write grammar registry")?;
136
137 let language = build_language_registry(config.language, dylib)?;
138 fs::write(
139 out_dir.join("language.registry.rs"),
140 prettyplease::unparse(&syn::parse2(language)?),
141 )?;
142
143 Ok(())
144}
145
146fn build_language_registry(
147 language_definition: Vec<LanguageDefinition>,
148 dylib: bool,
149) -> anyhow::Result<proc_macro2::TokenStream> {
150 let mut camel = Vec::new();
151 let mut grammars = Vec::new();
152
153 let mut globs = Vec::new();
154 let mut globs_to_camel = Vec::new();
155
156 let mut injection_regex = Vec::new();
157 let mut injection_regex_str_len = Vec::new();
158 let mut regex_to_camel = Vec::new();
159
160 for language in &language_definition {
161 if BLACKLISTED_MODULES.contains(&language.name.as_str())
162 || (!dylib && BLACKLISTED_FOR_STATIC_LINKING.contains(&language.name.as_str()))
163 {
164 continue;
165 }
166
167 let camel_cased_name = format_ident!("{}", language.name.to_upper_camel_case());
168 camel.push(camel_cased_name.clone());
169
170 let grammar = language
171 .grammar
172 .as_deref()
173 .unwrap_or(language.name.as_str());
174 grammars.push(format_ident!("{}", grammar.to_upper_camel_case()));
175
176 for ty in &language.file_types {
177 match ty {
178 FileType::Glob { glob } => globs.push(Cow::Borrowed(glob)),
179 FileType::Extension(ext) => globs.push(Cow::Owned(format!("*.{ext}"))),
180 }
181
182 globs_to_camel.push(camel_cased_name.clone());
183 }
184
185 if let Some(regex) = language.injection_regex.as_deref() {
186 injection_regex.push(format!("^{regex}$"));
187 injection_regex_str_len.push(regex.len());
188 regex_to_camel.push(camel_cased_name.clone());
189 }
190 }
191
192 let injection_regex_len = injection_regex.len();
193 let globs_array_len = globs.len();
194 let globs_string_len = globs.iter().map(|v| v.len()).collect::<Vec<_>>();
195
196 Ok(quote! {
197 #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
198 pub enum Language {
199 #(#camel),*
200 }
201
202 impl Language {
203 pub const VARIANTS: &[Self] = &[
204 #(Self::#camel),*
205 ];
206
207 pub const fn grammar(self) -> Grammar {
208 match self {
209 #(Self::#camel => Grammar::#grammars),*
210 }
211 }
212
213 pub fn from_file_name<P: AsRef<::std::path::Path>>(name: P) -> Option<Self> {
214 const LENGTHS: [usize; #globs_array_len] = [#(#globs_string_len),*];
215 const GLOB_TO_VARIANT: [Language; #globs_array_len] = [#(Language::#globs_to_camel),*];
216
217 thread_local! {
218 static GLOB: ::std::cell::LazyCell<::globset::GlobSet> = ::std::cell::LazyCell::new(|| {
219 ::globset::GlobSetBuilder::new()
220 #(.add(::globset::Glob::new(#globs).unwrap()))*
221 .build()
222 .unwrap()
223 });
224 }
225
226 let mut max = usize::MAX;
227 let mut curr = None;
228
229 GLOB.with(|glob| {
230 for m in glob.matches(name) {
231 let curr_length = LENGTHS[m];
232
233 if curr_length < max {
234 max = curr_length;
235 curr = Some(GLOB_TO_VARIANT[m]);
236 }
237 }
238 });
239
240 curr
241 }
242
243 pub fn from_injection(name: &str) -> Option<Self> {
244 const LENGTHS: [usize; #injection_regex_len] = [#(#injection_regex_str_len),*];
245 const REGEX_TO_VARIANT: [Language; #injection_regex_len] = [#(Language::#regex_to_camel),*];
246
247 thread_local! {
248 static REGEX: ::std::cell::LazyCell<::regex::RegexSet> = ::std::cell::LazyCell::new(|| {
249 ::regex::RegexSet::new([
250 #(#injection_regex),*
251 ])
252 .unwrap()
253 });
254 }
255
256 let mut max = usize::MAX;
257 let mut curr = None;
258
259 REGEX.with(|regex| {
260 for m in regex.matches(name) {
261 let curr_length = LENGTHS[m];
262
263 if curr_length < max {
264 max = curr_length;
265 curr = Some(REGEX_TO_VARIANT[m]);
266 }
267 }
268 });
269
270 curr
271 }
272 }
273 })
274}
275
276fn build_grammar_registry(
277 names: impl Iterator<Item = String>,
278 dylib: bool,
279) -> proc_macro2::TokenStream {
280 let (ids, plain, camel, snake) = names
281 .filter(|name| {
282 !BLACKLISTED_MODULES.contains(&name.as_str())
283 && (dylib || !BLACKLISTED_FOR_STATIC_LINKING.contains(&name.as_str()))
284 })
285 .enumerate()
286 .fold(
287 (Vec::new(), Vec::new(), Vec::new(), Vec::new()),
288 |(mut ids, mut plain_acc, mut camel_acc, mut snake_acc), (i, name)| {
289 camel_acc.push(format_ident!("{}", name.to_upper_camel_case()));
290
291 if name == "move" {
292 snake_acc.push(format_ident!("r#{}", name.to_snake_case()));
293 } else {
294 snake_acc.push(format_ident!("{}", name.to_snake_case()));
295 }
296
297 plain_acc.push(name);
298
299 ids.push(i);
300 (ids, plain_acc, camel_acc, snake_acc)
301 },
302 );
303
304 quote! {
305 #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
306 pub enum Grammar {
307 #(#camel),*
308 }
309
310 impl Grammar {
311 pub const VARIANTS: &[Self] = &[
312 #(Self::#camel),*
313 ];
314
315 pub const fn highlight_configuration_params(self) -> crate::HighlightConfigurationParams {
316 match self {
317 #(Self::#camel => crate::HighlightConfigurationParams {
318 language: crate::grammar::#snake::LANGUAGE,
319 name: #plain,
320 highlights_query: crate::grammar::#snake::HIGHLIGHTS_QUERY,
321 injection_query: crate::grammar::#snake::INJECTIONS_QUERY,
322 locals_query: crate::grammar::#snake::LOCALS_QUERY,
323 }),*
324 }
325 }
326
327 pub const fn idx(self) -> usize {
328 match self {
329 #(Self::#camel => #ids),*
330 }
331 }
332 }
333 }
334}
335
336fn build_language_module(
337 name: &str,
338 query_path: &Path,
339 dylib: bool,
340) -> anyhow::Result<Option<proc_macro2::TokenStream>> {
341 if BLACKLISTED_MODULES.contains(&name)
342 || (!dylib && BLACKLISTED_FOR_STATIC_LINKING.contains(&name))
343 {
344 return Ok(None);
345 }
346
347 let highlights_query = read_local_query(query_path, name, "highlights.scm");
348 let injections_query = read_local_query(query_path, name, "injections.scm");
349 let locals_query = read_local_query(query_path, name, "locals.scm");
350
351 let ffi = format_ident!("tree_sitter_{}", name.to_snake_case());
352 let name = if name == "move" {
353 format_ident!("r#{}", name.to_snake_case())
354 } else {
355 format_ident!("{}", name.to_snake_case())
356 };
357
358 Ok(Some(quote! {
359 pub mod #name {
360 unsafe extern "C" {
361 fn #ffi() -> *const ();
362 }
363
364 pub const LANGUAGE: tree_sitter_language::LanguageFn = unsafe { tree_sitter_language::LanguageFn::from_raw(#ffi) };
365 pub const HIGHLIGHTS_QUERY: &str = #highlights_query;
366 pub const INJECTIONS_QUERY: &str = #injections_query;
367 pub const LOCALS_QUERY: &str = #locals_query;
368 }
369 }))
370}
371
372// taken from https://github.com/helix-editor/helix/blob/2ce4c6d5fa3e50464b41a3d0190ad0e5ada2fc3c/helix-core/src/syntax.rs#L721
373fn read_local_query(query_path: &Path, language: &str, filename: &str) -> String {
374 static INHERITS_REGEX: LazyLock<regex::Regex> =
375 LazyLock::new(|| regex::Regex::new(r";+\s*inherits\s*:?\s*([a-z_,()-]+)\s*").unwrap());
376
377 let path = query_path.join(language).join(filename);
378
379 if !path.exists() {
380 return String::new();
381 }
382
383 let query =
384 fs::read_to_string(&path).unwrap_or_else(|e| panic!("failed to fetch {path:?}: {e:?}"));
385
386 INHERITS_REGEX
387 .replace_all(&query, |captures: ®ex::Captures| {
388 captures[1]
389 .split(',')
390 .fold(String::new(), |mut output, language| {
391 // `write!` to a String cannot fail.
392 write!(
393 output,
394 "\n{}\n",
395 read_local_query(query_path, language, filename)
396 )
397 .unwrap();
398 output
399 })
400 })
401 .to_string()
402}
403
404fn fetch_and_build_grammar(
405 grammars: Vec<GrammarDefinition>,
406 source_dir: &Path,
407) -> anyhow::Result<()> {
408 let pool = ThreadPool::new(std::thread::available_parallelism()?.get());
409
410 for grammar in grammars {
411 if BLACKLISTED_MODULES.contains(&grammar.name.as_str())
412 || BLACKLISTED_FOR_STATIC_LINKING.contains(&grammar.name.as_str())
413 {
414 continue;
415 }
416
417 let mut grammar_root = source_dir.join(&grammar.name);
418
419 pool.execute(move || {
420 let grammar_root = match grammar.source {
421 GrammarSource::Git {
422 remote,
423 revision,
424 subpath,
425 } => {
426 fetch_git_repository(&remote, &revision, &grammar_root)
427 .context(GRAMMAR_REPOSITORY_URL)
428 .expect("failed to fetch git repository");
429
430 if let Some(subpath) = subpath {
431 grammar_root.push(subpath);
432 }
433
434 grammar_root
435 }
436 GrammarSource::Local { path } => path,
437 };
438
439 let grammar_src = grammar_root.join("src");
440
441 let parser_file = Some(grammar_src.join("parser.c"))
442 .filter(|s| s.exists())
443 .or_else(|| Some(grammar_src.join("parser.cc")))
444 .filter(|s| s.exists());
445 let scanner_file = Some(grammar_src.join("scanner.c"))
446 .filter(|s| s.exists())
447 .or_else(|| Some(grammar_src.join("scanner.cc")))
448 .filter(|s| s.exists());
449
450 if let Some(parser_file) = parser_file {
451 cc::Build::new()
452 .cpp(parser_file.extension() == Some(OsStr::new("cc")))
453 .file(parser_file)
454 .flag_if_supported("-w")
455 .flag_if_supported("-s")
456 .include(&grammar_src)
457 .compile(&format!("{}-parser", grammar.name));
458 }
459
460 if let Some(scanner_file) = scanner_file {
461 cc::Build::new()
462 .cpp(scanner_file.extension() == Some(OsStr::new("cc")))
463 .file(scanner_file)
464 .flag_if_supported("-w")
465 .flag_if_supported("-s")
466 .include(&grammar_src)
467 .compile(&format!("{}-scanner", grammar.name));
468 }
469 });
470 }
471
472 pool.join();
473
474 Ok(())
475}
476
477fn fetch_git_repository(url: &str, ref_: &str, destination: &Path) -> anyhow::Result<()> {
478 if !destination.exists() {
479 let res = Command::new("git").arg("init").arg(destination).status()?;
480 if !res.success() {
481 bail!("git init failed with exit code {res}");
482 }
483
484 let res = Command::new("git")
485 .args(["remote", "add", "origin", url])
486 .current_dir(destination)
487 .status()?;
488 if !res.success() {
489 bail!("git remote failed with exit code {res}");
490 }
491 }
492
493 let res = Command::new("git")
494 .args(["rev-parse", "HEAD"])
495 .current_dir(destination)
496 .output()?
497 .stdout;
498 if res == ref_.as_bytes() {
499 return Ok(());
500 }
501
502 let res = Command::new("git")
503 .args(["fetch", "--depth", "1", "origin", ref_])
504 .current_dir(destination)
505 .status()?;
506 if !res.success() {
507 bail!("git fetch failed with exit code {res}");
508 }
509
510 let res = Command::new("git")
511 .args(["reset", "--hard", ref_])
512 .current_dir(destination)
513 .status()?;
514 if !res.success() {
515 bail!("git fetch failed with exit code {res}");
516 }
517
518 Ok(())
519}
520
521#[derive(Deserialize)]
522#[serde(rename_all = "kebab-case")]
523struct LanguageDefinition {
524 name: String,
525 injection_regex: Option<String>,
526 file_types: Vec<FileType>,
527 grammar: Option<String>,
528}
529
530#[derive(Deserialize)]
531#[serde(untagged)]
532pub enum FileType {
533 Glob { glob: String },
534 Extension(String),
535}
536
537#[derive(Deserialize, Clone)]
538#[serde(rename_all = "kebab-case")]
539pub struct GrammarDefinition {
540 name: String,
541 source: GrammarSource,
542}
543
544#[derive(Deserialize, Clone)]
545#[serde(rename_all = "lowercase", untagged)]
546enum GrammarSource {
547 Git {
548 #[serde(rename = "git")]
549 remote: String,
550 #[serde(rename = "rev")]
551 revision: String,
552 subpath: Option<String>,
553 },
554 Local {
555 path: PathBuf,
556 },
557}
558
559#[derive(Deserialize)]
560#[serde(rename_all = "kebab-case")]
561struct HelixLanguages {
562 language: Vec<LanguageDefinition>,
563 grammar: Vec<GrammarDefinition>,
564}