1use std::{
2 borrow::Cow,
3 ffi::OsStr,
4 fmt::Write,
5 fs,
6 path::{Path, PathBuf},
7 process::Command,
8 sync::LazyLock,
9};
10
11use anyhow::{bail, Context};
12use heck::{ToSnakeCase, ToUpperCamelCase};
13use quote::{format_ident, quote};
14use serde::Deserialize;
15use threadpool::ThreadPool;
16
17const GRAMMAR_REPOSITORY_URL: &str = "https://github.com/helix-editor/helix";
18const GRAMMAR_REPOSITORY_REF: &str = "82dd96369302f60a9c83a2d54d021458f82bcd36";
19const GRAMMAR_REPOSITORY_CONFIG_PATH: &str = "languages.toml";
20
21static BLACKLISTED_MODULES: &[&str] = &[
22 // these languages all don't have corresponding grammars
23 "cabal",
24 "idris",
25 "llvm-mir-yaml",
26 "prolog",
27 "mint",
28 "hare",
29 "wren",
30 // doesn't compile on macos
31 "gemini",
32];
33
34fn main() -> anyhow::Result<()> {
35 let out_dir = PathBuf::from(std::env::var("OUT_DIR").context("OUT_DIR not set by rustc")?);
36
37 let root = std::env::var("TREE_SITTER_GRAMMAR_LIB_DIR").ok();
38 println!("cargo::rerun-if-env-changed=TREE_SITTER_GRAMMAR_LIB_DIR");
39
40 let (root, dylib) = if let Some(root) = root.as_deref() {
41 (Path::new(root), true)
42 } else {
43 (out_dir.as_path(), false)
44 };
45
46 let (config, query_path) = if dylib {
47 let config = fs::read_to_string(root.join("languages.toml"))
48 .context("failed to read helix languages.toml")?;
49
50 // find the start of the language arrays & skip parsing everything before them (primarily because
51 // `basic_toml` can't handle some of the new syntax used in this file, and `toml` pulls in a lot
52 // of dependencies)
53 let language_defs_start = memchr::memmem::find(config.as_bytes(), b"[[language]]")
54 .context("languages.toml is missing languages")?;
55
56 let config = &config[language_defs_start..];
57
58 let config: HelixLanguages =
59 basic_toml::from_str(config).context("failed to parse helix languages.toml")?;
60
61 println!("cargo::rustc-link-search=native={}", root.display());
62
63 for grammar in &config.grammar {
64 if BLACKLISTED_MODULES.contains(&grammar.name.as_str()) {
65 continue;
66 }
67
68 println!("cargo::rustc-link-lib=dylib=tree-sitter-{}", grammar.name);
69 }
70
71 (config, root.join("queries"))
72 } else {
73 let sources = out_dir.join("sources");
74 fs::create_dir_all(&sources)?;
75
76 let helix_root = sources.join("helix");
77
78 fetch_git_repository(GRAMMAR_REPOSITORY_URL, GRAMMAR_REPOSITORY_REF, &helix_root)
79 .context(GRAMMAR_REPOSITORY_URL)?;
80
81 let config = fs::read_to_string(helix_root.join(GRAMMAR_REPOSITORY_CONFIG_PATH))
82 .context("failed to read helix languages.toml")?;
83
84 // find the start of the language arrays & skip parsing everything before them (primarily because
85 // `basic_toml` can't handle some of the new syntax used in this file, and `toml` pulls in a lot
86 // of dependencies)
87 let language_defs_start = memchr::memmem::find(config.as_bytes(), b"[[language]]")
88 .context("languages.toml is missing languages")?;
89
90 let config = &config[language_defs_start..];
91
92 let config: HelixLanguages =
93 basic_toml::from_str(config).context("failed to parse helix languages.toml")?;
94
95 fetch_and_build_grammar(config.grammar.clone(), &sources)?;
96
97 (config, helix_root.join("runtime/queries"))
98 };
99
100 let mut grammar_defs = Vec::new();
101 for grammar in &config.grammar {
102 let name = &grammar.name;
103 if let Some(tokens) =
104 build_language_module(name, query_path.as_path()).with_context(|| name.to_string())?
105 {
106 grammar_defs.push(tokens);
107 }
108 }
109 fs::write(
110 out_dir.join("grammar.defs.rs"),
111 prettyplease::unparse(
112 &syn::parse2(quote!(#(#grammar_defs)*)).context("failed to parse grammar defs")?,
113 ),
114 )
115 .context("failed to write grammar defs")?;
116
117 let registry = build_grammar_registry(config.grammar.iter().map(|v| v.name.clone()));
118 fs::write(
119 out_dir.join("grammar.registry.rs"),
120 prettyplease::unparse(&syn::parse2(registry).context("failed to parse grammar registry")?),
121 )
122 .context("failed to write grammar registry")?;
123
124 let language = build_language_registry(config.language)?;
125 fs::write(
126 out_dir.join("language.registry.rs"),
127 prettyplease::unparse(&syn::parse2(language)?),
128 )?;
129
130 Ok(())
131}
132
133fn build_language_registry(
134 language_definition: Vec<LanguageDefinition>,
135) -> anyhow::Result<proc_macro2::TokenStream> {
136 let mut camel = Vec::new();
137 let mut grammars = Vec::new();
138
139 let mut globs = Vec::new();
140 let mut globs_to_camel = Vec::new();
141
142 let mut injection_regex = Vec::new();
143 let mut injection_regex_str_len = Vec::new();
144 let mut regex_to_camel = Vec::new();
145
146 for language in &language_definition {
147 if BLACKLISTED_MODULES.contains(&language.name.as_str()) {
148 continue;
149 }
150
151 let camel_cased_name = format_ident!("{}", language.name.to_upper_camel_case());
152 camel.push(camel_cased_name.clone());
153
154 let grammar = language
155 .grammar
156 .as_deref()
157 .unwrap_or(language.name.as_str());
158 grammars.push(format_ident!("{}", grammar.to_upper_camel_case()));
159
160 for ty in &language.file_types {
161 match ty {
162 FileType::Glob { glob } => globs.push(Cow::Borrowed(glob)),
163 FileType::Extension(ext) => globs.push(Cow::Owned(format!("*.{ext}"))),
164 }
165
166 globs_to_camel.push(camel_cased_name.clone());
167 }
168
169 if let Some(regex) = language.injection_regex.as_deref() {
170 injection_regex.push(format!("^{regex}$"));
171 injection_regex_str_len.push(regex.len());
172 regex_to_camel.push(camel_cased_name.clone());
173 }
174 }
175
176 let injection_regex_len = injection_regex.len();
177 let globs_array_len = globs.len();
178 let globs_string_len = globs.iter().map(|v| v.len()).collect::<Vec<_>>();
179
180 Ok(quote! {
181 #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
182 pub enum Language {
183 #(#camel),*
184 }
185
186 impl Language {
187 pub const VARIANTS: &[Self] = &[
188 #(Self::#camel),*
189 ];
190
191 pub const fn grammar(self) -> Grammar {
192 match self {
193 #(Self::#camel => Grammar::#grammars),*
194 }
195 }
196
197 pub fn from_file_name<P: AsRef<::std::path::Path>>(name: P) -> Option<Self> {
198 const LENGTHS: [usize; #globs_array_len] = [#(#globs_string_len),*];
199 const GLOB_TO_VARIANT: [Language; #globs_array_len] = [#(Language::#globs_to_camel),*];
200
201 thread_local! {
202 static GLOB: ::std::cell::LazyCell<::globset::GlobSet> = ::std::cell::LazyCell::new(|| {
203 ::globset::GlobSetBuilder::new()
204 #(.add(::globset::Glob::new(#globs).unwrap()))*
205 .build()
206 .unwrap()
207 });
208 }
209
210 let mut max = usize::MAX;
211 let mut curr = None;
212
213 GLOB.with(|glob| {
214 for m in glob.matches(name) {
215 let curr_length = LENGTHS[m];
216
217 if curr_length < max {
218 max = curr_length;
219 curr = Some(GLOB_TO_VARIANT[m]);
220 }
221 }
222 });
223
224 curr
225 }
226
227 pub fn from_injection(name: &str) -> Option<Self> {
228 const LENGTHS: [usize; #injection_regex_len] = [#(#injection_regex_str_len),*];
229 const REGEX_TO_VARIANT: [Language; #injection_regex_len] = [#(Language::#regex_to_camel),*];
230
231 thread_local! {
232 static REGEX: ::std::cell::LazyCell<::regex::RegexSet> = ::std::cell::LazyCell::new(|| {
233 ::regex::RegexSet::new([
234 #(#injection_regex),*
235 ])
236 .unwrap()
237 });
238 }
239
240 let mut max = usize::MAX;
241 let mut curr = None;
242
243 REGEX.with(|regex| {
244 for m in regex.matches(name) {
245 let curr_length = LENGTHS[m];
246
247 if curr_length < max {
248 max = curr_length;
249 curr = Some(REGEX_TO_VARIANT[m]);
250 }
251 }
252 });
253
254 curr
255 }
256 }
257 })
258}
259
260fn build_grammar_registry(names: impl Iterator<Item = String>) -> proc_macro2::TokenStream {
261 let (ids, plain, camel, snake) = names
262 .filter(|name| !BLACKLISTED_MODULES.contains(&name.as_str()))
263 .enumerate()
264 .fold(
265 (Vec::new(), Vec::new(), Vec::new(), Vec::new()),
266 |(mut ids, mut plain_acc, mut camel_acc, mut snake_acc), (i, name)| {
267 camel_acc.push(format_ident!("{}", name.to_upper_camel_case()));
268
269 if name == "move" {
270 snake_acc.push(format_ident!("r#{}", name.to_snake_case()));
271 } else {
272 snake_acc.push(format_ident!("{}", name.to_snake_case()));
273 }
274
275 plain_acc.push(name);
276
277 ids.push(i);
278 (ids, plain_acc, camel_acc, snake_acc)
279 },
280 );
281
282 quote! {
283 #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
284 pub enum Grammar {
285 #(#camel),*
286 }
287
288 impl Grammar {
289 pub const VARIANTS: &[Self] = &[
290 #(Self::#camel),*
291 ];
292
293 pub const fn highlight_configuration_params(self) -> crate::HighlightConfigurationParams {
294 match self {
295 #(Self::#camel => crate::HighlightConfigurationParams {
296 language: crate::grammar::#snake::LANGUAGE,
297 name: #plain,
298 highlights_query: crate::grammar::#snake::HIGHLIGHTS_QUERY,
299 injection_query: crate::grammar::#snake::INJECTIONS_QUERY,
300 locals_query: crate::grammar::#snake::LOCALS_QUERY,
301 }),*
302 }
303 }
304
305 pub const fn idx(self) -> usize {
306 match self {
307 #(Self::#camel => #ids),*
308 }
309 }
310 }
311 }
312}
313
314fn build_language_module(
315 name: &str,
316 query_path: &Path,
317) -> anyhow::Result<Option<proc_macro2::TokenStream>> {
318 if BLACKLISTED_MODULES.contains(&name) {
319 return Ok(None);
320 }
321
322 let highlights_query = read_local_query(query_path, name, "highlights.scm");
323 let injections_query = read_local_query(query_path, name, "injections.scm");
324 let locals_query = read_local_query(query_path, name, "locals.scm");
325
326 let ffi = format_ident!("tree_sitter_{}", name.to_snake_case());
327 let name = if name == "move" {
328 format_ident!("r#{}", name.to_snake_case())
329 } else {
330 format_ident!("{}", name.to_snake_case())
331 };
332
333 Ok(Some(quote! {
334 pub mod #name {
335 extern "C" {
336 fn #ffi() -> *const ();
337 }
338
339 pub const LANGUAGE: tree_sitter_language::LanguageFn = unsafe { tree_sitter_language::LanguageFn::from_raw(#ffi) };
340 pub const HIGHLIGHTS_QUERY: &str = #highlights_query;
341 pub const INJECTIONS_QUERY: &str = #injections_query;
342 pub const LOCALS_QUERY: &str = #locals_query;
343 }
344 }))
345}
346
347// taken from https://github.com/helix-editor/helix/blob/2ce4c6d5fa3e50464b41a3d0190ad0e5ada2fc3c/helix-core/src/syntax.rs#L721
348fn read_local_query(query_path: &Path, language: &str, filename: &str) -> String {
349 static INHERITS_REGEX: LazyLock<regex::Regex> =
350 LazyLock::new(|| regex::Regex::new(r";+\s*inherits\s*:?\s*([a-z_,()-]+)\s*").unwrap());
351
352 let path = query_path.join(language).join(filename);
353
354 if !path.exists() {
355 return String::new();
356 }
357
358 let query =
359 fs::read_to_string(&path).unwrap_or_else(|e| panic!("failed to fetch {path:?}: {e:?}"));
360
361 INHERITS_REGEX
362 .replace_all(&query, |captures: ®ex::Captures| {
363 captures[1]
364 .split(',')
365 .fold(String::new(), |mut output, language| {
366 // `write!` to a String cannot fail.
367 write!(
368 output,
369 "\n{}\n",
370 read_local_query(query_path, language, filename)
371 )
372 .unwrap();
373 output
374 })
375 })
376 .to_string()
377}
378
379fn fetch_and_build_grammar(
380 grammars: Vec<GrammarDefinition>,
381 source_dir: &Path,
382) -> anyhow::Result<()> {
383 let pool = ThreadPool::new(std::thread::available_parallelism()?.get());
384
385 for grammar in grammars {
386 if BLACKLISTED_MODULES.contains(&grammar.name.as_str()) {
387 continue;
388 }
389
390 let mut grammar_root = source_dir.join(&grammar.name);
391
392 pool.execute(move || {
393 let grammar_root = match grammar.source {
394 GrammarSource::Git {
395 remote,
396 revision,
397 subpath,
398 } => {
399 fetch_git_repository(&remote, &revision, &grammar_root)
400 .context(GRAMMAR_REPOSITORY_URL)
401 .expect("failed to fetch git repository");
402
403 if let Some(subpath) = subpath {
404 grammar_root.push(subpath);
405 }
406
407 grammar_root
408 }
409 GrammarSource::Local { path } => path,
410 };
411
412 let grammar_src = grammar_root.join("src");
413
414 let parser_file = Some(grammar_src.join("parser.c"))
415 .filter(|s| s.exists())
416 .or_else(|| Some(grammar_src.join("parser.cc")))
417 .filter(|s| s.exists());
418 let scanner_file = Some(grammar_src.join("scanner.c"))
419 .filter(|s| s.exists())
420 .or_else(|| Some(grammar_src.join("scanner.cc")))
421 .filter(|s| s.exists());
422
423 if let Some(parser_file) = parser_file {
424 cc::Build::new()
425 .cpp(parser_file.extension() == Some(OsStr::new("cc")))
426 .file(parser_file)
427 .flag_if_supported("-w")
428 .flag_if_supported("-s")
429 .include(&grammar_src)
430 .compile(&format!("{}-parser", grammar.name));
431 }
432
433 if let Some(scanner_file) = scanner_file {
434 cc::Build::new()
435 .cpp(scanner_file.extension() == Some(OsStr::new("cc")))
436 .file(scanner_file)
437 .flag_if_supported("-w")
438 .flag_if_supported("-s")
439 .include(&grammar_src)
440 .compile(&format!("{}-scanner", grammar.name));
441 }
442 });
443 }
444
445 pool.join();
446
447 Ok(())
448}
449
450fn fetch_git_repository(url: &str, ref_: &str, destination: &Path) -> anyhow::Result<()> {
451 if !destination.exists() {
452 let res = Command::new("git").arg("init").arg(destination).status()?;
453 if !res.success() {
454 bail!("git init failed with exit code {res}");
455 }
456
457 let res = Command::new("git")
458 .args(["remote", "add", "origin", url])
459 .current_dir(destination)
460 .status()?;
461 if !res.success() {
462 bail!("git remote failed with exit code {res}");
463 }
464 }
465
466 let res = Command::new("git")
467 .args(["rev-parse", "HEAD"])
468 .current_dir(destination)
469 .output()?
470 .stdout;
471 if res == ref_.as_bytes() {
472 return Ok(());
473 }
474
475 let res = Command::new("git")
476 .args(["fetch", "--depth", "1", "origin", ref_])
477 .current_dir(destination)
478 .status()?;
479 if !res.success() {
480 bail!("git fetch failed with exit code {res}");
481 }
482
483 let res = Command::new("git")
484 .args(["reset", "--hard", ref_])
485 .current_dir(destination)
486 .status()?;
487 if !res.success() {
488 bail!("git fetch failed with exit code {res}");
489 }
490
491 Ok(())
492}
493
494#[derive(Deserialize)]
495#[serde(rename_all = "kebab-case")]
496struct LanguageDefinition {
497 name: String,
498 injection_regex: Option<String>,
499 file_types: Vec<FileType>,
500 grammar: Option<String>,
501}
502
503#[derive(Deserialize)]
504#[serde(untagged)]
505pub enum FileType {
506 Glob { glob: String },
507 Extension(String),
508}
509
510#[derive(Deserialize, Clone)]
511#[serde(rename_all = "kebab-case")]
512pub struct GrammarDefinition {
513 name: String,
514 source: GrammarSource,
515}
516
517#[derive(Deserialize, Clone)]
518#[serde(rename_all = "lowercase", untagged)]
519enum GrammarSource {
520 Git {
521 #[serde(rename = "git")]
522 remote: String,
523 #[serde(rename = "rev")]
524 revision: String,
525 subpath: Option<String>,
526 },
527 Local {
528 path: PathBuf,
529 },
530}
531
532#[derive(Deserialize)]
533#[serde(rename_all = "kebab-case")]
534struct HelixLanguages {
535 language: Vec<LanguageDefinition>,
536 grammar: Vec<GrammarDefinition>,
537}