From 4b863f4fb046cf41bcaa078de911e3ccde1795e0 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Sun, 21 Jul 2024 19:31:10 -0700 Subject: [PATCH 01/11] [skip ci] yolo --- crates/core/src/pattern_compiler/compiler.rs | 10 ++++++++++ crates/gritmodule/src/patterns_directory.rs | 15 ++++++--------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/crates/core/src/pattern_compiler/compiler.rs b/crates/core/src/pattern_compiler/compiler.rs index 8d9afe4bb..9951629f7 100644 --- a/crates/core/src/pattern_compiler/compiler.rs +++ b/crates/core/src/pattern_compiler/compiler.rs @@ -497,6 +497,13 @@ pub(crate) fn filter_libs( let mut filtered: BTreeMap = BTreeMap::new(); // gross but necessary due to running these patterns befor and after each file + println!( + "Starting filtering with {} and {} libs, with autowrap: {}", + src, + libs.len(), + will_autowrap + ); + let mut stack: Vec = if will_autowrap { let before_each_file = "before_each_file()"; let before_tree = @@ -545,6 +552,9 @@ pub(crate) fn filter_libs( } } } + + println!("Filtered to {:?} libs", filtered.len()); + Ok(filtered.into_iter().collect_vec()) } diff --git a/crates/gritmodule/src/patterns_directory.rs b/crates/gritmodule/src/patterns_directory.rs index 2e13d3ee6..a0d749099 100644 --- a/crates/gritmodule/src/patterns_directory.rs +++ b/crates/gritmodule/src/patterns_directory.rs @@ -119,7 +119,7 @@ impl PatternsDirectory { } } - pub fn get_language_directory(&self, lang: PatternLanguage) -> &BTreeMap { + fn get_language_directory(&self, lang: PatternLanguage) -> &BTreeMap { match lang { PatternLanguage::JavaScript => &self.java_script, PatternLanguage::TypeScript => &self.type_script, @@ -176,12 +176,6 @@ impl PatternsDirectory { self.get_language_and_universal_directory(language) } - fn get_language_directory_from_name(&self, name: &str) -> Option<&BTreeMap> { - self.pattern_to_language - .get(name) - .map(|l| self.get_language_directory(*l)) - } - // imo we should check if name matches [a-z][a-z0-9]* // as currently a pattern with no language header and an invalid pattern are // both treated as js patterns when the latter should be a not found error @@ -196,8 +190,11 @@ impl PatternsDirectory { } pub fn get(&self, name: &str) -> Option<&String> { - self.get_language_directory_from_name(name) - .and_then(|d| d.get(name)) + self.pattern_to_language + .get(name) + .map(|l| self.get_language_and_universal_directory(*l).ok()) + .flatten() + .and_then(|d| d.get(name).) } // do we want to do an overriding insert? From 1d188faf7b1dd40b409dd936bd001bfc01465603 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Sun, 21 Jul 2024 19:37:17 -0700 Subject: [PATCH 02/11] run the tests --- crates/gritmodule/src/patterns_directory.rs | 28 +++++++++++++++------ 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/crates/gritmodule/src/patterns_directory.rs b/crates/gritmodule/src/patterns_directory.rs index a0d749099..3a2c61cfa 100644 --- a/crates/gritmodule/src/patterns_directory.rs +++ b/crates/gritmodule/src/patterns_directory.rs @@ -147,6 +147,10 @@ impl PatternsDirectory { } } + fn get_universal(&self) -> &BTreeMap { + self.get_language_directory(PatternLanguage::Universal) + } + #[tracing::instrument] fn get_language_and_universal_directory( &self, @@ -157,9 +161,7 @@ impl PatternsDirectory { }; let lang_library = self.get_language_directory(language); let mut lang_library = lang_library.to_owned(); - let universal = self - .get_language_directory(PatternLanguage::Universal) - .to_owned(); + let universal = self.get_universal().to_owned(); let count = lang_library.len() + universal.len(); lang_library.extend(universal); if count != lang_library.len() { @@ -179,6 +181,7 @@ impl PatternsDirectory { // imo we should check if name matches [a-z][a-z0-9]* // as currently a pattern with no language header and an invalid pattern are // both treated as js patterns when the latter should be a not found error + #[deprecated = "use get_language_directory_or_default instead"] pub fn get_pattern_libraries(&self, root_pattern: &str) -> Result { let language = self .pattern_to_language @@ -189,12 +192,23 @@ impl PatternsDirectory { Ok(LanguageLibrary::new(language, library)) } - pub fn get(&self, name: &str) -> Option<&String> { + fn get_language_directory_from_name(&self, name: &str) -> Option<&BTreeMap> { self.pattern_to_language .get(name) - .map(|l| self.get_language_and_universal_directory(*l).ok()) - .flatten() - .and_then(|d| d.get(name).) + .map(|l| self.get_language_directory(*l)) + } + + pub fn get(&self, name: &str) -> Option<&String> { + if let Some(dir) = self.get_language_directory_from_name(name) { + if let Some(pattern) = dir.get(name) { + return Some(pattern); + } + } else if let Some(universal_dir) = self.get_universal() { + if let Some(pattern) = universal_dir.get(name) { + return Some(pattern); + } + } + None } // do we want to do an overriding insert? From a641417b8049bd352072e2d82c6e4d83c46e8779 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Sun, 21 Jul 2024 19:38:32 -0700 Subject: [PATCH 03/11] run the tests --- crates/gritmodule/src/patterns_directory.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/crates/gritmodule/src/patterns_directory.rs b/crates/gritmodule/src/patterns_directory.rs index 3a2c61cfa..1d2a88111 100644 --- a/crates/gritmodule/src/patterns_directory.rs +++ b/crates/gritmodule/src/patterns_directory.rs @@ -181,7 +181,6 @@ impl PatternsDirectory { // imo we should check if name matches [a-z][a-z0-9]* // as currently a pattern with no language header and an invalid pattern are // both treated as js patterns when the latter should be a not found error - #[deprecated = "use get_language_directory_or_default instead"] pub fn get_pattern_libraries(&self, root_pattern: &str) -> Result { let language = self .pattern_to_language @@ -203,10 +202,8 @@ impl PatternsDirectory { if let Some(pattern) = dir.get(name) { return Some(pattern); } - } else if let Some(universal_dir) = self.get_universal() { - if let Some(pattern) = universal_dir.get(name) { - return Some(pattern); - } + } else if let Some(pattern) = self.get_universal().get(name) { + return Some(pattern); } None } From 95c35088fdfdf0695ab984808d968848c6b98443 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Sun, 21 Jul 2024 19:55:15 -0700 Subject: [PATCH 04/11] run the tests --- crates/core/src/pattern_compiler/compiler.rs | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/crates/core/src/pattern_compiler/compiler.rs b/crates/core/src/pattern_compiler/compiler.rs index 9951629f7..d7169c524 100644 --- a/crates/core/src/pattern_compiler/compiler.rs +++ b/crates/core/src/pattern_compiler/compiler.rs @@ -495,15 +495,8 @@ pub(crate) fn filter_libs( foreign_functions: foreign_file, } = defs_to_filenames(libs, parser, tree.root_node())?; let mut filtered: BTreeMap = BTreeMap::new(); - // gross but necessary due to running these patterns befor and after each file - - println!( - "Starting filtering with {} and {} libs, with autowrap: {}", - src, - libs.len(), - will_autowrap - ); + // gross but necessary due to running these patterns befor and after each file let mut stack: Vec = if will_autowrap { let before_each_file = "before_each_file()"; let before_tree = @@ -553,8 +546,6 @@ pub(crate) fn filter_libs( } } - println!("Filtered to {:?} libs", filtered.len()); - Ok(filtered.into_iter().collect_vec()) } From e19607b0d9bb0d97330329237512e702647bcd31 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Sun, 21 Jul 2024 20:01:32 -0700 Subject: [PATCH 05/11] run the tests --- crates/cli/src/analyze.rs | 3 ++- crates/core/src/built_in_functions.rs | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/crates/cli/src/analyze.rs b/crates/cli/src/analyze.rs index 8522d66f2..edf3a4e94 100644 --- a/crates/cli/src/analyze.rs +++ b/crates/cli/src/analyze.rs @@ -49,7 +49,8 @@ impl<'b> RichPattern<'b> { ) -> Result { let lang = language.unwrap_or_default(); #[cfg(not(feature = "ai_builtins"))] - let injected_builtins: Option = None; + let injected_builtins: Option = + marzano_core::built_in_functions::get_ai_placeholder_functions(); #[cfg(feature = "ai_builtins")] let injected_builtins = Some(ai_builtins::ai_builtins::get_ai_built_in_functions()); diff --git a/crates/core/src/built_in_functions.rs b/crates/core/src/built_in_functions.rs index 737ecb8f5..4e4722cfb 100644 --- a/crates/core/src/built_in_functions.rs +++ b/crates/core/src/built_in_functions.rs @@ -494,3 +494,26 @@ fn length_fn<'a>( None => Err(anyhow!("length argument must be a list or string")), } } + +pub fn get_ai_placeholder_functions() -> Option { + Some( + vec![ + BuiltInFunction::new( + "llm_chat", + vec!["model", "messages", "pattern"], + Box::new(ai_fn_placholder), + ), + BuiltInFunction::new("embedding", vec!["target"], Box::new(ai_fn_placholder)), + ] + .into(), + ) +} + +fn ai_fn_placholder<'a>( + args: &'a [Option>], + context: &'a MarzanoContext<'a>, + state: &mut State<'a, MarzanoQueryContext>, + logs: &mut AnalysisLogs, +) -> Result> { + bail!("AI features are not supported in your GritQL distribution. Please upgrade to the Enterprise version to use AI features.") +} From 3e3652db4e8a399252788ca8559e561e06095a5c Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Sun, 21 Jul 2024 20:02:26 -0700 Subject: [PATCH 06/11] run the tests --- crates/lsp/src/util.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/lsp/src/util.rs b/crates/lsp/src/util.rs index 338a4aa21..7b8257c30 100644 --- a/crates/lsp/src/util.rs +++ b/crates/lsp/src/util.rs @@ -126,7 +126,7 @@ pub fn check_intersection(range1: &Range, range2: &Range) -> bool { pub(crate) fn get_ai_built_in_functions_for_feature() -> Option { #[cfg(not(feature = "ai_builtins"))] - return None; + return marzano_core::built_in_functions::get_ai_placeholder_functions(); #[cfg(feature = "ai_builtins")] return Some(ai_builtins::ai_builtins::get_ai_built_in_functions()); } From 4a46c95ffb19c86d1d8e80abb864a309f94f8d6f Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Sun, 21 Jul 2024 20:08:35 -0700 Subject: [PATCH 07/11] run the tests --- crates/core/src/built_in_functions.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/core/src/built_in_functions.rs b/crates/core/src/built_in_functions.rs index 4e4722cfb..ff9530c13 100644 --- a/crates/core/src/built_in_functions.rs +++ b/crates/core/src/built_in_functions.rs @@ -510,10 +510,10 @@ pub fn get_ai_placeholder_functions() -> Option { } fn ai_fn_placholder<'a>( - args: &'a [Option>], - context: &'a MarzanoContext<'a>, - state: &mut State<'a, MarzanoQueryContext>, - logs: &mut AnalysisLogs, + _args: &'a [Option>], + _context: &'a MarzanoContext<'a>, + _state: &mut State<'a, MarzanoQueryContext>, + _logs: &mut AnalysisLogs, ) -> Result> { bail!("AI features are not supported in your GritQL distribution. Please upgrade to the Enterprise version to use AI features.") } From 04843e213dec761be73ff51f7062ae5791899dee Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Sun, 21 Jul 2024 20:27:10 -0700 Subject: [PATCH 08/11] run the tests --- crates/core/src/problem.rs | 7 +++++-- crates/grit-pattern-matcher/src/context.rs | 7 +++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/crates/core/src/problem.rs b/crates/core/src/problem.rs index f8c7fa7da..3569dea65 100644 --- a/crates/core/src/problem.rs +++ b/crates/core/src/problem.rs @@ -68,11 +68,14 @@ impl Problem { } pub fn definitions(&self) -> StaticDefinitions<'_, MarzanoQueryContext> { - StaticDefinitions::new( + let mut defs = StaticDefinitions::new( &self.pattern_definitions, &self.predicate_definitions, &self.function_definitions, - ) + ); + // We use the first 3 indexes for auto-wrap stuff + defs.skippable_indexes = vec![0, 1, 2]; + defs } } diff --git a/crates/grit-pattern-matcher/src/context.rs b/crates/grit-pattern-matcher/src/context.rs index c58529ba3..294b853fd 100644 --- a/crates/grit-pattern-matcher/src/context.rs +++ b/crates/grit-pattern-matcher/src/context.rs @@ -75,6 +75,8 @@ pub struct StaticDefinitions<'a, Q: QueryContext> { pattern_definitions: &'a [PatternDefinition], predicate_definitions: &'a [PredicateDefinition], function_definitions: &'a [GritFunctionDefinition], + /// Pattern indexes we should skip during analysis (before_each_file / after_each_file) + pub skippable_indexes: Vec, } impl<'a, Q: QueryContext> StaticDefinitions<'a, Q> { @@ -87,10 +89,14 @@ impl<'a, Q: QueryContext> StaticDefinitions<'a, Q> { pattern_definitions, predicate_definitions, function_definitions, + skippable_indexes: vec![], } } pub fn get_pattern(&self, index: usize) -> Option<&PatternDefinition> { + if self.skippable_indexes.contains(&index) { + return None; + } self.pattern_definitions.get(index) } @@ -109,6 +115,7 @@ impl<'a, Q: QueryContext> Default for StaticDefinitions<'a, Q> { pattern_definitions: &[], predicate_definitions: &[], function_definitions: &[], + skippable_indexes: vec![], } } } From 5569b669df38d213b0ac7a429e0b387aba4d48b1 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Sun, 21 Jul 2024 20:28:04 -0700 Subject: [PATCH 09/11] run the tests --- crates/cli_bin/tests/snapshots/apply__output_jsonl.snap | 4 ++-- crates/core/src/problem.rs | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/cli_bin/tests/snapshots/apply__output_jsonl.snap b/crates/cli_bin/tests/snapshots/apply__output_jsonl.snap index 360059f3b..71d8cd51c 100644 --- a/crates/cli_bin/tests/snapshots/apply__output_jsonl.snap +++ b/crates/cli_bin/tests/snapshots/apply__output_jsonl.snap @@ -2,6 +2,6 @@ source: crates/cli_bin/tests/apply.rs expression: content --- -{"__typename":"Match","messages":[],"variables":[{"name":"$new_files","scopedName":"0_0_$new_files","ranges":[]},{"name":"$program","scopedName":"0_1_$program","ranges":[]},{"name":"$filename","scopedName":"0_2_$filename","ranges":[]},{"name":"$absolute_filename","scopedName":"0_3_$absolute_filename","ranges":[]},{"name":"$GLOBAL_IMPORTED_SOURCES","scopedName":"0_4_$GLOBAL_IMPORTED_SOURCES","ranges":[]},{"name":"$GLOBAL_IMPORTED_NAMES","scopedName":"0_5_$GLOBAL_IMPORTED_NAMES","ranges":[]},{"name":"$body","scopedName":"2_0_$body","ranges":[{"start":{"line":1,"column":1},"end":{"line":14,"column":2},"startByte":0,"endByte":256}]},{"name":"$p","scopedName":"7_0_$p","ranges":[]},{"name":"$all_imports","scopedName":"7_1_$all_imports","ranges":[]},{"name":"$h","scopedName":"7_2_$h","ranges":[]},{"name":"$statements","scopedName":"7_3_$statements","ranges":[]},{"name":"$anchor","scopedName":"7_4_$anchor","ranges":[]},{"name":"$p","scopedName":"8_0_$p","ranges":[]},{"name":"$all_imports","scopedName":"8_1_$all_imports","ranges":[]},{"name":"$our_source","scopedName":"8_2_$our_source","ranges":[]},{"name":"$imported_names","scopedName":"8_3_$imported_names","ranges":[]},{"name":"$joined_imported_names","scopedName":"8_4_$joined_imported_names","ranges":[]},{"name":"$p","scopedName":"9_0_$p","ranges":[]},{"name":"$our_source","scopedName":"9_1_$our_source","ranges":[]},{"name":"$imported_names","scopedName":"9_2_$imported_names","ranges":[]},{"name":"$all_imports","scopedName":"9_3_$all_imports","ranges":[]},{"name":"$name","scopedName":"9_4_$name","ranges":[]},{"name":"$our_source","scopedName":"10_0_$our_source","ranges":[]},{"name":"$joined_imported_names","scopedName":"10_1_$joined_imported_names","ranges":[]},{"name":"$imports","scopedName":"10_2_$imports","ranges":[]},{"name":"$source","scopedName":"10_3_$source","ranges":[]},{"name":"$statement","scopedName":"10_4_$statement","ranges":[]},{"name":"$imports","scopedName":"11_0_$imports","ranges":[]},{"name":"$source","scopedName":"11_1_$source","ranges":[]},{"name":"$match","scopedName":"12_0_$match","ranges":[{"start":{"line":4,"column":15},"end":{"line":9,"column":4},"startByte":81,"endByte":172},{"start":{"line":6,"column":17},"end":{"line":8,"column":6},"startByte":129,"endByte":168},{"start":{"line":13,"column":10},"end":{"line":13,"column":39},"startByte":223,"endByte":252}]}],"sourceFile":"small.ts","content":"// This is a smaller example\nfunction() { \n console.log(\"thing\");\n const foo = () => {\n console.log(\"bar\");\n const bar = () => {\n console.log(\"baz\");\n }\n };\n // Not this\n // Delay a bit\n // Wow\n handle(() => { console.log(\"foo\"); });\n}","ranges":[{"start":{"line":4,"column":15},"end":{"line":9,"column":4},"startByte":81,"endByte":172},{"start":{"line":6,"column":17},"end":{"line":8,"column":6},"startByte":129,"endByte":168},{"start":{"line":13,"column":10},"end":{"line":13,"column":39},"startByte":223,"endByte":252}],"reason":null,"id":"[UUID]"} -{"__typename":"Match","messages":[],"variables":[{"name":"$new_files","scopedName":"0_0_$new_files","ranges":[]},{"name":"$program","scopedName":"0_1_$program","ranges":[]},{"name":"$filename","scopedName":"0_2_$filename","ranges":[]},{"name":"$absolute_filename","scopedName":"0_3_$absolute_filename","ranges":[]},{"name":"$GLOBAL_IMPORTED_SOURCES","scopedName":"0_4_$GLOBAL_IMPORTED_SOURCES","ranges":[]},{"name":"$GLOBAL_IMPORTED_NAMES","scopedName":"0_5_$GLOBAL_IMPORTED_NAMES","ranges":[]},{"name":"$body","scopedName":"2_0_$body","ranges":[{"start":{"line":1,"column":1},"end":{"line":120,"column":1},"startByte":0,"endByte":3269}]},{"name":"$p","scopedName":"7_0_$p","ranges":[]},{"name":"$all_imports","scopedName":"7_1_$all_imports","ranges":[]},{"name":"$h","scopedName":"7_2_$h","ranges":[]},{"name":"$statements","scopedName":"7_3_$statements","ranges":[]},{"name":"$anchor","scopedName":"7_4_$anchor","ranges":[]},{"name":"$p","scopedName":"8_0_$p","ranges":[]},{"name":"$all_imports","scopedName":"8_1_$all_imports","ranges":[]},{"name":"$our_source","scopedName":"8_2_$our_source","ranges":[]},{"name":"$imported_names","scopedName":"8_3_$imported_names","ranges":[]},{"name":"$joined_imported_names","scopedName":"8_4_$joined_imported_names","ranges":[]},{"name":"$p","scopedName":"9_0_$p","ranges":[]},{"name":"$our_source","scopedName":"9_1_$our_source","ranges":[]},{"name":"$imported_names","scopedName":"9_2_$imported_names","ranges":[]},{"name":"$all_imports","scopedName":"9_3_$all_imports","ranges":[]},{"name":"$name","scopedName":"9_4_$name","ranges":[]},{"name":"$our_source","scopedName":"10_0_$our_source","ranges":[]},{"name":"$joined_imported_names","scopedName":"10_1_$joined_imported_names","ranges":[]},{"name":"$imports","scopedName":"10_2_$imports","ranges":[]},{"name":"$source","scopedName":"10_3_$source","ranges":[]},{"name":"$statement","scopedName":"10_4_$statement","ranges":[]},{"name":"$imports","scopedName":"11_0_$imports","ranges":[]},{"name":"$source","scopedName":"11_1_$source","ranges":[]},{"name":"$match","scopedName":"12_0_$match","ranges":[{"start":{"line":14,"column":39},"end":{"line":66,"column":4},"startByte":452,"endByte":1767},{"start":{"line":18,"column":19},"end":{"line":22,"column":6},"startByte":590,"endByte":710},{"start":{"line":24,"column":17},"end":{"line":26,"column":6},"startByte":729,"endByte":775},{"start":{"line":32,"column":26},"end":{"line":35,"column":6},"startByte":858,"endByte":933},{"start":{"line":36,"column":26},"end":{"line":39,"column":6},"startByte":961,"endByte":1036},{"start":{"line":40,"column":20},"end":{"line":51,"column":6},"startByte":1058,"endByte":1323},{"start":{"line":52,"column":20},"end":{"line":65,"column":6},"startByte":1345,"endByte":1761},{"start":{"line":71,"column":71},"end":{"line":71,"column":79},"startByte":1882,"endByte":1890},{"start":{"line":107,"column":25},"end":{"line":119,"column":2},"startByte":2904,"endByte":3267},{"start":{"line":109,"column":71},"end":{"line":109,"column":79},"startByte":2997,"endByte":3005}]}],"sourceFile":"big.ts","content":"import type { ChildProcessWithoutNullStreams } from 'child_process';\nimport { spawn } from 'child_process';\nimport type { stdlib } from '@getgrit/api';\n\nexport function wrap_exec({\n callback,\n cp,\n passOnNonZeroExitCode,\n}: {\n callback?: (write: (str: string) => void, end: () => void) => void;\n cp: ChildProcessWithoutNullStreams;\n passOnNonZeroExitCode: boolean | undefined;\n}): Promise {\n return new Promise((resolve) => {\n const allout = [] as string[];\n const stdout = [] as string[];\n const stderr = [] as string[];\n const write = (str: string): void => {\n process.stdin.cork();\n process.stdin.write(str);\n process.stdin.uncork();\n };\n\n const end = (): void => {\n process.stdin.end();\n };\n\n if (callback) {\n callback(write, end);\n }\n\n cp.stdout.on('data', (data: string) => {\n stdout.push(data);\n allout.push(data);\n });\n cp.stderr.on('data', (data: string) => {\n stderr.push(data);\n allout.push(data);\n });\n cp.on('error', (e) => {\n resolve({\n __typename: 'ShResult',\n kind: 'direct',\n success: false,\n code: -1,\n allout: allout.join(''),\n stdout: stdout.join(''),\n stderr: stderr.join(''),\n message: e.message,\n });\n });\n cp.on('close', (code: number) => {\n // Trailing info is more valuable than truncated.\n const message = stderr.join('').slice(-2000);\n resolve({\n __typename: 'ShResult',\n kind: 'direct',\n success: passOnNonZeroExitCode ? true : code === 0,\n code: code || 0,\n allout: allout.join(''),\n stdout: stdout.join(''),\n stderr: stderr.join(''),\n message,\n });\n });\n });\n}\n\nexport function baseSh(\n cmd: string,\n callback: (write: (str: string) => void, end: () => void) => void = () => {},\n cwd: string | URL | undefined = undefined,\n passOnNonZeroExitCode: boolean | undefined = undefined,\n env: NodeJS.ProcessEnv | undefined = undefined,\n detached?: boolean,\n): Promise {\n const cp = spawn(cmd, { shell: true, detached, cwd, env: { ...process.env, ...(env ?? {}) } });\n const interruptCP = function () {\n cp.emit('SIGINT');\n };\n\n try {\n const promise = wrap_exec({ callback, cp, passOnNonZeroExitCode });\n if (!detached) {\n return promise;\n }\n } finally {\n process.removeListener('SIGINT', interruptCP);\n }\n return Promise.resolve({\n __typename: 'ShResult',\n kind: 'detached',\n allout: 'Result is detached',\n stderr: '',\n stdout: 'Result is detached',\n code: 0,\n success: true,\n });\n}\n/**\n * The most basic sh. Sends output to console if anything goes wrong, that is, if status code != 0\n * @param cmd the command to run\n * @param callback the callback to run\n * @returns All output from the command.\n */\n\nexport const simpleSh = async (\n cmd: string,\n callback: (write: (str: string) => void, end: () => void) => void = () => {},\n cwd: string | URL | undefined = undefined,\n): Promise => {\n const res = await baseSh(cmd, callback, cwd);\n if (res.code !== 0) {\n console.log(res.stdout);\n console.error(res.stderr);\n throw new Error(res.stderr);\n }\n return res.allout;\n};\n","ranges":[{"start":{"line":14,"column":39},"end":{"line":66,"column":4},"startByte":452,"endByte":1767},{"start":{"line":18,"column":19},"end":{"line":22,"column":6},"startByte":590,"endByte":710},{"start":{"line":24,"column":17},"end":{"line":26,"column":6},"startByte":729,"endByte":775},{"start":{"line":32,"column":26},"end":{"line":35,"column":6},"startByte":858,"endByte":933},{"start":{"line":36,"column":26},"end":{"line":39,"column":6},"startByte":961,"endByte":1036},{"start":{"line":40,"column":20},"end":{"line":51,"column":6},"startByte":1058,"endByte":1323},{"start":{"line":52,"column":20},"end":{"line":65,"column":6},"startByte":1345,"endByte":1761},{"start":{"line":71,"column":71},"end":{"line":71,"column":79},"startByte":1882,"endByte":1890},{"start":{"line":107,"column":25},"end":{"line":119,"column":2},"startByte":2904,"endByte":3267},{"start":{"line":109,"column":71},"end":{"line":109,"column":79},"startByte":2997,"endByte":3005}],"reason":null,"id":"[UUID]"} +{"__typename":"Match","messages":[],"variables":[{"name":"$new_files","scopedName":"0_0_$new_files","ranges":[]},{"name":"$program","scopedName":"0_1_$program","ranges":[]},{"name":"$filename","scopedName":"0_2_$filename","ranges":[]},{"name":"$absolute_filename","scopedName":"0_3_$absolute_filename","ranges":[]},{"name":"$GLOBAL_REWRITE_INSTRUCTION","scopedName":"0_4_$GLOBAL_REWRITE_INSTRUCTION","ranges":[]},{"name":"$GLOBAL_IMPORTED_SOURCES","scopedName":"0_5_$GLOBAL_IMPORTED_SOURCES","ranges":[]},{"name":"$GLOBAL_IMPORTED_NAMES","scopedName":"0_6_$GLOBAL_IMPORTED_NAMES","ranges":[]},{"name":"$body","scopedName":"2_0_$body","ranges":[]},{"name":"$code_with_markers","scopedName":"2_1_$code_with_markers","ranges":[]},{"name":"$instruct","scopedName":"2_2_$instruct","ranges":[]},{"name":"$messages","scopedName":"2_3_$messages","ranges":[]},{"name":"$pattern","scopedName":"2_4_$pattern","ranges":[]},{"name":"$answer","scopedName":"2_5_$answer","ranges":[]},{"name":"$final","scopedName":"2_6_$final","ranges":[]},{"name":"$body","scopedName":"3_0_$body","ranges":[{"start":{"line":1,"column":1},"end":{"line":14,"column":2},"startByte":0,"endByte":256}]},{"name":"$p","scopedName":"8_0_$p","ranges":[]},{"name":"$all_imports","scopedName":"8_1_$all_imports","ranges":[]},{"name":"$h","scopedName":"8_2_$h","ranges":[]},{"name":"$statements","scopedName":"8_3_$statements","ranges":[]},{"name":"$anchor","scopedName":"8_4_$anchor","ranges":[]},{"name":"$p","scopedName":"9_0_$p","ranges":[]},{"name":"$all_imports","scopedName":"9_1_$all_imports","ranges":[]},{"name":"$our_source","scopedName":"9_2_$our_source","ranges":[]},{"name":"$imported_names","scopedName":"9_3_$imported_names","ranges":[]},{"name":"$joined_imported_names","scopedName":"9_4_$joined_imported_names","ranges":[]},{"name":"$p","scopedName":"10_0_$p","ranges":[]},{"name":"$our_source","scopedName":"10_1_$our_source","ranges":[]},{"name":"$imported_names","scopedName":"10_2_$imported_names","ranges":[]},{"name":"$all_imports","scopedName":"10_3_$all_imports","ranges":[]},{"name":"$name","scopedName":"10_4_$name","ranges":[]},{"name":"$our_source","scopedName":"11_0_$our_source","ranges":[]},{"name":"$joined_imported_names","scopedName":"11_1_$joined_imported_names","ranges":[]},{"name":"$imports","scopedName":"11_2_$imports","ranges":[]},{"name":"$source","scopedName":"11_3_$source","ranges":[]},{"name":"$statement","scopedName":"11_4_$statement","ranges":[]},{"name":"$imports","scopedName":"12_0_$imports","ranges":[]},{"name":"$source","scopedName":"12_1_$source","ranges":[]},{"name":"$match","scopedName":"13_0_$match","ranges":[{"start":{"line":4,"column":15},"end":{"line":9,"column":4},"startByte":81,"endByte":172},{"start":{"line":6,"column":17},"end":{"line":8,"column":6},"startByte":129,"endByte":168},{"start":{"line":13,"column":10},"end":{"line":13,"column":39},"startByte":223,"endByte":252}]}],"sourceFile":"small.ts","content":"// This is a smaller example\nfunction() { \n console.log(\"thing\");\n const foo = () => {\n console.log(\"bar\");\n const bar = () => {\n console.log(\"baz\");\n }\n };\n // Not this\n // Delay a bit\n // Wow\n handle(() => { console.log(\"foo\"); });\n}","ranges":[{"start":{"line":4,"column":15},"end":{"line":9,"column":4},"startByte":81,"endByte":172},{"start":{"line":6,"column":17},"end":{"line":8,"column":6},"startByte":129,"endByte":168},{"start":{"line":13,"column":10},"end":{"line":13,"column":39},"startByte":223,"endByte":252}],"reason":null,"id":"[UUID]"} +{"__typename":"Match","messages":[],"variables":[{"name":"$new_files","scopedName":"0_0_$new_files","ranges":[]},{"name":"$program","scopedName":"0_1_$program","ranges":[]},{"name":"$filename","scopedName":"0_2_$filename","ranges":[]},{"name":"$absolute_filename","scopedName":"0_3_$absolute_filename","ranges":[]},{"name":"$GLOBAL_REWRITE_INSTRUCTION","scopedName":"0_4_$GLOBAL_REWRITE_INSTRUCTION","ranges":[]},{"name":"$GLOBAL_IMPORTED_SOURCES","scopedName":"0_5_$GLOBAL_IMPORTED_SOURCES","ranges":[]},{"name":"$GLOBAL_IMPORTED_NAMES","scopedName":"0_6_$GLOBAL_IMPORTED_NAMES","ranges":[]},{"name":"$body","scopedName":"2_0_$body","ranges":[]},{"name":"$code_with_markers","scopedName":"2_1_$code_with_markers","ranges":[]},{"name":"$instruct","scopedName":"2_2_$instruct","ranges":[]},{"name":"$messages","scopedName":"2_3_$messages","ranges":[]},{"name":"$pattern","scopedName":"2_4_$pattern","ranges":[]},{"name":"$answer","scopedName":"2_5_$answer","ranges":[]},{"name":"$final","scopedName":"2_6_$final","ranges":[]},{"name":"$body","scopedName":"3_0_$body","ranges":[{"start":{"line":1,"column":1},"end":{"line":120,"column":1},"startByte":0,"endByte":3269}]},{"name":"$p","scopedName":"8_0_$p","ranges":[]},{"name":"$all_imports","scopedName":"8_1_$all_imports","ranges":[]},{"name":"$h","scopedName":"8_2_$h","ranges":[]},{"name":"$statements","scopedName":"8_3_$statements","ranges":[]},{"name":"$anchor","scopedName":"8_4_$anchor","ranges":[]},{"name":"$p","scopedName":"9_0_$p","ranges":[]},{"name":"$all_imports","scopedName":"9_1_$all_imports","ranges":[]},{"name":"$our_source","scopedName":"9_2_$our_source","ranges":[]},{"name":"$imported_names","scopedName":"9_3_$imported_names","ranges":[]},{"name":"$joined_imported_names","scopedName":"9_4_$joined_imported_names","ranges":[]},{"name":"$p","scopedName":"10_0_$p","ranges":[]},{"name":"$our_source","scopedName":"10_1_$our_source","ranges":[]},{"name":"$imported_names","scopedName":"10_2_$imported_names","ranges":[]},{"name":"$all_imports","scopedName":"10_3_$all_imports","ranges":[]},{"name":"$name","scopedName":"10_4_$name","ranges":[]},{"name":"$our_source","scopedName":"11_0_$our_source","ranges":[]},{"name":"$joined_imported_names","scopedName":"11_1_$joined_imported_names","ranges":[]},{"name":"$imports","scopedName":"11_2_$imports","ranges":[]},{"name":"$source","scopedName":"11_3_$source","ranges":[]},{"name":"$statement","scopedName":"11_4_$statement","ranges":[]},{"name":"$imports","scopedName":"12_0_$imports","ranges":[]},{"name":"$source","scopedName":"12_1_$source","ranges":[]},{"name":"$match","scopedName":"13_0_$match","ranges":[{"start":{"line":14,"column":39},"end":{"line":66,"column":4},"startByte":452,"endByte":1767},{"start":{"line":18,"column":19},"end":{"line":22,"column":6},"startByte":590,"endByte":710},{"start":{"line":24,"column":17},"end":{"line":26,"column":6},"startByte":729,"endByte":775},{"start":{"line":32,"column":26},"end":{"line":35,"column":6},"startByte":858,"endByte":933},{"start":{"line":36,"column":26},"end":{"line":39,"column":6},"startByte":961,"endByte":1036},{"start":{"line":40,"column":20},"end":{"line":51,"column":6},"startByte":1058,"endByte":1323},{"start":{"line":52,"column":20},"end":{"line":65,"column":6},"startByte":1345,"endByte":1761},{"start":{"line":71,"column":71},"end":{"line":71,"column":79},"startByte":1882,"endByte":1890},{"start":{"line":107,"column":25},"end":{"line":119,"column":2},"startByte":2904,"endByte":3267},{"start":{"line":109,"column":71},"end":{"line":109,"column":79},"startByte":2997,"endByte":3005}]}],"sourceFile":"big.ts","content":"import type { ChildProcessWithoutNullStreams } from 'child_process';\nimport { spawn } from 'child_process';\nimport type { stdlib } from '@getgrit/api';\n\nexport function wrap_exec({\n callback,\n cp,\n passOnNonZeroExitCode,\n}: {\n callback?: (write: (str: string) => void, end: () => void) => void;\n cp: ChildProcessWithoutNullStreams;\n passOnNonZeroExitCode: boolean | undefined;\n}): Promise {\n return new Promise((resolve) => {\n const allout = [] as string[];\n const stdout = [] as string[];\n const stderr = [] as string[];\n const write = (str: string): void => {\n process.stdin.cork();\n process.stdin.write(str);\n process.stdin.uncork();\n };\n\n const end = (): void => {\n process.stdin.end();\n };\n\n if (callback) {\n callback(write, end);\n }\n\n cp.stdout.on('data', (data: string) => {\n stdout.push(data);\n allout.push(data);\n });\n cp.stderr.on('data', (data: string) => {\n stderr.push(data);\n allout.push(data);\n });\n cp.on('error', (e) => {\n resolve({\n __typename: 'ShResult',\n kind: 'direct',\n success: false,\n code: -1,\n allout: allout.join(''),\n stdout: stdout.join(''),\n stderr: stderr.join(''),\n message: e.message,\n });\n });\n cp.on('close', (code: number) => {\n // Trailing info is more valuable than truncated.\n const message = stderr.join('').slice(-2000);\n resolve({\n __typename: 'ShResult',\n kind: 'direct',\n success: passOnNonZeroExitCode ? true : code === 0,\n code: code || 0,\n allout: allout.join(''),\n stdout: stdout.join(''),\n stderr: stderr.join(''),\n message,\n });\n });\n });\n}\n\nexport function baseSh(\n cmd: string,\n callback: (write: (str: string) => void, end: () => void) => void = () => {},\n cwd: string | URL | undefined = undefined,\n passOnNonZeroExitCode: boolean | undefined = undefined,\n env: NodeJS.ProcessEnv | undefined = undefined,\n detached?: boolean,\n): Promise {\n const cp = spawn(cmd, { shell: true, detached, cwd, env: { ...process.env, ...(env ?? {}) } });\n const interruptCP = function () {\n cp.emit('SIGINT');\n };\n\n try {\n const promise = wrap_exec({ callback, cp, passOnNonZeroExitCode });\n if (!detached) {\n return promise;\n }\n } finally {\n process.removeListener('SIGINT', interruptCP);\n }\n return Promise.resolve({\n __typename: 'ShResult',\n kind: 'detached',\n allout: 'Result is detached',\n stderr: '',\n stdout: 'Result is detached',\n code: 0,\n success: true,\n });\n}\n/**\n * The most basic sh. Sends output to console if anything goes wrong, that is, if status code != 0\n * @param cmd the command to run\n * @param callback the callback to run\n * @returns All output from the command.\n */\n\nexport const simpleSh = async (\n cmd: string,\n callback: (write: (str: string) => void, end: () => void) => void = () => {},\n cwd: string | URL | undefined = undefined,\n): Promise => {\n const res = await baseSh(cmd, callback, cwd);\n if (res.code !== 0) {\n console.log(res.stdout);\n console.error(res.stderr);\n throw new Error(res.stderr);\n }\n return res.allout;\n};\n","ranges":[{"start":{"line":14,"column":39},"end":{"line":66,"column":4},"startByte":452,"endByte":1767},{"start":{"line":18,"column":19},"end":{"line":22,"column":6},"startByte":590,"endByte":710},{"start":{"line":24,"column":17},"end":{"line":26,"column":6},"startByte":729,"endByte":775},{"start":{"line":32,"column":26},"end":{"line":35,"column":6},"startByte":858,"endByte":933},{"start":{"line":36,"column":26},"end":{"line":39,"column":6},"startByte":961,"endByte":1036},{"start":{"line":40,"column":20},"end":{"line":51,"column":6},"startByte":1058,"endByte":1323},{"start":{"line":52,"column":20},"end":{"line":65,"column":6},"startByte":1345,"endByte":1761},{"start":{"line":71,"column":71},"end":{"line":71,"column":79},"startByte":1882,"endByte":1890},{"start":{"line":107,"column":25},"end":{"line":119,"column":2},"startByte":2904,"endByte":3267},{"start":{"line":109,"column":71},"end":{"line":109,"column":79},"startByte":2997,"endByte":3005}],"reason":null,"id":"[UUID]"} {"__typename":"AllDone","processed":2,"found":2,"reason":"allMatchesFound"} diff --git a/crates/core/src/problem.rs b/crates/core/src/problem.rs index 3569dea65..727db5907 100644 --- a/crates/core/src/problem.rs +++ b/crates/core/src/problem.rs @@ -73,8 +73,10 @@ impl Problem { &self.predicate_definitions, &self.function_definitions, ); - // We use the first 3 indexes for auto-wrap stuff - defs.skippable_indexes = vec![0, 1, 2]; + // We use the first 3 indexes for auto-wrap stuff in production + if self.pattern_definitions.len() > 10 { + defs.skippable_indexes = vec![0, 1, 2]; + } defs } } From 1dd6b81575f675d74bb93b45e0ab4fe2b8e5bd5f Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Sun, 21 Jul 2024 20:31:55 -0700 Subject: [PATCH 10/11] run the tests --- crates/core/src/built_in_functions.rs | 6 +++--- crates/core/src/problem.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/core/src/built_in_functions.rs b/crates/core/src/built_in_functions.rs index ff9530c13..96314fa9e 100644 --- a/crates/core/src/built_in_functions.rs +++ b/crates/core/src/built_in_functions.rs @@ -501,15 +501,15 @@ pub fn get_ai_placeholder_functions() -> Option { BuiltInFunction::new( "llm_chat", vec!["model", "messages", "pattern"], - Box::new(ai_fn_placholder), + Box::new(ai_fn_placeholder), ), - BuiltInFunction::new("embedding", vec!["target"], Box::new(ai_fn_placholder)), + BuiltInFunction::new("embedding", vec!["target"], Box::new(ai_fn_placeholder)), ] .into(), ) } -fn ai_fn_placholder<'a>( +fn ai_fn_placeholder<'a>( _args: &'a [Option>], _context: &'a MarzanoContext<'a>, _state: &mut State<'a, MarzanoQueryContext>, diff --git a/crates/core/src/problem.rs b/crates/core/src/problem.rs index 727db5907..0d7ed3498 100644 --- a/crates/core/src/problem.rs +++ b/crates/core/src/problem.rs @@ -74,7 +74,7 @@ impl Problem { &self.function_definitions, ); // We use the first 3 indexes for auto-wrap stuff in production - if self.pattern_definitions.len() > 10 { + if self.pattern_definitions.len() >= 3 { defs.skippable_indexes = vec![0, 1, 2]; } defs From c81b8c5ac794819816b2e42f43ca092d7433e38d Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Sun, 21 Jul 2024 21:40:10 -0700 Subject: [PATCH 11/11] run the tests --- ...iable_ranges_in_snippet_with_multiple_contexts.snap | 4 ++-- ...ble_ranges_multiple_snippets_multiple_contexts.snap | 10 +++++----- .../snapshots/parse__parses_foreign_function.snap | 2 +- .../tests/snapshots/parse__parses_grit_file.snap | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/cli_bin/tests/snapshots/parse__correct_variable_ranges_in_snippet_with_multiple_contexts.snap b/crates/cli_bin/tests/snapshots/parse__correct_variable_ranges_in_snippet_with_multiple_contexts.snap index 95f964c1a..9464bfa84 100644 --- a/crates/cli_bin/tests/snapshots/parse__correct_variable_ranges_in_snippet_with_multiple_contexts.snap +++ b/crates/cli_bin/tests/snapshots/parse__correct_variable_ranges_in_snippet_with_multiple_contexts.snap @@ -18,7 +18,7 @@ variables: scopedName: 0_3_$absolute_filename ranges: [] - name: $body - scopedName: 12_0_$body + scopedName: 13_0_$body ranges: - start: line: 1 @@ -29,7 +29,7 @@ variables: startByte: 15 endByte: 20 - name: $match - scopedName: 12_1_$match + scopedName: 13_1_$match ranges: [] sourceFile: "`function () { $body }`" parsedPattern: "[..]" diff --git a/crates/cli_bin/tests/snapshots/parse__correct_variable_ranges_multiple_snippets_multiple_contexts.snap b/crates/cli_bin/tests/snapshots/parse__correct_variable_ranges_multiple_snippets_multiple_contexts.snap index 054284100..c5d1631e1 100644 --- a/crates/cli_bin/tests/snapshots/parse__correct_variable_ranges_multiple_snippets_multiple_contexts.snap +++ b/crates/cli_bin/tests/snapshots/parse__correct_variable_ranges_multiple_snippets_multiple_contexts.snap @@ -18,10 +18,10 @@ variables: scopedName: 0_3_$absolute_filename ranges: [] - name: $match - scopedName: 12_0_$match + scopedName: 13_0_$match ranges: [] - name: $body - scopedName: 13_0_$body + scopedName: 14_0_$body ranges: - start: line: 1 @@ -32,7 +32,7 @@ variables: startByte: 32 endByte: 37 - name: $args - scopedName: 13_1_$args + scopedName: 14_1_$args ranges: - start: line: 1 @@ -43,7 +43,7 @@ variables: startByte: 23 endByte: 28 - name: $body - scopedName: 14_0_$body + scopedName: 15_0_$body ranges: - start: line: 1 @@ -54,7 +54,7 @@ variables: startByte: 63 endByte: 68 - name: $args - scopedName: 14_1_$args + scopedName: 15_1_$args ranges: - start: line: 1 diff --git a/crates/cli_bin/tests/snapshots/parse__parses_foreign_function.snap b/crates/cli_bin/tests/snapshots/parse__parses_foreign_function.snap index 1b091cdbc..d7515d1cb 100644 --- a/crates/cli_bin/tests/snapshots/parse__parses_foreign_function.snap +++ b/crates/cli_bin/tests/snapshots/parse__parses_foreign_function.snap @@ -18,7 +18,7 @@ variables: scopedName: 0_3_$absolute_filename ranges: [] - name: $match - scopedName: 13_0_$match + scopedName: 14_0_$match ranges: [] sourceFile: "engine marzano(0.1)\nlanguage js\n\nfunction adder() js {\n console.log(\"We are in JavaScript now!\");\n return 10 % 3\n}\n\n`x` => adder()" parsedPattern: "[..]" diff --git a/crates/cli_bin/tests/snapshots/parse__parses_grit_file.snap b/crates/cli_bin/tests/snapshots/parse__parses_grit_file.snap index 5422cbbd3..68a216e0d 100644 --- a/crates/cli_bin/tests/snapshots/parse__parses_grit_file.snap +++ b/crates/cli_bin/tests/snapshots/parse__parses_grit_file.snap @@ -18,7 +18,7 @@ variables: scopedName: 0_3_$absolute_filename ranges: [] - name: $msg - scopedName: 12_0_$msg + scopedName: 13_0_$msg ranges: - start: line: 3 @@ -29,7 +29,7 @@ variables: startByte: 26 endByte: 30 - name: $match - scopedName: 12_1_$match + scopedName: 13_1_$match ranges: [] sourceFile: "language js\n\n`console.log($msg)`\n" parsedPattern: "[..]"