diff --git a/kclvm/Cargo.lock b/kclvm/Cargo.lock index 270ecf918..bf62f949d 100644 --- a/kclvm/Cargo.lock +++ b/kclvm/Cargo.lock @@ -571,6 +571,7 @@ dependencies = [ "libloading", "serde", "serde_json", + "threadpool", "walkdir", ] diff --git a/kclvm/runner/Cargo.lock b/kclvm/runner/Cargo.lock index b90f4cf39..204aaf374 100644 --- a/kclvm/runner/Cargo.lock +++ b/kclvm/runner/Cargo.lock @@ -114,6 +114,22 @@ dependencies = [ "lazy_static", "memchr", "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" + +[[package]] +name = "cast" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c24dab4283a142afa2fdca129b80ad2c6284e073930f964c3a1293c225ee39a" +dependencies = [ + "rustc_version", ] [[package]] @@ -171,6 +187,52 @@ dependencies = [ "libc", ] +[[package]] +name = "criterion" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1604dafd25fba2fe2d5895a9da139f8dc9b319a5fe5354ca137cbbce4e178d10" +dependencies = [ + "atty", + "cast", + "clap", + "criterion-plot", + "csv", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_cbor", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d00996de9f2f7559f7f4dc286073197f83e92256a59ed395f9aac01fe717da57" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.1" @@ -216,6 +278,28 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" +dependencies = [ + "bstr", + "csv-core", + "itoa 0.4.8", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + [[package]] name = "digest" version = "0.9.0" @@ -333,6 +417,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + [[package]] name = "hashbrown" version = "0.11.2" @@ -400,6 +490,12 @@ dependencies = [ "either", ] +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + [[package]] name = "itoa" version = "1.0.1" @@ -415,6 +511,15 @@ dependencies = [ "libc", ] +[[package]] +name = "js-sys" +version = "0.3.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "json_minimal" version = "0.1.3" @@ -530,6 +635,7 @@ name = "kclvm-runner" version = "0.1.0" dependencies = [ "clap", + "criterion", "fslock", "glob", "indexmap", @@ -544,6 +650,7 @@ dependencies = [ "libloading", "serde", "serde_json", + "threadpool", "walkdir", ] @@ -654,7 +761,7 @@ dependencies = [ "lazy_static", "libc", "regex", - "semver", + "semver 0.11.0", ] [[package]] @@ -767,6 +874,12 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + [[package]] name = "opaque-debug" version = "0.3.0" @@ -871,6 +984,34 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +[[package]] +name = "plotters" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a3fd9ec30b9749ce28cd91f255d569591cdf937fe280c312143e3c4bad6f2a" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d88417318da0eaf0fdcdb51a0ee6c3bed624333bff8f946733049380be67ac1c" + +[[package]] +name = "plotters-svg" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521fa9638fa597e1dc53e9412a4f9cefb01187ee1f7413076f9e6749e2885ba9" +dependencies = [ + "plotters-backend", +] + [[package]] name = "ppv-lite86" version = "0.2.16" @@ -978,6 +1119,30 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rayon" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + [[package]] name = "rdrand" version = "0.4.0" @@ -1133,6 +1298,15 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver 1.0.9", +] + [[package]] name = "ryu" version = "1.0.9" @@ -1169,6 +1343,12 @@ dependencies = [ "semver-parser", ] +[[package]] +name = "semver" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd" + [[package]] name = "semver-parser" version = "0.10.2" @@ -1187,6 +1367,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_cbor" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" +dependencies = [ + "half", + "serde", +] + [[package]] name = "serde_derive" version = "1.0.137" @@ -1204,7 +1394,7 @@ version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" dependencies = [ - "itoa", + "itoa 1.0.1", "ryu", "serde", ] @@ -1393,6 +1583,15 @@ dependencies = [ "syn", ] +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + [[package]] name = "time" version = "0.1.44" @@ -1404,6 +1603,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "toml" version = "0.5.9" @@ -1574,6 +1783,70 @@ version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +[[package]] +name = "wasm-bindgen" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" + +[[package]] +name = "web-sys" +version = "0.3.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/kclvm/runner/Cargo.toml b/kclvm/runner/Cargo.toml index 27190b8ab..6ef675d07 100644 --- a/kclvm/runner/Cargo.toml +++ b/kclvm/runner/Cargo.toml @@ -15,6 +15,7 @@ libc = "0.2.112" indexmap = "1.0" fslock = "0.2.1" libloading = "0.7.3" +threadpool = "1.0" kclvm-ast = {path = "../ast", version = "0.1.0"} kclvm-parser = {path = "../parser", version = "0.1.0"} @@ -23,3 +24,11 @@ kclvm-config = {path = "../config", version = "0.1.0"} kclvm-runtime = {path = "../runtime", version = "0.1.0"} kclvm-sema = {path = "../sema", version = "0.1.0"} kclvm-version = {path = "../version", version = "0.1.0"} + +[dev-dependencies] +kclvm-parser = {path = "../parser", version = "0.1.0"} +criterion = "0.3" + +[[bench]] +name = "bench_runner" +harness = false \ No newline at end of file diff --git a/kclvm/runner/benches/bench_runner.rs b/kclvm/runner/benches/bench_runner.rs new file mode 100644 index 000000000..d4d9e054b --- /dev/null +++ b/kclvm/runner/benches/bench_runner.rs @@ -0,0 +1,67 @@ +use std::collections::HashMap; + +use criterion::{criterion_group, criterion_main, Criterion}; +use kclvm_ast::ast::{Program, Module}; +use kclvm_runner::{runner::ExecProgramArgs, execute}; +use kclvm_sema::resolver::resolve_program; + +const MAIN_PKG_NAME: &str = "__main__"; +const TEST_CASE_PATH: &str = "/src/test_datas/init_check_order_0/main.k"; + +/// Load test kcl file to ast.Program +fn load_program(filename: String) -> Program { + let module = load_module(filename); + construct_program(module) +} + +/// Load test kcl file to ast.Module +fn load_module(filename: String) -> Module { + kclvm_parser::parse_file(&filename, None).unwrap() +} + +/// Construct ast.Program by ast.Module and default configuration. +/// Default configuration: +/// module.pkg = "__main__" +/// Program.root = "__main__" +/// Program.main = "__main__" +/// Program.cmd_args = [] +/// Program.cmd_overrides = [] +fn construct_program(mut module: Module) -> Program { + module.pkg = MAIN_PKG_NAME.to_string(); + let mut pkgs_ast = HashMap::new(); + pkgs_ast.insert(MAIN_PKG_NAME.to_string(), vec![module]); + Program { + root: MAIN_PKG_NAME.to_string(), + main: MAIN_PKG_NAME.to_string(), + pkgs: pkgs_ast, + cmd_args: vec![], + cmd_overrides: vec![], + } +} + +pub fn execute_for_test(kcl_path: &String) -> String{ + let plugin_agent = 0; + let args = ExecProgramArgs::default(); + // parse kcl file + let mut program = load_program(kcl_path.to_string()); + // resolve ast + let scope = resolve_program(&mut program); + scope.check_scope_diagnostics(); + // generate dylibs, link dylibs and execute. + execute(program, scope, plugin_agent, &args).unwrap() +} + +pub fn criterion_benchmark(c: &mut Criterion) { + let path = &format!("{}{}",std::env::current_dir().unwrap().to_str().unwrap(), TEST_CASE_PATH); + c.bench_function( + "kclvm-runner: ", |b| + b.iter(|| + { + execute_for_test(path); + } + ) + ); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); \ No newline at end of file diff --git a/kclvm/runner/src/assembler.rs b/kclvm/runner/src/assembler.rs new file mode 100644 index 000000000..e0eeb067a --- /dev/null +++ b/kclvm/runner/src/assembler.rs @@ -0,0 +1,161 @@ + +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + sync::mpsc::channel, +}; +use indexmap::IndexMap; +use threadpool::ThreadPool; + +use crate::command::Command; +use kclvm_ast::ast; +use kclvm_sema::resolver::scope::ProgramScope; +use kclvm_config::cache::{load_pkg_cache, save_pkg_cache, CacheOption}; +use kclvm_compiler::codegen::{llvm::emit_code, EmitOptions}; + +/// KclvmAssembler is mainly responsible for generating bytecode, +/// LLVM IR or dylib, and take the result of kclvm-parser and kclvm-sema +/// as input. +pub struct KclvmAssembler {} +const LL_FILE: &str = "_a.out"; +impl KclvmAssembler { + /// Generate the dylibs and return file paths from . + /// + /// In the method, 4 threads will be created to concurrently + /// generate dylibs under different package paths. + /// This method will generate “.out” and ".ll" files, + /// and return the file paths of the generated files in Vec. + pub fn gen_dylibs( + program: ast::Program, + scope: ProgramScope, + plugin_agent: u64, + ) -> Vec { + // gen bc or ll_file + let path = std::path::Path::new(LL_FILE); + if path.exists() { + std::fs::remove_file(path).unwrap(); + } + for entry in glob::glob(&format!("{}*.ll", LL_FILE)).unwrap() { + match entry { + Ok(path) => { + if path.exists() { + std::fs::remove_file(path).unwrap(); + } + } + Err(e) => println!("{:?}", e), + }; + } + + let cache_dir = Path::new(&program.root) + .join(".kclvm") + .join("cache") + .join(kclvm_version::get_full_version()); + if !cache_dir.exists() { + std::fs::create_dir_all(&cache_dir).unwrap(); + } + let mut compile_progs: IndexMap< + String, + ( + ast::Program, + IndexMap>, + PathBuf, + ), + > = IndexMap::default(); + for (pkgpath, modules) in program.pkgs { + let mut pkgs = HashMap::new(); + pkgs.insert(pkgpath.clone(), modules); + let compile_prog = ast::Program { + root: program.root.clone(), + main: program.main.clone(), + pkgs, + cmd_args: vec![], + cmd_overrides: vec![], + }; + compile_progs.insert( + pkgpath, + (compile_prog, scope.import_names.clone(), cache_dir.clone()), + ); + } + let pool = ThreadPool::new(4); + let (tx, rx) = channel(); + let prog_count = compile_progs.len(); + for (pkgpath, (compile_prog, import_names, cache_dir)) in compile_progs { + let tx = tx.clone(); + pool.execute(move || { + let root = &compile_prog.root; + let is_main_pkg = pkgpath == kclvm_ast::MAIN_PKG; + let file = if is_main_pkg { + PathBuf::from(&pkgpath) + } else { + cache_dir.join(&pkgpath) + }; + let ll_file = file.to_str().unwrap(); + let ll_path = format!("{}.ll", ll_file); + let dylib_path = format!("{}{}", ll_file, Command::get_lib_suffix()); + let mut ll_path_lock = + fslock::LockFile::open(&format!("{}.lock", ll_path)).unwrap(); + ll_path_lock.lock().unwrap(); + if Path::new(&ll_path).exists() { + std::fs::remove_file(&ll_path).unwrap(); + } + let dylib_path = if is_main_pkg { + emit_code( + &compile_prog, + import_names, + &EmitOptions { + from_path: None, + emit_path: Some(&ll_file), + no_link: true, + }, + ) + .expect("Compile KCL to LLVM error"); + let mut cmd = Command::new(plugin_agent); + cmd.run_clang_single(&ll_path, &dylib_path) + } else { + // If AST module has been modified, ignore the dylib cache + let dylib_relative_path: Option = + load_pkg_cache(root, &pkgpath, CacheOption::default()); + match dylib_relative_path { + Some(dylib_relative_path) => { + if dylib_relative_path.starts_with('.') { + dylib_relative_path.replacen(".", root, 1) + } else { + dylib_relative_path + } + } + None => { + emit_code( + &compile_prog, + import_names, + &EmitOptions { + from_path: None, + emit_path: Some(&ll_file), + no_link: true, + }, + ) + .expect("Compile KCL to LLVM error"); + let mut cmd = Command::new(plugin_agent); + let dylib_path = cmd.run_clang_single(&ll_path, &dylib_path); + let dylib_relative_path = dylib_path.replacen(root, ".", 1); + + save_pkg_cache( + root, + &pkgpath, + dylib_relative_path, + CacheOption::default(), + ); + dylib_path + } + } + }; + if Path::new(&ll_path).exists() { + std::fs::remove_file(&ll_path).unwrap(); + } + ll_path_lock.unlock().unwrap(); + tx.send(dylib_path) + .expect("channel will be there waiting for the pool"); + }); + } + rx.iter().take(prog_count).collect::>() + } +} \ No newline at end of file diff --git a/kclvm/runner/src/command.rs b/kclvm/runner/src/command.rs index ac18d6fe4..a4c2e4053 100644 --- a/kclvm/runner/src/command.rs +++ b/kclvm/runner/src/command.rs @@ -423,7 +423,6 @@ impl Command { } else { "clang" }; - if let Some(s) = Self::find_it(clang_exe) { return s.to_str().unwrap().to_string(); diff --git a/kclvm/runner/src/lib.rs b/kclvm/runner/src/lib.rs index e7a51a61d..abe33d061 100644 --- a/kclvm/runner/src/lib.rs +++ b/kclvm/runner/src/lib.rs @@ -1,2 +1,53 @@ +use kclvm_ast::ast::Program; +use kclvm_sema::resolver::scope::ProgramScope; +use runner::{ExecProgramArgs, KclvmRunner, KclvmRunnerOptions}; + pub mod command; +pub mod assembler; +pub mod linker; pub mod runner; + +#[cfg(test)] +pub mod tests; + +/// After the kcl program passed through kclvm-parser and kclvm-sema in the compiler frontend, +/// KCLVM needs to generate corresponding LLVM IR, dylibs or executable file for kcl program +/// in the compiler backend. +/// +/// Method “execute” is the entry point for the compiler backend. +/// +/// It returns the KCL program executing result as Result, +/// and mainly takes "program" (ast.Program returned by kclvm-parser) and "scope" +/// (ProgramScope returned by kclvm-sema) as input. +/// +/// "plugin_agent" is related to KCLVM plugin. +/// "args" is the items selected by the user in the KCLVM CLI. +/// +/// In the method, dylibs is generated by KclvmAssembler, and method "KclvmAssembler::gen_dylibs" +/// will return dylibs path in a "Vec"; +/// +/// After linking all dylibs by KclvmLinker, method "KclvmLinker::link_all_dylibs" will return a path +/// for dylib. +/// +/// At last, KclvmRunner will be constructed and call method "run" to execute the kcl program. +pub fn execute( + program: Program, + scope: ProgramScope, + plugin_agent: u64, + args: &ExecProgramArgs, +) -> Result { + // generate dylibs + let dylib_paths = assembler::KclvmAssembler::gen_dylibs(program, scope, plugin_agent); + + // link dylibsKclvmRunner + let dylib_path = linker::KclvmLinker::link_all_dylibs(dylib_paths, plugin_agent); + + // run + let runner = KclvmRunner::new( + dylib_path.as_str(), + Some(KclvmRunnerOptions { + plugin_agent_ptr: plugin_agent, + }), + ); + runner.run(&args) +} diff --git a/kclvm/runner/src/linker.rs b/kclvm/runner/src/linker.rs new file mode 100644 index 000000000..a5462c66b --- /dev/null +++ b/kclvm/runner/src/linker.rs @@ -0,0 +1,11 @@ +use crate::command::Command; + +/// KclvmLinker is mainly responsible for linking the dylibs generated by KclvmAssembler. +pub struct KclvmLinker {} +impl KclvmLinker { + /// Link the dylibs generated by method "gen_bc_or_ll_file". + pub fn link_all_dylibs(dylib_paths: Vec, plugin_agent: u64) -> String { + let mut cmd = Command::new(plugin_agent); + cmd.link_dylibs(&dylib_paths, "") + } +} \ No newline at end of file diff --git a/kclvm/runner/src/runner.rs b/kclvm/runner/src/runner.rs index f38efe6fe..b009f6067 100644 --- a/kclvm/runner/src/runner.rs +++ b/kclvm/runner/src/runner.rs @@ -1,5 +1,4 @@ use serde::{Deserialize, Serialize}; - use kclvm_ast::ast; #[allow(non_camel_case_types)] diff --git a/kclvm/runner/src/test_datas/init_check_order_0/main.k b/kclvm/runner/src/test_datas/init_check_order_0/main.k new file mode 100644 index 000000000..a76f90c8b --- /dev/null +++ b/kclvm/runner/src/test_datas/init_check_order_0/main.k @@ -0,0 +1,11 @@ +schema Person: + name: str + age: int + gender: str + info: str = "{}, {}, {} years old".format(name, gender, age) + +alice = Person { + "name": "alice", + "age": 10, + "gender": "female" +} diff --git a/kclvm/runner/src/test_datas/init_check_order_0/stdout.golden.json b/kclvm/runner/src/test_datas/init_check_order_0/stdout.golden.json new file mode 100644 index 000000000..c962e7309 --- /dev/null +++ b/kclvm/runner/src/test_datas/init_check_order_0/stdout.golden.json @@ -0,0 +1 @@ +{"alice": {"name": "alice", "age": 10, "gender": "female", "info": "alice, female, 10 years old", "__settings__": {"output_type": "INLINE", "__schema_type__": "__main__.Person"}}} \ No newline at end of file diff --git a/kclvm/runner/src/test_datas/init_check_order_1/main.k b/kclvm/runner/src/test_datas/init_check_order_1/main.k new file mode 100644 index 000000000..2711ed87e --- /dev/null +++ b/kclvm/runner/src/test_datas/init_check_order_1/main.k @@ -0,0 +1,61 @@ +schema Name: + mixin [UpperMixin] + firstName: str + lastName: str + upper: str + + # print("init name") + +schema Person(Name): + gender: str + title: str + info: str + + # print("init person") + +schema Girl(Person): + mixin [TitleMixin, InfoMixin] + gender: str = "female" + added: str = "some girl attr" + + # print("init girl") + + check: + gender == "female", "gender should be female in Girl" + +schema Boy(Person): + mixin [TitleMixin, InfoMixin] + gender: str = "male" + added: str = "some boy attr" + + # print("init boy") + + check: + gender == "male", "gender should be male in Boy" + +schema UpperMixin: + # print("init upperMixin") + upper: str = lastName.upper() + +schema TitleMixin: + # print("init title mixin") + if gender == "female": + title = "Ms.{}".format(lastName) + else: + title = "Mr.{}".format(lastName) + +schema InfoMixin: + # print("init info mixin") + info = "{}, {}".format(title, gender) + +alice = Girl { + "firstName": "Alice", + "lastName": "Smith" +} + +# print(" ===") + +bob = Boy { + "firstName": "Bob", + "lastName": "Green" +} diff --git a/kclvm/runner/src/test_datas/init_check_order_1/stdout.golden.json b/kclvm/runner/src/test_datas/init_check_order_1/stdout.golden.json new file mode 100644 index 000000000..1aad8c19b --- /dev/null +++ b/kclvm/runner/src/test_datas/init_check_order_1/stdout.golden.json @@ -0,0 +1 @@ +{"alice": {"firstName": "Alice", "lastName": "Smith", "upper": "SMITH", "__settings__": {"output_type": "INLINE", "__schema_type__": "__main__.InfoMixin"}, "gender": "female", "title": "Ms.Smith", "info": "Ms.Smith, female", "added": "some girl attr"}, "bob": {"firstName": "Bob", "lastName": "Green", "upper": "GREEN", "__settings__": {"output_type": "INLINE", "__schema_type__": "__main__.InfoMixin"}, "gender": "male", "title": "Mr.Green", "info": "Mr.Green, male", "added": "some boy attr"}} \ No newline at end of file diff --git a/kclvm/runner/src/test_datas/normal_2/main.k b/kclvm/runner/src/test_datas/normal_2/main.k new file mode 100644 index 000000000..2a7b43467 --- /dev/null +++ b/kclvm/runner/src/test_datas/normal_2/main.k @@ -0,0 +1,11 @@ + +schema NumberMap: + [num: str]: int + + check: + int(num) % 2 == 0 + +numMap = NumberMap { + str(0): 0 + str(2): 2 +} diff --git a/kclvm/runner/src/test_datas/normal_2/stdout.golden.json b/kclvm/runner/src/test_datas/normal_2/stdout.golden.json new file mode 100644 index 000000000..9958766ac --- /dev/null +++ b/kclvm/runner/src/test_datas/normal_2/stdout.golden.json @@ -0,0 +1 @@ +{"numMap":{"0":0,"2":2,"__settings__":{"__schema_type__":"__main__.NumberMap","output_type":"INLINE"}}} \ No newline at end of file diff --git a/kclvm/runner/src/test_datas/type_annotation_not_full_2/main.k b/kclvm/runner/src/test_datas/type_annotation_not_full_2/main.k new file mode 100644 index 000000000..5d277e7f0 --- /dev/null +++ b/kclvm/runner/src/test_datas/type_annotation_not_full_2/main.k @@ -0,0 +1,6 @@ +schema A[arg=1]: + a: int = arg + +a1 = A() +a2 = A(2) +a3 = A(arg=3) diff --git a/kclvm/runner/src/test_datas/type_annotation_not_full_2/stdout.golden.json b/kclvm/runner/src/test_datas/type_annotation_not_full_2/stdout.golden.json new file mode 100644 index 000000000..310d018aa --- /dev/null +++ b/kclvm/runner/src/test_datas/type_annotation_not_full_2/stdout.golden.json @@ -0,0 +1 @@ +{"a1":{"__settings__":{"__schema_type__":"__main__.A","output_type":"INLINE"},"a":1},"a2":{"__settings__":{"__schema_type__":"__main__.A","output_type":"INLINE"},"a":2},"a3":{"__settings__":{"__schema_type__":"__main__.A","output_type":"INLINE"},"a":3}} \ No newline at end of file diff --git a/kclvm/runner/src/tests.rs b/kclvm/runner/src/tests.rs new file mode 100644 index 000000000..232574ff9 --- /dev/null +++ b/kclvm/runner/src/tests.rs @@ -0,0 +1,83 @@ +use crate::{execute, runner::ExecProgramArgs}; +use kclvm_ast::ast::{Module, Program}; +use kclvm_sema::resolver::resolve_program; +use std::{collections::HashMap, fs::File}; + +const TEST_CASES: &[&'static str; 4] = &[ + "init_check_order_0", + "init_check_order_1", + "normal_2", + "type_annotation_not_full_2", +]; +const EXPECTED_FILE_NAME: &str = "stdout.golden.json"; +const TEST_CASE_PATH: &str = "./src/test_datas"; +const KCL_FILE_NAME: &str = "main.k"; +const MAIN_PKG_NAME: &str = "__main__"; + +/// Load test kcl file to ast.Program +pub fn load_program(filename: String) -> Program { + let module = load_module(filename); + construct_program(module) +} + +/// Load test kcl file to ast.Module +pub fn load_module(filename: String) -> Module { + kclvm_parser::parse_file(&filename, None).unwrap() +} + +/// Construct ast.Program by ast.Module and default configuration. +/// Default configuration: +/// module.pkg = "__main__" +/// Program.root = "__main__" +/// Program.main = "__main__" +/// Program.cmd_args = [] +/// Program.cmd_overrides = [] +pub fn construct_program(mut module: Module) -> Program { + module.pkg = MAIN_PKG_NAME.to_string(); + let mut pkgs_ast = HashMap::new(); + pkgs_ast.insert(MAIN_PKG_NAME.to_string(), vec![module]); + Program { + root: MAIN_PKG_NAME.to_string(), + main: MAIN_PKG_NAME.to_string(), + pkgs: pkgs_ast, + cmd_args: vec![], + cmd_overrides: vec![], + } +} + +/// Load the expect result from stdout.golden.json +pub fn load_expect_file(filename: String) -> String { + let f = File::open(filename).unwrap(); + let v: serde_json::Value = serde_json::from_reader(f).unwrap(); + v.to_string() +} + +/// Format str by json str +pub fn format_str_by_json(str: String) -> String { + let v: serde_json::Value = serde_json::from_str(&str).unwrap(); + v.to_string() +} + +pub fn execute_for_test(kcl_path: &String) -> String{ + let plugin_agent = 0; + let args = ExecProgramArgs::default(); + // parse kcl file + let mut program = load_program(kcl_path.to_string()); + // resolve ast + let scope = resolve_program(&mut program); + scope.check_scope_diagnostics(); + // generate dylibs, link dylibs and execute. + execute(program, scope, plugin_agent, &args).unwrap() +} + +#[test] +fn test_kclvm_runner_execute() { + for case in TEST_CASES { + let kcl_path = &format!("{}/{}/{}", TEST_CASE_PATH, case, KCL_FILE_NAME); + let expected_path = &format!("{}/{}/{}", TEST_CASE_PATH, case, EXPECTED_FILE_NAME); + let result = execute_for_test(kcl_path); + let expected_result = load_expect_file(expected_path.to_string()); + assert_eq!(expected_result, format_str_by_json(result)); + println!("{} - PASS", kcl_path); + } +} \ No newline at end of file diff --git a/kclvm/src/lib.rs b/kclvm/src/lib.rs index 291f5dc3a..d94d6ce68 100644 --- a/kclvm/src/lib.rs +++ b/kclvm/src/lib.rs @@ -1,19 +1,9 @@ extern crate serde; -use std::collections::HashMap; -use std::path::{Path, PathBuf}; -use std::sync::mpsc::channel; -use threadpool::ThreadPool; - -use indexmap::IndexMap; -use kclvm_ast::ast; -use kclvm_compiler::codegen::{llvm::emit_code, EmitOptions}; -use kclvm_config::cache::*; use kclvm_parser::load_program; -use kclvm_sema::resolver::resolve_program; - -use kclvm_runner::command::Command; +use kclvm_runner::execute; use kclvm_runner::runner::*; +use kclvm_sema::resolver::resolve_program; use kclvm_tools::query::apply_overrides; #[no_mangle] @@ -71,141 +61,11 @@ pub fn kclvm_cli_run_unsafe(args: *const i8, plugin_agent: *const i8) -> Result< // load ast let mut program = load_program(&files, Some(opts))?; apply_overrides(&mut program, &args.overrides, &[]); + + // resolve ast let scope = resolve_program(&mut program); scope.check_scope_diagnostics(); - // gen bc or ll file - let ll_file = "_a.out"; - let path = std::path::Path::new(ll_file); - if path.exists() { - std::fs::remove_file(path).unwrap(); - } - for entry in glob::glob(&format!("{}*.ll", ll_file)).unwrap() { - match entry { - Ok(path) => { - if path.exists() { - std::fs::remove_file(path).unwrap(); - } - } - Err(e) => println!("{:?}", e), - }; - } - - let cache_dir = Path::new(&program.root) - .join(".kclvm") - .join("cache") - .join(kclvm_version::get_full_version()); - if !cache_dir.exists() { - std::fs::create_dir_all(&cache_dir).unwrap(); - } - let mut compile_progs: IndexMap< - String, - ( - ast::Program, - IndexMap>, - PathBuf, - ), - > = IndexMap::default(); - for (pkgpath, modules) in program.pkgs { - let mut pkgs = HashMap::new(); - pkgs.insert(pkgpath.clone(), modules); - let compile_prog = ast::Program { - root: program.root.clone(), - main: program.main.clone(), - pkgs, - cmd_args: vec![], - cmd_overrides: vec![], - }; - compile_progs.insert( - pkgpath, - (compile_prog, scope.import_names.clone(), cache_dir.clone()), - ); - } - let pool = ThreadPool::new(4); - let (tx, rx) = channel(); - let prog_count = compile_progs.len(); - for (pkgpath, (compile_prog, import_names, cache_dir)) in compile_progs { - let tx = tx.clone(); - pool.execute(move || { - let root = &compile_prog.root; - let is_main_pkg = pkgpath == kclvm_ast::MAIN_PKG; - let file = if is_main_pkg { - PathBuf::from(&pkgpath) - } else { - cache_dir.join(&pkgpath) - }; - let ll_file = file.to_str().unwrap(); - let ll_path = format!("{}.ll", ll_file); - let dylib_path = format!("{}{}", ll_file, Command::get_lib_suffix()); - let mut ll_path_lock = fslock::LockFile::open(&format!("{}.lock", ll_path)).unwrap(); - ll_path_lock.lock().unwrap(); - if Path::new(&ll_path).exists() { - std::fs::remove_file(&ll_path).unwrap(); - } - let dylib_path = if is_main_pkg { - emit_code( - &compile_prog, - import_names, - &EmitOptions { - from_path: None, - emit_path: Some(&ll_file), - no_link: true, - }, - ) - .expect("Compile KCL to LLVM error"); - let mut cmd = Command::new(plugin_agent); - cmd.run_clang_single(&ll_path, &dylib_path) - } else { - // If AST module has been modified, ignore the dylib cache - let dylib_relative_path: Option = - load_pkg_cache(root, &pkgpath, CacheOption::default()); - match dylib_relative_path { - Some(dylib_relative_path) => { - if dylib_relative_path.starts_with('.') { - dylib_relative_path.replacen(".", root, 1) - } else { - dylib_relative_path - } - } - None => { - emit_code( - &compile_prog, - import_names, - &EmitOptions { - from_path: None, - emit_path: Some(&ll_file), - no_link: true, - }, - ) - .expect("Compile KCL to LLVM error"); - let mut cmd = Command::new(plugin_agent); - let dylib_path = cmd.run_clang_single(&ll_path, &dylib_path); - let dylib_relative_path = dylib_path.replacen(root, ".", 1); - - save_pkg_cache(root, &pkgpath, dylib_relative_path, CacheOption::default()); - dylib_path - } - } - }; - if Path::new(&ll_path).exists() { - std::fs::remove_file(&ll_path).unwrap(); - } - ll_path_lock.unlock().unwrap(); - tx.send(dylib_path) - .expect("channel will be there waiting for the pool"); - }); - } - let dylib_paths = rx.iter().take(prog_count).collect::>(); - let mut cmd = Command::new(plugin_agent); - // link all dylibs - let dylib_path = cmd.link_dylibs(&dylib_paths, ""); - // Config uild - // run dylib - let runner = KclvmRunner::new( - dylib_path.as_str(), - Some(KclvmRunnerOptions { - plugin_agent_ptr: plugin_agent, - }), - ); - runner.run(&args) + // generate dylibs, link dylibs and execute. + execute(program, scope, plugin_agent, &args) }