From e86954b86a83a3a5aeaad3c9c9209edddc3ce893 Mon Sep 17 00:00:00 2001 From: zong-zhe Date: Wed, 1 Jun 2022 18:19:15 +0800 Subject: [PATCH] refactor(kclvm-runner): encapsulate dylib generating, linking and executing in kclvm/lib.rs into kclvm-runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The assembler and linker of the current version of the compiler are not separately packaged. In order to support the reuse of modules such as dylibs generating，linking and executing, This modification separates dylibs generating，and encapsulate them individually into KclvmAssembler and KclvmLinker. Encapsulate dylibs generating, linking and executing into kclvm-runner/assembler.rs and kclvm-runner/linker.rs. Add struct "KclvmAssembler" in kclvm-runner/assembler.rs to provide method "gen_dylibs" for dylibs generating. Add struct "KclvmLinker" in kclvm-runner/linker.rs to provide method "link_all_dylibs" for dylib linking. Add method "execute" in kclvm-runner/lib.rs to encapsulate dylibs generating(gen_dylib), dylib linking(link_all_dylib) and running(runner.run) together. fix #67 --- kclvm/Cargo.lock | 1 + kclvm/runner/Cargo.lock | 277 +++++++++++++++++- kclvm/runner/Cargo.toml | 9 + kclvm/runner/benches/bench_runner.rs | 67 +++++ kclvm/runner/src/assembler.rs | 161 ++++++++++ kclvm/runner/src/command.rs | 1 - kclvm/runner/src/lib.rs | 51 ++++ kclvm/runner/src/linker.rs | 11 + kclvm/runner/src/runner.rs | 1 - .../src/test_datas/init_check_order_0/main.k | 11 + .../init_check_order_0/stdout.golden.json | 1 + .../src/test_datas/init_check_order_1/main.k | 61 ++++ .../init_check_order_1/stdout.golden.json | 1 + kclvm/runner/src/test_datas/normal_2/main.k | 11 + .../test_datas/normal_2/stdout.golden.json | 1 + .../type_annotation_not_full_2/main.k | 6 + .../stdout.golden.json | 1 + kclvm/runner/src/tests.rs | 83 ++++++ kclvm/src/lib.rs | 152 +--------- 19 files changed, 757 insertions(+), 150 deletions(-) create mode 100644 kclvm/runner/benches/bench_runner.rs create mode 100644 kclvm/runner/src/assembler.rs create mode 100644 kclvm/runner/src/linker.rs create mode 100644 kclvm/runner/src/test_datas/init_check_order_0/main.k create mode 100644 kclvm/runner/src/test_datas/init_check_order_0/stdout.golden.json create mode 100644 kclvm/runner/src/test_datas/init_check_order_1/main.k create mode 100644 kclvm/runner/src/test_datas/init_check_order_1/stdout.golden.json create mode 100644 kclvm/runner/src/test_datas/normal_2/main.k create mode 100644 kclvm/runner/src/test_datas/normal_2/stdout.golden.json create mode 100644 kclvm/runner/src/test_datas/type_annotation_not_full_2/main.k create mode 100644 kclvm/runner/src/test_datas/type_annotation_not_full_2/stdout.golden.json create mode 100644 kclvm/runner/src/tests.rs diff --git a/kclvm/Cargo.lock b/kclvm/Cargo.lock index 270ecf918..bf62f949d 100644 --- a/kclvm/Cargo.lock +++ b/kclvm/Cargo.lock @@ -571,6 +571,7 @@ dependencies = [ "libloading", "serde", "serde_json", + "threadpool", "walkdir", ] diff --git a/kclvm/runner/Cargo.lock b/kclvm/runner/Cargo.lock index b90f4cf39..204aaf374 100644 --- a/kclvm/runner/Cargo.lock +++ b/kclvm/runner/Cargo.lock @@ -114,6 +114,22 @@ dependencies = [ "lazy_static", "memchr", "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" + +[[package]] +name = "cast" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c24dab4283a142afa2fdca129b80ad2c6284e073930f964c3a1293c225ee39a" +dependencies = [ + "rustc_version", ] [[package]] @@ -171,6 +187,52 @@ dependencies = [ "libc", ] +[[package]] +name = "criterion" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1604dafd25fba2fe2d5895a9da139f8dc9b319a5fe5354ca137cbbce4e178d10" +dependencies = [ + "atty", + "cast", + "clap", + "criterion-plot", + "csv", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_cbor", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d00996de9f2f7559f7f4dc286073197f83e92256a59ed395f9aac01fe717da57" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.1" @@ -216,6 +278,28 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" +dependencies = [ + "bstr", + "csv-core", + "itoa 0.4.8", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + [[package]] name = "digest" version = "0.9.0" @@ -333,6 +417,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + [[package]] name = "hashbrown" version = "0.11.2" @@ -400,6 +490,12 @@ dependencies = [ "either", ] +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + [[package]] name = "itoa" version = "1.0.1" @@ -415,6 +511,15 @@ dependencies = [ "libc", ] +[[package]] +name = "js-sys" +version = "0.3.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "json_minimal" version = "0.1.3" @@ -530,6 +635,7 @@ name = "kclvm-runner" version = "0.1.0" dependencies = [ "clap", + "criterion", "fslock", "glob", "indexmap", @@ -544,6 +650,7 @@ dependencies = [ "libloading", "serde", "serde_json", + "threadpool", "walkdir", ] @@ -654,7 +761,7 @@ dependencies = [ "lazy_static", "libc", "regex", - "semver", + "semver 0.11.0", ] [[package]] @@ -767,6 +874,12 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + [[package]] name = "opaque-debug" version = "0.3.0" @@ -871,6 +984,34 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +[[package]] +name = "plotters" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a3fd9ec30b9749ce28cd91f255d569591cdf937fe280c312143e3c4bad6f2a" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d88417318da0eaf0fdcdb51a0ee6c3bed624333bff8f946733049380be67ac1c" + +[[package]] +name = "plotters-svg" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521fa9638fa597e1dc53e9412a4f9cefb01187ee1f7413076f9e6749e2885ba9" +dependencies = [ + "plotters-backend", +] + [[package]] name = "ppv-lite86" version = "0.2.16" @@ -978,6 +1119,30 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rayon" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + [[package]] name = "rdrand" version = "0.4.0" @@ -1133,6 +1298,15 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver 1.0.9", +] + [[package]] name = "ryu" version = "1.0.9" @@ -1169,6 +1343,12 @@ dependencies = [ "semver-parser", ] +[[package]] +name = "semver" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd" + [[package]] name = "semver-parser" version = "0.10.2" @@ -1187,6 +1367,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_cbor" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" +dependencies = [ + "half", + "serde", +] + [[package]] name = "serde_derive" version = "1.0.137" @@ -1204,7 +1394,7 @@ version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" dependencies = [ - "itoa", + "itoa 1.0.1", "ryu", "serde", ] @@ -1393,6 +1583,15 @@ dependencies = [ "syn", ] +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + [[package]] name = "time" version = "0.1.44" @@ -1404,6 +1603,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "toml" version = "0.5.9" @@ -1574,6 +1783,70 @@ version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +[[package]] +name = "wasm-bindgen" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" + +[[package]] +name = "web-sys" +version = "0.3.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/kclvm/runner/Cargo.toml b/kclvm/runner/Cargo.toml index 27190b8ab..6ef675d07 100644 --- a/kclvm/runner/Cargo.toml +++ b/kclvm/runner/Cargo.toml @@ -15,6 +15,7 @@ libc = "0.2.112" indexmap = "1.0" fslock = "0.2.1" libloading = "0.7.3" +threadpool = "1.0" kclvm-ast = {path = "../ast", version = "0.1.0"} kclvm-parser = {path = "../parser", version = "0.1.0"} @@ -23,3 +24,11 @@ kclvm-config = {path = "../config", version = "0.1.0"} kclvm-runtime = {path = "../runtime", version = "0.1.0"} kclvm-sema = {path = "../sema", version = "0.1.0"} kclvm-version = {path = "../version", version = "0.1.0"} + +[dev-dependencies] +kclvm-parser = {path = "../parser", version = "0.1.0"} +criterion = "0.3" + +[[bench]] +name = "bench_runner" +harness = false \ No newline at end of file diff --git a/kclvm/runner/benches/bench_runner.rs b/kclvm/runner/benches/bench_runner.rs new file mode 100644 index 000000000..d4d9e054b --- /dev/null +++ b/kclvm/runner/benches/bench_runner.rs @@ -0,0 +1,67 @@ +use std::collections::HashMap; + +use criterion::{criterion_group, criterion_main, Criterion}; +use kclvm_ast::ast::{Program, Module}; +use kclvm_runner::{runner::ExecProgramArgs, execute}; +use kclvm_sema::resolver::resolve_program; + +const MAIN_PKG_NAME: &str = "__main__"; +const TEST_CASE_PATH: &str = "/src/test_datas/init_check_order_0/main.k"; + +/// Load test kcl file to ast.Program +fn load_program(filename: String) -> Program { + let module = load_module(filename); + construct_program(module) +} + +/// Load test kcl file to ast.Module +fn load_module(filename: String) -> Module { + kclvm_parser::parse_file(&filename, None).unwrap() +} + +/// Construct ast.Program by ast.Module and default configuration. +/// Default configuration: +/// module.pkg = "__main__" +/// Program.root = "__main__" +/// Program.main = "__main__" +/// Program.cmd_args = [] +/// Program.cmd_overrides = [] +fn construct_program(mut module: Module) -> Program { + module.pkg = MAIN_PKG_NAME.to_string(); + let mut pkgs_ast = HashMap::new(); + pkgs_ast.insert(MAIN_PKG_NAME.to_string(), vec![module]); + Program { + root: MAIN_PKG_NAME.to_string(), + main: MAIN_PKG_NAME.to_string(), + pkgs: pkgs_ast, + cmd_args: vec![], + cmd_overrides: vec![], + } +} + +pub fn execute_for_test(kcl_path: &String) -> String{ + let plugin_agent = 0; + let args = ExecProgramArgs::default(); + // parse kcl file + let mut program = load_program(kcl_path.to_string()); + // resolve ast + let scope = resolve_program(&mut program); + scope.check_scope_diagnostics(); + // generate dylibs, link dylibs and execute. + execute(program, scope, plugin_agent, &args).unwrap() +} + +pub fn criterion_benchmark(c: &mut Criterion) { + let path = &format!("{}{}",std::env::current_dir().unwrap().to_str().unwrap(), TEST_CASE_PATH); + c.bench_function( + "kclvm-runner: ", |b| + b.iter(|| + { + execute_for_test(path); + } + ) + ); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); \ No newline at end of file diff --git a/kclvm/runner/src/assembler.rs b/kclvm/runner/src/assembler.rs new file mode 100644 index 000000000..e0eeb067a --- /dev/null +++ b/kclvm/runner/src/assembler.rs @@ -0,0 +1,161 @@ + +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + sync::mpsc::channel, +}; +use indexmap::IndexMap; +use threadpool::ThreadPool; + +use crate::command::Command; +use kclvm_ast::ast; +use kclvm_sema::resolver::scope::ProgramScope; +use kclvm_config::cache::{load_pkg_cache, save_pkg_cache, CacheOption}; +use kclvm_compiler::codegen::{llvm::emit_code, EmitOptions}; + +/// KclvmAssembler is mainly responsible for generating bytecode, +/// LLVM IR or dylib, and take the result of kclvm-parser and kclvm-sema +/// as input. +pub struct KclvmAssembler {} +const LL_FILE: &str = "_a.out"; +impl KclvmAssembler { + /// Generate the dylibs and return file paths from . + /// + /// In the method, 4 threads will be created to concurrently + /// generate dylibs under different package paths. + /// This method will generate “.out” and ".ll" files, + /// and return the file paths of the generated files in Vec. + pub fn gen_dylibs( + program: ast::Program, + scope: ProgramScope, + plugin_agent: u64, + ) -> Vec { + // gen bc or ll_file + let path = std::path::Path::new(LL_FILE); + if path.exists() { + std::fs::remove_file(path).unwrap(); + } + for entry in glob::glob(&format!("{}*.ll", LL_FILE)).unwrap() { + match entry { + Ok(path) => { + if path.exists() { + std::fs::remove_file(path).unwrap(); + } + } + Err(e) => println!("{:?}", e), + }; + } + + let cache_dir = Path::new(&program.root) + .join(".kclvm") + .join("cache") + .join(kclvm_version::get_full_version()); + if !cache_dir.exists() { + std::fs::create_dir_all(&cache_dir).unwrap(); + } + let mut compile_progs: IndexMap< + String, + ( + ast::Program, + IndexMap>, + PathBuf, + ), + > = IndexMap::default(); + for (pkgpath, modules) in program.pkgs { + let mut pkgs = HashMap::new(); + pkgs.insert(pkgpath.clone(), modules); + let compile_prog = ast::Program { + root: program.root.clone(), + main: program.main.clone(), + pkgs, + cmd_args: vec![], + cmd_overrides: vec![], + }; + compile_progs.insert( + pkgpath, + (compile_prog, scope.import_names.clone(), cache_dir.clone()), + ); + } + let pool = ThreadPool::new(4); + let (tx, rx) = channel(); + let prog_count = compile_progs.len(); + for (pkgpath, (compile_prog, import_names, cache_dir)) in compile_progs { + let tx = tx.clone(); + pool.execute(move || { + let root = &compile_prog.root; + let is_main_pkg = pkgpath == kclvm_ast::MAIN_PKG; + let file = if is_main_pkg { + PathBuf::from(&pkgpath) + } else { + cache_dir.join(&pkgpath) + }; + let ll_file = file.to_str().unwrap(); + let ll_path = format!("{}.ll", ll_file); + let dylib_path = format!("{}{}", ll_file, Command::get_lib_suffix()); + let mut ll_path_lock = + fslock::LockFile::open(&format!("{}.lock", ll_path)).unwrap(); + ll_path_lock.lock().unwrap(); + if Path::new(&ll_path).exists() { + std::fs::remove_file(&ll_path).unwrap(); + } + let dylib_path = if is_main_pkg { + emit_code( + &compile_prog, + import_names, + &EmitOptions { + from_path: None, + emit_path: Some(&ll_file), + no_link: true, + }, + ) + .expect("Compile KCL to LLVM error"); + let mut cmd = Command::new(plugin_agent); + cmd.run_clang_single(&ll_path, &dylib_path) + } else { + // If AST module has been modified, ignore the dylib cache + let dylib_relative_path: Option = + load_pkg_cache(root, &pkgpath, CacheOption::default()); + match dylib_relative_path { + Some(dylib_relative_path) => { + if dylib_relative_path.starts_with('.') { + dylib_relative_path.replacen(".", root, 1) + } else { + dylib_relative_path + } + } + None => { + emit_code( + &compile_prog, + import_names, + &EmitOptions { + from_path: None, + emit_path: Some(&ll_file), + no_link: true, + }, + ) + .expect("Compile KCL to LLVM error"); + let mut cmd = Command::new(plugin_agent); + let dylib_path = cmd.run_clang_single(&ll_path, &dylib_path); + let dylib_relative_path = dylib_path.replacen(root, ".", 1); + + save_pkg_cache( + root, + &pkgpath, + dylib_relative_path, + CacheOption::default(), + ); + dylib_path + } + } + }; + if Path::new(&ll_path).exists() { + std::fs::remove_file(&ll_path).unwrap(); + } + ll_path_lock.unlock().unwrap(); + tx.send(dylib_path) + .expect("channel will be there waiting for the pool"); + }); + } + rx.iter().take(prog_count).collect::>() + } +} \ No newline at end of file diff --git a/kclvm/runner/src/command.rs b/kclvm/runner/src/command.rs index ac18d6fe4..a4c2e4053 100644 --- a/kclvm/runner/src/command.rs +++ b/kclvm/runner/src/command.rs @@ -423,7 +423,6 @@ impl Command { } else { "clang" }; - if let Some(s) = Self::find_it(clang_exe) { return s.to_str().unwrap().to_string(); diff --git a/kclvm/runner/src/lib.rs b/kclvm/runner/src/lib.rs index e7a51a61d..abe33d061 100644 --- a/kclvm/runner/src/lib.rs +++ b/kclvm/runner/src/lib.rs @@ -1,2 +1,53 @@ +use kclvm_ast::ast::Program; +use kclvm_sema::resolver::scope::ProgramScope; +use runner::{ExecProgramArgs, KclvmRunner, KclvmRunnerOptions}; + pub mod command; +pub mod assembler; +pub mod linker; pub mod runner; + +#[cfg(test)] +pub mod tests; + +/// After the kcl program passed through kclvm-parser and kclvm-sema in the compiler frontend, +/// KCLVM needs to generate corresponding LLVM IR, dylibs or executable file for kcl program +/// in the compiler backend. +/// +/// Method “execute” is the entry point for the compiler backend. +/// +/// It returns the KCL program executing result as Result, +/// and mainly takes "program" (ast.Program returned by kclvm-parser) and "scope" +/// (ProgramScope returned by kclvm-sema) as input. +/// +/// "plugin_agent" is related to KCLVM plugin. +/// "args" is the items selected by the user in the KCLVM CLI. +/// +/// In the method, dylibs is generated by KclvmAssembler, and method "KclvmAssembler::gen_dylibs" +/// will return dylibs path in a "Vec"; +/// +/// After linking all dylibs by KclvmLinker, method "KclvmLinker::link_all_dylibs" will return a path +/// for dylib. +/// +/// At last, KclvmRunner will be constructed and call method "run" to execute the kcl program. +pub fn execute( + program: Program, + scope: ProgramScope, + plugin_agent: u64, + args: &ExecProgramArgs, +) -> Result { + // generate dylibs + let dylib_paths = assembler::KclvmAssembler::gen_dylibs(program, scope, plugin_agent); + + // link dylibsKclvmRunner + let dylib_path = linker::KclvmLinker::link_all_dylibs(dylib_paths, plugin_agent); + + // run + let runner = KclvmRunner::new( + dylib_path.as_str(), + Some(KclvmRunnerOptions { + plugin_agent_ptr: plugin_agent, + }), + ); + runner.run(&args) +} diff --git a/kclvm/runner/src/linker.rs b/kclvm/runner/src/linker.rs new file mode 100644 index 000000000..a5462c66b --- /dev/null +++ b/kclvm/runner/src/linker.rs @@ -0,0 +1,11 @@ +use crate::command::Command; + +/// KclvmLinker is mainly responsible for linking the dylibs generated by KclvmAssembler. +pub struct KclvmLinker {} +impl KclvmLinker { + /// Link the dylibs generated by method "gen_bc_or_ll_file". + pub fn link_all_dylibs(dylib_paths: Vec, plugin_agent: u64) -> String { + let mut cmd = Command::new(plugin_agent); + cmd.link_dylibs(&dylib_paths, "") + } +} \ No newline at end of file diff --git a/kclvm/runner/src/runner.rs b/kclvm/runner/src/runner.rs index f38efe6fe..b009f6067 100644 --- a/kclvm/runner/src/runner.rs +++ b/kclvm/runner/src/runner.rs @@ -1,5 +1,4 @@ use serde::{Deserialize, Serialize}; - use kclvm_ast::ast; #[allow(non_camel_case_types)] diff --git a/kclvm/runner/src/test_datas/init_check_order_0/main.k b/kclvm/runner/src/test_datas/init_check_order_0/main.k new file mode 100644 index 000000000..a76f90c8b --- /dev/null +++ b/kclvm/runner/src/test_datas/init_check_order_0/main.k @@ -0,0 +1,11 @@ +schema Person: + name: str + age: int + gender: str + info: str = "{}, {}, {} years old".format(name, gender, age) + +alice = Person { + "name": "alice", + "age": 10, + "gender": "female" +} diff --git a/kclvm/runner/src/test_datas/init_check_order_0/stdout.golden.json b/kclvm/runner/src/test_datas/init_check_order_0/stdout.golden.json new file mode 100644 index 000000000..c962e7309 --- /dev/null +++ b/kclvm/runner/src/test_datas/init_check_order_0/stdout.golden.json @@ -0,0 +1 @@ +{"alice": {"name": "alice", "age": 10, "gender": "female", "info": "alice, female, 10 years old", "__settings__": {"output_type": "INLINE", "__schema_type__": "__main__.Person"}}} \ No newline at end of file diff --git a/kclvm/runner/src/test_datas/init_check_order_1/main.k b/kclvm/runner/src/test_datas/init_check_order_1/main.k new file mode 100644 index 000000000..2711ed87e --- /dev/null +++ b/kclvm/runner/src/test_datas/init_check_order_1/main.k @@ -0,0 +1,61 @@ +schema Name: + mixin [UpperMixin] + firstName: str + lastName: str + upper: str + + # print("init name") + +schema Person(Name): + gender: str + title: str + info: str + + # print("init person") + +schema Girl(Person): + mixin [TitleMixin, InfoMixin] + gender: str = "female" + added: str = "some girl attr" + + # print("init girl") + + check: + gender == "female", "gender should be female in Girl" + +schema Boy(Person): + mixin [TitleMixin, InfoMixin] + gender: str = "male" + added: str = "some boy attr" + + # print("init boy") + + check: + gender == "male", "gender should be male in Boy" + +schema UpperMixin: + # print("init upperMixin") + upper: str = lastName.upper() + +schema TitleMixin: + # print("init title mixin") + if gender == "female": + title = "Ms.{}".format(lastName) + else: + title = "Mr.{}".format(lastName) + +schema InfoMixin: + # print("init info mixin") + info = "{}, {}".format(title, gender) + +alice = Girl { + "firstName": "Alice", + "lastName": "Smith" +} + +# print(" ===") + +bob = Boy { + "firstName": "Bob", + "lastName": "Green" +} diff --git a/kclvm/runner/src/test_datas/init_check_order_1/stdout.golden.json b/kclvm/runner/src/test_datas/init_check_order_1/stdout.golden.json new file mode 100644 index 000000000..1aad8c19b --- /dev/null +++ b/kclvm/runner/src/test_datas/init_check_order_1/stdout.golden.json @@ -0,0 +1 @@ +{"alice": {"firstName": "Alice", "lastName": "Smith", "upper": "SMITH", "__settings__": {"output_type": "INLINE", "__schema_type__": "__main__.InfoMixin"}, "gender": "female", "title": "Ms.Smith", "info": "Ms.Smith, female", "added": "some girl attr"}, "bob": {"firstName": "Bob", "lastName": "Green", "upper": "GREEN", "__settings__": {"output_type": "INLINE", "__schema_type__": "__main__.InfoMixin"}, "gender": "male", "title": "Mr.Green", "info": "Mr.Green, male", "added": "some boy attr"}} \ No newline at end of file diff --git a/kclvm/runner/src/test_datas/normal_2/main.k b/kclvm/runner/src/test_datas/normal_2/main.k new file mode 100644 index 000000000..2a7b43467 --- /dev/null +++ b/kclvm/runner/src/test_datas/normal_2/main.k @@ -0,0 +1,11 @@ + +schema NumberMap: + [num: str]: int + + check: + int(num) % 2 == 0 + +numMap = NumberMap { + str(0): 0 + str(2): 2 +} diff --git a/kclvm/runner/src/test_datas/normal_2/stdout.golden.json b/kclvm/runner/src/test_datas/normal_2/stdout.golden.json new file mode 100644 index 000000000..9958766ac --- /dev/null +++ b/kclvm/runner/src/test_datas/normal_2/stdout.golden.json @@ -0,0 +1 @@ +{"numMap":{"0":0,"2":2,"__settings__":{"__schema_type__":"__main__.NumberMap","output_type":"INLINE"}}} \ No newline at end of file diff --git a/kclvm/runner/src/test_datas/type_annotation_not_full_2/main.k b/kclvm/runner/src/test_datas/type_annotation_not_full_2/main.k new file mode 100644 index 000000000..5d277e7f0 --- /dev/null +++ b/kclvm/runner/src/test_datas/type_annotation_not_full_2/main.k @@ -0,0 +1,6 @@ +schema A[arg=1]: + a: int = arg + +a1 = A() +a2 = A(2) +a3 = A(arg=3) diff --git a/kclvm/runner/src/test_datas/type_annotation_not_full_2/stdout.golden.json b/kclvm/runner/src/test_datas/type_annotation_not_full_2/stdout.golden.json new file mode 100644 index 000000000..310d018aa --- /dev/null +++ b/kclvm/runner/src/test_datas/type_annotation_not_full_2/stdout.golden.json @@ -0,0 +1 @@ +{"a1":{"__settings__":{"__schema_type__":"__main__.A","output_type":"INLINE"},"a":1},"a2":{"__settings__":{"__schema_type__":"__main__.A","output_type":"INLINE"},"a":2},"a3":{"__settings__":{"__schema_type__":"__main__.A","output_type":"INLINE"},"a":3}} \ No newline at end of file diff --git a/kclvm/runner/src/tests.rs b/kclvm/runner/src/tests.rs new file mode 100644 index 000000000..232574ff9 --- /dev/null +++ b/kclvm/runner/src/tests.rs @@ -0,0 +1,83 @@ +use crate::{execute, runner::ExecProgramArgs}; +use kclvm_ast::ast::{Module, Program}; +use kclvm_sema::resolver::resolve_program; +use std::{collections::HashMap, fs::File}; + +const TEST_CASES: &[&'static str; 4] = &[ + "init_check_order_0", + "init_check_order_1", + "normal_2", + "type_annotation_not_full_2", +]; +const EXPECTED_FILE_NAME: &str = "stdout.golden.json"; +const TEST_CASE_PATH: &str = "./src/test_datas"; +const KCL_FILE_NAME: &str = "main.k"; +const MAIN_PKG_NAME: &str = "__main__"; + +/// Load test kcl file to ast.Program +pub fn load_program(filename: String) -> Program { + let module = load_module(filename); + construct_program(module) +} + +/// Load test kcl file to ast.Module +pub fn load_module(filename: String) -> Module { + kclvm_parser::parse_file(&filename, None).unwrap() +} + +/// Construct ast.Program by ast.Module and default configuration. +/// Default configuration: +/// module.pkg = "__main__" +/// Program.root = "__main__" +/// Program.main = "__main__" +/// Program.cmd_args = [] +/// Program.cmd_overrides = [] +pub fn construct_program(mut module: Module) -> Program { + module.pkg = MAIN_PKG_NAME.to_string(); + let mut pkgs_ast = HashMap::new(); + pkgs_ast.insert(MAIN_PKG_NAME.to_string(), vec![module]); + Program { + root: MAIN_PKG_NAME.to_string(), + main: MAIN_PKG_NAME.to_string(), + pkgs: pkgs_ast, + cmd_args: vec![], + cmd_overrides: vec![], + } +} + +/// Load the expect result from stdout.golden.json +pub fn load_expect_file(filename: String) -> String { + let f = File::open(filename).unwrap(); + let v: serde_json::Value = serde_json::from_reader(f).unwrap(); + v.to_string() +} + +/// Format str by json str +pub fn format_str_by_json(str: String) -> String { + let v: serde_json::Value = serde_json::from_str(&str).unwrap(); + v.to_string() +} + +pub fn execute_for_test(kcl_path: &String) -> String{ + let plugin_agent = 0; + let args = ExecProgramArgs::default(); + // parse kcl file + let mut program = load_program(kcl_path.to_string()); + // resolve ast + let scope = resolve_program(&mut program); + scope.check_scope_diagnostics(); + // generate dylibs, link dylibs and execute. + execute(program, scope, plugin_agent, &args).unwrap() +} + +#[test] +fn test_kclvm_runner_execute() { + for case in TEST_CASES { + let kcl_path = &format!("{}/{}/{}", TEST_CASE_PATH, case, KCL_FILE_NAME); + let expected_path = &format!("{}/{}/{}", TEST_CASE_PATH, case, EXPECTED_FILE_NAME); + let result = execute_for_test(kcl_path); + let expected_result = load_expect_file(expected_path.to_string()); + assert_eq!(expected_result, format_str_by_json(result)); + println!("{} - PASS", kcl_path); + } +} \ No newline at end of file diff --git a/kclvm/src/lib.rs b/kclvm/src/lib.rs index 291f5dc3a..d94d6ce68 100644 --- a/kclvm/src/lib.rs +++ b/kclvm/src/lib.rs @@ -1,19 +1,9 @@ extern crate serde; -use std::collections::HashMap; -use std::path::{Path, PathBuf}; -use std::sync::mpsc::channel; -use threadpool::ThreadPool; - -use indexmap::IndexMap; -use kclvm_ast::ast; -use kclvm_compiler::codegen::{llvm::emit_code, EmitOptions}; -use kclvm_config::cache::*; use kclvm_parser::load_program; -use kclvm_sema::resolver::resolve_program; - -use kclvm_runner::command::Command; +use kclvm_runner::execute; use kclvm_runner::runner::*; +use kclvm_sema::resolver::resolve_program; use kclvm_tools::query::apply_overrides; #[no_mangle] @@ -71,141 +61,11 @@ pub fn kclvm_cli_run_unsafe(args: *const i8, plugin_agent: *const i8) -> Result< // load ast let mut program = load_program(&files, Some(opts))?; apply_overrides(&mut program, &args.overrides, &[]); + + // resolve ast let scope = resolve_program(&mut program); scope.check_scope_diagnostics(); - // gen bc or ll file - let ll_file = "_a.out"; - let path = std::path::Path::new(ll_file); - if path.exists() { - std::fs::remove_file(path).unwrap(); - } - for entry in glob::glob(&format!("{}*.ll", ll_file)).unwrap() { - match entry { - Ok(path) => { - if path.exists() { - std::fs::remove_file(path).unwrap(); - } - } - Err(e) => println!("{:?}", e), - }; - } - - let cache_dir = Path::new(&program.root) - .join(".kclvm") - .join("cache") - .join(kclvm_version::get_full_version()); - if !cache_dir.exists() { - std::fs::create_dir_all(&cache_dir).unwrap(); - } - let mut compile_progs: IndexMap< - String, - ( - ast::Program, - IndexMap>, - PathBuf, - ), - > = IndexMap::default(); - for (pkgpath, modules) in program.pkgs { - let mut pkgs = HashMap::new(); - pkgs.insert(pkgpath.clone(), modules); - let compile_prog = ast::Program { - root: program.root.clone(), - main: program.main.clone(), - pkgs, - cmd_args: vec![], - cmd_overrides: vec![], - }; - compile_progs.insert( - pkgpath, - (compile_prog, scope.import_names.clone(), cache_dir.clone()), - ); - } - let pool = ThreadPool::new(4); - let (tx, rx) = channel(); - let prog_count = compile_progs.len(); - for (pkgpath, (compile_prog, import_names, cache_dir)) in compile_progs { - let tx = tx.clone(); - pool.execute(move || { - let root = &compile_prog.root; - let is_main_pkg = pkgpath == kclvm_ast::MAIN_PKG; - let file = if is_main_pkg { - PathBuf::from(&pkgpath) - } else { - cache_dir.join(&pkgpath) - }; - let ll_file = file.to_str().unwrap(); - let ll_path = format!("{}.ll", ll_file); - let dylib_path = format!("{}{}", ll_file, Command::get_lib_suffix()); - let mut ll_path_lock = fslock::LockFile::open(&format!("{}.lock", ll_path)).unwrap(); - ll_path_lock.lock().unwrap(); - if Path::new(&ll_path).exists() { - std::fs::remove_file(&ll_path).unwrap(); - } - let dylib_path = if is_main_pkg { - emit_code( - &compile_prog, - import_names, - &EmitOptions { - from_path: None, - emit_path: Some(&ll_file), - no_link: true, - }, - ) - .expect("Compile KCL to LLVM error"); - let mut cmd = Command::new(plugin_agent); - cmd.run_clang_single(&ll_path, &dylib_path) - } else { - // If AST module has been modified, ignore the dylib cache - let dylib_relative_path: Option = - load_pkg_cache(root, &pkgpath, CacheOption::default()); - match dylib_relative_path { - Some(dylib_relative_path) => { - if dylib_relative_path.starts_with('.') { - dylib_relative_path.replacen(".", root, 1) - } else { - dylib_relative_path - } - } - None => { - emit_code( - &compile_prog, - import_names, - &EmitOptions { - from_path: None, - emit_path: Some(&ll_file), - no_link: true, - }, - ) - .expect("Compile KCL to LLVM error"); - let mut cmd = Command::new(plugin_agent); - let dylib_path = cmd.run_clang_single(&ll_path, &dylib_path); - let dylib_relative_path = dylib_path.replacen(root, ".", 1); - - save_pkg_cache(root, &pkgpath, dylib_relative_path, CacheOption::default()); - dylib_path - } - } - }; - if Path::new(&ll_path).exists() { - std::fs::remove_file(&ll_path).unwrap(); - } - ll_path_lock.unlock().unwrap(); - tx.send(dylib_path) - .expect("channel will be there waiting for the pool"); - }); - } - let dylib_paths = rx.iter().take(prog_count).collect::>(); - let mut cmd = Command::new(plugin_agent); - // link all dylibs - let dylib_path = cmd.link_dylibs(&dylib_paths, ""); - // Config uild - // run dylib - let runner = KclvmRunner::new( - dylib_path.as_str(), - Some(KclvmRunnerOptions { - plugin_agent_ptr: plugin_agent, - }), - ); - runner.run(&args) + // generate dylibs, link dylibs and execute. + execute(program, scope, plugin_agent, &args) }