diff --git a/kclvm/Cargo.lock b/kclvm/Cargo.lock index 270ecf918..bf62f949d 100644 --- a/kclvm/Cargo.lock +++ b/kclvm/Cargo.lock @@ -571,6 +571,7 @@ dependencies = [ "libloading", "serde", "serde_json", + "threadpool", "walkdir", ] diff --git a/kclvm/runner/Cargo.lock b/kclvm/runner/Cargo.lock index b90f4cf39..79058b926 100644 --- a/kclvm/runner/Cargo.lock +++ b/kclvm/runner/Cargo.lock @@ -544,6 +544,7 @@ dependencies = [ "libloading", "serde", "serde_json", + "threadpool", "walkdir", ] @@ -1393,6 +1394,15 @@ dependencies = [ "syn", ] +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + [[package]] name = "time" version = "0.1.44" diff --git a/kclvm/runner/Cargo.toml b/kclvm/runner/Cargo.toml index 27190b8ab..d14c57c31 100644 --- a/kclvm/runner/Cargo.toml +++ b/kclvm/runner/Cargo.toml @@ -15,6 +15,7 @@ libc = "0.2.112" indexmap = "1.0" fslock = "0.2.1" libloading = "0.7.3" +threadpool = "1.0" kclvm-ast = {path = "../ast", version = "0.1.0"} kclvm-parser = {path = "../parser", version = "0.1.0"} diff --git a/kclvm/runner/src/command.rs b/kclvm/runner/src/command.rs index ac18d6fe4..a4c2e4053 100644 --- a/kclvm/runner/src/command.rs +++ b/kclvm/runner/src/command.rs @@ -423,7 +423,6 @@ impl Command { } else { "clang" }; - if let Some(s) = Self::find_it(clang_exe) { return s.to_str().unwrap().to_string(); diff --git a/kclvm/runner/src/eval.rs b/kclvm/runner/src/eval.rs new file mode 100644 index 000000000..cd2aa4c87 --- /dev/null +++ b/kclvm/runner/src/eval.rs @@ -0,0 +1,204 @@ +use indexmap::IndexMap; +use std::{collections::HashMap, path::Path, path::PathBuf, sync::mpsc::channel}; +use threadpool::ThreadPool; + +use kclvm_ast::ast::{self, Program}; +use kclvm_compiler::codegen::{llvm::emit_code, EmitOptions}; +use kclvm_config::cache::{load_pkg_cache, save_pkg_cache, CacheOption}; +use kclvm_sema::resolver::resolve_program; +use kclvm_sema::resolver::scope::ProgramScope; + +use crate::{ + command::Command, + runner::{ExecProgramArgs, KclvmRunner, KclvmRunnerOptions}, +}; + +const LL_FILE: &str = "_a.out"; + +/// Evaluator is used to resolve kcl ast, generate dylibs and execute. +pub struct Evaluator {} + +impl Evaluator { + pub fn new() -> Self { + Self {} + } + + /// Take ast.program as input，and resolve ast, generate and link dylib, execute. + pub fn eval( + &self, + mut program: Program, + plugin_agent: u64, + args: &ExecProgramArgs, + ) -> Result { + // resolve ast + let scope = resolve_program(&mut program); + scope.check_scope_diagnostics(); + + // generate dylibs + let dylib_paths = self.gen_dylibs(program, scope, plugin_agent); + + // link dylibs + let dylib_path = self.link_all_dylibs(dylib_paths, plugin_agent); + + // execute + self.run_dylib(dylib_path, plugin_agent, args) + } + + /// Generate the dylibs and return file paths. + fn gen_dylibs( + &self, + program: ast::Program, + scope: ProgramScope, + plugin_agent: u64, + ) -> Vec { + // gen bc or ll_file + let path = std::path::Path::new(LL_FILE); + if path.exists() { + std::fs::remove_file(path).unwrap(); + } + for entry in glob::glob(&format!("{}*.ll", LL_FILE)).unwrap() { + match entry { + Ok(path) => { + if path.exists() { + std::fs::remove_file(path).unwrap(); + } + } + Err(e) => println!("{:?}", e), + }; + } + + let cache_dir = Path::new(&program.root) + .join(".kclvm") + .join("cache") + .join(kclvm_version::get_full_version()); + if !cache_dir.exists() { + std::fs::create_dir_all(&cache_dir).unwrap(); + } + let mut compile_progs: IndexMap< + String, + ( + ast::Program, + IndexMap>, + PathBuf, + ), + > = IndexMap::default(); + for (pkgpath, modules) in program.pkgs { + let mut pkgs = HashMap::new(); + pkgs.insert(pkgpath.clone(), modules); + let compile_prog = ast::Program { + root: program.root.clone(), + main: program.main.clone(), + pkgs, + cmd_args: vec![], + cmd_overrides: vec![], + }; + compile_progs.insert( + pkgpath, + (compile_prog, scope.import_names.clone(), cache_dir.clone()), + ); + } + let pool = ThreadPool::new(4); + let (tx, rx) = channel(); + let prog_count = compile_progs.len(); + for (pkgpath, (compile_prog, import_names, cache_dir)) in compile_progs { + let tx = tx.clone(); + pool.execute(move || { + let root = &compile_prog.root; + let is_main_pkg = pkgpath == kclvm_ast::MAIN_PKG; + let file = if is_main_pkg { + PathBuf::from(&pkgpath) + } else { + cache_dir.join(&pkgpath) + }; + let ll_file = file.to_str().unwrap(); + let ll_path = format!("{}.ll", ll_file); + let dylib_path = format!("{}{}", ll_file, Command::get_lib_suffix()); + let mut ll_path_lock = + fslock::LockFile::open(&format!("{}.lock", ll_path)).unwrap(); + ll_path_lock.lock().unwrap(); + if Path::new(&ll_path).exists() { + std::fs::remove_file(&ll_path).unwrap(); + } + let dylib_path = if is_main_pkg { + emit_code( + &compile_prog, + import_names, + &EmitOptions { + from_path: None, + emit_path: Some(&ll_file), + no_link: true, + }, + ) + .expect("Compile KCL to LLVM error"); + let mut cmd = Command::new(plugin_agent); + cmd.run_clang_single(&ll_path, &dylib_path) + } else { + // If AST module has been modified, ignore the dylib cache + let dylib_relative_path: Option = + load_pkg_cache(root, &pkgpath, CacheOption::default()); + match dylib_relative_path { + Some(dylib_relative_path) => { + if dylib_relative_path.starts_with('.') { + dylib_relative_path.replacen(".", root, 1) + } else { + dylib_relative_path + } + } + None => { + emit_code( + &compile_prog, + import_names, + &EmitOptions { + from_path: None, + emit_path: Some(&ll_file), + no_link: true, + }, + ) + .expect("Compile KCL to LLVM error"); + let mut cmd = Command::new(plugin_agent); + let dylib_path = cmd.run_clang_single(&ll_path, &dylib_path); + let dylib_relative_path = dylib_path.replacen(root, ".", 1); + + save_pkg_cache( + root, + &pkgpath, + dylib_relative_path, + CacheOption::default(), + ); + dylib_path + } + } + }; + if Path::new(&ll_path).exists() { + std::fs::remove_file(&ll_path).unwrap(); + } + ll_path_lock.unlock().unwrap(); + tx.send(dylib_path) + .expect("channel will be there waiting for the pool"); + }); + } + rx.iter().take(prog_count).collect::>() + } + + /// Link the dylibs generated by method "gen_bc_or_ll_file". + fn link_all_dylibs(&self, dylib_paths: Vec, plugin_agent: u64) -> String { + let mut cmd = Command::new(plugin_agent); + cmd.link_dylibs(&dylib_paths, "") + } + + /// Execute the dylibs linked by method "link_all_dylibs". + fn run_dylib( + &self, + dylib_path: String, + plugin_agent: u64, + args: &ExecProgramArgs, + ) -> Result { + let runner = KclvmRunner::new( + dylib_path.as_str(), + Some(KclvmRunnerOptions { + plugin_agent_ptr: plugin_agent, + }), + ); + runner.run(&args) + } +} diff --git a/kclvm/runner/src/lib.rs b/kclvm/runner/src/lib.rs index e7a51a61d..0203b42a8 100644 --- a/kclvm/runner/src/lib.rs +++ b/kclvm/runner/src/lib.rs @@ -1,2 +1,3 @@ pub mod command; +pub mod eval; pub mod runner; diff --git a/kclvm/src/lib.rs b/kclvm/src/lib.rs index 291f5dc3a..bb679d819 100644 --- a/kclvm/src/lib.rs +++ b/kclvm/src/lib.rs @@ -1,5 +1,6 @@ extern crate serde; +use kclvm_runner::eval::Evaluator; use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::mpsc::channel; @@ -71,141 +72,6 @@ pub fn kclvm_cli_run_unsafe(args: *const i8, plugin_agent: *const i8) -> Result< // load ast let mut program = load_program(&files, Some(opts))?; apply_overrides(&mut program, &args.overrides, &[]); - let scope = resolve_program(&mut program); - scope.check_scope_diagnostics(); - // gen bc or ll file - let ll_file = "_a.out"; - let path = std::path::Path::new(ll_file); - if path.exists() { - std::fs::remove_file(path).unwrap(); - } - for entry in glob::glob(&format!("{}*.ll", ll_file)).unwrap() { - match entry { - Ok(path) => { - if path.exists() { - std::fs::remove_file(path).unwrap(); - } - } - Err(e) => println!("{:?}", e), - }; - } - - let cache_dir = Path::new(&program.root) - .join(".kclvm") - .join("cache") - .join(kclvm_version::get_full_version()); - if !cache_dir.exists() { - std::fs::create_dir_all(&cache_dir).unwrap(); - } - let mut compile_progs: IndexMap< - String, - ( - ast::Program, - IndexMap>, - PathBuf, - ), - > = IndexMap::default(); - for (pkgpath, modules) in program.pkgs { - let mut pkgs = HashMap::new(); - pkgs.insert(pkgpath.clone(), modules); - let compile_prog = ast::Program { - root: program.root.clone(), - main: program.main.clone(), - pkgs, - cmd_args: vec![], - cmd_overrides: vec![], - }; - compile_progs.insert( - pkgpath, - (compile_prog, scope.import_names.clone(), cache_dir.clone()), - ); - } - let pool = ThreadPool::new(4); - let (tx, rx) = channel(); - let prog_count = compile_progs.len(); - for (pkgpath, (compile_prog, import_names, cache_dir)) in compile_progs { - let tx = tx.clone(); - pool.execute(move || { - let root = &compile_prog.root; - let is_main_pkg = pkgpath == kclvm_ast::MAIN_PKG; - let file = if is_main_pkg { - PathBuf::from(&pkgpath) - } else { - cache_dir.join(&pkgpath) - }; - let ll_file = file.to_str().unwrap(); - let ll_path = format!("{}.ll", ll_file); - let dylib_path = format!("{}{}", ll_file, Command::get_lib_suffix()); - let mut ll_path_lock = fslock::LockFile::open(&format!("{}.lock", ll_path)).unwrap(); - ll_path_lock.lock().unwrap(); - if Path::new(&ll_path).exists() { - std::fs::remove_file(&ll_path).unwrap(); - } - let dylib_path = if is_main_pkg { - emit_code( - &compile_prog, - import_names, - &EmitOptions { - from_path: None, - emit_path: Some(&ll_file), - no_link: true, - }, - ) - .expect("Compile KCL to LLVM error"); - let mut cmd = Command::new(plugin_agent); - cmd.run_clang_single(&ll_path, &dylib_path) - } else { - // If AST module has been modified, ignore the dylib cache - let dylib_relative_path: Option = - load_pkg_cache(root, &pkgpath, CacheOption::default()); - match dylib_relative_path { - Some(dylib_relative_path) => { - if dylib_relative_path.starts_with('.') { - dylib_relative_path.replacen(".", root, 1) - } else { - dylib_relative_path - } - } - None => { - emit_code( - &compile_prog, - import_names, - &EmitOptions { - from_path: None, - emit_path: Some(&ll_file), - no_link: true, - }, - ) - .expect("Compile KCL to LLVM error"); - let mut cmd = Command::new(plugin_agent); - let dylib_path = cmd.run_clang_single(&ll_path, &dylib_path); - let dylib_relative_path = dylib_path.replacen(root, ".", 1); - - save_pkg_cache(root, &pkgpath, dylib_relative_path, CacheOption::default()); - dylib_path - } - } - }; - if Path::new(&ll_path).exists() { - std::fs::remove_file(&ll_path).unwrap(); - } - ll_path_lock.unlock().unwrap(); - tx.send(dylib_path) - .expect("channel will be there waiting for the pool"); - }); - } - let dylib_paths = rx.iter().take(prog_count).collect::>(); - let mut cmd = Command::new(plugin_agent); - // link all dylibs - let dylib_path = cmd.link_dylibs(&dylib_paths, ""); - - // Config uild - // run dylib - let runner = KclvmRunner::new( - dylib_path.as_str(), - Some(KclvmRunnerOptions { - plugin_agent_ptr: plugin_agent, - }), - ); - runner.run(&args) + // resolve ast，generate dylibs, link dylibs and execute. + Evaluator::new().eval(program, plugin_agent, &args) }