From 38cc97d1c4838029c8cf499aafa1c0058454f727 Mon Sep 17 00:00:00 2001 From: Yaraslau Tamashevich Date: Thu, 1 Feb 2024 12:07:11 +0200 Subject: [PATCH 1/6] Initial implementation of llvm backend --- CMakeLists.txt | 4 +- src/CoreVM/CMakeLists.txt | 2 +- src/CoreVM/transform/MergeBlockPass.cpp | 2 + src/shell/ASTPrinter.cpp | 6 +- src/shell/CMakeLists.txt | 33 ++- src/shell/Shell.cpp | 13 +- src/shell/llvm_executor.cpp | 285 ++++++++++++++++++++++++ src/shell/main.cpp | 4 +- 8 files changed, 340 insertions(+), 9 deletions(-) create mode 100644 src/shell/llvm_executor.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index d5921bb..8fe4535 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.28 FATAL_ERROR) -project(endo VERSION "0.0.0" LANGUAGES CXX) +project(endo VERSION "0.0.0") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") @@ -12,6 +12,8 @@ set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +option(ENDO_USE_LLVM "Use llvm as a backend for the shell" ON) + if(NOT DEFINED ENDO_TRACE_VM) option(ENDO_TRACE_VM "Enables debug output" OFF) if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug") diff --git a/src/CoreVM/CMakeLists.txt b/src/CoreVM/CMakeLists.txt index c48ea05..996075f 100644 --- a/src/CoreVM/CMakeLists.txt +++ b/src/CoreVM/CMakeLists.txt @@ -62,4 +62,4 @@ target_include_directories(CoreVM PUBLIC $ ) -target_link_libraries(CoreVM PUBLIC fmt::fmt-header-only range-v3::range-v3) +target_link_libraries(CoreVM PUBLIC fmt::fmt-header-only range-v3::range-v3 crispy::core) diff --git a/src/CoreVM/transform/MergeBlockPass.cpp b/src/CoreVM/transform/MergeBlockPass.cpp index 48bf735..79b5642 100644 --- a/src/CoreVM/transform/MergeBlockPass.cpp +++ b/src/CoreVM/transform/MergeBlockPass.cpp @@ -2,6 +2,8 @@ module; #include +#include + module CoreVM; namespace CoreVM diff --git a/src/shell/ASTPrinter.cpp b/src/shell/ASTPrinter.cpp index 93edc6b..98abed4 100644 --- a/src/shell/ASTPrinter.cpp +++ b/src/shell/ASTPrinter.cpp @@ -4,7 +4,7 @@ module; #include #include -#include +#include import Lexer; @@ -146,8 +146,8 @@ export class ASTPrinter: public Visitor } void visit(LiteralExpr const& node) override { _result += fmt::format("{}", node.value); } - void visit(SubstitutionExpr const& node) override { crispy::ignore_unused(node); } - void visit(CommandFileSubst const& node) override { crispy::ignore_unused(node); } + void visit(SubstitutionExpr const& node) override {} + void visit(CommandFileSubst const& node) override {} }; } // namespace endo::ast diff --git a/src/shell/CMakeLists.txt b/src/shell/CMakeLists.txt index 858dd30..40eebc5 100644 --- a/src/shell/CMakeLists.txt +++ b/src/shell/CMakeLists.txt @@ -15,6 +15,7 @@ target_sources(Shell ASTPrinter.cpp IRGenerator.cpp Parser.cpp + llvm_executor.cpp ) find_package(Threads REQUIRED) @@ -23,7 +24,36 @@ find_package(Threads REQUIRED) # see https://github.com/llvm/llvm-project/issues/75108 # and https://github.com/ericniebler/range-v3/blob/f013aef2ae81f3661a560e7922a968665bedebff/include/meta/meta_fwd.hpp#L37 target_compile_definitions(Shell PUBLIC META_WORKAROUND_LLVM_28385) -target_link_libraries(Shell PUBLIC fmt::fmt-header-only vtparser crispy::core CoreVM InputEditor Threads::Threads util) +set(shell_libs + fmt::fmt-header-only + vtparser + CoreVM + crispy::core + InputEditor + Threads::Threads + util) + + +if(ENDO_USE_LLVM) + find_package(LLVM) + if(LLVM_FOUND) + + message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") + message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") + include("${LLVM_DIR}/AddLLVM.cmake") + + include_directories(${LLVM_INCLUDE_DIRS}) + add_definitions(${LLVM_DEFINITIONS}) + llvm_map_components_to_libnames(llvm_libs analysis core executionengine instcombine object orcjit runtimedyld scalaropts support native) + # Link against LLVM libraries + set(shell_libs + ${shell_libs} + ${llvm_libs}) + + target_compile_definitions(Shell PUBLIC ENDO_USE_LLVM) + endif() +else() +endif() if(ENDO_TRACE_VM) @@ -36,6 +66,7 @@ if(ENDO_TRACE_LEXER) target_compile_definitions(Shell PUBLIC ENDO_TRACE_LEXER) endif() +target_link_libraries(Shell PUBLIC ${shell_libs}) add_executable(endo main.cpp diff --git a/src/shell/Shell.cpp b/src/shell/Shell.cpp index c642f2b..65e58f5 100644 --- a/src/shell/Shell.cpp +++ b/src/shell/Shell.cpp @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 module; #include - #include #include @@ -20,9 +19,10 @@ import Lexer; import ASTPrinter; import IRGenerator; import Parser; - import CoreVM; +import LLVMBack; + export module Shell; auto inline debugLog = logstore::category("debug ", "Debug log", logstore::category::state::Enabled); @@ -206,7 +206,16 @@ export class Shell final: public CoreVM::Runtime return _quit ? _exitCode : EXIT_SUCCESS; } + int execute(std::string const& lineBuffer) + { +#if defined(ENDO_USE_LLVM) + return llvm_backend::executeLLVM(lineBuffer); +#else + return executeCoreVM(lineBuffer); +#endif + } + int executeCoreVM(std::string const& lineBuffer) { try { diff --git a/src/shell/llvm_executor.cpp b/src/shell/llvm_executor.cpp new file mode 100644 index 0000000..cf097c5 --- /dev/null +++ b/src/shell/llvm_executor.cpp @@ -0,0 +1,285 @@ +module; + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "llvm/ADT/APFloat.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/CompileUtils.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" +#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" +#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" +#include "llvm/ExecutionEngine/Orc/LLJIT.h" +#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h" +#include "llvm/ExecutionEngine/SectionMemoryManager.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/Verifier.h" +#include "llvm/Passes/PassBuilder.h" +#include "llvm/Passes/StandardInstrumentations.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/InitLLVM.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Transforms/InstCombine/InstCombine.h" +#include "llvm/Transforms/Scalar.h" +#include "llvm/Transforms/Scalar/GVN.h" +#include "llvm/Transforms/Scalar/Reassociate.h" +#include "llvm/Transforms/Scalar/SimplifyCFG.h" +#include "llvm/Transforms/Utils.h" + +using namespace llvm; +using namespace llvm::orc; + +export module LLVMBack; + +class TestJIT +{ + private: + std::unique_ptr ES; + + DataLayout DL; + MangleAndInterner Mangle; + + RTDyldObjectLinkingLayer ObjectLayer; + IRCompileLayer CompileLayer; + + JITDylib& MainJD; + + public: + TestJIT(std::unique_ptr ES, JITTargetMachineBuilder JTMB, DataLayout DL): + ES(std::move(ES)), + DL(std::move(DL)), + Mangle(*this->ES, this->DL), + ObjectLayer(*this->ES, []() { return std::make_unique(); }), + CompileLayer(*this->ES, ObjectLayer, std::make_unique(std::move(JTMB))), + MainJD(this->ES->createBareJITDylib("
")) + { + MainJD.addGenerator( + cantFail(DynamicLibrarySearchGenerator::GetForCurrentProcess(DL.getGlobalPrefix()))); + if (JTMB.getTargetTriple().isOSBinFormatCOFF()) + { + ObjectLayer.setOverrideObjectFlagsWithResponsibilityFlags(true); + ObjectLayer.setAutoClaimResponsibilityForObjectSymbols(true); + } + } + + ~TestJIT() + { + if (auto Err = ES->endSession()) + ES->reportError(std::move(Err)); + } + + static Expected> Create() + { + auto EPC = SelfExecutorProcessControl::Create(); + if (!EPC) + return EPC.takeError(); + + auto ES = std::make_unique(std::move(*EPC)); + + JITTargetMachineBuilder JTMB(ES->getExecutorProcessControl().getTargetTriple()); + + auto DL = JTMB.getDefaultDataLayoutForTarget(); + if (!DL) + return DL.takeError(); + + return std::make_unique(std::move(ES), std::move(JTMB), std::move(*DL)); + } + + const DataLayout& getDataLayout() const { return DL; } + + JITDylib& getMainJITDylib() { return MainJD; } + + Error addModule(ThreadSafeModule TSM, ResourceTrackerSP RT = nullptr) + { + if (!RT) + RT = MainJD.getDefaultResourceTracker(); + return CompileLayer.add(RT, std::move(TSM)); + } + + Expected lookup(StringRef Name) { return ES->lookup({ &MainJD }, Mangle(Name.str())); } +}; + +std::vector tokenizer(const std::string& p_pcstStr, char delim) +{ + std::vector tokens; + std::stringstream mySstream(p_pcstStr); + std::string temp; + + while (getline(mySstream, temp, delim)) + { + tokens.push_back(temp); + } + + return tokens; +} + +std::tuple SeparateProg(const std::string input) +{ + + std::stringstream inStream(input); + std::string prog; + std::string args; + getline(inStream, prog, ' '); + getline(inStream, args, '\n'); + + return { prog, args }; +} + +extern "C" void execute(const char* prog, const char* params) +{ + std::vector argv; + argv.push_back(prog); + argv.push_back(params); + + pid_t const pid = fork(); + switch (pid) + { + case -1: return; + case 0: { + execvp(prog, const_cast(argv.data())); + } + default: int wstatus = 0; waitpid(pid, &wstatus, 0); + } +} + +int jitPart(std::string input) +{ + using namespace llvm; + using namespace llvm::orc; + + std::unique_ptr> builder; + + // Create an LLJIT instance. + ExitOnError ExitOnErr; + auto J = ExitOnErr(TestJIT::Create()); + + auto Context = std::make_unique(); + + builder = std::make_unique>(*Context); + + auto TheModule = std::make_unique("test", *Context); + TheModule->setDataLayout(J->getDataLayout()); + + // Create new pass and analysis managers. + auto TheFPM = std::make_unique(); + auto TheLAM = std::make_unique(); + auto TheFAM = std::make_unique(); + auto TheCGAM = std::make_unique(); + auto TheMAM = std::make_unique(); + auto ThePIC = std::make_unique(); + auto TheSI = std::make_unique(*Context, /*DebugLogging*/ true); + TheSI->registerCallbacks(*ThePIC, TheMAM.get()); + + // Add transform passes. + // Do simple "peephole" optimizations and bit-twiddling optzns. + TheFPM->addPass(InstCombinePass()); + // Reassociate expressions. + TheFPM->addPass(ReassociatePass()); + // Eliminate Common SubExpressions. + TheFPM->addPass(GVNPass()); + // Simplify the control flow graph (deleting unreachable blocks, etc). + TheFPM->addPass(SimplifyCFGPass()); + + // Register analysis passes used in these transform passes. + PassBuilder PB; + PB.registerModuleAnalyses(*TheMAM); + PB.registerFunctionAnalyses(*TheFAM); + PB.crossRegisterProxies(*TheLAM, *TheFAM, *TheCGAM, *TheMAM); + + auto byteptr = builder->getPtrTy(); + //llvm::Type::getInt8Ty(*Context)->getPo; + auto FunctionType = llvm::FunctionType::get(llvm::Type::getVoidTy(*Context), {byteptr, byteptr}, false); + + auto fun = TheModule->getOrInsertFunction("execute", FunctionType); + + Function* F = Function::Create(FunctionType, Function::ExternalLinkage, "__anon_expr", TheModule.get()); + + // Add a basic block to the function. As before, it automatically inserts + // because of the last argument. + BasicBlock* BB = BasicBlock::Create(*Context, "EntryBlock", F); + + builder->SetInsertPoint(BB); + + // Get pointers to the constant `1'. + + const auto [prog, args] = SeparateProg(input); + + auto CalRes = builder->CreateCall(fun,{builder->CreateGlobalString(prog),builder->CreateGlobalString(args)}); + + // Value *One = builder.getInt32(1); + // Value *Add = builder.CreateAdd(One, One); + + builder->CreateRet(builder->getInt32(1)); + + auto RT = J->getMainJITDylib().createResourceTracker(); + + auto M = ThreadSafeModule(std::move(TheModule), std::move(Context)); + + M.getModuleUnlocked()->print(llvm::outs(), nullptr); + + ExitOnErr(J->addModule(std::move(M), RT)); + + // Look up the JIT'd function, cast it to a function pointer, then call it. + auto ExprSymbol = ExitOnErr(J->lookup("__anon_expr")); + + void (*FP)() = ExprSymbol.getAddress().toPtr(); + FP(); + + return EXIT_SUCCESS; +} + +export namespace llvm_backend +{ + +int executeLLVM(std::string const& lineBuffer) +{ + try + { + return jitPart(lineBuffer); + } + catch (std::exception const& e) + { + std::cout << "Exception caught: " << e.what(); + return EXIT_FAILURE; + } +} + +void init() +{ + InitializeNativeTarget(); + InitializeNativeTargetAsmPrinter(); +} + +} // namespace llvm_backend diff --git a/src/shell/main.cpp b/src/shell/main.cpp index 6690ee3..762b3b4 100644 --- a/src/shell/main.cpp +++ b/src/shell/main.cpp @@ -4,7 +4,7 @@ using namespace std::string_literals; import Shell; - +import LLVMBack; std::string_view getEnvironment(std::string_view name, std::string_view defaultValue) { @@ -18,6 +18,8 @@ int main(int argc, char const* argv[]) setsid(); + llvm_backend::init(); + if (argc == 2) // This here only exists for early-development debugging purposes. return shell.execute(argv[1]); From 9e0d51a4599a7860cc9cd0056fd9dba173200df3 Mon Sep 17 00:00:00 2001 From: Yaraslau Tamashevich Date: Fri, 2 Feb 2024 09:44:45 +0200 Subject: [PATCH 2/6] Cleanup after comments --- .github/workflows/build.yml | 6 +-- src/shell/Shell.cpp | 4 +- src/shell/llvm_executor.cpp | 82 ++++++++++++++++++------------------- src/shell/main.cpp | 4 +- 4 files changed, 48 insertions(+), 48 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9d32f3c..cf92f68 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -80,15 +80,15 @@ jobs: run: clang++ --version - name: Check clang version - run: clang++-17 --version + run: clang++-18 --version - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type run: > cmake -B ${{ steps.strings.outputs.build-output-dir }} - -DCMAKE_CXX_COMPILER=clang++-17 - -DCMAKE_C_COMPILER=clang-17 + -DCMAKE_CXX_COMPILER=clang++-18 + -DCMAKE_C_COMPILER=clang-18 -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -GNinja -S ${{ github.workspace }} diff --git a/src/shell/Shell.cpp b/src/shell/Shell.cpp index 65e58f5..d71fbd9 100644 --- a/src/shell/Shell.cpp +++ b/src/shell/Shell.cpp @@ -21,7 +21,7 @@ import IRGenerator; import Parser; import CoreVM; -import LLVMBack; +import LLVMBackend; export module Shell; @@ -210,7 +210,7 @@ export class Shell final: public CoreVM::Runtime int execute(std::string const& lineBuffer) { #if defined(ENDO_USE_LLVM) - return llvm_backend::executeLLVM(lineBuffer); + return LLVMBackend::executeLLVM(lineBuffer); #else return executeCoreVM(lineBuffer); #endif diff --git a/src/shell/llvm_executor.cpp b/src/shell/llvm_executor.cpp index cf097c5..a3b33c5 100644 --- a/src/shell/llvm_executor.cpp +++ b/src/shell/llvm_executor.cpp @@ -18,50 +18,50 @@ module; #include -#include "llvm/ADT/APFloat.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringRef.h" -#include "llvm/ExecutionEngine/JITSymbol.h" -#include "llvm/ExecutionEngine/Orc/CompileUtils.h" -#include "llvm/ExecutionEngine/Orc/Core.h" -#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" -#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" -#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" -#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" -#include "llvm/ExecutionEngine/Orc/LLJIT.h" -#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" -#include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h" -#include "llvm/ExecutionEngine/SectionMemoryManager.h" -#include "llvm/IR/BasicBlock.h" -#include "llvm/IR/Constants.h" -#include "llvm/IR/DataLayout.h" -#include "llvm/IR/DerivedTypes.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/IRBuilder.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/LLVMContext.h" -#include "llvm/IR/Module.h" -#include "llvm/IR/PassManager.h" -#include "llvm/IR/Type.h" -#include "llvm/IR/Verifier.h" -#include "llvm/Passes/PassBuilder.h" -#include "llvm/Passes/StandardInstrumentations.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/InitLLVM.h" -#include "llvm/Support/TargetSelect.h" -#include "llvm/Support/raw_ostream.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Transforms/InstCombine/InstCombine.h" -#include "llvm/Transforms/Scalar.h" -#include "llvm/Transforms/Scalar/GVN.h" -#include "llvm/Transforms/Scalar/Reassociate.h" -#include "llvm/Transforms/Scalar/SimplifyCFG.h" -#include "llvm/Transforms/Utils.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include using namespace llvm; using namespace llvm::orc; -export module LLVMBack; +export module LLVMBackend; class TestJIT { @@ -260,7 +260,7 @@ int jitPart(std::string input) return EXIT_SUCCESS; } -export namespace llvm_backend +export namespace LLVMBackend { int executeLLVM(std::string const& lineBuffer) diff --git a/src/shell/main.cpp b/src/shell/main.cpp index 762b3b4..545a3d8 100644 --- a/src/shell/main.cpp +++ b/src/shell/main.cpp @@ -4,7 +4,7 @@ using namespace std::string_literals; import Shell; -import LLVMBack; +import LLVMBackend; std::string_view getEnvironment(std::string_view name, std::string_view defaultValue) { @@ -18,7 +18,7 @@ int main(int argc, char const* argv[]) setsid(); - llvm_backend::init(); + LLVMBackend::init(); if (argc == 2) // This here only exists for early-development debugging purposes. From fb119283be1e0dc17731b411f34a5c8211a608e4 Mon Sep 17 00:00:00 2001 From: Yaraslau Tamashevich Date: Fri, 2 Feb 2024 09:59:19 +0200 Subject: [PATCH 3/6] Update github actions --- .github/workflows/build.yml | 6 ++++-- CMakeLists.txt | 1 - src/shell/CMakeLists.txt | 31 +++++++++++++++---------------- src/shell/Shell.cpp | 11 ++++++++++- src/shell/main.cpp | 3 +-- 5 files changed, 30 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cf92f68..a96c6e2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ jobs: build: runs-on: ${{ matrix.os }} - name: "${{ matrix.cpp_compiler }}-${{ matrix.os }}-${{ matrix.build_type }}" + name: "${{ matrix.build_type }}; LLVM: ${{ matrix.llvm_backend }}" strategy: # Set fail-fast to false to ensure that feedback is delivered for all matrix combinations. Consider changing this to true when your workflow is stable. @@ -30,7 +30,8 @@ jobs: # To add more build types (Release, Debug, RelWithDebInfo, etc.) customize the build_type list. matrix: os: [ubuntu-latest] - build_type: [RelWithDebInfo] + build_type: [Release,Debug] + llvm_backend: [ON,OFF] steps: - uses: actions/checkout@v3 @@ -90,6 +91,7 @@ jobs: -DCMAKE_CXX_COMPILER=clang++-18 -DCMAKE_C_COMPILER=clang-18 -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} + -DENDO_USE_LLVM=${{ matrix.llvm_backend }} -GNinja -S ${{ github.workspace }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 8fe4535..9b5e01c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,6 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - option(ENDO_USE_LLVM "Use llvm as a backend for the shell" ON) if(NOT DEFINED ENDO_TRACE_VM) diff --git a/src/shell/CMakeLists.txt b/src/shell/CMakeLists.txt index 40eebc5..7c25b0c 100644 --- a/src/shell/CMakeLists.txt +++ b/src/shell/CMakeLists.txt @@ -15,7 +15,6 @@ target_sources(Shell ASTPrinter.cpp IRGenerator.cpp Parser.cpp - llvm_executor.cpp ) find_package(Threads REQUIRED) @@ -35,24 +34,24 @@ set(shell_libs if(ENDO_USE_LLVM) - find_package(LLVM) - if(LLVM_FOUND) + find_package(LLVM REQUIRED) + message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") + message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") + include("${LLVM_DIR}/AddLLVM.cmake") - message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") - message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") - include("${LLVM_DIR}/AddLLVM.cmake") - - include_directories(${LLVM_INCLUDE_DIRS}) - add_definitions(${LLVM_DEFINITIONS}) - llvm_map_components_to_libnames(llvm_libs analysis core executionengine instcombine object orcjit runtimedyld scalaropts support native) + include_directories(${LLVM_INCLUDE_DIRS}) + add_definitions(${LLVM_DEFINITIONS}) + llvm_map_components_to_libnames(llvm_libs analysis core executionengine instcombine object orcjit runtimedyld scalaropts support native) # Link against LLVM libraries - set(shell_libs - ${shell_libs} - ${llvm_libs}) + set(shell_libs + ${shell_libs} + ${llvm_libs}) - target_compile_definitions(Shell PUBLIC ENDO_USE_LLVM) - endif() -else() + target_sources(Shell + PUBLIC + FILE_SET CXX_MODULES FILES + llvm_executor.cpp) + target_compile_definitions(Shell PUBLIC ENDO_USE_LLVM) endif() diff --git a/src/shell/Shell.cpp b/src/shell/Shell.cpp index d71fbd9..628ff89 100644 --- a/src/shell/Shell.cpp +++ b/src/shell/Shell.cpp @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 module; #include + #include #include @@ -21,7 +22,9 @@ import IRGenerator; import Parser; import CoreVM; +#if defined(ENDO_USE_LLVM) import LLVMBackend; +#endif export module Shell; @@ -171,7 +174,13 @@ export class SystemEnvironment: public Environment export class Shell final: public CoreVM::Runtime { public: - Shell(): Shell(RealTTY::instance(), SystemEnvironment::instance()) {} + Shell(): Shell(RealTTY::instance(), SystemEnvironment::instance()) + { + +#if defined(ENDO_USE_LLVM) + LLVMBackend::init(); +#endif + } Shell(TTY& tty, Environment& env): _env { env }, _tty { tty } { diff --git a/src/shell/main.cpp b/src/shell/main.cpp index 545a3d8..2b8eba5 100644 --- a/src/shell/main.cpp +++ b/src/shell/main.cpp @@ -4,7 +4,7 @@ using namespace std::string_literals; import Shell; -import LLVMBackend; + std::string_view getEnvironment(std::string_view name, std::string_view defaultValue) { @@ -18,7 +18,6 @@ int main(int argc, char const* argv[]) setsid(); - LLVMBackend::init(); if (argc == 2) // This here only exists for early-development debugging purposes. From c2ae1a4820493565320cfd698a23abe9c4d23a58 Mon Sep 17 00:00:00 2001 From: Yaraslau Tamashevich Date: Sun, 4 Feb 2024 22:25:02 +0200 Subject: [PATCH 4/6] llvm backend part 1 --- src/shell/CMakeLists.txt | 2 +- src/shell/LLVMBackend.cpp | 354 +++++++++++++++++ src/shell/Shell.cpp | 770 +++++++++++++++++++----------------- src/shell/Shell_test.cpp | 22 +- src/shell/llvm_executor.cpp | 285 ------------- src/shell/main.cpp | 8 +- 6 files changed, 787 insertions(+), 654 deletions(-) create mode 100644 src/shell/LLVMBackend.cpp delete mode 100644 src/shell/llvm_executor.cpp diff --git a/src/shell/CMakeLists.txt b/src/shell/CMakeLists.txt index 7c25b0c..89dd921 100644 --- a/src/shell/CMakeLists.txt +++ b/src/shell/CMakeLists.txt @@ -50,7 +50,7 @@ if(ENDO_USE_LLVM) target_sources(Shell PUBLIC FILE_SET CXX_MODULES FILES - llvm_executor.cpp) + LLVMBackend.cpp) target_compile_definitions(Shell PUBLIC ENDO_USE_LLVM) endif() diff --git a/src/shell/LLVMBackend.cpp b/src/shell/LLVMBackend.cpp new file mode 100644 index 0000000..8f36589 --- /dev/null +++ b/src/shell/LLVMBackend.cpp @@ -0,0 +1,354 @@ +module; + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace llvm; +using namespace llvm::orc; + +export module LLVMBackend; + +class TestJIT +{ + private: + std::unique_ptr ES; + + DataLayout DL; + MangleAndInterner Mangle; + + RTDyldObjectLinkingLayer ObjectLayer; + IRCompileLayer CompileLayer; + + JITDylib& MainJD; + + public: + TestJIT(std::unique_ptr ES, JITTargetMachineBuilder JTMB, DataLayout DL): + ES(std::move(ES)), + DL(std::move(DL)), + Mangle(*this->ES, this->DL), + ObjectLayer(*this->ES, []() { return std::make_unique(); }), + CompileLayer(*this->ES, ObjectLayer, std::make_unique(std::move(JTMB))), + MainJD(this->ES->createBareJITDylib("
")) + { + MainJD.addGenerator( + cantFail(DynamicLibrarySearchGenerator::GetForCurrentProcess(DL.getGlobalPrefix()))); + if (JTMB.getTargetTriple().isOSBinFormatCOFF()) + { + ObjectLayer.setOverrideObjectFlagsWithResponsibilityFlags(true); + ObjectLayer.setAutoClaimResponsibilityForObjectSymbols(true); + } + } + + ~TestJIT() + { + if (auto Err = ES->endSession()) + ES->reportError(std::move(Err)); + } + + static Expected> Create() + { + auto EPC = SelfExecutorProcessControl::Create(); + if (!EPC) + return EPC.takeError(); + + auto ES = std::make_unique(std::move(*EPC)); + + JITTargetMachineBuilder JTMB(ES->getExecutorProcessControl().getTargetTriple()); + + auto DL = JTMB.getDefaultDataLayoutForTarget(); + if (!DL) + return DL.takeError(); + + return std::make_unique(std::move(ES), std::move(JTMB), std::move(*DL)); + } + + const DataLayout& getDataLayout() const { return DL; } + + JITDylib& getMainJITDylib() { return MainJD; } + + Error addModule(ThreadSafeModule TSM, ResourceTrackerSP RT = nullptr) + { + if (!RT) + RT = MainJD.getDefaultResourceTracker(); + return CompileLayer.add(RT, std::move(TSM)); + } + + Expected lookup(StringRef Name) { return ES->lookup({ &MainJD }, Mangle(Name.str())); } +}; + +std::vector tokenizer(const std::string& p_pcstStr, char delim) +{ + std::vector tokens; + std::stringstream mySstream(p_pcstStr); + std::string temp; + + while (getline(mySstream, temp, delim)) + { + tokens.push_back(temp); + } + + return tokens; +} + +extern "C" void execute_llvmbackend(const char* prog, const char* params, int stdinFd, int stdoutFd) +{ + + std::vector argv; + argv.push_back(prog); + argv.push_back(params); + + pid_t const pid = fork(); + + fmt::print("program to execute {} with args: {} \b", prog, params); + + switch (pid) + { + case -1: fmt::print("Failed to fork(): {} \n", strerror(errno)); return; + case 0: { + // child process + if (stdinFd != STDIN_FILENO) + dup2(stdinFd, STDIN_FILENO); + if (stdoutFd != STDOUT_FILENO) + dup2(stdoutFd, STDOUT_FILENO); + execvp(prog, const_cast(argv.data())); + } + default: { + // parent process + int wstatus = 0; + waitpid(pid, &wstatus, 0); + if (WIFSIGNALED(wstatus)) + fmt::print("child process exited with signal {} \n", WTERMSIG(wstatus)); + else if (WIFEXITED(wstatus)) + fmt::print("child process exited with code {} \n", WEXITSTATUS(wstatus)); + else if (WIFSTOPPED(wstatus)) + fmt::print("child process stopped with signal {} \n", WSTOPSIG(wstatus)); + else + fmt::print("child process exited with unknown status {} \n", wstatus); + break; + } + } +} + +// int jitPart(std::string input) +// { +// using namespace llvm; +// using namespace llvm::orc; + +// std::unique_ptr> builder; + +// // Create an LLJIT instance. +// ExitOnError ExitOnErr; +// auto J = ExitOnErr(TestJIT::Create()); + +// auto Context = std::make_unique(); + +// builder = std::make_unique>(*Context); + +// auto TheModule = std::make_unique("test", *Context); +// TheModule->setDataLayout(J->getDataLayout()); + +// auto byteptr = builder->getPtrTy(); +// // llvm::Type::getInt8Ty(*Context)->getPo; +// auto FunctionType = llvm::FunctionType::get(llvm::Type::getVoidTy(*Context), { byteptr, byteptr }, +// false); + +// auto fun = TheModule->getOrInsertFunction("execute", FunctionType); + +// Function* F = Function::Create(FunctionType, Function::ExternalLinkage, "__anon_expr", +// TheModule.get()); + +// // Add a basic block to the function. As before, it automatically inserts +// // because of the last argument. +// BasicBlock* BB = BasicBlock::Create(*Context, "EntryBlock", F); + +// builder->SetInsertPoint(BB); + +// // Get pointers to the constant `1'. + +// const auto [prog, args] = SeparateProg(input); + +// auto CalRes = +// builder->CreateCall(fun, { builder->CreateGlobalString(prog), builder->CreateGlobalString(args) }); + +// // Value *One = builder.getInt32(1); +// // Value *Add = builder.CreateAdd(One, One); + +// builder->CreateRet(builder->getInt32(1)); + +// auto RT = J->getMainJITDylib().createResourceTracker(); + +// auto M = ThreadSafeModule(std::move(TheModule), std::move(Context)); + +// M.getModuleUnlocked()->print(llvm::outs(), nullptr); + +// ExitOnErr(J->addModule(std::move(M), RT)); + +// // Look up the JIT'd function, cast it to a function pointer, then call it. +// auto ExprSymbol = ExitOnErr(J->lookup("__anon_expr")); + +// void (*FP)() = ExprSymbol.getAddress().toPtr(); +// FP(); + +// return EXIT_SUCCESS; +// } + +std::tuple SeparateProg(const std::string input) +{ + + std::stringstream inStream(input); + std::string prog; + std::string args; + getline(inStream, prog, ' '); + getline(inStream, args, '\n'); + + return { prog, args }; +} + +export class LLVMBackend +{ + public: + LLVMBackend() + { + InitializeNativeTarget(); + InitializeNativeTargetAsmPrinter(); + // Create an LLJIT instance. + } + + auto exec(std::string const& input, const int stdinFd, const int stdoutFd) + { + + using namespace llvm; + using namespace llvm::orc; + std::unique_ptr> builder; + + // Create an LLJIT instance. + ExitOnError ExitOnErr; + auto J = ExitOnErr(TestJIT::Create()); + + auto Context = std::make_unique(); + + builder = std::make_unique>(*Context); + + auto TheModule = std::make_unique("test", *Context); + TheModule->setDataLayout(J->getDataLayout()); + + // Create new pass and analysis managers. + auto TheFPM = std::make_unique(); + auto TheLAM = std::make_unique(); + auto TheFAM = std::make_unique(); + auto TheCGAM = std::make_unique(); + auto TheMAM = std::make_unique(); + auto ThePIC = std::make_unique(); + auto TheSI = std::make_unique(*Context, /*DebugLogging*/ true); + TheSI->registerCallbacks(*ThePIC, TheMAM.get()); + + TheFPM->addPass(ReassociatePass()); + TheFPM->addPass(GVNPass()); + TheFPM->addPass(SimplifyCFGPass()); + + PassBuilder PB; + PB.registerModuleAnalyses(*TheMAM); + PB.registerFunctionAnalyses(*TheFAM); + PB.crossRegisterProxies(*TheLAM, *TheFAM, *TheCGAM, *TheMAM); + + auto byteptr = builder->getPtrTy(); + auto inttype = builder->getInt64Ty(); + // llvm::Type::getInt8Ty(*Context)->getPo; + auto FunctionType = llvm::FunctionType::get( + llvm::Type::getVoidTy(*Context), { byteptr, byteptr, inttype, inttype }, false); + + auto fun = TheModule->getOrInsertFunction("execute_llvmbackend", FunctionType); + + Function* F = + Function::Create(FunctionType, Function::ExternalLinkage, "__anon_expr", TheModule.get()); + + // Add a basic block to the function. As before, it automatically inserts + // because of the last argument. + BasicBlock* BB = BasicBlock::Create(*Context, "EntryBlock", F); + + builder->SetInsertPoint(BB); + + // Get pointers to the constant `1'. + + const auto [prog, args] = SeparateProg(input); + + auto CalRes = builder->CreateCall(fun, + { builder->CreateGlobalString(prog), + builder->CreateGlobalString(args), + builder->getInt64(stdinFd), + builder->getInt64(stdoutFd) }); + + // Value *One = builder.getInt32(1); + // Value *Add = builder.CreateAdd(One, One); + + builder->CreateRet(builder->getInt32(1)); + + auto RT = J->getMainJITDylib().createResourceTracker(); + + auto M = ThreadSafeModule(std::move(TheModule), std::move(Context)); + + M.getModuleUnlocked()->print(llvm::outs(), nullptr); + + ExitOnErr(J->addModule(std::move(M), RT)); + + // Look up the JIT'd function, cast it to a function pointer, then call it. + auto ExprSymbol = ExitOnErr(J->lookup("__anon_expr")); + + (ExprSymbol.getAddress().toPtr())(); + } +}; diff --git a/src/shell/Shell.cpp b/src/shell/Shell.cpp index 628ff89..2ab9f17 100644 --- a/src/shell/Shell.cpp +++ b/src/shell/Shell.cpp @@ -37,7 +37,8 @@ using std::placeholders::_1; using std::placeholders::_2; using std::placeholders::_3; -std::vector constructArgv(CoreVM::CoreStringArray const& args) +template +std::vector constructArgv(T const& args) { std::vector argv; argv.reserve(args.size() + 1); @@ -171,37 +172,21 @@ export class SystemEnvironment: public Environment std::map _values; }; -export class Shell final: public CoreVM::Runtime +class ShellBase { public: - Shell(): Shell(RealTTY::instance(), SystemEnvironment::instance()) + ShellBase() = delete; + ShellBase(TTY& tty, Environment& env): _env { env }, _tty { tty } { -#if defined(ENDO_USE_LLVM) - LLVMBackend::init(); -#endif - } - - Shell(TTY& tty, Environment& env): _env { env }, _tty { tty } - { _currentPipelineBuilder.defaultStdinFd = _tty.inputFd(); _currentPipelineBuilder.defaultStdoutFd = _tty.outputFd(); _env.setAndExport("SHELL", "endo"); - - // NB: These lines could go away once we have a proper command line parser and - // the ability to set these options from the command line. - registerBuiltinFunctions(); - - // for (CoreVM::NativeCallback const* callback: builtins()) - // fmt::print("builtin: {}\n", callback->signature().to_s()); } + virtual ~ShellBase() = default; - [[nodiscard]] Environment& environment() noexcept { return _env; } - [[nodiscard]] Environment const& environment() const noexcept { return _env; } - - void setOptimize(bool optimize) { _optimize = optimize; } - + virtual int execute(std::string const& lineBuffer) = 0; int run() { while (!_quit && prompt.ready()) @@ -216,448 +201,515 @@ export class Shell final: public CoreVM::Runtime return _quit ? _exitCode : EXIT_SUCCESS; } - int execute(std::string const& lineBuffer) + [[nodiscard]] Environment& environment() noexcept { return _env; } + [[nodiscard]] Environment const& environment() const noexcept { return _env; } + + void setOptimize(bool optimize) { _optimize = optimize; } + + template + void error(fmt::format_string const& message, Args&&... args) { -#if defined(ENDO_USE_LLVM) - return LLVMBackend::executeLLVM(lineBuffer); -#else - return executeCoreVM(lineBuffer); -#endif + std::cerr << fmt::format(message, std::forward(args)...) + "\n"; } - int executeCoreVM(std::string const& lineBuffer) + + [[nodiscard]] std::optional resolveProgram(std::string const& program) const { - try - { - CoreVM::diagnostics::ConsoleReport report; - auto parser = endo::Parser(*this, report, std::make_unique(lineBuffer)); - auto const rootNode = parser.parse(); - if (!rootNode) - { - error("Failed to parse input"); - return EXIT_FAILURE; - } + auto const pathEnv = _env.get("PATH"); + if (!pathEnv.has_value()) + return std::nullopt; - debugLog()("Parsed & printed: {}", endo::ast::ASTPrinter::print(*rootNode)); + auto const pathEnvValue = pathEnv.value(); + auto const paths = crispy::split(pathEnvValue, ':'); - CoreVM::IRProgram* irProgram = IRGenerator::generate(*rootNode); - if (irProgram == nullptr) + for (auto const& pathStr: paths) + { + auto path = std::filesystem::path(pathStr); + auto const programPath = path / program; + if (std::filesystem::exists(programPath)) { - error("Failed to generate IR program"); - return EXIT_FAILURE; + debugLog()("Found program: {}", programPath.string()); + return programPath; } + } - if (_optimize) - { - CoreVM::PassManager pm; - - // clang-format off - pm.registerPass("eliminate-empty-blocks", &CoreVM::transform::emptyBlockElimination); - pm.registerPass("eliminate-linear-br", &CoreVM::transform::eliminateLinearBr); - pm.registerPass("eliminate-unused-blocks", &CoreVM::transform::eliminateUnusedBlocks); - pm.registerPass("eliminate-unused-instr", &CoreVM::transform::eliminateUnusedInstr); - pm.registerPass("fold-constant-condbr", &CoreVM::transform::foldConstantCondBr); - pm.registerPass("rewrite-br-to-exit", &CoreVM::transform::rewriteBrToExit); - pm.registerPass("rewrite-cond-br-to-same-branches", &CoreVM::transform::rewriteCondBrToSameBranches); - // clang-format on + return std::nullopt; + } - pm.run(irProgram); - } + void builtinCallProcess(std::string const& program, std::vector const& args) + { + std::vector const argv = constructArgv(args); + std::optional const programPath = resolveProgram(program); - debugLog()("================================================\n"); - debugLog()("Optimized IR program:\n"); - if (debugLog.is_enabled()) - irProgram->dump(); + int const stdinFd = _currentPipelineBuilder.defaultStdinFd; + int const stdoutFd = _currentPipelineBuilder.defaultStdoutFd; - _currentProgram = CoreVM::TargetCodeGenerator {}.generate(irProgram); - if (!_currentProgram) - { - error("Failed to generate target code"); - return EXIT_FAILURE; - } - _currentProgram->link(this, &report); - - debugLog()("================================================\n"); - debugLog()("Linked target code:\n"); - if (debugLog.is_enabled()) - _currentProgram->dump(); - - CoreVM::Handler* main = _currentProgram->findHandler("@main"); - assert(main != nullptr); - auto runner = - CoreVM::Runner(main, nullptr, &_globals, std::bind(&Shell::trace, this, _1, _2, _3)); - _runner = &runner; - runner.run(); - return _exitCode; - } - catch (std::exception const& e) + if (!programPath.has_value()) { - error("Exception caught: {}", e.what()); - return EXIT_FAILURE; + error("Failed to resolve program '{}'", program); + return; } - return EXIT_SUCCESS; + // TODO: setup redirects + // CoreVM::CoreIntArray const& outputRedirects = context.getIntArray(1); + // for (size_t i = 0; i + 2 < outputRedirects.size(); i += 2) + // debugLog()("redirect: {} -> {}\n", outputRedirects[i], outputRedirects[i + 1]); + + pid_t const pid = fork(); + switch (pid) + { + case -1: error("Failed to fork(): {}", strerror(errno)); return; + case 0: { + // child process + if (stdinFd != STDIN_FILENO) + dup2(stdinFd, STDIN_FILENO); + if (stdoutFd != STDOUT_FILENO) + dup2(stdoutFd, STDOUT_FILENO); + execvp(programPath->c_str(), const_cast(argv.data())); + error("Failed to execve({}): {}", programPath->string(), strerror(errno)); + _exit(EXIT_FAILURE); + } + default: + // parent process + int wstatus = 0; + waitpid(pid, &wstatus, 0); + if (WIFSIGNALED(wstatus)) + error("child process exited with signal {}", WTERMSIG(wstatus)); + else if (WIFEXITED(wstatus)) + error("child process exited with code {}", _exitCode = WEXITSTATUS(wstatus)); + else if (WIFSTOPPED(wstatus)) + error("child process stopped with signal {}", WSTOPSIG(wstatus)); + else + error("child process exited with unknown status {}", wstatus); + break; + } } + public: Prompt prompt; std::vector processGroups; + protected: + Environment& _env; + + TTY& _tty; + + PipelineBuilder _currentPipelineBuilder; + + // This stores the PIDs of all processes in the pipeline's process group. + std::vector _currentProcessGroupPids; + std::optional _leftPid; + std::optional _rightPid; + + // This stores the exit code of the last process in the pipeline. + // TODO: remember exit codes from all processes in the pipeline's process group + int _exitCode = -1; + + bool _optimize = false; + + bool _quit = false; +}; + +export class ShellLLVM final: public ShellBase +{ + public: + ShellLLVM(): ShellLLVM(RealTTY::instance(), SystemEnvironment::instance()) {} + + ShellLLVM(TTY& tty, Environment& env): ShellBase { tty, env }, _backend {} {} + + int execute(std::string const& lineBuffer) override; + private: - void registerBuiltinFunctions() + LLVMBackend _backend; +}; + +export class ShellCoreVM final: public CoreVM::Runtime, public ShellBase +{ + public: + ShellCoreVM(): ShellCoreVM(RealTTY::instance(), SystemEnvironment::instance()) {} + + ShellCoreVM(TTY& tty, Environment& env): ShellBase { tty, env } { - // clang-format off + // NB: These lines could go away once we have a proper command line parser and + // the ability to set these options from the command line. + registerBuiltinFunctions(); + + // for (CoreVM::NativeCallback const* callback: builtins()) + // fmt::print("builtin: {}\n", callback->signature().to_s()); + } + + int execute(std::string const& lineBuffer) override; + + void trace(CoreVM::Instruction instr, size_t ip, size_t sp) + { + debugLog()( + fmt::format("trace: {}\n", CoreVM::disassemble(instr, ip, sp, &_currentProgram->constants()))); + } + + private: + void registerBuiltinFunctions(); + // builtins that match to shell commands + void builtinExit(CoreVM::Params& context); + void builtinCallProcess(CoreVM::Params& context); + void builtinCallProcessShellPiped(CoreVM::Params& context); + void builtinChDir(CoreVM::Params& context); + void builtinChDirHome(CoreVM::Params& context); + void builtinSet(CoreVM::Params& context); + void builtinSetAndExport(CoreVM::Params& context); + void builtinExport(CoreVM::Params& context) { _env.exportVariable(context.getString(1)); } + void builtinTrue(CoreVM::Params& context) { context.setResult(true); } + void builtinFalse(CoreVM::Params& context) { context.setResult(false); } + void builtinReadDefault(CoreVM::Params& context); + void builtinRead(CoreVM::Params& context); + // helper-builtins for redirects and pipes + void builtinOpenRead(CoreVM::Params& context); + void builtinOpenWrite(CoreVM::Params& context); + + private: + std::unique_ptr _currentProgram; + CoreVM::Runner::Globals _globals; + + CoreVM::Runner* _runner = nullptr; +}; + +void ShellCoreVM::builtinOpenWrite(CoreVM::Params& context) +{ + std::string const& path = context.getString(1); + int const oflags = static_cast(context.getInt(2)); + int const fd = open(path.data(), oflags ? oflags : (O_WRONLY | O_CREAT | O_TRUNC)); + if (fd == -1) + { + error("Failed to open file '{}': {}", path, strerror(errno)); + context.setResult(CoreVM::CoreNumber(-1)); + return; + } + + context.setResult(CoreVM::CoreNumber(fd)); +} + +void ShellCoreVM::builtinOpenRead(CoreVM::Params& context) +{ + std::string const& path = context.getString(1); + int const fd = open(path.data(), O_RDONLY); + if (fd == -1) + { + error("Failed to open file '{}': {}", path, strerror(errno)); + context.setResult(CoreVM::CoreNumber(-1)); + return; + } + + context.setResult(CoreVM::CoreNumber(fd)); +} + +void ShellCoreVM::builtinRead(CoreVM::Params& context) +{ + CoreVM::CoreStringArray const& args = context.getStringArray(1); + std::string const& variable = args.at(0); + std::string const line = + readLine(_tty, fmt::format("{}read{}>{} ", "\033[1;34m", "\033[37;1m", "\033[m")); + _env.set(variable, line); + context.setResult(line); +} + +void ShellCoreVM::builtinReadDefault(CoreVM::Params& context) +{ + std::string const line = + readLine(_tty, fmt::format("{}read{}>{} ", "\033[1;34m", "\033[37;1m", "\033[m")); + _env.set("REPLY", line); + context.setResult(line); +} + +void ShellCoreVM::builtinSetAndExport(CoreVM::Params& context) +{ + _env.set(context.getString(1), context.getString(2)); + _env.exportVariable(context.getString(1)); +} + +void ShellCoreVM::builtinSet(CoreVM::Params& context) +{ + _env.set(context.getString(1), context.getString(2)); + context.setResult(true); +} + +void ShellCoreVM::builtinExit(CoreVM::Params& context) +{ + _exitCode = static_cast(context.getInt(1)); + _runner->suspend(); + _quit = true; +} + +void ShellCoreVM::builtinChDirHome(CoreVM::Params& context) +{ + auto const path = _env.get("HOME").value_or("/"); + _env.set("OLDPWD", std::filesystem::current_path().string()); + _env.set("PWD", path); + int const result = chdir(path.data()); + if (result != 0) + error("Failed to change directory to '{}'", path); + + context.setResult(result == 0); +} + +void ShellCoreVM::builtinChDir(CoreVM::Params& context) +{ + std::string const& path = context.getString(1); + + _env.set("OLDPWD", _env.get("PWD").value_or("")); + _env.set("PWD", path); + + int const result = chdir(path.data()); + if (result != 0) + error("Failed to change directory to '{}'", path); + + context.setResult(result == 0); +} + +void ShellCoreVM::builtinCallProcessShellPiped(CoreVM::Params& context) +{ + bool const lastInChain = context.getBool(1); + CoreVM::CoreStringArray const& args = context.getStringArray(2); + + std::vector const argv = constructArgv(args); + std::string const& program = args.at(0); + std::optional const programPath = resolveProgram(program); + + if (!programPath.has_value()) + { + error("Failed to resolve program '{}'", program); + context.setResult(CoreVM::CoreNumber(EXIT_FAILURE)); + return; + } + + auto const [stdinFd, stdoutFd] = _currentPipelineBuilder.requestShellPipe(lastInChain); + + // TODO: setup redirects + // CoreVM::CoreIntArray const& outputRedirects = context.getIntArray(1); + // for (size_t i = 0; i + 2 < outputRedirects.size(); i += 2) + // debugLog()("redirect: {} -> {}\n", outputRedirects[i], outputRedirects[i + 1]); + + pid_t const pid = fork(); + switch (pid) + { + case -1: + error("Failed to fork(): {}", strerror(errno)); + context.setResult(CoreVM::CoreNumber(EXIT_FAILURE)); + return; + case 0: { + // child process + setpgid(0, !_currentProcessGroupPids.empty() ? _currentProcessGroupPids.front() : 0); + if (stdinFd != STDIN_FILENO) + dup2(stdinFd, STDIN_FILENO); + if (stdoutFd != STDOUT_FILENO) + dup2(stdoutFd, STDOUT_FILENO); + execvp(programPath->c_str(), const_cast(argv.data())); + error("Failed to execve({}): {}", programPath->string(), strerror(errno)); + _exit(EXIT_FAILURE); + } + default: + // parent process + _leftPid = _rightPid; + _rightPid = pid; + _currentProcessGroupPids.push_back(pid); + if (lastInChain) + { + // This is the last process in the chain, so we need to wait for all + for (pid_t const pid: _currentProcessGroupPids) + { + int wstatus = 0; + waitpid(pid, &wstatus, 0); + if (WIFSIGNALED(wstatus)) + error("child process {}, exited with signal {}", pid, WTERMSIG(wstatus)); + else if (WIFEXITED(wstatus)) + error("child process {} exited with code {}", pid, _exitCode = WEXITSTATUS(wstatus)); + else if (WIFSTOPPED(wstatus)) + error("child process {} stopped with signal {}", pid, WSTOPSIG(wstatus)); + else + error("child process {} exited with unknown status {}", pid, wstatus); + } + _currentProcessGroupPids.clear(); + _leftPid = std::nullopt; + _rightPid = std::nullopt; + } + break; + } + + context.setResult(CoreVM::CoreNumber(_exitCode)); +} + +void ShellCoreVM::builtinCallProcess(CoreVM::Params& context) +{ + CoreVM::CoreStringArray const& args = context.getStringArray(1); + std::string const& program = args.at(0); + ShellBase::builtinCallProcess(program, args); +} + +void ShellCoreVM::registerBuiltinFunctions() +{ + // clang-format off registerFunction("exit") .param("code") .returnType(CoreVM::LiteralType::Void) - .bind(&Shell::builtinExit, this); + .bind(&ShellCoreVM::builtinExit, this); registerFunction("export") .param("name") .returnType(CoreVM::LiteralType::Void) - .bind(&Shell::builtinExport, this); + .bind(&ShellCoreVM::builtinExport, this); registerFunction("export") .param("name") .param("value") .returnType(CoreVM::LiteralType::Void) - .bind(&Shell::builtinSetAndExport, this); + .bind(&ShellCoreVM::builtinSetAndExport, this); registerFunction("true") .returnType(CoreVM::LiteralType::Boolean) - .bind(&Shell::builtinTrue, this); + .bind(&ShellCoreVM::builtinTrue, this); registerFunction("false") .returnType(CoreVM::LiteralType::Boolean) - .bind(&Shell::builtinFalse, this); + .bind(&ShellCoreVM::builtinFalse, this); registerFunction("cd") .returnType(CoreVM::LiteralType::Boolean) - .bind(&Shell::builtinChDirHome, this); + .bind(&ShellCoreVM::builtinChDirHome, this); registerFunction("cd") .param("path") .returnType(CoreVM::LiteralType::Boolean) - .bind(&Shell::builtinChDir, this); + .bind(&ShellCoreVM::builtinChDir, this); registerFunction("set") .param("name") .param("value") .returnType(CoreVM::LiteralType::Boolean) - .bind(&Shell::builtinSet, this); + .bind(&ShellCoreVM::builtinSet, this); registerFunction("callproc") .param>("args") //.param>("redirects") .returnType(CoreVM::LiteralType::Number) - .bind(&Shell::builtinCallProcess, this); + .bind(&ShellCoreVM::builtinCallProcess, this); registerFunction("callproc") .param("last_in_chain") .param>("args") //.param>("redirects") .returnType(CoreVM::LiteralType::Number) - .bind(&Shell::builtinCallProcessShellPiped, this); + .bind(&ShellCoreVM::builtinCallProcessShellPiped, this); registerFunction("read") .returnType(CoreVM::LiteralType::String) - .bind(&Shell::builtinReadDefault, this); + .bind(&ShellCoreVM::builtinReadDefault, this); registerFunction("read") .param>("args") .returnType(CoreVM::LiteralType::String) - .bind(&Shell::builtinRead, this); + .bind(&ShellCoreVM::builtinRead, this); // used to redirect file to stdin registerFunction("internal.open_read") .param("path") .returnType(CoreVM::LiteralType::Number) - .bind(&Shell::builtinOpenRead, this); + .bind(&ShellCoreVM::builtinOpenRead, this); // used for redirecting output to a file registerFunction("internal.open_write") .param("path") .param("oflags") .returnType(CoreVM::LiteralType::Number) - .bind(&Shell::builtinOpenWrite, this); - // clang-format on - } + .bind(&ShellCoreVM::builtinOpenWrite, this); + // clang-format on +} - // builtins that match to shell commands - void builtinExit(CoreVM::Params& context) - { - _exitCode = static_cast(context.getInt(1)); - _runner->suspend(); - _quit = true; - } - void builtinCallProcess(CoreVM::Params& context) +int ShellCoreVM::execute(std::string const& lineBuffer) +{ + try { - CoreVM::CoreStringArray const& args = context.getStringArray(1); - std::vector const argv = constructArgv(args); - std::string const& program = args.at(0); - std::optional const programPath = resolveProgram(program); - - int const stdinFd = _currentPipelineBuilder.defaultStdinFd; - int const stdoutFd = _currentPipelineBuilder.defaultStdoutFd; - - if (!programPath.has_value()) + CoreVM::diagnostics::ConsoleReport report; + auto parser = endo::Parser(*this, report, std::make_unique(lineBuffer)); + auto const rootNode = parser.parse(); + if (!rootNode) { - error("Failed to resolve program '{}'", program); - context.setResult(CoreVM::CoreNumber(EXIT_FAILURE)); - return; + error("Failed to parse input"); + return EXIT_FAILURE; } + debugLog()("Parsed & printed: {}", endo::ast::ASTPrinter::print(*rootNode)); - // TODO: setup redirects - // CoreVM::CoreIntArray const& outputRedirects = context.getIntArray(1); - // for (size_t i = 0; i + 2 < outputRedirects.size(); i += 2) - // debugLog()("redirect: {} -> {}\n", outputRedirects[i], outputRedirects[i + 1]); - - pid_t const pid = fork(); - switch (pid) + CoreVM::IRProgram* irProgram = IRGenerator::generate(*rootNode); + if (irProgram == nullptr) { - case -1: - error("Failed to fork(): {}", strerror(errno)); - context.setResult(CoreVM::CoreNumber(EXIT_FAILURE)); - return; - case 0: { - // child process - if (stdinFd != STDIN_FILENO) - dup2(stdinFd, STDIN_FILENO); - if (stdoutFd != STDOUT_FILENO) - dup2(stdoutFd, STDOUT_FILENO); - execvp(programPath->c_str(), const_cast(argv.data())); - error("Failed to execve({}): {}", programPath->string(), strerror(errno)); - _exit(EXIT_FAILURE); - } - default: - // parent process - int wstatus = 0; - waitpid(pid, &wstatus, 0); - if (WIFSIGNALED(wstatus)) - error("child process exited with signal {}", WTERMSIG(wstatus)); - else if (WIFEXITED(wstatus)) - error("child process exited with code {}", _exitCode = WEXITSTATUS(wstatus)); - else if (WIFSTOPPED(wstatus)) - error("child process stopped with signal {}", WSTOPSIG(wstatus)); - else - error("child process exited with unknown status {}", wstatus); - break; + error("Failed to generate IR program"); + return EXIT_FAILURE; } - context.setResult(CoreVM::CoreNumber(_exitCode)); - } - void builtinCallProcessShellPiped(CoreVM::Params& context) - { - bool const lastInChain = context.getBool(1); - CoreVM::CoreStringArray const& args = context.getStringArray(2); - - std::vector const argv = constructArgv(args); - std::string const& program = args.at(0); - std::optional const programPath = resolveProgram(program); - - if (!programPath.has_value()) + if (_optimize) { - error("Failed to resolve program '{}'", program); - context.setResult(CoreVM::CoreNumber(EXIT_FAILURE)); - return; - } - - auto const [stdinFd, stdoutFd] = _currentPipelineBuilder.requestShellPipe(lastInChain); + CoreVM::PassManager pm; - // TODO: setup redirects - // CoreVM::CoreIntArray const& outputRedirects = context.getIntArray(1); - // for (size_t i = 0; i + 2 < outputRedirects.size(); i += 2) - // debugLog()("redirect: {} -> {}\n", outputRedirects[i], outputRedirects[i + 1]); + // clang-format off + pm.registerPass("eliminate-empty-blocks", &CoreVM::transform::emptyBlockElimination); + pm.registerPass("eliminate-linear-br", &CoreVM::transform::eliminateLinearBr); + pm.registerPass("eliminate-unused-blocks", &CoreVM::transform::eliminateUnusedBlocks); + pm.registerPass("eliminate-unused-instr", &CoreVM::transform::eliminateUnusedInstr); + pm.registerPass("fold-constant-condbr", &CoreVM::transform::foldConstantCondBr); + pm.registerPass("rewrite-br-to-exit", &CoreVM::transform::rewriteBrToExit); + pm.registerPass("rewrite-cond-br-to-same-branches", &CoreVM::transform::rewriteCondBrToSameBranches); + // clang-format on - pid_t const pid = fork(); - switch (pid) - { - case -1: - error("Failed to fork(): {}", strerror(errno)); - context.setResult(CoreVM::CoreNumber(EXIT_FAILURE)); - return; - case 0: { - // child process - setpgid(0, !_currentProcessGroupPids.empty() ? _currentProcessGroupPids.front() : 0); - if (stdinFd != STDIN_FILENO) - dup2(stdinFd, STDIN_FILENO); - if (stdoutFd != STDOUT_FILENO) - dup2(stdoutFd, STDOUT_FILENO); - execvp(programPath->c_str(), const_cast(argv.data())); - error("Failed to execve({}): {}", programPath->string(), strerror(errno)); - _exit(EXIT_FAILURE); - } - default: - // parent process - _leftPid = _rightPid; - _rightPid = pid; - _currentProcessGroupPids.push_back(pid); - if (lastInChain) - { - // This is the last process in the chain, so we need to wait for all - for (pid_t const pid: _currentProcessGroupPids) - { - int wstatus = 0; - waitpid(pid, &wstatus, 0); - if (WIFSIGNALED(wstatus)) - error("child process {}, exited with signal {}", pid, WTERMSIG(wstatus)); - else if (WIFEXITED(wstatus)) - error("child process {} exited with code {}", - pid, - _exitCode = WEXITSTATUS(wstatus)); - else if (WIFSTOPPED(wstatus)) - error("child process {} stopped with signal {}", pid, WSTOPSIG(wstatus)); - else - error("child process {} exited with unknown status {}", pid, wstatus); - } - _currentProcessGroupPids.clear(); - _leftPid = std::nullopt; - _rightPid = std::nullopt; - } - break; + pm.run(irProgram); } - context.setResult(CoreVM::CoreNumber(_exitCode)); - } - - void builtinChDir(CoreVM::Params& context) - { - std::string const& path = context.getString(1); - - _env.set("OLDPWD", _env.get("PWD").value_or("")); - _env.set("PWD", path); - - int const result = chdir(path.data()); - if (result != 0) - error("Failed to change directory to '{}'", path); - - context.setResult(result == 0); - } - void builtinChDirHome(CoreVM::Params& context) - { - auto const path = _env.get("HOME").value_or("/"); - _env.set("OLDPWD", std::filesystem::current_path().string()); - _env.set("PWD", path); - int const result = chdir(path.data()); - if (result != 0) - error("Failed to change directory to '{}'", path); - - context.setResult(result == 0); - } - void builtinSet(CoreVM::Params& context) - { - _env.set(context.getString(1), context.getString(2)); - context.setResult(true); - } - void builtinSetAndExport(CoreVM::Params& context) - { - _env.set(context.getString(1), context.getString(2)); - _env.exportVariable(context.getString(1)); - } - void builtinExport(CoreVM::Params& context) { _env.exportVariable(context.getString(1)); } - void builtinTrue(CoreVM::Params& context) { context.setResult(true); } - void builtinFalse(CoreVM::Params& context) { context.setResult(false); } - void builtinReadDefault(CoreVM::Params& context) - { - std::string const line = - readLine(_tty, fmt::format("{}read{}>{} ", "\033[1;34m", "\033[37;1m", "\033[m")); - _env.set("REPLY", line); - context.setResult(line); - } - void builtinRead(CoreVM::Params& context) - { - CoreVM::CoreStringArray const& args = context.getStringArray(1); - std::string const& variable = args.at(0); - std::string const line = - readLine(_tty, fmt::format("{}read{}>{} ", "\033[1;34m", "\033[37;1m", "\033[m")); - _env.set(variable, line); - context.setResult(line); - } + debugLog()("================================================\n"); + debugLog()("Optimized IR program:\n"); + if (debugLog.is_enabled()) + irProgram->dump(); - // helper-builtins for redirects and pipes - void builtinOpenRead(CoreVM::Params& context) - { - std::string const& path = context.getString(1); - int const fd = open(path.data(), O_RDONLY); - if (fd == -1) + _currentProgram = CoreVM::TargetCodeGenerator {}.generate(irProgram); + if (!_currentProgram) { - error("Failed to open file '{}': {}", path, strerror(errno)); - context.setResult(CoreVM::CoreNumber(-1)); - return; + error("Failed to generate target code"); + return EXIT_FAILURE; } - - context.setResult(CoreVM::CoreNumber(fd)); + _currentProgram->link(this, &report); + + debugLog()("================================================\n"); + debugLog()("Linked target code:\n"); + if (debugLog.is_enabled()) + _currentProgram->dump(); + + CoreVM::Handler* main = _currentProgram->findHandler("@main"); + assert(main != nullptr); + auto runner = + CoreVM::Runner(main, nullptr, &_globals, std::bind(&ShellCoreVM::trace, this, _1, _2, _3)); + _runner = &runner; + runner.run(); + return _exitCode; } - - void builtinOpenWrite(CoreVM::Params& context) + catch (std::exception const& e) { - std::string const& path = context.getString(1); - int const oflags = static_cast(context.getInt(2)); - int const fd = open(path.data(), oflags ? oflags : (O_WRONLY | O_CREAT | O_TRUNC)); - if (fd == -1) - { - error("Failed to open file '{}': {}", path, strerror(errno)); - context.setResult(CoreVM::CoreNumber(-1)); - return; - } - - context.setResult(CoreVM::CoreNumber(fd)); + error("Exception caught: {}", e.what()); + return EXIT_FAILURE; } - [[nodiscard]] std::optional resolveProgram(std::string const& program) const - { - auto const pathEnv = _env.get("PATH"); - if (!pathEnv.has_value()) - return std::nullopt; - auto const pathEnvValue = pathEnv.value(); - auto const paths = crispy::split(pathEnvValue, ':'); - - for (auto const& pathStr: paths) - { - auto path = std::filesystem::path(pathStr); - auto const programPath = path / program; - if (std::filesystem::exists(programPath)) - { - debugLog()("Found program: {}", programPath.string()); - return programPath; - } - } - - return std::nullopt; - } + return EXIT_SUCCESS; +} - void trace(CoreVM::Instruction instr, size_t ip, size_t sp) +int ShellLLVM::execute(std::string const& lineBuffer) +{ + try { - debugLog()( - fmt::format("trace: {}\n", CoreVM::disassemble(instr, ip, sp, &_currentProgram->constants()))); + // Look up the JIT'd function, cast it to a function pointer, then call it. + _backend.exec( + lineBuffer, _currentPipelineBuilder.defaultStdinFd, _currentPipelineBuilder.defaultStdoutFd); + return _exitCode; } - - template - void error(fmt::format_string const& message, Args&&... args) + catch (std::exception const& e) { - std::cerr << fmt::format(message, std::forward(args)...) + "\n"; + error("Exception caught: {}", e.what()); + return EXIT_SUCCESS; } - private: - Environment& _env; - - TTY& _tty; - - std::unique_ptr _currentProgram; - CoreVM::Runner::Globals _globals; - - bool _optimize = false; - - PipelineBuilder _currentPipelineBuilder; - - // This stores the PIDs of all processes in the pipeline's process group. - std::vector _currentProcessGroupPids; - std::optional _leftPid; - std::optional _rightPid; - - // This stores the exit code of the last process in the pipeline. - // TODO: remember exit codes from all processes in the pipeline's process group - int _exitCode = -1; + return EXIT_SUCCESS; +} - CoreVM::Runner* _runner = nullptr; - bool _quit = false; -}; } // namespace endo diff --git a/src/shell/Shell_test.cpp b/src/shell/Shell_test.cpp index fee28ff..2d23746 100644 --- a/src/shell/Shell_test.cpp +++ b/src/shell/Shell_test.cpp @@ -11,6 +11,12 @@ using crispy::escape; import Shell; import TTY; +#if defined(ENDO_USE_LLVM) +using Shell = endo::ShellLLVM; +#else +using Shell = endo::ShellCoreVM; +#endif + namespace { struct TestShell @@ -19,7 +25,7 @@ struct TestShell endo::TestEnvironment env; int exitCode = -1; - endo::Shell shell { pty, env }; + Shell shell { pty, env }; std::string_view output() const noexcept { return pty.output(); } @@ -79,13 +85,13 @@ TEST_CASE("shell.builtin.set_variable") CHECK(shell.env.get("BRU").value_or("NONE") == "hello"); } -TEST_CASE("shell.builtin.get_variable") -{ - TestShell shell; - shell("set BRU hello"); - CHECK(shell.env.get("BRU").value_or("NONE") == "hello"); - shell("$BRU"); -} +// TEST_CASE("shell.builtin.get_variable") +// { +// TestShell shell; +// shell("set BRU hello"); +// CHECK(shell.env.get("BRU").value_or("NONE") == "hello"); +// shell("$BRU"); +// } // TEST_CASE("shell.builtin.set_and_export_variable") diff --git a/src/shell/llvm_executor.cpp b/src/shell/llvm_executor.cpp deleted file mode 100644 index a3b33c5..0000000 --- a/src/shell/llvm_executor.cpp +++ /dev/null @@ -1,285 +0,0 @@ -module; - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace llvm; -using namespace llvm::orc; - -export module LLVMBackend; - -class TestJIT -{ - private: - std::unique_ptr ES; - - DataLayout DL; - MangleAndInterner Mangle; - - RTDyldObjectLinkingLayer ObjectLayer; - IRCompileLayer CompileLayer; - - JITDylib& MainJD; - - public: - TestJIT(std::unique_ptr ES, JITTargetMachineBuilder JTMB, DataLayout DL): - ES(std::move(ES)), - DL(std::move(DL)), - Mangle(*this->ES, this->DL), - ObjectLayer(*this->ES, []() { return std::make_unique(); }), - CompileLayer(*this->ES, ObjectLayer, std::make_unique(std::move(JTMB))), - MainJD(this->ES->createBareJITDylib("
")) - { - MainJD.addGenerator( - cantFail(DynamicLibrarySearchGenerator::GetForCurrentProcess(DL.getGlobalPrefix()))); - if (JTMB.getTargetTriple().isOSBinFormatCOFF()) - { - ObjectLayer.setOverrideObjectFlagsWithResponsibilityFlags(true); - ObjectLayer.setAutoClaimResponsibilityForObjectSymbols(true); - } - } - - ~TestJIT() - { - if (auto Err = ES->endSession()) - ES->reportError(std::move(Err)); - } - - static Expected> Create() - { - auto EPC = SelfExecutorProcessControl::Create(); - if (!EPC) - return EPC.takeError(); - - auto ES = std::make_unique(std::move(*EPC)); - - JITTargetMachineBuilder JTMB(ES->getExecutorProcessControl().getTargetTriple()); - - auto DL = JTMB.getDefaultDataLayoutForTarget(); - if (!DL) - return DL.takeError(); - - return std::make_unique(std::move(ES), std::move(JTMB), std::move(*DL)); - } - - const DataLayout& getDataLayout() const { return DL; } - - JITDylib& getMainJITDylib() { return MainJD; } - - Error addModule(ThreadSafeModule TSM, ResourceTrackerSP RT = nullptr) - { - if (!RT) - RT = MainJD.getDefaultResourceTracker(); - return CompileLayer.add(RT, std::move(TSM)); - } - - Expected lookup(StringRef Name) { return ES->lookup({ &MainJD }, Mangle(Name.str())); } -}; - -std::vector tokenizer(const std::string& p_pcstStr, char delim) -{ - std::vector tokens; - std::stringstream mySstream(p_pcstStr); - std::string temp; - - while (getline(mySstream, temp, delim)) - { - tokens.push_back(temp); - } - - return tokens; -} - -std::tuple SeparateProg(const std::string input) -{ - - std::stringstream inStream(input); - std::string prog; - std::string args; - getline(inStream, prog, ' '); - getline(inStream, args, '\n'); - - return { prog, args }; -} - -extern "C" void execute(const char* prog, const char* params) -{ - std::vector argv; - argv.push_back(prog); - argv.push_back(params); - - pid_t const pid = fork(); - switch (pid) - { - case -1: return; - case 0: { - execvp(prog, const_cast(argv.data())); - } - default: int wstatus = 0; waitpid(pid, &wstatus, 0); - } -} - -int jitPart(std::string input) -{ - using namespace llvm; - using namespace llvm::orc; - - std::unique_ptr> builder; - - // Create an LLJIT instance. - ExitOnError ExitOnErr; - auto J = ExitOnErr(TestJIT::Create()); - - auto Context = std::make_unique(); - - builder = std::make_unique>(*Context); - - auto TheModule = std::make_unique("test", *Context); - TheModule->setDataLayout(J->getDataLayout()); - - // Create new pass and analysis managers. - auto TheFPM = std::make_unique(); - auto TheLAM = std::make_unique(); - auto TheFAM = std::make_unique(); - auto TheCGAM = std::make_unique(); - auto TheMAM = std::make_unique(); - auto ThePIC = std::make_unique(); - auto TheSI = std::make_unique(*Context, /*DebugLogging*/ true); - TheSI->registerCallbacks(*ThePIC, TheMAM.get()); - - // Add transform passes. - // Do simple "peephole" optimizations and bit-twiddling optzns. - TheFPM->addPass(InstCombinePass()); - // Reassociate expressions. - TheFPM->addPass(ReassociatePass()); - // Eliminate Common SubExpressions. - TheFPM->addPass(GVNPass()); - // Simplify the control flow graph (deleting unreachable blocks, etc). - TheFPM->addPass(SimplifyCFGPass()); - - // Register analysis passes used in these transform passes. - PassBuilder PB; - PB.registerModuleAnalyses(*TheMAM); - PB.registerFunctionAnalyses(*TheFAM); - PB.crossRegisterProxies(*TheLAM, *TheFAM, *TheCGAM, *TheMAM); - - auto byteptr = builder->getPtrTy(); - //llvm::Type::getInt8Ty(*Context)->getPo; - auto FunctionType = llvm::FunctionType::get(llvm::Type::getVoidTy(*Context), {byteptr, byteptr}, false); - - auto fun = TheModule->getOrInsertFunction("execute", FunctionType); - - Function* F = Function::Create(FunctionType, Function::ExternalLinkage, "__anon_expr", TheModule.get()); - - // Add a basic block to the function. As before, it automatically inserts - // because of the last argument. - BasicBlock* BB = BasicBlock::Create(*Context, "EntryBlock", F); - - builder->SetInsertPoint(BB); - - // Get pointers to the constant `1'. - - const auto [prog, args] = SeparateProg(input); - - auto CalRes = builder->CreateCall(fun,{builder->CreateGlobalString(prog),builder->CreateGlobalString(args)}); - - // Value *One = builder.getInt32(1); - // Value *Add = builder.CreateAdd(One, One); - - builder->CreateRet(builder->getInt32(1)); - - auto RT = J->getMainJITDylib().createResourceTracker(); - - auto M = ThreadSafeModule(std::move(TheModule), std::move(Context)); - - M.getModuleUnlocked()->print(llvm::outs(), nullptr); - - ExitOnErr(J->addModule(std::move(M), RT)); - - // Look up the JIT'd function, cast it to a function pointer, then call it. - auto ExprSymbol = ExitOnErr(J->lookup("__anon_expr")); - - void (*FP)() = ExprSymbol.getAddress().toPtr(); - FP(); - - return EXIT_SUCCESS; -} - -export namespace LLVMBackend -{ - -int executeLLVM(std::string const& lineBuffer) -{ - try - { - return jitPart(lineBuffer); - } - catch (std::exception const& e) - { - std::cout << "Exception caught: " << e.what(); - return EXIT_FAILURE; - } -} - -void init() -{ - InitializeNativeTarget(); - InitializeNativeTargetAsmPrinter(); -} - -} // namespace llvm_backend diff --git a/src/shell/main.cpp b/src/shell/main.cpp index 2b8eba5..f05cad3 100644 --- a/src/shell/main.cpp +++ b/src/shell/main.cpp @@ -5,6 +5,12 @@ using namespace std::string_literals; import Shell; +#if defined(ENDO_USE_LLVM) +using Shell = endo::ShellLLVM; +#else +using Shell = endo::ShellCoreVM; +#endif + std::string_view getEnvironment(std::string_view name, std::string_view defaultValue) { @@ -14,7 +20,7 @@ std::string_view getEnvironment(std::string_view name, std::string_view defaultV int main(int argc, char const* argv[]) { - auto shell = endo::Shell {}; + auto shell = Shell {}; setsid(); From 83e24a47a753d4338abcb2fb20594f43b9852421 Mon Sep 17 00:00:00 2001 From: Yaraslau Tamashevich Date: Mon, 5 Feb 2024 15:34:31 +0200 Subject: [PATCH 5/6] llvm backend part 2 --- src/shell/CMakeLists.txt | 1 + src/shell/LLVMBackend.cpp | 221 ++++++++++++++------------------------ src/shell/Shell.cpp | 63 ++++++++--- src/shell/main.cpp | 8 +- 4 files changed, 136 insertions(+), 157 deletions(-) diff --git a/src/shell/CMakeLists.txt b/src/shell/CMakeLists.txt index 89dd921..c1af8eb 100644 --- a/src/shell/CMakeLists.txt +++ b/src/shell/CMakeLists.txt @@ -23,6 +23,7 @@ find_package(Threads REQUIRED) # see https://github.com/llvm/llvm-project/issues/75108 # and https://github.com/ericniebler/range-v3/blob/f013aef2ae81f3661a560e7922a968665bedebff/include/meta/meta_fwd.hpp#L37 target_compile_definitions(Shell PUBLIC META_WORKAROUND_LLVM_28385) +target_compile_definitions(Shell PUBLIC ENDO_VERSION_STRING="0.0.1") set(shell_libs fmt::fmt-header-only vtparser diff --git a/src/shell/LLVMBackend.cpp b/src/shell/LLVMBackend.cpp index 8f36589..0a30b17 100644 --- a/src/shell/LLVMBackend.cpp +++ b/src/shell/LLVMBackend.cpp @@ -1,11 +1,16 @@ module; +#include + +#include + #include #include #include #include #include #include +#include #include #include #include @@ -13,7 +18,7 @@ module; #include #include #include -#include + #include #include @@ -58,11 +63,15 @@ module; #include #include +using std::unique_ptr; + using namespace llvm; using namespace llvm::orc; export module LLVMBackend; +auto inline llvmLog = logstore::category("llvm", "Debug log", logstore::category::state::Enabled); + class TestJIT { private: @@ -145,20 +154,21 @@ std::vector tokenizer(const std::string& p_pcstStr, char delim) return tokens; } -extern "C" void execute_llvmbackend(const char* prog, const char* params, int stdinFd, int stdoutFd) +extern "C" void execute(const char* prog, const char* params, int stdinFd, int stdoutFd) { - std::vector argv; argv.push_back(prog); - argv.push_back(params); + if (strlen(params) > 0) + argv.push_back(params); + argv.push_back(nullptr); + + fmt::print("program to execute {} with args: {} \n ", prog, params); pid_t const pid = fork(); - fmt::print("program to execute {} with args: {} \b", prog, params); - switch (pid) { - case -1: fmt::print("Failed to fork(): {} \n", strerror(errno)); return; + case -1: llvmLog()("Failed to fork(): {} \n", strerror(errno)); return; case 0: { // child process if (stdinFd != STDIN_FILENO) @@ -170,185 +180,120 @@ extern "C" void execute_llvmbackend(const char* prog, const char* params, int st default: { // parent process int wstatus = 0; - waitpid(pid, &wstatus, 0); - if (WIFSIGNALED(wstatus)) - fmt::print("child process exited with signal {} \n", WTERMSIG(wstatus)); - else if (WIFEXITED(wstatus)) - fmt::print("child process exited with code {} \n", WEXITSTATUS(wstatus)); - else if (WIFSTOPPED(wstatus)) - fmt::print("child process stopped with signal {} \n", WSTOPSIG(wstatus)); - else - fmt::print("child process exited with unknown status {} \n", wstatus); - break; + waitpid(pid, &wstatus, 0); + if (WIFSIGNALED(wstatus)) + llvmLog()("child process exited with signal {} \n", WTERMSIG(wstatus)); + else if (WIFEXITED(wstatus)) + llvmLog()("child process exited with code {} \n", WEXITSTATUS(wstatus)); + else if (WIFSTOPPED(wstatus)) + llvmLog()("child process stopped with signal {} \n", WSTOPSIG(wstatus)); + else + llvmLog()("child process exited with unknown status {} \n", wstatus); + break; } } } -// int jitPart(std::string input) -// { -// using namespace llvm; -// using namespace llvm::orc; - -// std::unique_ptr> builder; - -// // Create an LLJIT instance. -// ExitOnError ExitOnErr; -// auto J = ExitOnErr(TestJIT::Create()); - -// auto Context = std::make_unique(); - -// builder = std::make_unique>(*Context); - -// auto TheModule = std::make_unique("test", *Context); -// TheModule->setDataLayout(J->getDataLayout()); - -// auto byteptr = builder->getPtrTy(); -// // llvm::Type::getInt8Ty(*Context)->getPo; -// auto FunctionType = llvm::FunctionType::get(llvm::Type::getVoidTy(*Context), { byteptr, byteptr }, -// false); - -// auto fun = TheModule->getOrInsertFunction("execute", FunctionType); - -// Function* F = Function::Create(FunctionType, Function::ExternalLinkage, "__anon_expr", -// TheModule.get()); - -// // Add a basic block to the function. As before, it automatically inserts -// // because of the last argument. -// BasicBlock* BB = BasicBlock::Create(*Context, "EntryBlock", F); - -// builder->SetInsertPoint(BB); - -// // Get pointers to the constant `1'. - -// const auto [prog, args] = SeparateProg(input); - -// auto CalRes = -// builder->CreateCall(fun, { builder->CreateGlobalString(prog), builder->CreateGlobalString(args) }); - -// // Value *One = builder.getInt32(1); -// // Value *Add = builder.CreateAdd(One, One); - -// builder->CreateRet(builder->getInt32(1)); - -// auto RT = J->getMainJITDylib().createResourceTracker(); - -// auto M = ThreadSafeModule(std::move(TheModule), std::move(Context)); - -// M.getModuleUnlocked()->print(llvm::outs(), nullptr); - -// ExitOnErr(J->addModule(std::move(M), RT)); - -// // Look up the JIT'd function, cast it to a function pointer, then call it. -// auto ExprSymbol = ExitOnErr(J->lookup("__anon_expr")); - -// void (*FP)() = ExprSymbol.getAddress().toPtr(); -// FP(); - -// return EXIT_SUCCESS; -// } - std::tuple SeparateProg(const std::string input) { std::stringstream inStream(input); - std::string prog; + std::string program; std::string args; - getline(inStream, prog, ' '); + getline(inStream, program, ' '); getline(inStream, args, '\n'); - return { prog, args }; + return { "/usr/bin/" + program, args }; } export class LLVMBackend { + public: LLVMBackend() { InitializeNativeTarget(); InitializeNativeTargetAsmPrinter(); + InitializeNativeTargetAsmParser(); // Create an LLJIT instance. + _jit = ExitOnErr(TestJIT::Create()); } - auto exec(std::string const& input, const int stdinFd, const int stdoutFd) + void initializeModule() { + _context = std::make_unique(); + _builder = std::make_unique>(*_context); - using namespace llvm; - using namespace llvm::orc; - std::unique_ptr> builder; - - // Create an LLJIT instance. - ExitOnError ExitOnErr; - auto J = ExitOnErr(TestJIT::Create()); - - auto Context = std::make_unique(); - - builder = std::make_unique>(*Context); - - auto TheModule = std::make_unique("test", *Context); - TheModule->setDataLayout(J->getDataLayout()); - - // Create new pass and analysis managers. - auto TheFPM = std::make_unique(); - auto TheLAM = std::make_unique(); - auto TheFAM = std::make_unique(); - auto TheCGAM = std::make_unique(); - auto TheMAM = std::make_unique(); - auto ThePIC = std::make_unique(); - auto TheSI = std::make_unique(*Context, /*DebugLogging*/ true); - TheSI->registerCallbacks(*ThePIC, TheMAM.get()); - - TheFPM->addPass(ReassociatePass()); - TheFPM->addPass(GVNPass()); - TheFPM->addPass(SimplifyCFGPass()); + _module = std::make_unique("test", *_context); + _module->setDataLayout(_jit->getDataLayout()); + } - PassBuilder PB; - PB.registerModuleAnalyses(*TheMAM); - PB.registerFunctionAnalyses(*TheFAM); - PB.crossRegisterProxies(*TheLAM, *TheFAM, *TheCGAM, *TheMAM); + void HandleExternalCall(std::string const& input, const int stdinFd, const int stdoutFd) + { - auto byteptr = builder->getPtrTy(); - auto inttype = builder->getInt64Ty(); + auto byteptr = _builder->getPtrTy(); + auto inttype = _builder->getInt64Ty(); // llvm::Type::getInt8Ty(*Context)->getPo; - auto FunctionType = llvm::FunctionType::get( - llvm::Type::getVoidTy(*Context), { byteptr, byteptr, inttype, inttype }, false); + _execvpFunctionType = llvm::FunctionType::get( + llvm::Type::getVoidTy(*_context), { byteptr, byteptr, inttype, inttype }, false); - auto fun = TheModule->getOrInsertFunction("execute_llvmbackend", FunctionType); + _mainFunctionType = llvm::FunctionType::get( + llvm::Type::getVoidTy(*_context), { llvm::Type::getVoidTy(*_context) }, false); + + auto fun = _module->getOrInsertFunction("execute", _execvpFunctionType); Function* F = - Function::Create(FunctionType, Function::ExternalLinkage, "__anon_expr", TheModule.get()); + Function::Create(_mainFunctionType, Function::ExternalLinkage, "__anon_expr", _module.get()); // Add a basic block to the function. As before, it automatically inserts // because of the last argument. - BasicBlock* BB = BasicBlock::Create(*Context, "EntryBlock", F); + BasicBlock* BB = BasicBlock::Create(*_context, "EntryBlock", F); - builder->SetInsertPoint(BB); + _builder->SetInsertPoint(BB); // Get pointers to the constant `1'. const auto [prog, args] = SeparateProg(input); - auto CalRes = builder->CreateCall(fun, - { builder->CreateGlobalString(prog), - builder->CreateGlobalString(args), - builder->getInt64(stdinFd), - builder->getInt64(stdoutFd) }); + auto CalRes = _builder->CreateCall(fun, + { _builder->CreateGlobalString(prog), + _builder->CreateGlobalString(args), + _builder->getInt64(stdinFd), + _builder->getInt64(stdoutFd) }); - // Value *One = builder.getInt32(1); - // Value *Add = builder.CreateAdd(One, One); + _builder->CreateRet(_builder->getInt32(1)); + } - builder->CreateRet(builder->getInt32(1)); + auto exec(std::string const& input, const int stdinFd, const int stdoutFd) + { + initializeModule(); - auto RT = J->getMainJITDylib().createResourceTracker(); + HandleExternalCall(input, stdinFd, stdoutFd); - auto M = ThreadSafeModule(std::move(TheModule), std::move(Context)); + auto RT = _jit->getMainJITDylib().createResourceTracker(); - M.getModuleUnlocked()->print(llvm::outs(), nullptr); + auto M = ThreadSafeModule(std::move(_module), std::move(_context)); - ExitOnErr(J->addModule(std::move(M), RT)); + M.getModuleUnlocked()->print(llvm::outs(), nullptr); + ExitOnErr(_jit->addModule(std::move(M), RT)); // Look up the JIT'd function, cast it to a function pointer, then call it. - auto ExprSymbol = ExitOnErr(J->lookup("__anon_expr")); - + auto ExprSymbol = ExitOnErr(_jit->lookup("__anon_expr")); (ExprSymbol.getAddress().toPtr())(); + + // Delete the anonymous expression module from the JIT. + ExitOnErr(RT->remove()); } + + std::unique_ptr _context; + std::unique_ptr> _builder; + std::unique_ptr _jit; + std::unique_ptr _module; + ExitOnError ExitOnErr; + + // types of some functions + llvm::FunctionType* _execvpFunctionType; + llvm::FunctionType* _mainFunctionType; + + ~LLVMBackend() { llvm::llvm_shutdown(); } }; diff --git a/src/shell/Shell.cpp b/src/shell/Shell.cpp index 2ab9f17..cbaf9dc 100644 --- a/src/shell/Shell.cpp +++ b/src/shell/Shell.cpp @@ -2,6 +2,7 @@ module; #include +#include #include #include @@ -172,11 +173,17 @@ export class SystemEnvironment: public Environment std::map _values; }; -class ShellBase +class ShellBase: public crispy::app { public: - ShellBase() = delete; - ShellBase(TTY& tty, Environment& env): _env { env }, _tty { tty } + ShellBase(): + app("endo", "Endo shell", ENDO_VERSION_STRING, "Apache-2.0"), + _env { SystemEnvironment::instance() }, + _tty { RealTTY::instance() } + { + } + ShellBase(TTY& tty, Environment& env): + app("endo", "Endo shell", ENDO_VERSION_STRING, "Apache-2.0"), _env { env }, _tty { tty } { _currentPipelineBuilder.defaultStdinFd = _tty.inputFd(); @@ -185,22 +192,42 @@ class ShellBase _env.setAndExport("SHELL", "endo"); } virtual ~ShellBase() = default; + [[nodiscard]] crispy::cli::command parameterDefinition() const override + { + + return crispy::cli::command { + "endo", + "Endo Terminal " ENDO_VERSION_STRING + " - a shell for the 21st century", + crispy::cli::option_list {}, + crispy::cli::command_list {}, + }; + } virtual int execute(std::string const& lineBuffer) = 0; - int run() + int run(int argc, char const* argv[]) { - while (!_quit && prompt.ready()) + try { - auto const lineBuffer = prompt.read(); - debugLog()("input buffer: {}", lineBuffer); + customizeLogStoreOutput(); - _exitCode = execute(lineBuffer); - // _tty.writeToStdout("exit code: {}\n", _exitCode); - } + while (!_quit && prompt.ready()) + { + auto const lineBuffer = prompt.read(); + debugLog()("input buffer: {}", lineBuffer); - return _quit ? _exitCode : EXIT_SUCCESS; - } + _exitCode = execute(lineBuffer); + // _tty.writeToStdout("exit code: {}\n", _exitCode); + } + return _quit ? _exitCode : EXIT_SUCCESS; + } + catch (std::exception const& e) + { + std::cerr << fmt::format("Unhandled error caught. {}", e.what()) << '\n'; + return EXIT_FAILURE; + } + } [[nodiscard]] Environment& environment() noexcept { return _env; } [[nodiscard]] Environment const& environment() const noexcept { return _env; } @@ -698,6 +725,18 @@ int ShellLLVM::execute(std::string const& lineBuffer) { try { + auto tokens = Lexer::tokenize(std::make_unique(lineBuffer)); + for (auto const& tokenInfo: tokens) + { + if (tokenInfo.token == Token::Identifier) + if (tokenInfo.literal == "exit") + { + _quit = true; + return EXIT_SUCCESS; + } + debugLog()("token: {}\n", tokenInfo.literal); + } + // Look up the JIT'd function, cast it to a function pointer, then call it. _backend.exec( lineBuffer, _currentPipelineBuilder.defaultStdinFd, _currentPipelineBuilder.defaultStdoutFd); diff --git a/src/shell/main.cpp b/src/shell/main.cpp index f05cad3..2cd4d5a 100644 --- a/src/shell/main.cpp +++ b/src/shell/main.cpp @@ -11,7 +11,6 @@ using Shell = endo::ShellLLVM; using Shell = endo::ShellCoreVM; #endif - std::string_view getEnvironment(std::string_view name, std::string_view defaultValue) { auto const* const value = getenv(name.data()); @@ -24,10 +23,5 @@ int main(int argc, char const* argv[]) setsid(); - - if (argc == 2) - // This here only exists for early-development debugging purposes. - return shell.execute(argv[1]); - else - return shell.run(); + return shell.run(argc, argv); } From 668390671ab149a4596cb5a030e8a2f55cecb749 Mon Sep 17 00:00:00 2001 From: Yaraslau Tamashevich Date: Tue, 6 Feb 2024 18:49:19 +0200 Subject: [PATCH 6/6] llvm backend part 3 --- src/shell/LLVMBackend.cpp | 56 +++++++++++++++++++++------------------ src/shell/Shell.cpp | 14 +++------- 2 files changed, 34 insertions(+), 36 deletions(-) diff --git a/src/shell/LLVMBackend.cpp b/src/shell/LLVMBackend.cpp index 0a30b17..46339da 100644 --- a/src/shell/LLVMBackend.cpp +++ b/src/shell/LLVMBackend.cpp @@ -223,52 +223,58 @@ export class LLVMBackend { _context = std::make_unique(); _builder = std::make_unique>(*_context); - _module = std::make_unique("test", *_context); _module->setDataLayout(_jit->getDataLayout()); } - void HandleExternalCall(std::string const& input, const int stdinFd, const int stdoutFd) + void HandleExternalCall(std::string const& input) const { - auto byteptr = _builder->getPtrTy(); - auto inttype = _builder->getInt64Ty(); - // llvm::Type::getInt8Ty(*Context)->getPo; - _execvpFunctionType = llvm::FunctionType::get( + auto* byteptr = _builder->getPtrTy(); + auto* inttype = _builder->getInt64Ty(); + auto* execvpFunctionType = llvm::FunctionType::get( llvm::Type::getVoidTy(*_context), { byteptr, byteptr, inttype, inttype }, false); - _mainFunctionType = llvm::FunctionType::get( - llvm::Type::getVoidTy(*_context), { llvm::Type::getVoidTy(*_context) }, false); + auto fun = _module->getOrInsertFunction("execute", execvpFunctionType); - auto fun = _module->getOrInsertFunction("execute", _execvpFunctionType); + const auto [prog, args] = SeparateProg(input); + + auto* CalRes = _builder->CreateCall(fun, + { _builder->CreateGlobalString(prog), + _builder->CreateGlobalString(args), + _builder->getInt64(_stdinFd), + _builder->getInt64(_stdoutFd) }); + } + + void HandleAfterParse(std::string const& input) const { + HandleExternalCall(input); + } + void HandleGlobalPart(std::string const& input) const + { + auto* mainFunctionType = llvm::FunctionType::get( + llvm::Type::getVoidTy(*_context), { llvm::Type::getVoidTy(*_context) }, false); Function* F = - Function::Create(_mainFunctionType, Function::ExternalLinkage, "__anon_expr", _module.get()); + Function::Create(mainFunctionType, Function::ExternalLinkage, "__anon_expr", _module.get()); - // Add a basic block to the function. As before, it automatically inserts - // because of the last argument. BasicBlock* BB = BasicBlock::Create(*_context, "EntryBlock", F); _builder->SetInsertPoint(BB); - // Get pointers to the constant `1'. - - const auto [prog, args] = SeparateProg(input); - - auto CalRes = _builder->CreateCall(fun, - { _builder->CreateGlobalString(prog), - _builder->CreateGlobalString(args), - _builder->getInt64(stdinFd), - _builder->getInt64(stdoutFd) }); + HandleAfterParse(input); _builder->CreateRet(_builder->getInt32(1)); } auto exec(std::string const& input, const int stdinFd, const int stdoutFd) { + // TODO move it somewhere + _stdinFd = stdinFd; + _stdoutFd = stdoutFd; + initializeModule(); - HandleExternalCall(input, stdinFd, stdoutFd); + HandleGlobalPart(input); auto RT = _jit->getMainJITDylib().createResourceTracker(); @@ -291,9 +297,7 @@ export class LLVMBackend std::unique_ptr _module; ExitOnError ExitOnErr; - // types of some functions - llvm::FunctionType* _execvpFunctionType; - llvm::FunctionType* _mainFunctionType; - + int _stdinFd{}; + int _stdoutFd{}; ~LLVMBackend() { llvm::llvm_shutdown(); } }; diff --git a/src/shell/Shell.cpp b/src/shell/Shell.cpp index cbaf9dc..cfd7437 100644 --- a/src/shell/Shell.cpp +++ b/src/shell/Shell.cpp @@ -197,8 +197,7 @@ class ShellBase: public crispy::app return crispy::cli::command { "endo", - "Endo Terminal " ENDO_VERSION_STRING - " - a shell for the 21st century", + "Endo Terminal " ENDO_VERSION_STRING , crispy::cli::option_list {}, crispy::cli::command_list {}, }; @@ -726,15 +725,10 @@ int ShellLLVM::execute(std::string const& lineBuffer) try { auto tokens = Lexer::tokenize(std::make_unique(lineBuffer)); - for (auto const& tokenInfo: tokens) + if (tokens[0].literal == "exit") { - if (tokenInfo.token == Token::Identifier) - if (tokenInfo.literal == "exit") - { - _quit = true; - return EXIT_SUCCESS; - } - debugLog()("token: {}\n", tokenInfo.literal); + _quit = true; + return EXIT_SUCCESS; } // Look up the JIT'd function, cast it to a function pointer, then call it.