From b2525428e697bba0d52557ec69effd6b34e04e83 Mon Sep 17 00:00:00 2001 From: Chuan Liu Date: Mon, 19 Apr 2021 22:26:49 -0700 Subject: [PATCH 1/7] draft change --- production/catalog/parser/src/lexer.ll | 1 + production/catalog/parser/src/parser.yy | 2 +- production/inc/gaia_internal/catalog/catalog.hpp | 13 ++++++++++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/production/catalog/parser/src/lexer.ll b/production/catalog/parser/src/lexer.ll index 0fbd1561f70e..0c35964b4da5 100644 --- a/production/catalog/parser/src/lexer.ll +++ b/production/catalog/parser/src/lexer.ll @@ -69,6 +69,7 @@ comment ("--".*) "EXISTS" return yy::parser::make_EXISTS(loc); "RELATIONSHIP" return yy::parser::make_RELATIONSHIP(loc); "ACTIVE" return yy::parser::make_ACTIVE(loc); +"USE" return yy::parser::make_USE(loc); "BOOL" return yy::parser::make_BOOL(loc); "INT8" return yy::parser::make_INT8(loc); diff --git a/production/catalog/parser/src/parser.yy b/production/catalog/parser/src/parser.yy index 058f4c65e797..b3ecbdcdb865 100644 --- a/production/catalog/parser/src/parser.yy +++ b/production/catalog/parser/src/parser.yy @@ -61,7 +61,7 @@ %token BOOL INT8 UINT8 INT16 UINT16 INT32 UINT32 INT64 UINT64 FLOAT DOUBLE STRING // Word tokens -%token CREATE DROP DATABASE TABLE IF NOT EXISTS ACTIVE RELATIONSHIP +%token CREATE DROP DATABASE TABLE IF NOT EXISTS ACTIVE RELATIONSHIP USE // Symbols %token LPAREN "(" diff --git a/production/inc/gaia_internal/catalog/catalog.hpp b/production/inc/gaia_internal/catalog/catalog.hpp index e26dc8ae372a..96704d7894e3 100644 --- a/production/inc/gaia_internal/catalog/catalog.hpp +++ b/production/inc/gaia_internal/catalog/catalog.hpp @@ -110,7 +110,8 @@ enum class statement_type_t : uint8_t { create, drop, - alter + alter, + use }; struct statement_t @@ -191,6 +192,16 @@ enum class create_type_t : uint8_t create_relationship, }; +struct use_statement_t : statement_t +{ + explicit use_statement_t(std::string db_name) + : statement_t(statement_type_t::use), database(std::move(db_name)) + { + } + + std::string database; +}; + // TODO: refactoring create statements into sub types, pending index changes (create_index). struct create_statement_t : statement_t { From 5c537c0a1acef1b9fc0e57e0693ffe1a980167a8 Mon Sep 17 00:00:00 2001 From: Chuan Liu Date: Wed, 21 Apr 2021 23:45:30 -0700 Subject: [PATCH 2/7] Implement use statement execution --- production/catalog/parser/src/parser.yy | 9 ++++ production/catalog/src/catalog.cpp | 17 ++++--- production/catalog/src/ddl_executor.cpp | 47 ++++++++++++------- production/catalog/src/fbs_generator.cpp | 8 ++-- .../inc/gaia_internal/catalog/catalog.hpp | 14 ++++-- .../gaia_internal/catalog/ddl_execution.hpp | 5 ++ .../gaia_internal/catalog/ddl_executor.hpp | 13 +++++ 7 files changed, 83 insertions(+), 30 deletions(-) diff --git a/production/catalog/parser/src/parser.yy b/production/catalog/parser/src/parser.yy index b3ecbdcdb865..c68b5b851843 100644 --- a/production/catalog/parser/src/parser.yy +++ b/production/catalog/parser/src/parser.yy @@ -82,6 +82,7 @@ %type > statement %type > create_statement %type > drop_statement +%type > use_statement %type opt_array %type opt_if_exists @@ -97,6 +98,7 @@ %printer { yyo << "statement"; } statement %printer { yyo << "create_statement:" << $$->name; } create_statement %printer { yyo << "drop_statement:" << $$->name; } drop_statement +%printer { yyo << "use_statement:" << $$->name; } use_statement %printer { yyo << "filed_def:" << $$->name; } field_def %printer { yyo << "data_field_def:" << $$->name; } data_field_def %printer { yyo << "link_def:" << $$.name; } link_def @@ -135,6 +137,7 @@ opt_if_not_exists: IF NOT EXISTS { $$ = true; } | { $$ = false; }; statement: create_statement { $$ = std::unique_ptr{std::move($1)}; } | drop_statement { $$ = std::unique_ptr{std::move($1)}; } +| use_statement { $$ = std::unique_ptr{std::move($1)}; } ; create_statement: @@ -170,6 +173,12 @@ drop_statement: } ; +use_statement: + USE IDENTIFIER { + $$ = std::make_unique($2); + } +; + field_def_commalist: field_def { $$ = std::make_unique(); diff --git a/production/catalog/src/catalog.cpp b/production/catalog/src/catalog.cpp index 65c8915bf37e..6046dcbd077b 100644 --- a/production/catalog/src/catalog.cpp +++ b/production/catalog/src/catalog.cpp @@ -21,6 +21,11 @@ void initialize_catalog() ddl_executor_t::get(); } +void use_database(const string& name) +{ + ddl_executor_t::get().switch_db_context(name); +} + gaia_id_t create_database(const string& name, bool throw_on_exists) { return ddl_executor_t::get().create_database(name, throw_on_exists); @@ -32,12 +37,12 @@ gaia_id_t create_table(const string& name, const ddl::field_def_list_t& fields) } gaia_id_t create_table( - const string& dbname, + const string& db_name, const string& name, const ddl::field_def_list_t& fields, bool throw_on_exists) { - return ddl_executor_t::get().create_table(dbname, name, fields, throw_on_exists); + return ddl_executor_t::get().create_table(db_name, name, fields, throw_on_exists); } gaia_id_t create_relationship( @@ -59,14 +64,14 @@ void drop_table(const string& name, bool throw_unless_exists) return ddl_executor_t::get().drop_table("", name, throw_unless_exists); } -void drop_table(const string& dbname, const string& name, bool throw_unless_exists) +void drop_table(const string& db_name, const string& name, bool throw_unless_exists) { - return ddl_executor_t::get().drop_table(dbname, name, throw_unless_exists); + return ddl_executor_t::get().drop_table(db_name, name, throw_unless_exists); } -gaia_id_t find_db_id(const string& dbname) +gaia_id_t find_db_id(const string& db_name) { - return ddl_executor_t::get().find_db_id(dbname); + return ddl_executor_t::get().find_db_id(db_name); } vector list_fields(gaia_id_t table_id) diff --git a/production/catalog/src/ddl_executor.cpp b/production/catalog/src/ddl_executor.cpp index c23418346ae9..10e931fced6a 100644 --- a/production/catalog/src/ddl_executor.cpp +++ b/production/catalog/src/ddl_executor.cpp @@ -48,7 +48,6 @@ ddl_executor_t& ddl_executor_t::get() void ddl_executor_t::bootstrap_catalog() { - create_database("catalog", false); { // create table gaia_database (name string); @@ -233,6 +232,7 @@ void ddl_executor_t::reset() // Create the special global database. // Tables created without specifying a database name will belong to the global database. m_empty_db_id = create_database(c_empty_db_name, false); + m_db_context = c_empty_db_name; } void ddl_executor_t::clear_cache() @@ -323,11 +323,11 @@ gaia_id_t ddl_executor_t::create_relationship( return m_relationship_names.at(name); } - gaia_id_t link1_src_table_id = get_table_id(link1.from_database, link1.from_table); - gaia_id_t link1_dest_table_id = get_table_id(link1.to_database, link1.to_table); + gaia_id_t link1_src_table_id = get_table_id(in_context(link1.from_database), link1.from_table); + gaia_id_t link1_dest_table_id = get_table_id(in_context(link1.to_database), link1.to_table); - gaia_id_t link2_src_table_id = get_table_id(link2.from_database, link2.from_table); - gaia_id_t link2_dest_table_id = get_table_id(link2.to_database, link2.to_table); + gaia_id_t link2_src_table_id = get_table_id(in_context(link2.from_database), link2.from_table); + gaia_id_t link2_dest_table_id = get_table_id(in_context(link2.to_database), link2.to_table); if (link1_src_table_id != link2_dest_table_id) { @@ -558,7 +558,7 @@ void ddl_executor_t::drop_table(const string& db_name, const string& name, bool return; } - string full_table_name = get_full_table_name(db_name, name); + string full_table_name = get_full_table_name(in_context(db_name), name); if (m_table_names.find(full_table_name) == m_table_names.end()) { @@ -656,8 +656,8 @@ gaia_id_t ddl_executor_t::create_table_impl( throw db_not_exists(db_name); } - string full_table_name = get_full_table_name(db_name, table_name); - gaia_id_t db_id = find_db_id_no_lock(db_name); + string full_table_name = get_full_table_name(in_context(db_name), table_name); + gaia_id_t db_id = find_db_id_no_lock(in_context(db_name)); ASSERT_INVARIANT(db_id != c_invalid_gaia_id, "Invalid database id!"); if (m_table_names.find(full_table_name) != m_table_names.end()) @@ -673,8 +673,9 @@ gaia_id_t ddl_executor_t::create_table_impl( { // Log a warning message for skipping non-system table creation. // - // The warnning should not apply to system table creation - // because current implementation will try to re-create all + // The warning should not apply to system table creation + // because: 1) users cannot create system tables by desgin; 2) + // current catalog implementation will try to re-create all // system tables on every startup and expect the creation to be // skipped normally if the tables already exist. gaia_log::catalog().warn("Table '{}' (id: {}) already exists, skipping.", full_table_name, id); @@ -701,7 +702,7 @@ gaia_id_t ddl_executor_t::create_table_impl( field_names.insert(field_name); } - string fbs{generate_fbs(db_name, table_name, fields)}; + string fbs{generate_fbs(in_context(db_name), table_name, fields)}; const std::vector bfbs = generate_bfbs(fbs); const std::vector bin = generate_bin(fbs, generate_json(fields)); @@ -741,21 +742,21 @@ gaia_id_t ddl_executor_t::create_table_impl( return table_id; } -gaia_id_t ddl_executor_t::find_db_id(const string& dbname) const +gaia_id_t ddl_executor_t::find_db_id(const string& db_name) const { shared_lock lock(m_lock); - return find_db_id_no_lock(dbname); + return find_db_id_no_lock(db_name); } -inline gaia_id_t ddl_executor_t::find_db_id_no_lock(const string& dbname) const +inline gaia_id_t ddl_executor_t::find_db_id_no_lock(const string& db_name) const { - if (dbname.empty()) + if (db_name.empty()) { return m_empty_db_id; } - else if (m_db_names.count(dbname)) + else if (m_db_names.count(db_name)) { - return m_db_names.at(dbname); + return m_db_names.at(db_name); } else { @@ -775,5 +776,17 @@ string ddl_executor_t::get_full_table_name(const string& db, const string& table } } +void ddl_executor_t::switch_db_context(const string& db_name) +{ + unique_lock lock(m_lock); + + if (!db_name.empty() && m_db_names.find(db_name) == m_db_names.end()) + { + throw db_not_exists(db_name); + } + + m_db_context = db_name; +} + } // namespace catalog } // namespace gaia diff --git a/production/catalog/src/fbs_generator.cpp b/production/catalog/src/fbs_generator.cpp index 507631382c35..73c4f397d036 100644 --- a/production/catalog/src/fbs_generator.cpp +++ b/production/catalog/src/fbs_generator.cpp @@ -121,14 +121,14 @@ string generate_fbs(gaia_id_t table_id) return fbs; } -string generate_fbs(const string& dbname) +string generate_fbs(const string& db_name) { - gaia_id_t db_id = find_db_id(dbname); + gaia_id_t db_id = find_db_id(db_name); if (db_id == c_invalid_gaia_id) { - throw db_not_exists(dbname); + throw db_not_exists(db_name); } - string fbs = generate_fbs_namespace(dbname); + string fbs = generate_fbs_namespace(db_name); gaia::db::begin_transaction(); for (auto const& table : gaia_database_t::get(db_id).gaia_tables()) { diff --git a/production/inc/gaia_internal/catalog/catalog.hpp b/production/inc/gaia_internal/catalog/catalog.hpp index 96704d7894e3..d37999e7a906 100644 --- a/production/inc/gaia_internal/catalog/catalog.hpp +++ b/production/inc/gaia_internal/catalog/catalog.hpp @@ -194,12 +194,12 @@ enum class create_type_t : uint8_t struct use_statement_t : statement_t { - explicit use_statement_t(std::string db_name) - : statement_t(statement_type_t::use), database(std::move(db_name)) + explicit use_statement_t(std::string name) + : statement_t(statement_type_t::use), name(std::move(name)) { } - std::string database; + std::string name; }; // TODO: refactoring create statements into sub types, pending index changes (create_index). @@ -413,6 +413,14 @@ class many_to_many_not_supported : public gaia::common::gaia_exception */ void initialize_catalog(); +/** + * Switch to the database. + * + * @param name database name + * @throw db_not_exists + */ +void use_database(const std::string& name); + /** * Create a database in the catalog. * diff --git a/production/inc/gaia_internal/catalog/ddl_execution.hpp b/production/inc/gaia_internal/catalog/ddl_execution.hpp index 3b5430109924..afc67ddf8c2c 100644 --- a/production/inc/gaia_internal/catalog/ddl_execution.hpp +++ b/production/inc/gaia_internal/catalog/ddl_execution.hpp @@ -91,6 +91,11 @@ inline void execute(const std::string& db_name, std::vectorname, !drop_stmt->if_exists); } } + else if (stmt->is_type(ddl::statement_type_t::use)) + { + auto use_stmt = dynamic_cast(stmt.get()); + use_database(use_stmt->name); + } } } diff --git a/production/inc/gaia_internal/catalog/ddl_executor.hpp b/production/inc/gaia_internal/catalog/ddl_executor.hpp index 125582c621dd..fa0484d301e6 100644 --- a/production/inc/gaia_internal/catalog/ddl_executor.hpp +++ b/production/inc/gaia_internal/catalog/ddl_executor.hpp @@ -37,11 +37,13 @@ class ddl_executor_t * APIs for accessing catalog records */ gaia::common::gaia_id_t create_database(const std::string& name, bool throw_on_exist = true); + gaia::common::gaia_id_t create_table( const std::string& db_name, const std::string& name, const ddl::field_def_list_t& fields, bool throw_on_exist = true); + gaia::common::gaia_id_t create_relationship( const std::string& name, const ddl::link_def_t& link1, @@ -49,8 +51,11 @@ class ddl_executor_t bool thrown_on_exists = true); void drop_table(const std::string& db_name, const std::string& name, bool throw_unless_exists); + void drop_database(const std::string& name, bool throw_unless_exists); + void switch_db_context(const std::string& db_name); + gaia::common::gaia_id_t find_db_id(const std::string& dbname) const; // Initialize the catalog manager. @@ -130,6 +135,14 @@ class ddl_executor_t gaia::common::gaia_id_t m_empty_db_id; + // The DB context defines the database in which an entity like a table, an + // index, or a relationship will be referred to without a database name. + std::string m_db_context; + inline std::string in_context(const std::string& db) + { + return db.empty() ? m_db_context : db; + } + // Use the lock to ensure exclusive access to caches. mutable std::shared_mutex m_lock; }; From d35ef6f5ce5642ec598a4a82c39793e9340282bc Mon Sep 17 00:00:00 2001 From: Chuan Liu Date: Thu, 22 Apr 2021 10:36:01 -0700 Subject: [PATCH 3/7] Update gaiac command line usage for output path and db --- production/catalog/gaiac/src/main.cpp | 131 +++++++++--------- .../gaia_internal/catalog/ddl_execution.hpp | 69 +-------- .../inc/gaia_internal/db/db_test_helpers.hpp | 19 +-- .../schemas/test/addr_book/CMakeLists.txt | 2 +- .../schemas/test/addr_book/addr_book.ddl | 4 + .../schemas/test/airport/CMakeLists.txt | 2 +- production/schemas/test/airport/airport.ddl | 4 + .../schemas/test/incubator/CMakeLists.txt | 4 +- .../{barn_storage.ddl => incubator.ddl} | 4 + .../schemas/test/prerequisites/CMakeLists.txt | 2 +- .../test/prerequisites/prerequisites.ddl | 9 ++ .../tests/test_gaia_generate.cpp | 2 +- .../tools/gaia_translate/CMakeLists.txt | 2 +- .../tests/test_translation_engine.cpp | 2 +- 14 files changed, 108 insertions(+), 148 deletions(-) rename production/schemas/test/incubator/{barn_storage.ddl => incubator.ddl} (92%) diff --git a/production/catalog/gaiac/src/main.cpp b/production/catalog/gaiac/src/main.cpp index d279f568aceb..aa75431ed8f9 100644 --- a/production/catalog/gaiac/src/main.cpp +++ b/production/catalog/gaiac/src/main.cpp @@ -12,12 +12,14 @@ #include "flatbuffers/idl.h" +#include "gaia/common.hpp" #include "gaia/db/db.hpp" #include "gaia_internal/catalog/catalog.hpp" #include "gaia_internal/catalog/ddl_execution.hpp" #include "gaia_internal/common/gaia_version.hpp" #include "gaia_internal/common/logger_internal.hpp" +#include "gaia_internal/common/scope_guard.hpp" #include "gaia_internal/db/db_test_helpers.hpp" #include "command.hpp" @@ -42,9 +44,8 @@ enum class operate_mode_t loading, }; -void start_repl(parser_t& parser, const string& db_name) +void start_repl(parser_t& parser) { - gaia::db::begin_session(); initialize_catalog(); const auto prompt = "gaiac> "; @@ -78,7 +79,7 @@ void start_repl(parser_t& parser, const string& db_name) int parsing_result = parser.parse_line(line); if (parsing_result == EXIT_SUCCESS) { - execute(db_name, parser.statements); + execute(parser.statements); } else { @@ -90,8 +91,6 @@ void start_repl(parser_t& parser, const string& db_name) cerr << c_error_prompt << e.what() << endl; } } - - gaia::db::end_session(); } // From the database name and catalog contents, generate the FlatBuffers C++ header file(s). @@ -121,11 +120,10 @@ void generate_fbs_headers(const string& db_name, const string& output_path) } // From the database name and catalog contents, generate the Extended Data Class definition(s). -void generate_edc_headers(const string& db_name, const string& output_path) +void generate_edc_headers(const string& db_name, const filesystem::path& output_path) { - std::string header_path = output_path + "gaia" + (db_name.empty() ? "" : "_" + db_name) + ".h"; - - cout << "Generating EDC headers in: '" << std::filesystem::absolute(header_path).string() << "'." << endl; + filesystem::path header_path = output_path; + header_path /= "gaia" + (db_name.empty() ? "" : "_" + db_name) + ".h"; ofstream edc(header_path); try @@ -140,10 +138,25 @@ void generate_edc_headers(const string& db_name, const string& output_path) edc.close(); } -void generate_headers(const string& db_name, const string& output_path) +void generate_headers(const string& db_name, const filesystem::path& output_path) { - generate_fbs_headers(db_name, output_path); - generate_edc_headers(db_name, output_path); + filesystem::path absolute_output_path = filesystem::absolute(output_path); + absolute_output_path += filesystem::path::preferred_separator; + absolute_output_path = absolute_output_path.lexically_normal(); + + if (!filesystem::exists(absolute_output_path)) + { + filesystem::create_directory(output_path); + } + else if (!filesystem::is_directory(absolute_output_path)) + { + throw std::invalid_argument("Invalid output path: '" + output_path.string() + "'."); + } + + cout << "Generating headers in: '" << absolute_output_path << "'." << endl; + + generate_fbs_headers(db_name, absolute_output_path); + generate_edc_headers(db_name, absolute_output_path); } // Check if a database name is valid. @@ -172,15 +185,6 @@ bool valid_db_name(const string& db_name) return true; } -// Add a trailing '/' if not provided. -void terminate_path(string& path) -{ - if (path.back() != '/') - { - path.append("/"); - } -} - string usage() { std::stringstream ss; @@ -236,11 +240,12 @@ int main(int argc, char* argv[]) int res = EXIT_SUCCESS; server_t server; string output_path; - string db_name; + vector db_names; string ddl_filename; operate_mode_t mode = operate_mode_t::loading; parser_t parser; bool remove_persistent_store = false; + const char* path_to_db_server = nullptr; // If no arguments are specified print the help. if (argc == 1) @@ -274,16 +279,7 @@ int main(int argc, char* argv[]) cerr << c_error_prompt << "Missing path to db server." << endl; exit(EXIT_FAILURE); } - const char* path_to_db_server = argv[i]; - server.set_path(path_to_db_server); - if (remove_persistent_store) - { - server.start(); - } - else - { - server.start_and_retain_persistent_store(); - } + path_to_db_server = argv[i]; } else if (argv[i] == string("-o") || argv[i] == string("--output")) { @@ -293,7 +289,6 @@ int main(int argc, char* argv[]) exit(EXIT_FAILURE); } output_path = argv[i]; - terminate_path(output_path); } else if (argv[i] == string("-d") || argv[i] == string("--db-name")) { @@ -302,7 +297,15 @@ int main(int argc, char* argv[]) cerr << c_error_prompt << "Missing database name." << endl; exit(EXIT_FAILURE); } - db_name = argv[i]; + string db_name = argv[i]; + if (!valid_db_name(db_name)) + { + cerr << c_error_prompt + << "The database name '" + db_name + "' is invalid." + << endl; + exit(EXIT_FAILURE); + } + db_names.push_back(db_name); } else if (argv[i] == string("-h") || argv[i] == string("--help")) { @@ -324,55 +327,57 @@ int main(int argc, char* argv[]) } } - if (!valid_db_name(db_name)) + if (path_to_db_server) { - cerr << c_error_prompt - << "The database name '" + db_name + "' supplied from the command line is incorrectly formatted." - << endl; - exit(EXIT_FAILURE); - } - - // This indicates if we should try to create the database automatically. If - // the database name is derived from the ddl file name, we will try to - // create the database for the user. This is to keep backward compatible - // with existing build scripts. Use '-d ' to avoid this behavior. - // GAIAPLAT-585 tracks the work to remove this behavior. - bool create_db = false; - if (!ddl_filename.empty() && db_name.empty()) - { - db_name = get_db_name_from_filename(ddl_filename); - create_db = true; + server.set_path(path_to_db_server); + if (remove_persistent_store) + { + server.start(); + } + else + { + server.start_and_retain_persistent_store(); + } } - if (!valid_db_name(db_name)) - { - cerr << c_error_prompt - << "The database name '" + db_name + "' derived from the filename is incorrectly formatted." - << endl; - exit(EXIT_FAILURE); - } + gaia::db::begin_session(); + auto close_db_session = scope_guard::make_scope_guard([&]() { + gaia::db::end_session(); + }); if (mode == operate_mode_t::interactive) { - start_repl(parser, db_name); + start_repl(parser); } else { try { - gaia::db::begin_session(); initialize_catalog(); if (!ddl_filename.empty()) { - load_catalog(parser, ddl_filename, db_name, create_db); + load_catalog(parser, ddl_filename); + + if (output_path.empty()) + { + output_path = filesystem::path(ddl_filename).stem(); + } } if (mode == operate_mode_t::generation) { - generate_headers(db_name, output_path); + // Generate headers for the default global database if no database is given. + if (db_names.empty()) + { + db_names.emplace_back(""); + } + + for (const auto& db_name : db_names) + { + generate_headers(db_name, output_path); + } } - gaia::db::end_session(); } catch (gaia::common::system_error& e) { diff --git a/production/inc/gaia_internal/catalog/ddl_execution.hpp b/production/inc/gaia_internal/catalog/ddl_execution.hpp index afc67ddf8c2c..08d5594606db 100644 --- a/production/inc/gaia_internal/catalog/ddl_execution.hpp +++ b/production/inc/gaia_internal/catalog/ddl_execution.hpp @@ -16,7 +16,7 @@ namespace gaia namespace catalog { -inline void execute(const std::string& db_name, std::vector>& statements) +inline void execute(std::vector>& statements) { for (auto& stmt : statements) { @@ -30,14 +30,7 @@ inline void execute(const std::string& db_name, std::vectortype == ddl::create_type_t::create_table) { - if (!create_stmt->database.empty()) - { - create_table(create_stmt->database, create_stmt->name, create_stmt->fields, throw_on_exist); - } - else - { - create_table(db_name, create_stmt->name, create_stmt->fields, throw_on_exist); - } + create_table(create_stmt->database, create_stmt->name, create_stmt->fields, throw_on_exist); } else if (create_stmt->type == ddl::create_type_t::create_database) { @@ -45,26 +38,6 @@ inline void execute(const std::string& db_name, std::vectortype == ddl::create_type_t::create_relationship) { - if (!db_name.empty()) - { - if (create_stmt->relationship.first.from_database.empty()) - { - create_stmt->relationship.first.from_database = db_name; - } - if (create_stmt->relationship.first.to_database.empty()) - { - create_stmt->relationship.first.to_database = db_name; - } - if (create_stmt->relationship.second.from_database.empty()) - { - create_stmt->relationship.second.from_database = db_name; - } - if (create_stmt->relationship.second.to_database.empty()) - { - create_stmt->relationship.second.to_database = db_name; - } - } - create_relationship( create_stmt->name, create_stmt->relationship.first, @@ -77,14 +50,7 @@ inline void execute(const std::string& db_name, std::vector(stmt.get()); if (drop_stmt->type == ddl::drop_type_t::drop_table) { - if (!drop_stmt->database.empty()) - { - drop_table(drop_stmt->database, drop_stmt->name, !drop_stmt->if_exists); - } - else - { - drop_table(db_name, drop_stmt->name, !drop_stmt->if_exists); - } + drop_table(drop_stmt->database, drop_stmt->name, !drop_stmt->if_exists); } else if (drop_stmt->type == ddl::drop_type_t::drop_database) { @@ -99,26 +65,9 @@ inline void execute(const std::string& db_name, std::vector #include +#include #include #include #include @@ -126,7 +127,7 @@ class server_t } // Launch SE server in background. - std::string cmd = c_daemonize_command + m_server_path; + std::string cmd = c_daemonize_command + m_server_path.string(); if (m_disable_persistence) { cmd.append(" "); @@ -159,7 +160,7 @@ class server_t // Try to kill the SE server process. // REVIEW: we should be using a proper process library for this, so we can kill by PID. std::string cmd = "pkill -f -KILL "; - cmd.append(m_server_path); + cmd.append(m_server_path.string()); std::cerr << cmd << std::endl; ::system(cmd.c_str()); } @@ -178,8 +179,7 @@ class server_t else { m_server_path = db_server_path; - terminate_path(m_server_path); - m_server_path.append(gaia::db::c_db_server_exec_name); + m_server_path /= gaia::db::c_db_server_exec_name; } } @@ -189,16 +189,7 @@ class server_t } private: - // Add a trailing '/' if not provided. - static void inline terminate_path(std::string& path) - { - if (path.back() != '/') - { - path.append("/"); - } - } - - std::string m_server_path; + std::filesystem::path m_server_path; bool m_disable_persistence = false; bool m_server_started = false; }; diff --git a/production/schemas/test/addr_book/CMakeLists.txt b/production/schemas/test/addr_book/CMakeLists.txt index 103f2d2fb195..1d96a68e6266 100644 --- a/production/schemas/test/addr_book/CMakeLists.txt +++ b/production/schemas/test/addr_book/CMakeLists.txt @@ -14,7 +14,7 @@ set(ADDR_BOOK_DDL "${PROJECT_SOURCE_DIR}/addr_book.ddl") add_custom_command( COMMENT "Compiling addr_book.ddl..." OUTPUT ${GAIA_GENERATED_SCHEMAS}/gaia_addr_book.h - COMMAND ${GAIA_GENERATE} --destroy-db -t ${GAIA_PROD_BUILD}/db/core -o ${GAIA_GENERATED_SCHEMAS} -g ${ADDR_BOOK_DDL} + COMMAND ${GAIA_GENERATE} --destroy-db -t ${GAIA_PROD_BUILD}/db/core -o ${GAIA_GENERATED_SCHEMAS} -d addr_book -g ${ADDR_BOOK_DDL} DEPENDS gaiac DEPENDS ${ADDR_BOOK_DDL} ) diff --git a/production/schemas/test/addr_book/addr_book.ddl b/production/schemas/test/addr_book/addr_book.ddl index 077ab9109ed7..022854e4f093 100644 --- a/production/schemas/test/addr_book/addr_book.ddl +++ b/production/schemas/test/addr_book/addr_book.ddl @@ -3,6 +3,10 @@ -- All rights reserved. --------------------------------------------- +create database if not exists addr_book; + +use addr_book; + create table if not exists employee ( name_first string active, name_last string, diff --git a/production/schemas/test/airport/CMakeLists.txt b/production/schemas/test/airport/CMakeLists.txt index 02439c88167e..43bd74b71817 100644 --- a/production/schemas/test/airport/CMakeLists.txt +++ b/production/schemas/test/airport/CMakeLists.txt @@ -14,7 +14,7 @@ set(AIRPORT_DDL "${PROJECT_SOURCE_DIR}/airport.ddl") add_custom_command( COMMENT "Compiling airport.ddl..." OUTPUT ${GAIA_GENERATED_SCHEMAS}/gaia_airport.h - COMMAND ${GAIA_GENERATE} --destroy-db -t ${GAIA_PROD_BUILD}/db/core -o ${GAIA_GENERATED_SCHEMAS} -g ${AIRPORT_DDL} + COMMAND ${GAIA_GENERATE} --destroy-db -t ${GAIA_PROD_BUILD}/db/core -o ${GAIA_GENERATED_SCHEMAS} -d airport -g ${AIRPORT_DDL} DEPENDS gaiac DEPENDS ${AIRPORT_DDL} ) diff --git a/production/schemas/test/airport/airport.ddl b/production/schemas/test/airport/airport.ddl index 19596b0e72f8..513ee7606398 100644 --- a/production/schemas/test/airport/airport.ddl +++ b/production/schemas/test/airport/airport.ddl @@ -3,6 +3,10 @@ -- All rights reserved. --------------------------------------------- +create database if not exists airport; + +use airport; + create table if not exists airport ( name string, diff --git a/production/schemas/test/incubator/CMakeLists.txt b/production/schemas/test/incubator/CMakeLists.txt index 7c40e9e0559e..abe4c500a97c 100644 --- a/production/schemas/test/incubator/CMakeLists.txt +++ b/production/schemas/test/incubator/CMakeLists.txt @@ -6,7 +6,7 @@ # Set the project name. project(incubator_schema) -set(INCUBATOR_DDL "${PROJECT_SOURCE_DIR}/barn_storage.ddl") +set(INCUBATOR_DDL "${PROJECT_SOURCE_DIR}/incubator.ddl") # # Generate stand alone incubator. @@ -14,7 +14,7 @@ set(INCUBATOR_DDL "${PROJECT_SOURCE_DIR}/barn_storage.ddl") add_custom_command( COMMENT "Compiling incubator.ddl..." OUTPUT ${GAIA_GENERATED_SCHEMAS}/gaia_barn_storage.h - COMMAND ${GAIA_GENERATE} --destroy-db -t ${GAIA_PROD_BUILD}/db/core -o ${GAIA_GENERATED_SCHEMAS} -g ${INCUBATOR_DDL} + COMMAND ${GAIA_GENERATE} --destroy-db -t ${GAIA_PROD_BUILD}/db/core -o ${GAIA_GENERATED_SCHEMAS} -d barn_storage -g ${INCUBATOR_DDL} DEPENDS gaiac DEPENDS ${INCUBATOR_DDL} ) diff --git a/production/schemas/test/incubator/barn_storage.ddl b/production/schemas/test/incubator/incubator.ddl similarity index 92% rename from production/schemas/test/incubator/barn_storage.ddl rename to production/schemas/test/incubator/incubator.ddl index 0ba314f945ed..8e06f4ebdc7e 100644 --- a/production/schemas/test/incubator/barn_storage.ddl +++ b/production/schemas/test/incubator/incubator.ddl @@ -3,6 +3,10 @@ -- All rights reserved. --------------------------------------------- +CREATE DATABASE IF NOT EXISTS barn_storage; + +USE barn_storage; + CREATE TABLE IF NOT EXISTS incubator ( name STRING, min_temp FLOAT, diff --git a/production/schemas/test/prerequisites/CMakeLists.txt b/production/schemas/test/prerequisites/CMakeLists.txt index 5615bf6a8021..dd559150151a 100644 --- a/production/schemas/test/prerequisites/CMakeLists.txt +++ b/production/schemas/test/prerequisites/CMakeLists.txt @@ -14,7 +14,7 @@ set(PREREQUISITES_DDL "${PROJECT_SOURCE_DIR}/prerequisites.ddl") add_custom_command( COMMENT "Compiling prerequisites.ddl..." OUTPUT ${GAIA_GENERATED_SCHEMAS}/gaia_prerequisites.h - COMMAND ${GAIA_GENERATE} --destroy-db -t ${GAIA_PROD_BUILD}/db/core -o ${GAIA_GENERATED_SCHEMAS} -g ${PREREQUISITES_DDL} + COMMAND ${GAIA_GENERATE} --destroy-db -t ${GAIA_PROD_BUILD}/db/core -o ${GAIA_GENERATED_SCHEMAS} -d prerequisites -g ${PREREQUISITES_DDL} DEPENDS gaiac DEPENDS ${PREREQUISITES_DDL} ) diff --git a/production/schemas/test/prerequisites/prerequisites.ddl b/production/schemas/test/prerequisites/prerequisites.ddl index 82397676bd7c..b89978076f00 100644 --- a/production/schemas/test/prerequisites/prerequisites.ddl +++ b/production/schemas/test/prerequisites/prerequisites.ddl @@ -1,3 +1,12 @@ +--------------------------------------------- +-- Copyright (c) Gaia Platform LLC +-- All rights reserved. +--------------------------------------------- + +create database if not exists prerequisites; + +use prerequisites; + create table Student ( StudentId string, Surname string, diff --git a/production/tools/gaia_generate/tests/test_gaia_generate.cpp b/production/tools/gaia_generate/tests/test_gaia_generate.cpp index 7712f53db32c..2474577825b6 100644 --- a/production/tools/gaia_generate/tests/test_gaia_generate.cpp +++ b/production/tools/gaia_generate/tests/test_gaia_generate.cpp @@ -45,7 +45,7 @@ TEST_F(gaia_generate_test, parse_ddl) EXPECT_EQ(EXIT_SUCCESS, parser.parse_line("create table tmp_airport ( name string );")); create_database("tmp_airport"); - execute("tmp_airport", parser.statements); + execute(parser.statements); auto header_str = gaia_generate("tmp_airport"); EXPECT_NE(0, header_str.find("struct tmp_airport_t")); diff --git a/production/tools/gaia_translate/CMakeLists.txt b/production/tools/gaia_translate/CMakeLists.txt index 10b535eafed7..2773b735d1bc 100644 --- a/production/tools/gaia_translate/CMakeLists.txt +++ b/production/tools/gaia_translate/CMakeLists.txt @@ -50,7 +50,7 @@ set(TEST_INCLUDES ${GAIA_REPO}/production/tools/gaia_translate/tests ) -set(INCUBATOR_DDL "${GAIA_REPO}/production/schemas/test/incubator/barn_storage.ddl") +set(INCUBATOR_DDL "${GAIA_REPO}/production/schemas/test/incubator/incubator.ddl") set(TRANSLATION_RULESET "${GAIA_REPO}/production/tools/gaia_translate/tests/test.ruleset") set(TRANSLATION_RULESET_OUT "${GAIA_GENERATED_SCHEMAS}/test_translation.cpp") diff --git a/production/tools/gaia_translate/tests/test_translation_engine.cpp b/production/tools/gaia_translate/tests/test_translation_engine.cpp index 760e7c55bda5..918f02f95dac 100644 --- a/production/tools/gaia_translate/tests/test_translation_engine.cpp +++ b/production/tools/gaia_translate/tests/test_translation_engine.cpp @@ -31,7 +31,7 @@ class translation_engine_test : public db_catalog_test_base_t { public: translation_engine_test() - : db_catalog_test_base_t("barn_storage.ddl"){}; + : db_catalog_test_base_t("incubator.ddl"){}; gaia_id_t insert_incubator(const char* name, float min_temp, float max_temp) { From 4161110f8e8ec9e87468577ef2f93b21b8715b57 Mon Sep 17 00:00:00 2001 From: Chuan Liu Date: Thu, 22 Apr 2021 10:53:37 -0700 Subject: [PATCH 4/7] Update catalog README --- production/catalog/README.md | 53 ++++++++---------------------------- 1 file changed, 11 insertions(+), 42 deletions(-) diff --git a/production/catalog/README.md b/production/catalog/README.md index 21e257554ce3..20c2fc71ec9c 100644 --- a/production/catalog/README.md +++ b/production/catalog/README.md @@ -58,7 +58,7 @@ out the DDL. The DDL typed in will be executed, and fbs output if any will be printed out to the console output. Under generation mode (`-g`), the tool will generate the following two header -files either from an DDL file or a specified database. +files from specified database(s). - The FlatBuffers header for field access, `_generated.h` - The EDC header file `gaia_.h` @@ -77,11 +77,11 @@ Enter interactive mode. ``` -Execute DDL statements in `airport.ddl` file using the `airport` database, and -generate header files for tables in `airport` database. +Execute DDL statements in `airport.ddl` file, generate header files to the +`airport` directory for tables in the `airport` database. ``` - gaiac -g airport.ddl + gaiac -g -d airport airport.ddl ``` Generate catalog direct access APIs. This is the command used for bootstrapping. @@ -92,18 +92,12 @@ Generate catalog direct access APIs. This is the command used for bootstrapping. ## Databases -There are two ways to create a database and specifying a table in a database: -first, using DDL; second, using `gaiac` command. When both are specified, the -DDL definition will override the `gaiac` settings. - -### Use DDL - The DDL to create database is `create database`. To specifying a table in a database, using the composite name of the format `[database].[table]`. -#### Examples +### Examples A database `addr_book` can be created using the following statement. @@ -126,11 +120,14 @@ create table addr_book.employee ( ); ``` -As a syntactic sugar, the database name can be omitted when specifying a -reference to a table in the same database. +Switch to a database to make the DDL more succinct (by avoid the database name +when referring to a table) with the `use database` statement. In the following +example, the `address` table will be created in the `addr_book` database. ``` - create table addr_book.address ( + use addr_book; + + create table address ( street: string, apt_suite: string, city: string, @@ -138,37 +135,9 @@ reference to a table in the same database. postal: string, country: string, current: bool, - addresses references employee ); ``` -### Use `gaiac` - -This is the way to create and specify a database before the introduction of the -database to DDL. The `gaiac` command line usage already documented how to do -this. See the following examples for more explanation. - -#### Examples - -In the following command line example, an `airport` database will be created -automatically. When no database prefix is specified for table names, the tables -will be created in the `airport` database. (The database prefix can still be -used to override the database name derived from the DDL file name.) - -``` - gaiac -g airport.ddl - -``` - -The following command line will create all tables in `tmp_airport`. Because a -database name is specified via `-d`, the command will not create the database -`tmp_airport` automatically. - -``` - gaiac -d tmp_airport -g airport.ddl - -``` - ## Catalog bootstrapping ### Code generation From c462faaee6ea20bbc02f36cfdca3227008ae510a Mon Sep 17 00:00:00 2001 From: Chuan Liu Date: Fri, 23 Apr 2021 17:45:37 -0700 Subject: [PATCH 5/7] Address CR comments --- production/catalog/README.md | 9 +++++---- production/catalog/gaiac/src/main.cpp | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/production/catalog/README.md b/production/catalog/README.md index 20c2fc71ec9c..f6407fd169b7 100644 --- a/production/catalog/README.md +++ b/production/catalog/README.md @@ -77,7 +77,7 @@ Enter interactive mode. ``` -Execute DDL statements in `airport.ddl` file, generate header files to the +Execute DDL statements in `airport.ddl` file, generate header files in the `airport` directory for tables in the `airport` database. ``` @@ -92,7 +92,7 @@ Generate catalog direct access APIs. This is the command used for bootstrapping. ## Databases -The DDL to create database is `create database`. +The DDL to create database is `create `. To specifying a table in a database, using the composite name of the format `[database].[table]`. @@ -121,8 +121,9 @@ create table addr_book.employee ( ``` Switch to a database to make the DDL more succinct (by avoid the database name -when referring to a table) with the `use database` statement. In the following -example, the `address` table will be created in the `addr_book` database. +when referring to a table) with the `use ` statement. In the +following example, the `address` table will be created in the `addr_book` +database. ``` use addr_book; diff --git a/production/catalog/gaiac/src/main.cpp b/production/catalog/gaiac/src/main.cpp index aa75431ed8f9..16f6669963e3 100644 --- a/production/catalog/gaiac/src/main.cpp +++ b/production/catalog/gaiac/src/main.cpp @@ -153,7 +153,7 @@ void generate_headers(const string& db_name, const filesystem::path& output_path throw std::invalid_argument("Invalid output path: '" + output_path.string() + "'."); } - cout << "Generating headers in: '" << absolute_output_path << "'." << endl; + cout << "Generating headers in: " << absolute_output_path << "." << endl; generate_fbs_headers(db_name, absolute_output_path); generate_edc_headers(db_name, absolute_output_path); From 8e0035875b9abfcb52ccd2d759b2bb4002396d21 Mon Sep 17 00:00:00 2001 From: Chuan Liu Date: Mon, 26 Apr 2021 09:36:47 -0700 Subject: [PATCH 6/7] Minor doc and comment updates --- production/catalog/README.md | 28 ++++++++++++++++--------- production/catalog/src/ddl_executor.cpp | 2 +- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/production/catalog/README.md b/production/catalog/README.md index f6407fd169b7..5cb203f8cf24 100644 --- a/production/catalog/README.md +++ b/production/catalog/README.md @@ -53,12 +53,12 @@ By default without specifying any mode, `gaiac` will run under loading mode to execute the DDL statements--translating them into catalog records--without generating any output. -The interactive mode (`-i`) provides a REPL style command line interface to try -out the DDL. The DDL typed in will be executed, and fbs output if any will be -printed out to the console output. +The interactive mode (`--interactive` or `-i`) provides a REPL style command +line interface to try out the DDL. The DDL typed in will be executed, and fbs +output if any will be printed out to the console output. -Under generation mode (`-g`), the tool will generate the following two header -files from specified database(s). +Under generation mode (`--generate` or `-g`), the tool will generate the following two header +files from specified database(s) under the output path. - The FlatBuffers header for field access, `_generated.h` - The EDC header file `gaia_.h` @@ -66,14 +66,14 @@ files from specified database(s). With the two headers, a direct access source file gains access to the database as defined by the catalog. -Full command line usage can be shown with the `-h` option. +Full command line usage can be shown with the `--help` or `-h` option. #### Examples Enter interactive mode. ``` - gaiac -i + gaiac --interactive ``` @@ -81,13 +81,21 @@ Execute DDL statements in `airport.ddl` file, generate header files in the `airport` directory for tables in the `airport` database. ``` - gaiac -g -d airport airport.ddl + gaiac --generate --db-name airport airport.ddl ``` -Generate catalog direct access APIs. This is the command used for bootstrapping. +Execute DDL statements in `incubator.ddl` file. Generate the headers in the +`incubator` directory for tables in the `barn_storage` and `lab`databases. ``` - gaiac -d catalog -g + gaiac --db-name barn_storage --db-name lab --generate incubator.ddl +``` + +Generate catalog direct access APIs in the `catalog` directory. This is the +command used for bootstrapping. + +``` + gaiac --database catalog --generate --output catalog ``` ## Databases diff --git a/production/catalog/src/ddl_executor.cpp b/production/catalog/src/ddl_executor.cpp index 10e931fced6a..b9ce58e868aa 100644 --- a/production/catalog/src/ddl_executor.cpp +++ b/production/catalog/src/ddl_executor.cpp @@ -674,7 +674,7 @@ gaia_id_t ddl_executor_t::create_table_impl( // Log a warning message for skipping non-system table creation. // // The warning should not apply to system table creation - // because: 1) users cannot create system tables by desgin; 2) + // because: 1) users cannot create system tables by design; 2) // current catalog implementation will try to re-create all // system tables on every startup and expect the creation to be // skipped normally if the tables already exist. From aed4ab17ced06c46e3901afa5819acff766fd3e2 Mon Sep 17 00:00:00 2001 From: Chuan Liu Date: Mon, 26 Apr 2021 09:41:28 -0700 Subject: [PATCH 7/7] Update indentation --- production/catalog/README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/production/catalog/README.md b/production/catalog/README.md index 5cb203f8cf24..cd2267639cf5 100644 --- a/production/catalog/README.md +++ b/production/catalog/README.md @@ -110,7 +110,7 @@ To specifying a table in a database, using the composite name of the format A database `addr_book` can be created using the following statement. ``` - create database addr_book; +create database addr_book; ``` Use the following statement to create an `employee` table in `addr_book` @@ -134,17 +134,17 @@ following example, the `address` table will be created in the `addr_book` database. ``` - use addr_book; - - create table address ( - street: string, - apt_suite: string, - city: string, - state: string, - postal: string, - country: string, - current: bool, - ); +use addr_book; + +create table address ( + street: string, + apt_suite: string, + city: string, + state: string, + postal: string, + country: string, + current: bool, +); ``` ## Catalog bootstrapping