From 344f3ccfcbf578bb5ef27180a3a193a96161d975 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Fri, 10 May 2024 11:22:25 -0600 Subject: [PATCH] sessions: store session in database This allows a session to be restored after restart so users don't have to login. Somehow ended up pulling in SQLx for this as it provides a nicer database experience and also includes migrations and pooling, but that will be a work in progress. --- Cargo.lock | 626 +++++++++++++++++- Cargo.toml | 4 + .../configdb/migrations/V2__Sessions.sql | 11 + src/bin/evebox.rs | 8 +- src/commands/config/mod.rs | 4 +- src/commands/config/users.rs | 40 +- src/commands/oneshot.rs | 32 +- src/commands/sqlite/mod.rs | 16 +- src/eventrepo/mod.rs | 3 + src/server/api/login.rs | 17 +- src/server/main.rs | 49 +- src/server/session.rs | 2 +- src/sqlite/configrepo.rs | 229 ++++--- src/sqlite/connection.rs | 70 +- src/sqlite/eventrepo.rs | 94 ++- src/sqlite/importer.rs | 153 +++-- src/sqlite/mod.rs | 3 +- src/sqlite/pool.rs | 11 - 18 files changed, 1075 insertions(+), 297 deletions(-) create mode 100644 resources/configdb/migrations/V2__Sessions.sql delete mode 100644 src/sqlite/pool.rs diff --git a/Cargo.lock b/Cargo.lock index 6912734e..ca2cef24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -30,6 +30,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", + "getrandom", "once_cell", "version_check", "zerocopy", @@ -119,7 +120,16 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.63", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", ] [[package]] @@ -253,6 +263,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "bcrypt" version = "0.15.1" @@ -277,6 +293,9 @@ name = "bitflags" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +dependencies = [ + "serde", +] [[package]] name = "block-buffer" @@ -333,6 +352,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "num-traits", +] + [[package]] name = "cipher" version = "0.4.4" @@ -374,7 +402,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.63", ] [[package]] @@ -399,6 +427,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "cookie" version = "0.17.0" @@ -435,6 +478,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.4.0" @@ -444,6 +502,21 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" + [[package]] name = "crossterm" version = "0.25.0" @@ -519,6 +592,17 @@ dependencies = [ "deadpool-runtime", ] +[[package]] +name = "der" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + [[package]] name = "deranged" version = "0.3.11" @@ -535,7 +619,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", + "subtle", ] [[package]] @@ -559,12 +645,27 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dyn-clone" version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" +[[package]] +name = "either" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" +dependencies = [ + "serde", +] + [[package]] name = "encoding_rs" version = "0.8.34" @@ -590,6 +691,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + [[package]] name = "evebox" version = "0.19.0-dev" @@ -601,6 +713,7 @@ dependencies = [ "base64 0.12.3", "bcrypt", "bytes", + "chrono", "clap", "deadpool-sqlite", "directories", @@ -628,6 +741,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "sqlx", "suricatax-rule-parser", "tempfile", "thiserror", @@ -642,6 +756,17 @@ dependencies = [ "webbrowser", ] +[[package]] +name = "event-listener" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9944b8ca13534cdfb2800775f8dd4902ff3fc75a50101466decadfdf322a24" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + [[package]] name = "fallible-iterator" version = "0.3.0" @@ -672,6 +797,23 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "finl_unicode" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" + +[[package]] +name = "flume" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +dependencies = [ + "futures-core", + "futures-sink", + "spin 0.9.8", +] + [[package]] name = "fnv" version = "1.0.7" @@ -729,6 +871,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.30" @@ -743,7 +896,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.63", ] [[package]] @@ -901,6 +1054,30 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "home" version = "0.5.9" @@ -1094,6 +1271,15 @@ version = "1.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -1136,6 +1322,9 @@ name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +dependencies = [ + "spin 0.5.2", +] [[package]] name = "libc" @@ -1163,6 +1352,12 @@ dependencies = [ "rle-decode-fast", ] +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + [[package]] name = "libredox" version = "0.1.3" @@ -1242,6 +1437,16 @@ dependencies = [ "serde", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "md5" version = "0.7.0" @@ -1332,12 +1537,49 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + [[package]] name = "num-conv" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1345,6 +1587,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -1408,6 +1651,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "parking" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" + [[package]] name = "parking_lot" version = "0.12.2" @@ -1431,6 +1680,12 @@ dependencies = [ "windows-targets 0.52.5", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pem" version = "3.0.4" @@ -1441,6 +1696,15 @@ dependencies = [ "serde", ] +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -1464,7 +1728,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.63", ] [[package]] @@ -1479,6 +1743,27 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.30" @@ -1639,7 +1924,7 @@ dependencies = [ "quote", "refinery-core", "regex", - "syn 2.0.61", + "syn 2.0.63", ] [[package]] @@ -1738,7 +2023,7 @@ dependencies = [ "cfg-if", "getrandom", "libc", - "spin", + "spin 0.9.8", "untrusted", "windows-sys 0.52.0", ] @@ -1749,6 +2034,26 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" +[[package]] +name = "rsa" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rusqlite" version = "0.31.0" @@ -1766,9 +2071,9 @@ dependencies = [ [[package]] name = "rust-embed" -version = "8.3.0" +version = "8.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb78f46d0066053d16d4ca7b898e9343bc3530f71c61d5ad84cd404ada068745" +checksum = "19549741604902eb99a7ed0ee177a0663ee1eda51a29f71401f166e47e77806a" dependencies = [ "include-flate", "rust-embed-impl", @@ -1778,22 +2083,22 @@ dependencies = [ [[package]] name = "rust-embed-impl" -version = "8.3.0" +version = "8.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91ac2a3c6c0520a3fb3dd89321177c3c692937c4eb21893378219da10c44fc8" +checksum = "cb9f96e283ec64401f30d3df8ee2aaeb2561f34c824381efa24a35f79bf40ee4" dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.61", + "syn 2.0.63", "walkdir", ] [[package]] name = "rust-embed-utils" -version = "8.3.0" +version = "8.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f69089032567ffff4eada41c573fc43ff466c7db7c5688b2e7969584345581" +checksum = "38c74a686185620830701348de757fd36bef4aa9680fd23c49fc539ddcc1af32" dependencies = [ "sha2", "walkdir", @@ -1947,7 +2252,7 @@ checksum = "c5e405930b9796f1c00bee880d03fc7e0bb4b9a11afc776885ffe84320da2865" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.63", ] [[package]] @@ -2066,6 +2371,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core", +] + [[package]] name = "siphasher" version = "1.0.1" @@ -2086,6 +2401,9 @@ name = "smallvec" version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +dependencies = [ + "serde", +] [[package]] name = "socket2" @@ -2097,11 +2415,242 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "spin" version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlformat" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c" +dependencies = [ + "itertools", + "nom", + "unicode_categories", +] + +[[package]] +name = "sqlx" +version = "0.8.0-alpha.0" +source = "git+https://github.com/launchbadge/sqlx#5d6c33ed65cc2d4671a9f569c565ab18f1ea67aa" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.0-alpha.0" +source = "git+https://github.com/launchbadge/sqlx#5d6c33ed65cc2d4671a9f569c565ab18f1ea67aa" +dependencies = [ + "ahash", + "atoi", + "byteorder", + "bytes", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashlink", + "hex", + "indexmap", + "log", + "memchr", + "once_cell", + "paste", + "percent-encoding", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlformat", + "thiserror", + "tokio", + "tokio-stream", + "tracing", + "url", + "webpki-roots", +] + +[[package]] +name = "sqlx-macros" +version = "0.8.0-alpha.0" +source = "git+https://github.com/launchbadge/sqlx#5d6c33ed65cc2d4671a9f569c565ab18f1ea67aa" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.63", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.8.0-alpha.0" +source = "git+https://github.com/launchbadge/sqlx#5d6c33ed65cc2d4671a9f569c565ab18f1ea67aa" +dependencies = [ + "dotenvy", + "either", + "heck 0.5.0", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.63", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.0-alpha.0" +source = "git+https://github.com/launchbadge/sqlx#5d6c33ed65cc2d4671a9f569c565ab18f1ea67aa" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.5.0", + "byteorder", + "bytes", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.0-alpha.0" +source = "git+https://github.com/launchbadge/sqlx#5d6c33ed65cc2d4671a9f569c565ab18f1ea67aa" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.5.0", + "byteorder", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.8.0-alpha.0" +source = "git+https://github.com/launchbadge/sqlx#5d6c33ed65cc2d4671a9f569c565ab18f1ea67aa" +dependencies = [ + "atoi", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "tracing", + "url", +] + +[[package]] +name = "stringprep" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" +dependencies = [ + "finl_unicode", + "unicode-bidi", + "unicode-normalization", +] [[package]] name = "strsim" @@ -2140,9 +2689,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.61" +version = "2.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c993ed8ccba56ae856363b1845da7266a7cb78e1d146c8a32d54b45a8b831fc9" +checksum = "bf5be731623ca1a1fb7d8be6f261a3be6d3e2337b8a1f97be944d020c8fcb704" dependencies = [ "proc-macro2", "quote", @@ -2205,7 +2754,7 @@ checksum = "e2470041c06ec3ac1ab38d0356a6119054dedaea53e12fbefc0de730a1c08524" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.63", ] [[package]] @@ -2292,7 +2841,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.63", ] [[package]] @@ -2305,6 +2854,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.11" @@ -2419,7 +2979,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.63", ] [[package]] @@ -2527,6 +3087,12 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -2608,6 +3174,12 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.92" @@ -2629,7 +3201,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.63", "wasm-bindgen-shared", ] @@ -2663,7 +3235,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.63", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2717,6 +3289,16 @@ version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" +[[package]] +name = "whoami" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" +dependencies = [ + "redox_syscall 0.4.1", + "wasite", +] + [[package]] name = "winapi" version = "0.3.9" @@ -2998,7 +3580,7 @@ checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.61", + "syn 2.0.63", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index c99403f2..3a7a6421 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,9 @@ bcrypt = "0.15.0" bytes = "1.5.0" clap = { version = "4.5.4", features = ["env", "derive", "color"] } +# Trying SQLx, it wrap up all 3 of the other SQLite crates into one. +sqlx = { git = "https://github.com/launchbadge/sqlx", default-features = true, features = ["runtime-tokio", "sqlite", "postgres", "tls-rustls"] } + # These 3 somewhat depend on each other. deadpool-sqlite = { version = "= 0.8.0", features = ["rt_tokio_1"] } rusqlite = { version = "= 0.31", default-features = false, features = ["bundled", "serde_json"] } @@ -72,3 +75,4 @@ directories = "5.0.1" gethostname = "0.4.3" tempfile = "3.9.0" inquire = "0.6.2" +chrono = { version = "0.4.38", default-features = false, features = ["std", "now"] } diff --git a/resources/configdb/migrations/V2__Sessions.sql b/resources/configdb/migrations/V2__Sessions.sql new file mode 100644 index 00000000..34b2f75f --- /dev/null +++ b/resources/configdb/migrations/V2__Sessions.sql @@ -0,0 +1,11 @@ +DROP TABLE IF EXISTS sessions; + +CREATE TABLE sessions ( + rowid INTEGER PRIMARY KEY, + token STRING UNIQUE NOT NULL, + uuid STRING NOT NULL, + expires_at INTEGER NOT NULL, + FOREIGN KEY (uuid) REFERENCES users(uuid) +); + +CREATE INDEX sessions_token_index ON sessions (token); diff --git a/src/bin/evebox.rs b/src/bin/evebox.rs index 4e025310..f0993820 100644 --- a/src/bin/evebox.rs +++ b/src/bin/evebox.rs @@ -263,6 +263,12 @@ async fn evebox_main() -> Result<(), Box> { .action(ArgAction::SetTrue) .help("Don't open browser"), ) + .arg( + Arg::new("fts") + .long("fts") + .action(ArgAction::SetTrue) + .help("Enable full text search"), + ) .arg( Arg::new("no-wait") .long("no-wait") @@ -324,7 +330,7 @@ async fn evebox_main() -> Result<(), Box> { } Some(("oneshot", args)) => evebox::commands::oneshot::main(args).await, Some(("agent", args)) => evebox::commands::agent::main(args).await, - Some(("config", args)) => evebox::commands::config::main(args), + Some(("config", args)) => evebox::commands::config::main(args).await, Some(("print", args)) => evebox::commands::print::main(args), Some(("elastic", args)) => evebox::commands::elastic::main::main(args).await, Some(("sqlite", args)) => evebox::commands::sqlite::main(args).await, diff --git a/src/commands/config/mod.rs b/src/commands/config/mod.rs index 64fed959..7d07af81 100644 --- a/src/commands/config/mod.rs +++ b/src/commands/config/mod.rs @@ -23,9 +23,9 @@ pub fn config_subcommand() -> clap::Command { Args::command() } -pub fn main(args: &clap::ArgMatches) -> anyhow::Result<()> { +pub async fn main(args: &clap::ArgMatches) -> anyhow::Result<()> { match args.subcommand() { - Some(("users", args)) => users::main(args), + Some(("users", args)) => users::main(args).await, _ => Err(anyhow!("no subcommand provided")), } } diff --git a/src/commands/config/users.rs b/src/commands/config/users.rs index 085d5b8c..c249b233 100644 --- a/src/commands/config/users.rs +++ b/src/commands/config/users.rs @@ -53,23 +53,23 @@ struct AddArgs { data_directory: Option, } -pub(crate) fn main(args: &clap::ArgMatches) -> Result<()> { +pub(crate) async fn main(args: &clap::ArgMatches) -> Result<()> { let args = UsersCommands::from_arg_matches(args)?; match args { - UsersCommands::Add(args) => add(args), - UsersCommands::List { data_directory } => list(data_directory), + UsersCommands::Add(args) => add(args).await, + UsersCommands::List { data_directory } => list(data_directory).await, UsersCommands::Rm { username, data_directory, - } => remove(username, data_directory), + } => remove(username, data_directory).await, UsersCommands::Passwd { username, data_directory, - } => password(username, data_directory), + } => password(username, data_directory).await, } } -fn open_config_repo>(data_directory: Option

) -> Result { +async fn open_config_repo>(data_directory: Option

) -> Result { let data_directory = data_directory .map(|p| PathBuf::from(p.as_ref())) .or_else(crate::path::data_directory); @@ -81,21 +81,21 @@ fn open_config_repo>(data_directory: Option

) -> Result) -> Result<()> { - let repo = open_config_repo(dir.as_deref())?; - let users = repo.get_users()?; +async fn list(dir: Option) -> Result<()> { + let repo = open_config_repo(dir.as_deref()).await?; + let users = repo.get_users().await?; for user in users { println!("{}", serde_json::to_string(&user).unwrap()); } Ok(()) } -fn add(args: AddArgs) -> Result<()> { - let repo = open_config_repo(args.data_directory.as_deref())?; +async fn add(args: AddArgs) -> Result<()> { + let repo = open_config_repo(args.data_directory.as_deref()).await?; let username = if let Some(username) = args.username { username.to_string() @@ -117,30 +117,30 @@ fn add(args: AddArgs) -> Result<()> { .prompt()? }; - repo.add_user(&username, &password)?; + repo.add_user(&username, &password).await?; println!("User added: username=\"{username}\""); Ok(()) } -fn remove(username: String, dir: Option) -> Result<()> { - let repo = open_config_repo(dir.as_deref())?; - if repo.remove_user(&username)? == 0 { +async fn remove(username: String, dir: Option) -> Result<()> { + let repo = open_config_repo(dir.as_deref()).await?; + if repo.remove_user(&username).await? == 0 { return Err(anyhow!("user does not exist")); } println!("User removed: username=\"{username}\""); Ok(()) } -fn password(username: String, data_directory: Option) -> Result<()> { - let repo = open_config_repo(data_directory)?; - let user = repo.get_user_by_name(&username)?; +async fn password(username: String, data_directory: Option) -> Result<()> { + let repo = open_config_repo(data_directory).await?; + let user = repo.get_user_by_name(&username).await?; let password = inquire::Password::new("Password:") .with_display_toggle_enabled() .with_display_mode(inquire::PasswordDisplayMode::Masked) .with_validator(inquire::required!()) .prompt()?; - if repo.update_password_by_id(&user.uuid, &password)? { + if repo.update_password_by_id(&user.uuid, &password).await? { println!("Password has been updated."); Ok(()) } else { diff --git a/src/commands/oneshot.rs b/src/commands/oneshot.rs index ed3c36d3..dde7b5dc 100644 --- a/src/commands/oneshot.rs +++ b/src/commands/oneshot.rs @@ -21,6 +21,7 @@ pub async fn main(args: &clap::ArgMatches) -> anyhow::Result<()> { let db_filename: String = config_loader.get("database-filename")?.unwrap(); let host: String = config_loader.get("http.host")?.unwrap(); let input = args.get_one::("INPUT").unwrap().to_string(); + let fts: bool = config_loader.get_bool("fts")?; info!("Using database filename {}", &db_filename); @@ -29,13 +30,13 @@ pub async fn main(args: &clap::ArgMatches) -> anyhow::Result<()> { ))); let mut db = sqlite::ConnectionBuilder::filename(Some(&PathBuf::from(&db_filename))).open(true)?; - let fts = false; sqlite::init_event_db(&mut db)?; - let db = Arc::new(Mutex::new(db)); - let pool = sqlite::pool::open_pool(&db_filename).await?; + let pool = sqlite::connection::open_deadpool(Some(&db_filename))?; + let xpool = sqlite::connection::open_sqlx_pool(Some(&db_filename), false).await?; + let db = crate::sqlite::connection::open_sqlx_connection(Some(&db_filename), true).await?; + let db = Arc::new(tokio::sync::Mutex::new(db)); let import_task = { - let db = db.clone(); tokio::spawn(async move { if let Err(err) = run_import(db, limit, &input, fts).await { error!("Import failure: {}", err); @@ -59,8 +60,19 @@ pub async fn main(args: &clap::ArgMatches) -> anyhow::Result<()> { let mut port = 5636; loop { let connection = Arc::new(Mutex::new(db_connection_builder.open(false).unwrap())); - let sqlite_datastore = - sqlite::eventrepo::SqliteEventRepo::new(connection, pool.clone(), fts); + let xconnection = Arc::new(tokio::sync::Mutex::new( + db_connection_builder + .open_sqlx_connection(false) + .await + .unwrap(), + )); + let sqlite_datastore = sqlite::eventrepo::SqliteEventRepo::new( + xconnection, + connection, + xpool.clone(), + pool.clone(), + fts, + ); let ds = crate::eventrepo::EventRepo::SQLite(sqlite_datastore); let config = crate::server::ServerConfig { port, @@ -139,7 +151,7 @@ pub async fn main(args: &clap::ArgMatches) -> anyhow::Result<()> { } async fn run_import( - db: Arc>, + sqlx: Arc>, limit: u64, input: &str, fts: bool, @@ -148,11 +160,12 @@ async fn run_import( Ok(geoipdb) => Some(geoipdb), Err(_) => None, }; - let mut indexer = sqlite::importer::SqliteEventSink::new(db, fts); + let mut indexer = sqlite::importer::SqliteEventSink::new(sqlx, fts); let mut reader = eve::reader::EveReader::new(input.into()); info!("Reading {} ({} bytes)", input, reader.file_size()); let mut last_percent = 0; let mut count = 0; + let start = std::time::Instant::now(); loop { match reader.next_record() { Ok(None) | Err(_) => { @@ -181,6 +194,7 @@ async fn run_import( } } indexer.commit().await?; - info!("Read {} events", count); + let elapsed = start.elapsed(); + info!("Read {} events in {}s", count, elapsed.as_secs_f64()); Ok(()) } diff --git a/src/commands/sqlite/mod.rs b/src/commands/sqlite/mod.rs index 8279cbce..5ad0546a 100644 --- a/src/commands/sqlite/mod.rs +++ b/src/commands/sqlite/mod.rs @@ -4,8 +4,8 @@ use crate::{ elastic::AlertQueryOptions, sqlite::{ - eventrepo::SqliteEventRepo, info::Info, init_event_db, pool::open_pool, ConnectionBuilder, - SqliteExt, + connection::open_deadpool, eventrepo::SqliteEventRepo, info::Info, init_event_db, + ConnectionBuilder, SqliteExt, }, }; use anyhow::Result; @@ -275,9 +275,17 @@ fn query(filename: &str, sql: &str) -> Result<()> { async fn optimize(args: &OptimizeArgs) -> Result<()> { let conn = ConnectionBuilder::filename(Some(&args.filename)).open(false)?; let conn = Arc::new(Mutex::new(conn)); - let pool = open_pool(&args.filename).await?; + + let xconn = ConnectionBuilder::filename(Some(&args.filename)) + .open_sqlx_connection(false) + .await?; + let xconn = Arc::new(tokio::sync::Mutex::new(xconn)); + + let pool = open_deadpool(Some(&args.filename))?; pool.resize(1); - let repo = SqliteEventRepo::new(conn, pool.clone(), false); + let xpool = crate::sqlite::connection::open_sqlx_pool(Some(&args.filename), false).await?; + // TODO: Set size to 1. + let repo = SqliteEventRepo::new(xconn, conn, xpool.clone(), pool.clone(), false); info!("Running inbox style query"); let gte = time::OffsetDateTime::now_utc() - time::Duration::days(1); diff --git a/src/eventrepo/mod.rs b/src/eventrepo/mod.rs index 8abf741f..f075f8eb 100644 --- a/src/eventrepo/mod.rs +++ b/src/eventrepo/mod.rs @@ -52,6 +52,9 @@ pub enum DatastoreError { #[error("sql: {0}")] FromSql(#[from] rusqlite::types::FromSqlError), + #[error("sqlx: {0}")] + SqlxError(#[from] sqlx::Error), + // Fallback... #[error("error: {0}")] AnyhowError(#[from] anyhow::Error), diff --git a/src/server/api/login.rs b/src/server/api/login.rs index c156f4e3..d3acb8d1 100644 --- a/src/server/api/login.rs +++ b/src/server/api/login.rs @@ -77,6 +77,21 @@ pub(crate) async fn post( session.username = Some(user.username); let session = Arc::new(session); context.session_store.put(session.clone()).unwrap(); + + // Create expiry data one week in the future. + let expiry = chrono::Utc::now() + chrono::Duration::weeks(1); + if let Err(err) = context + .config_repo + .save_session( + session.session_id.as_ref().unwrap(), + &user.uuid, + expiry.timestamp(), + ) + .await + { + error!("Failed to save session: {:?}", err); + } + ( StatusCode::OK, Json(serde_json::json!({ @@ -87,7 +102,7 @@ pub(crate) async fn post( } } -pub(crate) async fn logout_new( +pub(crate) async fn logout( context: Extension>, SessionExtractor(session): SessionExtractor, ) -> impl IntoResponse { diff --git a/src/server/main.rs b/src/server/main.rs index af59e979..e191af2b 100644 --- a/src/server/main.rs +++ b/src/server/main.rs @@ -121,9 +121,9 @@ pub async fn main(args: &clap::ArgMatches) -> Result<()> { let datastore = configure_datastore(config.clone(), &server_config).await?; let mut context = build_context(server_config.clone(), datastore).await?; - if server_config.authentication_required && !context.config_repo.has_users()? { + if server_config.authentication_required && !context.config_repo.has_users().await? { warn!("Username/password authentication is required, but no users exist, creating a user"); - let (username, password) = create_admin_user(&context)?; + let (username, password) = create_admin_user(&context).await?; warn!( "Created administrator username and password: username={}, password={}", username, password @@ -268,7 +268,7 @@ fn is_authentication_required(config: &Config) -> bool { true } -fn create_admin_user(context: &ServerContext) -> Result<(String, String)> { +async fn create_admin_user(context: &ServerContext) -> Result<(String, String)> { use rand::Rng; let rng = rand::thread_rng(); let username = "admin"; @@ -277,7 +277,7 @@ fn create_admin_user(context: &ServerContext) -> Result<(String, String)> { .take(12) .map(char::from) .collect(); - context.config_repo.add_user(username, &password)?; + context.config_repo.add_user(username, &password).await?; Ok((username.to_string(), password)) } @@ -322,7 +322,7 @@ pub(crate) fn build_axum_service( "/api/1/login", post(api::login::post).get(api::login::options), ) - .route("/api/1/logout", post(api::login::logout_new)) + .route("/api/1/logout", post(api::login::logout)) .route("/api/1/config", get(api::config)) .route("/api/1/version", get(api::get_version)) .route("/api/1/user", get(api::get_user)) @@ -463,10 +463,10 @@ pub(crate) async fn build_context( let config_repo = if let Some(directory) = &config.data_directory { let filename = PathBuf::from(directory).join("config.sqlite"); info!("Configuration database filename: {:?}", filename); - ConfigRepo::new(Some(&filename))? + ConfigRepo::new(Some(&filename)).await? } else { info!("Using temporary in-memory configuration database"); - ConfigRepo::new(None)? + ConfigRepo::new(None).await? }; let mut context = ServerContext::new(config, Arc::new(config_repo), datastore); @@ -561,13 +561,21 @@ async fn configure_datastore(config: Config, server_config: &ServerConfig) -> Re let mut connection = connection_builder.open(true).unwrap(); sqlite::init_event_db(&mut connection)?; + let xconn = connection_builder.open_sqlx_connection(true).await.unwrap(); + let xconn = Arc::new(tokio::sync::Mutex::new(xconn)); let has_fts = connection.has_table("fts")?; info!("FTS enabled: {has_fts}"); let connection = Arc::new(Mutex::new(connection)); - let pool = sqlite::pool::open_pool(&db_filename).await?; - - let eventstore = - sqlite::eventrepo::SqliteEventRepo::new(connection.clone(), pool, has_fts); + let pool = sqlite::connection::open_deadpool(Some(&db_filename))?; + let xpool = sqlite::connection::open_sqlx_pool(Some(&db_filename), false).await?; + + let eventstore = sqlite::eventrepo::SqliteEventRepo::new( + xconn.clone(), + connection.clone(), + xpool, + pool, + has_fts, + ); // Start retention task. sqlite::retention::start_retention_task(config.clone(), connection.clone())?; @@ -637,6 +645,25 @@ where if let Some(session) = session { return Ok(SessionExtractor(session)); } + + debug!("Session not found in cache, checking database"); + + match context.config_repo.get_user_by_session(&session_id).await { + Ok(Some(user)) => { + info!("Found session for user {}", &user.username); + let session = Session { + session_id: Some(session_id.to_string()), + username: Some(user.username), + }; + let session = Arc::new(session); + let _ = context.session_store.put(session.clone()); + return Ok(SessionExtractor(session)); + } + Ok(None) => {} + Err(err) => { + error!("Failed to get user by session from database: {:?}", err); + } + } } use axum::headers::authorization::Basic; diff --git a/src/server/session.rs b/src/server/session.rs index feecdfe0..5a846e43 100644 --- a/src/server/session.rs +++ b/src/server/session.rs @@ -44,7 +44,7 @@ impl SessionStore { } } -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub(crate) struct Session { pub session_id: Option, pub username: Option, diff --git a/src/sqlite/configrepo.rs b/src/sqlite/configrepo.rs index c68e31e6..1d40f8cc 100644 --- a/src/sqlite/configrepo.rs +++ b/src/sqlite/configrepo.rs @@ -2,18 +2,19 @@ // SPDX-License-Identifier: MIT use std::path::PathBuf; -use std::sync::Arc; -use std::sync::Mutex; use rusqlite::params; +use serde::Serialize; +use sqlx::Row; use time::format_description::well_known::Rfc3339; use tracing::debug; use tracing::error; +use tracing::info; use crate::sqlite::ConnectionBuilder; #[derive(thiserror::Error, Debug)] -pub enum ConfigRepoError { +pub(crate) enum ConfigRepoError { #[error("username not found: {0}")] UsernameNotFound(String), #[error("bad password for user: {0}")] @@ -26,24 +27,26 @@ pub enum ConfigRepoError { JoinError(#[from] tokio::task::JoinError), #[error("user does not exist: {0}")] NoUser(String), + #[error("sql error: {0}")] + SqlxError(#[from] sqlx::Error), } -#[derive(Debug, Clone, serde::Serialize)] +#[derive(Debug, Clone, Serialize)] pub(crate) struct User { pub uuid: String, pub username: String, } pub(crate) struct ConfigRepo { - pub db: Arc>, + pool: sqlx::Pool, } impl ConfigRepo { - pub fn new(filename: Option<&PathBuf>) -> Result { + pub async fn new(filename: Option<&PathBuf>) -> Result { let mut conn = ConnectionBuilder::filename(filename).open(true)?; init_db(&mut conn)?; Ok(Self { - db: Arc::new(Mutex::new(conn)), + pool: crate::sqlite::connection::open_sqlx_pool(filename, false).await?, }) } @@ -52,105 +55,159 @@ impl ConfigRepo { username: &str, password_in: &str, ) -> Result { - let username = username.to_string(); - let password_in = password_in.to_string(); - let db = self.db.clone(); - tokio::task::spawn_blocking(move || { - let conn = db.lock().unwrap(); - let mut stmt = - conn.prepare("SELECT uuid, username, password FROM users WHERE username = ?1")?; - let mut rows = stmt.query(params![username])?; - if let Some(row) = rows.next()? { - let uuid: String = row.get(0)?; - let username: String = row.get(1)?; - let password_hash: String = row.get(2)?; - if bcrypt::verify(password_in, &password_hash)? { - Ok(User { uuid, username }) - } else { - Err(ConfigRepoError::BadPassword(username)) - } + let query = sqlx::query::( + "SELECT uuid, username, password FROM users WHERE username = ?", + ) + .bind(username); + if let Some(row) = query.fetch_optional(&self.pool).await? { + let uuid: String = row.try_get(0)?; + let username: String = row.try_get(1)?; + let password_hash: String = row.try_get(2)?; + if bcrypt::verify(password_in, &password_hash)? { + return Ok(User { uuid, username }); } else { - Err(ConfigRepoError::UsernameNotFound(username)) + return Err(ConfigRepoError::BadPassword(username)); } - }) - .await? + } + + Err(ConfigRepoError::UsernameNotFound(username.to_string())) } - pub fn get_user_by_name(&self, username: &str) -> Result { - let conn = self.db.lock().unwrap(); - let user = conn - .query_row( - "SELECT uuid, username FROM users WHERE username = ?", - params![username], - |row| { - Ok(User { - uuid: row.get(0)?, - username: row.get(1)?, - }) - }, - ) - .map_err(|err| match err { - rusqlite::Error::QueryReturnedNoRows => { - ConfigRepoError::NoUser(username.to_string()) - } - _ => err.into(), - })?; - Ok(user) + pub async fn get_user_by_name(&self, username: &str) -> Result { + let row = sqlx::query("SELECT uuid, username FROM users WHERE username = ?") + .bind(username) + .fetch_optional(&self.pool) + .await?; + if let Some(row) = row { + Ok(User { + uuid: row.try_get("uuid")?, + username: row.try_get("username")?, + }) + } else { + Err(ConfigRepoError::NoUser(username.to_string())) + } } - pub fn has_users(&self) -> Result { - let conn = self.db.lock().unwrap(); - let count: u64 = conn.query_row("SELECT count(*) FROM users", [], |row| row.get(0))?; + pub async fn has_users(&self) -> Result { + let (count,): (u64,) = sqlx::query_as("SELECT count(*) FROM users") + .fetch_one(&self.pool) + .await?; Ok(count > 0) } - pub fn get_users(&self) -> Result, ConfigRepoError> { - let conn = self.db.lock().unwrap(); - let mut stmt = conn.prepare("SELECT uuid, username FROM users")?; - let rows = stmt.query_map(params![], |row| { - Ok(User { - uuid: row.get(0)?, - username: row.get(1)?, + pub async fn get_users(&self) -> Result, ConfigRepoError> { + let rows: Vec<(String, String)> = sqlx::query_as("SELECT uuid, username FROM users") + .fetch_all(&self.pool) + .await?; + Ok(rows + .into_iter() + .map(|row| User { + uuid: row.0, + username: row.1, }) - })?; - let mut users = Vec::new(); - for row in rows { - users.push(row?); - } - Ok(users) + .collect()) } - pub fn add_user(&self, username: &str, password: &str) -> Result { + pub async fn add_user( + &self, + username: &str, + password: &str, + ) -> Result { let password_hash = bcrypt::hash(password, bcrypt::DEFAULT_COST)?; let user_id = uuid::Uuid::new_v4().to_string(); - let mut conn = self.db.lock().unwrap(); - let tx = conn.transaction()?; - tx.execute( - "INSERT INTO users (uuid, username, password) VALUES (?, ?, ?)", - params![user_id, username, password_hash], - )?; - tx.commit()?; + sqlx::query("INSERT INTO users (uuid, username, password) VALUES (?, ?, ?)") + .bind(&user_id) + .bind(username) + .bind(password_hash) + .execute(&self.pool) + .await?; Ok(user_id) } - pub fn remove_user(&self, username: &str) -> Result { - let mut conn = self.db.lock().unwrap(); - let tx = conn.transaction()?; - let n = tx.execute("DELETE FROM users WHERE username = ?", params![username])?; - tx.commit()?; - Ok(n) + pub async fn remove_user(&self, username: &str) -> Result { + Ok(sqlx::query("DELETE FROM users WHERE username = ?") + .bind(username) + .execute(&self.pool) + .await? + .rows_affected()) } - pub fn update_password_by_id(&self, id: &str, password: &str) -> Result { + pub async fn update_password_by_id( + &self, + id: &str, + password: &str, + ) -> Result { let password_hash = bcrypt::hash(password, bcrypt::DEFAULT_COST)?; - let mut conn = self.db.lock().unwrap(); - let tx = conn.transaction()?; - let n = tx.execute( - "UPDATE users SET password = ? where uuid = ?", - params![password_hash, id], - )?; - tx.commit()?; - Ok(n > 0) + let result = sqlx::query("UPDATE users SET password = ? WHERE uuid = ?") + .bind(&password_hash) + .bind(id) + .execute(&self.pool) + .await?; + Ok(result.rows_affected() > 0) + } + + pub async fn save_session( + &self, + token: &str, + uuid: &str, + expires: i64, + ) -> Result<(), ConfigRepoError> { + let sql = "INSERT INTO sessions (token, uuid, expires_at) VALUES (?, ?, ?)"; + sqlx::query(sql) + .bind(token) + .bind(uuid) + .bind(expires) + .execute(&self.pool) + .await?; + Ok(()) + } + + async fn expire_sessions(&self) -> Result { + let now = time::OffsetDateTime::now_utc().unix_timestamp(); + let result = sqlx::query("DELETE FROM sessions WHERE expires AT < ?") + .bind(now) + .execute(&self.pool) + .await?; + Ok(result.rows_affected()) + } + + pub async fn get_user_by_session(&self, token: &str) -> Result, ConfigRepoError> { + let sql = r#" + SELECT users.uuid, users.username, sessions.expires_at + FROM users + JOIN sessions ON + users.uuid = sessions.uuid + WHERE sessions.token = ?"#; + + // TODO: Remove transaction, not needed here but used as an example. + let mut tx = self.pool.begin().await?; + if let Some(row) = sqlx::query(sql) + .bind(token) + .fetch_optional(&mut *tx) + .await? + { + let uuid: String = row.try_get("uuid")?; + let username: String = row.try_get("username")?; + let expires_at: i64 = row.try_get("expires_at")?; + + let now = time::OffsetDateTime::now_utc().unix_timestamp(); + if now > expires_at { + match self.expire_sessions().await { + Ok(n) => { + if n > 0 { + info!("Expired {} sessions", n); + } + } + Err(err) => { + error!("Failed to expire sessions: {:?}", err); + } + } + return Ok(None); + } + tx.commit().await?; + return Ok(Some(User { uuid, username })); + } + Ok(None) } } diff --git a/src/sqlite/connection.rs b/src/sqlite/connection.rs index d6e0b627..0a541bbc 100644 --- a/src/sqlite/connection.rs +++ b/src/sqlite/connection.rs @@ -2,7 +2,9 @@ // SPDX-License-Identifier: MIT use crate::sqlite::{info::Info, util::fts_create, SqliteExt}; +use deadpool_sqlite::CreatePoolError; use rusqlite::{params, Connection, DatabaseName, OpenFlags}; +use sqlx::SqliteConnection; use std::path::PathBuf; use time::format_description::well_known::Rfc3339; use tracing::{debug, error, info, warn}; @@ -27,12 +29,76 @@ impl ConnectionBuilder { debug!("Opening database {}", filename.display()); rusqlite::Connection::open_with_flags(filename, flags) } else { - rusqlite::Connection::open_in_memory() + rusqlite::Connection::open("file::memory:?cache=shared") } } + + pub async fn open_sqlx_connection( + &self, + create: bool, + ) -> Result { + open_sqlx_connection(self.filename.clone(), create).await + } +} + +pub(crate) async fn open_sqlx_connection( + path: Option>, + create: bool, +) -> Result { + use sqlx::sqlite::SqliteConnectOptions; + use sqlx::sqlite::SqliteConnection; + use sqlx::Connection; + + let path = path + .map(|p| p.into()) + .unwrap_or_else(|| "file::memory:?cache=shared".into()); + + let options = SqliteConnectOptions::new() + .filename(path) + .shared_cache(true) + .create_if_missing(create); + SqliteConnection::connect_with(&options).await +} + +pub(crate) async fn open_sqlx_pool( + path: Option>, + create: bool, +) -> Result, sqlx::Error> { + use sqlx::sqlite::SqliteConnectOptions; + use sqlx::sqlite::SqlitePoolOptions; + + let path = path + .map(|p| p.into()) + .unwrap_or_else(|| "file::memory:?cache=shared".into()); + + let pool = SqlitePoolOptions::new() + .min_connections(4) + .max_connections(12); + + let options = SqliteConnectOptions::new() + .filename(path) + .shared_cache(true) + .create_if_missing(create); + pool.connect_with(options).await +} + +/// Open an SQLite connection pool with deadpool. +/// +/// Fortunately SQLites default connection options are good enough as +/// deadpool does not provide a way to customize them. See +/// https://github.com/bikeshedder/deadpool/issues/214. +pub(crate) fn open_deadpool>( + path: Option

, +) -> Result { + let path = path + .map(|p| p.into()) + .unwrap_or_else(|| "file::memory:?cache=shared".into()); + let config = deadpool_sqlite::Config::new(path); + let pool = config.create_pool(deadpool_sqlite::Runtime::Tokio1)?; + Ok(pool) } -pub fn init_event_db(db: &mut Connection) -> Result<(), rusqlite::Error> { +pub(crate) fn init_event_db(db: &mut Connection) -> Result<(), rusqlite::Error> { let auto_vacuum = Info::new(db).get_auto_vacuum()?; if auto_vacuum == 2 { info!("Change auto-vacuum from incremental to full"); diff --git a/src/sqlite/eventrepo.rs b/src/sqlite/eventrepo.rs index 656ef98f..40b5808d 100644 --- a/src/sqlite/eventrepo.rs +++ b/src/sqlite/eventrepo.rs @@ -22,6 +22,7 @@ pub(crate) struct SqliteEventRepo { pub connection: Arc>, pub importer: super::importer::SqliteEventSink, pub pool: deadpool_sqlite::Pool, + pub xpool: sqlx::Pool, pub fts: bool, } @@ -29,11 +30,18 @@ pub(crate) struct SqliteEventRepo { type QueryParam = dyn ToSql + Send + Sync + 'static; impl SqliteEventRepo { - pub fn new(connection: Arc>, pool: deadpool_sqlite::Pool, fts: bool) -> Self { + pub fn new( + xdb: Arc>, + connection: Arc>, + xpool: sqlx::Pool, + pool: deadpool_sqlite::Pool, + fts: bool, + ) -> Self { debug!("SQLite event store created: fts={fts}"); Self { connection: connection.clone(), - importer: super::importer::SqliteEventSink::new(connection, fts), + importer: super::importer::SqliteEventSink::new(xdb, fts), + xpool, pool, fts, } @@ -44,71 +52,45 @@ impl SqliteEventRepo { } pub async fn min_row_id(&self) -> Result { - Ok(self - .pool - .get() + let (id,): (u64,) = sqlx::query_as("SELECT MIN(rowid) FROM events") + .fetch_optional(&self.xpool) .await? - .interact(|conn| { - conn.query_row_and_then("select min(rowid) from events", [], |row| row.get(0)) - .unwrap_or(0) - }) - .await?) + .unwrap_or((0,)); + Ok(id) } pub async fn max_row_id(&self) -> Result { - Ok(self - .pool - .get() + let (id,): (u64,) = sqlx::query_as("SELECT MAX(rowid) FROM events") + .fetch_optional(&self.xpool) .await? - .interact(|conn| { - conn.query_row_and_then("select max(rowid) from events", [], |row| row.get(0)) - .unwrap_or(0) - }) - .await?) + .unwrap_or((0,)); + Ok(id) } pub async fn min_timestamp(&self) -> Result, DatastoreError> { - let timestamp = self - .pool - .get() - .await? - .interact(|conn| { - conn.query_row_and_then( - "select min(timestamp) from events", - [], - |row| -> anyhow::Result { - let timestamp: i64 = row.get(0)?; - Ok(time::OffsetDateTime::from_unix_timestamp_nanos( - timestamp as i128, - )?) - }, - ) - }) - .await? - .ok(); - Ok(timestamp) + let result: Option<(i64,)> = sqlx::query_as("SELECT MIN(timestamp) FROM events") + .fetch_optional(&self.xpool) + .await?; + if let Some((ts,)) = result { + Ok(Some(time::OffsetDateTime::from_unix_timestamp_nanos( + ts as i128, + )?)) + } else { + Ok(None) + } } pub async fn max_timestamp(&self) -> Result, DatastoreError> { - let timestamp = self - .pool - .get() - .await? - .interact(|conn| { - conn.query_row_and_then( - "select max(timestamp) from events", - [], - |row| -> anyhow::Result { - let timestamp: i64 = row.get(0)?; - Ok(time::OffsetDateTime::from_unix_timestamp_nanos( - timestamp as i128, - )?) - }, - ) - }) - .await? - .ok(); - Ok(timestamp) + let result: Option<(i64,)> = sqlx::query_as("SELECT MAX(timestamp) FROM events") + .fetch_optional(&self.xpool) + .await?; + if let Some((ts,)) = result { + Ok(Some(time::OffsetDateTime::from_unix_timestamp_nanos( + ts as i128, + )?)) + } else { + Ok(None) + } } pub async fn get_event_by_id( diff --git a/src/sqlite/importer.rs b/src/sqlite/importer.rs index c6ede909..4920b821 100644 --- a/src/sqlite/importer.rs +++ b/src/sqlite/importer.rs @@ -3,10 +3,8 @@ use super::builder::SqliteValue; use crate::eve::{self, Eve}; -use rusqlite::TransactionBehavior; -use std::error::Error; -use std::sync::{Arc, Mutex}; -use std::time::Instant; +use sqlx::Connection; +use std::sync::Arc; use time::macros::format_description; use tracing::{debug, error}; @@ -16,8 +14,13 @@ pub enum IndexError { TimestampParseError, #[error("event has no timestamp field")] TimestampMissing, - #[error("sqlite error: {0}")] - SQLiteError(#[from] rusqlite::Error), +} + +struct PreparedEvent { + ts: i64, + archived: u8, + source_values: String, + event: String, } pub(crate) struct QueuedStatement { @@ -26,17 +29,17 @@ pub(crate) struct QueuedStatement { } pub(crate) struct SqliteEventSink { - conn: Arc>, - queue: Vec, + xconn: Arc>, + xqueue: Vec, fts: bool, } impl Clone for SqliteEventSink { fn clone(&self) -> Self { Self { - conn: self.conn.clone(), - queue: Vec::new(), + xconn: self.xconn.clone(), fts: self.fts, + xqueue: Vec::new(), } } } @@ -46,9 +49,7 @@ pub(crate) fn prepare_sql( event: &mut serde_json::Value, fts: bool, ) -> Result, IndexError> { - let ts = event - .timestamp() - .ok_or_else(|| IndexError::TimestampMissing)?; + let ts = event.timestamp().ok_or(IndexError::TimestampMissing)?; reformat_timestamps(event); let source_values = extract_values(event); let mut archived = 0; @@ -99,77 +100,91 @@ pub(crate) fn prepare_sql( } impl SqliteEventSink { - pub fn new(conn: Arc>, fts: bool) -> Self { + pub fn new(xconn: Arc>, fts: bool) -> Self { Self { - conn, - queue: Vec::new(), + xconn, fts, + xqueue: Vec::new(), } } - pub async fn submit(&mut self, mut event: serde_json::Value) -> Result { - for statement in prepare_sql(&mut event, self.fts)? { - self.queue.push(statement); + fn prep(&mut self, event: &mut serde_json::Value) -> Result<(), IndexError> { + let ts = event.timestamp().ok_or(IndexError::TimestampMissing)?; + reformat_timestamps(event); + let source_values = extract_values(event); + let mut archived = 0; + + if let Some(actions) = event["alert"]["metadata"]["evebox-action"].as_array() { + for action in actions { + if let serde_json::Value::String(action) = action { + if action == "archive" { + archived = 1; + break; + } + } + } } + + let prepared = PreparedEvent { + ts: ts.unix_timestamp_nanos() as i64, + source_values, + event: event.to_string(), + archived, + }; + + self.xqueue.push(prepared); + + Ok(()) + } + + pub async fn submit(&mut self, mut event: serde_json::Value) -> Result { + self.prep(&mut event)?; Ok(false) } pub async fn commit(&mut self) -> anyhow::Result { - let mut conn = self.conn.lock().unwrap(); - loop { - match conn.transaction_with_behavior(TransactionBehavior::Immediate) { - Err(err) => { - error!("Failed to start transaction, will try again: {}", err); - std::thread::sleep(std::time::Duration::from_secs(1)); - } - Ok(tx) => { - let start = Instant::now(); - let n = self.queue.len(); - for r in &self.queue { - // Run the execute in a loop as we can get lock errors here as well. - // - // TODO: Break out to own function, but need to replace (or get rid of) - // the mutex with an async aware mutex. - loop { - let mut st = tx.prepare_cached(&r.statement)?; - match st.execute(rusqlite::params_from_iter(&r.params)) { - Ok(_) => { - break; - } - Err(err) => { - error!("Insert statement failed: {err}"); - if err.to_string().contains("locked") { - std::thread::sleep(std::time::Duration::from_millis(10)); - } else { - return Err(anyhow!("execute: {:?}", err)); - } - } - } - } - } - if let Err(err) = tx.commit() { - let source = err.source(); - error!( - "Failed to commit events: error={}, source={:?}", - err, source - ); - return Err(IndexError::SQLiteError(err).into()); - } else { - let count = if self.fts { n / 2 } else { n }; - debug!( - "Committed {count} events in {} ms", - start.elapsed().as_millis() - ); - } - self.queue.truncate(0); - return Ok(n); - } + debug!("Committing {} events with sqlx", self.xqueue.len()); + let mut xconn = self.xconn.lock().await; + let mut tx = xconn.begin().await.unwrap(); + + for event in &self.xqueue { + let _result = sqlx::query::( + r#" + INSERT INTO events (timestamp, archived, source, source_values) + VALUES (?, ?, ?, ?) + "#, + ) + .bind(event.ts) + .bind(event.archived) + .bind(&event.event) + .bind(&event.source_values) + .execute(&mut *tx) + .await + .unwrap(); + + if self.fts { + let _result = sqlx::query::( + r#" + INSERT INTO fts (rowid, timestamp, source_values) + VALUES (last_insert_rowid(), ?, ?)"#, + ) + .bind(event.ts) + .bind(&event.source_values) + .execute(&mut *tx) + .await + .unwrap(); } } + + tx.commit().await.unwrap(); + + let n = self.xqueue.len(); + self.xqueue.truncate(0); + Ok(n) } pub fn pending(&self) -> usize { - self.queue.len() + self.xqueue.len() } } diff --git a/src/sqlite/mod.rs b/src/sqlite/mod.rs index 3fa129b1..b91f5251 100644 --- a/src/sqlite/mod.rs +++ b/src/sqlite/mod.rs @@ -7,11 +7,10 @@ pub mod connection; pub mod eventrepo; pub mod importer; pub(crate) mod info; -pub mod pool; pub mod retention; pub mod util; -pub use connection::init_event_db; +pub(crate) use connection::init_event_db; pub(crate) use connection::ConnectionBuilder; use rusqlite::params; use rusqlite::OptionalExtension; diff --git a/src/sqlite/pool.rs b/src/sqlite/pool.rs deleted file mode 100644 index afac2cd9..00000000 --- a/src/sqlite/pool.rs +++ /dev/null @@ -1,11 +0,0 @@ -// SPDX-FileCopyrightText: (C) 2020 Jason Ish -// SPDX-License-Identifier: MIT - -use std::path::PathBuf; - -pub async fn open_pool>(filename: T) -> anyhow::Result { - use deadpool_sqlite::{Config, Runtime}; - let config = Config::new(filename); - let pool = config.create_pool(Runtime::Tokio1)?; - Ok(pool) -}