-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Began migration to fully usable database
- Loading branch information
1 parent
e35eca5
commit dcc5eb6
Showing
8 changed files
with
243 additions
and
60 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| package lightdb | ||
|
|
||
| import fabric.rw.RW | ||
|
|
||
| abstract class Collection[D <: Document[D]](implicit val rw: RW[D]) { | ||
| protected lazy val defaultCollectionName: String = getClass.getSimpleName.replace("$", "") | ||
| protected lazy val store: Store = db.createStore(collectionName) | ||
| protected lazy val indexStore: Store = db.createStore(s"$collectionName.indexes") | ||
|
|
||
| protected def db: LightDB | ||
| protected def collectionName: String = defaultCollectionName | ||
|
|
||
| // TODO: Triggers | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| package lightdb | ||
|
|
||
| trait Document[D <: Document[D]] { | ||
| def _id: Id[D] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| package lightdb | ||
|
|
||
| import fabric.rw._ | ||
|
|
||
| class Id[T](val value: String) extends AnyVal { | ||
| def bytes: Array[Byte] = { | ||
| val b = toString.getBytes("UTF-8") | ||
| assert(b.length <= 128, s"Must be 128 bytes or less, but was ${b.length} ($value)") | ||
| b | ||
| } | ||
|
|
||
| override def toString: String = value | ||
| } | ||
|
|
||
| object Id { | ||
| private lazy val _rw: RW[Id[_]] = RW.string(_.value, Id.apply) | ||
|
|
||
| implicit def rw[T]: RW[Id[T]] = _rw.asInstanceOf[RW[Id[T]]] | ||
|
|
||
| def apply[T](value: String = Unique()): Id[T] = new Id[T](value) | ||
|
|
||
| def toString[T](id: Id[T]): String = id.value | ||
|
|
||
| def fromString[T](s: String): Id[T] = apply[T](s) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| package lightdb | ||
|
|
||
| import java.nio.file.Path | ||
|
|
||
| abstract class LightDB(directory: Path, | ||
| indexThreads: Int = 2, | ||
| maxFileSize: Int = 1024 * 1024) { | ||
|
|
||
| private var stores = List.empty[Store] | ||
|
|
||
| protected[lightdb] def createStore(name: String): Store = synchronized { | ||
| // TODO: verifyInitialized() | ||
| val store = Store(directory.resolve(name), indexThreads, maxFileSize) | ||
| stores = store :: stores | ||
| store | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| package lightdb | ||
|
|
||
| import cats.effect.IO | ||
| import com.oath.halodb.{HaloDB, HaloDBOptions} | ||
| import fabric.io.{JsonFormatter, JsonParser} | ||
| import fabric.rw._ | ||
|
|
||
| import java.nio.file.{Files, Path} | ||
| import scala.jdk.CollectionConverters._ | ||
|
|
||
| case class Store(directory: Path, | ||
| indexThreads: Int, | ||
| maxFileSize: Int) { | ||
| private lazy val instance: HaloDB = { | ||
| val opts = new HaloDBOptions | ||
| opts.setBuildIndexThreads(indexThreads) | ||
| opts.setMaxFileSize(maxFileSize) | ||
|
|
||
| Files.createDirectories(directory.getParent) | ||
| HaloDB.open(directory.toAbsolutePath.toString, opts) | ||
| } | ||
|
|
||
| def keyStream[D]: fs2.Stream[IO, Id[D]] = fs2.Stream.fromBlockingIterator[IO](instance.newKeyIterator().asScala, 1024) | ||
| .map { record => | ||
| Id[D](record.getBytes.string) | ||
| } | ||
|
|
||
| def stream[D]: fs2.Stream[IO, (Id[D], Array[Byte])] = fs2.Stream.fromBlockingIterator[IO](instance.newIterator().asScala, 1024) | ||
| .map { record => | ||
| val key = record.getKey.string | ||
| Id[D](key) -> record.getValue | ||
| } | ||
|
|
||
| def get[D](id: Id[D]): IO[Option[Array[Byte]]] = IO { | ||
| Option(instance.get(id.bytes)) | ||
| } | ||
|
|
||
| def getJson[D: RW](id: Id[D]): IO[Option[D]] = get(id) | ||
| .map(_.map { bytes => | ||
| val jsonString = bytes.string | ||
| val json = JsonParser(jsonString) | ||
| json.as[D] | ||
| }) | ||
|
|
||
| def put[D](id: Id[D], value: Array[Byte]): IO[Boolean] = IO { | ||
| instance.put(id.bytes, value) | ||
| } | ||
|
|
||
| def putJson[D <: Document[D]](doc: D) | ||
| (implicit rw: RW[D]): IO[D] = IO { | ||
| val json = doc.json | ||
| JsonFormatter.Compact(json) | ||
| }.flatMap { jsonString => | ||
| put(doc._id, jsonString.getBytes).map(_ => doc) | ||
| } | ||
|
|
||
| def delete[D](id: Id[D]): IO[Unit] = IO { | ||
| instance.delete(id.bytes) | ||
| } | ||
|
|
||
| def size: IO[Int] = IO(instance.size().toInt) | ||
|
|
||
| def truncate(): IO[Unit] = keyStream[Any] | ||
| .evalMap { id => | ||
| delete(id) | ||
| } | ||
| .compile | ||
| .drain | ||
| .flatMap { _ => | ||
| size.flatMap { | ||
| case 0 => IO.unit | ||
| case _ => truncate() | ||
| } | ||
| } | ||
|
|
||
| def dispose(): IO[Unit] = IO(instance.close()) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| package lightdb | ||
|
|
||
| import java.security.SecureRandom | ||
| import java.util.concurrent.ThreadLocalRandom | ||
|
|
||
| /** | ||
| * Unique String generator | ||
| */ | ||
| object Unique { | ||
| lazy val LettersLower = "abcdefghijklmnopqrstuvwxyz" | ||
| lazy val LettersUpper = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" | ||
| lazy val Numbers = "0123456789" | ||
| lazy val Readable = "ABCDEFGHJKLMNPQRSTWXYZ23456789" | ||
| lazy val Hexadecimal = s"${Numbers}abcdef" | ||
| lazy val LettersAndNumbers = s"$LettersLower$Numbers" | ||
| lazy val AllLettersAndNumbers = s"$LettersLower$LettersUpper$Numbers" | ||
|
|
||
| private lazy val secure = new SecureRandom | ||
|
|
||
| /** | ||
| * Random number generator used to generate unique values. Defaults to `threadLocalRandom`. | ||
| */ | ||
| var random: Int => Int = threadLocalRandom | ||
|
|
||
| /** | ||
| * The default length to use for generating unique values. Defaults to 32. | ||
| */ | ||
| var defaultLength: Int = 32 | ||
|
|
||
| /** | ||
| * The default characters to use for generating unique values. Defaults to AllLettersAndNumbers. | ||
| */ | ||
| var defaultCharacters: String = AllLettersAndNumbers | ||
|
|
||
| /** | ||
| * True if randomization should be secure. Defaults to false. | ||
| */ | ||
| var defaultSecure: Boolean = false | ||
|
|
||
| /** | ||
| * Uses java.util.concurrent.ThreadLocalRandom to generate random numbers. | ||
| * | ||
| * @param max the maximum value to include | ||
| * @return random number between 0 and max | ||
| */ | ||
| final def threadLocalRandom(max: Int): Int = ThreadLocalRandom.current().nextInt(max) | ||
|
|
||
| final def secureRandom(max: Int): Int = synchronized { | ||
| secure.nextInt(max) | ||
| } | ||
|
|
||
| /** | ||
| * Generates a unique String using the characters supplied at the length defined. | ||
| * | ||
| * @param length the length of the resulting String. Defaults to Unique.defaultLength. | ||
| * @param characters the characters for use in the String. Defaults to Unique.defaultCharacters. | ||
| * @param secure true if the randomization should be secure. Defaults to Unique.defaultSecure. | ||
| * @return a unique String | ||
| */ | ||
| def apply(length: Int = defaultLength, characters: String = defaultCharacters, secure: Boolean = defaultSecure): String = { | ||
| val charMax = characters.length | ||
| val r = if (secure) secureRandom _ else random | ||
| (0 until length).map(i => characters.charAt(r(charMax))).mkString | ||
| } | ||
|
|
||
| /** | ||
| * Convenience functionality to generate a UUID (https://en.wikipedia.org/wiki/Universally_unique_identifier) | ||
| * | ||
| * 32 characters of unique hexadecimal values with dashes representing 36 total characters | ||
| */ | ||
| def uuid(secure: Boolean = false): String = { | ||
| val a = apply(8, Hexadecimal, secure) | ||
| val b = apply(4, Hexadecimal, secure) | ||
| val c = apply(3, Hexadecimal, secure) | ||
| val d = apply(1, "89ab", secure) | ||
| val e = apply(3, Hexadecimal, secure) | ||
| val f = apply(12, Hexadecimal, secure) | ||
| s"$a-$b-4$c-$d$e-$f" | ||
| } | ||
|
|
||
| /** | ||
| * Returns the number of possible values for a specific length and characters. | ||
| */ | ||
| def poolSize(length: Int = 32, characters: String = AllLettersAndNumbers): Long = { | ||
| math.pow(characters.length, length).toLong | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| package object lightdb { | ||
| implicit class ByteArrayExtras(val bytes: Array[Byte]) extends AnyVal { | ||
| def string: String = new String(bytes, "UTF-8") | ||
| } | ||
| } |