diff --git a/build.sbt b/build.sbt index e8d1d1e9..08679f41 100644 --- a/build.sbt +++ b/build.sbt @@ -15,7 +15,7 @@ val developerURL: String = "https://matthicks.com" name := projectName ThisBuild / organization := org -ThisBuild / version := "0.3.0-SNAPSHOT" +ThisBuild / version := "1.0.0-SNAPSHOT" ThisBuild / scalaVersion := scala213 ThisBuild / crossScalaVersions := allScalaVersions ThisBuild / scalacOptions ++= Seq("-unchecked", "-deprecation") @@ -53,25 +53,26 @@ val scalaTestVersion: String = "3.2.18" val catsEffectTestingVersion: String = "1.5.0" lazy val root = project.in(file(".")) - .aggregate(core.js, core.jvm, lucene, halo, mapdb, all) + .aggregate(core) .settings( name := projectName, publish := {}, publishLocal := {} ) -lazy val core = crossProject(JSPlatform, JVMPlatform) - .crossType(CrossType.Full) +lazy val core = project.in(file("core")) .settings( name := s"$projectName-core", libraryDependencies ++= Seq( - "com.outr" %%% "scribe" % scribeVersion, - "com.outr" %%% "scribe-cats" % scribeVersion, - "org.typelevel" %%% "cats-effect" % catsEffectVersion, - "org.typelevel" %%% "fabric-io" % fabricVersion, - "co.fs2" %%% "fs2-core" % fs2Version, - "org.scalatest" %%% "scalatest" % scalaTestVersion % Test, - "org.typelevel" %%% "cats-effect-testing-scalatest" % catsEffectTestingVersion % Test + "com.outr" %% "scribe" % scribeVersion, + "com.outr" %% "scribe-cats" % scribeVersion, + "org.typelevel" %% "cats-effect" % catsEffectVersion, + "org.typelevel" %% "fabric-io" % fabricVersion, + "co.fs2" %% "fs2-core" % fs2Version, + "com.outr" %% "scribe-slf4j" % scribeVersion, + "com.github.yahoo" % "HaloDB" % haloDBVersion, + "org.scalatest" %% "scalatest" % scalaTestVersion % Test, + "org.typelevel" %% "cats-effect-testing-scalatest" % catsEffectTestingVersion % Test ), libraryDependencies ++= ( if (scalaVersion.value.startsWith("3.")) { @@ -91,56 +92,8 @@ lazy val core = crossProject(JSPlatform, JVMPlatform) } ) -lazy val lucene = project.in(file("lucene")) - .dependsOn(core.jvm) - .settings( - name := s"$projectName-lucene", - libraryDependencies ++= Seq( - "org.apache.lucene" % "lucene-core" % luceneVersion, - "org.apache.lucene" % "lucene-queryparser" % luceneVersion, - "org.scalatest" %% "scalatest" % scalaTestVersion % Test, - "org.typelevel" %% "cats-effect-testing-scalatest" % catsEffectTestingVersion % Test - ) - ) - -lazy val halo = project.in(file("halo")) - .dependsOn(core.jvm) - .settings( - name := s"$projectName-halo", - libraryDependencies ++= Seq( - "com.outr" %% "scribe-slf4j" % scribeVersion, - "com.github.yahoo" % "HaloDB" % haloDBVersion, - "org.scalatest" %% "scalatest" % scalaTestVersion % Test, - "org.typelevel" %% "cats-effect-testing-scalatest" % catsEffectTestingVersion % Test - ), - fork := true - ) - -lazy val mapdb = project.in(file("mapdb")) - .dependsOn(core.jvm) - .settings( - name := s"$projectName-mapdb", - libraryDependencies ++= Seq( - "org.mapdb" % "mapdb" % "3.1.0", - "org.scalatest" %% "scalatest" % scalaTestVersion % Test, - "org.typelevel" %% "cats-effect-testing-scalatest" % catsEffectTestingVersion % Test - ), - fork := true - ) - -lazy val all = project.in(file("all")) - .dependsOn(lucene, halo, mapdb) - .settings( - name := s"$projectName-all", - libraryDependencies ++= Seq( - "org.scalatest" %% "scalatest" % scalaTestVersion % Test, - "org.typelevel" %% "cats-effect-testing-scalatest" % catsEffectTestingVersion % Test - ), - fork := true - ) - lazy val benchmark = project.in(file("benchmark")) - .dependsOn(all) + .dependsOn(core) .settings( name := s"$projectName-benchmark", fork := true, diff --git a/core/src/main/scala/lightdb/Collection.scala b/core/src/main/scala/lightdb/Collection.scala new file mode 100644 index 00000000..f01a50c4 --- /dev/null +++ b/core/src/main/scala/lightdb/Collection.scala @@ -0,0 +1,14 @@ +package lightdb + +import fabric.rw.RW + +abstract class Collection[D <: Document[D]](implicit val rw: RW[D]) { + protected lazy val defaultCollectionName: String = getClass.getSimpleName.replace("$", "") + protected lazy val store: Store = db.createStore(collectionName) + protected lazy val indexStore: Store = db.createStore(s"$collectionName.indexes") + + protected def db: LightDB + protected def collectionName: String = defaultCollectionName + + // TODO: Triggers +} \ No newline at end of file diff --git a/core/src/main/scala/lightdb/Document.scala b/core/src/main/scala/lightdb/Document.scala new file mode 100644 index 00000000..9d41fa40 --- /dev/null +++ b/core/src/main/scala/lightdb/Document.scala @@ -0,0 +1,5 @@ +package lightdb + +trait Document[D <: Document[D]] { + def _id: Id[D] +} diff --git a/core/src/main/scala/lightdb/Id.scala b/core/src/main/scala/lightdb/Id.scala new file mode 100644 index 00000000..6065f0f3 --- /dev/null +++ b/core/src/main/scala/lightdb/Id.scala @@ -0,0 +1,25 @@ +package lightdb + +import fabric.rw._ + +class Id[T](val value: String) extends AnyVal { + def bytes: Array[Byte] = { + val b = toString.getBytes("UTF-8") + assert(b.length <= 128, s"Must be 128 bytes or less, but was ${b.length} ($value)") + b + } + + override def toString: String = value +} + +object Id { + private lazy val _rw: RW[Id[_]] = RW.string(_.value, Id.apply) + + implicit def rw[T]: RW[Id[T]] = _rw.asInstanceOf[RW[Id[T]]] + + def apply[T](value: String = Unique()): Id[T] = new Id[T](value) + + def toString[T](id: Id[T]): String = id.value + + def fromString[T](s: String): Id[T] = apply[T](s) +} \ No newline at end of file diff --git a/core/src/main/scala/lightdb/LightDB.scala b/core/src/main/scala/lightdb/LightDB.scala new file mode 100644 index 00000000..21795fc2 --- /dev/null +++ b/core/src/main/scala/lightdb/LightDB.scala @@ -0,0 +1,17 @@ +package lightdb + +import java.nio.file.Path + +abstract class LightDB(directory: Path, + indexThreads: Int = 2, + maxFileSize: Int = 1024 * 1024) { + + private var stores = List.empty[Store] + + protected[lightdb] def createStore(name: String): Store = synchronized { + // TODO: verifyInitialized() + val store = Store(directory.resolve(name), indexThreads, maxFileSize) + stores = store :: stores + store + } +} diff --git a/core/src/main/scala/lightdb/Store.scala b/core/src/main/scala/lightdb/Store.scala new file mode 100644 index 00000000..84a25a36 --- /dev/null +++ b/core/src/main/scala/lightdb/Store.scala @@ -0,0 +1,77 @@ +package lightdb + +import cats.effect.IO +import com.oath.halodb.{HaloDB, HaloDBOptions} +import fabric.io.{JsonFormatter, JsonParser} +import fabric.rw._ + +import java.nio.file.{Files, Path} +import scala.jdk.CollectionConverters._ + +case class Store(directory: Path, + indexThreads: Int, + maxFileSize: Int) { + private lazy val instance: HaloDB = { + val opts = new HaloDBOptions + opts.setBuildIndexThreads(indexThreads) + opts.setMaxFileSize(maxFileSize) + + Files.createDirectories(directory.getParent) + HaloDB.open(directory.toAbsolutePath.toString, opts) + } + + def keyStream[D]: fs2.Stream[IO, Id[D]] = fs2.Stream.fromBlockingIterator[IO](instance.newKeyIterator().asScala, 1024) + .map { record => + Id[D](record.getBytes.string) + } + + def stream[D]: fs2.Stream[IO, (Id[D], Array[Byte])] = fs2.Stream.fromBlockingIterator[IO](instance.newIterator().asScala, 1024) + .map { record => + val key = record.getKey.string + Id[D](key) -> record.getValue + } + + def get[D](id: Id[D]): IO[Option[Array[Byte]]] = IO { + Option(instance.get(id.bytes)) + } + + def getJson[D: RW](id: Id[D]): IO[Option[D]] = get(id) + .map(_.map { bytes => + val jsonString = bytes.string + val json = JsonParser(jsonString) + json.as[D] + }) + + def put[D](id: Id[D], value: Array[Byte]): IO[Boolean] = IO { + instance.put(id.bytes, value) + } + + def putJson[D <: Document[D]](doc: D) + (implicit rw: RW[D]): IO[D] = IO { + val json = doc.json + JsonFormatter.Compact(json) + }.flatMap { jsonString => + put(doc._id, jsonString.getBytes).map(_ => doc) + } + + def delete[D](id: Id[D]): IO[Unit] = IO { + instance.delete(id.bytes) + } + + def size: IO[Int] = IO(instance.size().toInt) + + def truncate(): IO[Unit] = keyStream[Any] + .evalMap { id => + delete(id) + } + .compile + .drain + .flatMap { _ => + size.flatMap { + case 0 => IO.unit + case _ => truncate() + } + } + + def dispose(): IO[Unit] = IO(instance.close()) +} \ No newline at end of file diff --git a/core/src/main/scala/lightdb/Unique.scala b/core/src/main/scala/lightdb/Unique.scala new file mode 100644 index 00000000..ef7830a1 --- /dev/null +++ b/core/src/main/scala/lightdb/Unique.scala @@ -0,0 +1,87 @@ +package lightdb + +import java.security.SecureRandom +import java.util.concurrent.ThreadLocalRandom + +/** + * Unique String generator + */ +object Unique { + lazy val LettersLower = "abcdefghijklmnopqrstuvwxyz" + lazy val LettersUpper = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + lazy val Numbers = "0123456789" + lazy val Readable = "ABCDEFGHJKLMNPQRSTWXYZ23456789" + lazy val Hexadecimal = s"${Numbers}abcdef" + lazy val LettersAndNumbers = s"$LettersLower$Numbers" + lazy val AllLettersAndNumbers = s"$LettersLower$LettersUpper$Numbers" + + private lazy val secure = new SecureRandom + + /** + * Random number generator used to generate unique values. Defaults to `threadLocalRandom`. + */ + var random: Int => Int = threadLocalRandom + + /** + * The default length to use for generating unique values. Defaults to 32. + */ + var defaultLength: Int = 32 + + /** + * The default characters to use for generating unique values. Defaults to AllLettersAndNumbers. + */ + var defaultCharacters: String = AllLettersAndNumbers + + /** + * True if randomization should be secure. Defaults to false. + */ + var defaultSecure: Boolean = false + + /** + * Uses java.util.concurrent.ThreadLocalRandom to generate random numbers. + * + * @param max the maximum value to include + * @return random number between 0 and max + */ + final def threadLocalRandom(max: Int): Int = ThreadLocalRandom.current().nextInt(max) + + final def secureRandom(max: Int): Int = synchronized { + secure.nextInt(max) + } + + /** + * Generates a unique String using the characters supplied at the length defined. + * + * @param length the length of the resulting String. Defaults to Unique.defaultLength. + * @param characters the characters for use in the String. Defaults to Unique.defaultCharacters. + * @param secure true if the randomization should be secure. Defaults to Unique.defaultSecure. + * @return a unique String + */ + def apply(length: Int = defaultLength, characters: String = defaultCharacters, secure: Boolean = defaultSecure): String = { + val charMax = characters.length + val r = if (secure) secureRandom _ else random + (0 until length).map(i => characters.charAt(r(charMax))).mkString + } + + /** + * Convenience functionality to generate a UUID (https://en.wikipedia.org/wiki/Universally_unique_identifier) + * + * 32 characters of unique hexadecimal values with dashes representing 36 total characters + */ + def uuid(secure: Boolean = false): String = { + val a = apply(8, Hexadecimal, secure) + val b = apply(4, Hexadecimal, secure) + val c = apply(3, Hexadecimal, secure) + val d = apply(1, "89ab", secure) + val e = apply(3, Hexadecimal, secure) + val f = apply(12, Hexadecimal, secure) + s"$a-$b-4$c-$d$e-$f" + } + + /** + * Returns the number of possible values for a specific length and characters. + */ + def poolSize(length: Int = 32, characters: String = AllLettersAndNumbers): Long = { + math.pow(characters.length, length).toLong + } +} \ No newline at end of file diff --git a/core/src/main/scala/lightdb/package.scala b/core/src/main/scala/lightdb/package.scala new file mode 100644 index 00000000..ce07a68d --- /dev/null +++ b/core/src/main/scala/lightdb/package.scala @@ -0,0 +1,5 @@ +package object lightdb { + implicit class ByteArrayExtras(val bytes: Array[Byte]) extends AnyVal { + def string: String = new String(bytes, "UTF-8") + } +} \ No newline at end of file