Skip to content

Commit

Permalink
Merge pull request #426 from gnieh/json/jq
Browse files Browse the repository at this point in the history
Add Jq implementation
  • Loading branch information
satabin authored Oct 2, 2023
2 parents 944b64b + d14d35e commit f69a657
Show file tree
Hide file tree
Showing 29 changed files with 1,953 additions and 178 deletions.
55 changes: 22 additions & 33 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,8 @@ val commonSettings = List(
}
.toList
.flatten,
testFrameworks += new TestFramework("weaver.framework.CatsEffect")
testFrameworks += new TestFramework("weaver.framework.CatsEffect"),
tlBspCrossProjectPlatforms := Set(JVMPlatform)
)

val root = tlCrossRootProject
Expand Down Expand Up @@ -230,12 +231,29 @@ lazy val json = crossProject(JVMPlatform, JSPlatform, NativePlatform)
name := "fs2-data-json",
description := "Streaming JSON manipulation library",
libraryDependencies ++= List(
"org.typelevel" %%% "literally" % literallyVersion
"org.typelevel" %%% "literally" % literallyVersion,
"org.typelevel" %%% "cats-parse" % "0.3.9"
) ++ PartialFunction
.condOpt(CrossVersion.partialVersion(scalaVersion.value)) { case Some((2, _)) =>
"org.scala-lang" % "scala-reflect" % scalaVersion.value
}
.toList
.toList,
mimaBinaryIssueFilters ++= List(
// all these experimental classes have been made internal
ProblemFilters.exclude[MissingClassProblem]("fs2.data.json.jsonpath.internals.JsonTagger"),
ProblemFilters.exclude[MissingClassProblem]("fs2.data.json.jsonpath.internals.JsonTagger$"),
ProblemFilters.exclude[MissingClassProblem]("fs2.data.json.jsonpath.internals.TaggedJson"),
ProblemFilters.exclude[MissingClassProblem]("fs2.data.json.jsonpath.internals.TaggedJson$"),
ProblemFilters.exclude[MissingClassProblem]("fs2.data.json.jsonpath.internals.TaggedJson$EndArrayElement$"),
ProblemFilters.exclude[MissingClassProblem]("fs2.data.json.jsonpath.internals.TaggedJson$EndObjectValue$"),
ProblemFilters.exclude[MissingClassProblem]("fs2.data.json.jsonpath.internals.TaggedJson$Raw"),
ProblemFilters.exclude[MissingClassProblem]("fs2.data.json.jsonpath.internals.TaggedJson$Raw$"),
ProblemFilters.exclude[MissingClassProblem]("fs2.data.json.jsonpath.internals.TaggedJson$StartArrayElement"),
ProblemFilters.exclude[MissingClassProblem]("fs2.data.json.jsonpath.internals.TaggedJson$StartArrayElement$"),
ProblemFilters.exclude[MissingClassProblem]("fs2.data.json.jsonpath.internals.TaggedJson$StartObjectValue"),
ProblemFilters.exclude[MissingClassProblem]("fs2.data.json.jsonpath.internals.TaggedJson$StartObjectValue$"),
ProblemFilters.exclude[DirectMissingMethodProblem]("fs2.data.json.jsonpath.package.untag")
)
)
.nativeSettings(
tlVersionIntroduced := Map("3" -> "1.5.1", "2.13" -> "1.5.1", "2.12" -> "1.5.1")
Expand Down Expand Up @@ -416,36 +434,7 @@ lazy val finiteState = crossProject(JVMPlatform, JSPlatform, NativePlatform)
.settings(
name := "fs2-data-finite-state",
description := "Streaming finite state machines",
tlVersionIntroduced := Map("3" -> "1.6.0", "2.13" -> "1.6.0", "2.12" -> "1.6.0"),
mimaBinaryIssueFilters ++= List(
// all filters related to esp.Rhs.Captured* come from converting it from case class to case object
ProblemFilters.exclude[MissingClassProblem]("fs2.data.esp.Rhs$CapturedLeaf"),
ProblemFilters.exclude[MissingTypesProblem]("fs2.data.esp.Rhs$CapturedLeaf$"),
ProblemFilters.exclude[DirectMissingMethodProblem]("fs2.data.esp.Rhs#CapturedLeaf.apply"),
ProblemFilters.exclude[DirectMissingMethodProblem]("fs2.data.esp.Rhs#CapturedLeaf.unapply"),
ProblemFilters.exclude[DirectMissingMethodProblem]("fs2.data.esp.Rhs#CapturedTree.name"),
ProblemFilters.exclude[DirectMissingMethodProblem]("fs2.data.esp.Rhs#CapturedTree.copy"),
ProblemFilters.exclude[IncompatibleResultTypeProblem]("fs2.data.esp.Rhs#CapturedTree.copy$default$1"),
ProblemFilters.exclude[DirectMissingMethodProblem]("fs2.data.esp.Rhs#CapturedTree.copy$default$2"),
ProblemFilters.exclude[DirectMissingMethodProblem]("fs2.data.esp.Rhs#CapturedTree.this"),
ProblemFilters.exclude[DirectMissingMethodProblem]("fs2.data.esp.Rhs#CapturedTree.apply"),
ProblemFilters.exclude[IncompatibleResultTypeProblem]("fs2.data.esp.Rhs#CapturedLeaf.fromProduct"),
ProblemFilters.exclude[IncompatibleResultTypeProblem]("fs2.data.esp.Rhs#CapturedTree._1"),
ProblemFilters.exclude[DirectMissingMethodProblem]("fs2.data.esp.Rhs#CapturedTree._2"),
ProblemFilters.exclude[ReversedMissingMethodProblem](
"fs2.data.mft.MFTBuilder#Guardable.fs2$data$mft$MFTBuilder$Guardable$$$outer"),
// rules now only have number of parameters
ProblemFilters.exclude[IncompatibleMethTypeProblem]("fs2.data.mft.Rules.apply"),
ProblemFilters.exclude[DirectMissingMethodProblem]("fs2.data.mft.Rules.params"),
ProblemFilters.exclude[IncompatibleMethTypeProblem]("fs2.data.mft.Rules.copy"),
ProblemFilters.exclude[IncompatibleResultTypeProblem]("fs2.data.mft.Rules.copy$default$1"),
ProblemFilters.exclude[IncompatibleMethTypeProblem]("fs2.data.mft.Rules.this"),
ProblemFilters.exclude[IncompatibleMethTypeProblem]("fs2.data.mft.Rules.apply"),
ProblemFilters.exclude[IncompatibleResultTypeProblem]("fs2.data.mft.Rules._1"),
// Removal of experimental class
ProblemFilters.exclude[MissingFieldProblem]("fs2.data.esp.Tag.True"),
ProblemFilters.exclude[MissingClassProblem]("fs2.data.esp.Tag$True$")
)
tlMimaPreviousVersions := Set.empty // experimental module, no compatbility guarantess
)
.jsSettings(
scalaJSLinkerConfig ~= (_.withModuleKind(ModuleKind.CommonJSModule))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,4 +35,8 @@ object Query {
case class Leaf[Tag, Path](tag: Tag) extends Query[Tag, Path]
case class Sequence[Tag, Path](elements: NonEmptyList[Query[Tag, Path]]) extends Query[Tag, Path]
case class LeafFunction[Tag, Path](f: Tag => Either[String, Tag]) extends Query[Tag, Path]

def empty[Tag, Path]: Query[Tag, Path] = Empty()
def variable[Tag, Path](v: String): Query[Tag, Path] = Variable(v)
def node[Tag, Path](tag: Tag, child: Query[Tag, Path]): Query[Tag, Path] = Node(tag, child)
}
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,14 @@ import cats.data.NonEmptyList
* The compiler is based on the approach described in [[https://doi.org/10.1109/ICDE.2014.6816714 _XQuery Streaming by Forest Transducers_]]
* and generalized for the abstract query language on trees.
*/
private[fs2] abstract class QueryCompiler[Tag, Path] {
private[fs2] abstract class QueryCompiler[InTag, OutTag, Path] {

type Matcher
type Pattern
type Guard

protected val emitSelected: Boolean = true

/** A single char to be matched in a path */
type Char

Expand All @@ -56,7 +58,7 @@ private[fs2] abstract class QueryCompiler[Tag, Path] {
def cases(matcher: Matcher): List[(Pattern, List[Guard])]

/** Return the constructor tag of this pattern, or `None` if it is a wildcard. */
def tagOf(pattern: Pattern): Option[Tag]
def tagOf(pattern: Pattern): Option[InTag]

/** Compiles the `query` into an [[MFT Macro Forest Transducer]].
* The `credit` parameter defines the maximum number of optimization passes that
Expand All @@ -67,8 +69,8 @@ private[fs2] abstract class QueryCompiler[Tag, Path] {
*
* If you do not want to perform any optimization, you can set this value to `0`.
*/
def compile(query: Query[Tag, Path], credit: Int = 50): MFT[NonEmptyList[Guard], Tag, Tag] = {
val mft = dsl[NonEmptyList[Guard], Tag, Tag] { implicit builder =>
def compile(query: Query[OutTag, Path], credit: Int = 50): MFT[NonEmptyList[Guard], InTag, OutTag] = {
val mft = dsl[NonEmptyList[Guard], InTag, OutTag] { implicit builder =>
val q0 = state(args = 0, initial = true)
val qinit = state(args = 1)
val qcopy = state(args = 0)
Expand Down Expand Up @@ -122,9 +124,12 @@ private[fs2] abstract class QueryCompiler[Tag, Path] {
val pat: builder.Guardable = tagOf(pattern).fold(anyNode)(aNode(_))
if (!finalTgt) {
q1(pat.when(guard)) -> q2(x1, copyArgs: _*) ~ q1(x2, copyArgs: _*)
} else {
} else if (emitSelected) {
q1(pat.when(guard)) -> end(x1, (copyArgs :+ copy(qcopy(x1))): _*) ~ q2(x1, copyArgs: _*) ~
q1(x2, copyArgs: _*)
} else {
q1(pat.when(guard)) -> end(x1, (copyArgs :+ qcopy(x1)): _*) ~ q2(x1, copyArgs: _*) ~
q1(x2, copyArgs: _*)
}
states1
}
Expand All @@ -134,7 +139,7 @@ private[fs2] abstract class QueryCompiler[Tag, Path] {
}
}

def translate(query: Query[Tag, Path], vars: List[String], q: builder.StateBuilder): Unit =
def translate(query: Query[OutTag, Path], vars: List[String], q: builder.StateBuilder): Unit =
query match {
case Query.Empty() =>
q(any) -> eps
Expand Down Expand Up @@ -192,7 +197,7 @@ private[fs2] abstract class QueryCompiler[Tag, Path] {

// compile and sequence every query in the sequence
val rhs =
queries.foldLeft[Rhs[Tag]](eps) { (acc, query) =>
queries.foldLeft[Rhs[OutTag]](eps) { (acc, query) =>
val q1 = state(args = q.nargs)

// translate the query
Expand All @@ -211,7 +216,7 @@ private[fs2] abstract class QueryCompiler[Tag, Path] {
translate(query, List("$input"), qinit)
}
// apply some optimizations until nothing changes or credit is exhausted
def optimize(mft: MFT[NonEmptyList[Guard], Tag, Tag], credit: Int): MFT[NonEmptyList[Guard], Tag, Tag] =
def optimize(mft: MFT[NonEmptyList[Guard], InTag, OutTag], credit: Int): MFT[NonEmptyList[Guard], InTag, OutTag] =
if (credit > 0) {
val mft1 = mft.removeUnusedParameters.inlineStayMoves.removeUnreachableStates
if (mft1.rules == mft.rules)
Expand Down
4 changes: 2 additions & 2 deletions finite-state/shared/src/main/scala/fs2/data/pfsa/Pred.scala
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ package fs2.data.pfsa
trait Pred[P, Elt] {

/** Whether the element `e` satisfies the predicate. */
def satsifies(p: P)(e: Elt): Boolean
def satisfies(p: P)(e: Elt): Boolean

/** The predicate that is always true. */
def always: P
Expand Down Expand Up @@ -56,7 +56,7 @@ object Pred {

implicit class PredOps[P](val p1: P) extends AnyVal {
def satisfies[Elt](e: Elt)(implicit P: Pred[P, Elt]): Boolean =
P.satsifies(p1)(e)
P.satisfies(p1)(e)

def &&[Elt](p2: P)(implicit P: Pred[P, Elt]): P =
P.and(p1, p2)
Expand Down
10 changes: 7 additions & 3 deletions finite-state/shared/src/main/scala/fs2/data/pfsa/Regular.scala
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ sealed abstract class Regular[CharSet] {
Regular.empty
}

def ?(implicit CharSet: Pred[CharSet, _], eq: Eq[CharSet]): Regular[CharSet] =
this || Regular.empty

def &&(that: Regular[CharSet])(implicit CharSet: Pred[CharSet, _], eq: Eq[CharSet]): Regular[CharSet] =
(this, that) match {
case (Regular.And(re1, re2), _) => re1 && (re2 && that)
Expand Down Expand Up @@ -116,7 +119,7 @@ sealed abstract class Regular[CharSet] {
def derive[C](c: C)(implicit CharSet: Pred[CharSet, C], eq: Eq[CharSet]): Regular[CharSet] =
this match {
case Regular.Epsilon() => Regular.Chars(CharSet.never)
case Regular.Chars(set) if CharSet.satsifies(set)(c) => Regular.Epsilon()
case Regular.Chars(set) if CharSet.satisfies(set)(c) => Regular.Epsilon()
case Regular.Chars(_) => Regular.Chars(CharSet.never)
case Regular.Concatenation(re1, re2) if re1.acceptEpsilon =>
(re1.derive(c) ~ re2) || re2.derive(c)
Expand Down Expand Up @@ -184,7 +187,8 @@ sealed abstract class Regular[CharSet] {
transitions: Map[Int, List[(CharSet, Int)]],
re: Regular[CharSet]): (Chain[Regular[CharSet]], Map[Int, List[(CharSet, Int)]]) = {
val q = qs.size.toInt - 1
re.classes.foldLeft((qs, transitions)) { case ((qs, transitions), cs) =>
val cls = re.classes
cls.foldLeft((qs, transitions)) { case ((qs, transitions), cs) =>
goto(re, q, cs, qs, transitions)
}
}
Expand Down Expand Up @@ -231,7 +235,7 @@ object Regular {
implicit def pred[CharSet: Eq, C](implicit CharSet: Pred[CharSet, C]): Pred[Regular[CharSet], C] =
new Pred[Regular[CharSet], C] {

override def satsifies(p: Regular[CharSet])(e: C): Boolean =
override def satisfies(p: Regular[CharSet])(e: C): Boolean =
p match {
case Epsilon() => false
case Chars(set) => set.satisfies(e)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ abstract class QuerySpec(credit: Int) extends SimpleIOSuite {

}

object MiniXQueryCompiler extends QueryCompiler[String, MiniXPath] {
object MiniXQueryCompiler extends QueryCompiler[String, String, MiniXPath] {

type Matcher = Set[String]
type Char = String
Expand All @@ -66,7 +66,7 @@ abstract class QuerySpec(credit: Int) extends SimpleIOSuite {

override implicit object predicate extends Pred[Matcher, Char] {

override def satsifies(p: Matcher)(e: Char): Boolean = p.contains(e)
override def satisfies(p: Matcher)(e: Char): Boolean = p.contains(e)

override val always: Matcher = Set("a", "b", "c", "d", "doc")

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ object RegularSpec extends SimpleIOSuite with Checkers {

implicit object CharSetInstances extends Pred[Set[Char], Char] with Candidate[Set[Char], Char] {

override def satsifies(p: Set[Char])(e: Char): Boolean = p.contains(e)
override def satisfies(p: Set[Char])(e: Char): Boolean = p.contains(e)

override val always: Set[Char] = Set('a', 'b')

Expand Down
81 changes: 81 additions & 0 deletions json/src/main/scala-2/fs2/data/json/jq/literals.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
/*
* Copyright 2023 Lucas Satabin
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package fs2
package data
package json
package jq

import cats.data.NonEmptyList
import cats.syntax.all._
import org.typelevel.literally.Literally

import scala.annotation.unused
import scala.reflect.macros.blackbox.Context

object literals {

implicit class JqStringContext(val sc: StringContext) extends AnyVal {
def jq(args: Any*): Jq = macro JqInterpolator.make
}

trait LiftableImpls {
val c: Context
import c.universe._

implicit def nel[T](implicit @unused T: Liftable[T]): Liftable[NonEmptyList[T]] = Liftable[NonEmptyList[T]] {
case NonEmptyList(t, Nil) => q"_root_.cats.data.NonEmptyList.one($t)"
case NonEmptyList(t, tail) => q"_root_.cats.data.NonEmptyList($t, $tail)"
}

implicit lazy val jqLiftable: Liftable[Jq] = Liftable[Jq] {
case Jq.Root => q"_root_.fs2.data.json.jq.Jq.Root"
case Jq.Identity => q"_root_.fs2.data.json.jq.Jq.Identity"
case Jq.Field(name) => q"_root_.fs2.data.json.jq.Jq.Field($name)"
case Jq.Index(idx) => q"_root_.fs2.data.json.jq.Jq.Index($idx)"
case Jq.Slice(idx1, idx2) => q"_root_.fs2.data.json.jq.Jq.Slice($idx1, $idx2)"
case Jq.Child => q"_root_.fs2.data.json.jq.Jq.Child"
case Jq.RecursiveDescent => q"_root_.fs2.data.json.jq.Jq.RecursiveDescent"
case Jq.Sequence(qs) =>
q"_root_.fs2.data.json.jq.Jq.Sequence(_root_.cats.data.NonEmptyChain.fromNonEmptyList(${qs.toNonEmptyList.widen[Jq]}))"
case Jq.Iterator(filter, inner) => q"_root_.fs2.data.json.jq.Jq.Iterator(${filter: Jq}, $inner)"
case Jq.Arr(pfx, qs) => q"_root_.fs2.data.json.jq.Jq.Arr(${pfx: Jq}, $qs)"
case Jq.Obj(pfx, qs) => q"_root_.fs2.data.json.jq.Jq.Obj(${pfx: Jq}, $qs)"
case Jq.Num(n) => q"_root_.fs2.data.json.jq.Jq.Num($n)"
case Jq.Str(s) => q"_root_.fs2.data.json.jq.Jq.Str($s)"
case Jq.Bool(b) => q"_root_.fs2.data.json.jq.Jq.Bool($b)"
case Jq.Null => q"_root_.fs2.data.json.jq.Jq.Null"
}
}

object JqInterpolator extends Literally[Jq] {

def validate(ctx: Context)(string: String): Either[String, ctx.Expr[Jq]] = {
import ctx.universe._
val liftables = new LiftableImpls {
val c: ctx.type = ctx
}
import liftables._
JqParser
.either(string)
.leftMap(_.getMessage)
.map(p => c.Expr(q"$p"))
}

def make(c: Context)(args: c.Expr[Any]*): c.Expr[Jq] = apply(c)(args: _*)

}
}
Loading

0 comments on commit f69a657

Please sign in to comment.