From 0a5cb058ee878a39d77d2d686bcecbdfb0d2f3c4 Mon Sep 17 00:00:00 2001 From: eikek Date: Tue, 21 Jun 2022 23:31:37 +0200 Subject: [PATCH 1/4] First step to support different file backends Files can now stored in the filesystem, an s3 storage or the database (as before). --- .../scala/sharry/backend/BackendApp.scala | 9 +- .../sharry/backend/{ => config}/Config.scala | 14 +- .../scala/sharry/backend/config/Files.scala | 37 + .../src/main/scala/sharry/common/Banner.scala | 4 +- .../{pureconfig => config}/Implicits.scala | 24 +- .../sharry/logging/impl/ScribeConfigure.scala | 2 +- .../src/main/resources/reference.conf | 34 + .../main/scala/sharry/restserver/Main.scala | 5 +- .../scala/sharry/restserver/RestApp.scala | 1 + .../scala/sharry/restserver/RestAppImpl.scala | 1 + .../scala/sharry/restserver/RestServer.scala | 1 + .../restserver/{ => config}/Config.scala | 4 +- .../restserver/{ => config}/ConfigFile.scala | 32 +- .../restserver/http4s/ClientRequestInfo.scala | 2 +- .../sharry/restserver/routes/InfoRoutes.scala | 3 +- .../restserver/routes/LoginRoutes.scala | 1 + .../sharry/restserver/routes/MailRoutes.scala | 2 +- .../restserver/routes/NotifyRoutes.scala | 2 +- .../restserver/routes/OpenShareRoutes.scala | 2 +- .../restserver/routes/RegisterRoutes.scala | 2 +- .../routes/ShareDetailResponse.scala | 2 +- .../restserver/routes/ShareRoutes.scala | 2 +- .../restserver/routes/ShareUploadRoutes.scala | 2 +- .../restserver/routes/tus/TusRoutes.scala | 2 +- .../restserver/webapp/TemplateRoutes.scala | 3 +- .../main/scala/sharry/store/FileStore.scala | 63 +- .../scala/sharry/store/FileStoreConfig.scala | 33 + .../scala/sharry/store/FileStoreType.scala | 26 + .../src/main/scala/sharry/store/Store.scala | 3 +- .../scala/sharry/store/StoreFixture.scala | 2 +- modules/webapp/package-lock.json | 2374 ++++++++++++++++- project/Dependencies.scala | 4 +- 32 files changed, 2647 insertions(+), 51 deletions(-) rename modules/backend/src/main/scala/sharry/backend/{ => config}/Config.scala (64%) create mode 100644 modules/backend/src/main/scala/sharry/backend/config/Files.scala rename modules/common/src/main/scala/sharry/common/{pureconfig => config}/Implicits.scala (67%) rename modules/restserver/src/main/scala/sharry/restserver/{ => config}/Config.scala (95%) rename modules/restserver/src/main/scala/sharry/restserver/{ => config}/ConfigFile.scala (60%) create mode 100644 modules/store/src/main/scala/sharry/store/FileStoreConfig.scala create mode 100644 modules/store/src/main/scala/sharry/store/FileStoreType.scala diff --git a/modules/backend/src/main/scala/sharry/backend/BackendApp.scala b/modules/backend/src/main/scala/sharry/backend/BackendApp.scala index b63fa2a3..b09d2867 100644 --- a/modules/backend/src/main/scala/sharry/backend/BackendApp.scala +++ b/modules/backend/src/main/scala/sharry/backend/BackendApp.scala @@ -7,6 +7,7 @@ import cats.effect._ import sharry.backend.account._ import sharry.backend.alias.OAlias import sharry.backend.auth.Login +import sharry.backend.config.Config import sharry.backend.job.PeriodicCleanup import sharry.backend.mail.OMail import sharry.backend.share.OShare @@ -54,7 +55,13 @@ object BackendApp { connectEC: ExecutionContext ): Resource[F, BackendApp[F]] = for { - store <- Store.create(cfg.jdbc, cfg.share.chunkSize, connectEC, true) + store <- Store.create( + cfg.jdbc, + cfg.share.chunkSize, + cfg.files.defaultStoreConfig, + connectEC, + true + ) backend <- create(cfg, store) _ <- PeriodicCleanup.resource(cfg.cleanup, cfg.signup, backend.share, backend.signup) diff --git a/modules/backend/src/main/scala/sharry/backend/Config.scala b/modules/backend/src/main/scala/sharry/backend/config/Config.scala similarity index 64% rename from modules/backend/src/main/scala/sharry/backend/Config.scala rename to modules/backend/src/main/scala/sharry/backend/config/Config.scala index 206cfa6a..26aed8cb 100644 --- a/modules/backend/src/main/scala/sharry/backend/Config.scala +++ b/modules/backend/src/main/scala/sharry/backend/config/Config.scala @@ -1,4 +1,6 @@ -package sharry.backend +package sharry.backend.config + +import cats.data.ValidatedNec import sharry.backend.auth.AuthConfig import sharry.backend.job.CleanupConfig @@ -13,7 +15,11 @@ case class Config( auth: AuthConfig, share: ShareConfig, cleanup: CleanupConfig, - mail: MailConfig -) + mail: MailConfig, + files: Files +) { + + def validate: ValidatedNec[String, Config] = + files.validate.map(fc => copy(files = fc)) -object Config {} +} diff --git a/modules/backend/src/main/scala/sharry/backend/config/Files.scala b/modules/backend/src/main/scala/sharry/backend/config/Files.scala new file mode 100644 index 00000000..cf481db2 --- /dev/null +++ b/modules/backend/src/main/scala/sharry/backend/config/Files.scala @@ -0,0 +1,37 @@ +package sharry.backend.config + +import cats.data.{Validated, ValidatedNec} +import cats.syntax.all._ + +import sharry.common.Ident +import sharry.store.FileStoreConfig + +case class Files(defaultStore: Ident, stores: Map[Ident, FileStoreConfig]) { + + val enabledStores: Map[Ident, FileStoreConfig] = + stores.view.filter(_._2.enabled).toMap + + def defaultStoreConfig: FileStoreConfig = + enabledStores.getOrElse( + defaultStore, + sys.error(s"Store '${defaultStore.id}' not found. Is it enabled?") + ) + + def validate: ValidatedNec[String, Files] = { + val storesEmpty = + if (enabledStores.isEmpty) + Validated.invalidNec( + "No file stores defined! Make sure at least one enabled store is present." + ) + else Validated.validNec(()) + + val defaultStorePresent = + enabledStores.get(defaultStore) match { + case Some(_) => Validated.validNec(()) + case None => + Validated.invalidNec(s"Default file store not present: ${defaultStore}") + } + + (storesEmpty |+| defaultStorePresent).map(_ => this) + } +} diff --git a/modules/common/src/main/scala/sharry/common/Banner.scala b/modules/common/src/main/scala/sharry/common/Banner.scala index bed0f525..d70ad318 100644 --- a/modules/common/src/main/scala/sharry/common/Banner.scala +++ b/modules/common/src/main/scala/sharry/common/Banner.scala @@ -5,7 +5,8 @@ case class Banner( gitHash: Option[String], jdbcUrl: LenientUri, configFile: Option[String], - baseUrl: LenientUri + baseUrl: LenientUri, + fileStoreConfig: String ) { private val banner = @@ -23,6 +24,7 @@ case class Banner( s"Base-Url: ${baseUrl.asString}", s"Database: ${jdbcUrl.asString}", s"Config: ${configFile.getOrElse("")}", + s"FileRepo: $fileStoreConfig", "" ) diff --git a/modules/common/src/main/scala/sharry/common/pureconfig/Implicits.scala b/modules/common/src/main/scala/sharry/common/config/Implicits.scala similarity index 67% rename from modules/common/src/main/scala/sharry/common/pureconfig/Implicits.scala rename to modules/common/src/main/scala/sharry/common/config/Implicits.scala index 9a506415..22a13c3f 100644 --- a/modules/common/src/main/scala/sharry/common/pureconfig/Implicits.scala +++ b/modules/common/src/main/scala/sharry/common/config/Implicits.scala @@ -1,14 +1,22 @@ -package sharry.common.pureconfig +package sharry.common.config + +import java.nio.file.{Path => JPath} import scala.reflect.ClassTag +import fs2.io.file.Path + import sharry.common._ -import _root_.pureconfig._ -import _root_.pureconfig.error.{CannotConvert, FailureReason} +import pureconfig._ +import pureconfig.configurable.genericMapReader +import pureconfig.error.{CannotConvert, FailureReason} import scodec.bits.ByteVector -object Implicits { +trait Implicits { + implicit val pathReader: ConfigReader[Path] = + ConfigReader[JPath].map(Path.fromNioPath) + implicit val lenientUriReader: ConfigReader[LenientUri] = ConfigReader[String].emap(reason(LenientUri.parse)) @@ -21,6 +29,9 @@ object Implicits { implicit val identReader: ConfigReader[Ident] = ConfigReader[String].emap(reason(Ident.fromString)) + implicit def identMapReader[B: ConfigReader]: ConfigReader[Map[Ident, B]] = + genericMapReader[Ident, B](reason(Ident.fromString)) + implicit val byteVectorReader: ConfigReader[ByteVector] = ConfigReader[String].emap(reason { str => if (str.startsWith("hex:")) @@ -33,6 +44,9 @@ object Implicits { implicit val byteSizeReader: ConfigReader[ByteSize] = ConfigReader[String].emap(reason(ByteSize.parse)) + implicit val signupModeReader: ConfigReader[SignupMode] = + ConfigReader[String].emap(reason(SignupMode.fromString)) + def reason[A: ClassTag]( f: String => Either[String, A] ): String => Either[FailureReason, A] = @@ -41,3 +55,5 @@ object Implicits { CannotConvert(in, implicitly[ClassTag[A]].runtimeClass.toString, str) ) } + +object Implicits extends Implicits diff --git a/modules/logging/scribe/src/main/scala/sharry/logging/impl/ScribeConfigure.scala b/modules/logging/scribe/src/main/scala/sharry/logging/impl/ScribeConfigure.scala index 873939b5..005a93cb 100644 --- a/modules/logging/scribe/src/main/scala/sharry/logging/impl/ScribeConfigure.scala +++ b/modules/logging/scribe/src/main/scala/sharry/logging/impl/ScribeConfigure.scala @@ -24,7 +24,7 @@ object ScribeConfigure { unsafeConfigure(sharryLogger, cfg) unsafeConfigure(scribe.Logger("org.flywaydb"), cfg) unsafeConfigure(scribe.Logger("binny"), cfg) - unsafeConfigure(scribe.Logger("org.http4s"), cfg) + // unsafeConfigure(scribe.Logger("org.http4s"), cfg) } def getRootMinimumLevel: Level = diff --git a/modules/restserver/src/main/resources/reference.conf b/modules/restserver/src/main/resources/reference.conf index 6cf9df63..bfede73f 100644 --- a/modules/restserver/src/main/resources/reference.conf +++ b/modules/restserver/src/main/resources/reference.conf @@ -264,6 +264,40 @@ sharry.restserver { password = "" } + # How files are stored. + files { + # The id of an enabled store from the `stores` array that should + # be used. + default-store = "database" + + # A list of possible file stores. Each entry must have a unique + # id. The `type` is one of: default-database, filesystem, s3. + # + # All stores with enabled=false are + # removed from the list. The `default-store` must be enabled. + stores = { + database = + { enabled = true + type = "default-database" + } + + filesystem = + { enabled = false + type = "file-system" + directory = "/some/directory" + } + + minio = + { enabled = false + type = "s3" + endpoint = "http://localhost:9000" + access-key = "username" + secret-key = "password" + bucket = "sharry" + } + } + } + # Configuration for registering new users at the local database. # Accounts registered here are checked via the `internal' # authentication plugin as described above. diff --git a/modules/restserver/src/main/scala/sharry/restserver/Main.scala b/modules/restserver/src/main/scala/sharry/restserver/Main.scala index 84b23df8..54d87d8d 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/Main.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/Main.scala @@ -6,6 +6,7 @@ import cats.effect._ import sharry.common._ import sharry.logging.impl.ScribeConfigure +import sharry.restserver.config.ConfigFile object Main extends IOApp { private[this] val logger = sharry.logging.getLogger[IO] @@ -46,10 +47,12 @@ object Main extends IOApp { BuildInfo.gitHeadCommit, cfg.backend.jdbc.url, Option(System.getProperty("config.file")), - cfg.baseUrl + cfg.baseUrl, + cfg.backend.files.defaultStoreConfig.toString ) _ <- logger.info(s"\n${banner.render("***>")}") + _ <- logger.info(s"\n${cfg.backend.files.stores}\n") pools = connectEC.map(Pools.apply) _ <- diff --git a/modules/restserver/src/main/scala/sharry/restserver/RestApp.scala b/modules/restserver/src/main/scala/sharry/restserver/RestApp.scala index 38e179b0..8b10124a 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/RestApp.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/RestApp.scala @@ -1,6 +1,7 @@ package sharry.restserver import sharry.backend.BackendApp +import sharry.restserver.config.Config trait RestApp[F[_]] { diff --git a/modules/restserver/src/main/scala/sharry/restserver/RestAppImpl.scala b/modules/restserver/src/main/scala/sharry/restserver/RestAppImpl.scala index 69a24750..8fe04aa2 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/RestAppImpl.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/RestAppImpl.scala @@ -6,6 +6,7 @@ import cats.effect._ import cats.implicits._ import sharry.backend.BackendApp +import sharry.restserver.config.Config final class RestAppImpl[F[_]: Sync](val config: Config, val backend: BackendApp[F]) extends RestApp[F] { diff --git a/modules/restserver/src/main/scala/sharry/restserver/RestServer.scala b/modules/restserver/src/main/scala/sharry/restserver/RestServer.scala index 901e79f1..62261f2d 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/RestServer.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/RestServer.scala @@ -11,6 +11,7 @@ import fs2.Stream import sharry.backend.auth.AuthToken import sharry.common.LenientUri import sharry.logging.Logger +import sharry.restserver.config.Config import sharry.restserver.http4s.EnvMiddleware import sharry.restserver.routes._ import sharry.restserver.webapp._ diff --git a/modules/restserver/src/main/scala/sharry/restserver/Config.scala b/modules/restserver/src/main/scala/sharry/restserver/config/Config.scala similarity index 95% rename from modules/restserver/src/main/scala/sharry/restserver/Config.scala rename to modules/restserver/src/main/scala/sharry/restserver/config/Config.scala index 24d71477..28e0e776 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/Config.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/config/Config.scala @@ -1,6 +1,6 @@ -package sharry.restserver +package sharry.restserver.config -import sharry.backend.{Config => BackendConfig} +import sharry.backend.config.{Config => BackendConfig} import sharry.common._ import sharry.logging.LogConfig diff --git a/modules/restserver/src/main/scala/sharry/restserver/ConfigFile.scala b/modules/restserver/src/main/scala/sharry/restserver/config/ConfigFile.scala similarity index 60% rename from modules/restserver/src/main/scala/sharry/restserver/ConfigFile.scala rename to modules/restserver/src/main/scala/sharry/restserver/config/ConfigFile.scala index 1862a2e0..76cb15cd 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/ConfigFile.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/config/ConfigFile.scala @@ -1,28 +1,25 @@ -package sharry.restserver +package sharry.restserver.config -import cats.implicits._ - -import sharry.common.SignupMode -import sharry.common.pureconfig.Implicits._ +import sharry.common.config.Implicits._ import sharry.logging.{Level, LogConfig} +import sharry.store.{FileStoreConfig, FileStoreType} -import _root_.pureconfig._ -import _root_.pureconfig.generic.auto._ import emil.MailAddress import emil.SSLType import emil.javamail.syntax._ -import yamusca.imports._ +import pureconfig._ +import pureconfig.generic.auto._ +import pureconfig.generic.{CoproductHint, FieldCoproductHint} +import yamusca.imports.{Template, mustache} object ConfigFile { + import Implicits._ def loadConfig: Config = ConfigSource.default.at("sharry.restserver").loadOrThrow[Config].validOrThrow object Implicits { - implicit val signupModeReader: ConfigReader[SignupMode] = - ConfigReader[String].emap(reason(SignupMode.fromString)) - implicit val mailAddressReader: ConfigReader[Option[MailAddress]] = ConfigReader[String].emap( reason(s => @@ -45,7 +42,7 @@ object ConfigFile { implicit val templateReader: ConfigReader[Template] = ConfigReader[String].emap( reason(s => - mustache.parse(s).leftMap(err => s"Error parsing template at ${err._1.pos}") + mustache.parse(s).left.map(err => s"Error parsing template at ${err._1.pos}") ) ) @@ -54,5 +51,16 @@ object ConfigFile { implicit val logLevelReader: ConfigReader[Level] = ConfigReader[String].emap(reason(Level.fromString)) + + implicit val fileStoreTypeReader: ConfigReader[FileStoreType] = + ConfigReader[String].emap(reason(FileStoreType.fromString)) + + // the value "s-3" looks strange, this is to allow to write "s3" in the config + implicit val fileStoreCoproductHint: CoproductHint[FileStoreConfig] = + new FieldCoproductHint[FileStoreConfig]("type") { + override def fieldValue(name: String) = + if (name.equalsIgnoreCase("S3")) "s3" + else super.fieldValue(name) + } } } diff --git a/modules/restserver/src/main/scala/sharry/restserver/http4s/ClientRequestInfo.scala b/modules/restserver/src/main/scala/sharry/restserver/http4s/ClientRequestInfo.scala index fa685b84..3178f870 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/http4s/ClientRequestInfo.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/http4s/ClientRequestInfo.scala @@ -4,7 +4,7 @@ import cats.data.NonEmptyList import cats.implicits._ import sharry.common._ -import sharry.restserver.Config +import sharry.restserver.config.Config import org.http4s._ import org.http4s.headers._ diff --git a/modules/restserver/src/main/scala/sharry/restserver/routes/InfoRoutes.scala b/modules/restserver/src/main/scala/sharry/restserver/routes/InfoRoutes.scala index 75855613..fefdafe5 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/routes/InfoRoutes.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/routes/InfoRoutes.scala @@ -3,7 +3,8 @@ package sharry.restserver.routes import cats.effect._ import sharry.restapi.model._ -import sharry.restserver.{BuildInfo, Config} +import sharry.restserver.BuildInfo +import sharry.restserver.config.Config import org.http4s.HttpRoutes import org.http4s.circe.CirceEntityEncoder._ diff --git a/modules/restserver/src/main/scala/sharry/restserver/routes/LoginRoutes.scala b/modules/restserver/src/main/scala/sharry/restserver/routes/LoginRoutes.scala index 03b0c5d0..6adf9983 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/routes/LoginRoutes.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/routes/LoginRoutes.scala @@ -10,6 +10,7 @@ import sharry.backend.auth._ import sharry.common._ import sharry.restapi.model._ import sharry.restserver._ +import sharry.restserver.config.Config import sharry.restserver.http4s.ClientRequestInfo import sharry.restserver.oauth.CodeFlow diff --git a/modules/restserver/src/main/scala/sharry/restserver/routes/MailRoutes.scala b/modules/restserver/src/main/scala/sharry/restserver/routes/MailRoutes.scala index d7eacef1..690c8ad6 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/routes/MailRoutes.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/routes/MailRoutes.scala @@ -12,7 +12,7 @@ import sharry.common._ import sharry.restapi.model.BasicResult import sharry.restapi.model.MailTemplate import sharry.restapi.model.SimpleMail -import sharry.restserver.Config +import sharry.restserver.config.Config import sharry.restserver.http4s.ClientRequestInfo import emil.MailAddress diff --git a/modules/restserver/src/main/scala/sharry/restserver/routes/NotifyRoutes.scala b/modules/restserver/src/main/scala/sharry/restserver/routes/NotifyRoutes.scala index bc6552b9..bac07082 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/routes/NotifyRoutes.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/routes/NotifyRoutes.scala @@ -8,7 +8,7 @@ import sharry.backend.auth.AuthToken import sharry.backend.mail.NotifyResult import sharry.common._ import sharry.restapi.model.BasicResult -import sharry.restserver.Config +import sharry.restserver.config.Config import sharry.restserver.http4s.ClientRequestInfo import org.http4s.HttpRoutes diff --git a/modules/restserver/src/main/scala/sharry/restserver/routes/OpenShareRoutes.scala b/modules/restserver/src/main/scala/sharry/restserver/routes/OpenShareRoutes.scala index 10c93894..f7e6d578 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/routes/OpenShareRoutes.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/routes/OpenShareRoutes.scala @@ -5,7 +5,7 @@ import cats.effect._ import sharry.backend.BackendApp import sharry.backend.share._ import sharry.common._ -import sharry.restserver.Config +import sharry.restserver.config.Config import sharry.restserver.routes.headers.SharryPassword import org.http4s._ diff --git a/modules/restserver/src/main/scala/sharry/restserver/routes/RegisterRoutes.scala b/modules/restserver/src/main/scala/sharry/restserver/routes/RegisterRoutes.scala index ff76794c..e0c7475f 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/routes/RegisterRoutes.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/routes/RegisterRoutes.scala @@ -7,7 +7,7 @@ import sharry.backend.BackendApp import sharry.backend.signup.OSignup.RegisterData import sharry.backend.signup.{NewInviteResult, SignupResult} import sharry.restapi.model._ -import sharry.restserver.Config +import sharry.restserver.config.Config import org.http4s.HttpRoutes import org.http4s.circe.CirceEntityDecoder._ diff --git a/modules/restserver/src/main/scala/sharry/restserver/routes/ShareDetailResponse.scala b/modules/restserver/src/main/scala/sharry/restserver/routes/ShareDetailResponse.scala index e116ca50..823e7381 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/routes/ShareDetailResponse.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/routes/ShareDetailResponse.scala @@ -8,7 +8,7 @@ import sharry.backend.BackendApp import sharry.backend.share._ import sharry.common._ import sharry.restapi.model.{ShareDetail => ShareDetailDto, _} -import sharry.restserver.Config +import sharry.restserver.config.Config import sharry.restserver.http4s.ClientRequestInfo import org.http4s._ diff --git a/modules/restserver/src/main/scala/sharry/restserver/routes/ShareRoutes.scala b/modules/restserver/src/main/scala/sharry/restserver/routes/ShareRoutes.scala index 4186d7a1..8ba76f12 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/routes/ShareRoutes.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/routes/ShareRoutes.scala @@ -9,7 +9,7 @@ import sharry.backend.auth.AuthToken import sharry.backend.share._ import sharry.common._ import sharry.restapi.model._ -import sharry.restserver.Config +import sharry.restserver.config.Config import sharry.restserver.routes.headers.SharryPassword import sharry.store.AddResult diff --git a/modules/restserver/src/main/scala/sharry/restserver/routes/ShareUploadRoutes.scala b/modules/restserver/src/main/scala/sharry/restserver/routes/ShareUploadRoutes.scala index 57e3d38a..4a9255ac 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/routes/ShareUploadRoutes.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/routes/ShareUploadRoutes.scala @@ -11,7 +11,7 @@ import sharry.backend.share.{File, ShareData} import sharry.common._ import sharry.common.syntax.all._ import sharry.restapi.model._ -import sharry.restserver.Config +import sharry.restserver.config.Config import sharry.restserver.http4s.ClientRequestInfo import sharry.restserver.routes.tus.TusRoutes diff --git a/modules/restserver/src/main/scala/sharry/restserver/routes/tus/TusRoutes.scala b/modules/restserver/src/main/scala/sharry/restserver/routes/tus/TusRoutes.scala index f087cf73..f98d16d8 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/routes/tus/TusRoutes.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/routes/tus/TusRoutes.scala @@ -8,7 +8,7 @@ import sharry.backend.BackendApp import sharry.backend.auth.AuthToken import sharry.backend.share.{FileInfo, UploadResult} import sharry.common._ -import sharry.restserver.Config +import sharry.restserver.config.Config import org.http4s._ import org.http4s.dsl.Http4sDsl diff --git a/modules/restserver/src/main/scala/sharry/restserver/webapp/TemplateRoutes.scala b/modules/restserver/src/main/scala/sharry/restserver/webapp/TemplateRoutes.scala index 6806390c..4f952aeb 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/webapp/TemplateRoutes.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/webapp/TemplateRoutes.scala @@ -8,9 +8,10 @@ import cats.implicits._ import fs2._ import sharry.restapi.model.AppConfig +import sharry.restserver.BuildInfo +import sharry.restserver.config.Config import sharry.restserver.routes.InfoRoutes import sharry.restserver.webapp.YamuscaConverter._ -import sharry.restserver.{BuildInfo, Config} import _root_.io.circe.syntax._ import org.http4s.HttpRoutes diff --git a/modules/store/src/main/scala/sharry/store/FileStore.scala b/modules/store/src/main/scala/sharry/store/FileStore.scala index 9979a487..7f4a0f8a 100644 --- a/modules/store/src/main/scala/sharry/store/FileStore.scala +++ b/modules/store/src/main/scala/sharry/store/FileStore.scala @@ -13,7 +13,9 @@ import sharry.store.records.RFileMeta import _root_.doobie._ import binny._ +import binny.fs.{FsChunkedBinaryStore, FsChunkedStoreConfig} import binny.jdbc.{GenericJdbcStore, JdbcStoreConfig} +import binny.minio.{MinioChunkedBinaryStore, MinioConfig, S3KeyMapping} import binny.tika.TikaContentTypeDetect import binny.util.Logger @@ -35,7 +37,25 @@ trait FileStore[F[_]] { } object FileStore { - def apply[F[_]: Sync]( + + def apply[F[_]: Async]( + ds: DataSource, + xa: Transactor[F], + chunkSize: Int, + config: FileStoreConfig + ): FileStore[F] = + config match { + case FileStoreConfig.DefaultDatabase(_) => + forDatabase(ds, xa, chunkSize) + + case c: FileStoreConfig.S3 => + forS3(xa, c, chunkSize) + + case c: FileStoreConfig.FileSystem => + forFs(xa, c, chunkSize) + } + + def forDatabase[F[_]: Async]( ds: DataSource, xa: Transactor[F], chunkSize: Int @@ -47,6 +67,41 @@ object FileStore { new Impl[F](bs, as, chunkSize) } + def forFs[F[_]: Async]( + xa: Transactor[F], + fsCfg: FileStoreConfig.FileSystem, + chunkSize: Int + ): FileStore[F] = { + val as = AttributeStore(xa) + val logger = SharryLogger(sharry.logging.getLogger[F]) + val cfg = FsChunkedStoreConfig + .defaults(fsCfg.directory) + .copy(chunkSize = chunkSize) + .withContentTypeDetect(TikaContentTypeDetect.default) + val bs = FsChunkedBinaryStore(cfg, logger, as) + new Impl[F](bs, as, chunkSize) + } + + def forS3[F[_]: Async]( + xa: Transactor[F], + s3: FileStoreConfig.S3, + chunkSize: Int + ): FileStore[F] = { + val as = AttributeStore(xa) + val logger = SharryLogger(sharry.logging.getLogger[F]) + val cfg = MinioConfig + .default( + s3.endpoint, + s3.accessKey, + s3.secretKey, + S3KeyMapping.constant(s3.bucket) + ) + .copy(chunkSize = chunkSize) + .withContentTypeDetect(TikaContentTypeDetect.default) + val bs = MinioChunkedBinaryStore(cfg, as, logger) + new Impl[F](bs, as, chunkSize) + } + final private class Impl[F[_]: Sync]( bs: ChunkedBinaryStore[F], attrStore: AttributeStore[F], @@ -80,8 +135,10 @@ object FileStore { chunkDef: ChunkDef, data: Chunk[Byte] ): F[Unit] = - bs.insertChunk(BinaryId(id.id), chunkDef, hint, data.toByteVector) - .map(_ => ()) + bs.insertChunk(BinaryId(id.id), chunkDef, hint, data.toByteVector).flatMap { + case _: InsertChunkResult.Success => ().pure[F] + case fail => Sync[F].raiseError(new Exception(s"Inserting chunk failed: $fail")) + } } private object SharryLogger { diff --git a/modules/store/src/main/scala/sharry/store/FileStoreConfig.scala b/modules/store/src/main/scala/sharry/store/FileStoreConfig.scala new file mode 100644 index 00000000..0bd12850 --- /dev/null +++ b/modules/store/src/main/scala/sharry/store/FileStoreConfig.scala @@ -0,0 +1,33 @@ +package sharry.store + +import fs2.io.file.Path + +sealed trait FileStoreConfig { + def enabled: Boolean + def storeType: FileStoreType +} +object FileStoreConfig { + case class DefaultDatabase(enabled: Boolean) extends FileStoreConfig { + val storeType = FileStoreType.DefaultDatabase + } + + case class FileSystem( + enabled: Boolean, + directory: Path + ) extends FileStoreConfig { + val storeType = FileStoreType.FileSystem + } + + case class S3( + enabled: Boolean, + endpoint: String, + accessKey: String, + secretKey: String, + bucket: String + ) extends FileStoreConfig { + val storeType = FileStoreType.S3 + + override def toString = + s"S3(enabled=$enabled, endpoint=$endpoint, bucket=$bucket, accessKey=$accessKey, secretKey=***)" + } +} diff --git a/modules/store/src/main/scala/sharry/store/FileStoreType.scala b/modules/store/src/main/scala/sharry/store/FileStoreType.scala new file mode 100644 index 00000000..9976e4b9 --- /dev/null +++ b/modules/store/src/main/scala/sharry/store/FileStoreType.scala @@ -0,0 +1,26 @@ +package sharry.store + +import cats.data.NonEmptyList + +sealed trait FileStoreType { self: Product => + def name: String = + productPrefix.toLowerCase +} +object FileStoreType { + case object DefaultDatabase extends FileStoreType + + case object S3 extends FileStoreType + + case object FileSystem extends FileStoreType + + val all: NonEmptyList[FileStoreType] = + NonEmptyList.of(DefaultDatabase, S3, FileSystem) + + def fromString(str: String): Either[String, FileStoreType] = + all + .find(_.name.equalsIgnoreCase(str)) + .toRight(s"Invalid file store type: $str") + + def unsafeFromString(str: String): FileStoreType = + fromString(str).fold(sys.error, identity) +} diff --git a/modules/store/src/main/scala/sharry/store/Store.scala b/modules/store/src/main/scala/sharry/store/Store.scala index d7bc2e66..be8a440d 100644 --- a/modules/store/src/main/scala/sharry/store/Store.scala +++ b/modules/store/src/main/scala/sharry/store/Store.scala @@ -28,6 +28,7 @@ object Store { def create[F[_]: Async]( jdbc: JdbcConfig, chunkSize: ByteSize, + fileStoreCfg: FileStoreConfig, connectEC: ExecutionContext, runMigration: Boolean ): Resource[F, Store[F]] = @@ -42,7 +43,7 @@ object Store { ds.setDriverClassName(jdbc.driverClass) } xa <- Resource.pure(HikariTransactor[F](ds, connectEC)) - fs = FileStore[F](ds, xa, chunkSize.bytes.toInt) + fs = FileStore[F](ds, xa, chunkSize.bytes.toInt, fileStoreCfg) st = new StoreImpl[F](jdbc, fs, xa) _ <- if (runMigration) Resource.eval(st.migrate) else Resource.pure[F, Int](0) } yield st: Store[F] diff --git a/modules/store/src/test/scala/sharry/store/StoreFixture.scala b/modules/store/src/test/scala/sharry/store/StoreFixture.scala index 4e52400f..5ca5e13c 100644 --- a/modules/store/src/test/scala/sharry/store/StoreFixture.scala +++ b/modules/store/src/test/scala/sharry/store/StoreFixture.scala @@ -50,7 +50,7 @@ object StoreFixture { ds <- dataSource(jdbc) connectEC <- ExecutionContexts.cachedThreadPool[F] tx = Transactor.fromDataSource[F](ds, connectEC) - fs = FileStore[F](ds, tx, 64 * 1024) + fs = FileStore[F](ds, tx, 64 * 1024, FileStoreConfig.DefaultDatabase(true)) st = new StoreImpl[F](jdbc, fs, tx) _ <- Resource.eval(st.migrate) } yield st diff --git a/modules/webapp/package-lock.json b/modules/webapp/package-lock.json index 44dac279..e7ebf335 100644 --- a/modules/webapp/package-lock.json +++ b/modules/webapp/package-lock.json @@ -1,8 +1,2359 @@ { "name": "docspell-css", "version": "1.0.0", - "lockfileVersion": 1, + "lockfileVersion": 2, "requires": true, + "packages": { + "": { + "name": "docspell-css", + "version": "1.0.0", + "devDependencies": { + "@fortawesome/fontawesome-free": "^6.1.1", + "@tailwindcss/forms": "^0.5.0", + "autoprefixer": "^10.4.4", + "cssnano": "^5.1.7", + "flag-icon-css": "^3.5.0", + "postcss": "^8.4.12", + "postcss-cli": "^9.1.0", + "postcss-import": "^14.1.0", + "postcss-purgecss": "^2.0.3", + "tailwindcss": "^3.0.24" + } + }, + "node_modules/@fortawesome/fontawesome-free": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-free/-/fontawesome-free-6.1.1.tgz", + "integrity": "sha512-J/3yg2AIXc9wznaVqpHVX3Wa5jwKovVF0AMYSnbmcXTiL3PpRPfF58pzWucCwEiCJBp+hCNRLWClTomD8SseKg==", + "dev": true, + "hasInstallScript": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@tailwindcss/forms": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@tailwindcss/forms/-/forms-0.5.0.tgz", + "integrity": "sha512-KzWugryEBFkmoaYcBE18rs6gthWCFHHO7cAZm2/hv3hwD67AzwP7udSCa22E7R1+CEJL/FfhYsJWrc0b1aeSzw==", + "dev": true, + "dependencies": { + "mini-svg-data-uri": "^1.2.3" + }, + "peerDependencies": { + "tailwindcss": ">=3.0.0 || >= 3.0.0-alpha.1" + } + }, + "node_modules/@trysound/sax": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@trysound/sax/-/sax-0.2.0.tgz", + "integrity": "sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==", + "dev": true, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/acorn": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", + "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", + "dev": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-node": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/acorn-node/-/acorn-node-1.8.2.tgz", + "integrity": "sha512-8mt+fslDufLYntIoPAaIMUe/lrbrehIiwmR3t2k9LljIzoigEPF27eLk2hy8zSGzmR/ogr7zbRKINMo1u0yh5A==", + "dev": true, + "dependencies": { + "acorn": "^7.0.0", + "acorn-walk": "^7.0.0", + "xtend": "^4.0.2" + } + }, + "node_modules/acorn-walk": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-7.2.0.tgz", + "integrity": "sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==", + "dev": true, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anymatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", + "integrity": "sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==", + "dev": true, + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/arg": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.1.tgz", + "integrity": "sha512-e0hDa9H2Z9AwFkk2qDlwhoMYE4eToKarchkQHovNdLTCYMHZHeRjI71crOh+dio4K6u1IcwubQqo79Ga4CyAQA==", + "dev": true + }, + "node_modules/array-union": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-3.0.1.tgz", + "integrity": "sha512-1OvF9IbWwaeiM9VhzYXVQacMibxpXOMYVNIvMtKRyX9SImBXpKcFr8XvFDeEslCyuH/t6KRt7HEO94AlP8Iatw==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/autoprefixer": { + "version": "10.4.4", + "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.4.tgz", + "integrity": "sha512-Tm8JxsB286VweiZ5F0anmbyGiNI3v3wGv3mz9W+cxEDYB/6jbnj6GM9H9mK3wIL8ftgl+C07Lcwb8PG5PCCPzA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/autoprefixer" + } + ], + "dependencies": { + "browserslist": "^4.20.2", + "caniuse-lite": "^1.0.30001317", + "fraction.js": "^4.2.0", + "normalize-range": "^0.1.2", + "picocolors": "^1.0.0", + "postcss-value-parser": "^4.2.0" + }, + "bin": { + "autoprefixer": "bin/autoprefixer" + }, + "engines": { + "node": "^10 || ^12 || >=14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "node_modules/binary-extensions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=", + "dev": true + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, + "dependencies": { + "fill-range": "^7.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browserslist": { + "version": "4.20.2", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.20.2.tgz", + "integrity": "sha512-CQOBCqp/9pDvDbx3xfMi+86pr4KXIf2FDkTTdeuYw8OxS9t898LA1Khq57gtufFILXpfgsSx5woNgsBgvGjpsA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + } + ], + "dependencies": { + "caniuse-lite": "^1.0.30001317", + "electron-to-chromium": "^1.4.84", + "escalade": "^3.1.1", + "node-releases": "^2.0.2", + "picocolors": "^1.0.0" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/camelcase-css": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/camelcase-css/-/camelcase-css-2.0.1.tgz", + "integrity": "sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, + "node_modules/caniuse-api": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/caniuse-api/-/caniuse-api-3.0.0.tgz", + "integrity": "sha512-bsTwuIg/BZZK/vreVTYYbSWoe2F+71P7K5QGEX+pT250DZbfU1MQ5prOKpPR+LL6uWKK3KMwMCAS74QB3Um1uw==", + "dev": true, + "dependencies": { + "browserslist": "^4.0.0", + "caniuse-lite": "^1.0.0", + "lodash.memoize": "^4.1.2", + "lodash.uniq": "^4.5.0" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001332", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001332.tgz", + "integrity": "sha512-10T30NYOEQtN6C11YGg411yebhvpnC6Z102+B95eAsN0oB6KUs01ivE8u+G6FMIRtIrVlYXhL+LUwQ3/hXwDWw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + } + ] + }, + "node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/chalk/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/chalk/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/chalk/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + }, + "node_modules/chalk/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/chokidar": { + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", + "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + ], + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "dev": true, + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/colord": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/colord/-/colord-2.9.2.tgz", + "integrity": "sha512-Uqbg+J445nc1TKn4FoDPS6ZZqAvEDnwrH42yo8B40JSOgSLxMZ/gt3h4nmCtPLQeXhjJJkqBx7SCY35WnIixaQ==", + "dev": true + }, + "node_modules/commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "dev": true, + "engines": { + "node": ">= 10" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "dev": true + }, + "node_modules/css-declaration-sorter": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/css-declaration-sorter/-/css-declaration-sorter-6.2.2.tgz", + "integrity": "sha512-Ufadglr88ZLsrvS11gjeu/40Lw74D9Am/Jpr3LlYm5Q4ZP5KdlUhG+6u2EjyXeZcxmZ2h1ebCKngDjolpeLHpg==", + "dev": true, + "engines": { + "node": "^10 || ^12 || >=14" + }, + "peerDependencies": { + "postcss": "^8.0.9" + } + }, + "node_modules/css-select": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-4.3.0.tgz", + "integrity": "sha512-wPpOYtnsVontu2mODhA19JrqWxNsfdatRKd64kmpRbQgh1KtItko5sTnEpPdpSaJszTOhEMlF/RPz28qj4HqhQ==", + "dev": true, + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.0.1", + "domhandler": "^4.3.1", + "domutils": "^2.8.0", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-tree": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-1.1.3.tgz", + "integrity": "sha512-tRpdppF7TRazZrjJ6v3stzv93qxRcSsFmW6cX0Zm2NVKpxE1WV1HblnghVv9TreireHkqI/VDEsfolRF1p6y7Q==", + "dev": true, + "dependencies": { + "mdn-data": "2.0.14", + "source-map": "^0.6.1" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/css-what": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", + "dev": true, + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/cssesc": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", + "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", + "dev": true, + "bin": { + "cssesc": "bin/cssesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/cssnano": { + "version": "5.1.7", + "resolved": "https://registry.npmjs.org/cssnano/-/cssnano-5.1.7.tgz", + "integrity": "sha512-pVsUV6LcTXif7lvKKW9ZrmX+rGRzxkEdJuVJcp5ftUjWITgwam5LMZOgaTvUrWPkcORBey6he7JKb4XAJvrpKg==", + "dev": true, + "dependencies": { + "cssnano-preset-default": "^5.2.7", + "lilconfig": "^2.0.3", + "yaml": "^1.10.2" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/cssnano" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/cssnano-preset-default": { + "version": "5.2.7", + "resolved": "https://registry.npmjs.org/cssnano-preset-default/-/cssnano-preset-default-5.2.7.tgz", + "integrity": "sha512-JiKP38ymZQK+zVKevphPzNSGHSlTI+AOwlasoSRtSVMUU285O7/6uZyd5NbW92ZHp41m0sSHe6JoZosakj63uA==", + "dev": true, + "dependencies": { + "css-declaration-sorter": "^6.2.2", + "cssnano-utils": "^3.1.0", + "postcss-calc": "^8.2.3", + "postcss-colormin": "^5.3.0", + "postcss-convert-values": "^5.1.0", + "postcss-discard-comments": "^5.1.1", + "postcss-discard-duplicates": "^5.1.0", + "postcss-discard-empty": "^5.1.1", + "postcss-discard-overridden": "^5.1.0", + "postcss-merge-longhand": "^5.1.4", + "postcss-merge-rules": "^5.1.1", + "postcss-minify-font-values": "^5.1.0", + "postcss-minify-gradients": "^5.1.1", + "postcss-minify-params": "^5.1.2", + "postcss-minify-selectors": "^5.2.0", + "postcss-normalize-charset": "^5.1.0", + "postcss-normalize-display-values": "^5.1.0", + "postcss-normalize-positions": "^5.1.0", + "postcss-normalize-repeat-style": "^5.1.0", + "postcss-normalize-string": "^5.1.0", + "postcss-normalize-timing-functions": "^5.1.0", + "postcss-normalize-unicode": "^5.1.0", + "postcss-normalize-url": "^5.1.0", + "postcss-normalize-whitespace": "^5.1.1", + "postcss-ordered-values": "^5.1.1", + "postcss-reduce-initial": "^5.1.0", + "postcss-reduce-transforms": "^5.1.0", + "postcss-svgo": "^5.1.0", + "postcss-unique-selectors": "^5.1.1" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/cssnano-utils": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/cssnano-utils/-/cssnano-utils-3.1.0.tgz", + "integrity": "sha512-JQNR19/YZhz4psLX/rQ9M83e3z2Wf/HdJbryzte4a3NSuafyp9w/I4U+hx5C2S9g41qlstH7DEWnZaaj83OuEA==", + "dev": true, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/csso": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/csso/-/csso-4.2.0.tgz", + "integrity": "sha512-wvlcdIbf6pwKEk7vHj8/Bkc0B4ylXZruLvOgs9doS5eOsOpuodOV2zJChSpkp+pRpYQLQMeF04nr3Z68Sta9jA==", + "dev": true, + "dependencies": { + "css-tree": "^1.1.2" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/defined": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/defined/-/defined-1.0.0.tgz", + "integrity": "sha1-yY2bzvdWdBiOEQlpFRGZ45sfppM=", + "dev": true + }, + "node_modules/dependency-graph": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/dependency-graph/-/dependency-graph-0.11.0.tgz", + "integrity": "sha512-JeMq7fEshyepOWDfcfHK06N3MhyPhz++vtqWhMT5O9A3K42rdsEDpfdVqjaqaAhsw6a+ZqeDvQVtD0hFHQWrzg==", + "dev": true, + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/detective": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/detective/-/detective-5.2.0.tgz", + "integrity": "sha512-6SsIx+nUUbuK0EthKjv0zrdnajCCXVYGmbYYiYjFVpzcjwEs/JMDZ8tPRG29J/HhN56t3GJp2cGSWDRjjot8Pg==", + "dev": true, + "dependencies": { + "acorn-node": "^1.6.1", + "defined": "^1.0.0", + "minimist": "^1.1.1" + }, + "bin": { + "detective": "bin/detective.js" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/didyoumean": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz", + "integrity": "sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==", + "dev": true + }, + "node_modules/dir-glob": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "dev": true, + "dependencies": { + "path-type": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/dlv": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", + "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==", + "dev": true + }, + "node_modules/dom-serializer": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-1.4.1.tgz", + "integrity": "sha512-VHwB3KfrcOOkelEG2ZOfxqLZdfkil8PtJi4P8N2MMXucZq2yLp75ClViUlOVwyoHEDjYU433Aq+5zWP61+RGag==", + "dev": true, + "dependencies": { + "domelementtype": "^2.0.1", + "domhandler": "^4.2.0", + "entities": "^2.0.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ] + }, + "node_modules/domhandler": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-4.3.1.tgz", + "integrity": "sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==", + "dev": true, + "dependencies": { + "domelementtype": "^2.2.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-2.8.0.tgz", + "integrity": "sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==", + "dev": true, + "dependencies": { + "dom-serializer": "^1.0.1", + "domelementtype": "^2.2.0", + "domhandler": "^4.2.0" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.4.111", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.111.tgz", + "integrity": "sha512-/s3+fwhKf1YK4k7btOImOzCQLpUjS6MaPf0ODTNuT4eTM1Bg4itBpLkydhOzJmpmH6Z9eXFyuuK5czsmzRzwtw==", + "dev": true + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/entities": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz", + "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==", + "dev": true, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/fast-glob": { + "version": "3.2.11", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.11.tgz", + "integrity": "sha512-xrO3+1bxSo3ZVHAnqzyuewYT6aMFHRAd4Kcs92MAonjwQZLsK9d0SF1IyQ3k5PoirxTW0Oe/RqFgMQ6TcNE5Ew==", + "dev": true, + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.4" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fastq": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", + "integrity": "sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==", + "dev": true, + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/flag-icon-css": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/flag-icon-css/-/flag-icon-css-3.5.0.tgz", + "integrity": "sha512-pgJnJLrtb0tcDgU1fzGaQXmR8h++nXvILJ+r5SmOXaaL/2pocunQo2a8TAXhjQnBpRLPtZ1KCz/TYpqeNuE2ew==", + "deprecated": "The project has been renamed to flag-icons", + "dev": true + }, + "node_modules/fraction.js": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.2.0.tgz", + "integrity": "sha512-MhLuK+2gUcnZe8ZHlaaINnQLl0xRIGRfcGk2yl8xoQAfHrSsL3rYu6FCmBdkdbhc9EPlwyGHewaRsvwRMJtAlA==", + "dev": true, + "engines": { + "node": "*" + }, + "funding": { + "type": "patreon", + "url": "https://www.patreon.com/infusion" + } + }, + "node_modules/fs-extra": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.0.tgz", + "integrity": "sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "dev": true + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "dev": true + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-stdin": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-9.0.0.tgz", + "integrity": "sha512-dVKBjfWisLAicarI2Sf+JuBE/DghV4UzNAVe9yhEJuzeREd3JhOTE9cUaJTeSa77fsbQUK3pcOpJfM59+VKZaA==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/globby": { + "version": "12.2.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-12.2.0.tgz", + "integrity": "sha512-wiSuFQLZ+urS9x2gGPl1H5drc5twabmm4m2gTR27XDFyjUHJUNsS8o/2aKyIF6IoBaR630atdher0XJ5g6OMmA==", + "dev": true, + "dependencies": { + "array-union": "^3.0.1", + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.7", + "ignore": "^5.1.9", + "merge2": "^1.4.1", + "slash": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.9", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.9.tgz", + "integrity": "sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ==", + "dev": true + }, + "node_modules/has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/ignore": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz", + "integrity": "sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dev": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-core-module": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.8.1.tgz", + "integrity": "sha512-SdNCUs284hr40hFTFP6l0IfZ/RSrMXF3qgoRHd3/79unUTvrFO/JoXwkGm+5J/Oe3E/b5GsnG330uUNgRpu1PA==", + "dev": true, + "dependencies": { + "has": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dev": true, + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/lilconfig": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-2.0.4.tgz", + "integrity": "sha512-bfTIN7lEsiooCocSISTWXkiWJkRqtL9wYtYy+8EK3Y41qh3mpwPU0ycTOgjdY9ErwXCc8QyrQp82bdL0Xkm9yA==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/lodash.memoize": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", + "integrity": "sha1-vMbEmkKihA7Zl/Mj6tpezRguC/4=", + "dev": true + }, + "node_modules/lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha1-0CJTc662Uq3BvILklFM5qEJ1R3M=", + "dev": true + }, + "node_modules/mdn-data": { + "version": "2.0.14", + "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.14.tgz", + "integrity": "sha512-dn6wd0uw5GsdswPFfsgMp5NSB0/aDe6fK94YJV/AJDYXL6HVLWBsxeq7js7Ad+mU2K9LAlwpk6kN2D5mwCPVow==", + "dev": true + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.4.tgz", + "integrity": "sha512-pRmzw/XUcwXGpD9aI9q/0XOwLNygjETJ8y0ao0wdqprrzDa4YnxLcz7fQRZr8voh8V10kGhABbNcHVk5wHgWwg==", + "dev": true, + "dependencies": { + "braces": "^3.0.1", + "picomatch": "^2.2.3" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mini-svg-data-uri": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/mini-svg-data-uri/-/mini-svg-data-uri-1.4.3.tgz", + "integrity": "sha512-gSfqpMRC8IxghvMcxzzmMnWpXAChSA+vy4cia33RgerMS8Fex95akUyQZPbxJJmeBGiGmK7n/1OpUX8ksRjIdA==", + "dev": true, + "bin": { + "mini-svg-data-uri": "cli.js" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minimist": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==", + "dev": true + }, + "node_modules/nanoid": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.3.tgz", + "integrity": "sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==", + "dev": true, + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/node-releases": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.3.tgz", + "integrity": "sha512-maHFz6OLqYxz+VQyCAtA3PTX4UP/53pa05fyDNc9CwjvJ0yEh6+xBwKsgCxMNhS8taUKBFYxfuiaD9U/55iFaw==", + "dev": true + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/normalize-range": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/normalize-range/-/normalize-range-0.1.2.tgz", + "integrity": "sha1-LRDAa9/TEuqXd2laTShDlFa3WUI=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/normalize-url": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz", + "integrity": "sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/nth-check": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.0.1.tgz", + "integrity": "sha512-it1vE95zF6dTT9lBsYbxvqh0Soy4SPowchj0UBGj/V6cTPnXXtQOPUbhZ6CmGzAD/rW22LQK6E96pcdJXk4A4w==", + "dev": true, + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, + "node_modules/object-hash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", + "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dev": true, + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true + }, + "node_modules/path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/picocolors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", + "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", + "dev": true + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postcss": { + "version": "8.4.12", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.12.tgz", + "integrity": "sha512-lg6eITwYe9v6Hr5CncVbK70SoioNQIq81nsaG86ev5hAidQvmOeETBqs7jm43K2F5/Ley3ytDtriImV6TpNiSg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + } + ], + "dependencies": { + "nanoid": "^3.3.1", + "picocolors": "^1.0.0", + "source-map-js": "^1.0.2" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/postcss-calc": { + "version": "8.2.4", + "resolved": "https://registry.npmjs.org/postcss-calc/-/postcss-calc-8.2.4.tgz", + "integrity": "sha512-SmWMSJmB8MRnnULldx0lQIyhSNvuDl9HfrZkaqqE/WHAhToYsAvDq+yAsA/kIyINDszOp3Rh0GFoNuH5Ypsm3Q==", + "dev": true, + "dependencies": { + "postcss-selector-parser": "^6.0.9", + "postcss-value-parser": "^4.2.0" + }, + "peerDependencies": { + "postcss": "^8.2.2" + } + }, + "node_modules/postcss-cli": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/postcss-cli/-/postcss-cli-9.1.0.tgz", + "integrity": "sha512-zvDN2ADbWfza42sAnj+O2uUWyL0eRL1V+6giM2vi4SqTR3gTYy8XzcpfwccayF2szcUif0HMmXiEaDv9iEhcpw==", + "dev": true, + "dependencies": { + "chokidar": "^3.3.0", + "dependency-graph": "^0.11.0", + "fs-extra": "^10.0.0", + "get-stdin": "^9.0.0", + "globby": "^12.0.0", + "picocolors": "^1.0.0", + "postcss-load-config": "^3.0.0", + "postcss-reporter": "^7.0.0", + "pretty-hrtime": "^1.0.3", + "read-cache": "^1.0.0", + "slash": "^4.0.0", + "yargs": "^17.0.0" + }, + "bin": { + "postcss": "index.js" + }, + "engines": { + "node": ">=12" + }, + "peerDependencies": { + "postcss": "^8.0.0" + } + }, + "node_modules/postcss-colormin": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/postcss-colormin/-/postcss-colormin-5.3.0.tgz", + "integrity": "sha512-WdDO4gOFG2Z8n4P8TWBpshnL3JpmNmJwdnfP2gbk2qBA8PWwOYcmjmI/t3CmMeL72a7Hkd+x/Mg9O2/0rD54Pg==", + "dev": true, + "dependencies": { + "browserslist": "^4.16.6", + "caniuse-api": "^3.0.0", + "colord": "^2.9.1", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-convert-values": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/postcss-convert-values/-/postcss-convert-values-5.1.0.tgz", + "integrity": "sha512-GkyPbZEYJiWtQB0KZ0X6qusqFHUepguBCNFi9t5JJc7I2OTXG7C0twbTLvCfaKOLl3rSXmpAwV7W5txd91V84g==", + "dev": true, + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-discard-comments": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/postcss-discard-comments/-/postcss-discard-comments-5.1.1.tgz", + "integrity": "sha512-5JscyFmvkUxz/5/+TB3QTTT9Gi9jHkcn8dcmmuN68JQcv3aQg4y88yEHHhwFB52l/NkaJ43O0dbksGMAo49nfQ==", + "dev": true, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-discard-duplicates": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/postcss-discard-duplicates/-/postcss-discard-duplicates-5.1.0.tgz", + "integrity": "sha512-zmX3IoSI2aoenxHV6C7plngHWWhUOV3sP1T8y2ifzxzbtnuhk1EdPwm0S1bIUNaJ2eNbWeGLEwzw8huPD67aQw==", + "dev": true, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-discard-empty": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/postcss-discard-empty/-/postcss-discard-empty-5.1.1.tgz", + "integrity": "sha512-zPz4WljiSuLWsI0ir4Mcnr4qQQ5e1Ukc3i7UfE2XcrwKK2LIPIqE5jxMRxO6GbI3cv//ztXDsXwEWT3BHOGh3A==", + "dev": true, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-discard-overridden": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/postcss-discard-overridden/-/postcss-discard-overridden-5.1.0.tgz", + "integrity": "sha512-21nOL7RqWR1kasIVdKs8HNqQJhFxLsyRfAnUDm4Fe4t4mCWL9OJiHvlHPjcd8zc5Myu89b/7wZDnOSjFgeWRtw==", + "dev": true, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-import": { + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/postcss-import/-/postcss-import-14.1.0.tgz", + "integrity": "sha512-flwI+Vgm4SElObFVPpTIT7SU7R3qk2L7PyduMcokiaVKuWv9d/U+Gm/QAd8NDLuykTWTkcrjOeD2Pp1rMeBTGw==", + "dev": true, + "dependencies": { + "postcss-value-parser": "^4.0.0", + "read-cache": "^1.0.0", + "resolve": "^1.1.7" + }, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "postcss": "^8.0.0" + } + }, + "node_modules/postcss-js": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/postcss-js/-/postcss-js-4.0.0.tgz", + "integrity": "sha512-77QESFBwgX4irogGVPgQ5s07vLvFqWr228qZY+w6lW599cRlK/HmnlivnnVUxkjHnCu4J16PDMHcH+e+2HbvTQ==", + "dev": true, + "dependencies": { + "camelcase-css": "^2.0.1" + }, + "engines": { + "node": "^12 || ^14 || >= 16" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + "peerDependencies": { + "postcss": "^8.3.3" + } + }, + "node_modules/postcss-load-config": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-3.1.3.tgz", + "integrity": "sha512-5EYgaM9auHGtO//ljHH+v/aC/TQ5LHXtL7bQajNAUBKUVKiYE8rYpFms7+V26D9FncaGe2zwCoPQsFKb5zF/Hw==", + "dev": true, + "dependencies": { + "lilconfig": "^2.0.4", + "yaml": "^1.10.2" + }, + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + "peerDependencies": { + "ts-node": ">=9.0.0" + }, + "peerDependenciesMeta": { + "ts-node": { + "optional": true + } + } + }, + "node_modules/postcss-merge-longhand": { + "version": "5.1.4", + "resolved": "https://registry.npmjs.org/postcss-merge-longhand/-/postcss-merge-longhand-5.1.4.tgz", + "integrity": "sha512-hbqRRqYfmXoGpzYKeW0/NCZhvNyQIlQeWVSao5iKWdyx7skLvCfQFGIUsP9NUs3dSbPac2IC4Go85/zG+7MlmA==", + "dev": true, + "dependencies": { + "postcss-value-parser": "^4.2.0", + "stylehacks": "^5.1.0" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-merge-rules": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/postcss-merge-rules/-/postcss-merge-rules-5.1.1.tgz", + "integrity": "sha512-8wv8q2cXjEuCcgpIB1Xx1pIy8/rhMPIQqYKNzEdyx37m6gpq83mQQdCxgIkFgliyEnKvdwJf/C61vN4tQDq4Ww==", + "dev": true, + "dependencies": { + "browserslist": "^4.16.6", + "caniuse-api": "^3.0.0", + "cssnano-utils": "^3.1.0", + "postcss-selector-parser": "^6.0.5" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-minify-font-values": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/postcss-minify-font-values/-/postcss-minify-font-values-5.1.0.tgz", + "integrity": "sha512-el3mYTgx13ZAPPirSVsHqFzl+BBBDrXvbySvPGFnQcTI4iNslrPaFq4muTkLZmKlGk4gyFAYUBMH30+HurREyA==", + "dev": true, + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-minify-gradients": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/postcss-minify-gradients/-/postcss-minify-gradients-5.1.1.tgz", + "integrity": "sha512-VGvXMTpCEo4qHTNSa9A0a3D+dxGFZCYwR6Jokk+/3oB6flu2/PnPXAh2x7x52EkY5xlIHLm+Le8tJxe/7TNhzw==", + "dev": true, + "dependencies": { + "colord": "^2.9.1", + "cssnano-utils": "^3.1.0", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-minify-params": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/postcss-minify-params/-/postcss-minify-params-5.1.2.tgz", + "integrity": "sha512-aEP+p71S/urY48HWaRHasyx4WHQJyOYaKpQ6eXl8k0kxg66Wt/30VR6/woh8THgcpRbonJD5IeD+CzNhPi1L8g==", + "dev": true, + "dependencies": { + "browserslist": "^4.16.6", + "cssnano-utils": "^3.1.0", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-minify-selectors": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/postcss-minify-selectors/-/postcss-minify-selectors-5.2.0.tgz", + "integrity": "sha512-vYxvHkW+iULstA+ctVNx0VoRAR4THQQRkG77o0oa4/mBS0OzGvvzLIvHDv/nNEM0crzN2WIyFU5X7wZhaUK3RA==", + "dev": true, + "dependencies": { + "postcss-selector-parser": "^6.0.5" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-nested": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/postcss-nested/-/postcss-nested-5.0.6.tgz", + "integrity": "sha512-rKqm2Fk0KbA8Vt3AdGN0FB9OBOMDVajMG6ZCf/GoHgdxUJ4sBFp0A/uMIRm+MJUdo33YXEtjqIz8u7DAp8B7DA==", + "dev": true, + "dependencies": { + "postcss-selector-parser": "^6.0.6" + }, + "engines": { + "node": ">=12.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + "peerDependencies": { + "postcss": "^8.2.14" + } + }, + "node_modules/postcss-normalize-charset": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/postcss-normalize-charset/-/postcss-normalize-charset-5.1.0.tgz", + "integrity": "sha512-mSgUJ+pd/ldRGVx26p2wz9dNZ7ji6Pn8VWBajMXFf8jk7vUoSrZ2lt/wZR7DtlZYKesmZI680qjr2CeFF2fbUg==", + "dev": true, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-normalize-display-values": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/postcss-normalize-display-values/-/postcss-normalize-display-values-5.1.0.tgz", + "integrity": "sha512-WP4KIM4o2dazQXWmFaqMmcvsKmhdINFblgSeRgn8BJ6vxaMyaJkwAzpPpuvSIoG/rmX3M+IrRZEz2H0glrQNEA==", + "dev": true, + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-normalize-positions": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/postcss-normalize-positions/-/postcss-normalize-positions-5.1.0.tgz", + "integrity": "sha512-8gmItgA4H5xiUxgN/3TVvXRoJxkAWLW6f/KKhdsH03atg0cB8ilXnrB5PpSshwVu/dD2ZsRFQcR1OEmSBDAgcQ==", + "dev": true, + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-normalize-repeat-style": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/postcss-normalize-repeat-style/-/postcss-normalize-repeat-style-5.1.0.tgz", + "integrity": "sha512-IR3uBjc+7mcWGL6CtniKNQ4Rr5fTxwkaDHwMBDGGs1x9IVRkYIT/M4NelZWkAOBdV6v3Z9S46zqaKGlyzHSchw==", + "dev": true, + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-normalize-string": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/postcss-normalize-string/-/postcss-normalize-string-5.1.0.tgz", + "integrity": "sha512-oYiIJOf4T9T1N4i+abeIc7Vgm/xPCGih4bZz5Nm0/ARVJ7K6xrDlLwvwqOydvyL3RHNf8qZk6vo3aatiw/go3w==", + "dev": true, + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-normalize-timing-functions": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/postcss-normalize-timing-functions/-/postcss-normalize-timing-functions-5.1.0.tgz", + "integrity": "sha512-DOEkzJ4SAXv5xkHl0Wa9cZLF3WCBhF3o1SKVxKQAa+0pYKlueTpCgvkFAHfk+Y64ezX9+nITGrDZeVGgITJXjg==", + "dev": true, + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-normalize-unicode": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/postcss-normalize-unicode/-/postcss-normalize-unicode-5.1.0.tgz", + "integrity": "sha512-J6M3MizAAZ2dOdSjy2caayJLQT8E8K9XjLce8AUQMwOrCvjCHv24aLC/Lps1R1ylOfol5VIDMaM/Lo9NGlk1SQ==", + "dev": true, + "dependencies": { + "browserslist": "^4.16.6", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-normalize-url": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/postcss-normalize-url/-/postcss-normalize-url-5.1.0.tgz", + "integrity": "sha512-5upGeDO+PVthOxSmds43ZeMeZfKH+/DKgGRD7TElkkyS46JXAUhMzIKiCa7BabPeIy3AQcTkXwVVN7DbqsiCew==", + "dev": true, + "dependencies": { + "normalize-url": "^6.0.1", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-normalize-whitespace": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/postcss-normalize-whitespace/-/postcss-normalize-whitespace-5.1.1.tgz", + "integrity": "sha512-83ZJ4t3NUDETIHTa3uEg6asWjSBYL5EdkVB0sDncx9ERzOKBVJIUeDO9RyA9Zwtig8El1d79HBp0JEi8wvGQnA==", + "dev": true, + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-ordered-values": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/postcss-ordered-values/-/postcss-ordered-values-5.1.1.tgz", + "integrity": "sha512-7lxgXF0NaoMIgyihL/2boNAEZKiW0+HkMhdKMTD93CjW8TdCy2hSdj8lsAo+uwm7EDG16Da2Jdmtqpedl0cMfw==", + "dev": true, + "dependencies": { + "cssnano-utils": "^3.1.0", + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-purgecss": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/postcss-purgecss/-/postcss-purgecss-2.0.3.tgz", + "integrity": "sha512-cuQin5PgZzvDe7EjW4S27iM6p4ZNz4iBEPmBrAykXm2WyaBtri1sA4ZVn/zECN7x3uxeADwDq1u4VDY5C9iusg==", + "dev": true, + "dependencies": { + "postcss": "7.0.26", + "purgecss": "^2.0.3" + } + }, + "node_modules/postcss-purgecss/node_modules/commander": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz", + "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, + "node_modules/postcss-purgecss/node_modules/postcss": { + "version": "7.0.26", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-7.0.26.tgz", + "integrity": "sha512-IY4oRjpXWYshuTDFxMVkJDtWIk2LhsTlu8bZnbEJA4+bYT16Lvpo8Qv6EvDumhYRgzjZl489pmsY3qVgJQ08nA==", + "dev": true, + "dependencies": { + "chalk": "^2.4.2", + "source-map": "^0.6.1", + "supports-color": "^6.1.0" + }, + "engines": { + "node": ">=6.0.0" + }, + "funding": { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + } + }, + "node_modules/postcss-purgecss/node_modules/purgecss": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/purgecss/-/purgecss-2.3.0.tgz", + "integrity": "sha512-BE5CROfVGsx2XIhxGuZAT7rTH9lLeQx/6M0P7DTXQH4IUc3BBzs9JUzt4yzGf3JrH9enkeq6YJBe9CTtkm1WmQ==", + "dev": true, + "dependencies": { + "commander": "^5.0.0", + "glob": "^7.0.0", + "postcss": "7.0.32", + "postcss-selector-parser": "^6.0.2" + }, + "bin": { + "purgecss": "bin/purgecss" + } + }, + "node_modules/postcss-purgecss/node_modules/purgecss/node_modules/postcss": { + "version": "7.0.32", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-7.0.32.tgz", + "integrity": "sha512-03eXong5NLnNCD05xscnGKGDZ98CyzoqPSMjOe6SuoQY7Z2hIj0Ld1g/O/UQRuOle2aRtiIRDg9tDcTGAkLfKw==", + "dev": true, + "dependencies": { + "chalk": "^2.4.2", + "source-map": "^0.6.1", + "supports-color": "^6.1.0" + }, + "engines": { + "node": ">=6.0.0" + }, + "funding": { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + } + }, + "node_modules/postcss-reduce-initial": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/postcss-reduce-initial/-/postcss-reduce-initial-5.1.0.tgz", + "integrity": "sha512-5OgTUviz0aeH6MtBjHfbr57tml13PuedK/Ecg8szzd4XRMbYxH4572JFG067z+FqBIf6Zp/d+0581glkvvWMFw==", + "dev": true, + "dependencies": { + "browserslist": "^4.16.6", + "caniuse-api": "^3.0.0" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-reduce-transforms": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/postcss-reduce-transforms/-/postcss-reduce-transforms-5.1.0.tgz", + "integrity": "sha512-2fbdbmgir5AvpW9RLtdONx1QoYG2/EtqpNQbFASDlixBbAYuTcJ0dECwlqNqH7VbaUnEnh8SrxOe2sRIn24XyQ==", + "dev": true, + "dependencies": { + "postcss-value-parser": "^4.2.0" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-reporter": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/postcss-reporter/-/postcss-reporter-7.0.5.tgz", + "integrity": "sha512-glWg7VZBilooZGOFPhN9msJ3FQs19Hie7l5a/eE6WglzYqVeH3ong3ShFcp9kDWJT1g2Y/wd59cocf9XxBtkWA==", + "dev": true, + "dependencies": { + "picocolors": "^1.0.0", + "thenby": "^1.3.4" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-selector-parser": { + "version": "6.0.9", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.9.tgz", + "integrity": "sha512-UO3SgnZOVTwu4kyLR22UQ1xZh086RyNZppb7lLAKBFK8a32ttG5i87Y/P3+2bRSjZNyJ1B7hfFNo273tKe9YxQ==", + "dev": true, + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-svgo": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/postcss-svgo/-/postcss-svgo-5.1.0.tgz", + "integrity": "sha512-D75KsH1zm5ZrHyxPakAxJWtkyXew5qwS70v56exwvw542d9CRtTo78K0WeFxZB4G7JXKKMbEZtZayTGdIky/eA==", + "dev": true, + "dependencies": { + "postcss-value-parser": "^4.2.0", + "svgo": "^2.7.0" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-unique-selectors": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/postcss-unique-selectors/-/postcss-unique-selectors-5.1.1.tgz", + "integrity": "sha512-5JiODlELrz8L2HwxfPnhOWZYWDxVHWL83ufOv84NrcgipI7TaeRsatAhK4Tr2/ZiYldpK/wBvw5BD3qfaK96GA==", + "dev": true, + "dependencies": { + "postcss-selector-parser": "^6.0.5" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/postcss-value-parser": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", + "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", + "dev": true + }, + "node_modules/pretty-hrtime": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/pretty-hrtime/-/pretty-hrtime-1.0.3.tgz", + "integrity": "sha1-t+PqQkNaTJsnWdmeDyAesZWALuE=", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/quick-lru": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", + "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/read-cache": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", + "integrity": "sha1-5mTvMRYRZsl1HNvo28+GtftY93Q=", + "dev": true, + "dependencies": { + "pify": "^2.3.0" + } + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve": { + "version": "1.22.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.0.tgz", + "integrity": "sha512-Hhtrw0nLeSrFQ7phPp4OOcVjLPIeMnRlr5mcnVuMe7M/7eBn98A3hmFRLoFo3DLZkivSYwhRUJTyPyWAk56WLw==", + "dev": true, + "dependencies": { + "is-core-module": "^2.8.1", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "dev": true, + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/slash": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", + "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-js": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", + "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stable": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/stable/-/stable-0.1.8.tgz", + "integrity": "sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w==", + "deprecated": "Modern JS already guarantees Array#sort() is a stable sort, so this library is deprecated. See the compatibility table on MDN: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/sort#browser_compatibility", + "dev": true + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/stylehacks": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/stylehacks/-/stylehacks-5.1.0.tgz", + "integrity": "sha512-SzLmvHQTrIWfSgljkQCw2++C9+Ne91d/6Sp92I8c5uHTcy/PgeHamwITIbBW9wnFTY/3ZfSXR9HIL6Ikqmcu6Q==", + "dev": true, + "dependencies": { + "browserslist": "^4.16.6", + "postcss-selector-parser": "^6.0.4" + }, + "engines": { + "node": "^10 || ^12 || >=14.0" + }, + "peerDependencies": { + "postcss": "^8.2.15" + } + }, + "node_modules/supports-color": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz", + "integrity": "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/svgo": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/svgo/-/svgo-2.8.0.tgz", + "integrity": "sha512-+N/Q9kV1+F+UeWYoSiULYo4xYSDQlTgb+ayMobAXPwMnLvop7oxKMo9OzIrX5x3eS4L4f2UHhc9axXwY8DpChg==", + "dev": true, + "dependencies": { + "@trysound/sax": "0.2.0", + "commander": "^7.2.0", + "css-select": "^4.1.3", + "css-tree": "^1.1.3", + "csso": "^4.2.0", + "picocolors": "^1.0.0", + "stable": "^0.1.8" + }, + "bin": { + "svgo": "bin/svgo" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/tailwindcss": { + "version": "3.0.24", + "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.0.24.tgz", + "integrity": "sha512-H3uMmZNWzG6aqmg9q07ZIRNIawoiEcNFKDfL+YzOPuPsXuDXxJxB9icqzLgdzKNwjG3SAro2h9SYav8ewXNgig==", + "dev": true, + "dependencies": { + "arg": "^5.0.1", + "chokidar": "^3.5.3", + "color-name": "^1.1.4", + "detective": "^5.2.0", + "didyoumean": "^1.2.2", + "dlv": "^1.1.3", + "fast-glob": "^3.2.11", + "glob-parent": "^6.0.2", + "is-glob": "^4.0.3", + "lilconfig": "^2.0.5", + "normalize-path": "^3.0.0", + "object-hash": "^3.0.0", + "picocolors": "^1.0.0", + "postcss": "^8.4.12", + "postcss-js": "^4.0.0", + "postcss-load-config": "^3.1.4", + "postcss-nested": "5.0.6", + "postcss-selector-parser": "^6.0.10", + "postcss-value-parser": "^4.2.0", + "quick-lru": "^5.1.1", + "resolve": "^1.22.0" + }, + "bin": { + "tailwind": "lib/cli.js", + "tailwindcss": "lib/cli.js" + }, + "engines": { + "node": ">=12.13.0" + }, + "peerDependencies": { + "postcss": "^8.0.9" + } + }, + "node_modules/tailwindcss/node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/tailwindcss/node_modules/lilconfig": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-2.0.5.tgz", + "integrity": "sha512-xaYmXZtTHPAw5m+xLN8ab9C+3a8YmV3asNSPOATITbtwrfbwaLJj8h66H1WMIpALCkqsIzK3h7oQ+PdX+LQ9Eg==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/tailwindcss/node_modules/postcss-load-config": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-3.1.4.tgz", + "integrity": "sha512-6DiM4E7v4coTE4uzA8U//WhtPwyhiim3eyjEMFCnUpzbrkK9wJHgKDT2mR+HbtSrd/NubVaYTOpSpjUl8NQeRg==", + "dev": true, + "dependencies": { + "lilconfig": "^2.0.5", + "yaml": "^1.10.2" + }, + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + "peerDependencies": { + "postcss": ">=8.0.9", + "ts-node": ">=9.0.0" + }, + "peerDependenciesMeta": { + "postcss": { + "optional": true + }, + "ts-node": { + "optional": true + } + } + }, + "node_modules/tailwindcss/node_modules/postcss-selector-parser": { + "version": "6.0.10", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.10.tgz", + "integrity": "sha512-IQ7TZdoaqbT+LCpShg46jnZVlhWD2w6iQYAcYXfHARZ7X1t/UGhhceQDs5X0cGqKvYlHNOuv7Oa1xmb0oQuA3w==", + "dev": true, + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/thenby": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/thenby/-/thenby-1.3.4.tgz", + "integrity": "sha512-89Gi5raiWA3QZ4b2ePcEwswC3me9JIg+ToSgtE0JWeCynLnLxNr/f9G+xfo9K+Oj4AFdom8YNJjibIARTJmapQ==", + "dev": true + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "dev": true, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", + "dev": true + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "dev": true, + "engines": { + "node": ">=0.4" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/yaml": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz", + "integrity": "sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, + "node_modules/yargs": { + "version": "17.3.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.3.1.tgz", + "integrity": "sha512-WUANQeVgjLbNsEmGk20f+nlHgOqzRFpiGWVaBrYGYIGANIIu3lWjoyi0fNlFmJkvfhCZ6BXINe7/W2O2bV4iaA==", + "dev": true, + "dependencies": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.0.0.tgz", + "integrity": "sha512-z9kApYUOCwoeZ78rfRYYWdiU/iNL6mwwYlkkZfJoyMR1xps+NEBX5X7XmRpxkZHhXJ6+Ey00IwKxBBSW9FIjyA==", + "dev": true, + "engines": { + "node": ">=12" + } + } + }, "dependencies": { "@fortawesome/fontawesome-free": { "version": "6.1.1", @@ -309,7 +2660,8 @@ "version": "6.2.2", "resolved": "https://registry.npmjs.org/css-declaration-sorter/-/css-declaration-sorter-6.2.2.tgz", "integrity": "sha512-Ufadglr88ZLsrvS11gjeu/40Lw74D9Am/Jpr3LlYm5Q4ZP5KdlUhG+6u2EjyXeZcxmZ2h1ebCKngDjolpeLHpg==", - "dev": true + "dev": true, + "requires": {} }, "css-select": { "version": "4.3.0", @@ -398,7 +2750,8 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/cssnano-utils/-/cssnano-utils-3.1.0.tgz", "integrity": "sha512-JQNR19/YZhz4psLX/rQ9M83e3z2Wf/HdJbryzte4a3NSuafyp9w/I4U+hx5C2S9g41qlstH7DEWnZaaj83OuEA==", - "dev": true + "dev": true, + "requires": {} }, "csso": { "version": "4.2.0", @@ -957,25 +3310,29 @@ "version": "5.1.1", "resolved": "https://registry.npmjs.org/postcss-discard-comments/-/postcss-discard-comments-5.1.1.tgz", "integrity": "sha512-5JscyFmvkUxz/5/+TB3QTTT9Gi9jHkcn8dcmmuN68JQcv3aQg4y88yEHHhwFB52l/NkaJ43O0dbksGMAo49nfQ==", - "dev": true + "dev": true, + "requires": {} }, "postcss-discard-duplicates": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/postcss-discard-duplicates/-/postcss-discard-duplicates-5.1.0.tgz", "integrity": "sha512-zmX3IoSI2aoenxHV6C7plngHWWhUOV3sP1T8y2ifzxzbtnuhk1EdPwm0S1bIUNaJ2eNbWeGLEwzw8huPD67aQw==", - "dev": true + "dev": true, + "requires": {} }, "postcss-discard-empty": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/postcss-discard-empty/-/postcss-discard-empty-5.1.1.tgz", "integrity": "sha512-zPz4WljiSuLWsI0ir4Mcnr4qQQ5e1Ukc3i7UfE2XcrwKK2LIPIqE5jxMRxO6GbI3cv//ztXDsXwEWT3BHOGh3A==", - "dev": true + "dev": true, + "requires": {} }, "postcss-discard-overridden": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/postcss-discard-overridden/-/postcss-discard-overridden-5.1.0.tgz", "integrity": "sha512-21nOL7RqWR1kasIVdKs8HNqQJhFxLsyRfAnUDm4Fe4t4mCWL9OJiHvlHPjcd8zc5Myu89b/7wZDnOSjFgeWRtw==", - "dev": true + "dev": true, + "requires": {} }, "postcss-import": { "version": "14.1.0", @@ -1082,7 +3439,8 @@ "version": "5.1.0", "resolved": "https://registry.npmjs.org/postcss-normalize-charset/-/postcss-normalize-charset-5.1.0.tgz", "integrity": "sha512-mSgUJ+pd/ldRGVx26p2wz9dNZ7ji6Pn8VWBajMXFf8jk7vUoSrZ2lt/wZR7DtlZYKesmZI680qjr2CeFF2fbUg==", - "dev": true + "dev": true, + "requires": {} }, "postcss-normalize-display-values": { "version": "5.1.0", diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 3251b4c6..5d85d4dc 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -4,7 +4,7 @@ object Dependencies { val BcryptVersion = "0.4" val BetterMonadicForVersion = "0.3.1" - val BinnyVersion = "0.5.0" + val BinnyVersion = "0.5.0+5-d1bcbf48-SNAPSHOT" val CirceVersion = "0.14.2" val ClipboardJsVersion = "2.0.6" val DoobieVersion = "1.0.0-RC2" @@ -95,6 +95,8 @@ object Dependencies { val binny = Seq( "com.github.eikek" %% "binny-core" % BinnyVersion, "com.github.eikek" %% "binny-jdbc" % BinnyVersion, + "com.github.eikek" %% "binny-minio" % BinnyVersion, + "com.github.eikek" %% "binny-fs" % BinnyVersion, "com.github.eikek" %% "binny-tika-detect" % BinnyVersion ) From 0cc2cb029f51fdb6b03512ab9a867df235c8de47 Mon Sep 17 00:00:00 2001 From: eikek Date: Sat, 25 Jun 2022 11:39:42 +0200 Subject: [PATCH 2/4] Add support for multiple file storage backends Sharry can now store files in the filesystem, S3 or the database. --- .../scala/sharry/backend/BackendApp.scala | 1 + .../scala/sharry/backend/config/Config.scala | 11 +-- .../scala/sharry/backend/share/Queries.scala | 29 +------- .../src/main/resources/reference.conf | 14 ++++ .../sharry/restserver/config/Config.scala | 43 ++++++++--- .../scala/sharry/store/ComputeChecksum.scala | 71 ++++++++++++++++++ .../sharry/store/ComputeChecksumConfig.scala | 25 +++++++ .../main/scala/sharry/store/FileStore.scala | 73 +++++++++++++------ .../src/main/scala/sharry/store/Store.scala | 17 ++++- .../sharry/store/doobie/AttributeStore.scala | 13 ++-- .../sharry/store/records/RFileMeta.scala | 4 +- .../scala/sharry/store/StoreFixture.scala | 10 ++- project/Dependencies.scala | 2 +- 13 files changed, 234 insertions(+), 79 deletions(-) create mode 100644 modules/store/src/main/scala/sharry/store/ComputeChecksum.scala create mode 100644 modules/store/src/main/scala/sharry/store/ComputeChecksumConfig.scala diff --git a/modules/backend/src/main/scala/sharry/backend/BackendApp.scala b/modules/backend/src/main/scala/sharry/backend/BackendApp.scala index b09d2867..cfdf4c82 100644 --- a/modules/backend/src/main/scala/sharry/backend/BackendApp.scala +++ b/modules/backend/src/main/scala/sharry/backend/BackendApp.scala @@ -58,6 +58,7 @@ object BackendApp { store <- Store.create( cfg.jdbc, cfg.share.chunkSize, + cfg.computeChecksum, cfg.files.defaultStoreConfig, connectEC, true diff --git a/modules/backend/src/main/scala/sharry/backend/config/Config.scala b/modules/backend/src/main/scala/sharry/backend/config/Config.scala index 26aed8cb..df3cb8df 100644 --- a/modules/backend/src/main/scala/sharry/backend/config/Config.scala +++ b/modules/backend/src/main/scala/sharry/backend/config/Config.scala @@ -1,13 +1,13 @@ package sharry.backend.config import cats.data.ValidatedNec - +import cats.syntax.all._ import sharry.backend.auth.AuthConfig import sharry.backend.job.CleanupConfig import sharry.backend.mail.MailConfig import sharry.backend.share.ShareConfig import sharry.backend.signup.SignupConfig -import sharry.store.JdbcConfig +import sharry.store.{ComputeChecksumConfig, JdbcConfig} case class Config( jdbc: JdbcConfig, @@ -16,10 +16,11 @@ case class Config( share: ShareConfig, cleanup: CleanupConfig, mail: MailConfig, - files: Files + files: Files, + computeChecksum: ComputeChecksumConfig ) { def validate: ValidatedNec[String, Config] = - files.validate.map(fc => copy(files = fc)) - + (files.validate, computeChecksum.validate) + .mapN((fc, cc) => copy(files = fc, computeChecksum = cc)) } diff --git a/modules/backend/src/main/scala/sharry/backend/share/Queries.scala b/modules/backend/src/main/scala/sharry/backend/share/Queries.scala index 47b5998e..645bde9c 100644 --- a/modules/backend/src/main/scala/sharry/backend/share/Queries.scala +++ b/modules/backend/src/main/scala/sharry/backend/share/Queries.scala @@ -346,31 +346,8 @@ object Queries { } def deleteFile[F[_]: Async](store: Store[F])(fileMetaId: Ident) = { - def deleteChunk(fid: Ident, chunk: Int): F[Int] = - store - .transact( - Sql - .deleteFrom( - FileChunkCols.table, - Sql.and(FileChunkCols.fileId.is(fid), FileChunkCols.chunkNr.is(chunk)) - ) - .update - .run - ) - - // When deleting large files, doing it in one transaction may blow - // memory. It is not important to be all-or-nothing, so here each - // chunk is deleted in one tx. This is slow, of course, but can be - // moved to a background thread. The cleanup job also detects - // orphaned files and removes them. - def deleteFileData(fid: Ident): F[Unit] = - Stream - .iterate(0)(_ + 1) - .covary[F] - .evalMap(n => deleteChunk(fid, n)) - .takeWhile(_ > 0) - .compile - .drain + val deleteFileData = + store.fileStore.delete(fileMetaId) def deleteFileMeta(fid: Ident): F[Int] = store.transact(for { @@ -378,7 +355,7 @@ object Queries { c <- Sql.deleteFrom(RFileMeta.table, RFileMeta.Columns.id.is(fid)).update.run } yield a + c) - deleteFileData(fileMetaId) *> deleteFileMeta(fileMetaId) + deleteFileData *> deleteFileMeta(fileMetaId) } def deleteShare[F[_]: Async](share: Ident, background: Boolean)( diff --git a/modules/restserver/src/main/resources/reference.conf b/modules/restserver/src/main/resources/reference.conf index bfede73f..dc361e1a 100644 --- a/modules/restserver/src/main/resources/reference.conf +++ b/modules/restserver/src/main/resources/reference.conf @@ -298,6 +298,20 @@ sharry.restserver { } } + # Checksums of uploaded files are computed in the background. + compute-checksum = { + # How many ids to queue at most. If full, uploading blocks until the queue + capacity = 5000 + + # How many checksums to compute in parallel, must be >= 0. If 0, + # they are computed sequentially. + parallel = -1 + + # If true, the `parallel` option above is ignored and it will be + # set to the number of available cores - 1. + use-default = true + } + # Configuration for registering new users at the local database. # Accounts registered here are checked via the `internal' # authentication plugin as described above. diff --git a/modules/restserver/src/main/scala/sharry/restserver/config/Config.scala b/modules/restserver/src/main/scala/sharry/restserver/config/Config.scala index 28e0e776..30aa9988 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/config/Config.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/config/Config.scala @@ -1,5 +1,7 @@ package sharry.restserver.config +import cats.data.{Validated, ValidatedNec} +import cats.syntax.all._ import sharry.backend.config.{Config => BackendConfig} import sharry.common._ import sharry.logging.LogConfig @@ -14,25 +16,42 @@ case class Config( backend: BackendConfig ) { - def validate: List[String] = { + def validate: ValidatedNec[String, Config] = { val threshold = Duration.seconds(30) - List( - if (backend.auth.sessionValid >= (webapp.authRenewal + threshold)) "" + val validSession = + if (backend.auth.sessionValid >= (webapp.authRenewal + threshold)) + Validated.validNec(()) else - s"session-valid time (${backend.auth.sessionValid}) must be " + - s"at least 30s greater than webapp.auth-renewal (${webapp.authRenewal})", - if (backend.share.maxValidity >= webapp.defaultValidity) "" + Validated.invalidNec( + s"session-valid time (${backend.auth.sessionValid}) must be " + + s"at least 30s greater than webapp.auth-renewal (${webapp.authRenewal})" + ) + + val validValidity = + if (backend.share.maxValidity >= webapp.defaultValidity) Validated.validNec(()) else - s"Default validity (${webapp.defaultValidity}) is larger than maximum ${backend.share.maxValidity}!", - Config.validateTheme(webapp.initialTheme) - ).filter(_.nonEmpty) + Validated.invalidNec( + s"Default validity (${webapp.defaultValidity}) is larger than maximum ${backend.share.maxValidity}!" + ) + + val valdidTheme = + Config.validateTheme(webapp.initialTheme) match { + case "" => Validated.validNec(()) + case str => Validated.invalidNec(str) + } + + val validBackend = backend.validate.map(c => copy(backend = c)) + (validSession, validValidity, valdidTheme, validBackend) + .mapN((_, _, _, c) => c) } def validOrThrow: Config = validate match { - case Nil => this - case errs => - sys.error(s"Configuration is not valid: ${errs.mkString(", ")}") + case Validated.Valid(cfg) => cfg + case Validated.Invalid(errs) => + sys.error( + s"Configuration is not valid: ${errs.toNonEmptyList.toList.mkString(", ")}" + ) } } diff --git a/modules/store/src/main/scala/sharry/store/ComputeChecksum.scala b/modules/store/src/main/scala/sharry/store/ComputeChecksum.scala new file mode 100644 index 00000000..b1a0f588 --- /dev/null +++ b/modules/store/src/main/scala/sharry/store/ComputeChecksum.scala @@ -0,0 +1,71 @@ +package sharry.store + +import binny._ +import binny.util.Stopwatch +import cats.effect._ +import cats.effect.std.Queue +import cats.syntax.all._ +import fs2.{Pipe, Stream} +import sharry.common.{ByteSize, Ident, Timestamp} +import sharry.store.records.RFileMeta + +trait ComputeChecksum[F[_]] { + + def submit(id: BinaryId, hint: Hint): F[Unit] + + def computeSync(id: BinaryId, hint: Hint, attr: AttributeNameSet): F[RFileMeta] + + def consumeAll(attr: AttributeNameSet): Stream[F, RFileMeta] +} +object ComputeChecksum { + def apply[F[_]: Async]( + store: BinaryStore[F], + config: ComputeChecksumConfig + ): F[ComputeChecksum[F]] = + for { + queue <- Queue.bounded[F, Entry](config.capacity) + } yield new ComputeChecksum[F] { + private[this] val logger = sharry.logging.getLogger[F] + + def submit(id: BinaryId, hint: Hint): F[Unit] = + queue.offer(Entry(id, hint)) + + def computeSync(id: BinaryId, hint: Hint, select: AttributeNameSet): F[RFileMeta] = + for { + _ <- logger.debug(s"Compute $select for binary ${id.id}") + w <- Stopwatch.start + attr <- store + .computeAttr(id, hint) + .run(select) + .getOrElse(BinaryAttributes.empty) + _ <- Stopwatch.show(w)(time => + logger.debug(s"Computing $select for ${id.id} took $time") + ) + now <- Timestamp.current[F] + fm = RFileMeta( + Ident.unsafe(id.id), + now, + attr.contentType.contentType, + ByteSize(attr.length), + attr.sha256 + ) + } yield fm + + def consumeAll(attr: AttributeNameSet): Stream[F, RFileMeta] = + logger.stream + .info( + s"Starting computing checksum of submitted binaries: $config" + ) + .drain ++ + Stream.repeatEval(queue.take).through(computePipe(attr)) + + private def computePipe( + attr: AttributeNameSet + ): Pipe[F, Entry, RFileMeta] = in => + if (config.parallel > 1) + in.parEvalMap(config.parallel)(e => computeSync(e.id, e.hint, attr)) + else in.evalMap(e => computeSync(e.id, e.hint, attr)) + } + + private case class Entry(id: BinaryId, hint: Hint) +} diff --git a/modules/store/src/main/scala/sharry/store/ComputeChecksumConfig.scala b/modules/store/src/main/scala/sharry/store/ComputeChecksumConfig.scala new file mode 100644 index 00000000..e63369de --- /dev/null +++ b/modules/store/src/main/scala/sharry/store/ComputeChecksumConfig.scala @@ -0,0 +1,25 @@ +package sharry.store + +import cats.data.{Validated, ValidatedNec} + +case class ComputeChecksumConfig(capacity: Int, parallel: Int, useDefault: Boolean) { + + def validate: ValidatedNec[String, ComputeChecksumConfig] = + if (capacity <= 0) Validated.invalidNec("Capacity must be > 0!") + else if (useDefault) + Validated.validNec(copy(parallel = ComputeChecksumConfig.defaultParallel)) + else if (parallel < 0) Validated.invalidNec("Parallel must be >= 0!") + else Validated.validNec(this) + + override def toString: String = + s"ComputeChecksumConfig(capacity=$capacity, parallel=$parallel, default: $useDefault)" +} + +object ComputeChecksumConfig { + val defaultParallel: Int = math.min(8, math.max(parallelMin, parallelMax)) + + val default = ComputeChecksumConfig(5000, 0, true) + + private def parallelMin: Int = 1 + private def parallelMax: Int = Runtime.getRuntime.availableProcessors() - 1 +} diff --git a/modules/store/src/main/scala/sharry/store/FileStore.scala b/modules/store/src/main/scala/sharry/store/FileStore.scala index 7f4a0f8a..08938afe 100644 --- a/modules/store/src/main/scala/sharry/store/FileStore.scala +++ b/modules/store/src/main/scala/sharry/store/FileStore.scala @@ -33,7 +33,11 @@ trait FileStore[F[_]] { def insertMeta(meta: RFileMeta): F[Unit] + def updateChecksum(meta: RFileMeta): F[Unit] + def addChunk(id: Ident, hint: Hint, chunkDef: ChunkDef, data: Chunk[Byte]): F[Unit] + + def computeAttributes: ComputeChecksum[F] } object FileStore { @@ -42,51 +46,59 @@ object FileStore { ds: DataSource, xa: Transactor[F], chunkSize: Int, + computeChecksumConfig: ComputeChecksumConfig, config: FileStoreConfig - ): FileStore[F] = + ): F[FileStore[F]] = config match { case FileStoreConfig.DefaultDatabase(_) => - forDatabase(ds, xa, chunkSize) + forDatabase(ds, xa, chunkSize, computeChecksumConfig) case c: FileStoreConfig.S3 => - forS3(xa, c, chunkSize) + forS3(xa, c, chunkSize, computeChecksumConfig) case c: FileStoreConfig.FileSystem => - forFs(xa, c, chunkSize) + forFs(xa, c, chunkSize, computeChecksumConfig) } def forDatabase[F[_]: Async]( ds: DataSource, xa: Transactor[F], - chunkSize: Int - ): FileStore[F] = { + chunkSize: Int, + computeChecksumConfig: ComputeChecksumConfig + ): F[FileStore[F]] = { val cfg = JdbcStoreConfig("filechunk", chunkSize, TikaContentTypeDetect.default) val as = AttributeStore(xa) val logger = SharryLogger(sharry.logging.getLogger[F]) - val bs = GenericJdbcStore[F](ds, logger, cfg, as) - new Impl[F](bs, as, chunkSize) + val bs = GenericJdbcStore[F](ds, logger, cfg) + ComputeChecksum[F](bs, computeChecksumConfig).map(cc => + new Impl[F](bs, as, chunkSize, cc) + ) } def forFs[F[_]: Async]( xa: Transactor[F], fsCfg: FileStoreConfig.FileSystem, - chunkSize: Int - ): FileStore[F] = { + chunkSize: Int, + computeChecksumConfig: ComputeChecksumConfig + ): F[FileStore[F]] = { val as = AttributeStore(xa) val logger = SharryLogger(sharry.logging.getLogger[F]) val cfg = FsChunkedStoreConfig .defaults(fsCfg.directory) .copy(chunkSize = chunkSize) .withContentTypeDetect(TikaContentTypeDetect.default) - val bs = FsChunkedBinaryStore(cfg, logger, as) - new Impl[F](bs, as, chunkSize) + val bs = FsChunkedBinaryStore(logger, cfg) + ComputeChecksum[F](bs, computeChecksumConfig).map(cc => + new Impl[F](bs, as, chunkSize, cc) + ) } def forS3[F[_]: Async]( xa: Transactor[F], s3: FileStoreConfig.S3, - chunkSize: Int - ): FileStore[F] = { + chunkSize: Int, + computeChecksumConfig: ComputeChecksumConfig + ): F[FileStore[F]] = { val as = AttributeStore(xa) val logger = SharryLogger(sharry.logging.getLogger[F]) val cfg = MinioConfig @@ -98,14 +110,17 @@ object FileStore { ) .copy(chunkSize = chunkSize) .withContentTypeDetect(TikaContentTypeDetect.default) - val bs = MinioChunkedBinaryStore(cfg, as, logger) - new Impl[F](bs, as, chunkSize) + val bs = MinioChunkedBinaryStore(cfg, logger) + ComputeChecksum[F](bs, computeChecksumConfig).map(cc => + new Impl[F](bs, as, chunkSize, cc) + ) } final private class Impl[F[_]: Sync]( bs: ChunkedBinaryStore[F], attrStore: AttributeStore[F], - val chunkSize: Int + val chunkSize: Int, + val computeAttributes: ComputeChecksum[F] ) extends FileStore[F] { def delete(id: Ident): F[Unit] = @@ -119,16 +134,22 @@ object FileStore { def insert(data: Binary[F], hint: Hint, created: Timestamp): F[RFileMeta] = data - .through(bs.insert(hint)) - .evalTap(id => attrStore.updateCreated(id, created)) - .evalMap(id => attrStore.findMeta(id).value) - .unNoneTerminate + .through(bs.insert) + .evalMap { id => + computeAttributes.submit(id, hint) *> + computeAttributes + .computeSync(id, hint, AttributeName.excludeSha256) + .flatTap(insertMeta) + } .compile .lastOrError def insertMeta(meta: RFileMeta): F[Unit] = attrStore.saveMeta(meta) + def updateChecksum(meta: RFileMeta): F[Unit] = + attrStore.updateChecksum(meta.id, meta.checksum) + def addChunk( id: Ident, hint: Hint, @@ -136,8 +157,14 @@ object FileStore { data: Chunk[Byte] ): F[Unit] = bs.insertChunk(BinaryId(id.id), chunkDef, hint, data.toByteVector).flatMap { - case _: InsertChunkResult.Success => ().pure[F] - case fail => Sync[F].raiseError(new Exception(s"Inserting chunk failed: $fail")) + case InsertChunkResult.Complete => + computeAttributes.submit(BinaryId(id.id), hint) *> + computeAttributes + .computeSync(BinaryId(id.id), hint, AttributeName.excludeSha256) + .flatMap(insertMeta) + case InsertChunkResult.Incomplete => ().pure[F] + case fail: InsertChunkResult.Failure => + Sync[F].raiseError(new Exception(s"Inserting chunk failed: $fail")) } } diff --git a/modules/store/src/main/scala/sharry/store/Store.scala b/modules/store/src/main/scala/sharry/store/Store.scala index be8a440d..038b9731 100644 --- a/modules/store/src/main/scala/sharry/store/Store.scala +++ b/modules/store/src/main/scala/sharry/store/Store.scala @@ -1,15 +1,13 @@ package sharry.store import scala.concurrent.ExecutionContext - import cats.effect._ import fs2._ - import sharry.common.ByteSize import sharry.store.doobie.StoreImpl - import _root_.doobie._ import _root_.doobie.hikari.HikariTransactor +import binny.AttributeName import com.zaxxer.hikari.HikariDataSource trait Store[F[_]] { @@ -28,6 +26,7 @@ object Store { def create[F[_]: Async]( jdbc: JdbcConfig, chunkSize: ByteSize, + computeChecksumConfig: ComputeChecksumConfig, fileStoreCfg: FileStoreConfig, connectEC: ExecutionContext, runMigration: Boolean @@ -43,7 +42,17 @@ object Store { ds.setDriverClassName(jdbc.driverClass) } xa <- Resource.pure(HikariTransactor[F](ds, connectEC)) - fs = FileStore[F](ds, xa, chunkSize.bytes.toInt, fileStoreCfg) + fs <- Resource.eval( + FileStore[F](ds, xa, chunkSize.bytes.toInt, computeChecksumConfig, fileStoreCfg) + ) + _ <- Async[F].background( + fs.computeAttributes + .consumeAll(AttributeName.all) + .evalMap(fs.updateChecksum) + .compile + .drain + ) + st = new StoreImpl[F](jdbc, fs, xa) _ <- if (runMigration) Resource.eval(st.migrate) else Resource.pure[F, Int](0) } yield st: Store[F] diff --git a/modules/store/src/main/scala/sharry/store/doobie/AttributeStore.scala b/modules/store/src/main/scala/sharry/store/doobie/AttributeStore.scala index c57f885a..1635a015 100644 --- a/modules/store/src/main/scala/sharry/store/doobie/AttributeStore.scala +++ b/modules/store/src/main/scala/sharry/store/doobie/AttributeStore.scala @@ -3,16 +3,14 @@ package sharry.store.doobie import cats.data.OptionT import cats.effect._ import cats.implicits._ - import sharry.common._ import sharry.store.records.RFileMeta - import binny._ import doobie._ import doobie.implicits._ +import scodec.bits.ByteVector -final private[store] class AttributeStore[F[_]: Sync](xa: Transactor[F]) - extends BinaryAttributeStore[F] { +final private[store] class AttributeStore[F[_]: Sync](xa: Transactor[F]) { def saveAttr(id: BinaryId, attrs: F[BinaryAttributes]): F[Unit] = for { @@ -40,10 +38,13 @@ final private[store] class AttributeStore[F[_]: Sync](xa: Transactor[F]) OptionT(RFileMeta.findById(Ident.unsafe(id.id)).transact(xa)) def saveMeta(fm: RFileMeta): F[Unit] = - RFileMeta.upsert(fm).transact(xa).map(_ => ()) + RFileMeta.upsert(fm).transact(xa).void def updateCreated(id: BinaryId, created: Timestamp): F[Unit] = - RFileMeta.updateCreated(Ident.unsafe(id.id), created).transact(xa).map(_ => ()) + RFileMeta.updateCreated(Ident.unsafe(id.id), created).transact(xa).void + + def updateChecksum(id: Ident, checksum: ByteVector): F[Unit] = + RFileMeta.updateChecksum(id, checksum).transact(xa).void } object AttributeStore { diff --git a/modules/store/src/main/scala/sharry/store/records/RFileMeta.scala b/modules/store/src/main/scala/sharry/store/records/RFileMeta.scala index 751b651a..c4416d86 100644 --- a/modules/store/src/main/scala/sharry/store/records/RFileMeta.scala +++ b/modules/store/src/main/scala/sharry/store/records/RFileMeta.scala @@ -48,7 +48,6 @@ object RFileMeta { table, Columns.id.is(r.id), Sql.commas( - Columns.created.setTo(r.created), Columns.checksum.setTo(r.checksum), Columns.mimetype.setTo(r.mimetype), Columns.length.setTo(r.length) @@ -71,6 +70,9 @@ object RFileMeta { def updateCreated(id: Ident, created: Timestamp): ConnectionIO[Int] = Sql.updateRow(table, Columns.id.is(id), Columns.created.setTo(created)).update.run + def updateChecksum(id: Ident, checksum: ByteVector): ConnectionIO[Int] = + Sql.updateRow(table, Columns.id.is(id), Columns.checksum.setTo(checksum)).update.run + def delete(id: Ident): ConnectionIO[Int] = Sql.deleteFrom(table, Columns.id.is(id)).update.run } diff --git a/modules/store/src/test/scala/sharry/store/StoreFixture.scala b/modules/store/src/test/scala/sharry/store/StoreFixture.scala index 5ca5e13c..6ad6e06c 100644 --- a/modules/store/src/test/scala/sharry/store/StoreFixture.scala +++ b/modules/store/src/test/scala/sharry/store/StoreFixture.scala @@ -50,7 +50,15 @@ object StoreFixture { ds <- dataSource(jdbc) connectEC <- ExecutionContexts.cachedThreadPool[F] tx = Transactor.fromDataSource[F](ds, connectEC) - fs = FileStore[F](ds, tx, 64 * 1024, FileStoreConfig.DefaultDatabase(true)) + fs <- Resource.eval( + FileStore[F]( + ds, + tx, + 64 * 1024, + ComputeChecksumConfig.default, + FileStoreConfig.DefaultDatabase(true) + ) + ) st = new StoreImpl[F](jdbc, fs, tx) _ <- Resource.eval(st.migrate) } yield st diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 5d85d4dc..de6bb4ce 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -4,7 +4,7 @@ object Dependencies { val BcryptVersion = "0.4" val BetterMonadicForVersion = "0.3.1" - val BinnyVersion = "0.5.0+5-d1bcbf48-SNAPSHOT" + val BinnyVersion = "0.5.0+17-7dd06f69-SNAPSHOT" val CirceVersion = "0.14.2" val ClipboardJsVersion = "2.0.6" val DoobieVersion = "1.0.0-RC2" From 59351526f5552b78160f910d5fedd69be37015a1 Mon Sep 17 00:00:00 2001 From: eikek Date: Sat, 25 Jun 2022 23:53:13 +0200 Subject: [PATCH 3/4] Copy files between file stores --- .../scala/sharry/backend/BackendApp.scala | 6 ++ .../scala/sharry/backend/config/Config.scala | 3 +- .../backend/config/CopyFilesConfig.scala | 17 ++++ .../config/{Files.scala => FilesConfig.scala} | 23 ++++- .../scala/sharry/backend/files/OFiles.scala | 77 +++++++++++++++ modules/microsite/docs/doc/configure.md | 87 ++++++++++++++++ .../src/main/resources/reference.conf | 21 ++++ .../scala/sharry/restserver/RestApp.scala | 2 - .../scala/sharry/restserver/RestAppImpl.scala | 14 ++- .../scala/sharry/restserver/RestServer.scala | 3 +- .../sharry/restserver/config/Config.scala | 1 + .../scala/sharry/store/ComputeChecksum.scala | 6 +- .../main/scala/sharry/store/FileStore.scala | 99 ++----------------- .../scala/sharry/store/FileStoreConfig.scala | 62 ++++++++++++ .../src/main/scala/sharry/store/Store.scala | 12 +-- .../sharry/store/doobie/AttributeStore.scala | 2 + 16 files changed, 320 insertions(+), 115 deletions(-) create mode 100644 modules/backend/src/main/scala/sharry/backend/config/CopyFilesConfig.scala rename modules/backend/src/main/scala/sharry/backend/config/{Files.scala => FilesConfig.scala} (55%) create mode 100644 modules/backend/src/main/scala/sharry/backend/files/OFiles.scala diff --git a/modules/backend/src/main/scala/sharry/backend/BackendApp.scala b/modules/backend/src/main/scala/sharry/backend/BackendApp.scala index cfdf4c82..365221ac 100644 --- a/modules/backend/src/main/scala/sharry/backend/BackendApp.scala +++ b/modules/backend/src/main/scala/sharry/backend/BackendApp.scala @@ -8,6 +8,7 @@ import sharry.backend.account._ import sharry.backend.alias.OAlias import sharry.backend.auth.Login import sharry.backend.config.Config +import sharry.backend.files.OFiles import sharry.backend.job.PeriodicCleanup import sharry.backend.mail.OMail import sharry.backend.share.OShare @@ -29,6 +30,8 @@ trait BackendApp[F[_]] { def share: OShare[F] def mail: OMail[F] + + def files: OFiles[F] } object BackendApp { @@ -41,6 +44,7 @@ object BackendApp { aliasImpl <- OAlias[F](store) shareImpl <- OShare[F](store, cfg.share) mailImpl <- OMail[F](store, cfg.mail, JavaMailEmil[F]()) + filesImpl <- Resource.pure(OFiles[F](store, cfg.files)) } yield new BackendApp[F] { val login: Login[F] = loginImpl val signup: OSignup[F] = signupImpl @@ -48,6 +52,7 @@ object BackendApp { val alias: OAlias[F] = aliasImpl val share: OShare[F] = shareImpl val mail: OMail[F] = mailImpl + val files: OFiles[F] = filesImpl } def apply[F[_]: Async]( @@ -63,6 +68,7 @@ object BackendApp { connectEC, true ) + backend <- create(cfg, store) _ <- PeriodicCleanup.resource(cfg.cleanup, cfg.signup, backend.share, backend.signup) diff --git a/modules/backend/src/main/scala/sharry/backend/config/Config.scala b/modules/backend/src/main/scala/sharry/backend/config/Config.scala index df3cb8df..6a814622 100644 --- a/modules/backend/src/main/scala/sharry/backend/config/Config.scala +++ b/modules/backend/src/main/scala/sharry/backend/config/Config.scala @@ -2,6 +2,7 @@ package sharry.backend.config import cats.data.ValidatedNec import cats.syntax.all._ + import sharry.backend.auth.AuthConfig import sharry.backend.job.CleanupConfig import sharry.backend.mail.MailConfig @@ -16,7 +17,7 @@ case class Config( share: ShareConfig, cleanup: CleanupConfig, mail: MailConfig, - files: Files, + files: FilesConfig, computeChecksum: ComputeChecksumConfig ) { diff --git a/modules/backend/src/main/scala/sharry/backend/config/CopyFilesConfig.scala b/modules/backend/src/main/scala/sharry/backend/config/CopyFilesConfig.scala new file mode 100644 index 00000000..6b3529d0 --- /dev/null +++ b/modules/backend/src/main/scala/sharry/backend/config/CopyFilesConfig.scala @@ -0,0 +1,17 @@ +package sharry.backend.config + +import cats.data.{Validated, ValidatedNec} + +import sharry.common.Ident + +case class CopyFilesConfig( + enable: Boolean, + source: Ident, + target: Ident, + parallel: Int +) { + + def validate: ValidatedNec[String, Unit] = + if (source == target) Validated.invalidNec("Source and target must not be the same") + else Validated.validNec(()) +} diff --git a/modules/backend/src/main/scala/sharry/backend/config/Files.scala b/modules/backend/src/main/scala/sharry/backend/config/FilesConfig.scala similarity index 55% rename from modules/backend/src/main/scala/sharry/backend/config/Files.scala rename to modules/backend/src/main/scala/sharry/backend/config/FilesConfig.scala index cf481db2..6d8ffd9f 100644 --- a/modules/backend/src/main/scala/sharry/backend/config/Files.scala +++ b/modules/backend/src/main/scala/sharry/backend/config/FilesConfig.scala @@ -6,7 +6,11 @@ import cats.syntax.all._ import sharry.common.Ident import sharry.store.FileStoreConfig -case class Files(defaultStore: Ident, stores: Map[Ident, FileStoreConfig]) { +case class FilesConfig( + defaultStore: Ident, + stores: Map[Ident, FileStoreConfig], + copyFiles: CopyFilesConfig +) { val enabledStores: Map[Ident, FileStoreConfig] = stores.view.filter(_._2.enabled).toMap @@ -17,7 +21,7 @@ case class Files(defaultStore: Ident, stores: Map[Ident, FileStoreConfig]) { sys.error(s"Store '${defaultStore.id}' not found. Is it enabled?") ) - def validate: ValidatedNec[String, Files] = { + def validate: ValidatedNec[String, FilesConfig] = { val storesEmpty = if (enabledStores.isEmpty) Validated.invalidNec( @@ -32,6 +36,19 @@ case class Files(defaultStore: Ident, stores: Map[Ident, FileStoreConfig]) { Validated.invalidNec(s"Default file store not present: ${defaultStore}") } - (storesEmpty |+| defaultStorePresent).map(_ => this) + val validCopyStores = + if (!copyFiles.enable) Validated.validNec(()) + else { + val exist = enabledStores.contains(copyFiles.source) && + enabledStores.contains(copyFiles.target) + if (exist) Validated.validNec(()) + else + Validated.invalidNec( + s"The source or target name for the copy-files section doesn't exist in the list of enabled file stores." + ) + } + + (storesEmpty |+| defaultStorePresent |+| validCopyStores |+| copyFiles.validate) + .map(_ => this) } } diff --git a/modules/backend/src/main/scala/sharry/backend/files/OFiles.scala b/modules/backend/src/main/scala/sharry/backend/files/OFiles.scala new file mode 100644 index 00000000..be691012 --- /dev/null +++ b/modules/backend/src/main/scala/sharry/backend/files/OFiles.scala @@ -0,0 +1,77 @@ +package sharry.backend.files + +import cats.data.OptionT +import cats.effect._ +import cats.syntax.all._ + +import sharry.backend.config.FilesConfig +import sharry.common.Ident +import sharry.store.{FileStoreConfig, Store} + +import binny.{AttributeName, CopyTool} + +trait OFiles[F[_]] { + + def computeBackgroundChecksum: Resource[F, F[Outcome[F, Throwable, Unit]]] + + def copyFiles(source: FileStoreConfig, target: FileStoreConfig): F[Int] + + def copyFiles(source: Ident, target: Ident): F[Int] +} + +object OFiles { + + def apply[F[_]: Async]( + store: Store[F], + fileConfig: FilesConfig + ): OFiles[F] = + new OFiles[F] { + private[this] val logger = sharry.logging.getLogger[F] + + def computeBackgroundChecksum: Resource[F, F[Outcome[F, Throwable, Unit]]] = + Async[F].background( + store.fileStore.computeAttributes + .consumeAll(AttributeName.all) + .evalMap(store.fileStore.updateChecksum) + .compile + .drain + ) + + def copyFiles(source: Ident, target: Ident): F[Int] = + (for { + src <- OptionT.fromOption[F](fileConfig.enabledStores.get(source)) + trg <- OptionT.fromOption[F](fileConfig.enabledStores.get(target)) + r <- OptionT.liftF(copyFiles(src, trg)) + } yield r).getOrElseF( + Sync[F].raiseError( + new IllegalArgumentException( + s"Source or target store not found for keys: ${source.id} and ${target.id}" + ) + ) + ) + + def copyFiles(source: FileStoreConfig, target: FileStoreConfig): F[Int] = { + val src = store.fileStore.createBinaryStore(source) + val trg = store.fileStore.createBinaryStore(target) + val binnyLogger = FileStoreConfig.SharryLogger(logger) + + logger.info(s"Starting to copy $source -> $target") *> + CopyTool + .copyAll( + binnyLogger, + src, + trg, + store.fileStore.chunkSize, + fileConfig.copyFiles.parallel + ) + .flatTap { r => + logger.info( + s"Copied ${r.success} files, ${r.exist} existed already and ${r.notFound} were not found." + ) *> (if (r.failed.nonEmpty) + logger.warn(s"Failed to copy these files: ${r.failed}") + else ().pure[F]) + } + .map(_.success) + } + } +} diff --git a/modules/microsite/docs/doc/configure.md b/modules/microsite/docs/doc/configure.md index c34bf4ea..2ee66d39 100644 --- a/modules/microsite/docs/doc/configure.md +++ b/modules/microsite/docs/doc/configure.md @@ -138,6 +138,93 @@ The following steps must be done manually: Then add the above setting into your config file. Test files can be found [here](https://www.eicar.org/?page_id=3950). +### Files + +By default, the files are also stored in the configured database. This +works quite well, but you can also choose to store the files somewhere +else: either in the local filesystem or in an S3 compatible object +storage. + +This is configured in the `files` section: + +``` + # How files are stored. + files { + # The id of an enabled store from the `stores` array that should + # be used. + default-store = "database" + + # A list of possible file stores. Each entry must have a unique + # id. The `type` is one of: default-database, filesystem, s3. + # + # All stores with enabled=false are + # removed from the list. The `default-store` must be enabled. + stores = { + database = + { enabled = true + type = "default-database" + } + + filesystem = + { enabled = false + type = "file-system" + directory = "/some/directory" + } + + minio = + { enabled = false + type = "s3" + endpoint = "http://localhost:9000" + access-key = "username" + secret-key = "password" + bucket = "sharry" + } + } + ... + } + +``` + +This config section requires to define a file store in `stores` and +then reference the key in `default-store`. Within `stores` you can +define what kind of storage to use via the `type` attribute. This can +be one of: `s3`, `file-system` or `default-database`. Depending on +`type` more information is required. For example, the filesystem needs +the base directory to use, or the above example for +[Minio](https://min.io) requires credentials and a bucket. + +#### Changing file stores + +The last part in the `files` section looks like this: + +``` + # Allows to copy files from one store to the other *before* sharry + # will be available. It is recommended to set the `enabled` flag to + # false afterwards and restart sharry. + # + # Files are only copied, they are *not* removed from the source + # store. + copy-files = { + enable = false + + # A key in the `backend.files` config identifying the store to + # copy from. + source = "database" + + # A key in the `backend.files` config identifying the store to + # copy the files to. + target = "minio" + + # How many files to copy in parallel. + parallel = 2 + } +``` + +This allows you to have Sharry copy all files from one store to the +other on startup. So to change from `database` to `minio` as in the +example, set `enabled` to `true` and change the `default-store` to +`minio` (the target store). When starting up sharry it will first copy +all files to the `minio` store before it is available. ### Bind diff --git a/modules/restserver/src/main/resources/reference.conf b/modules/restserver/src/main/resources/reference.conf index dc361e1a..3158a930 100644 --- a/modules/restserver/src/main/resources/reference.conf +++ b/modules/restserver/src/main/resources/reference.conf @@ -296,6 +296,27 @@ sharry.restserver { bucket = "sharry" } } + + # Allows to copy files from one store to the other *before* sharry + # will be available. It is recommended to set the `enabled` flag to + # false afterwards and restart sharry. + # + # Files are only copied, they are *not* removed from the source + # store. + copy-files = { + enable = false + + # A key in the `backend.files` config identifying the store to + # copy from. + source = "database" + + # A key in the `backend.files` config identifying the store to + # copy the files to. + target = "minio" + + # How many files to copy in parallel. + parallel = 2 + } } # Checksums of uploaded files are computed in the background. diff --git a/modules/restserver/src/main/scala/sharry/restserver/RestApp.scala b/modules/restserver/src/main/scala/sharry/restserver/RestApp.scala index 8b10124a..a1d36056 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/RestApp.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/RestApp.scala @@ -7,7 +7,5 @@ trait RestApp[F[_]] { def config: Config - def init: F[Unit] - def backend: BackendApp[F] } diff --git a/modules/restserver/src/main/scala/sharry/restserver/RestAppImpl.scala b/modules/restserver/src/main/scala/sharry/restserver/RestAppImpl.scala index 8fe04aa2..19fe2895 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/RestAppImpl.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/RestAppImpl.scala @@ -11,8 +11,15 @@ import sharry.restserver.config.Config final class RestAppImpl[F[_]: Sync](val config: Config, val backend: BackendApp[F]) extends RestApp[F] { - def init: F[Unit] = - Sync[F].pure(()) + def init: Resource[F, Unit] = + for { + _ <- backend.files.computeBackgroundChecksum.void + cf = config.backend.files.copyFiles + _ <- + if (cf.enable) + Resource.eval(backend.files.copyFiles(cf.source, cf.target)) + else Resource.pure[F, Int](0) + } yield () def shutdown: F[Unit] = ().pure[F] @@ -28,7 +35,6 @@ object RestAppImpl { for { backend <- BackendApp(cfg.backend, connectEC) app = new RestAppImpl[F](cfg, backend) - appR <- Resource.make(app.init.map(_ => app))(_.shutdown) + appR <- app.init.onFinalize(app.shutdown).as(app) } yield appR - } diff --git a/modules/restserver/src/main/scala/sharry/restserver/RestServer.scala b/modules/restserver/src/main/scala/sharry/restserver/RestServer.scala index 62261f2d..ae146b1d 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/RestServer.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/RestServer.scala @@ -5,7 +5,7 @@ import scala.concurrent.duration._ import cats.data.Kleisli import cats.data.OptionT import cats.effect._ -import cats.implicits._ +import cats.syntax.all._ import fs2.Stream import sharry.backend.auth.AuthToken @@ -33,7 +33,6 @@ object RestServer { val templates = TemplateRoutes[F](cfg) val app = for { restApp <- RestAppImpl.create[F](cfg, pools.connectEC) - _ <- Resource.eval(restApp.init) client <- BlazeClientBuilder[F].resource httpApp = Router( diff --git a/modules/restserver/src/main/scala/sharry/restserver/config/Config.scala b/modules/restserver/src/main/scala/sharry/restserver/config/Config.scala index 30aa9988..9c9536cc 100644 --- a/modules/restserver/src/main/scala/sharry/restserver/config/Config.scala +++ b/modules/restserver/src/main/scala/sharry/restserver/config/Config.scala @@ -2,6 +2,7 @@ package sharry.restserver.config import cats.data.{Validated, ValidatedNec} import cats.syntax.all._ + import sharry.backend.config.{Config => BackendConfig} import sharry.common._ import sharry.logging.LogConfig diff --git a/modules/store/src/main/scala/sharry/store/ComputeChecksum.scala b/modules/store/src/main/scala/sharry/store/ComputeChecksum.scala index b1a0f588..99bbf0a9 100644 --- a/modules/store/src/main/scala/sharry/store/ComputeChecksum.scala +++ b/modules/store/src/main/scala/sharry/store/ComputeChecksum.scala @@ -1,14 +1,16 @@ package sharry.store -import binny._ -import binny.util.Stopwatch import cats.effect._ import cats.effect.std.Queue import cats.syntax.all._ import fs2.{Pipe, Stream} + import sharry.common.{ByteSize, Ident, Timestamp} import sharry.store.records.RFileMeta +import binny._ +import binny.util.Stopwatch + trait ComputeChecksum[F[_]] { def submit(id: BinaryId, hint: Hint): F[Unit] diff --git a/modules/store/src/main/scala/sharry/store/FileStore.scala b/modules/store/src/main/scala/sharry/store/FileStore.scala index 08938afe..2b907cff 100644 --- a/modules/store/src/main/scala/sharry/store/FileStore.scala +++ b/modules/store/src/main/scala/sharry/store/FileStore.scala @@ -13,11 +13,6 @@ import sharry.store.records.RFileMeta import _root_.doobie._ import binny._ -import binny.fs.{FsChunkedBinaryStore, FsChunkedStoreConfig} -import binny.jdbc.{GenericJdbcStore, JdbcStoreConfig} -import binny.minio.{MinioChunkedBinaryStore, MinioConfig, S3KeyMapping} -import binny.tika.TikaContentTypeDetect -import binny.util.Logger trait FileStore[F[_]] { @@ -38,6 +33,8 @@ trait FileStore[F[_]] { def addChunk(id: Ident, hint: Hint, chunkDef: ChunkDef, data: Chunk[Byte]): F[Unit] def computeAttributes: ComputeChecksum[F] + + def createBinaryStore: FileStoreConfig => ChunkedBinaryStore[F] } object FileStore { @@ -48,71 +45,12 @@ object FileStore { chunkSize: Int, computeChecksumConfig: ComputeChecksumConfig, config: FileStoreConfig - ): F[FileStore[F]] = - config match { - case FileStoreConfig.DefaultDatabase(_) => - forDatabase(ds, xa, chunkSize, computeChecksumConfig) - - case c: FileStoreConfig.S3 => - forS3(xa, c, chunkSize, computeChecksumConfig) - - case c: FileStoreConfig.FileSystem => - forFs(xa, c, chunkSize, computeChecksumConfig) - } - - def forDatabase[F[_]: Async]( - ds: DataSource, - xa: Transactor[F], - chunkSize: Int, - computeChecksumConfig: ComputeChecksumConfig - ): F[FileStore[F]] = { - val cfg = JdbcStoreConfig("filechunk", chunkSize, TikaContentTypeDetect.default) - val as = AttributeStore(xa) - val logger = SharryLogger(sharry.logging.getLogger[F]) - val bs = GenericJdbcStore[F](ds, logger, cfg) - ComputeChecksum[F](bs, computeChecksumConfig).map(cc => - new Impl[F](bs, as, chunkSize, cc) - ) - } - - def forFs[F[_]: Async]( - xa: Transactor[F], - fsCfg: FileStoreConfig.FileSystem, - chunkSize: Int, - computeChecksumConfig: ComputeChecksumConfig - ): F[FileStore[F]] = { - val as = AttributeStore(xa) - val logger = SharryLogger(sharry.logging.getLogger[F]) - val cfg = FsChunkedStoreConfig - .defaults(fsCfg.directory) - .copy(chunkSize = chunkSize) - .withContentTypeDetect(TikaContentTypeDetect.default) - val bs = FsChunkedBinaryStore(logger, cfg) - ComputeChecksum[F](bs, computeChecksumConfig).map(cc => - new Impl[F](bs, as, chunkSize, cc) - ) - } - - def forS3[F[_]: Async]( - xa: Transactor[F], - s3: FileStoreConfig.S3, - chunkSize: Int, - computeChecksumConfig: ComputeChecksumConfig ): F[FileStore[F]] = { + val create = FileStoreConfig.createBinaryStore[F](ds, chunkSize) _ + val bs = create(config) val as = AttributeStore(xa) - val logger = SharryLogger(sharry.logging.getLogger[F]) - val cfg = MinioConfig - .default( - s3.endpoint, - s3.accessKey, - s3.secretKey, - S3KeyMapping.constant(s3.bucket) - ) - .copy(chunkSize = chunkSize) - .withContentTypeDetect(TikaContentTypeDetect.default) - val bs = MinioChunkedBinaryStore(cfg, logger) ComputeChecksum[F](bs, computeChecksumConfig).map(cc => - new Impl[F](bs, as, chunkSize, cc) + new Impl[F](bs, as, chunkSize, cc, create) ) } @@ -120,7 +58,8 @@ object FileStore { bs: ChunkedBinaryStore[F], attrStore: AttributeStore[F], val chunkSize: Int, - val computeAttributes: ComputeChecksum[F] + val computeAttributes: ComputeChecksum[F], + val createBinaryStore: FileStoreConfig => ChunkedBinaryStore[F] ) extends FileStore[F] { def delete(id: Ident): F[Unit] = @@ -167,28 +106,4 @@ object FileStore { Sync[F].raiseError(new Exception(s"Inserting chunk failed: $fail")) } } - - private object SharryLogger { - - def apply[F[_]](log: sharry.logging.Logger[F]): Logger[F] = - new Logger[F] { - override def trace(msg: => String): F[Unit] = - log.trace(msg) - - override def debug(msg: => String): F[Unit] = - log.debug(msg) - - override def info(msg: => String): F[Unit] = - log.info(msg) - - override def warn(msg: => String): F[Unit] = - log.warn(msg) - - override def error(msg: => String): F[Unit] = - log.error(msg) - - override def error(ex: Throwable)(msg: => String): F[Unit] = - log.error(ex)(msg) - } - } } diff --git a/modules/store/src/main/scala/sharry/store/FileStoreConfig.scala b/modules/store/src/main/scala/sharry/store/FileStoreConfig.scala index 0bd12850..e2d795cc 100644 --- a/modules/store/src/main/scala/sharry/store/FileStoreConfig.scala +++ b/modules/store/src/main/scala/sharry/store/FileStoreConfig.scala @@ -1,7 +1,16 @@ package sharry.store +import javax.sql.DataSource + +import cats.effect.Async import fs2.io.file.Path +import binny.fs.{FsChunkedBinaryStore, FsChunkedStoreConfig} +import binny.jdbc.{GenericJdbcStore, JdbcStoreConfig} +import binny.minio.{MinioChunkedBinaryStore, MinioConfig, S3KeyMapping} +import binny.tika.TikaContentTypeDetect +import binny.util.Logger + sealed trait FileStoreConfig { def enabled: Boolean def storeType: FileStoreType @@ -30,4 +39,57 @@ object FileStoreConfig { override def toString = s"S3(enabled=$enabled, endpoint=$endpoint, bucket=$bucket, accessKey=$accessKey, secretKey=***)" } + + def createBinaryStore[F[_]: Async](ds: DataSource, chunkSize: Int)( + config: FileStoreConfig + ) = { + val logger = SharryLogger(sharry.logging.getLogger[F]) + config match { + case DefaultDatabase(_) => + val cfg = JdbcStoreConfig("filechunk", chunkSize, TikaContentTypeDetect.default) + GenericJdbcStore[F](ds, logger, cfg) + + case FileSystem(_, baseDir) => + val cfg = FsChunkedStoreConfig + .defaults(baseDir) + .copy(chunkSize = chunkSize) + .withContentTypeDetect(TikaContentTypeDetect.default) + FsChunkedBinaryStore(logger, cfg) + + case S3(_, endpoint, accessKey, secretKey, bucket) => + val cfg = MinioConfig + .default( + endpoint, + accessKey, + secretKey, + S3KeyMapping.constant(bucket) + ) + .copy(chunkSize = chunkSize) + .withContentTypeDetect(TikaContentTypeDetect.default) + MinioChunkedBinaryStore(cfg, logger) + } + } + + object SharryLogger { + def apply[F[_]](log: sharry.logging.Logger[F]): Logger[F] = + new Logger[F] { + override def trace(msg: => String): F[Unit] = + log.trace(msg) + + override def debug(msg: => String): F[Unit] = + log.debug(msg) + + override def info(msg: => String): F[Unit] = + log.info(msg) + + override def warn(msg: => String): F[Unit] = + log.warn(msg) + + override def error(msg: => String): F[Unit] = + log.error(msg) + + override def error(ex: Throwable)(msg: => String): F[Unit] = + log.error(ex)(msg) + } + } } diff --git a/modules/store/src/main/scala/sharry/store/Store.scala b/modules/store/src/main/scala/sharry/store/Store.scala index 038b9731..316527a6 100644 --- a/modules/store/src/main/scala/sharry/store/Store.scala +++ b/modules/store/src/main/scala/sharry/store/Store.scala @@ -1,13 +1,15 @@ package sharry.store import scala.concurrent.ExecutionContext + import cats.effect._ import fs2._ + import sharry.common.ByteSize import sharry.store.doobie.StoreImpl + import _root_.doobie._ import _root_.doobie.hikari.HikariTransactor -import binny.AttributeName import com.zaxxer.hikari.HikariDataSource trait Store[F[_]] { @@ -45,14 +47,6 @@ object Store { fs <- Resource.eval( FileStore[F](ds, xa, chunkSize.bytes.toInt, computeChecksumConfig, fileStoreCfg) ) - _ <- Async[F].background( - fs.computeAttributes - .consumeAll(AttributeName.all) - .evalMap(fs.updateChecksum) - .compile - .drain - ) - st = new StoreImpl[F](jdbc, fs, xa) _ <- if (runMigration) Resource.eval(st.migrate) else Resource.pure[F, Int](0) } yield st: Store[F] diff --git a/modules/store/src/main/scala/sharry/store/doobie/AttributeStore.scala b/modules/store/src/main/scala/sharry/store/doobie/AttributeStore.scala index 1635a015..c8e653f3 100644 --- a/modules/store/src/main/scala/sharry/store/doobie/AttributeStore.scala +++ b/modules/store/src/main/scala/sharry/store/doobie/AttributeStore.scala @@ -3,8 +3,10 @@ package sharry.store.doobie import cats.data.OptionT import cats.effect._ import cats.implicits._ + import sharry.common._ import sharry.store.records.RFileMeta + import binny._ import doobie._ import doobie.implicits._ From 222ec76f52b4d2868e54b333ab6154184a407fbf Mon Sep 17 00:00:00 2001 From: eikek Date: Sun, 26 Jun 2022 00:07:13 +0200 Subject: [PATCH 4/4] Update binny --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index de6bb4ce..fda57f7d 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -4,7 +4,7 @@ object Dependencies { val BcryptVersion = "0.4" val BetterMonadicForVersion = "0.3.1" - val BinnyVersion = "0.5.0+17-7dd06f69-SNAPSHOT" + val BinnyVersion = "0.6.0" val CirceVersion = "0.14.2" val ClipboardJsVersion = "2.0.6" val DoobieVersion = "1.0.0-RC2"