Deploy to prod

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,6 @@
+## Please include a summary of your changes.
+
+
+## Before merging:
+* [ ] If this PR targets dev branch, do a Squash-merge.
+* [ ] If the target branch is prod, do a Regular merge.

fetcher-web/Dockerfile

@@ -1,10 +1,12 @@
-FROM gradle:6.9-jdk11 AS builder
+FROM gradle:7.6-jdk11 AS builder
 COPY . /fetcher-web
 WORKDIR /fetcher-web
 RUN gradle build
 
 FROM openjdk:11-jre-slim
 
+RUN apt update && apt install git -y
+
 RUN mkdir /app
 
 COPY --from=builder /fetcher-web/build/libs/bible-translation-tools_fetcher.jar /app/fetcher.jar

fetcher-web/build.gradle

@@ -17,12 +17,13 @@ plugins {
     id 'io.gitlab.arturbosch.detekt' version "$detekt_ver"
     id 'org.jlleitschuh.gradle.ktlint' version "$ktlint_ver"
     id 'com.github.johnrengelman.shadow' version "$shadowjar_ver"
+    id 'com.apollographql.apollo3' version "$graphql_ver"
 }
 
 apply plugin: 'application'
 
 group 'org.bibletranslationtools'
-version '0.1.0'
+version '0.2.0'
 
 sourceCompatibility = 11
 
@@ -56,6 +57,7 @@ dependencies {
     implementation "me.xdrop:fuzzywuzzy:$fuzzy_wuzzy_ver"
     implementation "io.insert-koin:koin-ktor:$koin_ver"
     implementation "net.lingala.zip4j:zip4j:$zip4j_ver"
+    implementation "com.apollographql.apollo3:apollo-runtime:$graphql_ver"
 
     detektPlugins "io.gitlab.arturbosch.detekt:detekt-formatting:$detekt_ver"
     implementation "org.jetbrains.kotlinx:kotlinx-html-jvm:$kotlinx_html_ver" // detekt report
@@ -91,3 +93,9 @@ shadowJar {
         )
     }
 }
+
+apollo {
+    service("service") {
+        packageName.set("org.bibletranslationtools.fetcher.graphql.generated")
+    }
+}

fetcher-web/dependencies.gradle

@@ -3,10 +3,10 @@ ext {
     detekt_ver = '1.16.0'
     kotlinx_html_ver='0.7.2' // detekt report
     ktlint_ver = '9.2.1'
-    shadowjar_ver = '5.2.0'
+    shadowjar_ver = '7.1.2'
 
     // Libraries
-    kotlin_ver = '1.3.72'
+    kotlin_ver = '1.6.0'
     ktor_ver = '1.5.2'
     thymeleaf_ver = '3.0.11.RELEASE'
     jackson_csv_ver = '2.8.8'
@@ -21,6 +21,7 @@ ext {
     fuzzy_wuzzy_ver='1.3.1'
     koin_ver='3.0.1-beta-2'
     zip4j_ver='2.7.0'
+    graphql_ver='3.7.5'
 
     system_lambda_ver='1.2.0'
 }

fetcher-web/gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 #Fri Jun 26 15:59:28 EDT 2020
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.5-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStorePath=wrapper/dists

fetcher-web/src/main/graphql/GetPrimaryRepos.graphql

@@ -0,0 +1,21 @@
+query GetPrimaryReposQuery {
+  GitRepos: git_repo(
+    where: {
+      content: {
+        wa_content_metadata: {
+          status: {
+            _eq: "Primary"
+          }
+        }
+      }
+    }
+  ) {
+    repoUrl: repo_url
+    content {
+      resourceType: resource_type
+      language {
+        languageCode: ietf_code
+      }
+    }
+  }
+}

fetcher-web/src/main/graphql/GetPrimaryRepos.graphqls

@@ -0,0 +1,62 @@
+schema {
+    query: Query
+}
+
+input String_comparison_exp {
+    _eq: String
+}
+
+input git_repo_bool_exp {
+    _and: [git_repo_bool_exp!]
+    _not: git_repo_bool_exp
+    _or: [git_repo_bool_exp!]
+    content: content_bool_exp
+    repo_name: String_comparison_exp
+    repo_url: String_comparison_exp
+}
+
+input content_bool_exp {
+    _and: [content_bool_exp!]
+    _not: content_bool_exp
+    _or: [content_bool_exp!]
+    language: language_bool_exp
+    resource_type: String_comparison_exp
+    wa_content_metadata: wa_content_metadata_bool_exp
+}
+
+input wa_content_metadata_bool_exp {
+    _and: [wa_content_metadata_bool_exp!]
+    _not: wa_content_metadata_bool_exp
+    _or: [wa_content_metadata_bool_exp!]
+    status: String_comparison_exp
+}
+
+input language_bool_exp {
+    _and: [language_bool_exp!]
+    _not: language_bool_exp
+    _or: [language_bool_exp!]
+    ietf_code: String_comparison_exp
+}
+
+type Query {
+    git_repo(where: git_repo_bool_exp): [Repo!]!
+}
+
+type Repo {
+    repo_url: String
+    content: Content
+}
+
+type Content {
+    language: Language
+    resource_type: String
+    wa_content_metadata: [ContentMetadata]
+}
+
+type Language {
+    ietf_code: String
+}
+
+type ContentMetadata {
+    status: String
+}

fetcher-web/src/main/kotlin/org/bibletranslationtools/fetcher/di/DIModule.kt

@@ -2,28 +2,29 @@ package org.bibletranslationtools.fetcher.di
 
 import org.bibletranslationtools.fetcher.config.DevEnvironmentConfig
 import org.bibletranslationtools.fetcher.config.EnvironmentConfig
-import org.bibletranslationtools.fetcher.impl.repository.AvailabilityCacheAccessor
 import org.bibletranslationtools.fetcher.impl.repository.BookCatalogImpl
 import org.bibletranslationtools.fetcher.impl.repository.BookRepositoryImpl
 import org.bibletranslationtools.fetcher.impl.repository.ChapterCatalogImpl
-import org.bibletranslationtools.fetcher.impl.repository.ContentAvailabilityCacheBuilder
 import org.bibletranslationtools.fetcher.impl.repository.DirectoryProviderImpl
+import org.bibletranslationtools.fetcher.impl.repository.LangType
 import org.bibletranslationtools.fetcher.impl.repository.LanguageRepositoryImpl
-import org.bibletranslationtools.fetcher.impl.repository.PortGatewayLanguageCatalog
 import org.bibletranslationtools.fetcher.impl.repository.ProductCatalogImpl
+import org.bibletranslationtools.fetcher.impl.repository.RequestResourceContainerImpl
 import org.bibletranslationtools.fetcher.impl.repository.RCRepositoryImpl
+import org.bibletranslationtools.fetcher.impl.repository.SourceTextAccessorImpl
 import org.bibletranslationtools.fetcher.impl.repository.StorageAccessImpl
-import org.bibletranslationtools.fetcher.impl.repository.UnfoldingWordHeartLanguagesCatalog
+import org.bibletranslationtools.fetcher.impl.repository.UnfoldingWordLanguagesCatalog
 import org.bibletranslationtools.fetcher.io.LocalFileTransferClient
 import org.bibletranslationtools.fetcher.repository.BookCatalog
 import org.bibletranslationtools.fetcher.repository.BookRepository
 import org.bibletranslationtools.fetcher.repository.ChapterCatalog
-import org.bibletranslationtools.fetcher.repository.ContentCacheAccessor
 import org.bibletranslationtools.fetcher.repository.DirectoryProvider
 import org.bibletranslationtools.fetcher.repository.LanguageCatalog
 import org.bibletranslationtools.fetcher.repository.LanguageRepository
 import org.bibletranslationtools.fetcher.repository.ProductCatalog
+import org.bibletranslationtools.fetcher.repository.RequestResourceContainer
 import org.bibletranslationtools.fetcher.repository.ResourceContainerRepository
+import org.bibletranslationtools.fetcher.repository.SourceTextAccessor
 import org.bibletranslationtools.fetcher.repository.StorageAccess
 import org.koin.core.qualifier.named
 import org.koin.dsl.module
@@ -42,30 +43,22 @@ val appDependencyModule = module(createdAtStart = true) {
     single<StorageAccess> { StorageAccessImpl(get()) }
 
     single<ChapterCatalog> { ChapterCatalogImpl() }
-    single<LanguageCatalog>(named("GL")) { PortGatewayLanguageCatalog() }
-    single<LanguageCatalog>(named("HL")) { UnfoldingWordHeartLanguagesCatalog(get()) }
+    single<LanguageCatalog>(named(LangType.GL.name)) { UnfoldingWordLanguagesCatalog(get(), LangType.GL) }
+    single<LanguageCatalog>(named(LangType.HL.name)) { UnfoldingWordLanguagesCatalog(get(), LangType.HL) }
+    single<LanguageCatalog>(named(LangType.ALL.name)) { UnfoldingWordLanguagesCatalog(get(), LangType.ALL) }
     single<LanguageRepository> {
         LanguageRepositoryImpl(
-            get(named("GL")),
-            get(named("HL"))
+            get(named(LangType.GL.name)),
+            get(named(LangType.HL.name))
         )
     }
     single<ProductCatalog> { ProductCatalogImpl() }
     single<BookCatalog> { BookCatalogImpl() }
     single<BookRepository> { BookRepositoryImpl(get()) }
-    single<ResourceContainerRepository> { RCRepositoryImpl(get()) }
+    single<ResourceContainerRepository> { RCRepositoryImpl(get(), get()) }
 
     single<IDownloadClient> { LocalFileTransferClient(get()) }
+    single<RequestResourceContainer> { RequestResourceContainerImpl(get(), get(), get(), get()) }
 
-    single {
-        ContentAvailabilityCacheBuilder(
-            envConfig = get(),
-            languageCatalog = get(named("GL")),
-            productCatalog = get(),
-            chapterCatalog = get(),
-            bookRepository = get(),
-            storageAccess = get()
-        )
-    }
-    single<ContentCacheAccessor> { AvailabilityCacheAccessor(get()) }
+    single<SourceTextAccessor> { SourceTextAccessorImpl() }
 }

fetcher-web/src/main/kotlin/org/bibletranslationtools/fetcher/impl/repository/ChapterCatalogImpl.kt

@@ -3,8 +3,6 @@ package org.bibletranslationtools.fetcher.impl.repository
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.module.kotlin.KotlinModule
 import com.fasterxml.jackson.module.kotlin.readValue
-import io.ktor.client.request.get
-import io.ktor.util.error
 import java.io.IOException
 import java.net.HttpURLConnection
 import java.net.URL

fetcher-web/src/main/kotlin/org/bibletranslationtools/fetcher/impl/repository/RCRepositoryImpl.kt

@@ -1,12 +1,17 @@
 package org.bibletranslationtools.fetcher.impl.repository
 
-import java.io.File
 import org.bibletranslationtools.fetcher.repository.ResourceContainerRepository
+import org.bibletranslationtools.fetcher.repository.SourceTextAccessor
 import org.bibletranslationtools.fetcher.repository.StorageAccess
+import org.slf4j.LoggerFactory
+import java.io.File
+
 
 class RCRepositoryImpl(
-    private val storageAccess: StorageAccess
+    private val storageAccess: StorageAccess,
+    private val sourceTextAccessor: SourceTextAccessor
 ) : ResourceContainerRepository {
+    private val logger = LoggerFactory.getLogger(javaClass)
     private val rcTemplateName = "%s_%s"
 
     override fun getRC(
@@ -16,4 +21,30 @@ class RCRepositoryImpl(
         val repoName = String.format(rcTemplateName, languageCode, resourceId)
         return storageAccess.getRepoFromFileSystem(repoName)
     }
+
+    override fun downloadRC(
+        languageCode: String,
+        resourceId: String
+    ): File? {
+        return sourceTextAccessor.getRepoUrl(
+            languageCode,
+            resourceId
+        )?.let { repoUrl ->
+            val repoName = "${languageCode}_$resourceId"
+            val reposDirectory = storageAccess.getReposDir()
+
+            val process = ProcessBuilder()
+                .command("git", "clone", repoUrl, repoName)
+                .directory(reposDirectory)
+                .start()
+
+            val exit = process.waitFor()
+
+            if (exit != 0) {
+                logger.error("An error occurred in cloneRepo with exit code: $exit")
+            }
+
+            storageAccess.getRepoFromFileSystem(repoName)
+        }
+    }
 }

fetcher-web/src/main/kotlin/org/bibletranslationtools/fetcher/usecase/RequestResourceContainer.kt → fetcher-web/src/main/kotlin/org/bibletranslationtools/fetcher/impl/repository/RequestResourceContainerImpl.kt

@@ -1,13 +1,13 @@
-package org.bibletranslationtools.fetcher.usecase
+package org.bibletranslationtools.fetcher.impl.repository
 
 import java.io.File
 import org.bibletranslationtools.fetcher.config.EnvironmentConfig
 import org.bibletranslationtools.fetcher.data.Deliverable
 import org.bibletranslationtools.fetcher.data.RCDeliverable
-import org.bibletranslationtools.fetcher.impl.repository.RCUtils
-import org.bibletranslationtools.fetcher.impl.repository.StorageAccessImpl
+import org.bibletranslationtools.fetcher.repository.RequestResourceContainer
 import org.bibletranslationtools.fetcher.repository.ResourceContainerRepository
 import org.bibletranslationtools.fetcher.repository.StorageAccess
+import org.bibletranslationtools.fetcher.usecase.ProductFileQuality
 import org.wycliffeassociates.rcmediadownloader.RCMediaDownloader
 import org.wycliffeassociates.rcmediadownloader.data.MediaDivision
 import org.wycliffeassociates.rcmediadownloader.data.MediaType
@@ -19,17 +19,17 @@ import org.wycliffeassociates.resourcecontainer.entity.MediaManifest
 import org.wycliffeassociates.resourcecontainer.entity.MediaProject
 import java.util.zip.Adler32
 
-class RequestResourceContainer(
+class RequestResourceContainerImpl(
     envConfig: EnvironmentConfig,
     private val rcRepository: ResourceContainerRepository,
     private val storageAccess: StorageAccess,
     private val downloadClient: IDownloadClient
-) {
+) : RequestResourceContainer {
     private val baseRCUrl = envConfig.CDN_BASE_RC_URL
     private val baseContentUrl = envConfig.CONTENT_ROOT_DIR
     private val outputDir = envConfig.RC_OUTPUT_DIR
 
-    fun getResourceContainer(
+    override fun getResourceContainer(
         deliverable: Deliverable
     ): RCDeliverable? {
         val rcName = RCUtils.createRCFileName(deliverable, "")
@@ -74,6 +74,10 @@ class RequestResourceContainer(
         }
     }
 
+    override fun getResourceContainer(languageCode: String, resourceId: String): File? {
+        return rcRepository.getRC(languageCode, resourceId)
+    }
+
     private fun getChapterFiles(deliverable: Deliverable): List<File> {
         val templateRC = getTemplateRC(deliverable) ?: return listOf()
 
@@ -147,6 +151,9 @@ class RequestResourceContainer(
         val templateRC = rcRepository.getRC(
             deliverable.language.code,
             deliverable.resourceId
+        ) ?: rcRepository.downloadRC(
+            deliverable.language.code,
+            deliverable.resourceId
         )
 
         return if (templateRC != null && templateRC.exists()) {
@@ -247,7 +254,7 @@ class RequestResourceContainer(
         val mediaTypes = listOf(MediaType.MP3, MediaType.CUE)
 
         private val mediaQualityMap = mapOf(
-            MediaType.MP3.toString() to "hi",
+            MediaType.MP3.toString() to ProductFileQuality.HI.quality,
             MediaType.WAV.toString() to "",
             MediaType.CUE.toString() to ""
         )