Find source text repos in port and clone them

fetcher-web/Dockerfile

@@ -1,10 +1,12 @@
-FROM gradle:6.9-jdk11 AS builder
+FROM gradle:7.6-jdk11 AS builder
 COPY . /fetcher-web
 WORKDIR /fetcher-web
 RUN gradle build
 
 FROM openjdk:11-jre-slim
 
+RUN apt update && apt install git -y
+
 RUN mkdir /app
 
 COPY --from=builder /fetcher-web/build/libs/bible-translation-tools_fetcher.jar /app/fetcher.jar

fetcher-web/build.gradle

@@ -17,12 +17,13 @@ plugins {
     id 'io.gitlab.arturbosch.detekt' version "$detekt_ver"
     id 'org.jlleitschuh.gradle.ktlint' version "$ktlint_ver"
     id 'com.github.johnrengelman.shadow' version "$shadowjar_ver"
+    id 'com.apollographql.apollo3' version "$graphql_ver"
 }
 
 apply plugin: 'application'
 
 group 'org.bibletranslationtools'
-version '0.1.0'
+version '0.2.0'
 
 sourceCompatibility = 11
 
@@ -56,6 +57,7 @@ dependencies {
     implementation "me.xdrop:fuzzywuzzy:$fuzzy_wuzzy_ver"
     implementation "io.insert-koin:koin-ktor:$koin_ver"
     implementation "net.lingala.zip4j:zip4j:$zip4j_ver"
+    implementation "com.apollographql.apollo3:apollo-runtime:$graphql_ver"
 
     detektPlugins "io.gitlab.arturbosch.detekt:detekt-formatting:$detekt_ver"
     implementation "org.jetbrains.kotlinx:kotlinx-html-jvm:$kotlinx_html_ver" // detekt report
@@ -91,3 +93,9 @@ shadowJar {
         )
     }
 }
+
+apollo {
+    service("service") {
+        packageName.set("org.bibletranslationtools.fetcher.graphql.generated")
+    }
+}

fetcher-web/dependencies.gradle

@@ -3,10 +3,10 @@ ext {
     detekt_ver = '1.16.0'
     kotlinx_html_ver='0.7.2' // detekt report
     ktlint_ver = '9.2.1'
-    shadowjar_ver = '5.2.0'
+    shadowjar_ver = '7.1.2'
 
     // Libraries
-    kotlin_ver = '1.3.72'
+    kotlin_ver = '1.6.0'
     ktor_ver = '1.5.2'
     thymeleaf_ver = '3.0.11.RELEASE'
     jackson_csv_ver = '2.8.8'
@@ -21,6 +21,7 @@ ext {
     fuzzy_wuzzy_ver='1.3.1'
     koin_ver='3.0.1-beta-2'
     zip4j_ver='2.7.0'
+    graphql_ver='3.7.5'
 
     system_lambda_ver='1.2.0'
 }

fetcher-web/gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 #Fri Jun 26 15:59:28 EDT 2020
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.5-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStorePath=wrapper/dists

fetcher-web/src/main/graphql/GetPrimaryRepos.graphql

@@ -0,0 +1,21 @@
+query GetPrimaryReposQuery {
+  GitRepos: git_repo(
+    where: {
+      content: {
+        wa_content_metadata: {
+          status: {
+            _eq: "Primary"
+          }
+        }
+      }
+    }
+  ) {
+    repoUrl: repo_url
+    content {
+      resourceType: resource_type
+      language {
+        languageCode: ietf_code
+      }
+    }
+  }
+}

fetcher-web/src/main/graphql/GetPrimaryRepos.graphqls

@@ -0,0 +1,62 @@
+schema {
+    query: Query
+}
+
+input String_comparison_exp {
+    _eq: String
+}
+
+input git_repo_bool_exp {
+    _and: [git_repo_bool_exp!]
+    _not: git_repo_bool_exp
+    _or: [git_repo_bool_exp!]
+    content: content_bool_exp
+    repo_name: String_comparison_exp
+    repo_url: String_comparison_exp
+}
+
+input content_bool_exp {
+    _and: [content_bool_exp!]
+    _not: content_bool_exp
+    _or: [content_bool_exp!]
+    language: language_bool_exp
+    resource_type: String_comparison_exp
+    wa_content_metadata: wa_content_metadata_bool_exp
+}
+
+input wa_content_metadata_bool_exp {
+    _and: [wa_content_metadata_bool_exp!]
+    _not: wa_content_metadata_bool_exp
+    _or: [wa_content_metadata_bool_exp!]
+    status: String_comparison_exp
+}
+
+input language_bool_exp {
+    _and: [language_bool_exp!]
+    _not: language_bool_exp
+    _or: [language_bool_exp!]
+    ietf_code: String_comparison_exp
+}
+
+type Query {
+    git_repo(where: git_repo_bool_exp): [Repo!]!
+}
+
+type Repo {
+    repo_url: String
+    content: Content
+}
+
+type Content {
+    language: Language
+    resource_type: String
+    wa_content_metadata: [ContentMetadata]
+}
+
+type Language {
+    ietf_code: String
+}
+
+type ContentMetadata {
+    status: String
+}

fetcher-web/src/main/kotlin/org/bibletranslationtools/fetcher/di/DIModule.kt

@@ -9,7 +9,10 @@ import org.bibletranslationtools.fetcher.impl.repository.DirectoryProviderImpl
 import org.bibletranslationtools.fetcher.impl.repository.LangType
 import org.bibletranslationtools.fetcher.impl.repository.LanguageRepositoryImpl
 import org.bibletranslationtools.fetcher.impl.repository.ProductCatalogImpl
+import org.bibletranslationtools.fetcher.impl.repository.RequestResourceContainerImpl
 import org.bibletranslationtools.fetcher.impl.repository.RCRepositoryImpl
+import org.bibletranslationtools.fetcher.impl.repository.SourceAvailabilityCacheAccessor
+import org.bibletranslationtools.fetcher.impl.repository.SourceAvailabilityCacheBuilder
 import org.bibletranslationtools.fetcher.impl.repository.StorageAccessImpl
 import org.bibletranslationtools.fetcher.impl.repository.UnfoldingWordLanguagesCatalog
 import org.bibletranslationtools.fetcher.io.LocalFileTransferClient
@@ -20,7 +23,9 @@ import org.bibletranslationtools.fetcher.repository.DirectoryProvider
 import org.bibletranslationtools.fetcher.repository.LanguageCatalog
 import org.bibletranslationtools.fetcher.repository.LanguageRepository
 import org.bibletranslationtools.fetcher.repository.ProductCatalog
+import org.bibletranslationtools.fetcher.repository.RequestResourceContainer
 import org.bibletranslationtools.fetcher.repository.ResourceContainerRepository
+import org.bibletranslationtools.fetcher.repository.SourceCacheAccessor
 import org.bibletranslationtools.fetcher.repository.StorageAccess
 import org.koin.core.qualifier.named
 import org.koin.dsl.module
@@ -51,7 +56,11 @@ val appDependencyModule = module(createdAtStart = true) {
     single<ProductCatalog> { ProductCatalogImpl() }
     single<BookCatalog> { BookCatalogImpl() }
     single<BookRepository> { BookRepositoryImpl(get()) }
-    single<ResourceContainerRepository> { RCRepositoryImpl(get()) }
+    single<ResourceContainerRepository> { RCRepositoryImpl(get(), get()) }
 
     single<IDownloadClient> { LocalFileTransferClient(get()) }
+    single<RequestResourceContainer> { RequestResourceContainerImpl(get(), get(), get(), get()) }
+
+    single { SourceAvailabilityCacheBuilder() }
+    single<SourceCacheAccessor> { SourceAvailabilityCacheAccessor(get(), get()) }
 }

fetcher-web/src/main/kotlin/org/bibletranslationtools/fetcher/impl/repository/RCRepositoryImpl.kt

@@ -1,11 +1,14 @@
 package org.bibletranslationtools.fetcher.impl.repository
 
-import java.io.File
 import org.bibletranslationtools.fetcher.repository.ResourceContainerRepository
+import org.bibletranslationtools.fetcher.repository.SourceCacheAccessor
 import org.bibletranslationtools.fetcher.repository.StorageAccess
+import java.io.File
+
 
 class RCRepositoryImpl(
-    private val storageAccess: StorageAccess
+    private val storageAccess: StorageAccess,
+    private val sourceCacheAccessor: SourceCacheAccessor
 ) : ResourceContainerRepository {
     private val rcTemplateName = "%s_%s"
 
@@ -16,4 +19,19 @@ class RCRepositoryImpl(
         val repoName = String.format(rcTemplateName, languageCode, resourceId)
         return storageAccess.getRepoFromFileSystem(repoName)
     }
+
+    override fun downloadRC(
+        languageCode: String,
+        resourceId: String
+    ): File? {
+        return sourceCacheAccessor.getRepoUrl(
+            languageCode,
+            resourceId
+        )?.let { repoUrl ->
+            val repoName = "${languageCode}_$resourceId"
+
+            sourceCacheAccessor.downloadRepo(repoName, repoUrl)
+            storageAccess.getRepoFromFileSystem(repoName)
+        }
+    }
 }

fetcher-web/src/main/kotlin/org/bibletranslationtools/fetcher/usecase/RequestResourceContainer.kt → fetcher-web/src/main/kotlin/org/bibletranslationtools/fetcher/impl/repository/RequestResourceContainerImpl.kt

@@ -1,13 +1,13 @@
-package org.bibletranslationtools.fetcher.usecase
+package org.bibletranslationtools.fetcher.impl.repository
 
 import java.io.File
 import org.bibletranslationtools.fetcher.config.EnvironmentConfig
 import org.bibletranslationtools.fetcher.data.Deliverable
 import org.bibletranslationtools.fetcher.data.RCDeliverable
-import org.bibletranslationtools.fetcher.impl.repository.RCUtils
-import org.bibletranslationtools.fetcher.impl.repository.StorageAccessImpl
+import org.bibletranslationtools.fetcher.repository.RequestResourceContainer
 import org.bibletranslationtools.fetcher.repository.ResourceContainerRepository
 import org.bibletranslationtools.fetcher.repository.StorageAccess
+import org.bibletranslationtools.fetcher.usecase.ProductFileQuality
 import org.wycliffeassociates.rcmediadownloader.RCMediaDownloader
 import org.wycliffeassociates.rcmediadownloader.data.MediaDivision
 import org.wycliffeassociates.rcmediadownloader.data.MediaType
@@ -19,17 +19,17 @@ import org.wycliffeassociates.resourcecontainer.entity.MediaManifest
 import org.wycliffeassociates.resourcecontainer.entity.MediaProject
 import java.util.zip.Adler32
 
-class RequestResourceContainer(
+class RequestResourceContainerImpl(
     envConfig: EnvironmentConfig,
     private val rcRepository: ResourceContainerRepository,
     private val storageAccess: StorageAccess,
     private val downloadClient: IDownloadClient
-) {
+) : RequestResourceContainer {
     private val baseRCUrl = envConfig.CDN_BASE_RC_URL
     private val baseContentUrl = envConfig.CONTENT_ROOT_DIR
     private val outputDir = envConfig.RC_OUTPUT_DIR
 
-    fun getResourceContainer(
+    override fun getResourceContainer(
         deliverable: Deliverable
     ): RCDeliverable? {
         val rcName = RCUtils.createRCFileName(deliverable, "")
@@ -74,6 +74,10 @@ class RequestResourceContainer(
         }
     }
 
+    override fun getResourceContainer(languageCode: String, resourceId: String): File? {
+        return rcRepository.getRC(languageCode, resourceId)
+    }
+
     private fun getChapterFiles(deliverable: Deliverable): List<File> {
         val templateRC = getTemplateRC(deliverable) ?: return listOf()
 
@@ -147,6 +151,9 @@ class RequestResourceContainer(
         val templateRC = rcRepository.getRC(
             deliverable.language.code,
             deliverable.resourceId
+        ) ?: rcRepository.downloadRC(
+            deliverable.language.code,
+            deliverable.resourceId
         )
 
         return if (templateRC != null && templateRC.exists()) {

fetcher-web/src/main/kotlin/org/bibletranslationtools/fetcher/impl/repository/SourceAvailabilityCacheAccessor.kt

@@ -0,0 +1,44 @@
+package org.bibletranslationtools.fetcher.impl.repository
+
+import org.bibletranslationtools.fetcher.graphql.generated.GetPrimaryReposQuery
+import org.bibletranslationtools.fetcher.repository.SourceCacheAccessor
+import org.bibletranslationtools.fetcher.repository.StorageAccess
+import org.slf4j.LoggerFactory
+
+class SourceAvailabilityCacheAccessor(
+    private val cacheBuilder: SourceAvailabilityCacheBuilder,
+    private val storageAccess: StorageAccess
+) : SourceCacheAccessor {
+    private val logger = LoggerFactory.getLogger(javaClass)
+    private var cache: List<GetPrimaryReposQuery.GitRepo>
+
+    init {
+        cache = cacheBuilder.build()
+    }
+
+    override fun update() {
+        cache = cacheBuilder.build()
+    }
+
+    override fun getRepoUrl(languageCode: String, resourceId: String): String? {
+        return cache.singleOrNull {
+            it.content?.language?.languageCode == languageCode &&
+                    it.content.resourceType == resourceId
+        }?.repoUrl
+    }
+
+    override fun downloadRepo(repoName: String, repoUrl: String) {
+        val reposDirectory = storageAccess.getReposDir()
+
+        val process = ProcessBuilder()
+            .command("git", "clone", repoUrl, repoName)
+            .directory(reposDirectory)
+            .start()
+
+        val exit = process.waitFor()
+
+        if (exit != 0) {
+            logger.error("An error occurred in cloneRepo with exit code: $exit")
+        }
+    }
+}

fetcher-web/src/main/kotlin/org/bibletranslationtools/fetcher/impl/repository/SourceAvailabilityCacheBuilder.kt

@@ -0,0 +1,21 @@
+package org.bibletranslationtools.fetcher.impl.repository
+
+import com.apollographql.apollo3.ApolloClient
+import kotlinx.coroutines.runBlocking
+import org.bibletranslationtools.fetcher.graphql.generated.GetPrimaryReposQuery
+
+class SourceAvailabilityCacheBuilder {
+
+    @Synchronized
+    fun build(): List<GetPrimaryReposQuery.GitRepo> {
+        val client = ApolloClient.Builder()
+            .serverUrl("https://api-biel-dev.walink.org/v1/graphql")
+            .build()
+
+        return runBlocking {
+            val query = GetPrimaryReposQuery()
+            val response = client.query(query).execute()
+            response.data?.GitRepos ?: listOf()
+        }
+    }
+}

fetcher-web/src/main/kotlin/org/bibletranslationtools/fetcher/impl/repository/StorageAccessImpl.kt

@@ -66,6 +66,10 @@ class StorageAccessImpl(private val directoryProvider: DirectoryProvider) : Stor
         return directoryProvider.getContentRoot()
     }
 
+    override fun getReposDir(): File {
+        return directoryProvider.getRCRepositoriesDir()
+    }
+
     override fun hasLanguageContent(languageCode: String): Boolean {
         val sourceFileRootDir = directoryProvider.getContentRoot()
         val dirs = sourceFileRootDir.listFiles(File::isDirectory)

fetcher-web/src/main/kotlin/org/bibletranslationtools/fetcher/repository/RequestResourceContainer.kt

@@ -0,0 +1,10 @@
+package org.bibletranslationtools.fetcher.repository
+
+import org.bibletranslationtools.fetcher.data.Deliverable
+import org.bibletranslationtools.fetcher.data.RCDeliverable
+import java.io.File
+
+interface RequestResourceContainer {
+    fun getResourceContainer(deliverable: Deliverable): RCDeliverable?
+    fun getResourceContainer(languageCode: String, resourceId: String): File?
+}

fetcher-web/src/main/kotlin/org/bibletranslationtools/fetcher/repository/ResourceContainerRepository.kt

@@ -7,4 +7,9 @@ interface ResourceContainerRepository {
         languageCode: String,
         resourceId: String
     ): File?
+
+    fun downloadRC(
+        languageCode: String,
+        resourceId: String
+    ): File?
 }