Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Find source text repos in port and clone them #198

Merged
merged 27 commits into from
May 20, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
64abafb
Add support for heart languages
mXaln May 15, 2024
35ae528
replace # with javascript:void(0) to avoid some navigation bugs
mXaln May 15, 2024
8147085
fix test
mXaln May 15, 2024
baa8f32
imports and types
mXaln May 16, 2024
5128cce
add enum types
mXaln May 16, 2024
e55eae7
replace # with ''
AnonymousWalker May 16, 2024
a926fb8
use javascript:void(0) for href with empty url
AnonymousWalker May 16, 2024
52c5f32
Merge branch 'mm-hl-support-refactor' into aw-avoid-hashtag
mXaln May 16, 2024
13c0c2c
Merge pull request #196 from Bible-Translation-Tools/aw-avoid-hashtag
mXaln May 16, 2024
c94cadf
Merge branch 'refs/heads/audiobieldev.walink.org' into mm-port-source…
mXaln May 17, 2024
00493b7
Find source text repos in port and clone them
mXaln May 17, 2024
5c77f1c
Find source text repos in port and clone them
mXaln May 17, 2024
d11d340
Merge remote-tracking branch 'origin/mm-port-source-repos' into mm-po…
mXaln May 17, 2024
a546d77
bump up gradle version in docker image
mXaln May 17, 2024
8ad057d
bump up shadow plugin version
mXaln May 17, 2024
925e5b6
install git in fetcher web container
mXaln May 17, 2024
e730325
check if orature product has source text
mXaln May 17, 2024
e9e90cf
rename function
mXaln May 17, 2024
ec9c1b6
small refactoring
mXaln May 17, 2024
5b1a466
make RequestResourceContainer a singleton
mXaln May 17, 2024
6b042d0
import fix
mXaln May 17, 2024
19e53e8
implement SourceCacheAccessor to get repos of all languages in port
mXaln May 20, 2024
2ffce80
make sourceCacheBuilder.build synchronized
mXaln May 20, 2024
6810dad
Merge branch 'refs/heads/audiobieldev.walink.org' into mm-port-source…
mXaln May 20, 2024
aaf576c
refactored due to feedback
mXaln May 20, 2024
7dfb3ec
removed deleted import
mXaln May 20, 2024
344fde9
renamed class
mXaln May 20, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion fetcher-web/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
FROM gradle:6.9-jdk11 AS builder
FROM gradle:7.6-jdk11 AS builder
COPY . /fetcher-web
WORKDIR /fetcher-web
RUN gradle build

FROM openjdk:11-jre-slim

RUN apt update && apt install git -y

RUN mkdir /app

COPY --from=builder /fetcher-web/build/libs/bible-translation-tools_fetcher.jar /app/fetcher.jar
Expand Down
10 changes: 9 additions & 1 deletion fetcher-web/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,13 @@ plugins {
id 'io.gitlab.arturbosch.detekt' version "$detekt_ver"
id 'org.jlleitschuh.gradle.ktlint' version "$ktlint_ver"
id 'com.github.johnrengelman.shadow' version "$shadowjar_ver"
id 'com.apollographql.apollo3' version "$graphql_ver"
}

apply plugin: 'application'

group 'org.bibletranslationtools'
version '0.1.0'
version '0.2.0'

sourceCompatibility = 11

Expand Down Expand Up @@ -56,6 +57,7 @@ dependencies {
implementation "me.xdrop:fuzzywuzzy:$fuzzy_wuzzy_ver"
implementation "io.insert-koin:koin-ktor:$koin_ver"
implementation "net.lingala.zip4j:zip4j:$zip4j_ver"
implementation "com.apollographql.apollo3:apollo-runtime:$graphql_ver"

detektPlugins "io.gitlab.arturbosch.detekt:detekt-formatting:$detekt_ver"
implementation "org.jetbrains.kotlinx:kotlinx-html-jvm:$kotlinx_html_ver" // detekt report
Expand Down Expand Up @@ -91,3 +93,9 @@ shadowJar {
)
}
}

apollo {
service("service") {
packageName.set("org.bibletranslationtools.fetcher.graphql.generated")
}
}
5 changes: 3 additions & 2 deletions fetcher-web/dependencies.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ ext {
detekt_ver = '1.16.0'
kotlinx_html_ver='0.7.2' // detekt report
ktlint_ver = '9.2.1'
shadowjar_ver = '5.2.0'
shadowjar_ver = '7.1.2'

// Libraries
kotlin_ver = '1.3.72'
kotlin_ver = '1.6.0'
ktor_ver = '1.5.2'
thymeleaf_ver = '3.0.11.RELEASE'
jackson_csv_ver = '2.8.8'
Expand All @@ -21,6 +21,7 @@ ext {
fuzzy_wuzzy_ver='1.3.1'
koin_ver='3.0.1-beta-2'
zip4j_ver='2.7.0'
graphql_ver='3.7.5'

system_lambda_ver='1.2.0'
}
2 changes: 1 addition & 1 deletion fetcher-web/gradle/wrapper/gradle-wrapper.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#Fri Jun 26 15:59:28 EDT 2020
distributionUrl=https\://services.gradle.org/distributions/gradle-6.5-all.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStorePath=wrapper/dists
Expand Down
21 changes: 21 additions & 0 deletions fetcher-web/src/main/graphql/GetPrimaryRepos.graphql
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
query GetPrimaryReposQuery {
GitRepos: git_repo(
where: {
content: {
wa_content_metadata: {
status: {
_eq: "Primary"
}
}
}
}
) {
repoUrl: repo_url
content {
resourceType: resource_type
language {
languageCode: ietf_code
}
}
}
}
62 changes: 62 additions & 0 deletions fetcher-web/src/main/graphql/GetPrimaryRepos.graphqls
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
schema {
query: Query
}

input String_comparison_exp {
_eq: String
}

input git_repo_bool_exp {
_and: [git_repo_bool_exp!]
_not: git_repo_bool_exp
_or: [git_repo_bool_exp!]
content: content_bool_exp
repo_name: String_comparison_exp
repo_url: String_comparison_exp
}

input content_bool_exp {
_and: [content_bool_exp!]
_not: content_bool_exp
_or: [content_bool_exp!]
language: language_bool_exp
resource_type: String_comparison_exp
wa_content_metadata: wa_content_metadata_bool_exp
}

input wa_content_metadata_bool_exp {
_and: [wa_content_metadata_bool_exp!]
_not: wa_content_metadata_bool_exp
_or: [wa_content_metadata_bool_exp!]
status: String_comparison_exp
}

input language_bool_exp {
_and: [language_bool_exp!]
_not: language_bool_exp
_or: [language_bool_exp!]
ietf_code: String_comparison_exp
}

type Query {
git_repo(where: git_repo_bool_exp): [Repo!]!
}

type Repo {
repo_url: String
content: Content
}

type Content {
language: Language
resource_type: String
wa_content_metadata: [ContentMetadata]
}

type Language {
ietf_code: String
}

type ContentMetadata {
status: String
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@ import org.bibletranslationtools.fetcher.impl.repository.DirectoryProviderImpl
import org.bibletranslationtools.fetcher.impl.repository.LangType
import org.bibletranslationtools.fetcher.impl.repository.LanguageRepositoryImpl
import org.bibletranslationtools.fetcher.impl.repository.ProductCatalogImpl
import org.bibletranslationtools.fetcher.impl.repository.RequestResourceContainerImpl
import org.bibletranslationtools.fetcher.impl.repository.RCRepositoryImpl
import org.bibletranslationtools.fetcher.impl.repository.SourceTextAccessorImpl
import org.bibletranslationtools.fetcher.impl.repository.StorageAccessImpl
import org.bibletranslationtools.fetcher.impl.repository.UnfoldingWordLanguagesCatalog
import org.bibletranslationtools.fetcher.io.LocalFileTransferClient
Expand All @@ -20,7 +22,9 @@ import org.bibletranslationtools.fetcher.repository.DirectoryProvider
import org.bibletranslationtools.fetcher.repository.LanguageCatalog
import org.bibletranslationtools.fetcher.repository.LanguageRepository
import org.bibletranslationtools.fetcher.repository.ProductCatalog
import org.bibletranslationtools.fetcher.repository.RequestResourceContainer
import org.bibletranslationtools.fetcher.repository.ResourceContainerRepository
import org.bibletranslationtools.fetcher.repository.SourceTextAccessor
import org.bibletranslationtools.fetcher.repository.StorageAccess
import org.koin.core.qualifier.named
import org.koin.dsl.module
Expand Down Expand Up @@ -51,7 +55,10 @@ val appDependencyModule = module(createdAtStart = true) {
single<ProductCatalog> { ProductCatalogImpl() }
single<BookCatalog> { BookCatalogImpl() }
single<BookRepository> { BookRepositoryImpl(get()) }
single<ResourceContainerRepository> { RCRepositoryImpl(get()) }
single<ResourceContainerRepository> { RCRepositoryImpl(get(), get()) }

single<IDownloadClient> { LocalFileTransferClient(get()) }
single<RequestResourceContainer> { RequestResourceContainerImpl(get(), get(), get(), get()) }

single<SourceTextAccessor> { SourceTextAccessorImpl() }
}
Original file line number Diff line number Diff line change
@@ -1,12 +1,17 @@
package org.bibletranslationtools.fetcher.impl.repository

import java.io.File
import org.bibletranslationtools.fetcher.repository.ResourceContainerRepository
import org.bibletranslationtools.fetcher.repository.SourceTextAccessor
import org.bibletranslationtools.fetcher.repository.StorageAccess
import org.slf4j.LoggerFactory
import java.io.File


class RCRepositoryImpl(
private val storageAccess: StorageAccess
private val storageAccess: StorageAccess,
private val sourceTextAccessor: SourceTextAccessor
) : ResourceContainerRepository {
private val logger = LoggerFactory.getLogger(javaClass)
private val rcTemplateName = "%s_%s"

override fun getRC(
Expand All @@ -16,4 +21,30 @@ class RCRepositoryImpl(
val repoName = String.format(rcTemplateName, languageCode, resourceId)
return storageAccess.getRepoFromFileSystem(repoName)
}

override fun downloadRC(
languageCode: String,
resourceId: String
): File? {
return sourceTextAccessor.getRepoUrl(
languageCode,
resourceId
)?.let { repoUrl ->
val repoName = "${languageCode}_$resourceId"
val reposDirectory = storageAccess.getReposDir()

val process = ProcessBuilder()
.command("git", "clone", repoUrl, repoName)
.directory(reposDirectory)
.start()

val exit = process.waitFor()

if (exit != 0) {
logger.error("An error occurred in cloneRepo with exit code: $exit")
}

storageAccess.getRepoFromFileSystem(repoName)
}
}
}
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
package org.bibletranslationtools.fetcher.usecase
package org.bibletranslationtools.fetcher.impl.repository

import java.io.File
import org.bibletranslationtools.fetcher.config.EnvironmentConfig
import org.bibletranslationtools.fetcher.data.Deliverable
import org.bibletranslationtools.fetcher.data.RCDeliverable
import org.bibletranslationtools.fetcher.impl.repository.RCUtils
import org.bibletranslationtools.fetcher.impl.repository.StorageAccessImpl
import org.bibletranslationtools.fetcher.repository.RequestResourceContainer
import org.bibletranslationtools.fetcher.repository.ResourceContainerRepository
import org.bibletranslationtools.fetcher.repository.StorageAccess
import org.bibletranslationtools.fetcher.usecase.ProductFileQuality
import org.wycliffeassociates.rcmediadownloader.RCMediaDownloader
import org.wycliffeassociates.rcmediadownloader.data.MediaDivision
import org.wycliffeassociates.rcmediadownloader.data.MediaType
Expand All @@ -19,17 +19,17 @@ import org.wycliffeassociates.resourcecontainer.entity.MediaManifest
import org.wycliffeassociates.resourcecontainer.entity.MediaProject
import java.util.zip.Adler32

class RequestResourceContainer(
class RequestResourceContainerImpl(
envConfig: EnvironmentConfig,
private val rcRepository: ResourceContainerRepository,
private val storageAccess: StorageAccess,
private val downloadClient: IDownloadClient
) {
) : RequestResourceContainer {
private val baseRCUrl = envConfig.CDN_BASE_RC_URL
private val baseContentUrl = envConfig.CONTENT_ROOT_DIR
private val outputDir = envConfig.RC_OUTPUT_DIR

fun getResourceContainer(
override fun getResourceContainer(
deliverable: Deliverable
): RCDeliverable? {
val rcName = RCUtils.createRCFileName(deliverable, "")
Expand Down Expand Up @@ -74,6 +74,10 @@ class RequestResourceContainer(
}
}

override fun getResourceContainer(languageCode: String, resourceId: String): File? {
return rcRepository.getRC(languageCode, resourceId)
}

private fun getChapterFiles(deliverable: Deliverable): List<File> {
val templateRC = getTemplateRC(deliverable) ?: return listOf()

Expand Down Expand Up @@ -147,6 +151,9 @@ class RequestResourceContainer(
val templateRC = rcRepository.getRC(
deliverable.language.code,
deliverable.resourceId
) ?: rcRepository.downloadRC(
deliverable.language.code,
deliverable.resourceId
)

return if (templateRC != null && templateRC.exists()) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package org.bibletranslationtools.fetcher.impl.repository

import com.apollographql.apollo3.ApolloClient
import kotlinx.coroutines.runBlocking
import org.bibletranslationtools.fetcher.graphql.generated.GetPrimaryReposQuery
import org.bibletranslationtools.fetcher.repository.SourceTextAccessor

class SourceTextAccessorImpl : SourceTextAccessor {
private var cache: List<GetPrimaryReposQuery.GitRepo> = listOf()

init {
buildCache()
}

override fun update() {
buildCache()
}

override fun getRepoUrl(languageCode: String, resourceId: String): String? {
return cache.singleOrNull {
it.content?.language?.languageCode == languageCode &&
it.content.resourceType == resourceId
}?.repoUrl
}

private fun buildCache() {
val client = ApolloClient.Builder()
.serverUrl("https://api-biel-dev.walink.org/v1/graphql")
.build()

cache = runBlocking {
val query = GetPrimaryReposQuery()
val response = client.query(query).execute()
response.data?.GitRepos ?: listOf()
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,10 @@ class StorageAccessImpl(private val directoryProvider: DirectoryProvider) : Stor
return directoryProvider.getContentRoot()
}

override fun getReposDir(): File {
return directoryProvider.getRCRepositoriesDir()
}

override fun hasLanguageContent(languageCode: String): Boolean {
val sourceFileRootDir = directoryProvider.getContentRoot()
val dirs = sourceFileRootDir.listFiles(File::isDirectory)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package org.bibletranslationtools.fetcher.repository

import org.bibletranslationtools.fetcher.data.Deliverable
import org.bibletranslationtools.fetcher.data.RCDeliverable
import java.io.File

interface RequestResourceContainer {
fun getResourceContainer(deliverable: Deliverable): RCDeliverable?
fun getResourceContainer(languageCode: String, resourceId: String): File?
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,9 @@ interface ResourceContainerRepository {
languageCode: String,
resourceId: String
): File?

fun downloadRC(
languageCode: String,
resourceId: String
): File?
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
package org.bibletranslationtools.fetcher.repository

interface SourceTextAccessor {
fun update()
fun getRepoUrl(languageCode: String, resourceId: String): String?
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import java.io.File

interface StorageAccess {
fun getContentRoot(): File
fun getReposDir(): File
fun hasLanguageContent(languageCode: String): Boolean
fun hasProductContent(languageCode: String, fileExtensions: List<String>): Boolean
fun getBookFile(request: FileAccessRequest): File?
Expand Down
Loading
Loading