build.gradle

buildscript {
    repositories {
        mavenLocal()
        mavenCentral()
        maven {
            url 'https://plugins.gradle.org/m2/'
        }
    }
    dependencies {
//        classpath group: 'net.researchgate', name: 'gradle-release', version: '2.8.1'
        classpath 'gradle.plugin.org.kt3k.gradle.plugin:coveralls-gradle-plugin:2.12.0'
        classpath "gradle.plugin.com.github.jengelman.gradle.plugins:shadow:7.0.0"
        classpath 'com.adarshr:gradle-test-logger-plugin:2.0.0'
    }
}

repositories {
    mavenLocal()
    mavenCentral()
    maven { url "https://jitpack.io" }
}

//apply plugin: 'net.researchgate.release'
apply plugin: 'jacoco'

jacoco {
    toolVersion = '0.8.5'
}

allprojects {
    apply plugin: 'java-library'
    apply plugin: 'base'
    apply plugin: 'com.github.kt3k.coveralls'
    apply plugin: 'com.adarshr.test-logger'

    group = "org.grobid"

    tasks.withType(JavaCompile) {
        options.encoding = 'UTF-8'
    }
}

subprojects {
    apply plugin: 'java'
    apply plugin: 'maven-publish'
 //   apply plugin: 'com.jfrog.bintray'

    publishing {
        publications {
            mavenJava(MavenPublication) {
                from components.java
            }
        }
        repositories {
            mavenLocal()
        }
    }

    task install(dependsOn: publishToMavenLocal)

    sourceCompatibility = 1.8
    targetCompatibility = 1.8

    repositories {
        mavenCentral()
//        jcenter()
        maven {
            url new File(rootProject.rootDir, "grobid-core/localLibs")
        }
        maven { url "https://jitpack.io" }
    }

    configurations {
        all*.exclude group: 'org.slf4j', module: "slf4j-log4j12"
        //all*.exclude group: 'log4j', module: "log4j"
//        implementation.setCanBeResolved(true)
    }

    ext {
        // treating them separately, these jars will be flattened into grobid-core.jar on installing,
        // to avoid missing dependencies from the projects that include grobid-core (see 'jar' task in grobid-core)
        localLibs = ['crfpp-1.0.2.jar',
                     'langdetect-1.1-20120112.jar',
                     'wipo-analysers-0.0.2.jar',
                     'imageio-pnm-1.0.jar',
                     'wapiti-1.5.0.jar']
    }

    dependencies {
        // packaging local libs inside grobid-core.jar
        implementation fileTree(dir: new File(rootProject.rootDir, 'grobid-core/localLibs'), include: localLibs)

        testImplementation "junit:junit:4.12"
        testImplementation "org.easymock:easymock:3.4"
        testImplementation "org.powermock:powermock-api-easymock:2.0.7"
        testImplementation "org.powermock:powermock-module-junit4:2.0.7"
        testImplementation "xmlunit:xmlunit:1.6"
        testImplementation "org.hamcrest:hamcrest-all:1.3"

        implementation "com.cybozu.labs:langdetect:1.1-20120112"
        implementation "com.rockymadden.stringmetric:stringmetric-core_2.10:0.27.3"
        implementation "commons-pool:commons-pool:1.6"
        implementation "commons-io:commons-io:2.5"
        implementation "org.apache.commons:commons-lang3:3.6"
        implementation "org.apache.commons:commons-collections4:4.1"
        implementation 'org.apache.commons:commons-text:1.8'
        implementation "commons-dbutils:commons-dbutils:1.7"
        implementation "com.google.guava:guava:28.2-jre"
        implementation "org.apache.httpcomponents:httpclient:4.5.3"
        implementation "black.ninia:jep:4.0.2"

        implementation "com.fasterxml.jackson.core:jackson-core:2.10.1"
        implementation "com.fasterxml.jackson.core:jackson-databind:2.10.1"
        implementation "com.fasterxml.jackson.module:jackson-module-afterburner:2.10.1"
        //implementation 'com.fasterxml.jackson.core:jackson-annotations:2.10.1'
        implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.10.1"
    }

    task sourceJar(type: Jar) {
        description = 'A jar that contains source code'
        archiveClassifier = 'sources'
        from project.sourceSets.main.java
    }

    artifacts {
        archives sourceJar
        archives jar
    }


    /*
    // removing - this currently breaks the release 
    task gitRev(type: Exec) {
        commandLine 'git', 'rev-parse', '--short', 'HEAD'
        standardOutput = new ByteArrayOutputStream()
        ext.hash = {
            standardOutput.toString()
        }
    }

    // allows to upload non-snapshot versions that are tight to git revision (substitutes the version if it's a snapshot one)
    // usage "gradle uploadArchives -PversionWithGitRev"
    task changeVersionIfNeeded(dependsOn: gitRev) {
        doLast {
            def suffix = '-SNAPSHOT'
            if (project.hasProperty('versionWithGitRev') && project.version.endsWith(suffix)) {
                println project.version
                project.version = project.version.substring(0, project.version.length() - suffix.length()) + "-" + tasks.gitRev.hash()
                println "Version to be uploaded: " + project.version
            }
        }
    }*/

    //compileJava.dependsOn(changeVersionIfNeeded)

//    uploadArchives {
//        // if you want to enable uploading to some maven repo, add those properties to ~/.gradle/gradle.properties, e.g.:
//        /*
//            mavenRepoUserName=maven_username
//            mavenRepoPassword=super_secret
//            mavenRepoReleasesUrl=https://nexus3.example.org/repository/maven-releases/
//            mavenRepoSnapshotsUrl=https://nexus3.example.org/repository/maven-snapshots/
//        */
//        def user = project.hasProperty('mavenRepoUserName') ? project.findProperty('mavenRepoUserName') : ''
//        def password = project.hasProperty('mavenRepoPassword') ? project.findProperty('mavenRepoPassword') : ''
//        def rurl = project.hasProperty('mavenRepoReleasesUrl') ? project.findProperty('mavenRepoReleasesUrl') : ''
//        def surl = project.hasProperty('mavenRepoSnapshotsUrl') ? project.findProperty('mavenRepoSnapshotsUrl') : ''
//
//        repositories.mavenDeployer {
//            repository(url: rurl) {
//                authentication(userName: user, password: password)
//            }
//            snapshotRepository(url: surl) {
//                authentication(userName: user, password: password)
//            }
//
//        }
//    }

    /*
    bintray {
        dryRun = false // Whether to run this as dry-run, without deploying
        publish = true // If version should be auto published after an upload

        // read from ~/.gradle/gradle.properties. Set those up before releasing
        user = project.hasProperty('bintrayUser') ? bintrayUser : ''
        key = project.hasProperty('bintrayApiKey') ? bintrayApiKey : ''
        configurations = ['archives']
        pkg {
            name = 'grobid'
            repo = 'maven'
            userOrg = 'rookies'
            licenses = ['Apache-2.0']
            vcsUrl = 'https://github.com/kermitt2/grobid'
            issueTrackerUrl = 'https://github.com/kermitt2/grobid/issues'
            githubRepo = 'kermitt2/grobid'

            version {
                name = rootProject.version
                vcsTag = rootProject.version
                released = new Date()
            }
        }
    }
    */

    test {
        testLogging.showStandardStreams = true
        // enable for having separate test executor for different tests
        forkEvery = 1
        maxHeapSize = "1024m"
    }

    //rootProject.afterReleaseBuild.dependsOn subprojects.bintrayUpload
}

/** SUBPROJECTS **/

project("grobid-core") {
    apply plugin: 'com.github.johnrengelman.shadow'
    apply plugin: 'jacoco'

    configurations {
        shadedLib
    }

    dependencies {
        implementation(group: 'xml-apis', name: 'xml-apis') {
            // otherwise xml-apis 2.0.1 will come from XOM and will result in
            // java.lang.ClassNotFoundException: org.w3c.dom.ElementTraversal
            //TODO: sort out this problem better
            version {
                strictly '1.4.01'
            }
        }

        // Logs
        api 'org.slf4j:slf4j-api:1.7.25'
        api 'org.slf4j:slf4j-log4j12:1.7.25'
        runtimeOnly 'org.slf4j:slf4j-jdk14:1.7.25'

        implementation "org.apache.pdfbox:pdfbox:2.0.18"

        api "xerces:xercesImpl:2.12.0"
        api "net.arnx:jsonic:1.3.10"
        api "net.sf.saxon:Saxon-HE:9.6.0-9"
        api "xom:xom:1.3.2"
        api 'javax.xml.bind:jaxb-api:2.3.0'

        implementation "joda-time:joda-time:2.9.9"
        implementation "org.apache.lucene:lucene-analyzers-common:4.5.1"
        implementation 'black.ninia:jep:4.0.2'
        implementation 'org.apache.opennlp:opennlp-tools:1.9.1'
        implementation group: 'org.jruby', name: 'jruby-complete', version: '9.2.13.0'

        shadedLib "org.apache.lucene:lucene-analyzers-common:4.5.1"
    }

    jar {
        from {
            project.configurations.runtimeClasspath.collect {
                it.isDirectory() ? [] : localLibs.contains(it.getName()) ? zipTree(it) : []
            }
        }
        duplicatesStrategy = DuplicatesStrategy.EXCLUDE
    }

    shadowJar {
        archiveClassifier = 'onejar'
        mergeServiceFiles()
        zip64 true
        manifest {
            attributes 'Main-Class': 'org.grobid.core.main.batch.GrobidMain'
        }
        from sourceSets.main.output

        from {
            project.configurations.runtimeClasspath.collect {
                it.isDirectory() ? [] : localLibs.contains(it.getName()) ? zipTree(it) : []
            }
        }

        configurations = [project.configurations.shadedLib, project.configurations.runtimeClasspath]
        relocate 'org.apache.lucene', 'org.grobid.shaded.org.apache.lucene'
    }

    artifacts {
        archives shadowJar
        archives jar
    }

    processResources {
        filesMatching('grobid-version.txt') {
            filter {
                it.replace('project.version', project.property('version'))
            }
        }
    }
}

project("grobid-home") {
    task packageGrobidHome(type: Zip) {
        from('.') {
            include("config/*")
            include("language-detection/**")
            include("sentence-segmentation/**")
            include("lib/**")
            include("pdfalto/**")
            include("models/**")
            include("lexicon/**")
            include("schemas/**")
            include("scripts/**")
            exclude("models/**/*.old")
        }
        into("grobid-home")
    }
    artifacts {
        archives packageGrobidHome
    }
}

project(":grobid-service") {
    apply plugin: 'application'
    apply plugin: 'jacoco'
    apply plugin: 'com.github.johnrengelman.shadow'

    mainClassName = 'org.grobid.service.main.GrobidServiceApplication'
    tasks.run {
        workingDir = rootProject.rootDir
        systemProperty "java.library.path","${System.getProperty('java.library.path')}:${file("../grobid-home/lib/lin-64/jep").absolutePath}:${file("../grobid-home/lib/lin-64").absolutePath}"
    }

    configurations {
        all*.exclude group: 'org.slf4j', module: "slf4j-jdk14"
        all*.exclude group: 'org.slf4j', module: "slf4j-log4j12"
        all*.exclude group: 'log4j', module: "log4j"
    }

    tasks.distZip.enabled = true
    tasks.distTar.enabled = false
    tasks.shadowDistZip.enabled = false
    tasks.shadowDistTar.enabled = false

    distZip { duplicatesStrategy = DuplicatesStrategy.EXCLUDE }
    distTar { duplicatesStrategy = DuplicatesStrategy.EXCLUDE }

    dependencies {
        implementation project(':grobid-core')
        implementation project(':grobid-trainer')
        implementation "io.dropwizard:dropwizard-core:1.3.23"
        implementation "io.dropwizard:dropwizard-assets:1.3.23"
        implementation "com.hubspot.dropwizard:dropwizard-guicier:1.3.5.0"
        implementation "io.dropwizard:dropwizard-testing:1.3.23"
        implementation "io.dropwizard:dropwizard-forms:1.3.23"
        implementation "io.dropwizard:dropwizard-client:1.3.23"
        implementation "io.dropwizard:dropwizard-auth:1.3.23"
        implementation "org.apache.pdfbox:pdfbox:2.0.3"
        implementation "javax.activation:activation:1.1.1"
        implementation "io.prometheus:simpleclient_dropwizard:0.11.0"
        implementation "io.prometheus:simpleclient_servlet:0.11.0"

        testImplementation "io.dropwizard:dropwizard-testing:1.3.17"
    }

    shadowJar {
        archiveClassifier = 'onejar'
        mergeServiceFiles()
        zip64 true
        manifest {
            attributes 'Main-Class': 'org.grobid.core.main.batch.GrobidMain'
        }

        duplicatesStrategy = DuplicatesStrategy.EXCLUDE
    }

    artifacts {
        archives shadowJar
    }

    distributions {
        main {
            contents {
                from(new File(rootProject.rootDir, "grobid-service/README.md")) {
                    into "doc"
                }
                //from(new File(rootProject.rootDir, "../grobid-home/config/grobid.yaml")) {
                //    into "config"
                //}
            }
        }
    }

    task("run-server", dependsOn: 'classes', type: JavaExec, group: 'application') {
        main = 'org.grobid.service.main.GrobidServiceApplication'
        classpath = sourceSets.main.runtimeClasspath
        workingDir = ".."
        args 'server', 'grobid-home/config/grobid.yaml'
        jvmArgs '-Xmx3072m'
//        conda_env_path = "$System.env.CONDA_PREFIX"
        systemProperty "java.library.path","${System.getProperty('java.library.path')}:" +
            "${file("../grobid-home/lib/lin-64/jep").absolutePath}:" +
            "${file("../grobid-home/lib/lin-64").absolutePath}:" +
            "${file("../grobid-home/lib/mac-64").absolutePath}:" +
            "${System.env.CONDA_PREFIX}/lib:" +
            "${System.env.CONDA_PREFIX}/lib/python3.8/site-packages/jep"
    }
}

// /Users/lfoppiano/opt/anaconda3/envs/jep/lib/python3.8/site-packages/jep
// /Users/lfoppiano/opt/anaconda3/envs/jep/lib

project(":grobid-trainer") {
    apply plugin: 'com.github.johnrengelman.shadow'
    apply plugin: 'jacoco'

    dependencies {
        implementation(group: 'xml-apis', name: 'xml-apis') {
            // otherwise xml-apis 2.0.1 will come from XOM and will result in
            // java.lang.ClassNotFoundException: org.w3c.dom.ElementTraversal
            //TODO: sort out this problem better
            version {
                strictly '1.4.01'
            }
        }
        implementation project(':grobid-core')
        implementation "com.rockymadden.stringmetric:stringmetric-core_2.10:0.27.3"
        implementation "me.tongfei:progressbar:0.9.0"
        implementation 'org.slf4j:slf4j-log4j12:1.7.25'
        implementation 'org.slf4j:slf4j-api:1.7.25'
        implementation 'org.slf4j:slf4j-jdk14:1.7.25'
    }

    configurations {
        all*.exclude group: 'org.slf4j', module: "slf4j-jdk14"
        all*.exclude group: 'org.slf4j', module: "slf4j-log4j12"
        //all*.exclude group: 'log4j', module: "log4j"
    }

    jar {
        from {
//            project.configurations.implementation.collect {
            project.configurations.runtimeClasspath.collect {
                it.isDirectory() ? [] : localLibs.contains(it.getName()) ? zipTree(it) : []
            }
        }

        duplicatesStrategy = DuplicatesStrategy.EXCLUDE
    }

    shadowJar {
        archiveClassifier = 'onejar'
        mergeServiceFiles()
        zip64 true
        manifest {
            attributes 'Main-Class': 'org.grobid.trainer.TrainerRunner'
        }

        duplicatesStrategy = DuplicatesStrategy.EXCLUDE
    }

    artifacts {
        archives shadowJar
        archives jar
    }

    def trainerTasks = [
        "train_name_header"           : "org.grobid.trainer.NameHeaderTrainer",
        "train_name_citation"         : "org.grobid.trainer.NameCitationTrainer",
        "train_affiliation_address"   : "org.grobid.trainer.AffiliationAddressTrainer",
        "train_header"                : "org.grobid.trainer.HeaderTrainer",
        "train_fulltext"              : "org.grobid.trainer.FulltextTrainer",
        "train_shorttext"             : "org.grobid.trainer.ShorttextTrainer",
        "train_figure"                : "org.grobid.trainer.FigureTrainer",
        "train_table"                 : "org.grobid.trainer.TableTrainer",
        "train_citation"              : "org.grobid.trainer.CitationTrainer",
        "train_date"                  : "org.grobid.trainer.DateTrainer",
        "train_segmentation"          : "org.grobid.trainer.SegmentationTrainer",
        "train_reference_segmentation": "org.grobid.trainer.ReferenceSegmenterTrainer",
        "train_ebook_model"           : "org.grobid.trainer.EbookTrainer",
        "train_patent_citation"       : "org.grobid.trainer.PatentParserTrainer"
    ]

    trainerTasks.each { taskName, mainClassName ->
        tasks.create(name: taskName, type: JavaExec, group: 'modeltraining') {
            main = mainClassName
            classpath = sourceSets.main.runtimeClasspath
        }
    }

    // evaluation tasks
    ext.getArg = { propName, defaultVal ->
        return project.hasProperty(propName) ? project.getProperty(propName) : defaultVal;
    }

    // run like this:
    // ./gradlew jatsEval -Pp2t=/path/to/goldenSet
    // ./gradlew jatsEval -Pp2t=/path/to/goldenSet -Prun=1 -PfileRatio=0.1
    // ./gradlew teiEval -Pp2t=/path/to/goldenSet
    // ./gradlew PrepareDOIMatching -Pp2t=ABS_PATH_TO_PMC/PMC_sample_1943 
    // ./gradlew EvaluateDOIMatching -Pp2t=ABS_PATH_TO_PMC/PMC_sample_1943 
    task(jatsEval, dependsOn: 'classes', type: JavaExec, group: 'modelevaluation') {
        main = 'org.grobid.trainer.evaluation.EndToEndEvaluation'
        classpath = sourceSets.main.runtimeClasspath
        args 'nlm', getArg('p2t', '.'), getArg('run', '0'), getArg('fileRatio', '1.0')
        jvmArgs '-Xmx3072m'
        systemProperty "java.library.path","${System.getProperty('java.library.path')}:${file("../grobid-home/lib/lin-64/jep").absolutePath}:${file("../grobid-home/lib/lin-64").absolutePath}"
    }

    task(teiEval, dependsOn: 'classes', type: JavaExec, group: 'modelevaluation') {
        main = 'org.grobid.trainer.evaluation.EndToEndEvaluation'
        classpath = sourceSets.main.runtimeClasspath
        args 'tei', getArg('p2t', '.'), getArg('run', '0'), getArg('fileRatio', '1.0')
        jvmArgs '-Xmx3072m'
        systemProperty "java.library.path","${System.getProperty('java.library.path')}:${file("../grobid-home/lib/lin-64/jep").absolutePath}:${file("../grobid-home/lib/lin-64").absolutePath}"
    }

    task(PrepareDOIMatching, dependsOn: 'classes', type: JavaExec, group: 'modelevaluation') {
        main = 'org.grobid.trainer.evaluation.EvaluationDOIMatching'
        classpath = sourceSets.main.runtimeClasspath
        args 'data', getArg('p2t', '.')
        jvmArgs '-Xmx3072m'
        systemProperty "java.library.path","${System.getProperty('java.library.path')}:${file("../grobid-home/lib/lin-64/jep").absolutePath}:${file("../grobid-home/lib/lin-64").absolutePath}"
    }

    task(EvaluateDOIMatching, dependsOn: 'classes', type: JavaExec, group: 'modelevaluation') {
        main = 'org.grobid.trainer.evaluation.EvaluationDOIMatching'
        classpath = sourceSets.main.runtimeClasspath
        args 'eval', getArg('p2t', '.')
        jvmArgs '-Xmx3072m'
        systemProperty "java.library.path","${System.getProperty('java.library.path')}:${file("../grobid-home/lib/lin-64/jep").absolutePath}:${file("../grobid-home/lib/lin-64").absolutePath}"
    }
}

/** JACOCO **/

tasks.register("codeCoverageReport", JacocoReport) {
    // If a subproject applies the 'jacoco' plugin, add the result it to the report
    subprojects { subproject ->
        subproject.plugins.withType(JacocoPlugin).configureEach {
            subproject.tasks.matching({ t -> t.extensions.findByType(JacocoTaskExtension) }).configureEach { testTask ->
                sourceSets subproject.sourceSets.main
                executionData(testTask)
            }

            // To automatically run `test` every time `./gradlew codeCoverageReport` is called,
            // you may want to set up a task dependency between them as shown below.
            // Note that this requires the `test` tasks to be resolved eagerly (see `forEach`) which
            // may have a negative effect on the configuration time of your build.
            subproject.tasks.matching({ t -> t.extensions.findByType(JacocoTaskExtension) }).forEach {
                rootProject.tasks.codeCoverageReport.dependsOn(it)
            }
        }
    }

    // XML -> coveralls,
    // HTML -> for manual check
    reports {
        xml.enabled true
        html.enabled true
        csv.enabled true
    }

}

/** COVERALLS **/
coveralls {
    sourceDirs = files(subprojects.sourceSets.main.allSource.srcDirs).files.absolutePath
    jacocoReportPath = "build/reports/jacoco/codeCoverageReport/codeCoverageReport.xml"
}

tasks.coveralls {
    dependsOn codeCoverageReport
}

wrapper {
    gradleVersion "7.1.1"
}

build.dependsOn project.getSubprojects().collect({ it.getTasks().getByName("build") })