From c36249531832b9c6c7dfc9c9c47935bcde15ffbd Mon Sep 17 00:00:00 2001 From: Josh Feinberg Date: Mon, 16 May 2022 15:47:19 -0500 Subject: [PATCH 1/2] Update stone generation to support config cache --- build.gradle | 8 +- gradle/wrapper/gradle-wrapper.properties | 2 +- stone.gradle | 289 ++++++++++++++--------- 3 files changed, 186 insertions(+), 113 deletions(-) diff --git a/build.gradle b/build.gradle index f9aa1db1d..5bce89084 100644 --- a/build.gradle +++ b/build.gradle @@ -64,7 +64,7 @@ configurations { withoutOsgi.extendsFrom api } -processResources { +processResources { task -> filesMatching('**/*.crt') { fcd -> def inputstream = fcd.open() def certDatas = com.dropbox.maven.pem_converter.PemLoader.load( @@ -74,7 +74,7 @@ processResources { def crtPath = fcd.getPath() def rawPath = crtPath.substring(0, crtPath.length() - 4) + ".raw" - def rawFile = new File(getDestinationDir(), rawPath); + def rawFile = new File(task.getDestinationDir(), rawPath); rawFile.getParentFile().mkdirs(); def out = new DataOutputStream(new FileOutputStream(rawFile)) com.dropbox.maven.pem_converter.RawLoader.store(certDatas, out) @@ -206,7 +206,9 @@ jar { '*' def noeeProp = 'osgi.bnd.noee' - def noee = project.properties.get(noeeProp, System.properties.get(noeeProp, 'false')) + def noee = providers.gradleProperty(noeeProp).forUseAtConfigurationTime().getOrElse( + providers.systemProperty(noeeProp).forUseAtConfigurationTime().getOrElse('false') + ) instruction '-noee', noee } } diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index dc0c267b1..cf617f53c 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip diff --git a/stone.gradle b/stone.gradle index c1fdde901..b337aaab6 100644 --- a/stone.gradle +++ b/stone.gradle @@ -1,6 +1,6 @@ apply plugin: 'java' -class StoneConfig { +class StoneConfig implements Serializable { String packageName = 'com.dropbox.stone' String globalRouteFilter = null boolean dataTypesOnly = false @@ -8,7 +8,7 @@ class StoneConfig { String routeWhitelistFilter = null } -class ClientSpec { +class ClientSpec implements Serializable { String name = null String javadoc = null String routeFilter = null @@ -16,145 +16,216 @@ class ClientSpec { String unusedClassesToGenerate = null } -def runStoneGenerator(List configs, - File stoneDir, - File generatorFile, - Collection specFiles, - File outputDir, - String pythonCommand - ) { - def srcOutputDir = new File(outputDir, "src") - def refsFile = new File(outputDir, "refs/javadoc-refs.json") - def logFile = new File(outputDir, "log/stone.log") - - // delete output dir for a clean build - if (outputDir.exists()) { - if (!outputDir.deleteDir()) { - throw new GradleException("Failed to delete output directory: ${outputDir.absolutePath}") +abstract class StoneTask extends DefaultTask { + + @Input + protected abstract ListProperty getStoneConfigs(); + + @Input + protected abstract Property getGeneratorDir() + + @Input + protected abstract Property getSpecDir() + + @Optional + @Input + protected abstract Property getRouteWhitelistFilter() + + @Input + protected abstract Property getStoneDir() + + @Input + protected abstract Property getPythonCommand() + + @OutputDirectory + protected abstract DirectoryProperty getOutputDir(); + + @Inject + protected abstract ObjectFactory getObjectFactory(); + + @Inject + protected abstract ExecOperations getExecOperations(); + + + public void config(List configs) { + getStoneConfigs().set(configs) + } + + public void generatorDir(String generatorDir) { + getGeneratorDir().set(generatorDir) + } + + public void routeWhitelistFilter(String routeWhitelistFilter) { + getRouteWhitelistFilter().set(routeWhitelistFilter) + } + + public void stoneDir(String stoneDir) { + getStoneDir().set(stoneDir) + } + + public void pythonCommand(String pythonCommand) { + getPythonCommand().set(pythonCommand) + } + + public void specDir(String specDir) { + getSpecDir().set(specDir) + } + + public void outputDir(String outputDir) { + getOutputDir().set(new File(outputDir)) + } + + @TaskAction + public void processStone() { + def generatorFileTree = getObjectFactory().fileTree().from(getGeneratorDir().get()) + generatorFileTree.include('**/*stoneg.py') + def generatorFile = generatorFileTree.getSingleFile() + def specFiles = getSpecFiles(getObjectFactory(), getSpecDir().get()).getFiles() + for (StoneConfig config in getStoneConfigs().get()) { + config.routeWhitelistFilter = getRouteWhitelistFilter().getOrNull() } + runStoneGenerator( + getStoneConfigs().get(), + getObjectFactory().fileTree().from(getStoneDir().get()).getDir(), + generatorFile, + specFiles, + getOutputDir().getAsFile().get(), + getPythonCommand().get() + ) } - srcOutputDir.mkdirs() - logFile.parentFile.mkdirs() - refsFile.parentFile.mkdirs() + static FileTree getSpecFiles(ObjectFactory objectFactory, String specDir) { + def fileTree = objectFactory.fileTree().from(specDir) + fileTree.include('**/*.stone') + return fileTree + } - boolean isFirst = true + def runStoneGenerator( + List configs, + File stoneDir, + File generatorFile, + Collection specFiles, + File outputDir, + String pythonCommand + ) { + def srcOutputDir = new File(outputDir, "src") + def refsFile = new File(outputDir, "refs/javadoc-refs.json") + def logFile = new File(outputDir, "log/stone.log") - for (StoneConfig c : configs) { - boolean append = !isFirst - if (c.dataTypesOnly) { - // generate only data types. This is a much simpler call - if (c.client) { - throw new GradleException("Cannot specify dataTypesOnly and clients for Stone generation.") + // delete output dir for a clean build + if (outputDir.exists()) { + if (!outputDir.deleteDir()) { + throw new GradleException("Failed to delete output directory: ${outputDir.absolutePath}") } + } + srcOutputDir.mkdirs() + logFile.parentFile.mkdirs() + refsFile.parentFile.mkdirs() - project.exec { - standardOutput = new FileOutputStream(logFile, append) - commandLine pythonCommand, "-m", "stone.cli" - - environment PYTHONPATH: stoneDir.absolutePath + boolean isFirst = true - if (isFirst) { - args "--clean-build" + for (StoneConfig c : configs) { + boolean append = !isFirst + if (c.dataTypesOnly) { + // generate only data types. This is a much simpler call + if (c.client) { + throw new GradleException("Cannot specify dataTypesOnly and clients for Stone generation.") } - if (c.routeWhitelistFilter) { - args "--route-whitelist-filter", config.routeWhitelistFilter + + + getExecOperations().exec { + standardOutput = new FileOutputStream(logFile, append) + commandLine pythonCommand, "-m", "stone.cli" + + environment PYTHONPATH: stoneDir.absolutePath + + if (isFirst) { + args "--clean-build" + } + if (c.routeWhitelistFilter) { + args "--route-whitelist-filter", config.routeWhitelistFilter + } + args generatorFile.absolutePath + args srcOutputDir.absolutePath + args specFiles.collect({ f -> f.absolutePath }) + args "--" + args "--package", c.packageName + args "--data-types-only" } - args generatorFile.absolutePath - args srcOutputDir.absolutePath - args specFiles.collect({ f -> f.absolutePath }) - args "--" - args "--package", c.packageName - args "--data-types-only" - } - } else { - def client = c.client - def routeFilters = [c.globalRouteFilter, client.routeFilter] - def routeFilter = routeFilters\ + } else { + def client = c.client + def routeFilters = [c.globalRouteFilter, client.routeFilter] + def routeFilter = routeFilters\ .findAll { filter -> filter != null }\ .collect { filter -> "(${filter})" }\ .join " and " - project.exec { - standardOutput = new FileOutputStream(logFile, append) - commandLine pythonCommand, "-m", "stone.cli" + getExecOperations().exec { + standardOutput = new FileOutputStream(logFile, append) + commandLine pythonCommand, "-m", "stone.cli" - environment PYTHONPATH: stoneDir.absolutePath - if (isFirst) { - args "--clean-build" - } - args "--attribute", ":all" - if (routeFilter) { - args "--filter-by-route-attr", routeFilter - } - if (c.routeWhitelistFilter) { - args "--route-whitelist-filter", c.routeWhitelistFilter - } - args generatorFile.absolutePath - args srcOutputDir.absolutePath - args specFiles.collect({ f -> f.absolutePath }) - args "--" - args "--package", c.packageName - args "--javadoc-refs", refsFile.absolutePath - - if (client.name) { - args "--client-class", client.name - } - if (client.javadoc != null) { - args "--client-javadoc", client.javadoc - } - if (client.requestsClassnamePrefix != null) { - args "--requests-classname-prefix", client.requestsClassnamePrefix - } - if (client.unusedClassesToGenerate != null) { - args "--unused-classes-to-generate", client.unusedClassesToGenerate + environment PYTHONPATH: stoneDir.absolutePath + if (isFirst) { + args "--clean-build" + } + args "--attribute", ":all" + if (routeFilter) { + args "--filter-by-route-attr", routeFilter + } + if (c.routeWhitelistFilter) { + args "--route-whitelist-filter", c.routeWhitelistFilter + } + args generatorFile.absolutePath + args srcOutputDir.absolutePath + args specFiles.collect({ f -> f.absolutePath }) + args "--" + args "--package", c.packageName + args "--javadoc-refs", refsFile.absolutePath + + if (client.name) { + args "--client-class", client.name + } + if (client.javadoc != null) { + args "--client-javadoc", client.javadoc + } + if (client.requestsClassnamePrefix != null) { + args "--requests-classname-prefix", client.requestsClassnamePrefix + } + if (client.unusedClassesToGenerate != null) { + args "--unused-classes-to-generate", client.unusedClassesToGenerate + } } } + isFirst = false } - isFirst = false } } - - // add generateStone task for all source sets (e.g. generateTestStone, etc) project.sourceSets.all { SourceSet sourceSet -> def taskName = "main" == sourceSet.name ? "generateStone" : "generate${sourceSet.name.capitalize()}Stone" - task "${taskName}" { + tasks.register(taskName, StoneTask) { description "Generate Stone Java source files for ${sourceSet.name}." def specDirPropName = "com.dropbox.api.${sourceSet.name}.specDir".toString() + def mySpecDir = project.properties.get(specDirPropName, "src/${sourceSet.name}/stone") + specDir mySpecDir def routeWhitelistFilterPropName = "com.dropbox.api.${sourceSet.name}.routeWhitelistFilter".toString() - - ext { - configs = [] - generatorDir = 'generator' - stoneDir = 'stone' - specDir = project.properties.get(specDirPropName, "src/${sourceSet.name}/stone") - outputDir = "${project.buildDir}/generated/source/stone/${sourceSet.name}" - routeWhitelistFilter = project.properties.get(routeWhitelistFilterPropName, null) - pythonCommand = 'python' - } - - def getSpecFiles = { fileTree(dir: specDir, include: '**/*.stone') } + generatorDir 'generator' + stoneDir 'stone' + routeWhitelistFilter project.properties.get(routeWhitelistFilterPropName, null) + pythonCommand 'python' + outputDir "${project.buildDir}/generated/source/stone/${sourceSet.name}" inputs.dir { project.fileTree(dir: generatorDir, exclude: '**/*.pyc') }.withPropertyName("stone").withPathSensitivity(PathSensitivity.RELATIVE) - inputs.dir(getSpecFiles).withPathSensitivity(PathSensitivity.RELATIVE).withPropertyName("stoneSpec").skipWhenEmpty(true) - inputs.property "configs", { new groovy.json.JsonBuilder(configs).toString() } - outputs.dir { outputDir }.withPropertyName("generatedStone") + inputs.dir(getSpecFiles(objects, getSpecDir().get())).withPathSensitivity(PathSensitivity.RELATIVE).withPropertyName("stoneSpec").skipWhenEmpty(true) + inputs.property "configs", { new groovy.json.JsonBuilder(getStoneConfigs().get()).toString() } + outputs.dir { getOutputDir().get() }.withPropertyName("generatedStone") outputs.cacheIf { true } - doLast { - def generatorFile = fileTree(dir: generatorDir, include: '**/*stoneg.py').getSingleFile() - def specFiles = getSpecFiles().getFiles() - for (StoneConfig config in configs) { - config.routeWhitelistFilter = routeWhitelistFilter - } - runStoneGenerator(configs, file(stoneDir), generatorFile, specFiles, file(outputDir), pythonCommand) - } } - sourceSet.java.srcDir project.tasks."${taskName}".outputDir + "/src" + sourceSet.java.srcDir project.tasks."${taskName}".getOutputDir().get().toString() + "/src" Task compile = project.tasks.getByName(sourceSet.getCompileTaskName("java")) compile.dependsOn project.tasks."${taskName}" } @@ -164,7 +235,7 @@ generateStone { String unusedClassesToGenerate = 'AuthError, PathRoot, PathRootError, AccessError, RateLimitError' String packageName = 'com.dropbox.core.v2' String globalRouteFilter = 'alpha_group=null and beta_group=null' - configs = [ + config [ new StoneConfig( packageName: packageName, globalRouteFilter: globalRouteFilter, @@ -202,7 +273,7 @@ generateStone { generateTestStone { String packageName = 'com.dropbox.core.stone' - configs = [ + config [ new StoneConfig( packageName: packageName, dataTypesOnly: true, From 22db4be99c2af67c9c501c28aefbea47663f5290 Mon Sep 17 00:00:00 2001 From: Josh Feinberg Date: Mon, 16 May 2022 16:17:55 -0500 Subject: [PATCH 2/2] Upgrade to python3 --- .github/workflows/ci-examples.yml | 4 ++-- .github/workflows/ci-tests.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-examples.yml b/.github/workflows/ci-examples.yml index b1acdbb3d..25667ed9c 100644 --- a/.github/workflows/ci-examples.yml +++ b/.github/workflows/ci-examples.yml @@ -24,9 +24,9 @@ jobs: distribution: 'zulu' - uses: actions/checkout@v2 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: - python-version: '2.7' + python-version: '3.9.10' - run: python -m pip install ply && pip install six - name: Grant execute permissions run: chmod +x gradlew && chmod +x update-submodules diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 00736cd39..bb0d8a63a 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -24,9 +24,9 @@ jobs: distribution: 'zulu' - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: - python-version: '2.7' + python-version: '3.9.10' - run: python -m pip install ply && pip install six - name: Grant execute permissions