Skip to content

Commit

Permalink
More changes to cache format, now the dependency tree generation is c…
Browse files Browse the repository at this point in the history 
…ached
  • Loading branch information
@gbevin
gbevin committed Jul 20, 2024
1 parent d24e9d2 commit 3ec2cc6
Show file tree
Hide file tree
Showing 5 changed files with 257 additions and 19 deletions.
Binary file modified lib/bld/bld-wrapper.jar
View file
Binary file not shown.
5 changes: 3 additions & 2 deletions src/main/java/rife/bld/BaseProject.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1593,14 +1593,15 @@ public void executeAutoDownloadPurge()
private void performAutoDownloadPurge() {
var resolution = new VersionResolution(properties());
var cache = new BldCache(libBldDirectory(), resolution);
cache.fingerprintDependencies(repositories(), dependencies(), downloadSources(), downloadJavadoc());
if (cache.isDependenciesHashValid()) {
cache.fingerprintDependencies(repositories(), dependencies());
if (cache.isDependenciesCacheValid(downloadSources(), downloadJavadoc())) {
return;
}

try {
executeAutoDownloadPurge();

cache.cacheDependenciesDownloads(downloadSources(), downloadJavadoc());
cache.writeCache();
} catch (Exception e) {
throw new RuntimeException(e);
Expand Down
164 changes: 155 additions & 9 deletions src/main/java/rife/bld/BldCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,21 +33,42 @@ public class BldCache {

private static final String PROPERTY_SUFFIX_HASH = ".hash";
private static final String PROPERTY_SUFFIX_LOCAL = ".local";
private static final String PROPERTY_SUFFIX_DOWNLOAD_SOURCES = ".download.sources";
private static final String PROPERTY_SUFFIX_DOWNLOAD_JAVADOC = ".download.javadoc";
private static final String PROPERTY_SUFFIX_DEPENDENCY_TREE = ".dependency.tree";

private static final String WRAPPER_PROPERTIES_HASH = Wrapper.WRAPPER_PROPERTIES + PROPERTY_SUFFIX_HASH;
private static final String BLD_BUILD_HASH = "bld-build" + PROPERTY_SUFFIX_HASH;

private static final String PROPERTY_EXTENSIONS_PREFIX = "bld.extensions";
private static final String PROPERTY_EXTENSIONS_HASH = PROPERTY_EXTENSIONS_PREFIX + PROPERTY_SUFFIX_HASH;
private static final String PROPERTY_EXTENSIONS_LOCAL = PROPERTY_EXTENSIONS_PREFIX + PROPERTY_SUFFIX_LOCAL;
private static final String PROPERTY_EXTENSIONS_DOWNLOAD_SOURCES = PROPERTY_EXTENSIONS_PREFIX + PROPERTY_SUFFIX_DOWNLOAD_SOURCES;
private static final String PROPERTY_EXTENSIONS_DOWNLOAD_JAVADOC = PROPERTY_EXTENSIONS_PREFIX + PROPERTY_SUFFIX_DOWNLOAD_JAVADOC;
private static final String PROPERTY_EXTENSIONS_DEPENDENCY_TREE = PROPERTY_EXTENSIONS_PREFIX + PROPERTY_SUFFIX_DEPENDENCY_TREE;

private static final String PROPERTY_DEPENDENCIES_PREFIX = "bld.dependencies";
private static final String PROPERTY_DEPENDENCIES_HASH = PROPERTY_DEPENDENCIES_PREFIX + PROPERTY_SUFFIX_HASH;
private static final String PROPERTY_DEPENDENCIES_DOWNLOAD_SOURCES = PROPERTY_DEPENDENCIES_PREFIX + PROPERTY_SUFFIX_DOWNLOAD_SOURCES;
private static final String PROPERTY_DEPENDENCIES_DOWNLOAD_JAVADOC = PROPERTY_DEPENDENCIES_PREFIX + PROPERTY_SUFFIX_DOWNLOAD_JAVADOC;
private static final String PROPERTY_DEPENDENCIES_COMPILE_DEPENDENCY_TREE = PROPERTY_DEPENDENCIES_PREFIX + ".compile" + PROPERTY_SUFFIX_DEPENDENCY_TREE;
private static final String PROPERTY_DEPENDENCIES_PROVIDED_DEPENDENCY_TREE = PROPERTY_DEPENDENCIES_PREFIX + ".provided" + PROPERTY_SUFFIX_DEPENDENCY_TREE;
private static final String PROPERTY_DEPENDENCIES_RUNTIME_DEPENDENCY_TREE = PROPERTY_DEPENDENCIES_PREFIX + ".runtime" + PROPERTY_SUFFIX_DEPENDENCY_TREE;
private static final String PROPERTY_DEPENDENCIES_TEST_DEPENDENCY_TREE = PROPERTY_DEPENDENCIES_PREFIX + ".test" + PROPERTY_SUFFIX_DEPENDENCY_TREE;

private final File bldLibDir_;
private final VersionResolution resolution_;
private String extensionsHash_;
private Boolean extensionsDownloadSources_;
private Boolean extensionsDownloadJavadocs_;
private String extensionsDependencyTree_;
private String dependenciesHash_;
private Boolean dependenciesDownloadSources_;
private Boolean dependenciesDownloadJavadocs_;
private String dependenciesCompileDependencyTree_;
private String dependenciesProvidedDependencyTree_;
private String dependenciesRuntimeDependencyTree_;
private String dependenciesTestDependencyTree_;

public BldCache(File bldLibDir, VersionResolution resolution) {
bldLibDir_ = bldLibDir;
Expand All @@ -57,12 +78,12 @@ public BldCache(File bldLibDir, VersionResolution resolution) {
new File(bldLibDir, BLD_BUILD_HASH).delete();
}

public void fingerprintExtensions(Collection<String> repositories, Collection<String> extensions, boolean downloadSources, boolean downloadJavadoc) {
public void fingerprintExtensions(Collection<String> repositories, Collection<String> extensions) {
try {
var overrides_fp = String.join("\n", resolution_.versionOverrides().entrySet().stream().map(e -> e.getKey() + ":" + e.getValue()).toList());
var repositories_fp = String.join("\n", repositories);
var extensions_fp = String.join("\n", extensions);
var fingerprint = overrides_fp + "\n" + repositories_fp + "\n" + extensions_fp + "\n" + downloadSources + "\n" + downloadJavadoc;
var fingerprint = overrides_fp + "\n" + repositories_fp + "\n" + extensions_fp + "\n";
var digest = MessageDigest.getInstance("SHA-1");
digest.update(fingerprint.getBytes(StandardCharsets.UTF_8));

Expand All @@ -73,7 +94,20 @@ public void fingerprintExtensions(Collection<String> repositories, Collection<St
}
}

public void fingerprintDependencies(List<Repository> repositories, DependencyScopes dependencies, boolean downloadSources, boolean downloadJavadoc) {
public void cacheExtensionsDownloads(boolean downloadSources, boolean downloadJavadoc) {
extensionsDownloadSources_ = downloadSources;
extensionsDownloadJavadocs_ = downloadJavadoc;
}

public void cacheExtensionsDependencyTree(String dependencyTree) {
extensionsDependencyTree_ = dependencyTree;
}

public String getCachedExtensionsDependencyTree() {
return hashProperties().getProperty(PROPERTY_EXTENSIONS_DEPENDENCY_TREE);
}

public void fingerprintDependencies(List<Repository> repositories, DependencyScopes dependencies) {
var finger_print = new StringBuilder();
finger_print.append(String.join("\n", resolution_.versionOverrides().entrySet().stream().map(e -> e.getKey() + ":" + e.getValue()).toList()));
for (var repository : repositories) {
Expand All @@ -90,10 +124,6 @@ public void fingerprintDependencies(List<Repository> repositories, DependencySco
}
}
}
finger_print.append(downloadSources)
.append('\n')
.append(downloadJavadoc)
.append('\n');

try {
var digest = MessageDigest.getInstance("SHA-1");
Expand All @@ -105,10 +135,64 @@ public void fingerprintDependencies(List<Repository> repositories, DependencySco
}
}

public void cacheDependenciesDownloads(boolean downloadSources, boolean downloadJavadoc) {
dependenciesDownloadSources_ = downloadSources;
dependenciesDownloadJavadocs_ = downloadJavadoc;
}

public void cacheDependenciesCompileDependencyTree(String compileTree) {
dependenciesCompileDependencyTree_ = compileTree;
}

public String getCachedDependenciesCompileDependencyTree() {
return hashProperties().getProperty(PROPERTY_DEPENDENCIES_COMPILE_DEPENDENCY_TREE);
}

public void cacheDependenciesProvidedDependencyTree(String providedTree) {
dependenciesProvidedDependencyTree_ = providedTree;
}

public String getCachedDependenciesProvidedDependencyTree() {
return hashProperties().getProperty(PROPERTY_DEPENDENCIES_PROVIDED_DEPENDENCY_TREE);
}

public void cacheDependenciesRuntimeDependencyTree(String runtimeTree) {
dependenciesRuntimeDependencyTree_ = runtimeTree;
}

public String getCachedDependenciesRuntimeDependencyTree() {
return hashProperties().getProperty(PROPERTY_DEPENDENCIES_RUNTIME_DEPENDENCY_TREE);
}

public void cacheDependenciesTestDependencyTree(String testTree) {
dependenciesTestDependencyTree_ = testTree;
}

public String getCachedDependenciesTestDependencyTree() {
return hashProperties().getProperty(PROPERTY_DEPENDENCIES_TEST_DEPENDENCY_TREE);
}

public boolean isExtensionHashValid() {
return validateExtensionsHash(extensionsHash_);
}

public boolean isExtensionsCacheValid(boolean downloadSources, boolean downloadJavadoc) {
var properties = hashProperties();
if (properties.isEmpty()) {
return false;
}

if (downloadSources != Boolean.parseBoolean(properties.getProperty(PROPERTY_EXTENSIONS_DOWNLOAD_SOURCES))) {
return false;
}

if (downloadJavadoc != Boolean.parseBoolean(properties.getProperty(PROPERTY_EXTENSIONS_DOWNLOAD_JAVADOC))) {
return false;
}

return validateExtensionsHash(extensionsHash_);
}

private File getCacheFile() {
return new File(bldLibDir_, BLD_CACHE);
}
Expand Down Expand Up @@ -170,6 +254,23 @@ public boolean isDependenciesHashValid() {
return validateDependenciesHash(dependenciesHash_);
}

public boolean isDependenciesCacheValid(boolean downloadSources, boolean downloadJavadoc) {
var properties = hashProperties();
if (properties.isEmpty()) {
return false;
}

if (downloadSources != Boolean.parseBoolean(properties.getProperty(PROPERTY_DEPENDENCIES_DOWNLOAD_SOURCES))) {
return false;
}

if (downloadJavadoc != Boolean.parseBoolean(properties.getProperty(PROPERTY_DEPENDENCIES_DOWNLOAD_JAVADOC))) {
return false;
}

return validateDependenciesHash(dependenciesHash_);
}

private boolean validateDependenciesHash(String hash) {
var properties = hashProperties();
if (properties.isEmpty()) {
Expand All @@ -188,7 +289,22 @@ public void writeCache(List<File> extensionsLocalArtifacts) {

try {
if (extensionsHash_ != null) {
properties.put(PROPERTY_EXTENSIONS_HASH, extensionsHash_);
if (!extensionsHash_.equals(properties.get(PROPERTY_EXTENSIONS_HASH))) {
properties.put(PROPERTY_EXTENSIONS_HASH, extensionsHash_);
properties.remove(PROPERTY_EXTENSIONS_DEPENDENCY_TREE);
}

if (extensionsDependencyTree_ != null) {
properties.put(PROPERTY_EXTENSIONS_DEPENDENCY_TREE, extensionsDependencyTree_);
}
}

if (extensionsDownloadSources_ != null) {
properties.put(PROPERTY_EXTENSIONS_DOWNLOAD_SOURCES, String.valueOf(extensionsDownloadSources_));
}

if (extensionsDownloadJavadocs_ != null) {
properties.put(PROPERTY_EXTENSIONS_DOWNLOAD_JAVADOC, String.valueOf(extensionsDownloadJavadocs_));
}

if (extensionsLocalArtifacts != null) {
Expand All @@ -205,7 +321,37 @@ public void writeCache(List<File> extensionsLocalArtifacts) {
}

if (dependenciesHash_ != null) {
properties.put(PROPERTY_DEPENDENCIES_HASH, dependenciesHash_);
if (!dependenciesHash_.equals(properties.get(PROPERTY_DEPENDENCIES_HASH))) {
properties.put(PROPERTY_DEPENDENCIES_HASH, dependenciesHash_);
properties.remove(PROPERTY_DEPENDENCIES_COMPILE_DEPENDENCY_TREE);
properties.remove(PROPERTY_DEPENDENCIES_PROVIDED_DEPENDENCY_TREE);
properties.remove(PROPERTY_DEPENDENCIES_RUNTIME_DEPENDENCY_TREE);
properties.remove(PROPERTY_DEPENDENCIES_TEST_DEPENDENCY_TREE);
}

if (dependenciesCompileDependencyTree_ != null) {
properties.put(PROPERTY_DEPENDENCIES_COMPILE_DEPENDENCY_TREE, dependenciesCompileDependencyTree_);
}

if (dependenciesProvidedDependencyTree_ != null) {
properties.put(PROPERTY_DEPENDENCIES_PROVIDED_DEPENDENCY_TREE, dependenciesProvidedDependencyTree_);
}

if (dependenciesRuntimeDependencyTree_ != null) {
properties.put(PROPERTY_DEPENDENCIES_RUNTIME_DEPENDENCY_TREE, dependenciesRuntimeDependencyTree_);
}

if (dependenciesTestDependencyTree_ != null) {
properties.put(PROPERTY_DEPENDENCIES_TEST_DEPENDENCY_TREE, dependenciesTestDependencyTree_);
}
}

if (dependenciesDownloadSources_ != null) {
properties.put(PROPERTY_DEPENDENCIES_DOWNLOAD_SOURCES, String.valueOf(dependenciesDownloadSources_));
}

if (dependenciesDownloadJavadocs_ != null) {
properties.put(PROPERTY_DEPENDENCIES_DOWNLOAD_JAVADOC, String.valueOf(dependenciesDownloadJavadocs_));
}

bldLibDir_.mkdirs();
Expand Down
101 changes: 96 additions & 5 deletions src/main/java/rife/bld/operations/DependencyTreeOperation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@
package rife.bld.operations;

import rife.bld.BaseProject;
import rife.bld.BldCache;
import rife.bld.BldVersion;
import rife.bld.BuildExecutor;
import rife.bld.dependencies.*;
import rife.bld.wrapper.Wrapper;
import rife.ioc.HierarchicalProperties;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
Expand All @@ -34,6 +36,7 @@ public class DependencyTreeOperation extends AbstractOperation<DependencyTreeOpe
private final DependencyScopes extensionDependencies_ = new DependencyScopes();

private final StringBuilder dependencyTree_ = new StringBuilder();
private File libBldDir_ = null;

/**
* Performs the dependency tree operation.
Expand All @@ -46,11 +49,97 @@ public void execute() {
return;
}

var extensions_tree = executeGenerateExtensionsDependencies();
var compile_tree = executeGenerateCompileDependencies();
var provided_tree = executeGenerateProvidedDependencies();
var runtime_tree = executeGenerateRuntimeDependencies();
var test_tree = executeGenerateTestDependencies();
// calculate the dependency tree of the extensions, using the cache if possible

String extensions_tree = null;
BldCache extensions_cache = null;
if (libBldDir_ != null) {
extensions_cache = new BldCache(libBldDir_, new VersionResolution(extensionProperties()));
extensions_cache.fingerprintExtensions(
extensionRepositories().stream().map(Repository::toString).toList(),
extensionDependencies().scope(compile).stream().map(Dependency::toString).toList());
if (extensions_cache.isExtensionHashValid()) {
var cached_tree = extensions_cache.getCachedExtensionsDependencyTree();
if (cached_tree != null) {
extensions_tree = cached_tree;
}
}
}

if (extensions_tree == null) {
extensions_tree = executeGenerateExtensionsDependencies();
if (extensions_cache != null) {
extensions_cache.cacheExtensionsDependencyTree(extensions_tree);
extensions_cache.writeCache();
}
}

// calculate the dependency tree of the dependencies, using the cache if possible

String compile_tree = null;
String provided_tree = null;
String runtime_tree = null;
String test_tree = null;
BldCache dependencies_cache = null;
if (libBldDir_ != null) {
dependencies_cache = new BldCache(libBldDir_, new VersionResolution(properties()));
dependencies_cache.fingerprintDependencies(repositories(), dependencies());
if (dependencies_cache.isDependenciesHashValid()) {
var cached_compile_tree = dependencies_cache.getCachedDependenciesCompileDependencyTree();
if (cached_compile_tree != null) {
compile_tree = cached_compile_tree;
}
var cached_provided_tree = dependencies_cache.getCachedDependenciesProvidedDependencyTree();
if (cached_provided_tree != null) {
provided_tree = cached_provided_tree;
}
var cached_runtime_tree = dependencies_cache.getCachedDependenciesRuntimeDependencyTree();
if (cached_runtime_tree != null) {
runtime_tree = cached_runtime_tree;
}
var cached_test_tree = dependencies_cache.getCachedDependenciesTestDependencyTree();
if (cached_test_tree != null) {
test_tree = cached_test_tree;
}
}
}

var write_dependencies_cache = false;
if (compile_tree == null) {
compile_tree = executeGenerateCompileDependencies();
if (dependencies_cache != null) {
dependencies_cache.cacheDependenciesCompileDependencyTree(compile_tree);
write_dependencies_cache = true;
}
}
if (provided_tree == null) {
provided_tree = executeGenerateProvidedDependencies();
if (dependencies_cache != null) {
dependencies_cache.cacheDependenciesProvidedDependencyTree(provided_tree);
write_dependencies_cache = true;
}
}
if (runtime_tree == null) {
runtime_tree = executeGenerateRuntimeDependencies();
if (dependencies_cache != null) {
dependencies_cache.cacheDependenciesRuntimeDependencyTree(runtime_tree);
write_dependencies_cache = true;
}
}
if (test_tree == null) {
test_tree = executeGenerateTestDependencies();
if (dependencies_cache != null) {
dependencies_cache.cacheDependenciesTestDependencyTree(test_tree);
write_dependencies_cache = true;
}
}

if (write_dependencies_cache) {
dependencies_cache.writeCache();
}

// output the dependency trees

dependencyTree_.setLength(0);
dependencyTree_.append(extensions_tree);
dependencyTree_.append(System.lineSeparator());
Expand Down Expand Up @@ -143,6 +232,8 @@ protected String executeGenerateTestDependencies() {
* @since 1.5.21
*/
public DependencyTreeOperation fromProject(BaseProject project) {
libBldDir_ = project.libBldDirectory();

// add the repositories and dependencies from the extensions
var wrapper = new Wrapper();
wrapper.currentDir(project.workDirectory());
Expand Down
Loading

0 comments on commit 3ec2cc6

Please sign in to comment.