Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
use sip hash algorithm [ci fast]
Browse files Browse the repository at this point in the history
Signed-off-by: Paolo Di Tommaso <[email protected]>
pditommaso committed Nov 10, 2024
1 parent 76696f4 commit 7edc0e7
Showing 2 changed files with 21 additions and 11 deletions.
18 changes: 15 additions & 3 deletions modules/nextflow/src/main/groovy/nextflow/conda/CondaCache.groovy
Original file line number Diff line number Diff line change
@@ -22,6 +22,7 @@ import java.nio.file.Path
import java.nio.file.Paths
import java.util.concurrent.ConcurrentHashMap

import com.google.common.hash.Hashing
import groovy.transform.CompileStatic
import groovy.transform.PackageScope
import groovy.util.logging.Slf4j
@@ -32,7 +33,6 @@ import nextflow.file.FileMutex
import nextflow.util.CacheHelper
import nextflow.util.Duration
import nextflow.util.Escape
import org.yaml.snakeyaml.Yaml
/**
* Handle Conda environment creation and caching
*
@@ -166,6 +166,18 @@ class CondaCache {
str.endsWith('.txt') && !str.contains('\n')
}

static protected String sipHash(CharSequence data) {
Hashing
.sipHash24()
.newHasher()
.putUnencodedChars(data)
.hash()
.toString()
}

static protected String sipHash(Path path) {
sipHash(path.toAbsolutePath().normalize().toString())
}

/**
* Get the path on the file system where store a Conda environment
@@ -188,7 +200,7 @@ class CondaCache {
try {
final path = condaEnv as Path
content = path.text
name = 'env-' + CacheHelper.hasher(path.toString()).hash().toString()
name = 'env-' + sipHash(path)

}
catch( NoSuchFileException e ) {
@@ -202,7 +214,7 @@ class CondaCache {
try {
final path = condaEnv as Path
content = path.text
name = 'env-'+ CacheHelper.hasher(path.toString()).hash().toString()
name = 'env-' + sipHash(path)
}
catch( NoSuchFileException e ) {
throw new IllegalArgumentException("Conda environment file does not exist: $condaEnv")
Original file line number Diff line number Diff line change
@@ -19,8 +19,6 @@ package nextflow.conda
import java.nio.file.Files
import java.nio.file.Paths

import nextflow.util.CacheHelper

import spock.lang.Specification
/**
*
@@ -91,7 +89,7 @@ class CondaCacheTest extends Specification {
def cache = Spy(CondaCache)
def BASE = Paths.get('/conda/envs')
def ENV = folder.resolve('foo.yml')
def hash = CacheHelper.hasher(ENV.toString()).hash().toString()
def hash = CondaCache.sipHash(ENV)
ENV.text = '''
channels:
- bioconda
@@ -107,7 +105,7 @@ class CondaCacheTest extends Specification {
then:
1 * cache.isYamlFilePath(ENV.toString())
1 * cache.getCacheDir() >> BASE
prefix.toString() == "/conda/envs/env-$hash-9416240708c49c4e627414b46a743664"
prefix.toString() == "/conda/envs/env-${hash}-9416240708c49c4e627414b46a743664"

cleanup:
folder?.deleteDir()
@@ -120,7 +118,7 @@ class CondaCacheTest extends Specification {
def cache = Spy(CondaCache)
def BASE = Paths.get('/conda/envs')
def ENV = Files.createTempFile('test','.yml')
def hash = CacheHelper.hasher(ENV.toString()).hash().toString()
def hash = CondaCache.sipHash(ENV)
ENV.text = '''
name: my-env-1.1
channels:
@@ -138,7 +136,7 @@ class CondaCacheTest extends Specification {
then:
1 * cache.isYamlFilePath(ENV.toString())
1 * cache.getCacheDir() >> BASE
prefix.toString() == "/conda/envs/env-$hash-e7fafe40ca966397a2c0d9bed7181aa7"
prefix.toString() == "/conda/envs/env-${hash}-e7fafe40ca966397a2c0d9bed7181aa7"

}

@@ -149,7 +147,7 @@ class CondaCacheTest extends Specification {
def cache = Spy(CondaCache)
def BASE = Paths.get('/conda/envs')
def ENV = folder.resolve('bar.txt')
def hash = CacheHelper.hasher(ENV.toString()).hash().toString()
def hash = CondaCache.sipHash(ENV)
ENV.text = '''
star=2.5.4a
bwa=0.7.15
@@ -163,7 +161,7 @@ class CondaCacheTest extends Specification {
1 * cache.isYamlFilePath(ENV.toString())
1 * cache.isTextFilePath(ENV.toString())
1 * cache.getCacheDir() >> BASE
prefix.toString() == "/conda/envs/env-$hash-8a4aa7db8ddb8ce4eb4d450d4814a437"
prefix.toString() == "/conda/envs/env-${hash}-8a4aa7db8ddb8ce4eb4d450d4814a437"

cleanup:
folder?.deleteDir()

0 comments on commit 7edc0e7

Please sign in to comment.