From 1e3d38dbbce977c7da7158214b0247313c2fda69 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Thu, 20 Jul 2023 10:40:01 -0500 Subject: [PATCH 1/2] Treat HTTP headers as case insensitive Signed-off-by: Ben Sherman --- .../file/http/XFileSystemProvider.groovy | 19 ++++++++++++++----- .../file/http/XFileSystemProviderTest.groovy | 4 ++-- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/modules/nf-httpfs/src/main/nextflow/file/http/XFileSystemProvider.groovy b/modules/nf-httpfs/src/main/nextflow/file/http/XFileSystemProvider.groovy index 67e7f9bde0..328cef8d00 100644 --- a/modules/nf-httpfs/src/main/nextflow/file/http/XFileSystemProvider.groovy +++ b/modules/nf-httpfs/src/main/nextflow/file/http/XFileSystemProvider.groovy @@ -191,8 +191,8 @@ abstract class XFileSystemProvider extends FileSystemProvider { XAuthRegistry.instance.authorize(conn) } if ( conn instanceof HttpURLConnection && conn.getResponseCode() in [307, 308] && attempt < MAX_REDIRECT_HOPS ) { - def header = conn.getHeaderFields() - String location = header.get("Location")?.get(0) + def header = getNormalizedHeaders(conn) + String location = header.get("location")?.get(0) URL newPath = new URI(location).toURL() log.debug "Remote redirect URL: $newPath" return toConnection0(newPath, attempt+1) @@ -454,18 +454,27 @@ abstract class XFileSystemProvider extends FileSystemProvider { return new XFileAttributes(null,-1) } if ( conn instanceof HttpURLConnection && conn.getResponseCode() in [200, 301, 302, 307, 308]) { - def header = conn.getHeaderFields() + def header = getNormalizedHeaders(conn) return readHttpAttributes(header) } return null } protected XFileAttributes readHttpAttributes(Map> header) { - def lastMod = header.get("Last-Modified")?.get(0) - long contentLen = header.get("Content-Length")?.get(0)?.toLong() ?: -1 + def lastMod = header.get("last-modified")?.get(0) + long contentLen = header.get("content-length")?.get(0)?.toLong() ?: -1 def dateFormat = new SimpleDateFormat('E, dd MMM yyyy HH:mm:ss Z', Locale.ENGLISH) // <-- make sure date parse is not language dependent (for the week day) def modTime = lastMod ? FileTime.from(dateFormat.parse(lastMod).time, TimeUnit.MILLISECONDS) : (FileTime)null new XFileAttributes(modTime, contentLen) } + static protected Map> getNormalizedHeaders(URLConnection conn) { + final header = conn.getHeaderFields() + header.inject([:]) { accum, key, value -> + final normalized = key != null ? key.toLowerCase() : null + accum[normalized] = value + accum + } + } + } diff --git a/modules/nf-httpfs/src/test/nextflow/file/http/XFileSystemProviderTest.groovy b/modules/nf-httpfs/src/test/nextflow/file/http/XFileSystemProviderTest.groovy index 85c4c9479c..9bbabfa06e 100644 --- a/modules/nf-httpfs/src/test/nextflow/file/http/XFileSystemProviderTest.groovy +++ b/modules/nf-httpfs/src/test/nextflow/file/http/XFileSystemProviderTest.groovy @@ -63,7 +63,7 @@ class XFileSystemProviderTest extends Specification { def "should read file attributes from map"() { given: def fs = new HttpFileSystemProvider() - def attrMap = ['Last-Modified': ['Fri, 04 Nov 2016 21:50:34 GMT'], 'Content-Length': ['21729']] + def attrMap = ['last-modified': ['Fri, 04 Nov 2016 21:50:34 GMT'], 'content-length': ['21729']] when: def attrs = fs.readHttpAttributes(attrMap) @@ -85,7 +85,7 @@ class XFileSystemProviderTest extends Specification { def GERMAN = new Locale.Builder().setLanguage("de").setRegion("DE").build() Locale.setDefault(Locale.Category.FORMAT, GERMAN) def fs = new HttpFileSystemProvider() - def attrMap = ['Last-Modified': ['Fri, 04 Nov 2016 21:50:34 GMT'], 'Content-Length': ['21729']] + def attrMap = ['last-modified': ['Fri, 04 Nov 2016 21:50:34 GMT'], 'content-length': ['21729']] when: def attrs = fs.readHttpAttributes(attrMap) From 4fbcb8a0e4a39e4cd65667039058285341eebca2 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 20 Jul 2023 22:19:18 +0200 Subject: [PATCH 2/2] Use case insensitive map [ci fast] Signed-off-by: Paolo Di Tommaso --- .../main/nextflow/util/InsensitiveMap.groovy | 50 ++++++++++++++++ .../nextflow/util/InsensitiveMapTest.groovy | 60 +++++++++++++++++++ .../file/http/XFileSystemProvider.groovy | 21 +++---- 3 files changed, 117 insertions(+), 14 deletions(-) create mode 100644 modules/nf-commons/src/main/nextflow/util/InsensitiveMap.groovy create mode 100644 modules/nf-commons/src/test/nextflow/util/InsensitiveMapTest.groovy diff --git a/modules/nf-commons/src/main/nextflow/util/InsensitiveMap.groovy b/modules/nf-commons/src/main/nextflow/util/InsensitiveMap.groovy new file mode 100644 index 0000000000..0be4ef9059 --- /dev/null +++ b/modules/nf-commons/src/main/nextflow/util/InsensitiveMap.groovy @@ -0,0 +1,50 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.util + +import groovy.transform.CompileStatic + +/** + * A {@link Map} that handles keys in a case insensitive manner + * + * @author Paolo Di Tommaso + */ +@CompileStatic +class InsensitiveMap implements Map { + + @Delegate + private Map target + + private InsensitiveMap(Map map) { + this.target = map + } + + @Override + boolean containsKey(Object key) { + target.any( it -> key?.toString()?.toLowerCase() == it.key?.toString()?.toLowerCase()) + } + + @Override + V get(Object key) { + target.find(it -> key?.toString()?.toLowerCase() == it.key?.toString()?.toLowerCase())?.value + } + + static Map of(Map target) { + new InsensitiveMap(target) + } +} diff --git a/modules/nf-commons/src/test/nextflow/util/InsensitiveMapTest.groovy b/modules/nf-commons/src/test/nextflow/util/InsensitiveMapTest.groovy new file mode 100644 index 0000000000..99a3bb394c --- /dev/null +++ b/modules/nf-commons/src/test/nextflow/util/InsensitiveMapTest.groovy @@ -0,0 +1,60 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.util + +import spock.lang.Specification + +/** + * + * @author Paolo Di Tommaso + */ +class InsensitiveMapTest extends Specification { + + def 'should get value by case insensitive keys' () { + given: + def map = InsensitiveMap.of([alpha: 1, BETA: 2]) + + expect: + map.alpha == 1 + map.ALPHA == 1 + map.Alpha == 1 + and: + map.get('alpha') == 1 + map.get('ALPHA') == 1 + map.get('Alpha') == 1 + and: + map.beta == 2 + map.BETA == 2 + map.Beta == 2 + and: + map.get('beta') == 2 + map.get('BETA') == 2 + map.get('Beta') == 2 + and: + map.foo == null + and: + map.containsKey('alpha') + map.containsKey('ALPHA') + and: + map.containsKey('beta') + map.containsKey('BETA') + and: + !map.containsKey('foo') + } + +} diff --git a/modules/nf-httpfs/src/main/nextflow/file/http/XFileSystemProvider.groovy b/modules/nf-httpfs/src/main/nextflow/file/http/XFileSystemProvider.groovy index 328cef8d00..1fa43da22f 100644 --- a/modules/nf-httpfs/src/main/nextflow/file/http/XFileSystemProvider.groovy +++ b/modules/nf-httpfs/src/main/nextflow/file/http/XFileSystemProvider.groovy @@ -44,6 +44,7 @@ import groovy.transform.PackageScope import groovy.util.logging.Slf4j import nextflow.SysEnv import nextflow.extension.FilesEx +import nextflow.util.InsensitiveMap import sun.net.www.protocol.ftp.FtpURLConnection import static XFileSystemConfig.* @@ -191,8 +192,8 @@ abstract class XFileSystemProvider extends FileSystemProvider { XAuthRegistry.instance.authorize(conn) } if ( conn instanceof HttpURLConnection && conn.getResponseCode() in [307, 308] && attempt < MAX_REDIRECT_HOPS ) { - def header = getNormalizedHeaders(conn) - String location = header.get("location")?.get(0) + final header = InsensitiveMap.of(conn.getHeaderFields()) + String location = header.get("Location")?.get(0) URL newPath = new URI(location).toURL() log.debug "Remote redirect URL: $newPath" return toConnection0(newPath, attempt+1) @@ -454,27 +455,19 @@ abstract class XFileSystemProvider extends FileSystemProvider { return new XFileAttributes(null,-1) } if ( conn instanceof HttpURLConnection && conn.getResponseCode() in [200, 301, 302, 307, 308]) { - def header = getNormalizedHeaders(conn) + final header = conn.getHeaderFields() return readHttpAttributes(header) } return null } protected XFileAttributes readHttpAttributes(Map> header) { - def lastMod = header.get("last-modified")?.get(0) - long contentLen = header.get("content-length")?.get(0)?.toLong() ?: -1 + final header0 = InsensitiveMap.>of(header) + def lastMod = header0.get("Last-Modified")?.get(0) + long contentLen = header0.get("Content-Length")?.get(0)?.toLong() ?: -1 def dateFormat = new SimpleDateFormat('E, dd MMM yyyy HH:mm:ss Z', Locale.ENGLISH) // <-- make sure date parse is not language dependent (for the week day) def modTime = lastMod ? FileTime.from(dateFormat.parse(lastMod).time, TimeUnit.MILLISECONDS) : (FileTime)null new XFileAttributes(modTime, contentLen) } - static protected Map> getNormalizedHeaders(URLConnection conn) { - final header = conn.getHeaderFields() - header.inject([:]) { accum, key, value -> - final normalized = key != null ? key.toLowerCase() : null - accum[normalized] = value - accum - } - } - }