Skip to content

Commit

Permalink
🚧 Ongoing work on issue #3
Browse files Browse the repository at this point in the history
  • Loading branch information
@pmonks
pmonks committed Sep 4, 2023
1 parent fce6fa2 commit 8885937
Show file tree
Hide file tree
Showing 15 changed files with 672 additions and 361 deletions.
1 change: 1 addition & 0 deletions deps.edn
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
clj-xml-validation/clj-xml-validation {:mvn/version "1.0.2"}
tolitius/xml-in {:mvn/version "0.1.1"}
hato/hato {:mvn/version "0.9.0"}
dev.weavejester/medley {:mvn/version "1.7.0"}
miikka/clj-base62 {:mvn/version "0.1.1"}
com.github.pmonks/clj-spdx {:mvn/version "1.0.91"}
com.github.pmonks/rencg {:mvn/version "1.0.34"}}
Expand Down
38 changes: 24 additions & 14 deletions src/lice_comb/deps.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,8 @@
(when ga
[(symbol (first (s/split (str ga) #"\$"))) info]))

(defmulti dep->expressions
"Attempt to detect the SPDX license expression(s) (a set) in a tools.deps
(defmulti dep->expressions-info
"Attempt to detect the SPDX license expression(s) (a map) in a tools.deps
style dep (a MapEntry or two-element sequence of
`[groupId/artifactId dep-info]`).
Expand All @@ -63,7 +63,7 @@
{:arglists '([[ga info]])}
(fn [[_ info]] (:deps/manifest info)))

(defmethod dep->expressions :mvn
(defmethod dep->expressions-info :mvn
[dep]
(when dep
(let [[ga info] (normalise-dep dep)
Expand All @@ -73,38 +73,48 @@
; override
(let [pom-uri (lcmvn/pom-uri-for-gav group-id artifact-id version)
expressions ;(check-fallbacks ga
(if-let [expressions (lcmvn/pom->expressions pom-uri)]
(if-let [expressions (lcmvn/pom->expressions-info pom-uri)]
expressions
(apply lcimd/union (mapcat lcf/zip->expressions (:paths info))))];)] ; If we didn't find any licenses in the dep's POM, check the dep's JAR(s) too
(into {} (pmap #(lcimd/prepend-source (lcf/zip->expressions-info %) dep) (:paths info))))];)] ; If we didn't find any licenses in the dep's POM, check the dep's JAR(s) too
expressions))));)

(defmethod dep->expressions :deps
(defmethod dep->expressions-info :deps
[dep]
(when dep
(let [[ga info] (normalise-dep dep)
version (:git/sha info)]
; (if-let [override (check-overrides ga version)]
; override
; (check-fallbacks ga
(lcf/dir->expressions (:deps/root info)))));))
; (check-fallbacks ga
(lcf/dir->expressions-info (:deps/root info)))));))

(defmethod dep->expressions nil
(defmethod dep->expressions-info nil
[_])

(defmethod dep->expressions :default
(defmethod dep->expressions-info :default
[dep]
(throw (ex-info (str "Unexpected manifest type '" (:deps/manifest (second dep)) "' for dependency " dep)
{:dep dep})))

(defn dep->expressions
"Attempt to detect the SPDX license expression(s) (a set) in a tools.deps
style dep (a MapEntry or two-element sequence of
`[groupId/artifactId dep-info]`).
The result has metadata attached that describes how the identifiers in the
expression(s) were determined."
[dep]
(some-> (dep->expressions-info dep)
keys
set))

(defn deps-expressions
"Attempt to detect the SPDX license expression(s) in a tools.deps 'lib map',
returning a new lib map with the licenses assoc'ed in (in key
`:lice-comb/license-expressions`)"
`:lice-comb/license-info`)"
[deps]
(when deps
(into {}
;####TODO: CHECK WHETHER METADATA MAPS NEED TO BE MERGED!!!!
(pmap #(let [[k v] %] [k (assoc v :lice-comb/license-expressions (dep->expressions [k v]))]) deps))))
(into {} (pmap #(let [[k v] %] [k (assoc v :lice-comb/license-info (dep->expressions-info [k v]))]) deps))))

(defn init!
"Initialises this namespace upon first call (and does nothing on subsequent
Expand Down
103 changes: 76 additions & 27 deletions src/lice_comb/files.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
(throw (java.nio.file.NotDirectoryException. (str dir))))
(throw (java.io.FileNotFoundException. (str dir)))))))

(defn probable-license-file?
(defn- probable-license-file?
"Returns true if the given file-like thing (String, File, ZipEntry) is a
probable license file, false otherwise."
[f]
Expand All @@ -50,35 +50,52 @@
(or (contains? probable-license-filenames fname)
(s/ends-with? fname ".pom"))))))

(defn probable-license-files
(defn- probable-license-files
"Returns all probable license files in the given directory, recursively, as a
set of java.io.File objects. dir may be a String or a java.io.File, either of
which must refer to a readable directory."
[dir]
(when-let [dir (ensure-readable-dir dir)]
(lcu/nset (filter #(and (.isFile ^java.io.File %) (probable-license-file? %)) (file-seq dir)))))
(some-> (seq (filter #(and (.isFile ^java.io.File %) (probable-license-file? %)) (file-seq dir)))
set)))

(defn file->expressions
"Attempts to determine the SPDX license expression(s) (a set) from the given
(defn file->expressions-info
"Attempts to determine the SPDX license expression(s) (a map) from the given
file (an InputStream or something that can have an io/input-stream opened on
it). If an InputStream is provided, it must already be open and the associated
filename should also be provided as the second parameter (it is optional in
filepath should also be provided as the second parameter (it is optional in
other cases).
The result has metadata attached that describes how the identifiers in the
expression(s) were determined."
([f] (file->expressions f (lcu/filename f)))
([f fname]
(when (and f fname)
(let [lfname (s/lower-case fname)]
(lcimd/prepend-source (cond (= lfname "pom.xml") (lcmvn/pom->expressions f fname)
(s/ends-with? lfname ".pom") (lcmvn/pom->expressions f fname)
([f] (file->expressions-info f (lcu/filepath f)))
([f filepath]
(when (and f (not (s/blank? filepath)))
(let [fname (lcu/filename filepath)
lfname (s/lower-case fname)]
(lcimd/prepend-source (cond (= lfname "pom.xml") (lcmvn/pom->expressions-info f fname)
(s/ends-with? lfname ".pom") (lcmvn/pom->expressions-info f fname)
(instance? java.io.InputStream f) (lcmtch/text->ids f)
:else (with-open [is (io/input-stream f)] (doall (lcmtch/text->ids is)))) ; Default is to assume it's a plain text file containing license text(s)
fname)))))
filepath)))))

(defn zip->expressions
"Attempt to detect the SPDX license expression(s) in a ZIP file. zip may be a
(defn file->expressions
"Attempts to determine the SPDX license expression(s) (a set) from the given
file (an InputStream or something that can have an io/input-stream opened on
it). If an InputStream is provided, it must already be open and the associated
filepath should also be provided as the second parameter (it is optional in
other cases).
The result has metadata attached that describes how the identifiers in the
expression(s) were determined."
([f] (file->expressions f (lcu/filepath f)))
([f filepath]
(some-> (file->expressions-info f filepath)
keys
set)))

(defn zip->expressions-info
"Attempt to detect the SPDX license expression(s) (a map) in a ZIP file. zip may be a
String or a java.io.File, both of which must refer to a ZIP-format compressed
file.
Expand All @@ -91,27 +108,42 @@
(let [zip-file (io/file zip)]
(java.util.zip.ZipFile. zip-file) ; This no-op forces validation of the zip file - ZipInputStream does not reliably perform validation
(with-open [zip-is (java.util.zip.ZipInputStream. (io/input-stream zip-file))]
(loop [result #{}
(loop [result {}
entry (.getNextEntry zip-is)]
(if entry
(if (probable-license-file? entry)
(recur (lcimd/union result (lcimd/prepend-source (file->expressions zip-is (lcu/filename entry)) (lcu/filename zip-file)))
(recur (merge result (lcimd/prepend-source (file->expressions-info zip-is (lcu/filename entry)) (lcu/filepath zip-file)))
(.getNextEntry zip-is))
(recur result (.getNextEntry zip-is)))
(doall (some-> (seq result) set)))))))) ; De-lazy the result before we exit the with-open scope
(when-not (empty? result) result)))))))

(defn zip->expressions
"Attempt to detect the SPDX license expression(s) (a set) in a ZIP file. zip may be a
String or a java.io.File, both of which must refer to a ZIP-format compressed
file.
Throws on invalid zip file (doesn't exist, not readable, not ZIP format, etc.).
The result has metadata attached that describes how the identifiers in the
expression(s) were determined."
[zip]
(some-> (zip->expressions-info zip)
keys
set))

(defn- zip-compressed-files
"Returns all probable ZIP compressed files in the given directory,
recursively, as a set of java.io.File objects. dir may be a String or a
java.io.File, either of which must refer to a readable directory."
[dir]
(when-let [dir (ensure-readable-dir dir)]
(lcu/nset (filter #(and (.isFile ^java.io.File %)
(or (s/ends-with? (str %) ".zip")
(s/ends-with? (str %) ".jar")))
(file-seq dir)))))
(some-> (seq (filter #(and (.isFile ^java.io.File %)
(or (s/ends-with? (str %) ".zip")
(s/ends-with? (str %) ".jar")))
(file-seq dir)))
set)))

(defn dir->expressions
(defn dir->expressions-info
"Attempt to detect the SPDX license expression(s) (a set) in a directory. dir
may be a String or a java.io.File, both of which must refer to a
readable directory.
Expand All @@ -122,15 +154,32 @@
The result has metadata attached that describes how the identifiers in the
expression(s) were determined."
([dir] (dir->expressions dir nil))
([dir] (dir->expressions-info dir nil))
([dir {:keys [include-zips?] :or {include-zips? false}}]
(when dir
(let [file-expressions (apply lcimd/union (map file->expressions (probable-license-files dir)))]
(let [file-expressions (into {} (map file->expressions-info (probable-license-files dir)))]
(if include-zips?
(let [zip-expressions (apply lcimd/union (map #(try (zip->expressions %) (catch Exception _ nil)) (zip-compressed-files dir)))]
(lcimd/union file-expressions zip-expressions))
(let [zip-expressions (into {} (map #(try (zip->expressions-info %) (catch Exception _ nil)) (zip-compressed-files dir)))]
(merge file-expressions zip-expressions))
file-expressions)))))

(defn dir->expressions
"Attempt to detect the SPDX license expression(s) (a map) in a directory. dir
may be a String or a java.io.File, both of which must refer to a
readable directory.
The optional `opts` map has these keys:
* `include-zips?` (boolean, default false) - controls whether zip compressed
files found in the directory are included in the scan or not
The result has metadata attached that describes how the identifiers in the
expression(s) were determined."
([dir] (dir->expressions dir nil))
([dir opts]
(some-> (dir->expressions-info dir opts)
keys
set)))

(defn init!
"Initialises this namespace upon first call (and does nothing on subsequent
calls), returning nil. Consumers of this namespace are not required to call
Expand Down
17 changes: 9 additions & 8 deletions src/lice_comb/impl/http.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,15 @@
Note: does not throw - returns false on errors."
[uri]
(when (lcu/valid-http-uri? (str uri))
(try
(when-let [response (hc/head (str uri)
{:http-client @http-client-d
:header {"user agent" "com.github.pmonks/lice-comb"}})]
(= 200 (:status response)))
(catch Exception _
false))))
(boolean
(when (lcu/valid-http-uri? (str uri))
(try
(when-let [response (hc/head (str uri)
{:http-client @http-client-d
:header {"user agent" "com.github.pmonks/lice-comb"}})]
(= 200 (:status response)))
(catch Exception _
false)))))

(defn- cdn-uri
"Converts raw URIs into CDN URIs, for these 'known' hosts:
Expand Down
Loading

0 comments on commit 8885937

Please sign in to comment.