LibGit2: consult known hosts files to verify SSH server identity (Jul…

…iaLang#38580)

* LibGit2: consult known hosts files to verify SSH server identity

* SSH host verification: improved error message wording

* qualify libssh2 ccalls

stdlib/LibGit2/Project.toml

@@ -2,8 +2,10 @@ name = "LibGit2"
 uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
 
 [deps]
+Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 NetworkOptions = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 
 [extras]
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

stdlib/LibGit2/src/LibGit2.jl

@@ -7,8 +7,10 @@ module LibGit2
 
 import Base: ==
 using Base: something, notnothing
+using Base64: base64decode
+using NetworkOptions
 using Printf: @printf
-import NetworkOptions
+using SHA: sha1, sha256
 
 export with, GitRepo, GitConfig
 

stdlib/LibGit2/src/callbacks.jl

@@ -359,19 +359,179 @@ function fetchhead_foreach_callback(ref_name::Cstring, remote_url::Cstring,
     return Cint(0)
 end
 
+struct CertHostKey
+    parent :: Cint
+    mask   :: Cint
+    md5    :: NTuple{16,UInt8}
+    sha1   :: NTuple{20,UInt8}
+    sha256 :: NTuple{32,UInt8}
+end
+
+struct KeyHashes
+    sha1   :: Union{NTuple{20,UInt8}, Nothing}
+    sha256 :: Union{NTuple{32,UInt8}, Nothing}
+end
+
+function KeyHashes(cert_p::Ptr{CertHostKey})
+    cert = unsafe_load(cert_p)
+    return KeyHashes(
+        cert.mask & Consts.CERT_SSH_SHA1   != 0 ? cert.sha1   : nothing,
+        cert.mask & Consts.CERT_SSH_SHA256 != 0 ? cert.sha256 : nothing,
+    )
+end
+
+function verify_host_error(message::AbstractString)
+    printstyled(stderr, "$message\n", color = :cyan, bold = true)
+end
+
 function certificate_callback(
-    cert_p :: Ptr{Cvoid},
+    cert_p :: Ptr{CertHostKey},
     valid  :: Cint,
     host_p :: Ptr{Cchar},
-    user_p :: Ptr{Cvoid},
+    data_p :: Ptr{Cvoid},
 )::Cint
     valid != 0 && return Consts.CERT_ACCEPT
     host = unsafe_string(host_p)
     cert_type = unsafe_load(convert(Ptr{Cint}, cert_p))
     transport = cert_type == Consts.CERT_TYPE_TLS ? "TLS" :
                 cert_type == Consts.CERT_TYPE_SSH ? "SSH" : nothing
-    verify = NetworkOptions.verify_host(host, transport)
-    verify ? Consts.PASSTHROUGH : Consts.CERT_ACCEPT
+    if !NetworkOptions.verify_host(host, transport)
+        # user has opted out of host verification
+        return Consts.CERT_ACCEPT
+    end
+    if transport == "TLS"
+        # TLS verification is done before the callback and indicated with the
+        # incoming `valid` flag, so if we get here then host verification failed
+        verify_host_error("TLS host verification: the identity of the server `$host` could not be verified. Someone could be trying to man-in-the-middle your connection. It is also possible that the correct server is using an invalid certificate or that your system's certificate authority root store is misconfigured.")
+        return Consts.CERT_REJECT
+    elseif transport == "SSH"
+        # SSH verification has to be done here
+        files = [joinpath(homedir(), ".ssh", "known_hosts")]
+        check = ssh_knownhost_check(files, host, KeyHashes(cert_p))
+        valid = false
+        if check == Consts.SSH_HOST_KNOWN
+            valid = true
+        elseif check == Consts.SSH_HOST_UNKNOWN
+            if Sys.which("ssh-keyscan") !== nothing
+                msg = "Please run `ssh-keyscan $host >> $(files[1])` in order to add the server to your known hosts file and the try again."
+            else
+                msg = "Please connect once using `ssh $host` in order to add the server to your known hosts file and then try again. You may not be allowed to log in (wrong user and/or no login allowed), but ssh will prompt you to add a host key for the server which will allow libgit2 to verify the server."
+            end
+            verify_host_error("SSH host verification: the server `$host` is not a known host. $msg")
+        elseif check == Consts.SSH_HOST_MISMATCH
+            verify_host_error("SSH host verification: the identity of the server `$host` does not match its known hosts record. Someone could be trying to man-in-the-middle your connection. It is also possible that the server has changed its key, in which case you should check with the server administrator and if they confirm that the key has been changed, update your known hosts file.")
+        elseif check == Consts.SSH_HOST_BAD_HASH
+            verify_host_error("SSH host verification: no secure certificate hash available for `$host`, cannot verify server identity.")
+        else
+            @error("unexpected SSH known host check result", check)
+        end
+        return valid ? Consts.CERT_ACCEPT : Consts.CERT_REJECT
+    end
+    @error("unexpected transport encountered, refusing to validate", cert_type)
+    return Consts.CERT_REJECT
+end
+
+## SSH known host checking
+#
+# We can't use libssh2_knownhost_check because libgit2, for no good reason,
+# doesn't give us a host fingerprint that we can use for that and instead gives
+# us multiple hashes of that fingerprint instead. Moreover, since a host can
+# have multiple fingerprints in the known hosts file with different encryption
+# types (gitlab.com does this, for example), we need to iterate through all the
+# known hosts entries and manually check if any of them is a match.
+#
+# The fact that libgit2 won't give us a fingerprint also means that we cannot,
+# even if we wanted to, prompt the user for whether to add the fingerprint to
+# the known hosts file, since we don't have the fingerprint that should be
+# added. The only option is to instruct the user how to add it themselves.
+#
+# Check logic: if a host appears in a known hosts file at all then one of the
+# keys in that file must match or we declare a mismatch; if the host name
+# doesn't appear in the file at all, however, we will continue searching files.
+#
+# This allows adding a host to the system known hosts file to fully override
+# that host appearing in a bundled known hosts file. It is necessary to allow
+# any of multiple entries in a single file to match, however, to allow for the
+# possiblity that the file contains multiple fingerprints for the same host. If
+# libgit2 gave us the fucking fingerprint then we could search for only an entry
+# with the correct type, but we can't do that without the actual fingerprint.
+
+struct KnownHost
+    magic :: Cuint
+    node  :: Ptr{Cvoid}
+    name  :: Ptr{Cchar}
+    key   :: Ptr{Cchar}
+    type  :: Cint
+end
+
+function ssh_knownhost_check(
+    files  :: AbstractVector{<:AbstractString},
+    host   :: AbstractString,
+    hashes :: KeyHashes,
+)
+    hashes.sha1 === hashes.sha256 === nothing &&
+        return Consts.SSH_HOST_BAD_HASH
+    session = @ccall "libssh2".libssh2_session_init_ex(
+        C_NULL :: Ptr{Cvoid},
+        C_NULL :: Ptr{Cvoid},
+        C_NULL :: Ptr{Cvoid},
+        C_NULL :: Ptr{Cvoid},
+    ) :: Ptr{Cvoid}
+    for file in files
+        ispath(file) || continue
+        hosts = @ccall "libssh2".libssh2_knownhost_init(
+            session :: Ptr{Cvoid},
+        ) :: Ptr{Cvoid}
+        count = @ccall "libssh2".libssh2_knownhost_readfile(
+            hosts :: Ptr{Cvoid},
+            file  :: Cstring,
+            1     :: Cint, # standard OpenSSH format
+        ) :: Cint
+        if count < 0
+            @warn("Error parsing SSH known hosts file `$file`")
+            @ccall "libssh2".libssh2_knownhost_free(hosts::Ptr{Cvoid})::Cvoid
+            continue
+        end
+        name_match = false
+        prev = Ptr{KnownHost}(0)
+        store = Ref{Ptr{KnownHost}}()
+        while true
+            get = @ccall "libssh2".libssh2_knownhost_get(
+                hosts :: Ptr{Cvoid},
+                store :: Ptr{Ptr{KnownHost}},
+                prev  :: Ptr{KnownHost},
+            ) :: Cint
+            get < 0 && @warn("Error searching SSH known hosts file `$file`")
+            get == 0 || break # end of file or error
+            # got a known hosts record for host, now check its key hash
+            prev = store[]
+            known_host = unsafe_load(prev)
+            known_host.name == C_NULL && continue
+            host == unsafe_string(known_host.name) || continue
+            name_match = true # we've found some entry in this file
+            key_match = true # all available hashes must match
+            key = base64decode(unsafe_string(known_host.key))
+            if hashes.sha1 !== nothing
+                key_match &= sha1(key) == collect(hashes.sha1)
+            end
+            if hashes.sha256 !== nothing
+                key_match &= sha256(key) == collect(hashes.sha256)
+            end
+            key_match || continue
+            # name and key match found
+            @ccall "libssh2".libssh2_knownhost_free(hosts::Ptr{Cvoid})::Cvoid
+            @assert 0 == @ccall "libssh2".libssh2_session_free(session::Ptr{Cvoid})::Cint
+            return Consts.SSH_HOST_KNOWN
+        end
+        @ccall "libssh2".libssh2_knownhost_free(hosts::Ptr{Cvoid})::Cvoid
+        name_match || continue # no name match, search more files
+        # name match but no key match => host mismatch
+        @assert 0 == @ccall "libssh2".libssh2_session_free(session::Ptr{Cvoid})::Cint
+        return Consts.SSH_HOST_MISMATCH
+    end
+    # name not found in any known hosts files
+    @assert 0 == @ccall "libssh2".libssh2_session_free(session::Ptr{Cvoid})::Cint
+    return Consts.SSH_HOST_UNKNOWN
 end
 
 "C function pointer for `mirror_callback`"
@@ -381,4 +541,4 @@ credentials_cb() = @cfunction(credentials_callback, Cint, (Ptr{Ptr{Cvoid}}, Cstr
 "C function pointer for `fetchhead_foreach_callback`"
 fetchhead_foreach_cb() = @cfunction(fetchhead_foreach_callback, Cint, (Cstring, Cstring, Ptr{GitHash}, Cuint, Any))
 "C function pointer for `certificate_callback`"
-certificate_cb() = @cfunction(certificate_callback, Cint, (Ptr{Cvoid}, Cint, Ptr{Cchar}, Ptr{Cvoid}))
+certificate_cb() = @cfunction(certificate_callback, Cint, (Ptr{CertHostKey}, Cint, Ptr{Cchar}, Ptr{Cvoid}))

stdlib/LibGit2/src/consts.jl

@@ -317,6 +317,25 @@ const PASSTHROUGH = -30
 const CERT_REJECT = -1
 const CERT_ACCEPT =  0
 
+# certificate hash flags
+const CERT_SSH_MD5    = 1 << 0
+const CERT_SSH_SHA1   = 1 << 1
+const CERT_SSH_SHA256 = 1 << 2
+
+# libssh2 known host constants
+const LIBSSH2_KNOWNHOST_TYPE_PLAIN  = 1
+const LIBSSH2_KNOWNHOST_TYPE_SHA1   = 2
+const LIBSSH2_KNOWNHOST_TYPE_CUSTOM = 3
+
+const LIBSSH2_KNOWNHOST_KEYENC_RAW    = 1 << 16
+const LIBSSH2_KNOWNHOST_KEYENC_BASE64 = 2 << 16
+
+# internal constants for SSH host verification outcomes
+const SSH_HOST_KNOWN    = 0
+const SSH_HOST_UNKNOWN  = 1
+const SSH_HOST_MISMATCH = 2
+const SSH_HOST_BAD_HASH = 3
+
 @enum(GIT_SUBMODULE_IGNORE, SUBMODULE_IGNORE_UNSPECIFIED  = -1, # use the submodule's configuration
                             SUBMODULE_IGNORE_NONE         = 1,  # any change or untracked == dirty
                             SUBMODULE_IGNORE_UNTRACKED    = 2,  # dirty if tracked files change

stdlib/LibGit2/test/known_hosts

@@ -0,0 +1,4 @@
+github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
+gitlab.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCsj2bNKTBSpIYDEGk9KxsGh3mySTRgMtXL583qmBpzeQ+jqCMRgBqB98u3z++J1sKlXHWfM9dyhSevkMwSbhoR8XIq/U0tCNyokEi/ueaBMCvbcTHhO7FcwzY92WK4Yt0aGROY5qX2UKSeOvuP4D6TPqKF1onrSzH9bx9XUf2lEdWT/ia1NEKjunUqu1xOB/StKDHMoX4/OKyIzuS0q/T1zOATthvasJFoPrAjkohTyaDUz2LN5JoH839hViyEG82yB+MjcFV5MU3N1l1QL3cVUCh93xSaua1N85qivl+siMkPGbO5xR/En4iEY6K2XPASUEMaieWVNTRCtJ4S8H+9
+gitlab.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFSMqzJeV9rUzU4kWitGjeR4PWSa29SPqJ1fVkhtj3Hw9xjLVXVYrU9QlYWrOLXBpQ6KWjbjTDTdDkoohFzgbEY=
+gitlab.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfuCHKVTjquxvt6CM6tdG4SLp1Btn/nOeHHE5UOzRdf

stdlib/LibGit2/test/libgit2.jl

@@ -2390,6 +2390,103 @@ mktempdir() do dir
             Base.shred!(valid_p_cred)
         end
 
+        @testset "SSH known host checking" begin
+            key_hashes(sha1::String, sha256::String) = LibGit2.KeyHashes(
+                Tuple(hex2bytes(sha1)),
+                Tuple(hex2bytes(sha256)),
+            )
+            # randomly generated hashes matching no hosts
+            random_key_hashes = key_hashes(
+                "a9971372d02a67bdfea82e2b4808b4cf478b49c0",
+                "45aac5c20d5c7f8b998fee12fa9b75086c0d3ed6e33063f7ce940409ff4efbbc"
+            )
+            # hashes of the unique github.com fingerprint
+            github_key_hashes = key_hashes(
+                "bf6b6825d2977c511a475bbefb88aad54a92ac73",
+                "9d385b83a9175292561a5ec4d4818e0aca51a264f17420112ef88ac3a139498f"
+            )
+            # hashes of the middle github.com fingerprint
+            gitlab_key_hashes = key_hashes(
+                "4db6b9ab0209fcde106cbf0fc4560ad063a962ad",
+                "1db5b783ccd48cd4a4b056ea4e25163d683606ad71f3174652b9625c5cd29d4c"
+            )
+
+            # various key hash collections
+            partial_hashes(keys::LibGit2.KeyHashes) = [ keys,
+                LibGit2.KeyHashes(keys.sha1, nothing),
+                LibGit2.KeyHashes(nothing, keys.sha256),
+            ]
+            bad_hashes = LibGit2.KeyHashes(nothing, nothing)
+            random_hashes = partial_hashes(random_key_hashes)
+            github_hashes = partial_hashes(github_key_hashes)
+            gitlab_hashes = partial_hashes(gitlab_key_hashes)
+
+            # various known hosts files
+            no_file = tempname()
+            empty_file = tempname(); touch(empty_file)
+            known_hosts = joinpath(@__DIR__, "known_hosts")
+            wrong_hosts = tempname()
+            open(wrong_hosts, write=true) do io
+                for line in eachline(known_hosts)
+                    words = split(line)
+                    words[1] = words[1] == "github.com" ? "gitlab.com" :
+                               words[1] == "gitlab.com" ? "github.com" :
+                               words[1]
+                    println(io, join(words, " "))
+                end
+            end
+
+            @testset "bad hash errors" begin
+                hash = bad_hashes
+                for host in ["github.com", "gitlab.com", "unknown.host"],
+                    files in [[no_file], [empty_file], [known_hosts]]
+                    check = LibGit2.ssh_knownhost_check(files, host, hash)
+                    @test check == LibGit2.Consts.SSH_HOST_BAD_HASH
+                end
+            end
+
+            @testset "unknown hosts" begin
+                host = "unknown.host"
+                for hash in [github_hashes; gitlab_hashes; random_hashes],
+                    files in [[no_file], [empty_file], [known_hosts]]
+                    check = LibGit2.ssh_knownhost_check(files, host, hash)
+                    @test check == LibGit2.Consts.SSH_HOST_UNKNOWN
+                end
+            end
+
+            @testset "known hosts" begin
+                for (host, hashes) in [
+                        "github.com" => github_hashes,
+                        "gitlab.com" => gitlab_hashes,
+                    ], hash in hashes
+                    for files in [[no_file], [empty_file]]
+                        check = LibGit2.ssh_knownhost_check(files, host, hash)
+                        @test check == LibGit2.Consts.SSH_HOST_UNKNOWN
+                    end
+                    for files in [
+                            [known_hosts],
+                            [empty_file; known_hosts],
+                            [known_hosts; empty_file],
+                            [known_hosts; wrong_hosts],
+                        ]
+                        check = LibGit2.ssh_knownhost_check(files, host, hash)
+                        @test check == LibGit2.Consts.SSH_HOST_KNOWN
+                    end
+                    for files in [
+                            [wrong_hosts],
+                            [empty_file; wrong_hosts],
+                            [wrong_hosts; empty_file],
+                            [wrong_hosts; known_hosts],
+                        ]
+                        check = LibGit2.ssh_knownhost_check(files, host, hash)
+                        @test check == LibGit2.Consts.SSH_HOST_MISMATCH
+                    end
+                end
+            end
+
+            rm(empty_file)
+        end
+
         @testset "HTTPS credential prompt" begin
             url = "https://github.com/test/package.jl"