From c022585588321e8959c1451cb003f6bd91675fd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sat, 3 Mar 2018 22:40:10 +0000 Subject: [PATCH 1/2] Add MIME registry --- spec/std/data/mime.types | 1 + spec/std/mime_spec.cr | 176 +++++++++++++++++++ src/crystal/system/mime.cr | 12 ++ src/crystal/system/unix/mime.cr | 24 +++ src/crystal/system/win32/mime.cr | 8 + src/mime.cr | 287 +++++++++++++++++++++++++++++++ 6 files changed, 508 insertions(+) create mode 100644 spec/std/data/mime.types create mode 100644 spec/std/mime_spec.cr create mode 100644 src/crystal/system/mime.cr create mode 100644 src/crystal/system/unix/mime.cr create mode 100644 src/crystal/system/win32/mime.cr create mode 100644 src/mime.cr diff --git a/spec/std/data/mime.types b/spec/std/data/mime.types new file mode 100644 index 000000000000..42da870f2600 --- /dev/null +++ b/spec/std/data/mime.types @@ -0,0 +1 @@ +foo/bar foo diff --git a/spec/std/mime_spec.cr b/spec/std/mime_spec.cr new file mode 100644 index 000000000000..9a57ab434851 --- /dev/null +++ b/spec/std/mime_spec.cr @@ -0,0 +1,176 @@ +require "./spec_helper" +require "mime" + +module MIME + def self.initialized + @@initialized + end + + def self.reset + @@initialized = false + @@types = {} of String => String + @@types_lower = {} of String => String + @@extensions = {} of String => Set(String) + end +end + +describe MIME do + it ".from_extension" do + MIME.from_extension(".html").partition(';')[0].should eq "text/html" + MIME.from_extension(".HTML").partition(';')[0].should eq "text/html" + + expect_raises KeyError do + MIME.from_extension(".fooobar") + end + MIME.from_extension(".fooobar", "default/fooobar").should eq "default/fooobar" + MIME.from_extension(".fooobar") { "default/fooobar" }.should eq "default/fooobar" + end + + it ".from_extension?" do + MIME.from_extension?(".html").should eq MIME.from_extension(".html") + MIME.from_extension?(".HTML").should eq MIME.from_extension(".HTML") + + MIME.from_extension?(".fooobar").should be_nil + end + + it ".from_filename" do + MIME.from_filename("test.html").should eq MIME.from_extension(".html") + MIME.from_filename("foo/bar.not-exists", "foo/bar-exist").should eq "foo/bar-exist" + MIME.from_filename("foo/bar.not-exists") { "foo/bar-exist" }.should eq "foo/bar-exist" + end + + it ".from_filename" do + MIME.from_filename?("test.html").should eq MIME.from_extension(".html") + end + + describe ".register" do + it "registers new type" do + MIME.register(".Custom-Type", "text/custom-type") + + MIME.from_extension(".Custom-Type").should eq "text/custom-type" + MIME.from_extension(".custom-type").should eq "text/custom-type" + MIME.extensions("text/custom-type").should eq Set{".Custom-Type"} + + MIME.register(".custom-type2", "text/custom-type") + MIME.extensions("text/custom-type").should eq Set{".Custom-Type", ".custom-type2"} + + MIME.register(".custom-type", "text/custom-type-lower") + MIME.from_extension(".custom-type").should eq "text/custom-type-lower" + MIME.from_extension(".Custom-Type").should eq "text/custom-type" + end + + it "fails for invalid extension" do + expect_raises ArgumentError, "Extension does not start with a dot" do + MIME.register("foo", "text/foo") + end + + expect_raises ArgumentError, "String contains null byte" do + MIME.register(".foo\0", "text/foo") + end + end + end + + describe ".extensions" do + it "lists extensions" do + MIME.extensions("text/html").should contain ".htm" + MIME.extensions("text/html").should contain ".html" + end + + it "returns empty set" do + MIME.extensions("foo/bar").should eq Set(String).new + end + + it "recognizes overridden types" do + MIME.register(".custom-type-overridden", "text/custom-type-overridden") + MIME.register(".custom-type-overridden", "text/custom-type-override") + + MIME.extensions("text/custom-type-overridden").should eq Set(String).new + end + end + + it "parses media types" do + MIME.register(".parse-media-type1", "text/html; charset=utf-8") + MIME.extensions("text/html").should contain (".parse-media-type1") + + MIME.register(".parse-media-type2", "text/html; foo = bar; bar= foo ;") + MIME.extensions("text/html").should contain (".parse-media-type2") + + MIME.register(".parse-media-type3", "foo/bar") + MIME.extensions("foo/bar").should contain (".parse-media-type3") + + MIME.register(".parse-media-type4", " form-data ; name=foo") + MIME.extensions("form-data").should contain (".parse-media-type4") + + MIME.register(".parse-media-type41", %(FORM-DATA;name="foo")) + MIME.extensions("form-data").should contain (".parse-media-type41") + + MIME.register(".parse-media-type5", %( FORM-DATA ; name="foo")) + MIME.extensions("form-data").should contain (".parse-media-type5") + + expect_raises ArgumentError, "Invalid media type" do + MIME.register(".parse-media-type6", ": inline; attachment; filename=foo.html") + end + + expect_raises ArgumentError, "Invalid media type" do + MIME.register(".parse-media-type7", "filename=foo.html, filename=bar.html") + end + + expect_raises ArgumentError, "Invalid media type" do + MIME.register(".parse-media-type8", %("foo; filename=bar;baz"; filename=qux)) + end + + expect_raises ArgumentError, "Invalid media type" do + MIME.register(".parse-media-type9", "x=y; filename=foo.html") + end + + expect_raises ArgumentError, "Invalid media type" do + MIME.register(".parse-media-type10", "filename=foo.html") + end + end + + it ".load_mime_database" do + MIME.from_extension?(".bar").should be_nil + MIME.from_extension?(".fbaz").should be_nil + + MIME.load_mime_database IO::Memory.new <<-EOF + foo/bar bar + foo/baz baz fbaz #foobaz + # foo/foo foo + EOF + + MIME.from_extension?(".bar").should eq "foo/bar" + MIME.from_extension?(".fbaz").should eq "foo/baz" + MIME.from_extension?(".#foobaz").should be_nil + MIME.from_extension?(".foobaz").should be_nil + MIME.from_extension?(".foo").should be_nil + end + + describe ".init" do + it "loads defaults" do + MIME.reset + MIME.init + MIME.initialized.should be_true + MIME.from_extension(".html").partition(';')[0].should eq "text/html" + ensure + MIME.reset + end + + it "skips loading defaults" do + MIME.reset + MIME.init(load_defaults: false) + MIME.initialized.should be_true + MIME.from_extension?(".html").should be_nil + ensure + MIME.reset + end + + it "loads file" do + MIME.reset + MIME.initialized.should be_false + MIME.init(datapath("mime.types")) + MIME.from_extension?(".foo").should eq "foo/bar" + ensure + MIME.reset + end + end +end diff --git a/src/crystal/system/mime.cr b/src/crystal/system/mime.cr new file mode 100644 index 000000000000..9b456daa291c --- /dev/null +++ b/src/crystal/system/mime.cr @@ -0,0 +1,12 @@ +module Crystal::System::MIME + # Load MIME types from operating system source. + # def self.load +end + +{% if flag?(:unix) %} + require "./unix/mime" +{% elsif flag?(:win32) %} + require "./win32/mime" +{% else %} + {% raise "No Crystal::System::Mime implementation available" %} +{% end %} diff --git a/src/crystal/system/unix/mime.cr b/src/crystal/system/unix/mime.cr new file mode 100644 index 000000000000..53d30a91ecf5 --- /dev/null +++ b/src/crystal/system/unix/mime.cr @@ -0,0 +1,24 @@ +module Crystal::System::MIME + MIME_SOURCES = { + "/etc/mime.types", # Linux + "/etc/httpd/mime.types", # Apache on Mac OS X + "/usr/local/etc/mime.types", # FreeBSD + "/usr/share/misc/mime.types", # OpenBSD + "/etc/httpd/conf/mime.types", # Apache + "/etc/apache/mime.types", # Apache 1 + "/etc/apache2/mime.types", # Apache 2 + "/usr/local/lib/netscape/mime.types", # Netscape + "/usr/local/etc/httpd/conf/mime.types", # Apache 1.2 + } + + # Load MIME types from operating system source. + def self.load + MIME_SOURCES.each do |path| + next unless ::File.exists?(path) + ::File.open(path) do |file| + ::MIME.load_mime_database file + end + rescue + end + end +end diff --git a/src/crystal/system/win32/mime.cr b/src/crystal/system/win32/mime.cr new file mode 100644 index 000000000000..ac81f499044f --- /dev/null +++ b/src/crystal/system/win32/mime.cr @@ -0,0 +1,8 @@ +module Crystal::System::MIME + # Load MIME types from operating system source. + def self.load + # TODO: MIME types in Windows are provided by the registry. This needs to be + # implemented when registry access it is available. + # Until then, there will no system-provided MIME types in Windows. + end +end diff --git a/src/mime.cr b/src/mime.cr new file mode 100644 index 000000000000..9a369cde2f6b --- /dev/null +++ b/src/mime.cr @@ -0,0 +1,287 @@ +require "crystal/system/mime" + +# This module implements a global MIME registry. +# +# ``` +# require "mime" +# +# MIME.from_extension(".html") # => "text/html; charset=utf-8" +# MIME.from_filename("path/file.html") # => "text/html; charset=utf-8" +# ``` +# +# The registry will be populated with some default values (see `DEFAULT_TYPES`) +# as well as the operating system's MIME database. +# +# Default initialization can be skipped by calling `MIME.init(false)` before the first +# query to the MIME database. +# +# ## OS-provided MIME database +# +# On a POSIX system, the following files are tried to be read in sequential order, +# stopping at the first existing file. These values override those from `DEFAULT_TYPES`. +# +# ```plain +# /etc/mime.types +# /etc/httpd/mime.types # Mac OS X +# /etc/httpd/conf/mime.types # Apache +# /etc/apache/mime.types # Apache 1 +# /etc/apache2/mime.types # Apache 2 +# /usr/local/etc/httpd/conf/mime.types +# /usr/local/lib/netscape/mime.types +# /usr/local/etc/httpd/conf/mime.types # Apache 1.2 +# /usr/local/etc/mime.types # FreeBSD +# /usr/share/misc/mime.types # OpenBSD +# ``` +# +# ## Registering custom MIME types +# +# Applications can register their own MIME types: +# +# ``` +# require "mime" +# MIME.from_extension?(".cr") # => nil +# MIME.extensions("text/crystal") # => Set(String).new +# +# MIME.register(".cr", "text/crystal") +# MIME.from_extension?(".cr") # => "text/crystal" +# MIME.extensions("text/crystal") # => Set(String){".cr"} +# ``` +# +# ## Loading a custom MIME database +# +# To load a custom MIME database, `load_mime_database` can be called with an +# `IO` to read the database from. +# +# ``` +# # Load user-defined MIME types +# File.open("~/.mime.types") do |io| +# MIME.load_mime_database(io) +# end +# ``` +# +# Loaded values override previously defined mappings. +# +# The data format must follow the format of `mime.types`: Each line declares +# a MIME type followed by a whitespace-separated list of extensions mapped to +# this type. Everything following a `#` is considered a comment until the end of +# line. Empy line are ignored. +# +# ```plain +# text/html html htm +# +# # comment +# ``` +module MIME + class Error < Exception + end + + @@initialized = false + @@types = {} of String => String + @@types_lower = {} of String => String + @@extensions = {} of String => Set(String) + + # A limited set of default MIME types. + DEFAULT_TYPES = { + ".css" => "text/css; charset=utf-8", + ".gif" => "image/gif", + ".htm" => "text/html; charset=utf-8", + ".html" => "text/html; charset=utf-8", + ".jpg" => "image/jpeg", + ".jpeg" => "image/jpeg", + ".js" => "application/javascript; charset=utf-8", + ".pdf" => "application/pdf", + ".png" => "image/png", + ".svg" => "image/svg+xml", + ".txt" => "text/plain; charset=utf-8", + ".xml" => "text/xml; charset=utf-8", + } + + # Initializes the MIME database. + # + # The default behaviour is to load the internal defaults as well as the OS-provided + # MIME database. This can be disabled with *load_defaults* set to `false`. + # + # This method usually doesn't need to be called explicitly when the default behaviour is expected. + # It will be called implicitly with `load_defaults: true` when a query method + # is called and the MIME database has not been initialized before. + # + # Calling this method repeatedly is allowed. + def self.init(load_defaults : Bool = true) : Nil + @@initialized = true + + if load_defaults + DEFAULT_TYPES.each do |ext, type| + register ext, type + end + + Crystal::System::MIME.load + end + end + + # Initializes the MIME database loading contents from a file. + # + # This will neither load the internal defaults nor the OS-provided MIME database, + # only the database at *filename* (using `.load_mime_database`). + # + # Callig this method repeatedly is allowed. + def self.init(filename : String) : Nil + init(load_defaults: false) + + File.open(filename, "r") do |file| + load_mime_database(file) + end + end + + private def self.initialize_types + init unless @@initialized + end + + # Looks up the MIME type associated with *extension*. + # + # A case sensitive search is tried first, if this yields no result, it is + # matched case-insensitive. Returns *default* if *extension* is not registered. + def self.from_extension(extension : String, default) : String + from_extension(extension) { default } + end + + # Looks up the MIME type associated with *extension*. + # + # A case sensitive search is tried first, if this yields no result, it is + # matched case-insensitive. Raises `KeyError` if *extension* is not registered. + def self.from_extension(extension : String) : String + from_extension(extension) { raise KeyError.new("Missing MIME type for extension #{extension.inspect}") } + end + + # Looks up the MIME type associated with *extension*. + # + # A case sensitive search is tried first, if this yields no result, it is + # matched case-insensitive. Returns `nil` if *extension* is not registered. + def self.from_extension?(extension : String) : String? + from_extension(extension) { nil } + end + + # Looks up the MIME type associated with *extension*. + # + # A case sensitive search is tried first, if this yields no result, it is + # matched case-insensitive. Runs the fiven block if *extension* is not registered. + def self.from_extension(extension : String, &block) + initialize_types + + @@types.fetch(extension) { @@types_lower.fetch(extension.downcase) { yield extension } } + end + + # Looks up the MIME type associated with the extension in *filename*. + # + # A case sensitive search is tried first, if this yields no result, it is + # matched case-insensitive. Returns *default* if *extension* is not registered. + def self.from_filename(filename : String, default) : String + from_extension(File.extname(filename), default) + end + + # Looks up the MIME type associated with the extension in *filename*. + # + # A case sensitive search is tried first, if this yields no result, it is + # matched case-insensitive. Raises `KeyError` if extension is not registered. + def self.from_filename(filename : String) : String + from_extension(File.extname(filename)) + end + + # Looks up the MIME type associated with the extension in *filename*. + # + # A case sensitive search is tried first, if this yields no result, it is + # matched case-insensitive. Returns `nil` if extension is not registered. + def self.from_filename?(filename : String) : String? + from_extension?(File.extname(filename)) + end + + # Looks up the MIME type associated with the extension in *filename*. + # + # A case sensitive search is tried first, if this yields no result, it is + # matched case-insensitive. Runs the fiven block if extension is not registered. + def self.from_filename(filename : String, &block) + from_extension(File.extname(filename)) { |extension| yield extension } + end + + # Register *type* for *extension*. + # + # *extension* must start with a dot (`.`) and must not contain any null bytes. + def self.register(extension : String, type : String) : Nil + raise ArgumentError.new("Extension does not start with a dot: #{extension.inspect}") unless extension.starts_with?('.') + extension.check_no_null_byte + + initialize_types + + # If the same extension had a different type registered before, it needs to + # be removed from the extensions list. + if previous_type = @@types[extension]? + if extensions = @@extensions[parse_media_type(previous_type)]? + extensions.delete(extension) + end + end + + mediatype = parse_media_type(type) || raise ArgumentError.new "Invalid media type: #{type}" + + @@types[extension] = type + @@types_lower[extension.downcase] = type + + type_extensions = @@extensions[mediatype] ||= Set(String).new + type_extensions << extension + end + + # Returns all extensions registered for *type*. + def self.extensions(type : String) : Set(String) + initialize_types + + @@extensions.fetch(type) { Set(String).new } + end + + # tspecial as defined by RFC 1521 and RFC 2045 + private TSPECIAL_CHARACTERS = {'(', ')', '<', '>', '@', ',', ';', ':', '\\', '"', '/', '[', ']', '?', '='} + + private def self.parse_media_type(type : String) : String? + reader = Char::Reader.new(type) + + sub_type_start = -1 + while reader.has_next? + case char = reader.current_char + when ';' + break + when '/' + return nil if sub_type_start > -1 + sub_type_start = reader.pos + reader.next_char + else + if TSPECIAL_CHARACTERS.includes?(char) || 0x20 > char.ord > 0x7F + return nil + end + + reader.next_char + end + end + + if reader.pos == 0 + return nil + end + + type.byte_slice(0, reader.pos).strip.downcase + end + + # Reads MIME type mappings from an IO and registers the extension-to-type + # relation (see `.register`). + # + # The format follows that of `mime.types`: Each line is list of MIME type and + # zero or more extensions, separated by whitespace. + def self.load_mime_database(io : IO) : Nil + while line = io.gets + fields = line.split + + fields.each_with_index do |field, i| + extension = field + break if extension.starts_with?('#') + next if i == 0 # first index contains the media type + + register ".#{extension}", fields[0] + end + end + end +end From e4a128fe6707a21b3e37397315ce25df521d508c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20M=C3=BCller?= Date: Sat, 3 Mar 2018 22:40:40 +0000 Subject: [PATCH 2/2] Use MIME in HTTP::StaticFileHandler --- src/http/server/handlers/static_file_handler.cr | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/http/server/handlers/static_file_handler.cr b/src/http/server/handlers/static_file_handler.cr index 0ff33006eb15..08276768f798 100644 --- a/src/http/server/handlers/static_file_handler.cr +++ b/src/http/server/handlers/static_file_handler.cr @@ -1,6 +1,7 @@ require "ecr/macros" require "html" require "uri" +require "mime" # A simple handler that lists directories and serves files under a given public directory. class HTTP::StaticFileHandler @@ -72,7 +73,7 @@ class HTTP::StaticFileHandler return end - context.response.content_type = mime_type(file_path) + context.response.content_type = MIME.from_filename(file_path, "application/octet-stream") context.response.content_length = File.size(file_path) File.open(file_path) do |file| IO.copy(file, context.response) @@ -126,18 +127,6 @@ class HTTP::StaticFileHandler File.info(file_path).modification_time end - private def mime_type(path) - case File.extname(path) - when ".txt" then "text/plain" - when ".htm", ".html" then "text/html" - when ".css" then "text/css" - when ".js" then "application/javascript" - when ".svg" then "image/svg+xml" - when ".wasm" then "application/wasm" - else "application/octet-stream" - end - end - record DirectoryListing, request_path : String, path : String do @escaped_request_path : String?