Add Container and Block for Text

src/FastAI.jl

@@ -110,6 +110,8 @@ export Vision
 include("Tabular/Tabular.jl")
 @reexport using .Tabular
 
+include("Text/Text.jl")
+@reexport using .Text
 
 include("deprecations.jl")
 export
@@ -173,6 +175,7 @@ export
     TableRow,
     Continuous,
     Image,
+    TextBlock,
 
     # encodings
     encode,

src/Text/Text.jl

@@ -0,0 +1,35 @@
+module Text
+
+
+using ..FastAI
+using ..FastAI:
+    # blocks
+    Block, WrapperBlock, AbstractBlock, OneHotTensor, OneHotTensorMulti, Label,
+    LabelMulti, wrapped, Continuous, getencodings, getblocks, encodetarget, encodeinput,
+    # encodings
+    Encoding, StatefulEncoding, OneHot,
+    # visualization
+    ShowText,
+    # other
+    Context, Training, Validation, FASTAI_METHOD_REGISTRY, registerlearningtask!
+
+import Requires: @require
+
+include("recipes.jl")
+include("blocks/text.jl")
+include("transform.jl")
+
+function __init__()
+    _registerrecipes()
+    @require Makie="ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" begin
+        import .Makie
+        import .Makie: @recipe, @lift
+        import .FastAI: ShowMakie
+        include("makie.jl")
+    end
+end
+
+export TextBlock, TextFolders, replace_all_caps, replace_sentence_case,
+    convert_lowercase
+end
+

src/Text/blocks/text.jl

@@ -0,0 +1 @@
+struct TextBlock <: Block end

src/Text/makie.jl

@@ -0,0 +1 @@
+# # No Makie recipes yet, text is better I guess

src/Text/recipes.jl

@@ -0,0 +1,45 @@
+"""
+    TextFolders(textfile; labelfn = parentname, split = false)
+
+Recipe for loading a single-label text classification dataset
+stored in hierarchical folder format. 
+"""
+
+Base.@kwdef struct TextFolders <: Datasets.DatasetRecipe
+    labelfn = parentname
+    split::Bool = false
+    filefilterfn = _ -> true
+end
+
+Datasets.recipeblocks(::Type{TextFolders}) = Tuple{TextBlock, Label}
+
+function Datasets.loadrecipe(recipe::TextFolders, path)
+    isdir(path) || error("$path is not a directory")
+    data = loadfolderdata(
+        path,
+        filterfn=f -> istextfile(f) && recipe.filefilterfn(f),
+        loadfn=(loadfile, recipe.labelfn),
+        splitfn=recipe.split ? grandparentname : nothing)
+
+    (recipe.split ? length(data) > 0 : nobs(data) > 0) || error("No text files found in $path")
+
+    labels = recipe.split ? first(values(data))[2] : data[2]
+    blocks = TextBlock(), Label(unique(eachobs(labels)))
+    length(blocks[2].classes) > 1 || error("Expected multiple different labels, got: $(blocks[2].classes))")
+    return data, blocks
+end
+
+# Registering recipes
+
+const RECIPES = Dict{String,Vector{Datasets.DatasetRecipe}}(
+    "imdb" => [TextFolders(
+        filefilterfn = f->!contains(f, "tmp_clas") && !contains(f, "tmp_lm") && !contains(f, "unsup")
+        )],
+)
+
+function _registerrecipes()
+    for (name, recipes) in RECIPES, recipe in recipes
+        Datasets.registerrecipe!(Datasets.FASTAI_DATA_REGISTRY, name, recipe)
+    end
+end
+

src/Text/transform.jl

@@ -0,0 +1,25 @@
+"""
+    replace_all_caps(String)
+
+Replace tokens in ALL CAPS by their lower version and add xxup before.
+"""
+
+function replace_all_caps(t::String)
+    t = replace(t, r"([A-Z]+[^a-z\s]*)(?=(\s|$))" => s"xxup \1")
+    t = replace(t, r"([A-Z]*[^a-z\s]+)(?=(\s|$))" => lowercase)
+end
+
+"""
+    replace_sentence_case(String)
+
+Replace tokens in Sentence Case by their lower verions and add xxmaj before.
+"""
+
+function replace_sentence_case(t::String)
+    t = replace(t, r"(?<!\w)([A-Z][A-Z0-9]*[a-z0-9]+)(?!\w)" => s"xxmaj \1")
+    t = replace(t, r"(?<!\w)([A-Z][A-Z0-9]*[a-z0-9]+)(?!\w)" => lowercase)
+end
+
+function convert_lowercase(t::String)
+    string("xxbos ", lowercase(t))
+end

src/datasets/Datasets.jl

@@ -66,6 +66,7 @@ export
 
     # utilities
     isimagefile,
+    istextfile,
     matches,
     loadfile,
     loadmask,

src/datasets/containers.jl

@@ -38,6 +38,8 @@ function loadfile(file::String)
         return FileIO.load(file, view = true)
     elseif endswith(file, ".csv")
         return DataFrame(CSV.File(file))
+    elseif endswith(file, ".txt")
+        return String(read(file))
     else
         return FileIO.load(file)
     end

src/datasets/load.jl

@@ -45,6 +45,8 @@ matches(re::Regex) = f -> matches(re, f)
 matches(re::Regex, f) = !isnothing(match(re, f))
 const RE_IMAGEFILE = r".*\.(gif|jpe?g|tiff?|png|webp|bmp)$"i
 isimagefile(f) = matches(RE_IMAGEFILE, f)
+const RE_TEXTFILE = r".*\.(txt|csv|json|md|html?|xml|yaml|toml)$"i
+istextfile(f) = matches(RE_TEXTFILE, f)
 
 
 maskfromimage(a::AbstractArray{<:Gray{T}}, classes) where T = maskfromimage(reinterpret(T, a), classes)