From 3db4b304e985d7c4aab12aedbfe0e8739878dd29 Mon Sep 17 00:00:00 2001
From: Kevin Day <kday@library.tamu.edu>
Date: Wed, 11 Jan 2023 10:04:41 -0600
Subject: [PATCH 1/2] Issue 481: Use case insensitive filter and add case
 insensitive string type.

Cannot a filter to the `solr.StrField`.
According to the SOLR documentation, a filter can only be added to something tokenized and a `solr.StrField` does not allow tokenization.
This uses a `solr.TextField` instead.
Several fields need to have case insensitive searches.
A new type is added that uses the `KeywordTokenizer`, called `string_ci` and `strings_ci`.
The `KeywordTokenizer` essentialy is a pretend token.
It tokenizes the whole string, which is effectively the same as not having a tokenizer.
The documentation even references the `KeywordTokenizer` as the method of disabling the tokenizer.

Fields that should be case insensitive are moved from `string` to `string_ci` and `strings` to `strings_ci` respectively.

There are potential performance concerns with using `solr.TextField` rather than `solr.StrField` due to the loss of the docvalues optimization feature.

see: https://solr.apache.org/guide/7_7/field-types-included-with-solr.html#field-types-included-with-solr
see: https://solr.apache.org/guide/7_7/field-type-definitions-and-properties.html#field-type-definitions-and-properties
see: https://solr.apache.org/guide/7_7/field-properties-by-use-case.html#field-properties-by-use-case
see: https://solr.apache.org/guide/7_7/tokenizers.html#keyword-tokenizer
see: https://solr.apache.org/guide/7_7/docvalues.html
---
 solr/config/managed-schema | 51 ++++++++++++++++++++++++--------------
 1 file changed, 33 insertions(+), 18 deletions(-)

diff --git a/solr/config/managed-schema b/solr/config/managed-schema
index 0d723813..22776bdb 100644
--- a/solr/config/managed-schema
+++ b/solr/config/managed-schema
@@ -123,14 +123,14 @@
     <field name="thumbnail" type="string" indexed="true" stored="true" multiValued="false"/>
     <field name="resource" type="string" indexed="true" stored="true" multiValued="false"/>
     <field name="manifest" type="string" indexed="true" stored="true" multiValued="false"/>
-    <field name="application_type" type="string" indexed="true" stored="true" multiValued="false"/>
+    <field name="application_type" type="string_ci" indexed="true" stored="true" multiValued="false"/>
 
     <!-- Mandatory (M) -->
     <field name="title" type="sorting_string" indexed="true" stored="true" multiValued="false"/>
-    <field name="content_type" type="strings" indexed="true" stored="true" multiValued="true"/>
-    <field name="digital_publisher" type="strings" indexed="true" stored="true" multiValued="true"/>
-    <field name="rights_access" type="strings" indexed="true" stored="true" multiValued="true"/>
-    <field name="reformatting" type="strings" indexed="true" stored="true" multiValued="true"/>
+    <field name="content_type" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
+    <field name="digital_publisher" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
+    <field name="rights_access" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
+    <field name="reformatting" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
     <field name="filename" type="sorting_string" indexed="true" stored="true" multiValued="true"/>
 
     <copyField source="title" dest="_text_"/>
@@ -142,13 +142,13 @@
     <field name="creator" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
     <field name="date_published" type="pdate" indexed="true" stored="true" multiValued="false"/>
     <!--<field name="date_created" type="pdate" indexed="true" stored="true" multiValued="false"/>-->
-    <field name="date_created" type="string" indexed="true" stored="true" multiValued="false"/>
-    <field name="summary_abstract" type="strings" indexed="true" stored="true" multiValued="true"/>
+    <field name="date_created" type="strings_ci" indexed="true" stored="true" multiValued="false"/>
+    <field name="summary_abstract" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
     <field name="language" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
     <field name="institution_department" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
-    <field name="standard_digital_identifier" type="strings" indexed="true" stored="true" multiValued="true"/>
-    <field name="local_digital_identifier" type="strings" indexed="true" stored="true" multiValued="true"/>
-    <field name="edition_revision_information" type="strings" indexed="true" stored="true" multiValued="true"/>
+    <field name="standard_digital_identifier" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
+    <field name="local_digital_identifier" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
+    <field name="edition_revision_information" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
 
     <copyField source="subject" dest="_text_"/>
     <copyField source="creator" dest="_text_"/>
@@ -158,13 +158,13 @@
     <copyField source="summary_abstract" dest="summary_abstract_txt_en_split_mv"/>
 
     <!-- Non-core Recommended -->
-    <field name="alternative_title" type="strings" indexed="true" stored="true" multiValued="true"/>
+    <field name="alternative_title" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
     <field name="genre" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
-    <field name="table_of_contents" type="strings" indexed="true" stored="true" multiValued="true"/>
+    <field name="table_of_contents" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
     <field name="contributor" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
     <field name="related_resource" type="strings" indexed="true" stored="true" multiValued="true"/>
-    <field name="original_publisher" type="strings" indexed="true" stored="true" multiValued="true"/>
-    <field name="physical_extent" type="strings" indexed="true" stored="true" multiValued="true"/>
+    <field name="original_publisher" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
+    <field name="physical_extent" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
     <field name="sponsor" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
     <field name="details" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
     <field name="spatial" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
@@ -180,11 +180,11 @@
     <!-- Non-core Optional -->
     <field name="source_collection" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
     <field name="original_resource" type="strings" indexed="true" stored="true" multiValued="true"/>
-    <field name="notes" type="strings" indexed="true" stored="true" multiValued="true"/>
-    <field name="origin" type="strings" indexed="true" stored="true" multiValued="true"/>
-    <field name="audience_level" type="strings" indexed="true" stored="true" multiValued="true"/>
+    <field name="notes" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
+    <field name="origin" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
+    <field name="audience_level" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
     <field name="classification" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
-    <field name="physical_item_identifier" type="strings" indexed="true" stored="true" multiValued="true"/>
+    <field name="physical_item_identifier" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
     <field name="physical_item_location" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
 
     <copyField source="notes" dest="_text_"/>
@@ -248,6 +248,7 @@
     <fieldType name="whole_strings" class="solr.TextField" omitNorms="true" sortMissingLast="true" multiValued="true">
       <analyzer>
         <tokenizer class="solr.KeywordTokenizerFactory"/>
+        <filter class="solr.LowerCaseFilterFactory"/>
       </analyzer>
     </fieldType>
 
@@ -255,6 +256,20 @@
     <fieldType name="string" class="solr.StrField" sortMissingLast="true" docValues="true" />
     <fieldType name="strings" class="solr.StrField" sortMissingLast="true" multiValued="true" docValues="true" />
 
+    <!-- The TextField type can be analyzed and indexed/stored. -->
+    <fieldType name="string_ci" class="solr.TextField" sortMissingLast="true">
+      <analyzer>
+        <tokenizer class="solr.KeywordTokenizerFactory" />
+        <filter class="solr.LowerCaseFilterFactory" />
+      </analyzer>
+    </fieldType>
+    <fieldType name="strings_ci" class="solr.TextField" sortMissingLast="true" multiValued="true">
+      <analyzer>
+        <tokenizer class="solr.KeywordTokenizerFactory" />
+        <filter class="solr.LowerCaseFilterFactory" />
+      </analyzer>
+    </fieldType>
+
     <!-- boolean type: "true" or "false" -->
     <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
     <fieldType name="booleans" class="solr.BoolField" sortMissingLast="true" multiValued="true"/>

From 753b83a3880f538eb0b6eb32f91f1b9a5c937ff7 Mon Sep 17 00:00:00 2001
From: Kevin Day <kday@library.tamu.edu>
Date: Wed, 11 Jan 2023 14:14:14 -0600
Subject: [PATCH 2/2] Issue 481: Re-use whole_strings, rename string_ci to
 whole_string, fix date_created.

The `strings_ci` is close enough to `whole_strings`, just use `whole_strings`.
There is no `whole_string`.
Rename `string_ci` to `whole_string`.
To better prevent future problems, document these custom field types.

The date_created is not multi-valued so use `whole_string`.
---
 solr/config/managed-schema | 64 ++++++++++++++++++--------------------
 1 file changed, 31 insertions(+), 33 deletions(-)

diff --git a/solr/config/managed-schema b/solr/config/managed-schema
index 22776bdb..96cc6d02 100644
--- a/solr/config/managed-schema
+++ b/solr/config/managed-schema
@@ -123,14 +123,14 @@
     <field name="thumbnail" type="string" indexed="true" stored="true" multiValued="false"/>
     <field name="resource" type="string" indexed="true" stored="true" multiValued="false"/>
     <field name="manifest" type="string" indexed="true" stored="true" multiValued="false"/>
-    <field name="application_type" type="string_ci" indexed="true" stored="true" multiValued="false"/>
+    <field name="application_type" type="whole_string" indexed="true" stored="true" multiValued="false"/>
 
     <!-- Mandatory (M) -->
     <field name="title" type="sorting_string" indexed="true" stored="true" multiValued="false"/>
-    <field name="content_type" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
-    <field name="digital_publisher" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
-    <field name="rights_access" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
-    <field name="reformatting" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
+    <field name="content_type" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
+    <field name="digital_publisher" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
+    <field name="rights_access" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
+    <field name="reformatting" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
     <field name="filename" type="sorting_string" indexed="true" stored="true" multiValued="true"/>
 
     <copyField source="title" dest="_text_"/>
@@ -142,13 +142,13 @@
     <field name="creator" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
     <field name="date_published" type="pdate" indexed="true" stored="true" multiValued="false"/>
     <!--<field name="date_created" type="pdate" indexed="true" stored="true" multiValued="false"/>-->
-    <field name="date_created" type="strings_ci" indexed="true" stored="true" multiValued="false"/>
-    <field name="summary_abstract" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
+    <field name="date_created" type="whole_string" indexed="true" stored="true" multiValued="false"/>
+    <field name="summary_abstract" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
     <field name="language" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
     <field name="institution_department" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
-    <field name="standard_digital_identifier" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
-    <field name="local_digital_identifier" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
-    <field name="edition_revision_information" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
+    <field name="standard_digital_identifier" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
+    <field name="local_digital_identifier" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
+    <field name="edition_revision_information" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
 
     <copyField source="subject" dest="_text_"/>
     <copyField source="creator" dest="_text_"/>
@@ -158,13 +158,13 @@
     <copyField source="summary_abstract" dest="summary_abstract_txt_en_split_mv"/>
 
     <!-- Non-core Recommended -->
-    <field name="alternative_title" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
+    <field name="alternative_title" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
     <field name="genre" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
-    <field name="table_of_contents" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
+    <field name="table_of_contents" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
     <field name="contributor" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
     <field name="related_resource" type="strings" indexed="true" stored="true" multiValued="true"/>
-    <field name="original_publisher" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
-    <field name="physical_extent" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
+    <field name="original_publisher" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
+    <field name="physical_extent" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
     <field name="sponsor" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
     <field name="details" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
     <field name="spatial" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
@@ -180,11 +180,11 @@
     <!-- Non-core Optional -->
     <field name="source_collection" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
     <field name="original_resource" type="strings" indexed="true" stored="true" multiValued="true"/>
-    <field name="notes" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
-    <field name="origin" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
-    <field name="audience_level" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
+    <field name="notes" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
+    <field name="origin" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
+    <field name="audience_level" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
     <field name="classification" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
-    <field name="physical_item_identifier" type="strings_ci" indexed="true" stored="true" multiValued="true"/>
+    <field name="physical_item_identifier" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
     <field name="physical_item_location" type="whole_strings" indexed="true" stored="true" multiValued="true"/>
 
     <copyField source="notes" dest="_text_"/>
@@ -244,7 +244,19 @@
       </analyzer>
     </fieldType>
 
-    <!-- whole strings type -->
+    <!--
+      An alternative to string type that is utilizes a lower case filter.
+      The solr.StrField does not allow this.
+      Base types like solr.StrField have omitNorms="true" by default.
+      This sets omitNorms="true" to more closely match the solr.StrField type.
+      The docValues="true" from solrStrField cannot be preserved because solr.TextField does not support it.
+    -->
+    <fieldType name="whole_string" class="solr.TextField" omitNorms="true" sortMissingLast="true">
+      <analyzer>
+        <tokenizer class="solr.KeywordTokenizerFactory"/>
+        <filter class="solr.LowerCaseFilterFactory"/>
+      </analyzer>
+    </fieldType>
     <fieldType name="whole_strings" class="solr.TextField" omitNorms="true" sortMissingLast="true" multiValued="true">
       <analyzer>
         <tokenizer class="solr.KeywordTokenizerFactory"/>
@@ -256,20 +268,6 @@
     <fieldType name="string" class="solr.StrField" sortMissingLast="true" docValues="true" />
     <fieldType name="strings" class="solr.StrField" sortMissingLast="true" multiValued="true" docValues="true" />
 
-    <!-- The TextField type can be analyzed and indexed/stored. -->
-    <fieldType name="string_ci" class="solr.TextField" sortMissingLast="true">
-      <analyzer>
-        <tokenizer class="solr.KeywordTokenizerFactory" />
-        <filter class="solr.LowerCaseFilterFactory" />
-      </analyzer>
-    </fieldType>
-    <fieldType name="strings_ci" class="solr.TextField" sortMissingLast="true" multiValued="true">
-      <analyzer>
-        <tokenizer class="solr.KeywordTokenizerFactory" />
-        <filter class="solr.LowerCaseFilterFactory" />
-      </analyzer>
-    </fieldType>
-
     <!-- boolean type: "true" or "false" -->
     <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
     <fieldType name="booleans" class="solr.BoolField" sortMissingLast="true" multiValued="true"/>