diff --git a/.circleci/config.yml b/.circleci/config.yml index c976f5612..dfb28f768 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,24 +1,56 @@ -version: 2 -jobs: - build: - # Primary command image where all commands run +--- +version: 2.1 +executors: + orangelight-executor: docker: - image: circleci/ruby:2.6-node-browsers environment: RAILS_ENV: test ORANGELIGHT_HOST: localhost ORANGELIGHT_USER: postgres + SOLR_URL: http://solr:SolrRocks@localhost:8983/solr/orangelight-core-test NVM_HOME: ~/.nvm - - # Service container image available at 'host: localhost' - image: postgres:10 environment: POSTGRES_USER: orangelight POSTGRES_DB: orangelight_test POSTGRES_HOST_AUTH_METHOD: trust + - image: pulibrary/orangelight-solr:8.4 + command: server/scripts/ci-start.sh + working_directory: ~/orangelight + basic-executor: + docker: + - image: circleci/ruby:2.6-node-browsers + environment: + RAILS_ENV: test + NVM_HOME: ~/.nvm working_directory: ~/orangelight + +commands: + setup-bundler: steps: + - run: gem install bundler -v '2.2.14' + # Restore bundle cache + - restore_cache: + keys: + - orangelight-{{ checksum "Gemfile.lock" }} + - orangelight- + # Bundle install dependencies + - run: + name: Install dependencies + command: bundle check --path=vendor/bundle || bundle install --path=vendor/bundle --jobs 4 --retry 3 + - save_cache: + key: orangelight-{{ checksum "Gemfile.lock" }} + paths: + - ./vendor/bundle +jobs: + build: + executor: basic-executor + steps: + - attach_workspace: + at: '~/orangelight' - checkout + - setup-bundler - run: name: Install the NVM command: | @@ -37,43 +69,6 @@ jobs: command: | source ~/.nvm/nvm.sh curl -o- -L https://yarnpkg.com/install.sh | bash - - run: - name: Install Java 8 - command: | - sudo apt update - sudo apt install software-properties-common apt-transport-https - wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | sudo apt-key add - - sudo add-apt-repository --yes https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ - sudo apt update - sudo apt install adoptopenjdk-8-hotspot - sudo update-alternatives --set java /usr/lib/jvm/adoptopenjdk-8-hotspot-amd64/bin/java - #set timezone to est - - run: - name: Set Timezone to EST - command: echo 'America/New_York' = /etc/timezone - # Install Bundler - - run: gem install bundler -v '2.2.14' - # Restore bundle cache - - restore_cache: - keys: - - orangelight-{{ checksum "Gemfile.lock" }} - - orangelight- - # Bundle install dependencies - - run: - name: Install dependencies - command: bundle check --path=vendor/bundle || bundle install --path=vendor/bundle --jobs 4 --retry 3 - - run: - name: Install Chrome - command: | - wget -q -O - https://dl.google.com/linux/linux_signing_key.pub | sudo apt-key add - - sudo sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' - sudo apt-get update - sudo apt-get -y install google-chrome-stable - - run: sudo apt install postgresql-client - - save_cache: - key: orangelight-{{ checksum "Gemfile.lock" }} - paths: - - ./vendor/bundle # Only necessary if app uses webpacker or yarn in some other way - restore_cache: keys: @@ -89,32 +84,66 @@ jobs: key: orangelight-{{ checksum "yarn.lock" }} paths: - ~/.cache/yarn - - run: - name: Start Solr test instance - command: bundle exec rake server:test - background: true - - run: bin/jetty_wait - - run: - name: npm install - command: npm install + - persist_to_workspace: + root: '~/orangelight' + paths: '*' + + test: + executor: orangelight-executor + steps: + - attach_workspace: + at: '~/orangelight' + - setup-bundler - run: name: Wait for DB command: dockerize -wait tcp://localhost:5432 -timeout 1m + - run: sudo apt install postgresql-client - run: name: Database setup command: bundle exec rake db:setup + #set timezone to est - run: - name: Run rubocop - command: bundle exec rake rubocop + name: Set Timezone to EST + command: echo 'America/New_York' = /etc/timezone + - run: + name: Install Chrome + command: | + wget -q -O - https://dl.google.com/linux/linux_signing_key.pub | sudo apt-key add - + sudo sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' + sudo apt-get update + sudo apt-get -y install google-chrome-stable + - run: + name: Load config into solr + command: | + cd solr/conf + zip -1 -r solr_config.zip ./* + curl -H "Content-type:application/octet-stream" --data-binary @solr_config.zip "http://solr:SolrRocks@127.0.0.1:8983/solr/admin/configs?action=UPLOAD&name=orangelight" + curl -H 'Content-type: application/json' http://solr:SolrRocks@127.0.0.1:8983/api/collections/ -d '{create: {name: orangelight-core-test, config: orangelight, numShards: 1}}' + - run: + name: Index Test Data + command: bundle exec rake pulsearch:solr:index - run: name: Run Rspec command: bundle exec rspec spec - run: name: Run JS unit tests command: bundle exec yarn test + + rubocop: + executor: basic-executor + steps: + - attach_workspace: + at: '~/orangelight' + - setup-bundler + - run: bundle exec rubocop + workflows: - version: 2 - build_accept_deploy: + build_accept: jobs: - build - + - rubocop: + requires: + - build + - test: + requires: + - build diff --git a/.gitignore b/.gitignore index 42cbbe7ef..486379fc5 100644 --- a/.gitignore +++ b/.gitignore @@ -25,10 +25,7 @@ coverage/ .byebug_history # Ignore dreaded .DS_Store .DS_Store -# Ignore sorl config and libs downloaded from https://github.com/pulibrary/pul_solr/ -solr/conf/*.txt -solr/conf/*.xml -solr/conf/*.jar + /public/packs /public/packs-test /node_modules diff --git a/.lando.yml b/.lando.yml index b96e30473..2d5ae0d04 100644 --- a/.lando.yml +++ b/.lando.yml @@ -1,13 +1,19 @@ name: orangelight services: orangelight_test_solr: - type: solr:8.4 + type: solr:custom + overrides: + image: pulibrary/orangelight-solr:8.4 + command: server/scripts/lando-start.sh portforward: true core: orangelight-core-test config: dir: "solr/conf" orangelight_development_solr: - type: solr:8.4 + type: solr:custom + overrides: + image: pulibrary/orangelight-solr:8.4 + command: server/scripts/lando-start.sh portforward: true core: orangelight-core-dev config: diff --git a/.solr_wrapper b/.solr_wrapper deleted file mode 100644 index f50133d5a..000000000 --- a/.solr_wrapper +++ /dev/null @@ -1 +0,0 @@ -version: 8.4.1 \ No newline at end of file diff --git a/Gemfile b/Gemfile index d1b75bcb0..6ff8ce872 100644 --- a/Gemfile +++ b/Gemfile @@ -55,7 +55,6 @@ gem 'faraday', '~> 0.17' gem 'faraday-cookie_jar' gem 'global' gem 'omniauth-cas' -gem 'solr_wrapper', '~> 2.0' gem 'yajl-ruby', '>= 1.3.1', require: 'yajl' gem 'babel-transpiler' diff --git a/Gemfile.lock b/Gemfile.lock index 29b8aae8e..88a623130 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -400,7 +400,6 @@ GEM responders (3.0.1) actionpack (>= 5.0) railties (>= 5.0) - retriable (3.1.2) reverse_markdown (1.4.0) nokogiri rsolr (2.2.1) @@ -494,11 +493,6 @@ GEM thor (~> 0.19, >= 0.19.4) tilt (~> 2.0) yard (~> 0.9) - solr_wrapper (2.2.0) - faraday - retriable - ruby-progressbar - rubyzip spring (2.1.0) sprockets (3.7.2) concurrent-ruby (~> 1.0) @@ -628,7 +622,6 @@ DEPENDENCIES sitemap_generator (~> 6.0) sneakers solargraph - solr_wrapper (~> 2.0) spring sprockets-es6 string_rtl diff --git a/README.md b/README.md index 3ddc90a8f..8ef6031f4 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,14 @@ it to the bottom of `spec/fixtures/current_fixtures.json`. Note that file contains a list so you have to make sure you add a comma to the end of the last record and keep the closing bracket at the end of the file. Then run `rake pulsearch:solr:index` for both the dev and the test environment, as specified above. +## Update Solr configuration + +Run the following command to update pull in Solr configuration updates from the pul_solr repo: + +```bash +rake pulsearch:solr:update +``` + ## Local development with Figgy Orangelight is configured to use two environment variables in order to query and retrieve [IIIF Manifests](https://iiif.io/api/presentation/2.1/#manifest) for resources linked to catalog records in the [Figgy digital object repository](https://github.com/pulibrary/figgy). By default, these are provided with the following values: @@ -110,7 +118,7 @@ GRAPHQL_API_URL=https://figgy-staging.princeton.edu/graphql FIGGY_URL=https://fi To start up a copy of the project with a solr index of fixture data ```bash -bundle exec rake server +rake servers:start ``` Then, in another terminal window build browse index csv files in /tmp: ```bash diff --git a/Rakefile b/Rakefile index a6ea8a7be..100435196 100644 --- a/Rakefile +++ b/Rakefile @@ -3,7 +3,6 @@ require File.expand_path('../config/application', __FILE__) require 'rubocop/rake_task' if Rails.env.development? || Rails.env.test? -require 'solr_wrapper/rake_task' require 'honeybadger/init/ruby' require 'sneakers/tasks' @@ -19,6 +18,3 @@ if defined? RuboCop desc 'Run test suite and style checker' task spec: :rubocop end - -Rake::Task[:default].clear -task default: :ci diff --git a/config/blacklight.yml b/config/blacklight.yml index 94770a84e..5186269b6 100644 --- a/config/blacklight.yml +++ b/config/blacklight.yml @@ -15,7 +15,7 @@ development: url: <%= ENV['SOLR_URL'] || "http://#{ENV["lando_orangelight_development_solr_conn_host"] || "127.0.0.1"}:#{ENV["lando_orangelight_development_solr_conn_port"] || 8983}/solr/orangelight-core-dev" %> test: &test adapter: solr - url: <%= "http://#{ENV["lando_orangelight_test_solr_conn_host"] || "127.0.0.1"}:#{ENV['SOLR_TEST_PORT'] || ENV["lando_orangelight_test_solr_conn_port"] || 8888}/solr/orangelight-core-test" %> + url: <%= ENV['SOLR_URL'] || "http://#{ENV["lando_orangelight_test_solr_conn_host"] || "127.0.0.1"}:#{ENV['SOLR_TEST_PORT'] || ENV["lando_orangelight_test_solr_conn_port"] || 8888}/solr/orangelight-core-test" %> staging: adapter: solr url: <%= ENV['SOLR_URL'] || "http://127.0.0.1:8983/solr/blacklight-core" %> diff --git a/lib/orangelight/browse_lists.rb b/lib/orangelight/browse_lists.rb index 7589675f2..fa71c208c 100644 --- a/lib/orangelight/browse_lists.rb +++ b/lib/orangelight/browse_lists.rb @@ -20,26 +20,44 @@ def connection port = config['port'] sql_command = "PGPASSWORD=#{password} psql -U #{dbuser} -h #{dbhost} -p #{port} #{dbname} -c" - # changes for different facet queries - facet_request = "#{core_url}select?q=*%3A*&fl=id&wt=json&indent=true&defType=edismax&facet.sort=asc&facet.limit=-1&facet.field=" - solr_url = Blacklight.connection_config[:url] - - conn = Faraday.new(url: solr_url) do |faraday| + conn = Faraday.new(url: solr_connection.to_s) do |faraday| faraday.options[:open_timeout] = 2000 faraday.options[:timeout] = 2000 faraday.request :url_encoded # form-encode POST params faraday.response :logger # log requests to STDOUT faraday.adapter Faraday.default_adapter # make requests with Net::HTTP + faraday.basic_auth(solr_user, solr_password) if basic_auth? # enable Solr auth end + [sql_command, facet_request, conn] end + def facet_request + "#{core_url}select?q=*%3A*&fl=id&wt=json&indent=true&defType=edismax&facet.sort=asc&facet.limit=-1&facet.field=" + end + + def solr_connection + Blacklight.default_index.connection.uri + end + + def solr_user + solr_connection.user + end + + def solr_password + solr_connection.password + end + + def basic_auth? + solr_user && solr_password + end + def output_root Pathname.new('/tmp') end def core_url - Blacklight.default_index.connection.uri.to_s.gsub(%r{^.*\/solr}, '/solr') + solr_connection.to_s.gsub(%r{^.*\/solr}, '/solr') end def browse_facet(_sql_command, facet_request, conn, facet_field, table_name) diff --git a/lib/tasks/server.rake b/lib/tasks/server.rake index 1d9871e2d..787f0feca 100644 --- a/lib/tasks/server.rake +++ b/lib/tasks/server.rake @@ -1,20 +1,5 @@ # frozen_string_literal: true -require 'solr_wrapper' - -desc 'Run test suite' -task :ci do - if Rails.env.test? - run_solr('test', port: '8985') do - Rake::Task['pulsearch:solr:index'].invoke - Rake::Task['spec'].invoke - Rake::Task['yarn:test'].invoke - end - else - system('rake ci RAILS_ENV=test') - end -end - namespace :yarn do desc 'Run jest tests' task :test do @@ -22,14 +7,6 @@ namespace :yarn do end end -desc 'Run solr and orangelight for interactive development' -task :server, [:rails_server_args] do |_t, args| - run_solr('development', port: '8983') do - Rake::Task['pulsearch:solr:index'].invoke - system "bundle exec rails s #{args[:rails_server_args]}" - end -end - namespace :servers do task initialize: :environment do Rake::Task["db:create"].invoke @@ -40,7 +17,6 @@ namespace :servers do desc "Start the Apache Solr and PostgreSQL container services using Lando." task start: :environment do - Rake::Task["pulsearch:solr:update"].invoke system("lando start") system("rake servers:initialize") system("rake servers:initialize RAILS_ENV=test") @@ -55,51 +31,17 @@ end namespace :server do desc 'Run development solr' task :dev do - if ENV["lando_orangelight_development_solr_conn_port"] - Rake::Task['pulsearch:solr:index'].invoke - puts("Indexing to Lando. Running at http://localhost:#{ENV['lando_orangelight_development_solr_conn_port']}") - else - run_solr('development', port: '8983') do - Rake::Task['pulsearch:solr:index'].invoke - sleep - end - end + Rake::Task['pulsearch:solr:index'].invoke + puts("Indexing to Lando. Running at http://localhost:#{ENV['lando_orangelight_development_solr_conn_port']}") end desc 'Run test solr' task :test do if Rails.env.test? - if ENV["lando_orangelight_test_solr_conn_port"] - Rake::Task['pulsearch:solr:index'].invoke - puts("Indexing to Lando. Running at http://localhost:#{ENV['lando_orangelight_test_solr_conn_port']}") - else - run_solr('test', port: '8888') do - Rake::Task['pulsearch:solr:index'].invoke - sleep - end - end + Rake::Task['pulsearch:solr:index'].invoke + puts("Indexing to Lando. Running at http://localhost:#{ENV['lando_orangelight_test_solr_conn_port']}") else system('rake server:test RAILS_ENV=test') end end end - -def run_solr(environment, solr_params) - solr_dir = File.join(File.expand_path('.', File.dirname(__FILE__)), '../../', 'solr') - SolrWrapper.wrap(solr_params) do |solr| - ENV['SOLR_TEST_PORT'] = solr.port - - # additional solr configuration - Rake::Task['pulsearch:solr:update'].invoke(solr_dir) - solr.with_collection(name: "orangelight-core-#{environment}", dir: File.join(solr_dir, 'conf')) do - puts "\n#{environment.titlecase} solr server running: http://localhost:#{solr.port}/solr/#/orangelight-core-#{environment}" - puts "\n^C to stop" - puts ' ' - begin - yield - rescue Interrupt - puts 'Shutting down...' - end - end - end -end diff --git a/solr/conf/.keep b/solr/conf/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/solr/conf/CJKFoldingFilter.jar b/solr/conf/CJKFoldingFilter.jar new file mode 100644 index 000000000..0ba3634ec Binary files /dev/null and b/solr/conf/CJKFoldingFilter.jar differ diff --git a/solr/conf/lucene-umich-solr-filters-6.0.0-SNAPSHOT.jar b/solr/conf/lucene-umich-solr-filters-6.0.0-SNAPSHOT.jar new file mode 100644 index 000000000..a7a93b718 Binary files /dev/null and b/solr/conf/lucene-umich-solr-filters-6.0.0-SNAPSHOT.jar differ diff --git a/solr/conf/mapping-ISOLatin1Accent.txt b/solr/conf/mapping-ISOLatin1Accent.txt new file mode 100644 index 000000000..725b62c45 --- /dev/null +++ b/solr/conf/mapping-ISOLatin1Accent.txt @@ -0,0 +1,246 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Syntax: +# "source" => "target" +# "source".length() > 0 (source cannot be empty.) +# "target".length() >= 0 (target can be empty.) + +# example: +# "??" => "A" +# "\u00C0" => "A" +# "\u00C0" => "\u0041" +# "??" => "ss" +# "\t" => " " +# "\n" => "" + +# ?? => A +"\u00C0" => "A" + +# ?? => A +"\u00C1" => "A" + +# ?? => A +"\u00C2" => "A" + +# ?? => A +"\u00C3" => "A" + +# ?? => A +"\u00C4" => "A" + +# ?? => A +"\u00C5" => "A" + +# ?? => AE +"\u00C6" => "AE" + +# ?? => C +"\u00C7" => "C" + +# ?? => E +"\u00C8" => "E" + +# ?? => E +"\u00C9" => "E" + +# ?? => E +"\u00CA" => "E" + +# ?? => E +"\u00CB" => "E" + +# ?? => I +"\u00CC" => "I" + +# ?? => I +"\u00CD" => "I" + +# ?? => I +"\u00CE" => "I" + +# ?? => I +"\u00CF" => "I" + +# ?? => IJ +"\u0132" => "IJ" + +# ?? => D +"\u00D0" => "D" + +# ?? => N +"\u00D1" => "N" + +# ?? => O +"\u00D2" => "O" + +# ?? => O +"\u00D3" => "O" + +# ?? => O +"\u00D4" => "O" + +# ?? => O +"\u00D5" => "O" + +# ?? => O +"\u00D6" => "O" + +# ?? => O +"\u00D8" => "O" + +# ?? => OE +"\u0152" => "OE" + +# ?? +"\u00DE" => "TH" + +# ?? => U +"\u00D9" => "U" + +# ?? => U +"\u00DA" => "U" + +# ?? => U +"\u00DB" => "U" + +# ?? => U +"\u00DC" => "U" + +# ?? => Y +"\u00DD" => "Y" + +# ?? => Y +"\u0178" => "Y" + +# ?? => a +"\u00E0" => "a" + +# ?? => a +"\u00E1" => "a" + +# ?? => a +"\u00E2" => "a" + +# ?? => a +"\u00E3" => "a" + +# ?? => a +"\u00E4" => "a" + +# ?? => a +"\u00E5" => "a" + +# ?? => ae +"\u00E6" => "ae" + +# ?? => c +"\u00E7" => "c" + +# ?? => e +"\u00E8" => "e" + +# ?? => e +"\u00E9" => "e" + +# ?? => e +"\u00EA" => "e" + +# ?? => e +"\u00EB" => "e" + +# ?? => i +"\u00EC" => "i" + +# ?? => i +"\u00ED" => "i" + +# ?? => i +"\u00EE" => "i" + +# ?? => i +"\u00EF" => "i" + +# ?? => ij +"\u0133" => "ij" + +# ?? => d +"\u00F0" => "d" + +# ?? => n +"\u00F1" => "n" + +# ?? => o +"\u00F2" => "o" + +# ?? => o +"\u00F3" => "o" + +# ?? => o +"\u00F4" => "o" + +# ?? => o +"\u00F5" => "o" + +# ?? => o +"\u00F6" => "o" + +# ?? => o +"\u00F8" => "o" + +# ?? => oe +"\u0153" => "oe" + +# ?? => ss +"\u00DF" => "ss" + +# ?? => th +"\u00FE" => "th" + +# ?? => u +"\u00F9" => "u" + +# ?? => u +"\u00FA" => "u" + +# ?? => u +"\u00FB" => "u" + +# ?? => u +"\u00FC" => "u" + +# ?? => y +"\u00FD" => "y" + +# ?? => y +"\u00FF" => "y" + +# ??? => ff +"\uFB00" => "ff" + +# ??? => fi +"\uFB01" => "fi" + +# ??? => fl +"\uFB02" => "fl" + +# ??? => ffi +"\uFB03" => "ffi" + +# ??? => ffl +"\uFB04" => "ffl" + +# ??? => ft +"\uFB05" => "ft" + +# ??? => st +"\uFB06" => "st" diff --git a/solr/conf/protwords.txt b/solr/conf/protwords.txt new file mode 100644 index 000000000..3f0dfb9b6 --- /dev/null +++ b/solr/conf/protwords.txt @@ -0,0 +1,24 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# Use a protected word file to protect against the stemmer reducing two +# unrelated words to the same base word. + +perthes +constanter +bible +oversize +overseer +populism +musical +musicals diff --git a/solr/conf/schema.xml b/solr/conf/schema.xml new file mode 100644 index 000000000..aa5773597 --- /dev/null +++ b/solr/conf/schema.xml @@ -0,0 +1,879 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/solr/conf/solrconfig.xml b/solr/conf/solrconfig.xml new file mode 100644 index 000000000..c400a51e4 --- /dev/null +++ b/solr/conf/solrconfig.xml @@ -0,0 +1,521 @@ + + + + + + 8.4.1 + + + + + + + + + + + ${solr.core0.data.dir:} + + + + + + + ${solr.blacklight-core.data.dir:} + + + + + + + ${solr.data.dir:} + + + + + + + dedupe + + + + + + solrpingquery + + + all + + + + + + true + hashed_id_s + false + id + solr.processor.Lookup3Signature + + + + + + + + true + json + true + + + + + + solr + + + + + + + + + + true + + 100 + 500 + + + *:*score desc, pub_date_start_sort desc, title_sort asc + *:*pub_date_start_sort desc, title_sort asc + *:*pub_date_start_sort asc, title_sort asc + *:*author_sort asc, title_sort asc + *:*title_sort asc, pub_date_start_sort desc + naturescore desc, pub_date_start_sort desc, title_sort asc + timescore desc, pub_date_start_sort desc, title_sort asc + bookscore desc, pub_date_start_sort desc, title_sort asc + sciencescore desc, pub_date_start_sort desc, title_sort asc + musicscore desc, pub_date_start_sort desc, title_sort asc + + format:Audio + format:Book + format:Data File + format:Journal + format:Manuscript + format:Map + format:Mixed Material + format:Musical Score + format:Senior Thesis + format:Thesis + format:Video/Projected Medium + format:Visual Material + + location:Architecture Library + location:East Asian Library + location:Engineering Library + location:Fine Annex + location:Firestone Library + location:Forrestal Annex + location:Harold P. Furth Plasma Physics Library + location:Lewis Library + location:Marquand Library + location:Mendel Music Library + location:Mudd Manuscript Library + location:Rare Books and Special Collections + location:ReCAP + location:Stokes Library + location:Video Library + + cataloged_tdt:[NOW/DAY-7DAYS TO NOW/DAY+1DAY] + cataloged_tdt:[NOW/DAY-14DAYS TO NOW/DAY+1DAY] + cataloged_tdt:[NOW/DAY-21DAYS TO NOW/DAY+1DAY] + cataloged_tdt:[NOW/DAY-1MONTH TO NOW/DAY+1DAY] + cataloged_tdt:[NOW/DAY-2MONTHS TO NOW/DAY+1DAY] + cataloged_tdt:[NOW/DAY-3MONTHS TO NOW/DAY+1DAY] + cataloged_tdt:[NOW/DAY-6MONTHS TO NOW/DAY+1DAY] + + language_facet:English + language_facet:German + language_facet:French + language_facet:Spanish + language_facet:Chinese + language_facet:Italian + language_facet:Russian + language_facet:Arabic + language_facet:Japanese + language_facet:Latin + + subject_era_facet:20th century + subject_era_facet:19th century + subject_era_facet:18th century + subject_era_facet:21st century + subject_era_facet:17th century + + lc_1letter_facet:A - General Works + lc_1letter_facet:B - Philosophy, Psychology, Religion + lc_1letter_facet:C - Historical Sciences (Archaeology, Genealogy) + lc_1letter_facet:D - World History + lc_1letter_facet:E - History of the Americas (General) + lc_1letter_facet:F - History of the Americas (Local) + lc_1letter_facet:G - Geography, Anthropology, Recreation + lc_1letter_facet:H - Social Sciences + lc_1letter_facet:J - Political Science + lc_1letter_facet:K - Law + lc_1letter_facet:L - Education + lc_1letter_facet:M - Music + lc_1letter_facet:N - Fine Arts + lc_1letter_facet:P - Language & Literature + lc_1letter_facet:Q - Science + lc_1letter_facet:R - Medicine + lc_1letter_facet:S - Agriculture + lc_1letter_facet:T - Technology + lc_1letter_facet:U - Military Science + lc_1letter_facet:Z - Bibliography, Library Science, Information Resources + + subject_topic_facet:History + subject_topic_facet:United States + subject_topic_facet:Politics and government + subject_topic_facet:History and criticism + subject_topic_facet:Great Britain + subject_topic_facet:United States. Congress + subject_topic_facet:Claims + subject_topic_facet:Bills, Private + subject_topic_facet:China + subject_topic_facet:France + + genre_facet:Biography + genre_facet:Congresses + genre_facet:Periodicals + genre_facet:Private bills + genre_facet:Exhibitions + genre_facet:Early works to 1800 + genre_facet:Fiction + genre_facet:Maps + genre_facet:Bibliography + genre_facet:Catalogs + + sudoc_facet:X/Y - Congress + + acces_facet:Online + acces_facet:In the Library + + + + + + + + + edismax + explicit + 10 + on + *:* + 6<90% + + + if(field(numeric_id_b),50,0) + + + + + title_a_index^1500 + author_main_unstem_search^1000 + title_unstem_search^40 + title_display^40 + author_unstem_search^40 + subject_topic_unstem_search^18 + subject_unstem_search^15 + subject_topic_index^12 + subject_t^10 + subject_addl_unstem_search^8 + subject_addl_t^4 + isbn_t^3 + issn_s^3 + lccn_s^3 + text + description_t + cjk_all + + + title_245a_lr^16000 + title_245_lr^16000 + title_a_index^12000 + author_main_unstem_search^10000 + title_unstem_search^400 + title_display^400 + author_unstem_search^400 + subject_topic_unstem_search^180 + subject_unstem_search^150 + subject_topic_index^120 + subject_t^100 + subject_addl_unstem_search^80 + subject_addl_t^40 + isbn_t^30 + issn_s^30 + lccn_s^30 + text^10 + description_t^10 + cjk_all^10 + + + author_main_unstem_search^20 + author_unstem_search^10 + cjk_author + + + author_main_unstem_search^200 + author_unstem_search^100 + cjk_author^10 + + + title_245a_la^50 + title_245_la^10 + title_la^2 + title_addl_la + + + title_245a_lr^600 + title_245_lr^600 + title_245a_la^500 + title_245_la^100 + title_lr^100 + title_la^20 + title_addl_la^10 + + + more_in_this_series_la + + + more_in_this_series_la + + + pub_created_unstem_search + cjk_publisher + + + pub_created_unstem_search + cjk_publisher + + + notes_index + cjk_notes + + + notes_index + cjk_notes + + + series_title_index^5 + series_ae_index + series_statement_index + linked_series_title_index + linked_series_index + original_version_series_index + cjk_series_title + + + series_title_index^50 + series_ae_index^10 + series_statement_index^10 + linked_series_title_index^10 + linked_series_index^10 + original_version_series_index^10 + cjk_series_title^10 + + + title_a_index^500 + title_unstem_search^100 + title_display^50 + other_title_index^5 + series_title_index^5 + uniform_title_s^5 + title_vern_display + content_title_index + contains_title_index + linked_title_index + series_ae_index + series_statement_index + linked_series_title_index + linked_series_index + original_version_series_index + cjk_title + + + title_245a_lr^5500 + title_245_lr^5500 + title_a_index^5000 + title_unstem_search^1000 + title_display^500 + other_title_index^50 + series_title_index^50 + uniform_title_s^50 + title_vern_display^10 + content_title_index^10 + contains_title_index^10 + linked_title_index^10 + series_ae_index^10 + series_statement_index^10 + linked_series_title_index^10 + linked_series_index^10 + original_version_series_index^10 + cjk_title^10 + + + subject_topic_unstem_search^25 + subject_unstem_search^20 + cjk_subject + + + subject_topic_unstem_search^250 + subject_unstem_search^200 + cjk_subject^10 + + + 3 + 0.01 + + + id, + score, + author_display, + marc_relator_display, + format, + pub_created_display, + title_display, + title_vern_display, + isbn_s, + oclc_s, + lccn_s, + holdings_1display, + electronic_access_1display, + electronic_portfolio_s, + cataloged_tdt + + + true + 1 + 10 + format + language_facet + pub_date_start_sort + advanced_location_s + index + index + 1000 + 500 + + + + + + + + + + + + + + + + + + all + * + 1 + {!raw f=id v=$id} + + + diff --git a/solr/conf/spellings.txt b/solr/conf/spellings.txt new file mode 100644 index 000000000..162a044d5 --- /dev/null +++ b/solr/conf/spellings.txt @@ -0,0 +1,2 @@ +pizza +history diff --git a/solr/conf/stopwords.txt b/solr/conf/stopwords.txt new file mode 100644 index 000000000..b5824da32 --- /dev/null +++ b/solr/conf/stopwords.txt @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# a couple of test stopwords to test that the words are really being +# configured from this file: +stopworda +stopwordb + +#Standard english stop words taken from Lucene's StopAnalyzer +a +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +s +such +t +that +the +their +then +there +these +they +this +to +was +will +with + diff --git a/solr/conf/stopwords_en.txt b/solr/conf/stopwords_en.txt new file mode 100644 index 000000000..b5824da32 --- /dev/null +++ b/solr/conf/stopwords_en.txt @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +# a couple of test stopwords to test that the words are really being +# configured from this file: +stopworda +stopwordb + +#Standard english stop words taken from Lucene's StopAnalyzer +a +an +and +are +as +at +be +but +by +for +if +in +into +is +it +no +not +of +on +or +s +such +t +that +the +their +then +there +these +they +this +to +was +will +with + diff --git a/solr/conf/synonyms.txt b/solr/conf/synonyms.txt new file mode 100644 index 000000000..23002fec5 --- /dev/null +++ b/solr/conf/synonyms.txt @@ -0,0 +1,31 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +#some test synonym mappings unlikely to appear in real input text +#aaa => aaaa +#bbb => bbbb1 bbbb2 +#ccc => cccc1,cccc2 +#a\=>a => b\=>b +#a\,a => b\,b +#fooaaa,baraaa,bazaaa + +# Some synonym groups specific to this example +GB,gib,gigabyte,gigabytes +MB,mib,megabyte,megabytes +Television, Televisions, TV, TVs +#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming +#after us won't split it into two words. + +# Synonym mappings can be used for spelling correction too +#pixima => pixma + diff --git a/solr/docker/CJKFoldingFilter.jar b/solr/docker/CJKFoldingFilter.jar new file mode 100644 index 000000000..0ba3634ec Binary files /dev/null and b/solr/docker/CJKFoldingFilter.jar differ diff --git a/solr/docker/Dockerfile b/solr/docker/Dockerfile new file mode 100644 index 000000000..188ff914c --- /dev/null +++ b/solr/docker/Dockerfile @@ -0,0 +1,4 @@ +FROM pulibrary/ci-solr:8.4 + +COPY --chown=$SOLR_USER:$SOLR_USER CJKFoldingFilter.jar /opt/solr/contrib/analysis-extras/lib/. +COPY --chown=$SOLR_USER:$SOLR_USER lucene-umich-solr-filters-6.0.0-SNAPSHOT.jar /opt/solr/contrib/analysis-extras/lib/. diff --git a/solr/docker/lucene-umich-solr-filters-6.0.0-SNAPSHOT.jar b/solr/docker/lucene-umich-solr-filters-6.0.0-SNAPSHOT.jar new file mode 100644 index 000000000..a7a93b718 Binary files /dev/null and b/solr/docker/lucene-umich-solr-filters-6.0.0-SNAPSHOT.jar differ diff --git a/spec/lib/orangelight/browse_lists/call_number_csv_spec.rb b/spec/lib/orangelight/browse_lists/call_number_csv_spec.rb index b81697211..f2d7c1a6f 100644 --- a/spec/lib/orangelight/browse_lists/call_number_csv_spec.rb +++ b/spec/lib/orangelight/browse_lists/call_number_csv_spec.rb @@ -4,7 +4,14 @@ require 'orangelight/browse_lists/call_number_csv' RSpec.describe BrowseLists::CallNumberCSV do - let(:solr_url) { Blacklight.connection_config[:url] } + let(:solr_url) do + # Remove Solr auth username and password used in CI. + # Faraday adds as Base64 encoded credentials in the + # Authorization header. Causes issues with webmocks + uri = Blacklight.default_index.connection.uri.dup + uri.user = nil + uri.to_s + end describe "#write" do let(:output_root) { Rails.root.join("tmp", "spec") } @@ -14,17 +21,17 @@ WebMock.disable_net_connect! stub_request( - :get, "#{solr_url}/select?defType=edismax&facet.field=call_number_browse_s&facet.limit=-1&facet.mincount=2&facet.sort=asc&fl=id&indent=true&q=*:*&wt=json" + :get, "#{solr_url}select?defType=edismax&facet.field=call_number_browse_s&facet.limit=-1&facet.mincount=2&facet.sort=asc&fl=id&indent=true&q=*:*&wt=json" ) .to_return(status: 200, body: file_fixture("call_number_browse/multi_call_numbers.json"), headers: {}) stub_request( - :get, "#{solr_url}/select?cursorMark=*&defType=edismax&facet=false&fl=call_number_browse_s,title_display,title_vern_display,author_display,author_s,id,pub_created_vern_display,pub_created_display,holdings_1display&indent=true&q=*:*&rows=500&sort=id%20asc&wt=json" + :get, "#{solr_url}select?cursorMark=*&defType=edismax&facet=false&fl=call_number_browse_s,title_display,title_vern_display,author_display,author_s,id,pub_created_vern_display,pub_created_display,holdings_1display&indent=true&q=*:*&rows=500&sort=id%20asc&wt=json" ) .to_return(status: 200, body: file_fixture("call_number_browse/index_entries_rows500_cursor_p1.json"), headers: {}) stub_request( - :get, "#{solr_url}/select?cursorMark=AoEoMTAwMDA1NjY=&defType=edismax&facet=false&fl=call_number_browse_s,title_display,title_vern_display,author_display,author_s,id,pub_created_vern_display,pub_created_display,holdings_1display&indent=true&q=*:*&rows=500&sort=id%20asc&wt=json" + :get, "#{solr_url}select?cursorMark=AoEoMTAwMDA1NjY=&defType=edismax&facet=false&fl=call_number_browse_s,title_display,title_vern_display,author_display,author_s,id,pub_created_vern_display,pub_created_display,holdings_1display&indent=true&q=*:*&rows=500&sort=id%20asc&wt=json" ) .to_return(status: 200, body: file_fixture("call_number_browse/index_entries_rows500_cursor_p2.json"), headers: {}) end @@ -57,7 +64,7 @@ before do allow(Rails.logger).to receive(:error) stub_request( - :get, "#{solr_url}/select?cursorMark=*&defType=edismax&facet=false&fl=call_number_browse_s,title_display,title_vern_display,author_display,author_s,id,pub_created_vern_display,pub_created_display,holdings_1display&indent=true&q=*:*&rows=500&sort=id%20asc&wt=json" + :get, "#{solr_url}select?cursorMark=*&defType=edismax&facet=false&fl=call_number_browse_s,title_display,title_vern_display,author_display,author_s,id,pub_created_vern_display,pub_created_display,holdings_1display&indent=true&q=*:*&rows=500&sort=id%20asc&wt=json" ) .to_return(status: 200, body: {}.to_json, headers: {}) end