Merge to master: Release 2.2.0 - Add index all data step to the processing queue

.github/workflows/ruby-unit-tests.yml

@@ -9,22 +9,35 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        backend: ['ncbo_cron'] # ruby runs tests with 4store backend and ruby-agraph runs with AllegroGraph backend
+        goo-slice: [ '20', '100', '500' ]
+        ruby-version: [ '2.7' ]
+        triplestore: [ 'fs', 'ag', 'vo', 'gb' ]
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - name: copy config.rb file from template
-      run:  cp config/config.rb.sample config/config.rb
-    - name: Build docker-compose
-      run: docker-compose build
-    - name: Run unit tests
-      run: |
-        ci_env=`bash <(curl -s https://codecov.io/env)`
-        docker-compose run $ci_env -e CI --rm ${{ matrix.backend }} bundle exec rake test TESTOPTS='-v'
-    - name: Upload coverage reports to Codecov
-      uses: codecov/codecov-action@v3
-      with:
-        flags: unittests
-        verbose: true
-        fail_ci_if_error: false # optional (default = false)
-
+      - uses: actions/checkout@v3
+      - name: create config.rb file
+        run: cp config/config.rb.sample config/config.rb
+      - name: Install Dependencies
+        run: sudo apt-get update && sudo apt-get -y install raptor2-utils
+      - name: Set up JDK 11
+        uses: actions/setup-java@v2
+        with:
+          java-version: '11'
+          distribution: 'adopt'
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+      - name: Run unit tests
+        # unit tests are run inside a container
+        # http://docs.codecov.io/docs/testing-with-docker
+        run: |
+          ci_env=`bash <(curl -s https://codecov.io/env)`
+          GOO_SLICES=${{ matrix.goo-slice }} bundle exec rake test:docker:${{ matrix.triplestore }} TESTOPTS="-v"
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          flags: unittests
+          verbose: true
+          fail_ci_if_error: false # optional (default = false)

Gemfile

@@ -21,14 +21,16 @@ gem 'redis'
 gem 'rest-client'
 gem 'sys-proctable'
 gem 'request_store'
+gem 'parallel'
+gem 'json-ld'
 
 # Monitoring
 gem 'cube-ruby', require: 'cube'
 
-gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'master'
-gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master'
-gem 'ontologies_linked_data', github: 'ontoportal-lirmm/ontologies_linked_data', branch: 'master'
-gem 'ncbo_annotator', github: 'ontoportal-lirmm/ncbo_annotator', branch: 'master'
+gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'development'
+gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'development'
+gem 'ontologies_linked_data', github: 'ontoportal-lirmm/ontologies_linked_data', branch: 'development'
+gem 'ncbo_annotator', github: 'ontoportal-lirmm/ncbo_annotator', branch: 'development'
 # Testing
 group :test do
   gem 'email_spec'

Gemfile.lock

@@ -1,12 +1,15 @@
 GIT
   remote: https://github.com/ontoportal-lirmm/goo.git
-  revision: 8ddd2d719617ad082c6964a9efdac153cdd2b48e
-  branch: master
+  revision: b2a635fb1e8206e6e3010be4dbe033b47eb58481
+  branch: development
   specs:
     goo (0.0.2)
       addressable (~> 2.8)
       pry
-      rdf (= 1.0.8)
+      rdf (= 3.2.11)
+      rdf-raptor
+      rdf-rdfxml
+      rdf-vocab
       redis
       rest-client
       rsolr
@@ -15,8 +18,8 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/ncbo_annotator.git
-  revision: 57204d8e54432ba660af4c49806e2a3019a23fa2
-  branch: master
+  revision: 1eb751b65d10ae23d45c74e0516c78754a8419f0
+  branch: development
   specs:
     ncbo_annotator (0.0.1)
       goo
@@ -26,8 +29,8 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git
-  revision: 95e77989e4e8ea2fde86cf4f048f7d6cd7a6829f
-  branch: master
+  revision: 0855adfce5365c6ef7a494e3d3ad9af2611974d9
+  branch: development
   specs:
     ontologies_linked_data (0.0.1)
       activesupport
@@ -46,13 +49,12 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/sparql-client.git
-  revision: aed51baf4106fd0f3d0e3f9238f0aad9406aa3f0
-  branch: master
+  revision: 59251e59346c9a69a67c88552ba55a1244eec602
+  branch: development
   specs:
-    sparql-client (1.0.1)
-      json_pure (>= 1.4)
-      net-http-persistent (= 2.9.4)
-      rdf (>= 1.0)
+    sparql-client (3.2.2)
+      net-http-persistent (~> 4.0, >= 4.0.2)
+      rdf (~> 3.2, >= 3.2.11)
 
 PATH
   remote: .
@@ -79,9 +81,9 @@ GEM
       public_suffix (>= 2.0.2, < 6.0)
     base64 (0.2.0)
     bcrypt (3.1.20)
-    bigdecimal (3.1.6)
-    binding_of_caller (1.0.0)
-      debug_inspector (>= 0.0.1)
+    bigdecimal (3.1.8)
+    binding_of_caller (1.0.1)
+      debug_inspector (>= 1.2.0)
     builder (3.2.4)
     coderay (1.1.3)
     concurrent-ruby (1.2.3)
@@ -101,7 +103,7 @@ GEM
       faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
     faraday-net_http (3.0.2)
-    faraday-retry (2.2.0)
+    faraday-retry (2.2.1)
       faraday (~> 2.0)
     ffi (1.16.3)
     gapic-common (0.21.1)
@@ -112,43 +114,47 @@ GEM
       googleapis-common-protos-types (>= 1.11.0, < 2.a)
       googleauth (~> 1.9)
       grpc (~> 1.59)
-    google-analytics-data (0.5.0)
+    google-analytics-data (0.6.0)
       google-analytics-data-v1beta (>= 0.11, < 2.a)
       google-cloud-core (~> 1.6)
-    google-analytics-data-v1beta (0.11.2)
+    google-analytics-data-v1beta (0.12.0)
       gapic-common (>= 0.21.1, < 2.a)
       google-cloud-errors (~> 1.0)
-    google-apis-analytics_v3 (0.14.0)
-      google-apis-core (>= 0.12.0, < 2.a)
-    google-apis-core (0.13.0)
+    google-apis-analytics_v3 (0.16.0)
+      google-apis-core (>= 0.15.0, < 2.a)
+    google-apis-core (0.15.0)
       addressable (~> 2.5, >= 2.5.1)
       googleauth (~> 1.9)
       httpclient (>= 2.8.1, < 3.a)
       mini_mime (~> 1.0)
       representable (~> 3.0)
       retriable (>= 2.0, < 4.a)
       rexml
-    google-cloud-core (1.6.1)
+    google-cloud-core (1.7.0)
       google-cloud-env (>= 1.0, < 3.a)
       google-cloud-errors (~> 1.0)
     google-cloud-env (2.1.1)
       faraday (>= 1.0, < 3.a)
-    google-cloud-errors (1.3.1)
-    google-protobuf (3.25.2-x86_64-linux)
-    googleapis-common-protos (1.4.0)
-      google-protobuf (~> 3.14)
-      googleapis-common-protos-types (~> 1.2)
-      grpc (~> 1.27)
-    googleapis-common-protos-types (1.11.0)
+    google-cloud-errors (1.4.0)
+    google-protobuf (3.25.3-x86_64-darwin)
+    google-protobuf (3.25.3-x86_64-linux)
+    googleapis-common-protos (1.5.0)
       google-protobuf (~> 3.18)
-    googleauth (1.10.0)
+      googleapis-common-protos-types (~> 1.7)
+      grpc (~> 1.41)
+    googleapis-common-protos-types (1.14.0)
+      google-protobuf (~> 3.18)
+    googleauth (1.11.0)
       faraday (>= 1.0, < 3.a)
       google-cloud-env (~> 2.1)
       jwt (>= 1.4, < 3.0)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (>= 0.16, < 2.a)
-    grpc (1.61.0-x86_64-linux)
+    grpc (1.63.0-x86_64-darwin)
+      google-protobuf (~> 3.25)
+      googleapis-common-protos-types (~> 1.0)
+    grpc (1.63.0-x86_64-linux)
       google-protobuf (~> 3.25)
       googleapis-common-protos-types (~> 1.0)
     htmlentities (4.3.4)
@@ -158,72 +164,97 @@ GEM
     httpclient (2.8.3)
     i18n (0.9.5)
       concurrent-ruby (~> 1.0)
-    json (2.7.1)
-    json_pure (2.7.1)
-    jwt (2.7.1)
+    json (2.7.2)
+    json-canonicalization (0.4.0)
+    json-ld (3.2.5)
+      htmlentities (~> 4.3)
+      json-canonicalization (~> 0.3, >= 0.3.2)
+      link_header (~> 0.0, >= 0.0.8)
+      multi_json (~> 1.15)
+      rack (>= 2.2, < 4)
+      rdf (~> 3.2, >= 3.2.10)
+    jwt (2.8.1)
+      base64
     launchy (2.5.2)
       addressable (~> 2.8)
-    libxml-ruby (5.0.2)
+    libxml-ruby (5.0.3)
+    link_header (0.0.8)
     logger (1.6.0)
     macaddr (1.7.2)
       systemu (~> 2.6.5)
     mail (2.6.6)
       mime-types (>= 1.16, < 4)
-    method_source (1.0.0)
+    method_source (1.1.0)
     mime-types (3.5.2)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2024.0206)
+    mime-types-data (3.2024.0507)
     mini_mime (1.1.5)
     minitest (4.7.5)
     mlanett-redis-lock (0.2.7)
       redis
     multi_json (1.15.0)
-    net-http-persistent (2.9.4)
+    net-http-persistent (4.0.2)
+      connection_pool (~> 2.2)
     netrc (0.11.0)
     oj (3.16.3)
       bigdecimal (>= 3.0)
     omni_logger (0.1.4)
       logger
     os (1.1.4)
+    parallel (1.24.0)
     parseconfig (1.1.2)
     pony (1.13.1)
       mail (>= 2.0)
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    public_suffix (5.0.4)
-    rack (3.0.9)
+    public_suffix (5.0.5)
+    rack (3.0.11)
     rack-test (2.1.0)
       rack (>= 1.3)
-    rake (13.1.0)
-    rdf (1.0.8)
-      addressable (>= 2.2)
-    redis (5.0.8)
-      redis-client (>= 0.17.0)
-    redis-client (0.19.1)
+    rake (13.2.1)
+    rdf (3.2.11)
+      link_header (~> 0.0, >= 0.0.8)
+    rdf-raptor (3.2.0)
+      ffi (~> 1.15)
+      rdf (~> 3.2)
+    rdf-rdfxml (3.2.2)
+      builder (~> 3.2)
+      htmlentities (~> 4.3)
+      rdf (~> 3.2)
+      rdf-xsd (~> 3.2)
+    rdf-vocab (3.2.7)
+      rdf (~> 3.2, >= 3.2.4)
+    rdf-xsd (3.2.1)
+      rdf (~> 3.2)
+      rexml (~> 3.2)
+    redis (5.2.0)
+      redis-client (>= 0.22.0)
+    redis-client (0.22.2)
       connection_pool
     representable (3.2.0)
       declarative (< 0.1.0)
       trailblazer-option (>= 0.1.1, < 0.2.0)
       uber (< 0.2.0)
-    request_store (1.5.1)
+    request_store (1.7.0)
       rack (>= 1.4)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
     retriable (3.1.2)
-    rexml (3.2.6)
-    rsolr (2.5.0)
+    rexml (3.2.8)
+      strscan (>= 3.0.9)
+    rsolr (2.6.0)
       builder (>= 2.1.2)
       faraday (>= 0.9, < 3, != 2.0.0)
     ruby-xxHash (0.4.0.2)
     ruby2_keywords (0.0.5)
     rubyzip (2.3.2)
     rufus-scheduler (2.0.24)
       tzinfo (>= 0.3.22)
-    signet (0.18.0)
+    signet (0.19.0)
       addressable (~> 2.8)
       faraday (>= 0.17.5, < 3.a)
       jwt (>= 1.5, < 3.0)
@@ -237,6 +268,7 @@ GEM
       simplecov (~> 0.19)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
+    strscan (3.1.0)
     sys-proctable (1.3.0)
       ffi (~> 1.1)
     systemu (2.6.5)
@@ -250,6 +282,7 @@ GEM
       macaddr (~> 1.0)
 
 PLATFORMS
+  x86_64-darwin-23
   x86_64-linux
 
 DEPENDENCIES
@@ -260,13 +293,15 @@ DEPENDENCIES
   goo!
   google-analytics-data
   google-apis-analytics_v3
+  json-ld
   mail (= 2.6.6)
   minitest (< 5.0)
   multi_json
   ncbo_annotator!
   ncbo_cron!
   oj (~> 3.0)
   ontologies_linked_data!
+  parallel
   parseconfig
   pony
   pry
@@ -281,4 +316,4 @@ DEPENDENCIES
   test-unit-minitest
 
 BUNDLED WITH
-   2.3.23
+   2.4.22