diff --git a/.github/workflows/ruby-unit-tests.yml b/.github/workflows/ruby-unit-tests.yml index c4268d19..aefeb2c0 100644 --- a/.github/workflows/ruby-unit-tests.yml +++ b/.github/workflows/ruby-unit-tests.yml @@ -12,19 +12,19 @@ jobs: matrix: backend: ['ruby', 'ruby-agraph'] # api runs tests with 4store backend and api-agraph runs with AllegroGraph backend steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: copy config file run: cp config/config.test.rb config/config.rb - - name: Build docker-compose - run: docker-compose build + - name: Build docker compose + run: docker compose build - name: Run unit tests # unit tests are run inside a container # http://docs.codecov.io/docs/testing-with-docker run: | ci_env=`bash <(curl -s https://codecov.io/env)` - docker-compose run $ci_env -e CI ${{ matrix.backend }} bundle exec rake test TESTOPTS='-v' + docker compose run $ci_env -e CI ${{ matrix.backend }} bundle exec rake test TESTOPTS='-v' - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: flags: unittests verbose: true diff --git a/Dockerfile b/Dockerfile index 5016faa1..819141f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,15 +8,12 @@ RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends \ raptor2-utils \ && rm -rf /var/lib/apt/lists/* -# The Gemfile Caching Trick -RUN mkdir -p /srv/ontoportal/ncbo_annotator -COPY *.gemspec Gemfile* /srv/ontoportal/ncbo_annotator/ - WORKDIR /srv/ontoportal/ncbo_annotator +COPY Gemfile* *.gemspec /srv/ontoportal/ncbo_annotator/ -# set bundler to v2.4.22 which is the last version supported by ruby 2.7 -RUN gem install bundler -v 2.4.22 -ENV BUNDLE_PATH /bundle +RUN gem install bundler +ENV BUNDLE_PATH=/bundle RUN bundle install + COPY . /srv/ontoportal/ncbo_annotator CMD ["/bin/bash"] diff --git a/Gemfile b/Gemfile index 6c521f89..f45945cb 100644 --- a/Gemfile +++ b/Gemfile @@ -1,11 +1,10 @@ source 'https://rubygems.org' gem 'cube-ruby' -gem 'faraday', '~> 1.9' +gem 'faraday' gem 'ffi' gem 'oj', '~> 3.0' gem 'rake', '~> 10.0' -gem 'redis' gem 'ruby-xxHash' group :development do @@ -16,6 +15,6 @@ group :development do end # NCBO gems (can be from a local dev path or from rubygems/git) -gem 'goo', github: 'ncbo/goo', branch: 'master' -gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'master' -gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'master' +gem 'goo', github: 'ncbo/goo', branch: 'develop' +gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'develop' +gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'develop' diff --git a/Gemfile.lock b/Gemfile.lock index 0d5034cb..f706d744 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,13 +1,14 @@ GIT remote: https://github.com/ncbo/goo.git - revision: 75436fe8e387febc53e34ee31ff0e6dd837a9d3f - branch: master + revision: 70007faf43d0d96292809f89f15a27a757ccaa25 + branch: develop specs: goo (0.0.2) addressable (~> 2.8) pry rdf (= 1.0.8) redis + request_store rest-client rsolr sparql-client @@ -15,8 +16,8 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: ee0013f0ee23876076bff9d9258b46371ec3b248 - branch: master + revision: c30887ee907acd39500cae6c12f45edb06fe0802 + branch: develop specs: ontologies_linked_data (0.0.1) activesupport @@ -35,8 +36,8 @@ GIT GIT remote: https://github.com/ncbo/sparql-client.git - revision: d418d56a6c9ff5692f925b45739a2a1c66bca851 - branch: master + revision: 1657f0dd69fd4b522d3549a6848670175f5e98cc + branch: develop specs: sparql-client (1.0.1) json_pure (>= 1.4) @@ -52,51 +53,34 @@ GEM multi_json (~> 1.3) thread_safe (~> 0.1) tzinfo (~> 0.3.37) - addressable (2.8.6) - public_suffix (>= 2.0.2, < 6.0) + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + base64 (0.2.0) bcrypt (3.1.20) - bigdecimal (3.1.5) - builder (3.2.4) + bigdecimal (3.1.8) + builder (3.3.0) coderay (1.1.3) - concurrent-ruby (1.2.2) + concurrent-ruby (1.3.4) connection_pool (2.4.1) cube-ruby (0.0.3) date (3.3.4) - docile (1.4.0) + docile (1.4.1) domain_name (0.6.20240107) - faraday (1.10.3) - faraday-em_http (~> 1.0) - faraday-em_synchrony (~> 1.0) - faraday-excon (~> 1.1) - faraday-httpclient (~> 1.0) - faraday-multipart (~> 1.0) - faraday-net_http (~> 1.0) - faraday-net_http_persistent (~> 1.0) - faraday-patron (~> 1.0) - faraday-rack (~> 1.0) - faraday-retry (~> 1.0) + faraday (2.8.1) + base64 + faraday-net_http (>= 2.0, < 3.1) ruby2_keywords (>= 0.0.4) - faraday-em_http (1.0.0) - faraday-em_synchrony (1.0.0) - faraday-excon (1.1.0) - faraday-httpclient (1.0.1) - faraday-multipart (1.0.4) - multipart-post (~> 2) - faraday-net_http (1.0.1) - faraday-net_http_persistent (1.2.0) - faraday-patron (1.0.0) - faraday-rack (1.0.0) - faraday-retry (1.0.3) - ffi (1.16.3) + faraday-net_http (3.0.2) + ffi (1.17.0) http-accept (1.7.0) - http-cookie (1.0.5) + http-cookie (1.0.7) domain_name (~> 0.5) i18n (0.9.5) concurrent-ruby (~> 1.0) - json (2.7.1) - json_pure (2.7.1) - libxml-ruby (5.0.2) - logger (1.6.0) + json (2.7.2) + json_pure (2.7.2) + libxml-ruby (5.0.3) + logger (1.6.1) macaddr (1.7.2) systemu (~> 2.6.5) mail (2.8.1) @@ -104,52 +88,55 @@ GEM net-imap net-pop net-smtp - method_source (1.0.0) + method_source (1.1.0) mime-types (3.5.2) mime-types-data (~> 3.2015) - mime-types-data (3.2023.1205) + mime-types-data (3.2024.0903) mini_mime (1.1.5) minitest (4.7.5) multi_json (1.15.0) - multipart-post (2.3.0) net-http-persistent (2.9.4) - net-imap (0.4.9.1) + net-imap (0.4.16) date net-protocol net-pop (0.1.2) net-protocol net-protocol (0.2.2) timeout - net-smtp (0.4.0.1) + net-smtp (0.5.0) net-protocol netrc (0.11.0) - oj (3.16.3) + oj (3.16.6) bigdecimal (>= 3.0) + ostruct (>= 0.2) omni_logger (0.1.4) logger + ostruct (0.6.0) pony (1.13.1) mail (>= 2.0) pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (5.0.4) - rack (3.0.8) + public_suffix (5.1.1) + rack (3.1.7) rack-test (2.1.0) rack (>= 1.3) rake (10.5.0) rdf (1.0.8) addressable (>= 2.2) - redis (5.0.8) - redis-client (>= 0.17.0) - redis-client (0.19.1) + redis (5.3.0) + redis-client (>= 0.22.0) + redis-client (0.22.2) connection_pool + request_store (1.7.0) + rack (>= 1.4) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - rexml (3.2.6) - rsolr (2.5.0) + rexml (3.3.7) + rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) ruby-xxHash (0.4.0.2) @@ -162,7 +149,7 @@ GEM simplecov-cobertura (2.1.0) rexml simplecov (~> 0.19) - simplecov-html (0.12.3) + simplecov-html (0.13.1) simplecov_json_formatter (0.1.4) systemu (2.6.5) thread_safe (0.3.6) @@ -172,12 +159,13 @@ GEM macaddr (~> 1.0) PLATFORMS - ruby + aarch64-linux + arm64-darwin-22 x86_64-linux DEPENDENCIES cube-ruby - faraday (~> 1.9) + faraday ffi goo! minitest (~> 4.0) @@ -185,7 +173,6 @@ DEPENDENCIES ontologies_linked_data! pry rake (~> 10.0) - redis ruby-xxHash simplecov simplecov-cobertura diff --git a/dip.yml b/dip.yml new file mode 100644 index 00000000..8bf3a609 --- /dev/null +++ b/dip.yml @@ -0,0 +1,44 @@ +version: '7.1' + +# Define default environment variables to pass +# to Docker Compose +#environment: +# RAILS_ENV: development + +interaction: + # This command spins up a container with the required dependencies (solr, 4store, etc), + # and opens a terminal within it. + runner: + description: Open a Bash shell within a ncbo_annotator container (with dependencies up) + service: ruby + command: /bin/bash + + # Run a container without any dependent services + bash: + description: Run an arbitrary script within a container (or open a shell without deps) + service: ruby + command: /bin/bash + compose_run_options: [ no-deps ] + + # A shortcut to run Bundler commands + bundle: + description: Run Bundler commands + service: ruby + command: bundle + compose_run_options: [ no-deps ] + + # A shortcut to run unit tests + test: + description: Run minitest unit tests + service: ruby + command: bundle exec rake test TESTOPTS='-v' + + test-agraph: + description: Run minitest unit tests + service: ruby-agraph + command: bundle exec rake test TESTOPTS='-v' + + 'redis-cli': + description: Run Redis console + service: redis-ut + command: redis-cli -h redis-ut diff --git a/docker-compose.yml b/docker-compose.yml index 3b343882..6c55b6bd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,30 +1,34 @@ -version: '3.8' - x-app: &app - build: - context: . - args: - RUBY_VERSION: '2.7' - environment: &env - COVERAGE: 'true' # enable simplecov code coverage - REDIS_HOST: redis-ut - REDIS_PORT: 6379 - MGREP_HOST: mgrep-ut - MGREP_PORT: 55556 - SOLR_TERM_SEARCH_URL: http://solr-ut:8983/solr/term_search_core1 - SOLR_PROP_SEARCH_URL: http://solr-ut:8983/solr/prop_search_core1 - volumes: - - .:/srv/ontoportal/ncbo_annotator - stdin_open: true - tty: true - command: "bundle exec rake test TESTOPTS='-v'" - depends_on: &depends_on - solr-ut: - condition: service_healthy - redis-ut: - condition: service_healthy - mgrep-ut: - condition: service_healthy + image: ncbo_annotator:0.0.1 + build: + context: . + args: + RUBY_VERSION: '3.0' + environment: &env + # default bundle config resolves to /usr/local/bundle/config inside of the container + # we are setting it to local app directory if we need to use 'bundle config local' + BUNDLE_APP_CONFIG: /srv/ontoportal/ncbo_annotator/.bundle + BUNDLE_PATH: /bundle + COVERAGE: 'true' # enable simplecov code coverage + REDIS_HOST: redis-ut + REDIS_PORT: 6379 + MGREP_HOST: mgrep-ut + MGREP_PORT: 55556 + SOLR_TERM_SEARCH_URL: http://solr-ut:8983/solr/term_search_core1 + SOLR_PROP_SEARCH_URL: http://solr-ut:8983/solr/prop_search_core1 + volumes: + - .:/srv/ontoportal/ncbo_annotator + - bundle:/bundle + stdin_open: true + tty: true + command: "bundle exec rake test TESTOPTS='-v'" + depends_on: &depends_on + solr-ut: + condition: service_healthy + redis-ut: + condition: service_healthy + mgrep-ut: + condition: service_healthy services: ruby: @@ -48,9 +52,9 @@ services: GOO_BACKEND_NAME: ag GOO_PORT: 10035 GOO_HOST: agraph-ut - GOO_PATH_QUERY: /repositories/bioportal_test - GOO_PATH_DATA: /repositories/bioportal_test/statements - GOO_PATH_UPDATE: /repositories/bioportal_test/statements + GOO_PATH_QUERY: /repositories/ontoportal_test + GOO_PATH_DATA: /repositories/ontoportal_test/statements + GOO_PATH_UPDATE: /repositories/ontoportal_test/statements depends_on: <<: *depends_on agraph-ut: @@ -59,7 +63,7 @@ services: - agraph solr-ut: - image: ontoportal/solr-ut:0.0.2 + image: ontoportal/solr-ut:0.2.0 healthcheck: test: ["CMD-SHELL", "curl -sf http://localhost:8983/solr/term_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1"] start_period: 3s @@ -79,7 +83,7 @@ services: image: ontoportal/mgrep:0.0.2 platform: linux/amd64 healthcheck: - test: ["CMD", "nc", "-z", "-v", "localhost", "55556"] + test: ["CMD", "nc", "-z", "-v", "127.0.0.1", "55556"] start_period: 3s interval: 10s timeout: 5s @@ -96,7 +100,7 @@ services: - 4store agraph-ut: - image: franzinc/agraph:v8.0.0 + image: franzinc/agraph:v8.2.0 platform: linux/amd64 environment: - AGRAPH_SUPER_USER=test @@ -104,12 +108,12 @@ services: shm_size: 1g command: > bash -c "/agraph/bin/agraph-control --config /agraph/etc/agraph.cfg start - ; agtool repos create bioportal_test + ; agtool repos create --supersede ontoportal_test ; agtool users add anonymous - ; agtool users grant anonymous root:bioportal_test:rw + ; agtool users grant anonymous root:ontoportal_test:rw ; tail -f /agraph/data/agraph.log" healthcheck: - test: ["CMD-SHELL", "agtool storage-report bioportal_test || exit 1"] + test: ["CMD-SHELL", "agtool storage-report ontoportal_test || exit 1"] start_period: 30s interval: 10s timeout: 5s diff --git a/lib/ncbo_annotator.rb b/lib/ncbo_annotator.rb index 4631b53a..e1e76c3b 100644 --- a/lib/ncbo_annotator.rb +++ b/lib/ncbo_annotator.rb @@ -142,10 +142,10 @@ def generate_dictionary_file() end outFile.close - redis_mgrep_dict_refresh_timestamp() + redis_mgrep_dict_refresh_timestamp end - @logger.info("Completed generating a dictionary file in #{(time/60).round(1)} minutes.") + @logger.info("Completed generating a dictionary file in #{time.round(2)} seconds.") end def create_term_cache_from_ontologies(ontologies, delete_cache=false, redis_prefix=nil) diff --git a/run-unit-tests.sh b/run-unit-tests.sh index 476c2b83..e25a7a8b 100755 --- a/run-unit-tests.sh +++ b/run-unit-tests.sh @@ -5,6 +5,6 @@ [ -f config/config.rb ] || cp config/config.test.rb config/config.rb docker-compose build -docker-compose run --rm ruby bundle exec rake test TESTOPTS='-v' -#docker-compose run --rm ruby-agraph bundle exec rake test TESTOPTS='-v' -docker-compose --profile agraph --profile 4store kill +docker compose run --rm ruby bundle exec rake test TESTOPTS='-v' +#docker compose run --rm ruby-agraph bundle exec rake test TESTOPTS='-v' +docker compose --profile agraph --profile 4store stop diff --git a/test/test_annotator.rb b/test/test_annotator.rb index 755c99c1..f63e0235 100644 --- a/test/test_annotator.rb +++ b/test/test_annotator.rb @@ -71,6 +71,7 @@ def test_all_classes_in_cache def test_generate_dictionary_file start_timestamp = Time.now + sleep(1) ontologies = AnnotatorUnit.ontologies.dup class_pages = TestAnnotator.all_classes(ontologies) assert class_pages.length > 100, "No classes in system ???" diff --git a/test/test_case.rb b/test/test_case.rb index 52476b5d..aa9817f2 100644 --- a/test/test_case.rb +++ b/test/test_case.rb @@ -48,8 +48,8 @@ def self.ontologies def before_suites # code to run before the very first test LinkedData::SampleData::Ontology.delete_ontologies_and_submissions - @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies - @@sty = LinkedData::SampleData::Ontology.load_semantic_types_ontology + @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies(process_submission: true) + @@sty = LinkedData::SampleData::Ontology.load_semantic_types_ontology(process_submission: true) annotator = Annotator::Models::NcboAnnotator.new annotator.init_redis_for_tests annotator.create_term_cache_from_ontologies(@@ontologies, true)