Merge branch 'master' into pl/feature-instances

.dockerignore

@@ -0,0 +1,11 @@
+# Git
+.git
+.gitignore
+# Logs
+log/*
+# Temp files
+tmp/*
+# Editor temp files
+*.swp
+*.swo
+test/solr

.github/workflows/ruby-unit-tests.yml

@@ -10,11 +10,14 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Set up solr configsets
-      run: test/solr/generate_ncbo_configsets.sh
+      working-directory: ./test
+      run: solr/generate_ncbo_configsets.sh
     - name: create config.rb file
       run: cp config/config.rb.sample config/config.rb
     - name: Build docker-compose
+      working-directory: ./test
       run: docker-compose build
     - name: Run unit tests
-      run: docker-compose up --exit-code-from unit-test
+      working-directory: ./test
+      run: docker-compose run ld-unit-test wait-for-it ld-solr-ut:8983 -- bundle exec rake test TESTOPTS='-v'
 

Dockerfile

@@ -1,12 +1,12 @@
 FROM ruby:2.6
 
-RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends vim openjdk-11-jre-headless raptor2-utils
+RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends openjdk-11-jre-headless raptor2-utils wait-for-it
 # The Gemfile Caching Trick
 RUN mkdir -p /srv/ontoportal/ontologies_linked_data
-COPY Gemfile* /srv/ontoportal/ontologies_linked_data/
+COPY Gemfile* *.gemspec /srv/ontoportal/ontologies_linked_data/
 WORKDIR /srv/ontoportal/ontologies_linked_data
 RUN gem install bundler -v "$(grep -A 1 "BUNDLED WITH" Gemfile.lock | tail -n 1)"
-RUN bundle install --binstubs
+RUN bundle install
 
 COPY . /srv/ontoportal/ontologies_linked_data
 

Gemfile.lock

@@ -1,6 +1,6 @@
 GIT
   remote: https://github.com/ncbo/goo.git
-  revision: d26b364dc5e8a22753cf9f8495a77c398112683f
+  revision: d0dd816a0afd5b85a02c4b909502cf3a93e6be58
   branch: master
   specs:
     goo (0.0.2)
@@ -73,7 +73,7 @@ GEM
     ethon (0.15.0)
       ffi (>= 1.15.0)
     eventmachine (1.2.7)
-    faraday (1.9.3)
+    faraday (1.10.0)
       faraday-em_http (~> 1.0)
       faraday-em_synchrony (~> 1.0)
       faraday-excon (~> 1.1)
@@ -133,7 +133,7 @@ GEM
     omni_logger (0.1.4)
       logger
     parallel (1.21.0)
-    parser (3.1.0.0)
+    parser (3.1.1.0)
       ast (~> 2.4.1)
     pony (1.13.1)
       mail (>= 2.0)
@@ -151,7 +151,7 @@ GEM
       addressable (>= 2.2)
     redis (4.6.0)
     ref (2.0.0)
-    regexp_parser (2.2.0)
+    regexp_parser (2.2.1)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
@@ -160,7 +160,7 @@ GEM
     rexml (3.2.5)
     rsolr (1.1.2)
       builder (>= 2.1.2)
-    rubocop (1.25.0)
+    rubocop (1.25.1)
       parallel (~> 1.10)
       parser (>= 3.1.0.0)
       rainbow (>= 2.2.2, < 4.0)
@@ -169,19 +169,19 @@ GEM
       rubocop-ast (>= 1.15.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.15.1)
-      parser (>= 3.0.1.1)
+    rubocop-ast (1.16.0)
+      parser (>= 3.1.1.0)
     ruby-progressbar (1.11.0)
     ruby-xxHash (0.4.0.2)
     ruby2_keywords (0.0.5)
     rubyzip (1.3.0)
-    sequel (5.53.0)
+    sequel (5.54.0)
     simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.3)
+    simplecov_json_formatter (0.1.4)
     systemu (2.6.5)
     test-unit-minitest (0.9.1)
       minitest (~> 4.7)

config/config.rb.sample

@@ -6,7 +6,7 @@ LinkedData.config do |config|
   config.repository_folder = "./test/data/ontology_files/repo"
   config.rest_url_prefix   = "http://data.bioontology.org/"
   config.enable_security   = false
-
+  config.java_max_heap_size         = '10240M'
   #PURL server config parameters
   config.enable_purl            = false
   config.purl_host              = "purl.bioontology.org"

lib/ontologies_linked_data/config/config.rb

@@ -28,10 +28,12 @@ def config(&block)
     @settings.search_server_url             ||= 'http://localhost:8983/solr/term_search_core1'
     @settings.property_search_server_url    ||= 'http://localhost:8983/solr/prop_search_core1'
     @settings.repository_folder             ||= './test/data/ontology_files/repo'
-    @settings.rest_url_prefix                ||= DEFAULT_PREFIX
+    @settings.rest_url_prefix                ||= 'http://data.bioontology.org/'
     @settings.enable_security               ||= false
     @settings.enable_slices                 ||= false
 
+    # Java/JVM options
+    @settings.java_max_heap_size            ||= '10240M'
     ### these params should be not ussed any more
     # removed so that dependencies shout
     #
@@ -40,8 +42,9 @@ def config(&block)
     # ###
 
     @settings.ui_host                       ||= 'bioportal.bioontology.org'
-    @settings.replace_url_prefix             ||= false
-    @settings.id_url_prefix                  ||= DEFAULT_PREFIX
+    @settings.replace_url_prefix            ||= false
+    @settings.id_url_prefix                 ||= 'http://data.bioontology.org/'
+
     @settings.queries_debug                 ||= false
     @settings.enable_monitoring             ||= false
     @settings.cube_host                     ||= 'localhost'
@@ -67,10 +70,10 @@ def config(&block)
     @settings.purl_username                 ||= ''
     @settings.purl_password                 ||= ''
     @settings.purl_maintainers              ||= ''
-    @settings.purl_target_url_prefix         ||= 'http://bioportal.bioontology.org'
+    @settings.purl_target_url_prefix        ||= 'http://bioportal.bioontology.org'
 
     # Email settings
-    @settings.enable_notifications           ||= false
+    @settings.enable_notifications          ||= false
     @settings.email_sender                  ||= 'admin@example.org' # Default sender for emails
     @settings.email_override                ||= 'test.email@example.org' # By default, all email gets sent here. Disable with email_override_disable.
     @settings.email_disable_override        ||= false
@@ -94,17 +97,17 @@ def config(&block)
     unless @settings.redis_host.nil?
       puts "Error: 'redis_host' is not a valid conf parameter."
       puts '        Redis databases were split into multiple hosts (09/22/13).'
-      raise StandardError, 'redis_host is not a valid conf parameter.'
     end
-
     # Check to make sure url prefix has trailing slash
     @settings.rest_url_prefix = "#{@settings.rest_url_prefix}/" unless @settings.rest_url_prefix[-1].eql?('/')
 
     puts "(LD) >> Using rdf store #{@settings.goo_host}:#{@settings.goo_port}"
     puts "(LD) >> Using term search server at #{@settings.search_server_url}"
     puts "(LD) >> Using property search server at #{@settings.property_search_server_url}"
-    puts "(LD) >> Using HTTP Redis instance at #{@settings.http_redis_host}:#{@settings.http_redis_port}"
-    puts "(LD) >> Using Goo Redis instance at #{@settings.goo_redis_host}:#{@settings.goo_redis_port}"
+    puts '(LD) >> Using HTTP Redis instance at '+
+            "#{@settings.http_redis_host}:#{@settings.http_redis_port}"
+    puts '(LD) >> Using Goo Redis instance at '+
+            "#{@settings.goo_redis_host}:#{@settings.goo_redis_port}"
 
     connect_goo unless overide_connect_goo
   end

lib/ontologies_linked_data/diff/bubastis_diff.rb

@@ -43,6 +43,7 @@ def initialize(input_fileOld, input_fileNew)
         @input_fileNew = input_fileNew
         @output_repo = File.expand_path(@input_fileNew).gsub(File.basename(@input_fileNew),'')
         @file_diff_path = nil
+        @java_heap_size = LinkedData.settings.java_max_heap_size
       end
 
       def setup_environment
@@ -83,7 +84,7 @@ def call_bubastis_java_cmd
         end
         errors_log = File.join([@output_repo, "bubastis_diff_errors.log"])
         File.delete errors_log if File.exist? errors_log
-        java_cmd = "java -DentityExpansionLimit=1500000 -Xmx5120M -jar #{@bubastis_jar_path} #{options.join(' ')}"
+        java_cmd = "java -DentityExpansionLimit=1500000 -Xmx#{@java_heap_size} -jar #{@bubastis_jar_path} #{options.join(' ')}"
         Diff.logger.info("Java call [#{java_cmd}]")
         stdout,stderr,status = Open3.capture3(java_cmd)
         if not status.success?

lib/ontologies_linked_data/parser/owlapi.rb

@@ -13,14 +13,15 @@ class RDFFileNotGeneratedException < Parser::ParserException
 
     class OWLAPICommand
       def initialize(input_file, output_repo, opts = {})
-        @owlapi_wrapper_jar_path = LinkedData.bindir + "/owlapi-wrapper-1.3.7.jar"
+        @owlapi_wrapper_jar_path = LinkedData.bindir + "/owlapi-wrapper-1.3.8.jar"
         @input_file = input_file
         @output_repo = output_repo
         @master_file = opts[:master_file]
         @logger = opts[:logger] || Parser.logger
         @file_triples_path = nil
         @missing_imports = nil
         @reasoning = true
+        @java_heap_size = LinkedData.settings.java_max_heap_size
       end
 
       def setup_environment
@@ -71,7 +72,7 @@ def call_owlapi_java_command
         if File.exist? errors_log
           File.delete errors_log
         end
-        command_call = "java -DentityExpansionLimit=2500000 -Xmx10240M -jar #{@owlapi_wrapper_jar_path} #{options}"
+        command_call = "java -DentityExpansionLimit=2500000 -Xmx#{@java_heap_size} -jar #{@owlapi_wrapper_jar_path} #{options}"
         @logger.info("Java call [#{command_call}]")
         Open3.popen3(command_call) do |i,o,e,w|
           i.close

docker-compose.yml → test/docker-compose.yml

@@ -1,50 +1,42 @@
 version: '3'
 
 services:
-  unit-test:
-    # volumes:
-    #  - .:/srv/ontoportal/ontologies_linked_data
-    build: .
-    #command: "bundle exec rake test TESTOPTS='-v' TEST='./test/parser/test_owl_api_command.rb'"
+  ld-unit-test:
+    build: ../.
     environment:
+      - OVERRIDE_CONNECT_GOO=true
       - GOO_BACKEND_NAME=4store
       - GOO_PORT=9000
-      - GOO_HOST=4store
+      - GOO_HOST=4store-ut
       - GOO_PATH_QUERY=/sparql/
       - GOO_PATH_DATA=/data/
       - GOO_PATH_UPDATE=/update/
-      - OVERRIDE_CONNECT_GOO=true
-      - REDIS_HOST=redis
+      - REDIS_HOST=redis-ut
       - REDIS_PORT=6379
-      - SOLR_HOST=solr
-    command: "bundle exec rake test TESTOPTS='-v'"
+      - SOLR_HOST=ld-solr-ut
+    command: "rake test TESTOPTS='-v'"
     depends_on:
-      - solr
-      - redis
-      - 4store
+      - ld-solr-ut
+      - redis-ut
+      - 4store-ut
 
-  redis:
+  redis-ut:
     image: redis
 
-  4store:
+  4store-ut:
     image: bde2020/4store
-    ports:
-        - "9000:9000"
     command: >
       bash -c "4s-backend-setup --segments 4 ontoportal_kb
       && 4s-backend ontoportal_kb
       && 4s-httpd -D -s-1 -p 9000 ontoportal_kb"
 
-  solr:
-    image: solr:8.8
+  ld-solr-ut:
+    image: solr:8
     volumes:
-      - ./test/solr/configsets:/configsets:ro
-    ports:
-      - "8983:8983"
+      - ./solr/configsets:/configsets:ro
+    #ports:
+    #  - "8983:8983"
     command: >
       bash -c "precreate-core term_search_core1 /configsets/term_search
       && precreate-core prop_search_core1 /configsets/property_search
       && solr-foreground"
-
-#  mgrep:
-#    image: ontoportal/mgrep-ncbo:0.1

test/run-unit-tests.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# sample script to run unit tests with docker
+
+# add config for unit testing
+[ -f ../config/config.rb ] || cp ../config/config.rb.sample ../config/config.rb
+
+#generate solr configsets for solr container
+solr/generate_ncbo_configsets.sh
+
+# build docker containers
+docker-compose build
+
+#docker-compose up --exit-code-from unit-test
+docker-compose run --rm ld-unit-test wait-for-it ld-solr-ut:8983 -- bundle exec rake test TESTOPTS='-v'
+

test/solr/generate_ncbo_configsets.sh

@@ -2,18 +2,22 @@
 # generates solr configsets by merging _default configset with config files in config/solr
 # _default is copied from sorl distribuion solr-8.10.1/server/solr/configsets/_default/
 
+#cd solr/configsets
+ld_config='../config/solr'
+[ -d solr/configsets/property_search ] && rm -Rf solr/configsets/property_search
+[ -d solr/configsets/term_search ] && rm -Rf solr/configsets/term_search
+if [[ ! -d ${ld_config}/property_search ]]; then
+  echo 'cant find ld solr config sets'
+  exit 1
+fi
+if [[ ! -d solr/configsets/_default/conf ]]; then
+  echo 'cant find default solr configset' 
+  exit 1
+fi
+mkdir -p solr/configsets/property_search/conf
+mkdir -p solr/configsets/term_search/conf
+cp -a solr/configsets/_default/conf/* solr/configsets/property_search/conf/
+cp -a solr/configsets/_default/conf/* solr/configsets/term_search/conf/
+cp -a $ld_config/property_search/* solr/configsets/property_search/conf
+cp -a $ld_config/term_search/* solr/configsets/term_search/conf
 
-pushd test/solr/configsets
-ld_config='../../../config/solr/'
-ls -l $ld_config
-pwd
-[ -d property_search ] && rm -Rf property_search
-[ -d term_search ] && rm -Rf property_search
-[ -d $ld_config/property_search ] || echo "cant find ontologies_linked_data project" 
-mkdir -p property_search/conf
-mkdir -p term_search/conf
-cp -a _default/conf/* property_search/conf/
-cp -a _default/conf/* term_search/conf/
-cp -a $ld_config/property_search/* property_search/conf
-cp -a $ld_config/term_search/* term_search/conf
-popd