Merge pull request #91 from ARGOeu/devel

Version 1.0.3

Jenkinsfile

@@ -0,0 +1,63 @@
+pipeline {
+    agent { 
+        docker { 
+            image 'node:buster' 
+        }
+    }
+    options {
+        checkoutToSubdirectory('eosc-recommender-metrics')
+        newContainerPerStage()
+    }
+    environment {
+        PROJECT_DIR='eosc-recommender-metrics'
+        GH_USER = 'newgrnetci'
+        GH_EMAIL = '<argo@grnet.gr>'
+   }
+    stages {
+
+        stage ('Deploy Docs') {
+            when {
+                anyOf {
+                    branch 'master'
+                    branch 'devel'
+                }
+            }
+            steps {
+                echo 'Publish eosc-recommender-metrics docs...'
+                sh '''
+                    cd $WORKSPACE/$PROJECT_DIR
+                    cd website
+                    npm install
+                '''
+                sshagent (credentials: ['jenkins-master']) {
+                    sh '''
+                        cd $WORKSPACE/$PROJECT_DIR/website
+                        mkdir ~/.ssh && ssh-keyscan -H github.com > ~/.ssh/known_hosts
+                        git config --global user.email ${GH_EMAIL}
+                        git config --global user.name ${GH_USER}
+                        GIT_USER=${GH_USER} USE_SSH=true npm run deploy
+                    '''
+                }
+            }
+        } 
+    }
+    post{
+        always {
+            cleanWs()
+        }
+        success {
+            script{
+                if ( env.BRANCH_NAME == 'master' || env.BRANCH_NAME == 'devel' ) {
+                    slackSend( message: ":rocket: New version of eosc-recommender-metrics docs deployed! Job: $JOB_NAME !\n <https://argoeu.github.io/eosc-recommender-metrics|See them here...>")
+                }
+            }
+        }
+        failure {
+            script{
+                if ( env.BRANCH_NAME == 'master' || env.BRANCH_NAME == 'devel' ) {
+                    slackSend( message: ":rain_cloud: Deployment of eosc-recommender-metrics docs failed! Job: $JOB_NAME")
+                }
+            }
+        }
+    }
+}

README.md

@@ -22,112 +22,30 @@ A framework for counting the recommender metrics
 1. Install Conda from here: https://docs.conda.io/projects/conda/en/latest/user-guide/install/linux.html. Tested on conda v 4.10.3.
 2. Run from terminal: `conda env create -f environment.yml`
 3. Run from terminal: `conda activate rsmetrics`
-4. Run from terminal: `chmod +x ./preprocessor.py ./rsmetrics.py`
+4. Run from terminal: `chmod +x ./preprocessor.py ./preprocessor_common.py ./rsmetrics.py`
 
 # Usage
-7. Run from terminal: `./preprocessor.py` in order to prepare the data for the RSmetrics
+5. Configure `./preprocessor_common.py`, `./preprocessor.py` and `./rsmetrics.py` by editting the `config.yaml` or providing another with `-c`.
+6. Run from terminal: `./preprocessor_common.py` in order to gather `users` and `resources` and store them in the `Datastore`:
 ```bash
-
-  _____                                                       
- |  __ \                                                      
- | |__) | __ ___ _ __  _ __ ___   ___ ___  ___ ___  ___  _ __ 
- |  ___/ '__/ _ \ '_ \| '__/ _ \ / __/ _ \/ __/ __|/ _ \| '__|
- | |   | | |  __/ |_) | | | (_) | (_|  __/\__ \__ \ (_) | |   
- |_|   |_|  \___| .__/|_|  \___/ \___\___||___/___/\___/|_|   
-                | |                                           
-                |_|                                           
-
-Version: 0.2
-© 2022, National Infrastructures for Research and Technology (GRNET)
-
-usage: preprocessor [-c [FILEPATH]] [-o [DIRPATH]] [-s [DATETIME]] [-e [DATETIME]] [-h]
-                    [-v]
-
-Prepare data for the EOSC Marketplace RS metrics calculation
-
-optional arguments:
-  -c [FILEPATH], --config [FILEPATH]
-                        override default configuration file (./config.yaml)
-  -o [DIRPATH], --output [DIRPATH]
-                        override default output dir path (./data)
-  -s [DATETIME], --starttime [DATETIME]
-                        process data starting from given datetime in ISO format (UTC)
-                        e.g. YYYY-MM-DD
-  -e [DATETIME], --endtime [DATETIME]
-                        process data ending to given datetime in ISO format (UTC) e.g.
-                        YYYY-MM-DD
-  -h, --help            show this help message and exit
-  -v, --version         show program's version number and exit
+./preprocessor_common.py # this will ingest users and resources [from scratch] by retrieving the data from 'cyfronet' provider (which is specified in the config file
+./preprocessor_common.py -p cyfronet # equivalent to first one
+./preprocessor_common.py -p cyfronet --use-cache # equivalent to first one but use the cache file to read resources instead of downloading them via the EOSC Marketplace
+./preprocessor_common.py -p athena # currently is not working since users collection only exist in 'cyfronet'
 ```
-
-8. Configure `./preprocessor.py` by editting the `config.yaml` or providing another with `-c`:
-```yaml
-
-# Set the desired connector (e.g. MongoDB)
-Source:
-    MongoDB:
-        host: localhost
-        port: 27017
-        db: recommender_dev
-
-User:
-    export: true
-
-Service:
-    # if true it keeps only published, otherwise all
-    # this has an effect in exporting when from is set to 'source' 
-    # and also in metrics calculations where service is considered
-    published: true
-
-    # Use the EOSC-Marketplace webpage
-    # to associate page_id and service_id
-    download: true
-    path: ./page_map
-
-    export: true
-    from: 'page_map' # or 'source'
-
-# Calculate source's metrics
-Metrics: true
-
+7. Run from terminal: `./preprocessor.py -p <provider>` in order to gather `user_actions` and `recommendations` from the particular provider and store them in the `Datastore`:
+```bash
+./preprocessor.py # this will ingest user_actions and recommendations [from scratch] by retrieving the data from 'cyfronet' provider (which is specified in the config file
+./preprocessor.py -p cyfronet # equivalent to first one
+./preprocessor.py -p athena # same procedure as the first one but for 'athena' provider
 ```
-
-9. Run from terminal: `./rsmetrics.py --users --services` to run RSmetrics and include the `users.csv` and `services.csv` files generated by the Preprocessor
+9. Run from terminal: `./rsmetrics.py -p <provider>` in order to gather the respective data (`users`, `resources`, `user_actions` and `recommendations`), calculate `statistics` and `metrics` and store them in the `Datastore`, concerning that particular provider:
 ```bash
-  _____   _____                _        _          
- |  __ \ / ____|              | |      (_)         
- | |__) | (___  _ __ ___   ___| |_ _ __ _  ___ ___ 
- |  _  / \___ \| '_ ` _ \ / _ \ __| '__| |/ __/ __|
- | | \ \ ____) | | | | | |  __/ |_| |  | | (__\__ \
- |_|  \_\_____/|_| |_| |_|\___|\__|_|  |_|\___|___/
-
-Version: 0.2
-© 2022, National Infrastructures for Research and Technology (GRNET)
-
-usage: rsmetrics [-i [FILEPATH]] [-s [DATETIME]] [-e [DATETIME]] [--users] [--services]
-                 [-h] [-v]
-
-Calculate metrics for the EOSC Marketplace RS
-
-optional arguments:
-  -i [FILEPATH], --input [FILEPATH]
-                        override default output dir (./data)
-  -s [DATETIME], --starttime [DATETIME]
-                        calculate metrics starting from given datetime in ISO format
-                        (UTC) e.g. YYYY-MM-DD
-  -e [DATETIME], --endtime [DATETIME]
-                        calculate metrics ending to given datetime in ISO format (UTC)
-                        e.g. YYYY-MM-DD
-  --users               enable reading total users from users.csv, otherwise it will be
-                        calculated according to the user actions
-  --services            enable reading total services from services.csv, otherwise it
-                        will be calculated according to the user actions
-  -h, --help            show this help message and exit
-  -v, --version         show program's version number and exit
-
+./rsmetrics.py # this will calculate and store statistics and metrics concerning data (users, resources, user_actions and recommendations) concerning the specified provider (which by default is 'cyfronet')
+./rsmetrics.py -p cyfronet # equivalent to first one
+./rsmetrics.py -p athena # same procedure as the first one for 'athena' provider
 ```
 
-
 ## Reporting
 
 The reporting script generates an evalutation report in html format automatically served from a spawed localserver (default: http://localhost:8080)
@@ -203,3 +121,4 @@ This by default honors this repo's folder structure and directs to the root `/da
 You can override this by editing the `.env` file inside the `/webservice` folder, or specificy the `RS_EVAL_METRIC_SOURCE` variable accordingly before executing the `flask run` command
 
 _Tested with python 3.9_
+

config.yaml

@@ -1,25 +1,20 @@
-# Set the desired connector (e.g. MongoDB)
-Source:
-    MongoDB:
-        host: localhost
-        port: 27017
-        db: recommender_dev
+# Set the desired connector
+# a list of:
+# name/tag, type of database, hostname, port, database
+providers:
+   - name: cyfronet
+     db: "mongodb://localhost:27017/recommender_dev"
+   - name: athena
+     db: "mongodb://localhost:27017/athena_dev"
 
 # The database where the Preprocessor's
 # and RSmetrics data are stored
-Datastore:
-    MongoDB:
-        host: localhost
-        port: 27017
-        db: rsmetrics
+datastore: "mongodb://localhost:27017/rsmetrics"
 
-Service:
+service:
     # Use the EOSC-Marketplace webpage
-    # to retrieve resources and 
-    # associate the page_id and the service_id
-    Portal:
-        download: true
-        path: ./page_map
+    # to retrieve resources and associate the page_id and the service_id
+    store: './page_map' # or null
 
     # if true it keeps only published, otherwise all
     # this has an effect in exporting when from is set to 'source' 
@@ -32,9 +27,4 @@ Service:
     # - 'page_map': use the EOSC Marketplace
     from: 'page_map' # or 'source'
 
-# Calculate source's metrics (pre-metrics)
-Metrics: true
-
-
-
 

environment.yml

@@ -44,12 +44,11 @@ dependencies:
     - pytz==2022.1
     - PyYAML==6.0
     - requests==2.27.1
-    - scikit-surprise==1.1.1
     - scipy==1.8.0
     - six==1.16.0
     - soupsieve==2.3.2
-    - surprise==0.1
     - urllib3==1.26.9
     - Werkzeug==2.1.2
     - zipp==3.8.0
     - flask-pymongo==2.3.0
+    - pymongoarrow==0.6.2