cd.yml

name: CD
on: workflow_dispatch
#  schedule:
#    - cron: '0 * * * *'

defaults:
  run:
    working-directory: .

jobs:
  Scrape:
    runs-on: windows-latest
    steps:
      - uses: actions/checkout@v2

      - uses: szenius/set-timezone@v1.0
        with:
          timezoneLinux: "Asia/Singapore"
          timezoneMacos: "Asia/Singapore"
          timezoneWindows: "Singapore Standard Time"

      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.6

      - name: Install dependencies
        run: |
            python -m pip install --upgrade pip
            pip install -r requirements.txt

      - name: Scrape for new data and append to file
        run: |
            echo "Dynamically get current datetime - 2 hour and save it in `$since variable"
            $since = [DateTime]::Now.AddHours(-2).ToString('yyyy-MM-dd HH:mm:ss')
            echo "Scrape since $since and save scraped output to `$output variable"
            echo "The output should be appended to the same file"
            $output = twint -s \#dhl --since $since --hashtags --stats -l en -o new_data_appended.txt
            echo "Printing out the output to console"
            $output

      - name: Commit files
        run: |
          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git config --local user.name "github-actions[bot]"
          git pull
          git add .
          git commit -m "CD - Scrape new data $(date)"
          git push --set-upstream origin dev
        env:
          REPO_KEY: ${{secrets.GITHUB_TOKEN}}
          username: github-actions
  
  Clean:
    needs: Scrape
    runs-on: windows-latest
    steps:
      - uses: actions/checkout@v2

      - uses: szenius/set-timezone@v1.0
        with:
          timezoneLinux: "Asia/Singapore"
          timezoneMacos: "Asia/Singapore"
          timezoneWindows: "Singapore Standard Time"

      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.6

      - name: Install dependencies
        run: |
            python -m pip install --upgrade pip
            pip install -r requirements.txt
            
      - name: Setup Cloud SDK
        uses: google-github-actions/setup-gcloud@v0.2.0
        with:
          service_account_key: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
          service_account_email: ${{ secrets.GCP_COMPUTE_ENGINE_EMAIL  }}
          project_id: ${{ secrets.GCP_PROJECT_ID }}
          export_default_credentials: true

      - name: Run clean.py
        env:
          MLFLOW_TRACKING_URI: ${{ secrets.MLFLOW_TRACKING_URI }}
          MLFLOW_TRACKING_USERNAME: ${{ secrets.MLFLOW_TRACKING_USERNAME }}
          MLFLOW_TRACKING_PASSWORD: ${{ secrets.MLFLOW_TRACKING_PASSWORD }}
        run: |
            python pipeline/clean.py

      - name: Commit files
        run: |
          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git config --local user.name "github-actions[bot]"
          git pull
          git add new_data_cleaned.csv
          git commit -m "CD - Clean new data $(date)"
          git push --set-upstream origin dev
        env:
          REPO_KEY: ${{secrets.GITHUB_TOKEN}}
          username: github-actions
          
  Append:
    needs: Clean
    runs-on: windows-latest
    steps:
      - uses: actions/checkout@v2

      - uses: szenius/set-timezone@v1.0
        with:
          timezoneLinux: "Asia/Singapore"
          timezoneMacos: "Asia/Singapore"
          timezoneWindows: "Singapore Standard Time"

      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.6

      - name: Install dependencies
        run: |
            python -m pip install --upgrade pip
            pip install -r requirements.txt

      - name: Run append.py
        run: |
            python pipeline/append.py

      - name: Remove fasttext model
        uses: JesseTG/rm@v1.0.0
        with:
          path: ./lid.176.ftz
          
      - name: Remove model.pickle
        uses: JesseTG/rm@v1.0.0
        with:
          path: ./model.pickle
          
      - name: Commit files
        run: |
          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git config --local user.name "github-actions[bot]"
          git pull
          git add .
          git commit -m "CD - Append new data $(date)"
          git push --set-upstream origin dev
        env:
          REPO_KEY: ${{secrets.GITHUB_TOKEN}}
          username: github-actions
  
  Update_data:
    needs: Append
    runs-on: windows-latest
    
    steps:
      - uses: actions/checkout@v2

      - uses: szenius/set-timezone@v1.0
        with:
          timezoneLinux: "Asia/Singapore"
          timezoneMacos: "Asia/Singapore"
          timezoneWindows: "Singapore Standard Time"

      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.6

      - name: Install dependencies
        run: |
            python -m pip install --upgrade pip
            pip install -r requirements.txt

      - name: Run update.py
        env:
          GOOGLEDRIVE_ACCESS_TOKEN: ${{ secrets.GOOGLEDRIVE_ACCESS_TOKEN }}
          GOOGLEDRIVE_CLIENT_ID: ${{ secrets.GOOGLEDRIVE_CLIENT_ID }}
          GOOGLEDRIVE_CLIENT_SECRET: ${{ secrets.GOOGLEDRIVE_CLIENT_SECRET }}
          GOOGLEDRIVE_REFRESH_TOKEN: ${{ secrets.GOOGLEDRIVE_REFRESH_TOKEN }}
          UPDATE_FILENAME: ${{ 'data.txt' }}
        run: |
            python pipeline/update.py

      - name: Commit files
        run: |
          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git config --local user.name "github-actions[bot]"
          git pull
          git add .
          git commit -m "CD - Update data on GDrive $(date)"
          git push --set-upstream origin dev
        env:
          REPO_KEY: ${{secrets.GITHUB_TOKEN}}
          username: github-actions