dvc.yaml

vars:
  - dataset:
      year: '2019'
  - mlflow:
      experiment_name: 'final-presentation'
  - gpu:
      CUDA_VISIBLE_DEVICES: 0

stages:
  merge_lcfa_2019:
    cmd: 
      - julia scripts/dataset/merge_lcfa.jl ../GLM-L2-LCFA/2019
    deps:
      - scripts/dataset/merge_lcfa.jl
      - src/dataset/l2_lcfa_merge.jl
      - src/structs/FlashRecords.jl
    outs:
      - data/exp_pro/GLM-L2-LCFA/2019.jld2:
          persist: true
          cache: true

  merge_lcfa_2020:
    cmd: 
      - julia scripts/dataset/merge_lcfa.jl ../GLM-L2-LCFA/2020
    deps:
      - scripts/dataset/merge_lcfa.jl
      - src/dataset/l2_lcfa_merge.jl
      - src/structs/FlashRecords.jl
    outs:
      - data/exp_pro/GLM-L2-LCFA/2020.jld2:
          persist: true
          cache: true

  merge_lcfa_2021:
    cmd: 
      - julia scripts/dataset/merge_lcfa.jl ../GLM-L2-LCFA/2021
    deps:
      - scripts/dataset/merge_lcfa.jl
      - src/dataset/l2_lcfa_merge.jl
      - src/structs/FlashRecords.jl
    outs:
      - data/exp_pro/GLM-L2-LCFA/2021.jld2:
          persist: true
          cache: true

  grid_dataset:
    cmd:
      - julia scripts/dataset/grid_fed.jl -s ${ds.resolution.spatial} -t ${ds.resolution.temporal} -f data/exp_pro/GLM-L2-LCFA/${dataset.year}.jld2
    deps:
      - scripts/dataset/grid_fed.jl
      - src/dataset/fed_grid.jl
      - src/dataset/l2_lcfa_merge.jl
      - data/exp_pro/GLM-L2-LCFA/2019.jld2
      - data/exp_pro/GLM-L2-LCFA/2020.jld2
      - data/exp_pro/GLM-L2-LCFA/2021.jld2
    outs:
      - data/exp_pro/GLM-L2-LCFA-GRID/spatial=${ds.resolution.spatial}_temporal=${ds.resolution.temporal}_year=${dataset.year}:
          persist: true
          cache: true
    params:
      - ds.resolution

  cluster_dataset:
    cmd:
      - julia scripts/dataset/cluster_grid.jl
        --folder=data/exp_pro/GLM-L2-LCFA-GRID/spatial=${ds.resolution.spatial}_temporal=${ds.resolution.temporal}_year=${dataset.year}
        --threshold=${ds.cluster.threshold}
        --binary
        --radius=${ds.cluster.radius}
        --time_scale=${ds.cluster.t_scale}
        --windows=${ds.cluster.windows}
        --min_neighbors=${ds.cluster.min_neighbors}
        --min_cluster_size=${ds.cluster.min_cluster_size}
        --dimensions=${ds.cluster.dimensions}
        --padding=${ds.cluster.padding}
        --single-file=dataset.h5
    deps:
      - scripts/dataset/cluster_grid.jl
      - src/dataset/fed_grid.jl
      - src/dataset/cluster_dbscan.jl
      - data/exp_pro/GLM-L2-LCFA-GRID/spatial=${ds.resolution.spatial}_temporal=${ds.resolution.temporal}_year=${dataset.year}
    outs:
      - data/training/dataset.h5:
          persist: true
          cache: true
    params:
      - ds.resolution
      - ds.cluster

  train:
    cmd:
      - export $(grep -v '^#' .env | xargs) &&
        export MLFLOW_EXPERIMENT_NAME=${mlflow.experiment_name} &&
        export CUDA_VISIBLE_DEVICES=${gpu.CUDA_VISIBLE_DEVICES} &&
        git add . && 
        git commit -m "automatic commit" &&
        julia scripts/training/train.jl
        --dataset=type=${train.ds.type}_splitratio=${train.ds.splitratio}_batchsize=${train.ds.batchsize}_N=${train.ds.N}
        --dataset_path=data/training/dataset.h5
        --architecture=type=${train.arch.type}_out=${train.arch.out}_dropout=${train.arch.dropout}
        --optimiser="${train.optimiser}"
        --epochs=${train.epochs}
        --loss=${train.loss}
        --metrics=mse,csi,binarycrossentropy
        --device=gpu
        --early_stop=10
        --plateau=10
        --throttle=120

    deps:
      - scripts/training/train.jl
      - src/dataset/${train.ds.type}.jl
      - src/architecture/${train.arch.type}.jl
      - src/optimisers/optimiser.jl
      - src/utils/logging.jl
      - src/training/train.jl
      - src/evaluation/loss.jl
      - data/training/dataset.h5
    outs:
      - data/experiments/${mlflow.experiment_name}:
          persist: true
    params:
      - train