move download path to under mnt

Signed-off-by: Sunyanan Choochotkaew <sunyanan.choochotkaew1@ibm.com>

cmd/main.py

@@ -13,9 +13,6 @@
 UTC_OFFSET_TIMEDELTA = datetime.datetime.utcnow() - datetime.datetime.now()
 data_path = os.getenv("CPE_DATAPATH", data_path)
 
-# set model top path to data path
-os.environ['MODEL_PATH'] = data_path
-
 cur_path = os.path.join(os.path.dirname(__file__), '.')
 sys.path.append(cur_path)
 src_path = os.path.join(os.path.dirname(__file__), '..', 'src')
@@ -749,6 +746,9 @@ def export(args):
 
 
 if __name__ == "__main__":
+    # set model top path to data path
+    os.environ['MODEL_PATH'] = data_path
+
     # Create an ArgumentParser object
     parser = argparse.ArgumentParser(description="Kepler model server entrypoint")
     parser.add_argument("command", type=str, help="The command to execute.")

manifests/base/patch/patch-estimator-sidecar.yaml

@@ -37,6 +37,10 @@ spec:
           readOnly: true
         - mountPath: /tmp
           name: tmp
+        - mountPath: /mnt
+          name: mnt
       volumes:
       - emptyDir: {}
-        name: tmp
+        name: tmp
+      - emptyDir: {}
+        name: mnt

manifests/offline-trainer/offline-trainer.yaml

@@ -1,11 +1,3 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kepler-model-server-cfm
-  namespace: system
-data:
-  MODEL_PATH: /data/models
----
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -31,7 +23,7 @@ spec:
         configMap:
           name: kepler-model-server-cfm
       - emptyDir: {}
-        name: model-data
+        name: mnt
       containers:
       - name: offline-trainer
         image: kepler_model_server
@@ -43,8 +35,8 @@ spec:
           - name: cfm
             mountPath: /etc/kepler/kepler.config
             readOnly: true
-          - name: model-data
-            mountPath: /data
+          - name: mnt
+            mountPath: /mnt
             readOnly: false
         command: [ "python3.8" ]
         args: ["-u", "src/train/offline_trainer.py" ]

manifests/server/online-train/patch-trainer.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: kepler-model-server-cfm
-  namespace: system
+  namespace: kepler
 data:
   PROM_SERVER: 'http://prometheus-k8s.monitoring.svc.cluster.local:9090'
   PROM_QUERY_INTERVAL: '20'
@@ -14,7 +14,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: kepler-model-server
-  namespace: system
+  namespace: kepler
 spec:
   template:
     spec:
@@ -27,8 +27,8 @@ spec:
           - name: cfm
             mountPath: /etc/kepler/kepler.config
             readOnly: true
-          - name: model-data
-            mountPath: /data
+          - name: mnt
+            mountPath: /mnt
             readOnly: false
         command: [ "python3.8" ]
         args: ["-u", "src/train/online_trainer.py" ]

manifests/server/server.yaml

@@ -3,8 +3,6 @@ kind: ConfigMap
 metadata:
   name: kepler-model-server-cfm
   namespace: system
-data:
-  MODEL_PATH: /data/models
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -31,7 +29,7 @@ spec:
         configMap:
           name: kepler-model-server-cfm
       - emptyDir: {}
-        name: model-data
+        name: mnt
       containers:
       - name: server-api
         image: kepler_model_server
@@ -43,8 +41,8 @@ spec:
           - name: cfm
             mountPath: /etc/kepler/kepler.config
             readOnly: true
-          - name: model-data
-            mountPath: /data
+          - name: mnt
+            mountPath: /mnt
             readOnly: false
         command: [ "python3.8" ]
         args: ["-u", "src/server/model_server.py" ]

src/estimate/estimator.py

@@ -37,7 +37,7 @@ def __init__(self, metrics, values, output_type, source, system_features, system
 from archived_model import get_achived_model
 from model import load_downloaded_model
 from loader import get_download_output_path
-from config import set_env_from_model_config, SERVE_SOCKET
+from config import set_env_from_model_config, SERVE_SOCKET, download_path
 from train_types import is_support_output_type
 
 loaded_model = dict()
@@ -56,7 +56,7 @@ def handle_request(data):
     output_type = ModelOutputType[power_request.output_type]
 
     if output_type.name not in loaded_model:
-        output_path = get_download_output_path(power_request.energy_source, output_type)
+        output_path = get_download_output_path(download_path, power_request.energy_source, output_type)
         if not os.path.exists(output_path):
             # try connecting to model server
             output_path = make_request(power_request)

src/estimate/model/model.py

@@ -11,6 +11,7 @@
 sys.path.append(cur_path)
 
 from loader import load_metadata, get_download_output_path
+from config import download_path
 from prom_types import TIMESTAMP_COL, valid_container_query
 
 from scikit_model import ScikitModel
@@ -132,5 +133,5 @@ def load_model(model_path):
 
 # download model folder has no subfolder of energy source and feature group because it has been already determined by model request
 def load_downloaded_model(energy_source, output_type):
-    model_path = get_download_output_path(energy_source, output_type)
+    model_path = get_download_output_path(download_path, energy_source, output_type)
     return load_model(model_path)

src/estimate/model_server_connector.py

@@ -9,8 +9,8 @@
 util_path = os.path.join(os.path.dirname(__file__), '..', 'util')
 sys.path.append(util_path)
 
-from config import is_model_server_enabled, get_model_server_req_endpoint, get_model_server_list_endpoint
-from loader import get_download_path, get_download_output_path
+from config import is_model_server_enabled, get_model_server_req_endpoint, get_model_server_list_endpoint, download_path
+from loader import get_download_output_path
 from train_types import ModelOutputType
 
 def make_model_request(power_request):
@@ -19,8 +19,8 @@ def make_model_request(power_request):
 TMP_FILE = 'tmp.zip'
 
 def unpack(energy_source, output_type, response, replace=True):
-    output_path = get_download_output_path(energy_source, output_type)
-    tmp_filepath = os.path.join(get_download_path(), TMP_FILE)
+    output_path = get_download_output_path(download_path, energy_source, output_type)
+    tmp_filepath = os.path.join(download_path, TMP_FILE)
     if os.path.exists(output_path):
         if not replace:
             # delete downloaded file

src/server/model_server.py

@@ -13,8 +13,8 @@
 sys.path.append(util_path)
 
 from util.train_types import get_valid_feature_groups, ModelOutputType, FeatureGroups, FeatureGroup
-from util.config import getConfig, model_toppath, ERROR_KEY, MODEL_SERVER_MODEL_REQ_PATH, MODEL_SERVER_MODEL_LIST_PATH, initial_pipeline_url
-from util.loader import parse_filters, is_valid_model, load_json, load_weight, get_model_group_path, get_archived_file, METADATA_FILENAME, CHECKPOINT_FOLDERNAME, get_pipeline_path, any_node_type, is_matched_type
+from util.config import getConfig, model_toppath, ERROR_KEY, MODEL_SERVER_MODEL_REQ_PATH, MODEL_SERVER_MODEL_LIST_PATH, initial_pipeline_url, download_path
+from util.loader import parse_filters, is_valid_model, load_json, load_weight, get_model_group_path, get_archived_file, METADATA_FILENAME, CHECKPOINT_FOLDERNAME, get_pipeline_path
 
 ###############################################
 # model request 
@@ -179,15 +179,16 @@ def load_init_pipeline():
     # unpack pipeline
     try:
         TMP_FILE = 'tmp.zip'
-        with codecs.open(TMP_FILE, 'wb') as f:
+        tmp_filepath = os.path.join(download_path, TMP_FILE)
+        with codecs.open(tmp_filepath, 'wb') as f:
             f.write(response.content)
-        shutil.unpack_archive(TMP_FILE, default_pipeline)
-    except:
-        print("failed to unpack downloaded pipeline.")
+        shutil.unpack_archive(tmp_filepath, default_pipeline)
+    except Exception as e:
+        print("failed to unpack downloaded pipeline: ", e)
         return
 
     # remove downloaded zip
-    os.remove(TMP_FILE)
+    os.remove(tmp_filepath)
     print("initial pipeline is loaded to {}".format(default_pipeline))
 
 if __name__ == '__main__':

src/util/config.py

@@ -21,6 +21,9 @@
 # can be read only (for configmap mount)
 CONFIG_PATH = "/etc/kepler/kepler.config"
 
+DOWNLOAD_FOLDERNAME = "download"
+MODEL_FOLDERNAME = "models"
+
 DEFAULT_TOTAL_SOURCE = "acpi"
 DEFAULT_COMPONENTS_SOURCE = "rapl"
 TOTAL_KEY = "TOTAL"
@@ -62,11 +65,8 @@ def getPath(subpath):
 
 initial_pipeline_url = getConfig('INITIAL_PIPELINE_URL', get_pipeline_url())
 
-default_model_path = os.path.join(os.path.dirname(__file__), '..', 'models')
-model_toppath =  getPath(getConfig('MODEL_PATH', default_model_path))
-
-if not os.path.exists(model_toppath):
-    os.mkdir(model_toppath)
+model_toppath =  getConfig('MODEL_PATH', getPath(MODEL_FOLDERNAME))
+download_path = getConfig('MODEL_PATH', getPath(DOWNLOAD_FOLDERNAME))
 
 ERROR_KEY = 'mae'
 ERROR_KEY = getConfig('ERROR_KEY', ERROR_KEY)

src/util/loader.py

@@ -15,7 +15,6 @@
 
 DEFAULT_PIPELINE = 'default'
 CHECKPOINT_FOLDERNAME = 'checkpoint'
-DOWNLOAD_FOLDERNAME = 'download'
 PREPROCESS_FOLDERNAME = "preprocessed_data"
 
 default_init_model_url = "https://raw.githubusercontent.com/sustainable-computing-io/kepler-model-db/main/models/"
@@ -233,12 +232,8 @@ def load_pipeline_metadata(pipeline_path, energy_source, model_type):
     pipeline_metadata_filename = _pipeline_model_metadata_filename(energy_source, model_type)
     return load_csv(pipeline_path, pipeline_metadata_filename)
 
-def get_download_path():
-    path = os.path.join(os.path.dirname(__file__), DOWNLOAD_FOLDERNAME)
-    return assure_path(path)
-
-def get_download_output_path(energy_source, output_type):
-    energy_source_path = assure_path(os.path.join(get_download_path(), energy_source))
+def get_download_output_path(download_path, energy_source, output_type):
+    energy_source_path = assure_path(os.path.join(download_path, energy_source))
     return os.path.join(energy_source_path, output_type.name)
 
 def get_url(output_type, feature_group=default_feature_group, trainer_name=default_trainer_name, node_type=default_node_type, model_topurl=default_init_model_url, energy_source="rapl", pipeline_name=default_init_pipeline_name):

tests/README.md

@@ -153,7 +153,7 @@ Optional arguments:
 
  Run:
  ```
- python estimate_mode_test.py
+ python estimator_model_test.py
  ```
 
  Reuse (test_model):

tests/estimator_model_request_test.py

@@ -33,7 +33,7 @@
 from estimate.estimator import handle_request, loaded_model, PowerRequest
 from estimate.model_server_connector import list_all_models
 from estimate.archived_model import get_achived_model, reset_failed_list
-from config import get_init_model_url, set_env_from_model_config, get_url
+from config import get_init_model_url, set_env_from_model_config, get_url, download_path
 from extractor_test import test_energy_source
 
 from estimator_power_request_test import generate_request
@@ -50,7 +50,7 @@
     print("Available Models:", available_models)
     for output_type_name, valid_fgs in available_models.items():
         output_type = ModelOutputType[output_type_name]
-        output_path = get_download_output_path(energy_source, output_type)
+        output_path = get_download_output_path(download_path, energy_source, output_type)
         for fg_name, best_model in valid_fgs.items():
             if os.path.exists(output_path):
                 shutil.rmtree(output_path)
@@ -73,7 +73,7 @@
             print("Download: ", url)
             response = requests.get(url)
             if response.status_code == 200:
-                output_path = get_download_output_path(energy_source, output_type)
+                output_path = get_download_output_path(download_path, energy_source, output_type)
                 if output_type_name in loaded_model:
                     del loaded_model[output_type_name]
                 if os.path.exists(output_path):
@@ -93,7 +93,7 @@
     # test getting model from archived
     os.environ['MODEL_SERVER_ENABLE'] = "false"
     output_type = ModelOutputType[output_type_name]
-    output_path = get_download_output_path(energy_source, output_type)
+    output_path = get_download_output_path(download_path, energy_source, output_type)
     if output_type_name in loaded_model:
         del loaded_model[output_type_name]
     if os.path.exists(output_path):
@@ -121,7 +121,7 @@
     reset_failed_list()
     if output_type_name in loaded_model:
         del loaded_model[output_type_name]
-    output_path = get_download_output_path(energy_source, output_type)
+    output_path = get_download_output_path(download_path, energy_source, output_type)
     if os.path.exists(output_path):
         shutil.rmtree(output_path)
     request_json = generate_request(None, n=10, metrics=FeatureGroups[FeatureGroup.KubeletOnly], output_type=output_type_name)

tests/model_server_test.py

@@ -17,13 +17,12 @@
 
 from train_types import FeatureGroup, FeatureGroups, ModelOutputType
 from model_server import MODEL_SERVER_PORT
+from config import download_path
 
 def get_model_request_json(metrics, output_type, node_type, weight, trainer_name, energy_source):
     return {"metrics": metrics, "output_type": output_type.name, "node_type": node_type, "weight": weight, "trainer_name": trainer_name, "source": energy_source}
 
 TMP_FILE = 'download.zip'
-DOWNLOAD_FOLDER = 'download'
-download_path = os.path.join(os.path.dirname(__file__), DOWNLOAD_FOLDER)
 
 def make_request(metrics, output_type, node_type=-1, weight=False, trainer_name="", energy_source='rapl'):
     model_request = get_model_request_json(metrics, output_type, node_type, weight, trainer_name, energy_source)

tests/weight_model_request_test.py

@@ -24,7 +24,7 @@
 from train_types import FeatureGroups, FeatureGroup, ModelOutputType
 from loader import get_download_output_path
 from estimate.model_server_connector import list_all_models
-from config import get_model_server_req_endpoint
+from config import get_model_server_req_endpoint, download_path
 from extractor_test import test_energy_source
 from estimator_power_request_test import generate_request
 
@@ -50,7 +50,7 @@
 
     for output_type_name, valid_fgs in available_models.items():
         output_type = ModelOutputType[output_type_name]
-        output_path = get_download_output_path(energy_source, output_type)
+        output_path = get_download_output_path(download_path, energy_source, output_type)
         for fg_name, best_model in valid_fgs.items():
             for trainer in weight_available_trainers:
                 print("feature group: ", fg_name)