From 0c037acbce71d2048049bd56808d11d92e4b4334 Mon Sep 17 00:00:00 2001 From: MrXXX <2998480931@qq.com> Date: Fri, 11 Oct 2024 11:14:36 +0800 Subject: [PATCH] no more showing figures in table to save time, delete stucked experiments, add config file to cache db path --- pyerm/database/experiment.py | 6 +-- pyerm/database/tables.py | 10 ++--- pyerm/database/utils.py | 16 +++++++- pyerm/webUI/__init__.py | 3 ++ pyerm/webUI/app.py | 27 +++++++++++-- pyerm/webUI/home.py | 16 ++++++-- pyerm/webUI/tables.py | 76 ++++++++++++++++++++---------------- setup.py | 4 +- 8 files changed, 105 insertions(+), 53 deletions(-) diff --git a/pyerm/database/experiment.py b/pyerm/database/experiment.py index 18caeb5..682a42a 100644 --- a/pyerm/database/experiment.py +++ b/pyerm/database/experiment.py @@ -20,7 +20,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -# Version: 0.2.4 +# Version: 0.2.7 import os import typing @@ -32,9 +32,9 @@ from .dbbase import Database from .tables import ExperimentTable, MethodTable, ResultTable, DetailTable, DataTable +PYERM_HOME = os.path.join(os.path.expanduser('~'), '.pyerm') __all__ = ['Experiment'] -USER_HOME = os.path.expanduser('~') class Experiment: """ @@ -75,7 +75,7 @@ class Experiment: """ def __init__(self, db_path:str=None): if db_path is None: - db_path = os.path.join(USER_HOME, 'experiment.db') + db_path = os.path.join(PYERM_HOME, 'experiment.db') self._db = Database(db_path) self.experiment_table = ExperimentTable(self._db) self.parameter_table = None diff --git a/pyerm/database/tables.py b/pyerm/database/tables.py index ecffbd5..e64bc1b 100644 --- a/pyerm/database/tables.py +++ b/pyerm/database/tables.py @@ -20,7 +20,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -# Version: 0.2.6 +# Version: 0.2.7 from PIL import Image from io import BytesIO @@ -72,10 +72,10 @@ def experiment_failed(self, experiment_id:int, error_info:str=None, end_time:flo end_time = time() end_time = localtime(end_time) end_time = strftime("%Y-%m-%d %H:%M:%S", end_time) - print(error_info) + # print(error_info) if error_info is None: error_info = traceback.format_exc() - print(error_info) + # print(error_info) super().update(f"id={experiment_id}", end_time=strftime(end_time), status='failed', failed_reason=error_info) def get_experiment(self, experiment_id:int) -> dict: @@ -105,7 +105,7 @@ def insert(self, **kwargs): query = f"SELECT data_id FROM {self.table_name} WHERE {condition}" id_list = self.db.cursor.execute(query, values).fetchall() - print(kwargs) + # print(kwargs) if id_list == []: return super().insert(**kwargs) @@ -134,7 +134,7 @@ def insert(self, **kwargs): query = f"SELECT method_id FROM {self.table_name} WHERE {condition}" id_list = self.db.cursor.execute(query, values).fetchall() - print(kwargs) + # print(kwargs) if id_list == []: return super().insert(**kwargs) else: diff --git a/pyerm/database/utils.py b/pyerm/database/utils.py index a06a1d8..c58528c 100644 --- a/pyerm/database/utils.py +++ b/pyerm/database/utils.py @@ -20,7 +20,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -# Version: 0.2.6 +# Version: 0.2.7 + +from time import time from .dbbase import Database @@ -32,4 +34,14 @@ def delete_failed_experiments(db:Database): task = experiment[1] task_table = db[f"result_{task}"] task_table.delete(f"experiment_id={experiment_id}") - experiment_table.delete(f"id={experiment_id}") \ No newline at end of file + experiment_table.delete(f"id={experiment_id}") + + # # delete stuck running experiments that have been running for more than 24 hours + # running_experiments = experiment_table.select('id', 'task', 'start_time', where="status='running'") + # for experiment in running_experiments: + # experiment_id = experiment[0] + # task = experiment[1] + # if time() - experiment[2] > 86400: + # task_table = db[f"result_{task}"] + # task_table.delete(f"experiment_id={experiment_id}") + # experiment_table.delete(f"id={experiment_id}") \ No newline at end of file diff --git a/pyerm/webUI/__init__.py b/pyerm/webUI/__init__.py index 12793a4..69e6012 100644 --- a/pyerm/webUI/__init__.py +++ b/pyerm/webUI/__init__.py @@ -21,3 +21,6 @@ # SOFTWARE. # Version: 0.2.4 +import os + +PYERM_HOME = os.path.join(os.path.expanduser('~'), '.pyerm') \ No newline at end of file diff --git a/pyerm/webUI/app.py b/pyerm/webUI/app.py index ff4a052..b9c57fe 100644 --- a/pyerm/webUI/app.py +++ b/pyerm/webUI/app.py @@ -20,19 +20,34 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -# Version: 0.2.4 +# Version: 0.2.7 import streamlit as st import os +import configparser from home import home from tables import tables -USER_HOME = os.path.expanduser('~') +from pyerm.webUI import PYERM_HOME + def init(): + config = configparser.ConfigParser() + if not os.path.exists(PYERM_HOME): + os.makedirs(PYERM_HOME) + config['DEFAULT']['db_path'] = os.path.join(PYERM_HOME, 'experiment.db') + else: + config.read(os.path.join(PYERM_HOME, 'config.ini')) + + cache_dir = os.path.join(PYERM_HOME, '.cache') + if not os.path.exists(cache_dir): + os.makedirs(cache_dir) + if 'db_path' not in st.session_state: - st.session_state.db_path = os.path.join(USER_HOME, 'experiment.db') + st.session_state.db_path = config.get('DEFAULT', 'db_path') + else: + config.set('DEFAULT', 'db_path', st.session_state.db_path) if 'table_name' not in st.session_state: st.session_state.table_name = None if 'sql' not in st.session_state: @@ -41,6 +56,12 @@ def init(): st.session_state.zip = None if 'last_version' not in st.session_state: st.session_state.last_version = None + if 'selected_row' not in st.session_state: + st.session_state.selected_row = None + + with open(os.path.join(PYERM_HOME, 'config.ini'), 'w') as f: + config.write(f) + def main(): diff --git a/pyerm/webUI/home.py b/pyerm/webUI/home.py index 2f7e860..32cb645 100644 --- a/pyerm/webUI/home.py +++ b/pyerm/webUI/home.py @@ -20,7 +20,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -# Version: 0.2.6 +# Version: 0.2.7 import streamlit as st import os @@ -28,8 +28,7 @@ from importlib.metadata import version from pyerm.database.dbbase import Database - -USER_HOME = os.path.expanduser("~") +from pyerm.webUI import PYERM_HOME REPO_URL = "https://github.com/Mr-SGXXX/pyerm" @@ -42,6 +41,7 @@ def home(): download_zip() if st.checkbox('Download raw db file'): download_db() + clean_cache() def title(): @@ -68,7 +68,7 @@ def load_db(): st.write(f"Database not found. Please input the correct path.") def export_data(): - output_dir_path = f"{USER_HOME}/.tmp" + output_dir_path = f"{PYERM_HOME}/.tmp" if not os.path.exists(output_dir_path): os.makedirs(output_dir_path) db_name = os.path.basename(st.session_state.db_path) @@ -105,3 +105,11 @@ def download_db(): ) +def clean_cache(): + st.write('## Clean Cache') + st.write('Cache is used to store temporary files, such as result images.') + if st.checkbox('I want to clean cache'): + st.write('This will delete the cache folder and all its contents, which cannot be undone.') + if st.button('Confirm'): + subprocess.run(["rm", "-rf", os.path.join(PYERM_HOME, '.cache')]) + st.write('Cache cleaned successfully.') \ No newline at end of file diff --git a/pyerm/webUI/tables.py b/pyerm/webUI/tables.py index 1ac37f5..4074420 100644 --- a/pyerm/webUI/tables.py +++ b/pyerm/webUI/tables.py @@ -20,7 +20,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -# Version: 0.2.6 +# Version: 0.2.7 import pandas as pd from PIL import Image @@ -29,10 +29,11 @@ import streamlit as st import os import re -import tempfile +from time import strftime, gmtime from pyerm.database.utils import delete_failed_experiments from pyerm.database.dbbase import Database +from pyerm.webUI import PYERM_HOME def tables(): title() @@ -52,23 +53,27 @@ def detect_tables(): st.session_state.table_name = table_name def select_tables(): - def image_to_base64(img): - buffered = BytesIO(img) - img_str = base64.b64encode(buffered.getvalue()).decode() - return img_str + # def image_to_base64(img, desc): + # buffered = BytesIO(img) + # img_str = base64.b64encode(buffered.getvalue()).decode() + # return img_str + + # def make_image_clickable(image_name, image, desc): + # img_str = image_to_base64(image, desc) + # if img_str: + # return f'' + # else: + # return f'' - def make_image_clickable(image_name, image): - img_str = image_to_base64(image) - return f'' - def fold_detail_row(row, col_name): if row[col_name]: - return f'
Details{row[col_name]}
' + detail = row[col_name].replace("\n", "
") + return f'
Details{detail}
' else: return 'None' db = Database(st.session_state.db_path, output_info=False) - table_name = st.session_state.table_name + table_name:str = st.session_state.table_name if st.session_state.sql is not None: try: table_name = st.session_state.table_name @@ -83,37 +88,40 @@ def fold_detail_row(row, col_name): columns = [column[0] for column in db.cursor.description] df = pd.DataFrame(data, columns=columns) - # special process for image columns - columns_keep = [col for col in df.columns if not col.startswith("image_")] - pattern = re.compile(r'image_(\d+)') - max_image_num = -1 - for name in df.columns: - match = pattern.match(name) - if match: - max_image_num = max(max_image_num, int(match.group(1))) - for i in range(max_image_num+1): - if f'image_{i}' in df.columns and not df[f'image_{i}_name'].isnull().all(): - df[f'image_{i}'] = df.apply(lambda x: make_image_clickable(x[f'image_{i}_name'], x[f'image_{i}']), axis=1) - columns_keep.append(f'image_{i}') - - if "failed_reason" in df.columns: - df['failed_reason'] = df.apply(lambda x: fold_detail_row(x, 'failed_reason'), axis=1) - df = df[columns_keep] - if st.button('Refresh'): + if st.button('Refresh', key='refresh1'): st.session_state.table_name = table_name st.write('## Table:', table_name) + + columns_keep = [col for col in df.columns if not col.startswith("image_")] + if table_name == 'experiment_list': - if st.checkbox('Delete all failed records'): + if st.checkbox('Delete all failed and stuck records'): st.write('**Warning: This operation will delete all failed records and their results, which cannot be undone.**') if st.button(f"Confirm"): delete_failed_experiments(db) st.session_state.table_name = table_name - st.write(df.to_html(escape=False, columns=columns_keep), unsafe_allow_html=True) - - # st.dataframe(df[columns_keep]) - + df['failed_reason'] = df.apply(lambda x: fold_detail_row(x, 'failed_reason'), axis=1) + df['useful_time_cost'] = df['useful_time_cost'].apply(lambda x: strftime('%H:%M:%S', gmtime(x)) if not pd.isnull(x) else x) + df['total_time_cost'] = df['total_time_cost'].apply(lambda x: strftime('%H:%M:%S', gmtime(x)) if not pd.isnull(x) else x) + # elif table_name.startswith("result_"): + # # special process for image columns + # pattern = re.compile(r'image_(\d+)') + # max_image_num = -1 + # for name in df.columns: + # match = pattern.match(name) + # if match: + # max_image_num = max(max_image_num, int(match.group(1))) + + # for i in range(max_image_num+1): + # if f'image_{i}' in df.columns and not df[f'image_{i}_name'].isnull().all(): + # df[f'image_{i}'] = df.apply(lambda x: make_image_clickable(x[f'image_{i}_name'], x[f'image_{i}'], desc=f"{x[f'experiment_id']}_{i}"), axis=1) + # columns_keep.append(f'image_{i}') + df = df[columns_keep] + st.write(df.to_html(escape=False, columns=columns_keep), unsafe_allow_html=True) + if st.button('Refresh', key='refresh2'): + st.session_state.table_name = table_name def input_sql(): st.sidebar.write('You can also set the columns and condition for construct a select SQL sentense for the current table here.') diff --git a/setup.py b/setup.py index 3b8b337..c9d74a1 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -# Version: 0.2.6 +# Version: 0.2.7 from setuptools import setup, find_packages with open("README.md", "r", encoding="utf-8") as f: @@ -28,7 +28,7 @@ setup( name='pyerm', - version='0.2.6', + version='0.2.7', author='Yuxuan Shao', author_email='yx_shao@qq.com', description='This project is an local experiment record manager for python based on SQLite DMS, suitable for recording experiment and analysing experiment data with a web UI, which can help you efficiently save your experiment settings and results for later analysis.',