From 30e18ae7b08183ae99f9dad02b200220a7f75fc7 Mon Sep 17 00:00:00 2001 From: Mr_SGXXX <2998480931@qq.com> Date: Tue, 22 Oct 2024 23:36:59 +0800 Subject: [PATCH] display update, full sql support, config file cache position show&delete, other small fix --- pyerm/database/dbbase.py | 3 +- pyerm/webUI/__init__.py | 2 +- pyerm/webUI/app.py | 6 +- pyerm/webUI/details.py | 165 +++++++++++++++++++++++++++------------ pyerm/webUI/home.py | 43 +++++++--- pyerm/webUI/tables.py | 30 ++++--- setup.py | 6 +- 7 files changed, 177 insertions(+), 78 deletions(-) diff --git a/pyerm/database/dbbase.py b/pyerm/database/dbbase.py index 2463610..76fc90f 100644 --- a/pyerm/database/dbbase.py +++ b/pyerm/database/dbbase.py @@ -129,7 +129,8 @@ def add_column(self, column_name:str, column_definition:str) -> None: @property def columns(self): if self._column is None: - self._column = [column[1] for column in self.db.cursor.execute(f'PRAGMA table_info({self.table_name})').fetchall()] + self._column = self.db.cursor.execute(f"SELECT * FROM {self.table_name}").description + self._column = [column[0] for column in self._column] return self._column @property diff --git a/pyerm/webUI/__init__.py b/pyerm/webUI/__init__.py index 69e6012..eabf40f 100644 --- a/pyerm/webUI/__init__.py +++ b/pyerm/webUI/__init__.py @@ -23,4 +23,4 @@ # Version: 0.2.4 import os -PYERM_HOME = os.path.join(os.path.expanduser('~'), '.pyerm') \ No newline at end of file +PYERM_HOME = os.path.join(os.path.expanduser('~'), 'pyerm') \ No newline at end of file diff --git a/pyerm/webUI/app.py b/pyerm/webUI/app.py index 3cdee5d..1b9e5ec 100644 --- a/pyerm/webUI/app.py +++ b/pyerm/webUI/app.py @@ -40,10 +40,6 @@ def init(): config['DEFAULT']['db_path'] = os.path.join(PYERM_HOME, 'experiment.db') else: config.read(os.path.join(PYERM_HOME, 'config.ini')) - - cache_dir = os.path.join(PYERM_HOME, '.cache') - if not os.path.exists(cache_dir): - os.makedirs(cache_dir) if 'db_path' not in st.session_state: st.session_state.db_path = config.get('DEFAULT', 'db_path') @@ -61,6 +57,8 @@ def init(): st.session_state.selected_row = None if 'cur_detail_id' not in st.session_state: st.session_state.cur_detail_id = None + if 'cur_detail_img_id' not in st.session_state: + st.session_state.cur_detail_img_id = None with open(os.path.join(PYERM_HOME, 'config.ini'), 'w') as f: config.write(f) diff --git a/pyerm/webUI/details.py b/pyerm/webUI/details.py index d10a5a9..2cba078 100644 --- a/pyerm/webUI/details.py +++ b/pyerm/webUI/details.py @@ -22,101 +22,166 @@ # Version: 0.2.8 +import pandas as pd import streamlit as st import os +from PIL import Image +import re +import io from pyerm.database.dbbase import Database from pyerm.webUI import PYERM_HOME def details(): title() + if st.button('Refresh', key='refresh1'): + st.rerun() if os.path.exists(st.session_state.db_path) and st.session_state.db_path.endswith('.db'): db = Database(st.session_state.db_path, output_info=False) - experiment_list = db['experiment_list'] - max_id, min_id = experiment_list.select('MAX(id)', 'MIN(id)')[0] - if st.session_state.cur_detail_id is None: - detect_experiment_id(db) + experiment_table = db['experiment_list'] + max_id, min_id = experiment_table.select('MAX(id)', 'MIN(id)')[0] + if st.session_state.cur_detail_id is None: - st.write('No experiment found.') + cur_id = detect_experiment_id(db) + else: + cur_id = st.session_state.cur_detail_id + if cur_id is None: + st.write('No experiment found. Please make sure any experiment has been run.') + else: + st.sidebar.markdown('## Choose Experiment') + cur_id_last = cur_id + cur_id = int(st.sidebar.text_input('Experiment ID', key='input_exp_id', value=cur_id)) + st.session_state.cur_detail_id = cur_id + if cur_id_last != cur_id: + st.session_state.cur_detail_img_id = 0 + cols_sidebar = st.sidebar.columns(2) + cur_id_last = cur_id + with cols_sidebar[0]: + if st.button('Last\nExperiment', disabled=cur_id <= min_id): + cur_id = detect_experiment_id(db, False) + with cols_sidebar[1]: + if st.button('Next\nExperiment', disabled=cur_id >= max_id): + cur_id = detect_experiment_id(db) + if cur_id_last != cur_id: + st.rerun() + + + if experiment_table.select(where=f'id={cur_id}') == []: + st.markdown('## Experiment Information') + st.markdown(f'### Current Experiment ID: {cur_id}') + st.write(f'Experiment with id:{cur_id} does not exist. Please check the ID.') else: - cols_top = st.columns(2) - st.markdown(f'## Current Experiment ID: {st.session_state.cur_detail_id}') - with cols_top[0]: - if st.button('Previous', disabled=st.session_state.cur_detail_id == min_id): - detect_experiment_id(db, False) - with cols_top[1]: - if st.button('Next', disabled=st.session_state.cur_detail_id == max_id): - detect_experiment_id(db) st.markdown('## Experiment Information') + st.markdown(f'### Current Experiment ID: {cur_id}') basic_info, method_info, data_info, result_info = detect_experiment_info(db) - st.write('Basic Information:') - st.write(basic_info[0]) + st.write('---') + st.write('### Basic Information:') + st.write(basic_info) + st.write('---') + st.write('### Method Information:') if method_info is not None: - st.write('Method Information:') - st.write(method_info[0]) + st.write(method_info) + else: + st.write('No hyperparam setting used by current method.') + + st.write('---') + st.write('### Data Information:') if data_info is not None: - st.write('Data Information:') - st.write(data_info[0]) + st.write(data_info) + else: + st.write('No hyperparam setting used by current dataset.') + + st.write('---') + st.write('### Result Information:') if result_info is not None: - st.write('Result Information:') - st.write(result_info[0]) + result_info, image_dict = split_result_info(result_info) + st.write(result_info) + selected_img = st.selectbox('**Select Experiment Result Image**', list(image_dict.keys()), key='select_img') + if selected_img: + st.image(Image.open(io.BytesIO(image_dict[selected_img]))) + else: + st.write(f'No image detected in experiment with id:{cur_id}.') + else: + st.write('No result found. Please check the status of the experiment.') + st.write('---') st.markdown('## Delete Current Experiment') st.markdown('**Warning**: This operation will delete current experiment record and result, which is irreversible.') if st.checkbox('Delete Current Experiment'): if st.button('Confirm'): delete_current_experiment(db) - cols_bottom = st.columns(2) - - with cols_bottom[0]: - if st.button('Previous', disabled=st.session_state.cur_detail_id == min_id): - detect_experiment_id(db, False) - with cols_bottom[1]: - if st.button('Next', disabled=st.session_state.cur_detail_id == max_id): - detect_experiment_id(db) - def detect_experiment_id(db, next_flag=True): - experiment_list = db['experiment_list'] + experiment_table = db['experiment_list'] if st.session_state.cur_detail_id is None: - basic_info = experiment_list.select(other='ORDER BY id DESC LIMIT 1') + basic_info = experiment_table.select(other='ORDER BY id DESC LIMIT 1') if len(basic_info) == 0: return None - st.session_state.cur_detail_id = basic_info[0]['id'] + # st.write(basic_info) + st.session_state.cur_detail_id = basic_info[0][0] + st.session_state.cur_detail_img_id = 0 else: - basic_info = experiment_list.select(where=f'WHERE id{">" if next_flag else "<"}{st.session_state.cur_detail_id}', other='ORDER BY id ASC LIMIT 1') - st.session_state.cur_detail_id = basic_info[0]['id'] + basic_info = experiment_table.select(where=f'id{">" if next_flag else "<"}{st.session_state.cur_detail_id}', other=f'ORDER BY id {"ASC" if next_flag else "DESC"} LIMIT 1') + st.session_state.cur_detail_id = basic_info[0][0] + st.session_state.cur_detail_img_id = 0 + + return st.session_state.cur_detail_id def detect_experiment_info(db): - experiment_list = db['experiment_list'] - basic_info = experiment_list.select(where=f'WHERE id={st.session_state.cur_detail_id}') - method = basic_info[0]['method'] - method_id = basic_info[0]['method_id'] + experiment_table = db['experiment_list'] + basic_info = experiment_table.select(where=f'id={st.session_state.cur_detail_id}') + basic_columns = experiment_table.columns + method = basic_info[0][2] + method_id = basic_info[0][3] method_info = None - data = basic_info[0]['data'] - data_id = basic_info[0]['data_id'] + data = basic_info[0][4] + data_id = basic_info[0][5] data_info = None result_info = None - task = basic_info[0]['task'] + task = basic_info[0][6] if method_id != -1: method_table = db[f'method_{method}'] - method_info = method_table.select(where=f'WHERE id={method_id}') + method_info = method_table.select(where=f'method_id={method_id}') + method_columns = method_table.columns + method_info = pd.DataFrame(method_info, columns=method_columns) if data_id != -1: data_table = db[f'data_{data}'] - data_info = data_table.select(where=f'WHERE id={data_id}') - if basic_info[0]["status"] == 'finished': + data_info = data_table.select(where=f'data_id={data_id}') + data_columns = data_table.columns + data_info = pd.DataFrame(data_info, columns=data_columns) + if basic_info[0][13] == 'finished': result_table = db[f'result_{task}'] - result_info = result_table.select(where=f'WHERE id={st.session_state.cur_detail_id}') + result_info = result_table.select(where=f'experiment_id={st.session_state.cur_detail_id}') + result_columns = result_table.columns + result_info = pd.DataFrame(result_info, columns=result_columns) + basic_info = pd.DataFrame(basic_info, columns=basic_columns) return basic_info, method_info, data_info, result_info + + +def split_result_info(result_info): + columns_keep = [col for col in result_info.columns if not col.startswith("image_")] + pattern = re.compile(r'image_(\d+)$') + image_dict = {} + for name in result_info.columns: + match = pattern.match(name) + if match and not result_info[name].isnull().all(): + image_dict[result_info[f"{name}_name"][0]] = result_info[name][0] + elif result_info[name].isnull().all(): + break + + return result_info[columns_keep], image_dict + + + def delete_current_experiment(db): - experiment_list = db['experiment_list'] - task = experiment_list.select(where=f'id={st.session_state.cur_detail_id}')[0]['task'] - experiment_list.delete(f'id={st.session_state.cur_detail_id}') + experiment_table = db['experiment_list'] + task = experiment_table.select(where=f'id={st.session_state.cur_detail_id}')[0][6] + experiment_table.delete(f'id={st.session_state.cur_detail_id}') result_table = db[f'result_{task}'] - result_table.delete(f'id={st.session_state.cur_detail_id}') + result_table.delete(f'experiment_id={st.session_state.cur_detail_id}') db.conn.commit() st.session_state.cur_detail_id = None diff --git a/pyerm/webUI/home.py b/pyerm/webUI/home.py index 32cb645..8735b08 100644 --- a/pyerm/webUI/home.py +++ b/pyerm/webUI/home.py @@ -24,6 +24,8 @@ import streamlit as st import os +import platform +import math import subprocess from importlib.metadata import version @@ -34,6 +36,7 @@ def home(): title() + st.write("---") load_db() if os.path.exists(st.session_state.db_path) and st.session_state.db_path.endswith('.db'): st.markdown('Export Experiment Data') @@ -41,19 +44,21 @@ def home(): download_zip() if st.checkbox('Download raw db file'): download_db() + st.write("---") clean_cache() def title(): st.title('Python Experiment Record Manager WebUI') st.markdown(f"**Version**: {version('pyerm')}") - st.markdown(f"**Repository**: [{REPO_URL}]({REPO_URL})") + st.markdown(f"**GitHub Repository**: [{REPO_URL}]({REPO_URL})") + st.markdown(f"**PyPI**: [PyERM](https://pypi.org/project/pyerm/)") st.markdown(f"**License**: MIT") st.markdown(f"**Author**: Yuxuan Shao") st.markdown(f"**Description**:") st.markdown(f"PyERM is a Python package for managing experiment records.") st.markdown(f"This is the web user interface of the PyERM.") - st.markdown(f"**Disclaimer**: This is a demo version. The actual version is not available yet.") + st.markdown(f"**Disclaimer**: This is a demo version. Bugs may exist. Use at your own risk.") def load_db(): st.markdown('## Load Database') @@ -74,11 +79,11 @@ def export_data(): db_name = os.path.basename(st.session_state.db_path) db_name = os.path.splitext(db_name)[0] zip_path = os.path.join(output_dir_path, f"{db_name}.zip") - subprocess.run(["export_zip", st.session_state.db_path, output_dir_path]) - with open(zip_path, "rb") as file: - zip = file.read() - - subprocess.run(["rm", "-f", zip_path]) + result1 = subprocess.run(["export_zip", st.session_state.db_path, output_dir_path]) + if result1.returncode == 0: + with open(zip_path, "rb") as file: + zip = file.read() + os.remove(zip_path) return zip def download_zip(): @@ -107,9 +112,27 @@ def download_db(): def clean_cache(): st.write('## Clean Cache') - st.write('Cache is used to store temporary files, such as result images.') + st.write('Cache is used to store temporary files, such as configure files.__') + st.write(f'Cache folder is located at: **{PYERM_HOME}**') + st.write(f'Cache folder size: {get_folder_size(PYERM_HOME)}') if st.checkbox('I want to clean cache'): st.write('This will delete the cache folder and all its contents, which cannot be undone.') if st.button('Confirm'): - subprocess.run(["rm", "-rf", os.path.join(PYERM_HOME, '.cache')]) - st.write('Cache cleaned successfully.') \ No newline at end of file + subprocess.run(["rm", "-rf", PYERM_HOME]) + st.write('Cache cleaned successfully.') + +def get_folder_size(folder_path): + total_size = 0 + for dirpath, dirnames, filenames in os.walk(folder_path): + for filename in filenames: + filepath = os.path.join(dirpath, filename) + total_size += os.path.getsize(filepath) + return format_size(total_size) + +def format_size(size_bytes): + units = ["B", "KB", "MB", "GB", "TB"] + if size_bytes == 0: + return "0 B" + index = min(len(units) - 1, int(math.log(size_bytes, 1024))) + size = size_bytes / (1024 ** index) + return f"{size:.2f} {units[index]}" \ No newline at end of file diff --git a/pyerm/webUI/tables.py b/pyerm/webUI/tables.py index 4074420..9bc6a37 100644 --- a/pyerm/webUI/tables.py +++ b/pyerm/webUI/tables.py @@ -37,19 +37,22 @@ def tables(): title() - st.sidebar.markdown('## Detected Tables') if os.path.exists(st.session_state.db_path) and st.session_state.db_path.endswith('.db'): detect_tables() - if st.sidebar.checkbox('Use SQL condition & columns', False): + st.sidebar.write('## SQL Query') + if st.sidebar.checkbox('Use Full SQL Sentense For Total DB', False): + input_full_sql() + if st.sidebar.checkbox('Use SQL By Columns & Conditions & Tables', False): input_sql() select_tables() def detect_tables(): + st.sidebar.markdown('## Detected Tables') db = Database(st.session_state.db_path, output_info=False) if len(db.table_names) == 0: st.write('No tables detected in the database.') return - table_name = st.sidebar.radio('Table to select:', db.table_names + db.view_names) + table_name = st.sidebar.radio('**Table to select**:', db.table_names + db.view_names) st.session_state.table_name = table_name def select_tables(): @@ -74,13 +77,14 @@ def fold_detail_row(row, col_name): db = Database(st.session_state.db_path, output_info=False) table_name:str = st.session_state.table_name + st.write('## Table:', table_name) if st.session_state.sql is not None: try: - table_name = st.session_state.table_name df = pd.read_sql_query(st.session_state.sql, db.conn) + st.write(f'### Used SQL: ({st.session_state.sql})') st.session_state.sql = None except Exception as e: - st.write('Error:', e) + st.write('SQL Error:', e) st.session_state.sql = None return else: @@ -91,7 +95,8 @@ def fold_detail_row(row, col_name): if st.button('Refresh', key='refresh1'): st.session_state.table_name = table_name - st.write('## Table:', table_name) + st.rerun() + columns_keep = [col for col in df.columns if not col.startswith("image_")] @@ -122,18 +127,25 @@ def fold_detail_row(row, col_name): st.write(df.to_html(escape=False, columns=columns_keep), unsafe_allow_html=True) if st.button('Refresh', key='refresh2'): st.session_state.table_name = table_name + st.rerun() def input_sql(): - st.sidebar.write('You can also set the columns and condition for construct a select SQL sentense for the current table here.') + st.sidebar.write('You can set the columns and condition for construct a select SQL sentense for the current table here.') condition = st.sidebar.text_input("Condition", value='', help='The condition for the select SQL sentense.') columns = st.sidebar.text_input("Columns", value='*', help='The columns for the select SQL sentense.') st.session_state.table_name = st.sidebar.text_input("Table", value=st.session_state.table_name, help='The table, view or query for the select SQL sentense.') - if st.sidebar.button('Run'): + if st.sidebar.button('Run', key="run_table_sql"): st.session_state.sql = f"SELECT {columns} FROM {st.session_state.table_name} WHERE {condition}" if condition else f"SELECT {columns} FROM {st.session_state.table_name}" +def input_full_sql(): + st.sidebar.write('You can input a full SQL sentense here to select what you need and link different tables or views.') + sql = st.sidebar.text_area('SQL', value=None, height=200) + if st.sidebar.button('Run', key='run_full_sql'): + st.session_state.sql = sql + st.session_state.table_name = 'SQL Query Results' def title(): - st.title('Tables of the experiments') + st.title('Tables of the experiment records') if os.path.exists(st.session_state.db_path) and st.session_state.db_path.endswith('.db'): st.write(f'Database Loaded (In {st.session_state.db_path})') else: diff --git a/setup.py b/setup.py index c9d74a1..dac3d0a 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -# Version: 0.2.7 +# Version: 0.2.8 from setuptools import setup, find_packages with open("README.md", "r", encoding="utf-8") as f: @@ -28,7 +28,7 @@ setup( name='pyerm', - version='0.2.7', + version='0.2.8', author='Yuxuan Shao', author_email='yx_shao@qq.com', description='This project is an local experiment record manager for python based on SQLite DMS, suitable for recording experiment and analysing experiment data with a web UI, which can help you efficiently save your experiment settings and results for later analysis.', @@ -55,7 +55,7 @@ "xlsxwriter", "numpy", "matplotlib", - "streamlit>=1.31.1" + "streamlit>=1.39.0" ], python_requires='>=3.9', )