Skip to content

Commit

Permalink
no more showing figures in table to save time, delete stucked experim…
Browse files Browse the repository at this point in the history 
…ents, add config file to cache db path
  • Loading branch information
@Mr-SGXXX
Mr-SGXXX committed Oct 11, 2024
1 parent 7ea8b62 commit 0c037ac
Show file tree
Hide file tree
Showing 8 changed files with 105 additions and 53 deletions.
6 changes: 3 additions & 3 deletions pyerm/database/experiment.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# Version: 0.2.4
# Version: 0.2.7

import os
import typing
Expand All @@ -32,9 +32,9 @@
from .dbbase import Database
from .tables import ExperimentTable, MethodTable, ResultTable, DetailTable, DataTable

PYERM_HOME = os.path.join(os.path.expanduser('~'), '.pyerm')
__all__ = ['Experiment']

USER_HOME = os.path.expanduser('~')

class Experiment:
"""
Expand Down Expand Up @@ -75,7 +75,7 @@ class Experiment:
"""
def __init__(self, db_path:str=None):
if db_path is None:
db_path = os.path.join(USER_HOME, 'experiment.db')
db_path = os.path.join(PYERM_HOME, 'experiment.db')
self._db = Database(db_path)
self.experiment_table = ExperimentTable(self._db)
self.parameter_table = None
Expand Down
10 changes: 5 additions & 5 deletions pyerm/database/tables.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# Version: 0.2.6
# Version: 0.2.7

from PIL import Image
from io import BytesIO
Expand Down Expand Up @@ -72,10 +72,10 @@ def experiment_failed(self, experiment_id:int, error_info:str=None, end_time:flo
end_time = time()
end_time = localtime(end_time)
end_time = strftime("%Y-%m-%d %H:%M:%S", end_time)
print(error_info)
# print(error_info)
if error_info is None:
error_info = traceback.format_exc()
print(error_info)
# print(error_info)
super().update(f"id={experiment_id}", end_time=strftime(end_time), status='failed', failed_reason=error_info)

def get_experiment(self, experiment_id:int) -> dict:
Expand Down Expand Up @@ -105,7 +105,7 @@ def insert(self, **kwargs):

query = f"SELECT data_id FROM {self.table_name} WHERE {condition}"
id_list = self.db.cursor.execute(query, values).fetchall()
print(kwargs)
# print(kwargs)

if id_list == []:
return super().insert(**kwargs)
Expand Down Expand Up @@ -134,7 +134,7 @@ def insert(self, **kwargs):

query = f"SELECT method_id FROM {self.table_name} WHERE {condition}"
id_list = self.db.cursor.execute(query, values).fetchall()
print(kwargs)
# print(kwargs)
if id_list == []:
return super().insert(**kwargs)
else:
Expand Down
16 changes: 14 additions & 2 deletions pyerm/database/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# Version: 0.2.6
# Version: 0.2.7

from time import time

from .dbbase import Database

Expand All @@ -32,4 +34,14 @@ def delete_failed_experiments(db:Database):
task = experiment[1]
task_table = db[f"result_{task}"]
task_table.delete(f"experiment_id={experiment_id}")
experiment_table.delete(f"id={experiment_id}")
experiment_table.delete(f"id={experiment_id}")

# # delete stuck running experiments that have been running for more than 24 hours
# running_experiments = experiment_table.select('id', 'task', 'start_time', where="status='running'")
# for experiment in running_experiments:
# experiment_id = experiment[0]
# task = experiment[1]
# if time() - experiment[2] > 86400:
# task_table = db[f"result_{task}"]
# task_table.delete(f"experiment_id={experiment_id}")
# experiment_table.delete(f"id={experiment_id}")
3 changes: 3 additions & 0 deletions pyerm/webUI/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,6 @@
# SOFTWARE.

# Version: 0.2.4
import os

PYERM_HOME = os.path.join(os.path.expanduser('~'), '.pyerm')
27 changes: 24 additions & 3 deletions pyerm/webUI/app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,19 +20,34 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# Version: 0.2.4
# Version: 0.2.7

import streamlit as st
import os
import configparser

from home import home
from tables import tables

USER_HOME = os.path.expanduser('~')
from pyerm.webUI import PYERM_HOME


def init():
config = configparser.ConfigParser()
if not os.path.exists(PYERM_HOME):
os.makedirs(PYERM_HOME)
config['DEFAULT']['db_path'] = os.path.join(PYERM_HOME, 'experiment.db')
else:
config.read(os.path.join(PYERM_HOME, 'config.ini'))

cache_dir = os.path.join(PYERM_HOME, '.cache')
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)

if 'db_path' not in st.session_state:
st.session_state.db_path = os.path.join(USER_HOME, 'experiment.db')
st.session_state.db_path = config.get('DEFAULT', 'db_path')
else:
config.set('DEFAULT', 'db_path', st.session_state.db_path)
if 'table_name' not in st.session_state:
st.session_state.table_name = None
if 'sql' not in st.session_state:
Expand All @@ -41,6 +56,12 @@ def init():
st.session_state.zip = None
if 'last_version' not in st.session_state:
st.session_state.last_version = None
if 'selected_row' not in st.session_state:
st.session_state.selected_row = None

with open(os.path.join(PYERM_HOME, 'config.ini'), 'w') as f:
config.write(f)



def main():
Expand Down
16 changes: 12 additions & 4 deletions pyerm/webUI/home.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,15 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# Version: 0.2.6
# Version: 0.2.7

import streamlit as st
import os
import subprocess
from importlib.metadata import version

from pyerm.database.dbbase import Database

USER_HOME = os.path.expanduser("~")
from pyerm.webUI import PYERM_HOME

REPO_URL = "https://github.com/Mr-SGXXX/pyerm"

Expand All @@ -42,6 +41,7 @@ def home():
download_zip()
if st.checkbox('Download raw db file'):
download_db()
clean_cache()


def title():
Expand All @@ -68,7 +68,7 @@ def load_db():
st.write(f"Database not found. Please input the correct path.")

def export_data():
output_dir_path = f"{USER_HOME}/.tmp"
output_dir_path = f"{PYERM_HOME}/.tmp"
if not os.path.exists(output_dir_path):
os.makedirs(output_dir_path)
db_name = os.path.basename(st.session_state.db_path)
Expand Down Expand Up @@ -105,3 +105,11 @@ def download_db():
)


def clean_cache():
st.write('## Clean Cache')
st.write('Cache is used to store temporary files, such as result images.')
if st.checkbox('I want to clean cache'):
st.write('This will delete the cache folder and all its contents, which cannot be undone.')
if st.button('Confirm'):
subprocess.run(["rm", "-rf", os.path.join(PYERM_HOME, '.cache')])
st.write('Cache cleaned successfully.')
76 changes: 42 additions & 34 deletions pyerm/webUI/tables.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# Version: 0.2.6
# Version: 0.2.7

import pandas as pd
from PIL import Image
Expand All @@ -29,10 +29,11 @@
import streamlit as st
import os
import re
import tempfile
from time import strftime, gmtime

from pyerm.database.utils import delete_failed_experiments
from pyerm.database.dbbase import Database
from pyerm.webUI import PYERM_HOME

def tables():
title()
Expand All @@ -52,23 +53,27 @@ def detect_tables():
st.session_state.table_name = table_name

def select_tables():
def image_to_base64(img):
buffered = BytesIO(img)
img_str = base64.b64encode(buffered.getvalue()).decode()
return img_str
# def image_to_base64(img, desc):
# buffered = BytesIO(img)
# img_str = base64.b64encode(buffered.getvalue()).decode()
# return img_str

# def make_image_clickable(image_name, image, desc):
# img_str = image_to_base64(image, desc)
# if img_str:
# return f'<a href="data:image/jpeg;base64,{img_str}" target="_blank" title="{image_name}"><img src="data:image/jpeg;base64,{img_str}" width="100"></a>'
# else:
# return f'<a href="#" title="None"><img src="#" width="100"></a>'

def make_image_clickable(image_name, image):
img_str = image_to_base64(image)
return f'<a href="data:image/jpeg;base64,{img_str}" target="_blank" title="{image_name}"><img src="data:image/jpeg;base64,{img_str}" width="100"></a>'

def fold_detail_row(row, col_name):
if row[col_name]:
return f'<details><summary>Details</summary>{row[col_name]}</details>'
detail = row[col_name].replace("\n", "<br>")
return f'<details><summary>Details</summary>{detail}</details>'
else:
return 'None'

db = Database(st.session_state.db_path, output_info=False)
table_name = st.session_state.table_name
table_name:str = st.session_state.table_name
if st.session_state.sql is not None:
try:
table_name = st.session_state.table_name
Expand All @@ -83,37 +88,40 @@ def fold_detail_row(row, col_name):
columns = [column[0] for column in db.cursor.description]
df = pd.DataFrame(data, columns=columns)

# special process for image columns
columns_keep = [col for col in df.columns if not col.startswith("image_")]
pattern = re.compile(r'image_(\d+)')
max_image_num = -1
for name in df.columns:
match = pattern.match(name)
if match:
max_image_num = max(max_image_num, int(match.group(1)))
for i in range(max_image_num+1):
if f'image_{i}' in df.columns and not df[f'image_{i}_name'].isnull().all():
df[f'image_{i}'] = df.apply(lambda x: make_image_clickable(x[f'image_{i}_name'], x[f'image_{i}']), axis=1)
columns_keep.append(f'image_{i}')

if "failed_reason" in df.columns:
df['failed_reason'] = df.apply(lambda x: fold_detail_row(x, 'failed_reason'), axis=1)
df = df[columns_keep]

if st.button('Refresh'):
if st.button('Refresh', key='refresh1'):
st.session_state.table_name = table_name
st.write('## Table:', table_name)

columns_keep = [col for col in df.columns if not col.startswith("image_")]

if table_name == 'experiment_list':
if st.checkbox('Delete all failed records'):
if st.checkbox('Delete all failed and stuck records'):
st.write('**Warning: This operation will delete all failed records and their results, which cannot be undone.**')
if st.button(f"Confirm"):
delete_failed_experiments(db)
st.session_state.table_name = table_name
st.write(df.to_html(escape=False, columns=columns_keep), unsafe_allow_html=True)

# st.dataframe(df[columns_keep])

df['failed_reason'] = df.apply(lambda x: fold_detail_row(x, 'failed_reason'), axis=1)
df['useful_time_cost'] = df['useful_time_cost'].apply(lambda x: strftime('%H:%M:%S', gmtime(x)) if not pd.isnull(x) else x)
df['total_time_cost'] = df['total_time_cost'].apply(lambda x: strftime('%H:%M:%S', gmtime(x)) if not pd.isnull(x) else x)
# elif table_name.startswith("result_"):
# # special process for image columns
# pattern = re.compile(r'image_(\d+)')
# max_image_num = -1
# for name in df.columns:
# match = pattern.match(name)
# if match:
# max_image_num = max(max_image_num, int(match.group(1)))

# for i in range(max_image_num+1):
# if f'image_{i}' in df.columns and not df[f'image_{i}_name'].isnull().all():
# df[f'image_{i}'] = df.apply(lambda x: make_image_clickable(x[f'image_{i}_name'], x[f'image_{i}'], desc=f"{x[f'experiment_id']}_{i}"), axis=1)
# columns_keep.append(f'image_{i}')

df = df[columns_keep]
st.write(df.to_html(escape=False, columns=columns_keep), unsafe_allow_html=True)
if st.button('Refresh', key='refresh2'):
st.session_state.table_name = table_name

def input_sql():
st.sidebar.write('You can also set the columns and condition for construct a select SQL sentense for the current table here.')
Expand Down
4 changes: 2 additions & 2 deletions setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,15 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# Version: 0.2.6
# Version: 0.2.7
from setuptools import setup, find_packages

with open("README.md", "r", encoding="utf-8") as f:
long_description = f.read()

setup(
name='pyerm',
version='0.2.6',
version='0.2.7',
author='Yuxuan Shao',
author_email='yx_shao@qq.com',
description='This project is an local experiment record manager for python based on SQLite DMS, suitable for recording experiment and analysing experiment data with a web UI, which can help you efficiently save your experiment settings and results for later analysis.',
Expand Down

0 comments on commit 0c037ac

Please sign in to comment.