calculate prepare LLM dataset total size

[huseinzol05]

Notebook https://github.com/huseinzol05/malaysian-dataset/blob/master/prepare-llm/calculate-size.ipynb,

from bs4 import BeautifulSoup
import requests
import re
import json

r = requests.get('https://github.com/users/huseinzol05/projects/1/views/1')
soup = BeautifulSoup(r.content, "lxml")

data = json.loads(soup.find('script', {'id': 'memex-items-data'}).contents[0])
len(data)

parsed = []
for d in data:
    t = d['memexProjectColumnValues'][0]['value']['title']['raw']
    if '(' in t and ')' in t:
        parsed.append(t)

sizes = []

units = {
    'MB': 1e6,
    'GB': 1e9,
    'KB': 1e4,
}

for string in parsed:
    for r in re.finditer(r'\([^()]*\)', string):
        span = r.span()
        subs = string[span[0] + 1: span[1] - 1]
        s, unit = subs.split()
        sizes.append(float(s) * units[unit])
        
sum(sizes) / 1e9

[huseinzol05]

Currently 89.27078 GB.

[huseinzol05]

Currently 97.054874 GB.

[huseinzol05]

97.751634, freezing dataset to train LLM.