From 4431ec415e781018b22539585b6fb491f3c61336 Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Sat, 18 Mar 2023 05:40:35 +0000 Subject: [PATCH 01/30] preserve Cantonese hyphenation --- server/cantonese.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/server/cantonese.py b/server/cantonese.py index a5698c1..915dcd6 100644 --- a/server/cantonese.py +++ b/server/cantonese.py @@ -5,7 +5,7 @@ # cantonese.py - Python functions for processing Cantonese transliterations # (uses eSpeak and Gradint for help with some of them) -# v1.42 (c) 2013-15,2017-23 Silas S. Brown. License: GPL +# v1.43 (c) 2013-15,2017-23 Silas S. Brown. License: GPL cache = {} # to avoid repeated eSpeak runs, # zi -> jyutping or (pinyin,) -> translit @@ -64,7 +64,7 @@ def hanzi_only(unitext): return u"".join(filter(lambda x:0x4e00<=ord(x)<0xa700 o def py2nums(pinyin): if not type(pinyin)==type(u""): pinyin = pinyin.decode('utf-8') - assert pinyin.strip(), "blank pinyin" # saves figuring out a findall TypeError + if not pinyin.strip(): return "" global pinyin_dryrun if pinyin_dryrun: pinyin_dryrun = list(pinyin_dryrun) @@ -291,6 +291,9 @@ def songSubst(l): pinyin = pinyin.decode('utf-8') if pinyin and not (pinyin,) in cache: pinyin_dryrun.add(pinyin) + for w in pinyin.split(): + for h in w.split('-'): + pinyin_dryrun.add(h) dryrun_mode = False for l in lines: if '#' in l: l,pinyin = l.split('#') @@ -300,7 +303,7 @@ def songSubst(l): elif pinyin: jyutping = adjust_jyutping_for_pinyin(l,jyutping,pinyin) groupLens = [0] - for syl,space in re.findall('([A-Za-z]*[1-5])( *)',py2nums(pinyin)): + for syl,space in re.findall('([A-Za-z]*[1-5])( *)',' '.join('-'.join(py2nums(h) for h in w.split('-')) for w in pinyin.split())): # doing it this way so we're not relying on espeak transliterate_multiple to preserve spacing and hyphenation groupLens[-1] += 1 if space: groupLens.append(0) if not groupLens[-1]: groupLens=groupLens[:-1] From b391a56629eef26814a98ce38997887e795fa818 Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Sat, 1 Jul 2023 11:45:52 +0100 Subject: [PATCH 02/30] clarify comment --- server/cantonese.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/cantonese.py b/server/cantonese.py index 915dcd6..f59160e 100644 --- a/server/cantonese.py +++ b/server/cantonese.py @@ -91,7 +91,7 @@ def adjust_jyutping_for_pinyin(hanzi,jyutping,pinyin): i = 0 ; tones = re.finditer('[1-7]',jyutping) ; j2 = [] for h,p in zip(list(hanzi),pinyin): try: j = getNext(tones).end() - except StopIteration: return jyutping # one of the zin has no Cantonese reading, which we'll pick up later on "failed to fix" + except StopIteration: return jyutping # one of the hanzi has no Cantonese reading in our data: we'll warn "failed to fix" below j2.append(jyutping[i:j]) ; i = j if h in py2j and p.lower() in py2j[h]: j2[-1]=j2[-1][:re.search("[A-Za-z]*[1-7]$",j2[-1]).start()]+py2j[h][p.lower()] return "".join(j2)+jyutping[i:] From bc7528d3af1858268bad0ff971286ef5d10caeb9 Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Sun, 9 Jul 2023 21:39:59 +0100 Subject: [PATCH 03/30] Sidney Lau aa->a rule was wrong (typo in the example I used as a guide) --- server/cantonese.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/cantonese.py b/server/cantonese.py index f59160e..d9ae50a 100644 --- a/server/cantonese.py +++ b/server/cantonese.py @@ -5,7 +5,7 @@ # cantonese.py - Python functions for processing Cantonese transliterations # (uses eSpeak and Gradint for help with some of them) -# v1.43 (c) 2013-15,2017-23 Silas S. Brown. License: GPL +# v1.44 (c) 2013-15,2017-23 Silas S. Brown. License: GPL cache = {} # to avoid repeated eSpeak runs, # zi -> jyutping or (pinyin,) -> translit @@ -153,7 +153,7 @@ def adjust_jyutping_for_pinyin(hanzi,jyutping,pinyin): def jyutping_to_lau(j): j = S(j).lower().replace("j","y").replace("z","j") for k,v in jlRep: j=j.replace(k,v) - return j.lower().replace("aa","a").replace("ohek","euk") + return j.lower().replace("ohek","euk") def jyutping_to_lau_java(jyutpingNo=2,lauNo=1): # for annogen.py 3.29+ --annotation-postprocess to ship Jyutping and generate Lau at runtime return 'if(annotNo=='+str(jyutpingNo)+'||annotNo=='+str(lauNo)+'){m=Pattern.compile("(.*?)").matcher(r);sb=new StringBuffer();while(m.find()){String r2=(annotNo=='+str(jyutpingNo)+'?m.group(1).replaceAll("([1-7])(.)","$1­$2"):(m.group(1)+" ").toLowerCase().replace("j","y").replace("z","j")'+''.join('.replace("'+k+'","'+v+'")' for k,v in jlRep)+'.toLowerCase().replace("aa","a").replace("ohek","euk").replaceAll("([1-7])","$1-").replace("- "," ").replaceAll(" $","")),tmp=m.group(1).substring(0,1);if(annotNo=='+str(lauNo)+'&&tmp.equals(tmp.toUpperCase()))r2=r2.substring(0,1).toUpperCase()+r2.substring(1);m.appendReplacement(sb,""+r2+"");}m.appendTail(sb); r=sb.toString();}' # TODO: can probably go faster with mapping for some of this From d451c6ca827c199fa088c6de9c70f743d6de9f52 Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Mon, 10 Jul 2023 07:56:31 +0100 Subject: [PATCH 04/30] Sidney Lau also fix in Java --- server/cantonese.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/cantonese.py b/server/cantonese.py index d9ae50a..53a71f1 100644 --- a/server/cantonese.py +++ b/server/cantonese.py @@ -5,7 +5,7 @@ # cantonese.py - Python functions for processing Cantonese transliterations # (uses eSpeak and Gradint for help with some of them) -# v1.44 (c) 2013-15,2017-23 Silas S. Brown. License: GPL +# v1.45 (c) 2013-15,2017-23 Silas S. Brown. License: GPL cache = {} # to avoid repeated eSpeak runs, # zi -> jyutping or (pinyin,) -> translit @@ -156,7 +156,7 @@ def jyutping_to_lau(j): return j.lower().replace("ohek","euk") def jyutping_to_lau_java(jyutpingNo=2,lauNo=1): # for annogen.py 3.29+ --annotation-postprocess to ship Jyutping and generate Lau at runtime - return 'if(annotNo=='+str(jyutpingNo)+'||annotNo=='+str(lauNo)+'){m=Pattern.compile("(.*?)").matcher(r);sb=new StringBuffer();while(m.find()){String r2=(annotNo=='+str(jyutpingNo)+'?m.group(1).replaceAll("([1-7])(.)","$1­$2"):(m.group(1)+" ").toLowerCase().replace("j","y").replace("z","j")'+''.join('.replace("'+k+'","'+v+'")' for k,v in jlRep)+'.toLowerCase().replace("aa","a").replace("ohek","euk").replaceAll("([1-7])","$1-").replace("- "," ").replaceAll(" $","")),tmp=m.group(1).substring(0,1);if(annotNo=='+str(lauNo)+'&&tmp.equals(tmp.toUpperCase()))r2=r2.substring(0,1).toUpperCase()+r2.substring(1);m.appendReplacement(sb,""+r2+"");}m.appendTail(sb); r=sb.toString();}' # TODO: can probably go faster with mapping for some of this + return 'if(annotNo=='+str(jyutpingNo)+'||annotNo=='+str(lauNo)+'){m=Pattern.compile("(.*?)").matcher(r);sb=new StringBuffer();while(m.find()){String r2=(annotNo=='+str(jyutpingNo)+'?m.group(1).replaceAll("([1-7])(.)","$1­$2"):(m.group(1)+" ").toLowerCase().replace("j","y").replace("z","j")'+''.join('.replace("'+k+'","'+v+'")' for k,v in jlRep)+'.toLowerCase().replace("ohek","euk").replaceAll("([1-7])","$1-").replace("- "," ").replaceAll(" $","")),tmp=m.group(1).substring(0,1);if(annotNo=='+str(lauNo)+'&&tmp.equals(tmp.toUpperCase()))r2=r2.substring(0,1).toUpperCase()+r2.substring(1);m.appendReplacement(sb,""+r2+"");}m.appendTail(sb); r=sb.toString();}' # TODO: can probably go faster with mapping for some of this def incomplete_lau_to_jyutping(l): # incomplete: assumes Lau didn't do the "aa" -> "a" rule l = S(l).lower().replace("euk","ohek") From 18ff7132b6bc417ce0586aaf01866d73c79c1ebd Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Thu, 21 Sep 2023 07:30:57 +0100 Subject: [PATCH 05/30] fix Cantonese transliteration duplicating pre-included jyutping --- src/synth.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/synth.py b/src/synth.py index 3177631..4b6ecc8 100644 --- a/src/synth.py +++ b/src/synth.py @@ -678,6 +678,7 @@ def transliterate_multiple(self,lang,textList,forPartials=1,keepIndexList=0): delete_last_r_if_blank = 1 # this depends on original Jonathan Duddington eSpeak and is not compatible with eSpeak NG which does not always give us blank lines. TODO: if want eSpeak NG compatibility (which might be a good idea as mainstream GNU/Linux distributions now ship eSpeak NG and just call it "espeak"), we might want to pick up on THIS l ending with B("[_^_]") and next Translate being an identical syllable. That might not be the only problem. else: en_words[toAppend]=1 else: # not Translate + if lang=="zhy" and l.endswith(B("[_^_]")): del r[-1] # will be duplicated if lang=="zh" and l.startswith(B("Found: ")) and ((l[7:8]==l[9:10]==B("'") and B("a")<=l[8:9]<=B("z")) or (l[8:9]==B(" ") and B("a")<=l[7:8]<=B("z"))): # an alphabetical letter - we can say this as a Chinese letter and it should be compatible with more partials-based synths. But DON'T do this if going to give it to a unit-selection synth - 'me1' and 'ne1' don't have hanzi and some synths will have difficulty saying them. if l[8:9]==B(' '): letter=l[7:8] else: letter=l[8:9] From e0fdf29fccb8d75c3d4f23f406174a2a6e3204d8 Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Fri, 22 Sep 2023 07:58:42 +0100 Subject: [PATCH 06/30] properly fix last commit: it was a Python 3 compatibility issue --- src/synth.py | 2 +- src/top.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/synth.py b/src/synth.py index 4b6ecc8..0a29abb 100644 --- a/src/synth.py +++ b/src/synth.py @@ -659,7 +659,7 @@ def transliterate_multiple(self,lang,textList,forPartials=1,keepIndexList=0): if int0: if int0 > thisgroup_max_priority: thisgroup_max_priority = int0 - if lWords[-1]=="[_^_]": thisgroup_enWord_priority = int0 # so far it looks like this is going to be an English word + if lWords[-1]==B("[_^_]"): thisgroup_enWord_priority = int0 # so far it looks like this is going to be an English word else: # a split between the groups if thisgroup_enWord_priority == thisgroup_max_priority: # the choice with the highest priority was the one containing the [_^_] to put the word into English en_words[r[-1]]=1 diff --git a/src/top.py b/src/top.py index 3f88721..e74fc1b 100644 --- a/src/top.py +++ b/src/top.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # (Python 2 or Python 3, but more fully tested on 2) -program_name = "gradint v3.075 (c) 2002-23 Silas S. Brown. GPL v3+." +program_name = "gradint v3.076 (c) 2002-23 Silas S. Brown. GPL v3+." # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From babc5893b9ef42a7ebb7f5778103ea37704b3585 Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Fri, 22 Sep 2023 08:06:12 +0100 Subject: [PATCH 07/30] this was also supposed to be in last commit --- src/synth.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/synth.py b/src/synth.py index 0a29abb..e35ea22 100644 --- a/src/synth.py +++ b/src/synth.py @@ -678,7 +678,6 @@ def transliterate_multiple(self,lang,textList,forPartials=1,keepIndexList=0): delete_last_r_if_blank = 1 # this depends on original Jonathan Duddington eSpeak and is not compatible with eSpeak NG which does not always give us blank lines. TODO: if want eSpeak NG compatibility (which might be a good idea as mainstream GNU/Linux distributions now ship eSpeak NG and just call it "espeak"), we might want to pick up on THIS l ending with B("[_^_]") and next Translate being an identical syllable. That might not be the only problem. else: en_words[toAppend]=1 else: # not Translate - if lang=="zhy" and l.endswith(B("[_^_]")): del r[-1] # will be duplicated if lang=="zh" and l.startswith(B("Found: ")) and ((l[7:8]==l[9:10]==B("'") and B("a")<=l[8:9]<=B("z")) or (l[8:9]==B(" ") and B("a")<=l[7:8]<=B("z"))): # an alphabetical letter - we can say this as a Chinese letter and it should be compatible with more partials-based synths. But DON'T do this if going to give it to a unit-selection synth - 'me1' and 'ne1' don't have hanzi and some synths will have difficulty saying them. if l[8:9]==B(' '): letter=l[7:8] else: letter=l[8:9] From 3260b950f3d81d68e82c481af336b84d0a326369 Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Sun, 24 Sep 2023 07:50:53 +0100 Subject: [PATCH 08/30] fix Cantonese Mandarin-driven override table (bian/pian was swapped) --- server/cantonese.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/cantonese.py b/server/cantonese.py index 53a71f1..cee50e3 100644 --- a/server/cantonese.py +++ b/server/cantonese.py @@ -5,7 +5,7 @@ # cantonese.py - Python functions for processing Cantonese transliterations # (uses eSpeak and Gradint for help with some of them) -# v1.45 (c) 2013-15,2017-23 Silas S. Brown. License: GPL +# v1.46 (c) 2013-15,2017-23 Silas S. Brown. License: GPL cache = {} # to avoid repeated eSpeak runs, # zi -> jyutping or (pinyin,) -> translit @@ -101,7 +101,7 @@ def adjust_jyutping_for_pinyin(hanzi,jyutping,pinyin): u"\u4E50\u6A02":{"le4":"lok6","yue4":"ngok6"}, u"\u4EB2\u89AA":{"qin1":"can1","qing4":"can3"}, u"\u4F20\u50B3":{"chuan2":"cyun4","zhuan4":"zyun6"}, -u"\u4FBF":{"bian4":"pin4","pian2":"bin6"}, +u"\u4FBF":{"bian4":"bin6","pian2":"pin4"}, u"\u5047":{"jia3":"gaa2","jia4":"gaa3"}, u"\u5174\u8208":{"xing1":"hing1","xing4":"hing3"}, # u"\u5207":{"qie4":"cai3","qie1":"cit3"}, # WRONG (rm'd v1.17). It's cit3 in re4qie4. It just wasn't in yiqie4 (which zhy_list has as an exception anyway) From f8e99988731f3a0317bf0b6d74d941b3e0831cb8 Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Sun, 3 Dec 2023 06:18:16 +0000 Subject: [PATCH 09/30] Cantonese readings from shen/shi --- server/cantonese.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/cantonese.py b/server/cantonese.py index cee50e3..921642e 100644 --- a/server/cantonese.py +++ b/server/cantonese.py @@ -5,7 +5,7 @@ # cantonese.py - Python functions for processing Cantonese transliterations # (uses eSpeak and Gradint for help with some of them) -# v1.46 (c) 2013-15,2017-23 Silas S. Brown. License: GPL +# v1.47 (c) 2013-15,2017-23 Silas S. Brown. License: GPL cache = {} # to avoid repeated eSpeak runs, # zi -> jyutping or (pinyin,) -> translit @@ -100,6 +100,7 @@ def adjust_jyutping_for_pinyin(hanzi,jyutping,pinyin): u"\u4E3A\u70BA":{"wei2":"wai4","wei4":"wai6"}, u"\u4E50\u6A02":{"le4":"lok6","yue4":"ngok6"}, u"\u4EB2\u89AA":{"qin1":"can1","qing4":"can3"}, +u"\u4EC0":{"shen2":"sam6","shi2":"sap6"}, # unless zaap6 u"\u4F20\u50B3":{"chuan2":"cyun4","zhuan4":"zyun6"}, u"\u4FBF":{"bian4":"bin6","pian2":"pin4"}, u"\u5047":{"jia3":"gaa2","jia4":"gaa3"}, From c42c89222152fee7550d6ee741c623d206c1251c Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Tue, 5 Dec 2023 08:55:00 +0000 Subject: [PATCH 10/30] web: try harder to stop users accidentally playing two lessons at once --- Makefile | 1 - server/gradint.cgi | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 537592e..6bed3d9 100644 --- a/Makefile +++ b/Makefile @@ -232,7 +232,6 @@ publish: $(All_Versions) gradint.py grep ^program_name < src/top.py|head -1|sed -e 's/.*radint v/v/' -e 's/ .*/./' > ~/homepage/public/gradint/latest-version.txt make clean ~/homepage/update - ssh st0rage "cd eGuidedog/ssb22/gradint; screen -d -m /bin/bash -c 'sleep 60;. build-sync.sh'" gradint-build.7z: mkdir /tmp/gradint-build00 diff --git a/server/gradint.cgi b/server/gradint.cgi index 9df0bc9..a0a0bd5 100755 --- a/server/gradint.cgi +++ b/server/gradint.cgi @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # (either Python 2 or Python 3) -program_name = "gradint.cgi v1.32 (c) 2011,2015,2017-22 Silas S. Brown. GPL v3+" +program_name = "gradint.cgi v1.33 (c) 2011,2015,2017-23 Silas S. Brown. GPL v3+" # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -142,7 +142,7 @@ def main(): try: v=open(gradint.vocabFile).read() except: v="" # (shouldn't get here unless they hack URLs) htmlOut('

|
',"Text edit your vocab list") - elif "lesson" in query: # make lesson + elif "lesson" in query: # make lesson ("Start lesson" button) setup_userID() gradint.maxNewWords = int(query.getfirst("new")) # (shouldn't need sensible-range check here if got a dropdown; if they really want to hack the URL then ok...) gradint.maxLenOfLesson = int(float(query.getfirst("mins"))*60) @@ -415,7 +415,7 @@ def listVocab(hasList): # main screen if data: hasList = "

"+"".join(["%s" % (num,gradint.secondLanguage,htmlize(dest,gradint.secondLanguage),gradint.firstLanguage,htmlize(src,gradint.firstLanguage),deleteLink(src,dest)) for num,src,dest in data])+"
"+localise("Your word list",1)+" ("+localise("click for audio",1)+")
"+localise("Repeats",1)+""+localise(gradint.secondLanguage,1)+""+localise(gradint.firstLanguage,1)+"
%d%s%s
" else: hasList="" else: hasList="" - if hasList: body += '

'+numSelect('new',range(2,10),gradint.maxNewWords)+' '+localise("new words in")+' '+numSelect('mins',[15,20,25,30],int(gradint.maxLenOfLesson/60))+' '+localise('mins')+"""
""" + if hasList: body += '

'+numSelect('new',range(2,10),gradint.maxNewWords)+' '+localise("new words in")+' '+numSelect('mins',[15,20,25,30],int(gradint.maxLenOfLesson/60))+' '+localise('mins')+"""
""" if "dictionary" in query: if query.getfirst("dictionary")=="1": body += '' # apparently it is -1, not -2; the redirect doesn't count as one (TODO are there any JS browsers that do count it as 2?) else: body += '

'+localise("Back to dictionary",1)+'' # TODO check for cross-site scripting From 0bd9f2b367bf5fb5f93c3bef9008c5ff54d3b090 Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Wed, 6 Dec 2023 06:19:03 +0000 Subject: [PATCH 11/30] server avoid spurious settings files; update identifier wording --- server/gradint.cgi | 18 +++++++----- src/frontend.py | 14 ++++----- src/synth.py | 8 ++--- thindown.py | 73 ++++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 88 insertions(+), 25 deletions(-) diff --git a/server/gradint.cgi b/server/gradint.cgi index a0a0bd5..c70c60c 100755 --- a/server/gradint.cgi +++ b/server/gradint.cgi @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # (either Python 2 or Python 3) -program_name = "gradint.cgi v1.33 (c) 2011,2015,2017-23 Silas S. Brown. GPL v3+" +program_name = "gradint.cgi v1.34 (c) 2011,2015,2017-23 Silas S. Brown. GPL v3+" # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,7 +19,7 @@ path_add = "$HOME/gradint/bin" # include sox, lame, espeak, maybe oggenc lib_path_add = "$HOME/gradint/lib" espeak_data_path = "$HOME/gradint" -import os, os.path, sys, cgi, urllib, time +import os, os.path, sys, cgi, urllib, time, re try: from commands import getoutput # Python 2 except: from subprocess import getoutput # Python 3 try: from urllib import quote,quote_plus,unquote # Python 2 @@ -74,8 +74,10 @@ reinit_gradint() def main(): if "id" in query: # e.g. from redirectHomeKeepCookie - os.environ["HTTP_COOKIE"]="id="+query.getfirst("id") - print ('Set-Cookie: id=' + query.getfirst("id")+'; expires=Wed, 1 Dec 2036 23:59:59 GMT') + queryID = query.getfirst("id") + if not re.match("[A-Za-z0-9_.-]",queryID): return htmlOut("Bad query.  Bad, bad query.") # to avoid cluttering the disk if we're being given random queries by an attacker. IDs we generate are numeric only, but allow alphanumeric in case server admin wants to generate them. Don't allow =, parens, etc (likely random SQL query) + os.environ["HTTP_COOKIE"]="id="+queryID + print ('Set-Cookie: id=' + queryID+'; expires=Wed, 1 Dec 2036 23:59:59 GMT') # TODO: S2G if has_userID(): setup_userID() # always, even for justSynth, as it may include a voice selection (TODO consequently being called twice in many circumstances, could make this more efficient) filetype="" if "filetype" in query: filetype=query.getfirst("filetype") @@ -95,7 +97,7 @@ def main(): gradint.justSynthesize="0" if "l2w" in query and query.getfirst("l2w"): gradint.startBrowser=lambda *args:0 - if query.getfirst("l2")=="zh" and gradint.sanityCheck(query.getfirst("l2w"),"zh"): gradint.justSynthesize += "#en Pinyin needs tones. Please go back and add tone numbers." # speaking it because alert box might not work and we might be being called from HTML5 Audio stuff (TODO maybe duplicate sanityCheck in js, if so don't call HTML5 audio, then we can have an on-screen message here) + if query.getfirst("l2")=="zh" and gradint.generalCheck(query.getfirst("l2w"),"zh"): gradint.justSynthesize += "#en Pinyin needs tones. Please go back and add tone numbers." # speaking it because alert box might not work and we might be being called from HTML5 Audio stuff (TODO maybe duplicate generalCheck in js, if so don't call HTML5 audio, then we can have an on-screen message here) else: gradint.justSynthesize += "#"+query.getfirst("l2").replace("#","").replace('"','')+" "+query.getfirst("l2w").replace("#","").replace('"','') if "l1w" in query and query.getfirst("l1w"): gradint.justSynthesize += "#"+query.getfirst("l1").replace("#","").replace('"','')+" "+query.getfirst("l1w").replace("#","").replace('"','') if gradint.justSynthesize=="0": return htmlOut('You must type a word in the box before pressing the Speak button.'+backLink) # TODO maybe add a Javascript test to the form also, IF can figure out if window.alert works @@ -103,9 +105,9 @@ def main(): elif "add" in query: # add to vocab (l1,l2 the langs, l1w,l2w the words) if "l2w" in query and query.getfirst("l2w") and "l1w" in query and query.getfirst("l1w"): gradint.startBrowser=lambda *args:0 - if query.getfirst("l2")=="zh": scmsg=gradint.sanityCheck(query.getfirst("l2w"),"zh") - else: scmsg=None - if scmsg: htmlOut(gradint.B(scmsg)+gradint.B(backLink)) + if query.getfirst("l2")=="zh": gcmsg=gradint.generalCheck(query.getfirst("l2w"),"zh") + else: gcmsg=None + if gcmsg: htmlOut(gradint.B(gcmsg)+gradint.B(backLink)) else: addWord(query.getfirst("l1w"),query.getfirst("l2w"),query.getfirst("l1"),query.getfirst("l2")) else: htmlOut('You must type words in both boxes before pressing the Add button.'+backLink) # TODO maybe add a Javascript test to the form also, IF can figure out a way to tell whether window.alert() works or not elif "bulkadd" in query: # bulk adding, from authoring options diff --git a/src/frontend.py b/src/frontend.py index 3d4699b..a2088db 100644 --- a/src/frontend.py +++ b/src/frontend.py @@ -1295,10 +1295,10 @@ def openDirectory(dir,inGuiThread=0): if inGuiThread: tkMessageBox.showinfo(app.master.title(),msg) else: waitOnMessage(msg) -def sanityCheck(text,language,pauseOnError=0): # text is utf-8; returns error message if any +def generalCheck(text,language,pauseOnError=0): # text is utf-8; returns error message if any if not text: return # always OK empty strings if pauseOnError: - ret = sanityCheck(text,language) + ret = generalCheck(text,language) if ret: waitOnMessage(ret) return ret if language=="zh": @@ -1329,7 +1329,7 @@ def s60_addVocab(): result = appuifw.multi_query(label1,label2) # unfortunately multi_query can't take default items (and sometimes no T9!), but Form is too awkward (can't see T9 mode + requires 2-button save via Options) and non-multi query would be even more modal if not result: return # cancelled l2,l1 = result # guaranteed to both be populated - while sanityCheck(l2.encode('utf-8'),secondLanguage,1): + while generalCheck(l2.encode('utf-8'),secondLanguage,1): l2=appuifw.query(label1,"text",u"") if not l2: return # cancelled # TODO detect duplicates like Tk GUI does? @@ -1371,7 +1371,7 @@ def s60_viewVocab(): oldL1,oldL2 = l1,l2 if action==2: first=1 - while first or (l2 and sanityCheck(l2.encode('utf-8'),secondLanguage,1)): + while first or (l2 and generalCheck(l2.encode('utf-8'),secondLanguage,1)): first=0 ; l2=appuifw.query(ensure_unicode(secondLanguage),"text",l2) if not l2: continue elif action==3: @@ -1386,7 +1386,7 @@ def s60_viewVocab(): def android_addVocab(): while True: l2 = None - while not l2 or sanityCheck(l2.encode('utf-8'),secondLanguage,1): + while not l2 or generalCheck(l2.encode('utf-8'),secondLanguage,1): l2 = android.dialogGetInput("Add word","Word in %s" % localise(secondLanguage)).result if not l2: return # cancelled l1 = android.dialogGetInput("Add word","Meaning in %s" % localise(firstLanguage)).result @@ -1585,7 +1585,7 @@ def gui_event_loop(): if not text1 and not text2: app.todo.alert=u"Before pressing the "+localise("Speak")+u" button, you need to type the text you want to hear into the box." else: if text1.startswith(B('#')): msg="" # see below - else: msg=sanityCheck(text1,secondLanguage) + else: msg=generalCheck(text1,secondLanguage) if msg: app.todo.alert=ensure_unicode(msg) else: app.set_watch_cursor = 1 ; app.toRestore = [] @@ -1706,7 +1706,7 @@ def scanDirs(): app.todo.alert=msg+" "+localise("Repeat count is 0, so we cannot reduce it for extra revision.") elif menu_response=="add": text1 = asUnicode(app.Text1.get()).encode('utf-8') ; text2 = asUnicode(app.Text2.get()).encode('utf-8') - msg=sanityCheck(text1,secondLanguage) + msg=generalCheck(text1,secondLanguage) if msg: app.todo.alert=ensure_unicode(msg) else: o=appendVocabFileInRightLanguages() diff --git a/src/synth.py b/src/synth.py index e35ea22..18871e3 100644 --- a/src/synth.py +++ b/src/synth.py @@ -1162,7 +1162,7 @@ def abspath_from_start(p): # for just_synthesize to check for paths relative to os.chdir(d) return r -def just_synthesize(callSanityCheck=0,lastLang_override=None): +def just_synthesize(callGeneralCheck=0,lastLang_override=None): # Handle the justSynthesize setting (see advanced.txt) global startAnnouncement,endAnnouncement,logFile,synth_partials_cache synth_partials_cache = {} # to stop 'memory leak' when running from the GUI @@ -1197,7 +1197,7 @@ def checkCanSynth(fname): r = repr(l[0]) if r[:1]=="b": r=r[1:] show_warning("Assuming that %s is a word to synthesize in language '%s'" % (r,lastLanguage)) - if callSanityCheck and sanityCheck(l[0],lastLanguage,1): return + if callGeneralCheck and generalCheck(l[0],lastLanguage,1): return event = checkCanSynth("!synth:"+S(l[0])+"_"+S(lastLanguage)) if not event: continue # couldn't synth called_synth = 1 @@ -1217,10 +1217,10 @@ def checkCanSynth(fname): lastLanguage=lang ; continue # otherwise, user might have omitted lang by mistake show_warning("Assuming %s was meant to be synthesized in language '%s'" % (cond(B('#') in B(justSynthesize) or len(repr(line))<10,"that '"+repr(line)+"'","this line"),lastLanguage)) - if callSanityCheck and sanityCheck(line,lastLanguage,1): return + if callGeneralCheck and generalCheck(line,lastLanguage,1): return event = checkCanSynth("!synth:"+S(line)+"_"+S(lastLanguage)) else: - if callSanityCheck and sanityCheck(text,lang,1): return + if callGeneralCheck and generalCheck(text,lang,1): return event = checkCanSynth(fname) lastLanguage = lang if not event: continue diff --git a/thindown.py b/thindown.py index 1c0c92a..05737ef 100644 --- a/thindown.py +++ b/thindown.py @@ -1,3 +1,9 @@ +#!/usr/bin/env python +# (works on either Python 2 or Python 3) + +# program to "thin down" the gradint .py for low memory environments +# by taking out some of the code that's unused on that platform + # This file is part of the source code of Gradint # (c) Silas S. Brown. # This program is free software; you can redistribute it and/or modify @@ -8,11 +14,6 @@ # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -#!/usr/bin/env python -# (works on either Python 2 or Python 3) - -# program to "thin down" the gradint .py for low memory environments -# by taking out some of the code that's unused on that platform import sys, re @@ -230,7 +231,67 @@ to_omit = desktop_only + S60_only + android_only + android_or_S60 + not_winCE + riscos_only + mac_only elif "core" in sys.argv: # experimental "core code only" for 'minimal embedded porting' starting point (no UI, no synth, limited file I/O; you'll probably have to load up the event data yourself) version = "core" - to_omit = tk_only + not_S60_or_android + not_android + riscos_only + mac_only + desktop_only + winCE_only + S60_only + android_only + android_or_S60 + ["def main():","def rest_of_main():",'if __name__=="__main__":',"def transliterates_differently(text,lang):","def primitive_synthloop():","def appendVocabFileInRightLanguages():",'def delOrReplace(L2toDel,L1toDel,newL2,newL1,action="delete"):',"def sanityCheck(text,language,pauseOnError=0):","def localise(s):","def singular(number,s):","def readText(l):","def asUnicode(x):","def updateSettingsFile(fname,newVals):","def clearScreen():","def startBrowser(url):",'def getYN(msg,defaultIfEof="n"):',"def waitOnMessage(msg):","def interrupt_instructions():","def parseSynthVocab(fname,forGUI=0):","def scanSamples_inner(directory,retVal,doLimit):","def getLsDic(directory):","def check_has_variants(directory,ls):","def exec_in_a_func(x):","def scanSamples(directory=None):","def synth_from_partials(text,lang,voice=None,isStart=1):","def partials_langname(lang):","if partialsDirectory and isDirectory(partialsDirectory):",'for zipToCheck in ["yali-voice","yali-lower","cameron-voice"]:','def stripPuncEtc(text):','def can_be_synthesized(fname,dirBase=None,lang=None):','def synthcache_lookup(fname,dirBase=None,printErrors=0,justQueryCache=0,lang=None):','def textof(fname):','if synthCache and transTbl in synthCache_contents:','if synthCache:','class Partials_Synth(Synth):','def abspath_from_start(p):','class SynthEvent(Event):','def pinyin_uColon_to_V(pinyin):','def synth_event(language,text,is_prompt=0):','def get_synth_if_possible(language,warn=1,to_transliterate=False):','if wavPlayer_override or (unix and not macsound and not (oss_sound_device=="/dev/sound/dsp" or oss_sound_device=="/dev/dsp")):','def fix_compatibility(utext):','def read_chinese_number(num):','def preprocess_chinese_numbers(utext,isCant=0):','def intor0(v):','def fix_pinyin(pinyin,en_words):','def fix_commas(text):','def shell_escape(text):','class SimpleZhTransliterator(object):','def sort_out_pinyin_3rd_tones(pinyin):','def ensure_unicode(text):','def unzip_and_delete(f,specificFiles="",ignore_fail=0):','class Synth(object):','def quickGuess(letters,lettersPerSec):',"def changeToDirOf(file,winsound_also=0):",'if app or appuifw or android:','def subst_some_synth_for_synthcache(events):','def decide_subst_synth(cache_fname):','if winsound or winCEsound or mingw32 or riscos_sound or not hasattr(os,"tempnam") or android:','if len(sys.argv)>1:','def readSettings(f):','def exc_info(inGradint=True):','if not fileExists(configFiles[0]):','def u8strip(d):',] + to_omit = tk_only + not_S60_or_android + not_android + riscos_only + mac_only + desktop_only + winCE_only + S60_only + android_only + android_or_S60 + [ +"def main():", +"def rest_of_main():", +'if __name__=="__main__":', +"def transliterates_differently(text,lang):", +"def primitive_synthloop():", +"def appendVocabFileInRightLanguages():", +'def delOrReplace(L2toDel,L1toDel,newL2,newL1,action="delete"):', +"def generalCheck(text,language,pauseOnError=0):", +"def localise(s):", +"def singular(number,s):", +"def readText(l):", +"def asUnicode(x):", +"def updateSettingsFile(fname,newVals):", +"def clearScreen():", +"def startBrowser(url):",'def getYN(msg,defaultIfEof="n"):',"def waitOnMessage(msg):", +"def interrupt_instructions():", +"def parseSynthVocab(fname,forGUI=0):", +"def scanSamples_inner(directory,retVal,doLimit):", +"def getLsDic(directory):", +"def check_has_variants(directory,ls):", +"def exec_in_a_func(x):", +"def scanSamples(directory=None):", +"def synth_from_partials(text,lang,voice=None,isStart=1):", +"def partials_langname(lang):", +"if partialsDirectory and isDirectory(partialsDirectory):", +'for zipToCheck in ["yali-voice","yali-lower","cameron-voice"]:', +'def stripPuncEtc(text):', +'def can_be_synthesized(fname,dirBase=None,lang=None):', +'def synthcache_lookup(fname,dirBase=None,printErrors=0,justQueryCache=0,lang=None):', +'def textof(fname):', +'if synthCache and transTbl in synthCache_contents:', +'if synthCache:', +'class Partials_Synth(Synth):', +'def abspath_from_start(p):', +'class SynthEvent(Event):', +'def pinyin_uColon_to_V(pinyin):', +'def synth_event(language,text,is_prompt=0):', +'def get_synth_if_possible(language,warn=1,to_transliterate=False):', +'if wavPlayer_override or (unix and not macsound and not (oss_sound_device=="/dev/sound/dsp" or oss_sound_device=="/dev/dsp")):', +'def fix_compatibility(utext):', +'def read_chinese_number(num):', +'def preprocess_chinese_numbers(utext,isCant=0):', +'def intor0(v):', +'def fix_pinyin(pinyin,en_words):', +'def fix_commas(text):', +'def shell_escape(text):', +'class SimpleZhTransliterator(object):', +'def sort_out_pinyin_3rd_tones(pinyin):', +'def ensure_unicode(text):', +'def unzip_and_delete(f,specificFiles="",ignore_fail=0):', +'class Synth(object):', +'def quickGuess(letters,lettersPerSec):',"def changeToDirOf(file,winsound_also=0):",'if app or appuifw or android:', +'def subst_some_synth_for_synthcache(events):', +'def decide_subst_synth(cache_fname):', +'if winsound or winCEsound or mingw32 or riscos_sound or not hasattr(os,"tempnam") or android:', +'if len(sys.argv)>1:', +'def readSettings(f):', +'def exc_info(inGradint=True):', +'if not fileExists(configFiles[0]):', +'def u8strip(d):'] else: assert 0, "Unrecognised version on command line" revertToIndent = lastIndentLevel = indentLevel = -1 From 6b0896d24689ca1a7244a8ad8f94226b764a3c54 Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Sun, 10 Dec 2023 09:36:02 +0000 Subject: [PATCH 12/30] Mac: fix launcher for macOS 14 --- mac/start-gradint.app/Contents/MacOS/start-gradint | 1 + 1 file changed, 1 insertion(+) diff --git a/mac/start-gradint.app/Contents/MacOS/start-gradint b/mac/start-gradint.app/Contents/MacOS/start-gradint index 1b697b7..e159ee9 100755 --- a/mac/start-gradint.app/Contents/MacOS/start-gradint +++ b/mac/start-gradint.app/Contents/MacOS/start-gradint @@ -1,5 +1,6 @@ #!/bin/bash export PATH="$PATH:/usr/local/bin" # in case lame etc is there +cd "${BASH_SOURCE%/*}/../.." # needed on macOS 14, possibly 13 if sw_vers 2>/dev/null|grep ^ProductVersion.*1[2-9]; then # macOS 12+ if test $(python3 -c 'import tkinter,sys;print(sys.version_info[:3]>=(3,10,1))' 2>/dev/null) = "True"; then exec python3 gradint.py; fi osascript -e "tell application (path to frontmost application as text) to display dialog \"macOS 12 bundled a broken version of the GUI libraries: please install Python 3 from python.org before running Gradint\" buttons {\"OK\"} with icon stop" From 706d688d276f153444e957f5e11f49dca6553f24 Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Tue, 12 Dec 2023 08:54:49 +0000 Subject: [PATCH 13/30] fix occasional Python 3 crash --- src/makeevent.py | 2 +- src/play.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/makeevent.py b/src/makeevent.py index ea13e5d..f5d1875 100644 --- a/src/makeevent.py +++ b/src/makeevent.py @@ -363,7 +363,7 @@ def toDict(l): # make the list of filenames into a dict of short-key -> [(long-k except OSError: pass if partials_raw_mode: (wtype,wrate,wchannels,wframes,wbits) = sndhdr.what(partialsDirectory+os.sep+"header"+dotwav) - partials_raw_0bytes = int(betweenPhrasePause*wrate)*wchannels*(wbits/8) + partials_raw_0bytes = int(betweenPhrasePause*wrate)*wchannels*int(wbits/8) else: synth_partials_voices,partials_raw_mode = {},None if checkIn("cant",synth_partials_voices): synth_partials_voices["zhy"]=synth_partials_voices["zh-yue"]=synth_partials_voices["cant"] diff --git a/src/play.py b/src/play.py index 5e730f4..e3da285 100644 --- a/src/play.py +++ b/src/play.py @@ -450,7 +450,7 @@ def pcmlen(file): if wrate==6144: # might be a .3gp from android_recordFile d = open(file).read() if 'mdat' in d: return (len(d)-d.index('mdat'))/1500.0 # this assumes the bitrate is roughly the same as in my tests, TODO figure it out properly - divisor = wrate*wchannels*wbits/8 # do NOT optimise with (wbits>>3), because wbits could be 4 + divisor = wrate*wchannels*int(wbits/8) # do NOT optimise with (wbits>>3), because wbits could be 4 if not divisor: raise IOError("Cannot parse sample format of '%s': %s" % (file,repr(header))) return (filelen(file) - 44.0) / divisor # 44 is a typical header length, and .0 to convert to floating-point From 61f718cc4e6f438d47edc6c961284cc9e36ca1d8 Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Tue, 19 Dec 2023 09:00:39 +0000 Subject: [PATCH 14/30] fix Mac afplay partials-synth on Python 3 --- src/top.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/top.py b/src/top.py index e74fc1b..b41c2d9 100644 --- a/src/top.py +++ b/src/top.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # (Python 2 or Python 3, but more fully tested on 2) -program_name = "gradint v3.076 (c) 2002-23 Silas S. Brown. GPL v3+." +program_name = "gradint v3.077 (c) 2002-23 Silas S. Brown. GPL v3+." # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -32,7 +32,9 @@ def sort(l,c): l.sort(key=cmp_to_key(c)) def chr(x): return unichr(x).encode('latin1') from subprocess import getoutput popenRB,popenWB = "r","w" - def unicode(b,enc): return b.decode(enc) + def unicode(b,enc): + if type(b)==str: return b + return b.decode(enc) else: # Python 2 def sort(l,c): l.sort(c) popenRB,popenWB = "rb","wb" From 08fd2de4054e445e78b5fb2f7fe2a6f49a36dee5 Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Tue, 2 Jan 2024 08:34:16 +0000 Subject: [PATCH 15/30] Rasbperry Pi OS 12 Bluetooth settings --- samples/utils/player.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/utils/player.py b/samples/utils/player.py index e09ed01..521af15 100644 --- a/samples/utils/player.py +++ b/samples/utils/player.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # (should work in both Python 2 and Python 3) -# Simple sound-playing server v1.56 +# Simple sound-playing server v1.57 # Silas S. Brown - public domain - no warranty # connect to port 8124 (assumes behind firewall) @@ -13,8 +13,8 @@ import socket, select, os, sys, os.path, time, re for a in sys.argv[1:]: - if a.startswith("--rpi-bluetooth-setup"): # tested on Raspberry Pi 400 with Raspbian 11; also tested on Raspberry Pi Zero W with Raspbian 10 Lite (with the device already paired: needed to say "scan on", "discovery on", remove + pair in bluetoothctl). Send Eth=(bluetooth Ethernet addr) to start. Note that the setup command reboots the system. - os.system('if [ -e /etc/xdg/lxsession/LXDE-pi/autostart ]; then mkdir -p /home/pi/.config/lxsession/LXDE-pi && cp /etc/xdg/lxsession/LXDE-pi/autostart /home/pi/.config/lxsession/LXDE-pi/ && echo sudo ethtool --set-eee eth0 eee off >> /home/pi/.config/lxsession/LXDE-pi/autostart && echo python '+os.path.join(os.getcwd(),sys.argv[0])+' >> /home/pi/.config/lxsession/LXDE-pi/autostart; else (echo "[Unit]";echo "Descrption=Gradint player utility";echo "[Service]";echo "Type=oneshot";echo "ExecStart='+os.path.join(os.getcwd(),sys.argv[0])+'";echo "[Install]";echo "WantedBy=multi-user.target") > player.service && sudo mv player.service /etc/systemd/system/ && sudo systemctl daemon-reload && sudo systemctl enable player && chmod +x '+sys.argv[0]+' && awk '+"'"+'// {print} /^import / {print "os.system('+"'"+'"'+"'"+'"'+"'"+'pulseaudio --start'+"'"+'"'+"'"+'"'+"'"+')"}'+"'"+' < '+sys.argv[0]+' > .playerTMP && mv .playerTMP '+sys.argv[0]+'; fi && sudo "apt-get -y install sox mpg123 pulseaudio pulseaudio-module-bluetooth && usermod -G bluetooth -a pi && (echo load-module module-switch-on-connect;echo load-module module-bluetooth-policy;echo load-module module-bluetooth-discover) >> /etc/pulse/default.pa && (echo [General];echo FastConnectable = true) >> /etc/bluetooth/main.conf && reboot"') # (eee off: improves reliability of gigabit ethernet on RPi400) + if a.startswith("--rpi-bluetooth-setup"): # tested on Raspberry Pi 400 with Raspbian 11 and Raspbian 12; also tested on Raspberry Pi Zero W with Raspbian 10 Lite (with the device already paired: needed to say "scan on", "discovery on", remove + pair in bluetoothctl). Send Eth=(bluetooth Ethernet addr) to start. Note that the setup command reboots the system. + os.system('(echo "[Unit]";echo "Description=Gradint player utility";echo "[Service]";echo "Type=oneshot";echo "ExecStart='+os.path.join(os.getcwd(),sys.argv[0])+'";echo "WorkingDirectory='+os.path.getcwd()+'";echo User="$(whoami)";echo "[Install]";echo "WantedBy=multi-user.target") > player.service && sudo mv player.service /etc/systemd/system/ && sudo systemctl daemon-reload && sudo systemctl enable player && chmod +x '+sys.argv[0]+' && awk '+"'"+'// {print} /^import / {print "os.system('+"'"+'"'+"'"+'"'+"'"+'pulseaudio --start'+"'"+'"'+"'"+'"'+"'"+')"}'+"'"+' < '+sys.argv[0]+' > .playerTMP && cat .playerTMP > '+sys.argv[0]+' && rm .playerTMP && sudo bash -c "apt-get -y install sox mpg123 pulseaudio pulseaudio-module-bluetooth && usermod -G bluetooth -a $USER && (echo load-module module-switch-on-connect;echo load-module module-bluetooth-policy;echo load-module module-bluetooth-discover) >> /etc/pulse/default.pa && (echo [General];echo FastConnectable = true) >> /etc/bluetooth/main.conf && reboot"') # (eee off: improves reliability of gigabit ethernet on RPi400) elif a=="--aplay": use_aplay = True # aplay and madplay, for older embedded devices, NOT tested together with --rpi-bluetooth-* above elif a.startswith("--delegate="): delegate_to_check=a.split('=')[1] # will ping that IP and delegate all sound to it when it's up. E.g. if it has better amplification but it's not always switched on. elif a.startswith("--chime="): chime_mp3=a.split('=')[1] # if clock bell desired, e.g. echo '$i-14vfff$c48o0l1b- @'|mwr2ly > chime.ly && lilypond chime.ly && timidity -Ow chime.midi && audacity chime.wav (amplify + trim) + mp3-encode (keep default 44100 sample rate so ~38 frames per sec). Not designed to work with --delegate. Pi1's 3.5mm o/p doesn't sound very good with this bell. From 3ff3eeaf7118e03728a46c55543023eda3a303c0 Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Sun, 14 Jan 2024 00:25:12 +0000 Subject: [PATCH 16/30] player does not need to start pulseaudio on startup --- samples/utils/player.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/utils/player.py b/samples/utils/player.py index 521af15..ff4d941 100644 --- a/samples/utils/player.py +++ b/samples/utils/player.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # (should work in both Python 2 and Python 3) -# Simple sound-playing server v1.57 +# Simple sound-playing server v1.58 # Silas S. Brown - public domain - no warranty # connect to port 8124 (assumes behind firewall) @@ -14,7 +14,7 @@ import socket, select, os, sys, os.path, time, re for a in sys.argv[1:]: if a.startswith("--rpi-bluetooth-setup"): # tested on Raspberry Pi 400 with Raspbian 11 and Raspbian 12; also tested on Raspberry Pi Zero W with Raspbian 10 Lite (with the device already paired: needed to say "scan on", "discovery on", remove + pair in bluetoothctl). Send Eth=(bluetooth Ethernet addr) to start. Note that the setup command reboots the system. - os.system('(echo "[Unit]";echo "Description=Gradint player utility";echo "[Service]";echo "Type=oneshot";echo "ExecStart='+os.path.join(os.getcwd(),sys.argv[0])+'";echo "WorkingDirectory='+os.path.getcwd()+'";echo User="$(whoami)";echo "[Install]";echo "WantedBy=multi-user.target") > player.service && sudo mv player.service /etc/systemd/system/ && sudo systemctl daemon-reload && sudo systemctl enable player && chmod +x '+sys.argv[0]+' && awk '+"'"+'// {print} /^import / {print "os.system('+"'"+'"'+"'"+'"'+"'"+'pulseaudio --start'+"'"+'"'+"'"+'"'+"'"+')"}'+"'"+' < '+sys.argv[0]+' > .playerTMP && cat .playerTMP > '+sys.argv[0]+' && rm .playerTMP && sudo bash -c "apt-get -y install sox mpg123 pulseaudio pulseaudio-module-bluetooth && usermod -G bluetooth -a $USER && (echo load-module module-switch-on-connect;echo load-module module-bluetooth-policy;echo load-module module-bluetooth-discover) >> /etc/pulse/default.pa && (echo [General];echo FastConnectable = true) >> /etc/bluetooth/main.conf && reboot"') # (eee off: improves reliability of gigabit ethernet on RPi400) + os.system('(echo "[Unit]";echo "Description=Gradint player utility";echo "[Service]";echo "Type=oneshot";echo "ExecStart='+os.path.join(os.getcwd(),sys.argv[0])+'";echo "WorkingDirectory='+os.path.getcwd()+'";echo User="$(whoami)";echo "[Install]";echo "WantedBy=multi-user.target") > player.service && sudo mv player.service /etc/systemd/system/ && sudo systemctl daemon-reload && sudo systemctl enable player && chmod +x '+sys.argv[0]+' && sudo bash -c "apt-get -y install sox mpg123 pulseaudio pulseaudio-module-bluetooth && usermod -G bluetooth -a $USER && (echo load-module module-switch-on-connect;echo load-module module-bluetooth-policy;echo load-module module-bluetooth-discover) >> /etc/pulse/default.pa && (echo [General];echo FastConnectable = true) >> /etc/bluetooth/main.conf && reboot"') # (eee off: improves reliability of gigabit ethernet on RPi400) elif a=="--aplay": use_aplay = True # aplay and madplay, for older embedded devices, NOT tested together with --rpi-bluetooth-* above elif a.startswith("--delegate="): delegate_to_check=a.split('=')[1] # will ping that IP and delegate all sound to it when it's up. E.g. if it has better amplification but it's not always switched on. elif a.startswith("--chime="): chime_mp3=a.split('=')[1] # if clock bell desired, e.g. echo '$i-14vfff$c48o0l1b- @'|mwr2ly > chime.ly && lilypond chime.ly && timidity -Ow chime.midi && audacity chime.wav (amplify + trim) + mp3-encode (keep default 44100 sample rate so ~38 frames per sec). Not designed to work with --delegate. Pi1's 3.5mm o/p doesn't sound very good with this bell. @@ -69,9 +69,9 @@ continue elif d=='QUIT': s.close() ; break - elif d=="Eth=": # Eth=ethernet address, to connect via Bluetooth, tested on Raspberry Pi 400 with Raspbian 11 + elif d=="Eth=": # Eth=ethernet address to connect via Bluetooth (see --rpi-bluetooth-setup above) eth = S(c.recv(17)) - assert re.match("^[A-Fa-f0-9:]*$",eth) + assert re.match("^[A-Fa-f0-9:]+$",eth) os.system("M=/dev/null;E="+eth+";if ! pacmd list-sinks | grep "+eth.replace(":","_")+" >$M; then while true; do bluetoothctl --timeout 1 disconnect | grep Missing >$M||sleep 5;T=5;while ! bluetoothctl --timeout $T connect $E | tee $M | egrep \"Connection successful|Device $E Connected: yes\"; do sleep 5; T=10;M=/dev/stderr;bluetoothctl --timeout 1 devices;echo Retrying $E; done ; Got=0; for Try in 1 2 3 4 5 6 7 8 9 a b c d e f g h i j k l m n o p q r s t u v w x y z; do if pacmd list-sinks | grep "+eth.replace(":","_")+" >/dev/null; then Got=1; break; fi; sleep 1; done; if [ $Got = 1 ] ; then break; fi; done; fi; pacmd set-default-sink bluez_sink."+eth.replace(":","_")+".a2dp_sink") # ; play /usr/share/scratch/Media/Sounds/Animal/Dog1.wav # (not really necessary if using 'close the socket' to signal we're ready) c.close() ; continue elif d=="Eth0": From 6491bd3a550faf442771d00e999da255294964e1 Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Tue, 16 Jan 2024 13:26:43 +0000 Subject: [PATCH 17/30] player RPi12 comment --- samples/utils/player.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/utils/player.py b/samples/utils/player.py index ff4d941..409d814 100644 --- a/samples/utils/player.py +++ b/samples/utils/player.py @@ -13,7 +13,8 @@ import socket, select, os, sys, os.path, time, re for a in sys.argv[1:]: - if a.startswith("--rpi-bluetooth-setup"): # tested on Raspberry Pi 400 with Raspbian 11 and Raspbian 12; also tested on Raspberry Pi Zero W with Raspbian 10 Lite (with the device already paired: needed to say "scan on", "discovery on", remove + pair in bluetoothctl). Send Eth=(bluetooth Ethernet addr) to start. Note that the setup command reboots the system. + if a.startswith("--rpi-bluetooth-setup"): # tested on Raspberry Pi 400 with OS versions 11 and 12; also tested on Raspberry Pi Zero W with Raspbian 10 Lite (with the device already paired: needed to say "scan on", "discovery on", remove + pair in bluetoothctl). Send Eth=(bluetooth Ethernet addr) to start. Note that the setup command reboots the system. + # NOTE: If running on Pi with OS 12 and you've also done "raspi-config" to set things back to PulseAudio (as needed for example for language-synchronised Bluetooth playing in http://ssb22.user.srcf.net/s60/video.html notes), you might need to replace 'ExecStart=' with 'ExecStart=bash -c "while ! ssh localhost true; do sleep 1; done; ssh localhost ' below (and add a " at end of line), and do an ssh-keygen and add to authorized_keys, so player is run in a separate session from systemd (even though the user is the same; it's not clear why this is needed) os.system('(echo "[Unit]";echo "Description=Gradint player utility";echo "[Service]";echo "Type=oneshot";echo "ExecStart='+os.path.join(os.getcwd(),sys.argv[0])+'";echo "WorkingDirectory='+os.path.getcwd()+'";echo User="$(whoami)";echo "[Install]";echo "WantedBy=multi-user.target") > player.service && sudo mv player.service /etc/systemd/system/ && sudo systemctl daemon-reload && sudo systemctl enable player && chmod +x '+sys.argv[0]+' && sudo bash -c "apt-get -y install sox mpg123 pulseaudio pulseaudio-module-bluetooth && usermod -G bluetooth -a $USER && (echo load-module module-switch-on-connect;echo load-module module-bluetooth-policy;echo load-module module-bluetooth-discover) >> /etc/pulse/default.pa && (echo [General];echo FastConnectable = true) >> /etc/bluetooth/main.conf && reboot"') # (eee off: improves reliability of gigabit ethernet on RPi400) elif a=="--aplay": use_aplay = True # aplay and madplay, for older embedded devices, NOT tested together with --rpi-bluetooth-* above elif a.startswith("--delegate="): delegate_to_check=a.split('=')[1] # will ping that IP and delegate all sound to it when it's up. E.g. if it has better amplification but it's not always switched on. From be2aebfb70f6360abe2fe119821ed7fadd705e5c Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Sun, 21 Jan 2024 08:43:54 +0000 Subject: [PATCH 18/30] Cantonese handle emoji pass-through on Python 2 --- server/cantonese.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server/cantonese.py b/server/cantonese.py index 921642e..ecc4576 100644 --- a/server/cantonese.py +++ b/server/cantonese.py @@ -5,7 +5,7 @@ # cantonese.py - Python functions for processing Cantonese transliterations # (uses eSpeak and Gradint for help with some of them) -# v1.47 (c) 2013-15,2017-23 Silas S. Brown. License: GPL +# v1.48 (c) 2013-15,2017-24 Silas S. Brown. License: GPL cache = {} # to avoid repeated eSpeak runs, # zi -> jyutping or (pinyin,) -> translit @@ -237,7 +237,10 @@ def mysub(z,l): z = re.sub(re.escape(x)+r"(.)",r"\1"+y,z) return z if type(u"")==type(""): U=str # Python 3 - else: U=unicode # Python 2 + else: # Python 2 + def U(x): + try: return x.decode('utf-8') # might be an emoji pass-through + except: return x # already Unicode return unicodedata.normalize('NFC',mysub(U(jyutping_to_yale_TeX(j).replace(r"\i{}","i").replace(r"\I{}","I")),[(r"\`",u"\u0300"),(r"\'",u"\u0301"),(r"\=",u"\u0304")])).encode('utf-8') def superscript_digits_TeX(j): From 1f96861b23cf06a007e78b19f646d847423af6ed Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Sat, 23 Mar 2024 06:38:28 +0000 Subject: [PATCH 19/30] server script modernisations --- server/email-lesson.sh | 48 +++++++++++++++++++++--------------------- src/frontend.py | 6 +++--- src/play.py | 4 ++-- src/system.py | 2 +- src/top.py | 2 +- 5 files changed, 31 insertions(+), 31 deletions(-) diff --git a/server/email-lesson.sh b/server/email-lesson.sh index 8406ee7..17e0d95 100755 --- a/server/email-lesson.sh +++ b/server/email-lesson.sh @@ -3,9 +3,9 @@ # email-lesson.sh: a script that can help you to # automatically distribute daily Gradint lessons # to students using a web server with reminder -# emails. Version 1.15 +# emails. Version 1.16 -# (C) 2007-2010,2020-2022 Silas S. Brown, License: GPL +# (C) 2007-2010,2020-2022,2024 Silas S. Brown, License: GPL # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -39,7 +39,7 @@ elif which mutt >/dev/null 2>/dev/null; then DefaultMailProg="mutt -x" else DefaultMailProg="ssh example.org mail" fi -if test "a$1" == "a--run"; then +if [ "$1" == "--run" ]; then set -o pipefail # make sure errors in pipes are reported if ! [ -d email_lesson_users ]; then echo "Error: script does not seem to have been set up yet" @@ -61,14 +61,14 @@ if test "a$1" == "a--run"; then while true; do ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS -n -o ControlMaster=yes $ControlPath $(echo "$PUBLIC_HTML"|sed -e 's/:.*//') sleep 86400; sleep 10; done & MasterPid=$! else unset MasterPid fi - (while ! bash -c "$CAT_LOGS_COMMAND"; do echo "cat-logs failed, re-trying in 61 seconds" 1>&2;sleep 61; done) | grep '/user\.' > "$TMPDIR/._email_lesson_logs" + (while ! bash -c "$CAT_LOGS_COMMAND"; do echo "cat-logs failed, re-trying in 61 seconds" >&2;sleep 61; done) | grep '/user\.' > "$TMPDIR/._email_lesson_logs" # (note: sleeping odd numbers of seconds so we can tell where it is if it gets stuck in one of these loops) Users="$(echo user.*)" cd .. unset NeedRunMirror for U in $Users; do . email_lesson_users/config - if ! test "a$GLOBAL_GRADINT_OPTIONS" == a; then GLOBAL_GRADINT_OPTIONS="$GLOBAL_GRADINT_OPTIONS ;"; fi + if [ "$GLOBAL_GRADINT_OPTIONS" ]; then GLOBAL_GRADINT_OPTIONS="$GLOBAL_GRADINT_OPTIONS ;"; fi # set some (but not all!) variables to defaults in case not set in profile SUBJECT_LINE="$DEFAULT_SUBJECT_LINE" FORGOT_YESTERDAY="$DEFAULT_FORGOT_YESTERDAY" @@ -85,7 +85,7 @@ if test "a$1" == "a--run"; then mv "email_lesson_users/$U/profile.removeCR" "email_lesson_users/$U/profile" fi . "email_lesson_users/$U/profile" - if test "a$Use_M3U" == ayes; then FILE_TYPE_2=m3u + if [ "$Use_M3U" == yes ]; then FILE_TYPE_2=m3u else FILE_TYPE_2=$FILE_TYPE; fi if echo "$MailProg" | grep ssh >/dev/null; then # ssh discards a level of quoting, so we need to be more careful @@ -94,7 +94,7 @@ if test "a$1" == "a--run"; then Extra_Mailprog_Params2="\"$Extra_Mailprog_Params2\"" fi if [ -e "email_lesson_users/$U/lastdate" ]; then - if test "$(cat "email_lesson_users/$U/lastdate")" == "$(date +%Y%m%d)"; then + if [ "$(cat "email_lesson_users/$U/lastdate")" == "$(date +%Y%m%d)" ]; then # still on same day - do nothing with this user this time continue fi @@ -114,10 +114,10 @@ if test "a$1" == "a--run"; then fi else Did_Download=1; fi rm -f "email_lesson_users/$U/rollback" - if test $Did_Download == 0; then + if [ $Did_Download == 0 ]; then # send a reminder DaysOld="$(python -c "import os,time;print(int((time.time()-os.stat('email_lesson_users/$U/lastdate').st_mtime)/3600/24))")" - if test $DaysOld -lt 5 || test $(date +%u) == 1; then # (remind only on Mondays if not checked for 5 days, to avoid filling up inboxes when people are away and can't get to email) + if [ $DaysOld -lt 5 ] || [ $(date +%u) == 1 ]; then # (remind only on Mondays if not checked for 5 days, to avoid filling up inboxes when people are away and can't get to email) while ! $MailProg -s "$SUBJECT_LINE" "$STUDENT_EMAIL" "$Extra_Mailprog_Params1" "$Extra_Mailprog_Params2" </dev/null; then OUTDIR=$TMPDIR else OUTDIR=$PUBLIC_HTML; fi USER_GRADINT_OPTIONS="$GLOBAL_GRADINT_OPTIONS $GRADINT_OPTIONS samplesDirectory='email_lesson_users/$U/samples'; progressFile='email_lesson_users/$U/progress.txt'; pickledProgressFile='email_lesson_users/$U/progress.bin'; vocabFile='email_lesson_users/$U/vocab.txt';saveLesson='';loadLesson=0;progressFileBackup='email_lesson_users/$U/progress.bak';outputFile=" @@ -147,14 +147,14 @@ do echo "mail sending failed; retrying in 62 seconds"; sleep 62; done; fi tail -$NumLines "email_lesson_users/$U/podcasts-to-send" > "email_lesson_users/$U/podcasts-to-send2" mv "email_lesson_users/$U/podcasts-to-send" "email_lesson_users/$U/podcasts-to-send.old" mv "email_lesson_users/$U/podcasts-to-send2" "email_lesson_users/$U/podcasts-to-send" - if test $NumLines == 0; then + if [ $NumLines == 0 ]; then echo "$U" | $MailProg -s Warning:email-lesson-run-out-of-podcasts $ADMIN_EMAIL fi else rm -f "email_lesson_users/$U/podcasts-to-send.old" # won't be a rollback after this fi - if test "$ENCODE_ON_REMOTE_HOST" == 1; then + if [ "$ENCODE_ON_REMOTE_HOST" == 1 ]; then ToSleep=123 - while ! if test "a$Send_Podcast_Instead" == a; then + while ! if [ ! "$Send_Podcast_Instead" ]; then python gradint.py "$USER_GRADINT_OPTIONS '-.sh'" "$TMPDIR/__stderr" | ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo "$PUBLIC_HTML"|sed -e 's/:.*//') "mkdir -p $REMOTE_WORKING_DIR; cd $REMOTE_WORKING_DIR; cat > __gradint.sh;chmod +x __gradint.sh;PATH=$SOX_PATH ./__gradint.sh|$ENCODING_COMMAND $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$CurDate.$FILE_TYPE;rm -f __gradint.sh"; else cd "email_lesson_users/$U" ; cat "$Send_Podcast_Instead" | ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo "$PUBLIC_HTML"|sed -e 's/:.*//') "cat > $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$CurDate.$FILE_TYPE"; cd ../..; @@ -166,18 +166,18 @@ do echo "mail sending failed; retrying in 62 seconds"; sleep 62; done; fi sleep $ToSleep ; ToSleep=$[$ToSleep*1.5] # (increasing-time retries) done rm "$TMPDIR/__stderr" - if test "a$Use_M3U" == ayes; then + if [ "$Use_M3U" == yes ]; then while ! ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo "$PUBLIC_HTML"|sed -e 's/:.*//') "echo $OUTSIDE_LOCATION/$U-$CurDate.$FILE_TYPE > $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$CurDate.m3u"; do sleep 63; done fi else # not ENCODE_ON_REMOTE_HOST - if ! test "a$Send_Podcast_Instead" == a; then + if [ "$Send_Podcast_Instead" ]; then (cd "email_lesson_users/$U" ; cat "$Send_Podcast_Instead") > "$OUTDIR/$U-$CurDate.$FILE_TYPE" elif ! python gradint.py "$USER_GRADINT_OPTIONS '$OUTDIR/$U-$CurDate.$FILE_TYPE'" "$OUTDIR/$U-$CurDate.m3u" fi if echo "$PUBLIC_HTML" | grep : >/dev/null; then @@ -200,14 +200,14 @@ EOF do echo "mail sending failed; retrying in 65 seconds"; sleep 65; done echo "$CurDate" > "email_lesson_users/$U/lastdate" unset AdminNote - if test "a$Send_Podcast_Instead" == a; then - if test "$(zgrep -H -m 1 lessonsLeft "email_lesson_users/$U/progress.txt"|sed -e 's/.*=//')" == 0; then AdminNote="Note: $U has run out of new words"; fi + if [ "$Send_Podcast_Instead" == a ]; then + if [ "$(zgrep -H -m 1 lessonsLeft "email_lesson_users/$U/progress.txt"|sed -e 's/.*=//')" == 0 ]; then AdminNote="Note: $U has run out of new words"; fi elif ! [ -e "email_lesson_users/$U/podcasts-to-send" ]; then AdminNote="Note: $U has run out of podcasts"; fi - if ! test "a$AdminNote" == a; then + if [ "$AdminNote" ]; then while ! echo "$AdminNote"|$MailProg -s gradint-user-ran-out "$ADMIN_EMAIL"; do echo "Mail sending failed; retrying in 67 seconds"; sleep 67; done fi done # end of per-user loop - if test "a$NeedRunMirror" == "a1" && ! test "a$PUBLIC_HTML_MIRROR_COMMAND" == a; then + if [ "$NeedRunMirror" == "1" ] && [ "$PUBLIC_HTML_MIRROR_COMMAND" ]; then while ! $PUBLIC_HTML_MIRROR_COMMAND; do echo "PUBLIC_HTML_MIRROR_COMMAND failed; retrying in 79 seconds" echo As subject | $MailProg -s "PUBLIC_HTML_MIRROR_COMMAND failed, will retry" "$ADMIN_EMAIL" || true # ignore errors @@ -215,9 +215,9 @@ do echo "mail sending failed; retrying in 65 seconds"; sleep 65; done done fi rm -f "$TMPDIR/._email_lesson_logs" - if ! test a$MasterPid == a; then + if [ $MasterPid ] ; then kill $MasterPid - kill $(ps axwww|grep "$TMPDIR/__gradint_ctrl"|sed -e 's/^ *//' -e 's/ .*//') 2>/dev/null + kill $(pgrep -f "$TMPDIR/__gradint_ctrl") 2>/dev/null rm -f "$TMPDIR/__gradint_ctrl" # in case ssh doesn't fi rm -f "$Gradint_Dir/.email-lesson-running" @@ -227,7 +227,7 @@ fi echo "After setting up users, run this script daily with --run on the command line." echo "As --run was not specified, it will now go into setup mode." # Setup: -if test "a$EDITOR" == a; then +if ! [ "$EDITOR" ]; then echo "Error: No EDITOR environment variable set"; exit 1 fi if ! [ -e email_lesson_users/config ]; then @@ -286,7 +286,7 @@ while true; do echo "Type a user alias (or just press Enter) to add a new user, or Ctrl-C to quit" read Alias ID=$(mktemp -d user.$(python -c 'import random; print(random.random())')XXXXXX) # (newer versions of mktemp allow more than 6 X's so the python step isn't necessary, but just in case we want to make sure that it's hard to guess the ID) - if ! test "a$Alias" == a; then ln -s "$ID" "$Alias"; fi + if [ "$Alias" ]; then ln -s "$ID" "$Alias"; fi cd "$ID" || exit 1 cat > profile <&2") # (1>&2 in case using stdout for something else) + else: os.system("clear >&2") # (>&2 in case using stdout for something else) return True cancelledFiles = [] @@ -1482,9 +1482,9 @@ def downloadLAME(): fi if grep downloads.sourceforge lame.tar.gz 2>/dev/null; then Link="$(cat lame.tar.gz|grep downloads.sourceforge|head -1)" - echo "Got HTML: $Link" 1>&2 + echo "Got HTML: $Link" >&2 Link="$(echo "$Link"|sed -e 's/.*http/http/' -e 's,.*/projects,http://sourceforge.net/projects,' -e 's/".*//')" - echo "Following link to $Link" 1>&2 + echo "Following link to $Link" >&2 if ! $Curl "$Link" > lame.tar.gz; then rm -f lame.tar.gz; exit 1 fi diff --git a/src/play.py b/src/play.py index e3da285..9994c2a 100644 --- a/src/play.py +++ b/src/play.py @@ -599,7 +599,7 @@ def beepCmd(soxParams,fname): class ShSoundCollector(object): def __init__(self): self.file2command = {} - self.commands = ["C() { echo -n $1% completed $'\r' 1>&2;}"] + self.commands = ["C() { echo -n $1% completed $'\r' >&2;}"] self.seconds = self.lastProgress = 0 if write_to_stdout: self.o=sys.stdout else: self.o = open(outputFile,"wb") @@ -656,7 +656,7 @@ def addFile(self,file,length): def finished(self): if outputFile_appendSilence: self.addSilence(outputFile_appendSilence,False) outfile_writeBytes(self.o,"\n") # so "tail" has a start of a line - self.commands.append("C 100;echo 1>&2;exit") + self.commands.append("C 100;echo >&2;exit") for c in self.commands: outfile_writeBytes(self.o,c+"\n") outfile_writeBytes(self.o,"tail -%d \"$S\" | bash\n" % (len(self.commands)+1)) if not write_to_stdout: diff --git a/src/system.py b/src/system.py index ec983c2..3476165 100644 --- a/src/system.py +++ b/src/system.py @@ -63,7 +63,7 @@ class ShellExecuteInfo(ctypes.Structure): _fields_ = [("cbSize",wintypes.DWORD), try: ctypes.cdll.commdlg except: WMstandard = True -if macsound and __name__=="__main__": os.system("clear 1>&2") # so warnings etc start with a clear terminal (1>&2 just in case using stdout for something else) +if macsound and __name__=="__main__": os.system("clear >&2") # so warnings etc start with a clear terminal (>&2 just in case using stdout for something else) if riscos_sound: sys.stderr.write("Loading Gradint...\n") # in case it takes a while try: import androidhelper as android diff --git a/src/top.py b/src/top.py index b41c2d9..629d92f 100644 --- a/src/top.py +++ b/src/top.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # (Python 2 or Python 3, but more fully tested on 2) -program_name = "gradint v3.077 (c) 2002-23 Silas S. Brown. GPL v3+." +program_name = "gradint v3.078 (c) 2002-24 Silas S. Brown. GPL v3+." # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From f9e8f3ca9765b36be7ada8b2f49fc630a346561e Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Sun, 9 Jun 2024 10:36:29 +0100 Subject: [PATCH 20/30] fix write to closed file on transliteration cache in Python 3 + ready for sndhdr removal in 3.13 --- advanced.txt | 2 +- samples/utils/autosplit.py | 0 samples/utils/cache-synth.py | 0 samples/utils/cleanup-cache.py | 0 samples/utils/diagram.py | 0 samples/utils/list-synth.py | 0 samples/utils/list2cache.py | 0 samples/utils/manual-splitter.py | 0 samples/utils/player.py | 0 samples/utils/recover-unavail.py | 0 samples/utils/synth-batchconvert-helper.py | 0 samples/utils/trace.py | 0 samples/utils/transliterate.py | 0 server/cantonese.py | 0 server/lesson-table.py | 0 server/safety-check-progressfile.py | 0 server/vocab2html.py | 0 src/makeevent.py | 5 +++-- src/play.py | 18 +++++++++++++----- src/system.py | 2 +- src/top.py | 6 +++--- thindown.py | 0 22 files changed, 21 insertions(+), 12 deletions(-) mode change 100644 => 100755 samples/utils/autosplit.py mode change 100644 => 100755 samples/utils/cache-synth.py mode change 100644 => 100755 samples/utils/cleanup-cache.py mode change 100644 => 100755 samples/utils/diagram.py mode change 100644 => 100755 samples/utils/list-synth.py mode change 100644 => 100755 samples/utils/list2cache.py mode change 100644 => 100755 samples/utils/manual-splitter.py mode change 100644 => 100755 samples/utils/player.py mode change 100644 => 100755 samples/utils/recover-unavail.py mode change 100644 => 100755 samples/utils/synth-batchconvert-helper.py mode change 100644 => 100755 samples/utils/trace.py mode change 100644 => 100755 samples/utils/transliterate.py mode change 100644 => 100755 server/cantonese.py mode change 100644 => 100755 server/lesson-table.py mode change 100644 => 100755 server/safety-check-progressfile.py mode change 100644 => 100755 server/vocab2html.py mode change 100644 => 100755 src/top.py mode change 100644 => 100755 thindown.py diff --git a/advanced.txt b/advanced.txt index 0dc41ae..3926c87 100644 --- a/advanced.txt +++ b/advanced.txt @@ -95,7 +95,7 @@ prefer_espeak = "en" # "zh" for Zhongwen (Mandarin). # - You can improve eSpeak's English by installing # Festival's dictionary and using lexconvert to convert -# it, see http://ssb22.user.srcf.net/gradint/lexconvert.html +# it, see http://ssb22.user.srcf.net/lexconvert/ # (this has already been done in the bundled version). # - eSpeak is not very natural-sounding, but it is very # clear and accurate in English and some other languages diff --git a/samples/utils/autosplit.py b/samples/utils/autosplit.py old mode 100644 new mode 100755 diff --git a/samples/utils/cache-synth.py b/samples/utils/cache-synth.py old mode 100644 new mode 100755 diff --git a/samples/utils/cleanup-cache.py b/samples/utils/cleanup-cache.py old mode 100644 new mode 100755 diff --git a/samples/utils/diagram.py b/samples/utils/diagram.py old mode 100644 new mode 100755 diff --git a/samples/utils/list-synth.py b/samples/utils/list-synth.py old mode 100644 new mode 100755 diff --git a/samples/utils/list2cache.py b/samples/utils/list2cache.py old mode 100644 new mode 100755 diff --git a/samples/utils/manual-splitter.py b/samples/utils/manual-splitter.py old mode 100644 new mode 100755 diff --git a/samples/utils/player.py b/samples/utils/player.py old mode 100644 new mode 100755 diff --git a/samples/utils/recover-unavail.py b/samples/utils/recover-unavail.py old mode 100644 new mode 100755 diff --git a/samples/utils/synth-batchconvert-helper.py b/samples/utils/synth-batchconvert-helper.py old mode 100644 new mode 100755 diff --git a/samples/utils/trace.py b/samples/utils/trace.py old mode 100644 new mode 100755 diff --git a/samples/utils/transliterate.py b/samples/utils/transliterate.py old mode 100644 new mode 100755 diff --git a/server/cantonese.py b/server/cantonese.py old mode 100644 new mode 100755 diff --git a/server/lesson-table.py b/server/lesson-table.py old mode 100644 new mode 100755 diff --git a/server/safety-check-progressfile.py b/server/safety-check-progressfile.py old mode 100644 new mode 100755 diff --git a/server/vocab2html.py b/server/vocab2html.py old mode 100644 new mode 100755 diff --git a/src/makeevent.py b/src/makeevent.py index f5d1875..3857ada 100644 --- a/src/makeevent.py +++ b/src/makeevent.py @@ -362,7 +362,7 @@ def toDict(l): # make the list of filenames into a dict of short-key -> [(long-k except IOError: pass # ignore write errors as it's only a cache except OSError: pass if partials_raw_mode: - (wtype,wrate,wchannels,wframes,wbits) = sndhdr.what(partialsDirectory+os.sep+"header"+dotwav) + (wtype,wrate,wchannels,wframes,wbits) = swhat(partialsDirectory+os.sep+"header"+dotwav) partials_raw_0bytes = int(betweenPhrasePause*wrate)*wchannels*int(wbits/8) else: synth_partials_voices,partials_raw_mode = {},None @@ -476,7 +476,8 @@ def optimise_partial_playing(ce): return s else: return ce # can't figure out an optimisation in these circumstances def simplified_header(fname): - h=sndhdr.what(fname) + # called by optimise_partial_playing(_list) + h=swhat(fname) # ignore num frames i.e. h[3], just compare formats if h: return h[:3]+h[4:] def optimise_partial_playing_list(ceList): diff --git a/src/play.py b/src/play.py index 9994c2a..80e9856 100644 --- a/src/play.py +++ b/src/play.py @@ -439,12 +439,20 @@ def lengthOfSound(file): if B(file).lower().endswith(B(dotmp3)): return rough_guess_mp3_length(file) else: return pcmlen(file) +if type("")==type(u""): # Python 3 + import wave + def swhat(file): + if file.lower().endswith(os.extsep+"wav"): + o = wave.open(file,'rb') + return "wav",o.getframerate(),o.getnchannels(),o.getnframes(),8*o.getsampwidth() + else: # fallback non-WAV + import sndhdr # before Python 3.13 + return sndhdr.what(file) +else: # Python 2 + import sndhdr + swhat = sndhdr.what def pcmlen(file): - header = sndhdr.what(file) - if not header: - # some Python 3 installations seem less able to run sndhdr - if gotSox: return len(readB(os.popen("sox \""+file+"\" -t raw "+sox_8bit+" "+sox_signed+" -c 1 -r 8000 - ",popenRB)))/8000.0 - else: raise IOError("sndhdr can't analyse file '%s'" % (file,)) + header = swhat(file) (wtype,wrate,wchannels,wframes,wbits) = header if android: if wrate==6144: # might be a .3gp from android_recordFile diff --git a/src/system.py b/src/system.py index 3476165..eb90715 100644 --- a/src/system.py +++ b/src/system.py @@ -224,7 +224,7 @@ def wspstrip(s): # directory should be OK by now if sys.platform.find("ymbian")>-1: sys.path.insert(0,os.getcwd()+os.sep+"lib") -import time,sched,sndhdr,random,math,pprint,codecs +import time,sched,random,math,pprint,codecs def exc_info(inGradint=True): import sys # in case it's been gc'd diff --git a/src/top.py b/src/top.py old mode 100644 new mode 100755 index 629d92f..a7dd143 --- a/src/top.py +++ b/src/top.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # (Python 2 or Python 3, but more fully tested on 2) -program_name = "gradint v3.078 (c) 2002-24 Silas S. Brown. GPL v3+." +program_name = "gradint v3.079 (c) 2002-24 Silas S. Brown. GPL v3+." # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -45,11 +45,11 @@ def sort(l,c): l.sort(c) try: True except: exec("True = 1 ; False = 0") def readB(f,m=None): - if hasattr(f,"buffer"): f=f.buffer # Python 3 non-"b" file + if hasattr(f,"buffer"): f0,f=f,f.buffer # Python 3 non-"b" file if m: return f.read(m) else: return f.read() # no "None" in Python 2 def writeB(f,b): - if hasattr(f,"buffer"): f=f.buffer # Python 3 non-"b" file + if hasattr(f,"buffer"): f0,f=f,f.buffer # Python 3 non-"b" file f.write(b) def B(x): if type(x)==bytes: return x diff --git a/thindown.py b/thindown.py old mode 100644 new mode 100755 From c4d0dbafc93e618a50b5979892d6719bda86cbc5 Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Sat, 22 Jun 2024 08:29:01 +0100 Subject: [PATCH 21/30] fix executable flag --- src/top.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 src/top.py diff --git a/src/top.py b/src/top.py old mode 100755 new mode 100644 From af62f7e97e848a342c4354f2bdea58fda18266ee Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Fri, 5 Jul 2024 06:41:43 +0100 Subject: [PATCH 22/30] cgi: zoom control; script variants; lesson timer; error localisations; tweak dark-mode boxes; rm spurious phone number links + withdrawn OU app --- advanced.txt | 3 ++- server/gradint.cgi | 61 +++++++++++++++++++++++++++++++++++----------- 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/advanced.txt b/advanced.txt index 3926c87..87533df 100644 --- a/advanced.txt +++ b/advanced.txt @@ -47,7 +47,8 @@ otherLanguages = ["cant","ko","jp"] # able to tell the difference between cant_en.wav and an # ordinary English prompt and might use it wrongly. -possible_otherLanguages = ["cant","ko","jp","en","zh"] +possible_otherLanguages = ["cant","ko","jp","en","zh", + "zhy","zh-yue"] # You can also fill in otherFirstLanguages below # (using the same ["item","item"] format) to diff --git a/server/gradint.cgi b/server/gradint.cgi index c70c60c..7149d10 100755 --- a/server/gradint.cgi +++ b/server/gradint.cgi @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # (either Python 2 or Python 3) -program_name = "gradint.cgi v1.34 (c) 2011,2015,2017-23 Silas S. Brown. GPL v3+" +program_name = "gradint.cgi v1.35 (c) 2011,2015,2017-24 Silas S. Brown. GPL v3+" # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -58,7 +58,7 @@ def reinit_gradint(): # if calling again, also redo setup_userID after langFullName = {} for l in gradint.ESpeakSynth().describe_supported_languages().split(): abbr,name = gradint.S(l).split("=") - langFullName[abbr]=name + langFullName[abbr]=name.replace("_","-") # Try to work out probable default language: lang = os.environ.get("HTTP_ACCEPT_LANGUAGE","") if lang: @@ -100,7 +100,7 @@ def main(): if query.getfirst("l2")=="zh" and gradint.generalCheck(query.getfirst("l2w"),"zh"): gradint.justSynthesize += "#en Pinyin needs tones. Please go back and add tone numbers." # speaking it because alert box might not work and we might be being called from HTML5 Audio stuff (TODO maybe duplicate generalCheck in js, if so don't call HTML5 audio, then we can have an on-screen message here) else: gradint.justSynthesize += "#"+query.getfirst("l2").replace("#","").replace('"','')+" "+query.getfirst("l2w").replace("#","").replace('"','') if "l1w" in query and query.getfirst("l1w"): gradint.justSynthesize += "#"+query.getfirst("l1").replace("#","").replace('"','')+" "+query.getfirst("l1w").replace("#","").replace('"','') - if gradint.justSynthesize=="0": return htmlOut('You must type a word in the box before pressing the Speak button.'+backLink) # TODO maybe add a Javascript test to the form also, IF can figure out if window.alert works + if gradint.justSynthesize=="0": return htmlOut(withLocalise('You must type a word in the box before pressing the Speak button.')+backLink) # TODO maybe add a Javascript test to the form also, IF can figure out if window.alert works serveAudio(stream = len(gradint.justSynthesize)>100, filetype=filetype) elif "add" in query: # add to vocab (l1,l2 the langs, l1w,l2w the words) if "l2w" in query and query.getfirst("l2w") and "l1w" in query and query.getfirst("l1w"): @@ -109,7 +109,7 @@ def main(): else: gcmsg=None if gcmsg: htmlOut(gradint.B(gcmsg)+gradint.B(backLink)) else: addWord(query.getfirst("l1w"),query.getfirst("l2w"),query.getfirst("l1"),query.getfirst("l2")) - else: htmlOut('You must type words in both boxes before pressing the Add button.'+backLink) # TODO maybe add a Javascript test to the form also, IF can figure out a way to tell whether window.alert() works or not + else: htmlOut(withLocalise('You must type words in both boxes before pressing the Add button.')+backLink) # TODO maybe add a Javascript test to the form also, IF can figure out a way to tell whether window.alert() works or not elif "bulkadd" in query: # bulk adding, from authoring options dirID = setup_userID() def isOK(x): @@ -126,7 +126,7 @@ def main(): redirectHomeKeepCookie(dirID,"&dictionary=1") # '1' is special value for JS-only back link; don't try to link to referer as it might be a generated page elif "clang" in query: # change languages (l1,l2) dirID = setup_userID() - if (gradint.firstLanguage,gradint.secondLanguage) == (query.getfirst("l1"),query.getfirst("l2")) and not query.getfirst("clang")=="ignore-unchanged": return htmlOut('You must change the settings before pressing the Change Languages button.'+backLink) # (external scripts can set clang=ignore-unchanged) + if (gradint.firstLanguage,gradint.secondLanguage) == (query.getfirst("l1"),query.getfirst("l2")) and not query.getfirst("clang")=="ignore-unchanged": return htmlOut(withLocalise('You must change the settings before pressing the Change Languages button.')+backLink) # (external scripts can set clang=ignore-unchanged) gradint.updateSettingsFile(gradint.settingsFile,{"firstLanguage": query.getfirst("l1"),"secondLanguage":query.getfirst("l2")}) redirectHomeKeepCookie(dirID) elif "swaplang" in query: # swap languages @@ -150,6 +150,18 @@ def main(): gradint.maxLenOfLesson = int(float(query.getfirst("mins"))*60) # TODO save those settings for next time also? serveAudio(stream = True, inURL = False, filetype=filetype) + elif "bigger" in query or "smaller" in query: + u = setup_userID() ; global zoom + if "bigger" in query: zoom = int(zoom*1.1) + else: zoom = int(zoom/1.1 + 0.5) + open(u+"-zoom.txt","w").write("%d\n" % zoom) + listVocab(True) + elif any("variant"+str(c) in query for c in range(max(len(gradint.GUI_translations[v]) for v in gradint.GUI_translations.keys() if v.startswith("@variants-")))): + for c in range(max(len(gradint.GUI_translations[v]) for v in gradint.GUI_translations.keys() if v.startswith("@variants-"))): #TODO duplicate code + if "variant"+str(c) in query: break + u = setup_userID() + gradint.updateSettingsFile(u+"-settings.txt",{"scriptVariants":{gradint.GUI_languages.get(gradint.firstLanguage,gradint.firstLanguage):c}}) + setup_userID() ; listVocab(True) elif "voNormal" in query: # voice option = normal setup_userID() gradint.voiceOption="" @@ -234,12 +246,14 @@ def justsynthLink(text,lang=""): # assumes written function h5a return ''+gradint.S(text)+'' # TODO if h5a's canPlayType etc works, cld o/p a lesson as a JS web page that does its own 'take out of event stream' and 'progress write-back'. wld need to code that HERE by inspecting the finished Lesson object, don't call play(). +zoom = 100 # in case browser device lacks a zoom UI, we'll provide one def htmlOut(body_u8,title_extra="",links=1): print ("Content-type: text/html; charset=utf-8\n") if title_extra: title_extra=": "+title_extra print ('Gradint Web edition'+title_extra+'') print ('') - print ('') + print ('') + if not zoom==100: print('' % zoom) print ('') if type(body_u8)==type(u""): body_u8=body_u8.encode('utf-8') if hasattr(sys.stdout,'buffer'): # Python 3 @@ -251,7 +265,6 @@ def htmlOut(body_u8,title_extra="",links=1): if links: print ('This is Gradint Web edition. If you need recorded words or additional functions, please download the full version of Gradint.') # TODO @ low-priority: Android 3 - if "iPhone" in os.environ.get("HTTP_USER_AGENT","") and gradint.secondLanguage=="zh": print ('

You can also try the Open University Chinese Characters First Steps iPhone application.') print ('

'+program_name[:program_name.index("(")]+"using "+gradint.program_name[:gradint.program_name.index("(")]) print ("") backLink = ' Back' # TODO may want to add a random= to the non-js HREF @@ -324,7 +337,7 @@ def addWord(l1w,l2w,l1,l2,out=True): if not ((gradint.firstLanguage,gradint.secondLanguage) == (l2,l1) and "HTTP_REFERER" in os.environ and not cginame in os.environ["HTTP_REFERER"]): gradint.updateSettingsFile(gradint.settingsFile,{"firstLanguage": l1,"secondLanguage":l2}) gradint.firstLanguage,gradint.secondLanguage = l1,l2 if (l1w+"_"+l1,l2w+"_"+l2) in map(lambda x:x[1:],gradint.parseSynthVocab(gradint.vocabFile,forGUI=1)): - if out: htmlOut('This word is already in your list.'+backLink) + if out: htmlOut(withLocalise('This word is already in your list.')+backLink) return gradint.appendVocabFileInRightLanguages().write(gradint.B(l2w)+gradint.B("=")+gradint.B(l1w)+gradint.B("\n")) if not out: return @@ -357,10 +370,18 @@ for k,v in {"Swap":{"zh":u"交换","zh2":u"交換"}, "click for audio":{"zh":u"击某词就听声音","zh2":u"擊某詞就聽聲音"}, "Repeats":{"zh":u"重复计数","zh2":u"重複計數"}, "To edit this list on another computer, type":{"zh":u"要是想在其他的电脑或手机编辑这个词汇表，请在别的设备打","zh2":u"要是想在其他的電腦或手機編輯這個詞彙表，請在別的設備打"}, + "Please wait while the lesson starts to play":{"zh":u"稍等本课正开始播放","zh2":u"稍等本課正開始播放"}, + "Bigger":{"zh":u"大"},"Smaller":{"zh":u"小"}, + 'You must type a word in the box before pressing the Speak button.':{"zh":u"按‘发音’前，应该框里打字。","zh2":u"按‘發音’前，應該框裡打字。"}, + 'You must type words in both boxes before pressing the Add button.':{"zh":u"按‘添加’前，应该在两框里打字。","zh2":u"按‘添加’前，應該在兩框裡打字。"}, + 'You must change the settings before pressing the Change Languages button.':{"zh":u"按‘选择其他语言’前，应该转换语言设定。","zh2":u"按‘選擇其他語言’前，應該轉換語言設定。"}, + 'This word is already in your list.':{"zh":u"本词已经在您的词汇表。","zh2":u"本詞已經在您的詞彙表。"}, "Your word list is empty.":{"zh":u"词汇表没有词汇，加一些吧","zh2":u"詞彙表沒有詞彙，加一些吧"} }.items(): if not k in gradint.GUI_translations: gradint.GUI_translations[k]=v +def withLocalise(x): return x+" "+localise(x,1) + def h5a(): body = """

'+localise("Your first language",1)+': '+langSelect('l1',firstLanguage)+' '+localise("second",1)+': '+langSelect('l2',secondLanguage)+' ' # onfocus..onblur updating onsubmit is needed for iOS "Go" button + body += (localise("Word in %s",1) % localise(secondLanguage))+':
'+(localise("Meaning in %s",1) % localise(firstLanguage))+':

'+localise("Your first language",1)+': '+langSelect('l1',firstLanguage)+' '+localise("second",1)+': '+langSelect('l2',secondLanguage)+' ' # onfocus..onblur updating onsubmit is needed for iOS "Go" button def htmlize(l,lang): if type(l)==type([]) or type(l)==type(()): return htmlize(l[-1],lang) l = gradint.B(l) @@ -417,13 +448,13 @@ def listVocab(hasList): # main screen if data: hasList = "

"+"".join(["%s" % (num,gradint.secondLanguage,htmlize(dest,gradint.secondLanguage),gradint.firstLanguage,htmlize(src,gradint.firstLanguage),deleteLink(src,dest)) for num,src,dest in data])+"
"+localise("Your word list",1)+" ("+localise("click for audio",1)+")
"+localise("Repeats",1)+""+localise(gradint.secondLanguage,1)+""+localise(gradint.firstLanguage,1)+"
%d%s%s
" else: hasList="" else: hasList="" - if hasList: body += '

'+numSelect('new',range(2,10),gradint.maxNewWords)+' '+localise("new words in")+' '+numSelect('mins',[15,20,25,30],int(gradint.maxLenOfLesson/60))+' '+localise('mins')+"""
""" + if hasList: body += '

'+numSelect('new',range(2,10),gradint.maxNewWords)+' '+localise("new words in")+' '+numSelect('mins',[15,20,25,30],int(gradint.maxLenOfLesson/60))+' '+localise('mins')+"""
""" # when lesson ended, refresh with lFinish which saves progress (interrupts before then cancel it) if "dictionary" in query: if query.getfirst("dictionary")=="1": body += '' # apparently it is -1, not -2; the redirect doesn't count as one (TODO are there any JS browsers that do count it as 2?) else: body += '

'+localise("Back to dictionary",1)+'' # TODO check for cross-site scripting if hasList: - if "SCRIPT_URI" in os.environ: hasList += "

"+localise("To edit this list on another computer, type",1)+" "+os.environ["SCRIPT_URI"]+"?id="+getCookieId()+"" - else: hasList="

"+localise("Your word list is empty.",1) + if "SCRIPT_URI" in os.environ: hasList += "

"+localise("To edit this list on another computer, type",1)+" "+os.environ["SCRIPT_URI"]+"?id="+re.sub("([0-9]{4})(?!$)",r"\1",getCookieId())+"" # span needed for iOS at least + else: hasList="

"+localise("Your word list is empty.",1) body += hasList htmlOut(body+'') @@ -464,6 +495,8 @@ def setup_userID(): if need_write: gradint.updateSettingsFile(gradint.settingsFile,{'firstLanguage':gradint.firstLanguage,'secondLanguage':gradint.secondLanguage}) else: gradint.readSettings(gradint.settingsFile) gradint.auto_advancedPrompt=1 # prompt in L2 if we don't have L1 prompts on the server, what else can we do... + if os.path.exists(userID+"-zoom.txt"): + global zoom ; zoom = int(open(userID+"-zoom.txt").read().strip()) return userID try: main() From 7fe0510987262dac4c451d1f0d95dea4bda1619c Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Wed, 10 Jul 2024 07:47:04 +0100 Subject: [PATCH 23/30] idiomatic Cantonese hanzi prompts that eSpeak zh-yue can transliterate --- hanzi-prompts/begin_zh-yue.txt | 1 + hanzi-prompts/end_zh-yue.txt | 1 + hanzi-prompts/longpause_zh-yue.txt | 1 + hanzi-prompts/meaningis_zh-yue.txt | 1 + hanzi-prompts/nowPleaseSay_zh-yue.txt | 1 + hanzi-prompts/pleaseSay_zh-yue.txt | 1 + hanzi-prompts/repeatAfterMe_zh-yue.txt | 1 + hanzi-prompts/sayAgain_zh-yue.txt | 1 + hanzi-prompts/tryToSay_zh-yue.txt | 1 + hanzi-prompts/whatSay_zh-yue.txt | 1 + hanzi-prompts/whatmean_zh-yue.txt | 1 + 11 files changed, 11 insertions(+) create mode 100644 hanzi-prompts/begin_zh-yue.txt create mode 100644 hanzi-prompts/end_zh-yue.txt create mode 100644 hanzi-prompts/longpause_zh-yue.txt create mode 100644 hanzi-prompts/meaningis_zh-yue.txt create mode 100644 hanzi-prompts/nowPleaseSay_zh-yue.txt create mode 100644 hanzi-prompts/pleaseSay_zh-yue.txt create mode 100644 hanzi-prompts/repeatAfterMe_zh-yue.txt create mode 100644 hanzi-prompts/sayAgain_zh-yue.txt create mode 100644 hanzi-prompts/tryToSay_zh-yue.txt create mode 100644 hanzi-prompts/whatSay_zh-yue.txt create mode 100644 hanzi-prompts/whatmean_zh-yue.txt diff --git a/hanzi-prompts/begin_zh-yue.txt b/hanzi-prompts/begin_zh-yue.txt new file mode 100644 index 0000000..62cad49 --- /dev/null +++ b/hanzi-prompts/begin_zh-yue.txt @@ -0,0 +1 @@ +開頭 diff --git a/hanzi-prompts/end_zh-yue.txt b/hanzi-prompts/end_zh-yue.txt new file mode 100644 index 0000000..679afff --- /dev/null +++ b/hanzi-prompts/end_zh-yue.txt @@ -0,0 +1 @@ +今日個堂上完啦 diff --git a/hanzi-prompts/longpause_zh-yue.txt b/hanzi-prompts/longpause_zh-yue.txt new file mode 100644 index 0000000..18d9f6c --- /dev/null +++ b/hanzi-prompts/longpause_zh-yue.txt @@ -0,0 +1 @@ +而家我哋要等一陣，然後翻溫。喺第一課我哋仲未學習好多嘅詞語，所以停頓會比較長，但係喺未來嘅課程，我哋唔會有咁長嘅停頓 diff --git a/hanzi-prompts/meaningis_zh-yue.txt b/hanzi-prompts/meaningis_zh-yue.txt new file mode 100644 index 0000000..a4c75cb --- /dev/null +++ b/hanzi-prompts/meaningis_zh-yue.txt @@ -0,0 +1 @@ +意思係 diff --git a/hanzi-prompts/nowPleaseSay_zh-yue.txt b/hanzi-prompts/nowPleaseSay_zh-yue.txt new file mode 100644 index 0000000..92923db --- /dev/null +++ b/hanzi-prompts/nowPleaseSay_zh-yue.txt @@ -0,0 +1 @@ +而家請講 diff --git a/hanzi-prompts/pleaseSay_zh-yue.txt b/hanzi-prompts/pleaseSay_zh-yue.txt new file mode 100644 index 0000000..cce3b70 --- /dev/null +++ b/hanzi-prompts/pleaseSay_zh-yue.txt @@ -0,0 +1 @@ +請講 diff --git a/hanzi-prompts/repeatAfterMe_zh-yue.txt b/hanzi-prompts/repeatAfterMe_zh-yue.txt new file mode 100644 index 0000000..09aaa03 --- /dev/null +++ b/hanzi-prompts/repeatAfterMe_zh-yue.txt @@ -0,0 +1 @@ +請跟住講 diff --git a/hanzi-prompts/sayAgain_zh-yue.txt b/hanzi-prompts/sayAgain_zh-yue.txt new file mode 100644 index 0000000..13ca92f --- /dev/null +++ b/hanzi-prompts/sayAgain_zh-yue.txt @@ -0,0 +1 @@ +再講一次 diff --git a/hanzi-prompts/tryToSay_zh-yue.txt b/hanzi-prompts/tryToSay_zh-yue.txt new file mode 100644 index 0000000..d43c674 --- /dev/null +++ b/hanzi-prompts/tryToSay_zh-yue.txt @@ -0,0 +1 @@ +試吓講 diff --git a/hanzi-prompts/whatSay_zh-yue.txt b/hanzi-prompts/whatSay_zh-yue.txt new file mode 100644 index 0000000..aed1a57 --- /dev/null +++ b/hanzi-prompts/whatSay_zh-yue.txt @@ -0,0 +1 @@ +點講 diff --git a/hanzi-prompts/whatmean_zh-yue.txt b/hanzi-prompts/whatmean_zh-yue.txt new file mode 100644 index 0000000..0aaf415 --- /dev/null +++ b/hanzi-prompts/whatmean_zh-yue.txt @@ -0,0 +1 @@ +乜嘢意思？ From c489f5119d97bb627317f954c36af09749efe80f Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Mon, 15 Jul 2024 06:33:43 +0100 Subject: [PATCH 24/30] cgi: big-print layout tweaks + nginx/fcgiwrap compatibility + fix Python 3 compatibility --- hanzi-prompts/whatmean_zh-yue_2.txt | 1 + hanzi-prompts/whatmean_zh-yue_3.txt | 1 + server/gradint.cgi | 59 +++++++++++++++++++---------- 3 files changed, 40 insertions(+), 21 deletions(-) create mode 100644 hanzi-prompts/whatmean_zh-yue_2.txt create mode 100644 hanzi-prompts/whatmean_zh-yue_3.txt diff --git a/hanzi-prompts/whatmean_zh-yue_2.txt b/hanzi-prompts/whatmean_zh-yue_2.txt new file mode 100644 index 0000000..87e6d63 --- /dev/null +++ b/hanzi-prompts/whatmean_zh-yue_2.txt @@ -0,0 +1 @@ +係乜嘢意思？ diff --git a/hanzi-prompts/whatmean_zh-yue_3.txt b/hanzi-prompts/whatmean_zh-yue_3.txt new file mode 100644 index 0000000..da79d2e --- /dev/null +++ b/hanzi-prompts/whatmean_zh-yue_3.txt @@ -0,0 +1 @@ +乜嘢意思呢？ diff --git a/server/gradint.cgi b/server/gradint.cgi index 7149d10..9d5c165 100755 --- a/server/gradint.cgi +++ b/server/gradint.cgi @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # (either Python 2 or Python 3) -program_name = "gradint.cgi v1.35 (c) 2011,2015,2017-24 Silas S. Brown. GPL v3+" +program_name = "gradint.cgi v1.36 (c) 2011,2015,2017-24 Silas S. Brown. GPL v3+" # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,6 +24,8 @@ try: from commands import getoutput # Python 2 except: from subprocess import getoutput # Python 3 try: from urllib import quote,quote_plus,unquote # Python 2 except: from urllib.parse import quote,quote_plus,unquote # Python 3 +try: from importlib import reload # Python 3 +except: pass home = os.environ.get("HOME","") if not home: try: @@ -52,7 +54,9 @@ sys.stderr=open("/dev/null","w") ; sys.argv = [] gradint = None def reinit_gradint(): # if calling again, also redo setup_userID after global gradint,langFullName - if gradint: gradint = reload(gradint) + if gradint: + if sys.version_info[0]>2: gradint.map,gradint.filter,gradint.chr=gradint._map,gradint._filter,gradint.unichr # undo Python 3 workaround in preparation for it to be done again, because reload doesn't do this (at least not on all Python versions) + gradint = reload(gradint) else: import gradint gradint.waitOnMessage = lambda *args:False langFullName = {} @@ -213,8 +217,10 @@ def allLinesHaveEquals(lines): for l in lines: if not '=' in l: return False return True +gradintUrl = os.environ.get("SCRIPT_URI","") # will be http:// or https:// as appropriate +if not gradintUrl and all(x in os.environ for x in ["REQUEST_SCHEME","SERVER_NAME","SCRIPT_NAME"]): gradintUrl = os.environ["REQUEST_SCHEME"]+"://"+os.environ["SERVER_NAME"]+os.environ["SCRIPT_NAME"] +if not gradintUrl: gradintUrl = "gradint.cgi" # guessing def authorWordList(lines,l1,l2): - gradintUrl = os.environ["SCRIPT_URI"] # will be http:// or https:// as appropriate r=[] ; count = 0 # could have target="gradint" in the following, but it may be in a background tab (target="_blank" not recommended as could accumulate many) r.append('

' % gradintUrl) @@ -278,28 +284,37 @@ def serveAudio(stream=0, filetype="mp3", inURL=1): if inURL: print ("Last-Modified: Sun, 06 Jul 2008 13:20:05 GMT") print ("Expires: Wed, 1 Dec 2036 23:59:59 GMT") # TODO: S2G + print ("Content-disposition: attachment; filename=gradint."+filetype+"\n") # helps with some browsers that can't really do streaming gradint.out_type = filetype + gradint.waitBeforeStart = 0 def mainOrSynth(): oldProgress = None ; rollback = False if not gradint.justSynthesize and 'h5a' in query: # TODO: if os.environ.get('HTTP_RANGE','')=='bytes=0-1' then that'll be '\xff' for mp3 but would need to stop the web server from adding a Content-Length etc (flush stdout and wait indefinitely for server to terminate the cgi process??) - try: oldProgress = open(gradint.progressFile).read() + try: oldProgress = open(gradint.progressFile,'rb').read() except: pass rollback = True if 'lesson' in query: random.seed(query.getfirst('lesson')) # so clients that re-GET same lesson from partway through can work if os.environ.get('HTTP_X_PLAYBACK_SESSION_ID',''): # seen on iOS: assumes the stream is a live broadcast and reconnecting to it continues where it left off. TODO: cache the mp3 output? (but don't delay the initial response) Recalculating for now with sox trim: if os.path.exists(gradint.progressFile+'-ts'): trimTo = time.time() - os.stat(gradint.progressFile+'-ts').st_mtime - if trimTo < gradint.maxLenOfLesson: - cin,cout = os.popen2("sox "+(gradint.soundCollector.soxParams()+' - ')*2+" trim "+str(int(trimTo))) - gradint.soundCollector.o,copyTo = cin,gradint.soundCollector.o - def copyStream(a,b): - while True: - try: x = a.read(1024) - except EOFError: break - b.write(x) - b.close() - import thread ; thread.start_new(copyStream,(cout,copyTo)) + if 15 < trimTo < gradint.maxLenOfLesson: # lower limit added due to Safari requesting whole lesson twice within the first few seconds and before the 1st one had finished generating: will result in 1st seconds missing if we trim + cmd = "sox "+(gradint.soundCollector.soxParams()+' - ')*2+" trim "+str(int(trimTo)) + if type("")==type(u""): # Python 3 + from subprocess import Popen,PIPE + global p # please don't gc it + p=Popen(cmd.split(),stdin=PIPE,stdout=gradint.soundCollector.o) + gradint.soundCollector.o = p.stdin + else: # Python 2 + cin,cout = os.popen2(cmd) + gradint.soundCollector.o,copyTo = cin,gradint.soundCollector.o + def copyStream(a,b): + while True: + try: x = a.read(1024) + except EOFError: break + b.write(x) + b.close() + import thread ; thread.start_new(copyStream,(cout,copyTo)) else: open(gradint.progressFile+'-ts','w') # previous one was abandoned, restart else: open(gradint.progressFile+'-ts','w') # create 1st one # end of if HTTP_X_PLAYBACK_SESSION_ID @@ -310,12 +325,14 @@ def serveAudio(stream=0, filetype="mp3", inURL=1): reinit_gradint() ; setup_userID() gradint.write_to_stdout,gradint.outputFile = o1,o2 gradint.setSoundCollector(gradint.SoundCollector()) - gradint.justSynthesize = "en Problem generating the lesson. Check we have prompts for those languages." ; gradint.main() ; oldProgress = None + gradint.justSynthesize = "en Problem generating the lesson. Check we have prompts for those languages." ; gradint.main() + if oldProgress: open(gradint.progressFile,'wb').write(oldProgress) + rollback = oldProgress = None if rollback: # roll back pending lFinish os.rename(gradint.progressFile,gradint.progressFile+'-new') - if oldProgress: open(gradint.progressFile,'w').write(oldProgress) + if oldProgress: open(gradint.progressFile,'wb').write(oldProgress) + # end of def mainOrSynth if stream: - print ("Content-disposition: attachment; filename=gradint.mp3\n") # helps with some browsers that can't really do streaming sys.stdout.flush() gradint.write_to_stdout = 1 gradint.outputFile="-."+filetype ; gradint.setSoundCollector(gradint.SoundCollector()) @@ -324,7 +341,6 @@ def serveAudio(stream=0, filetype="mp3", inURL=1): tempdir = getoutput("mktemp -d") gradint.write_to_stdout = 0 gradint.outputFile=tempdir+"/serveThis."+filetype ; gradint.setSoundCollector(gradint.SoundCollector()) - gradint.waitBeforeStart = 0 mainOrSynth() print ("Content-Length: "+repr(os.stat(tempdir+"/serveThis."+filetype).st_size)+"\n") sys.stdout.flush() @@ -436,7 +452,8 @@ def listVocab(hasList): # main screen def deleteLink(l1,l2): r = [] for l in [l2,l1]: - if type(l)==type([]) or type(l)==type(()) or not gradint.B("!synth:") in l: return "" # Web-GUI delete in poetry etc not yet supported + if type(l)==type([]) or type(l)==type(()) or not gradint.B("!synth:") in gradint.B(l): return "" # Web-GUI delete in poetry etc not yet supported + l = gradint.B(l) r.append(gradint.S(quote(l[l.index(gradint.B("!synth:"))+7:l.rfind(gradint.B("_"))]))) r.append(localise("Delete",2)) return ('') % tuple(r) @@ -448,12 +465,12 @@ def listVocab(hasList): # main screen if data: hasList = "

Click on each word for audio
"+"".join(["%s" % (num,gradint.secondLanguage,htmlize(dest,gradint.secondLanguage),gradint.firstLanguage,htmlize(src,gradint.firstLanguage),deleteLink(src,dest)) for num,src,dest in data])+"
"+localise("Your word list",1)+" ("+localise("click for audio",1)+")
"+localise("Repeats",1)+""+localise(gradint.secondLanguage,1)+""+localise(gradint.firstLanguage,1)+"
%d%s%s
" else: hasList="" else: hasList="" - if hasList: body += '

'+numSelect('new',range(2,10),gradint.maxNewWords)+' '+localise("new words in")+' '+numSelect('mins',[15,20,25,30],int(gradint.maxLenOfLesson/60))+' '+localise('mins')+"""
""" # when lesson ended, refresh with lFinish which saves progress (interrupts before then cancel it) + if hasList: body += '

'+numSelect('new',range(2,10),gradint.maxNewWords)+' '+localise("new words in")+' '+numSelect('mins',[15,20,25,30],int(gradint.maxLenOfLesson/60))+' '+localise('mins')+"""
""" # when lesson ended, refresh with lFinish which saves progress (interrupts before then cancel it), but don't save progress if somehow got the ended event in 1st minute as that could be a browser issue if "dictionary" in query: if query.getfirst("dictionary")=="1": body += '' # apparently it is -1, not -2; the redirect doesn't count as one (TODO are there any JS browsers that do count it as 2?) else: body += '

'+localise("Back to dictionary",1)+'' # TODO check for cross-site scripting if hasList: - if "SCRIPT_URI" in os.environ: hasList += "

"+localise("To edit this list on another computer, type",1)+" "+os.environ["SCRIPT_URI"]+"?id="+re.sub("([0-9]{4})(?!$)",r"\1",getCookieId())+"" # span needed for iOS at least + if "://" in gradintUrl: hasList += "

"+localise("To edit this list on another computer, type",1)+" "+gradintUrl.replace(".",".").replace("/","/")+"?id="+re.sub("([0-9]{4})(?!$)",r"\1",getCookieId())+"" # span needed for iOS at least else: hasList="

"+localise("Your word list is empty.",1) body += hasList htmlOut(body+'

') From 70d28fcf9e81101a94c69cd21a8ebf6e016310f3 Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Sat, 20 Jul 2024 08:30:09 +0100 Subject: [PATCH 25/30] fix Python 3 compatibility issue in Unicode progress files --- src/lessonplan.py | 7 ++++++- src/synth.py | 1 + src/top.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/lessonplan.py b/src/lessonplan.py index 1d64301..f2cc2b2 100644 --- a/src/lessonplan.py +++ b/src/lessonplan.py @@ -104,15 +104,19 @@ def _load_from_text(self,fromString=0): self._py3_fix() def _py3_fix(self): if not type("")==type(u""): return - # we're Python 3, and we might have just loaded data from Python 2 + # we're Python 3, and we might have just loaded data from Python 2. Might have to encode as Latin-1 then decode as UTF-8. But don't do this if file was in fact saved by Python 3. + if any(ord(c) > 255 for l in [self.data,self.unavail] for i in l for j in i[1:] for k in ([j] if type(j)==str else j) for c in k): return # must have been written by the Python 3 version for l in [self.data,self.unavail]: for i in range(len(l)): for j in [1,2]: if type(l[i][j])==str: l[i]=l[i][:j]+(S2(LB(l[i][j])),)+l[i][j+1:] elif type(l[i][j])==list: l[i]=l[i][:j]+(map(lambda x:S2(LB(x)),l[i][j]),)+l[i][j+1:] + def _py3_fix_on_save(self): + if type("")==type(u"") and not(any(ord(c) > 255 for l in [self.data,self.unavail] for i in l for j in i[1:] for k in ([j] if type(j)==str else j) for c in k)): self.unavail.append((1,u"\u2014","[Py3]")) # ensure there's at least one, to prevent a py3_fix redo def save(self,partial=0): if need_say_where_put_progress: show_info("Saving "+cond(partial,"partial ","")+"progress to "+progressFile+"... ") else: show_info("Saving "+cond(partial,"partial ","")+"progress... ") + self._py3_fix_on_save() global progressFileBackup # Remove 0-repeated items (helps editing by hand) data = [] # don't use self.data - may want to make another lesson after saving @@ -159,6 +163,7 @@ def save(self,partial=0): if not app and not appuifw and not android: show_info("done\n") def save_binary(self,data): # save a pickled version if possible (no error if not) if not (pickledProgressFile and pickle): return + self._py3_fix_on_save() try: if compress_progress_file: if paranoid_file_management: fn=os.tempnam() diff --git a/src/synth.py b/src/synth.py index 18871e3..9f01bb3 100644 --- a/src/synth.py +++ b/src/synth.py @@ -672,6 +672,7 @@ def transliterate_multiple(self,lang,textList,forPartials=1,keepIndexList=0): foundLetter=0 if l.startswith(B("Translate ")): toAppend=l[l.index(B("'"))+1:-1].replace(LB("\xc3\xbc"),B("v")) + if toAppend==LB("\xc2\xa0"): continue # stray no-break space (don't let this interfere with being able to do partials) if not (checkIn(toAppend,en_words) and r and toAppend==r[-1]): # TODO what about partial English words? e.g. try "kao3 testing" - translate 'testing' results in a translate of 'test' also (which assumes it's already in en mode), resulting in a spurious word "test" added to the text box; not sure how to pick this up without parsing the original text and comparing with the Replace rules that occurred r.append(toAppend) diff --git a/src/top.py b/src/top.py index a7dd143..8f268bf 100644 --- a/src/top.py +++ b/src/top.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # (Python 2 or Python 3, but more fully tested on 2) -program_name = "gradint v3.079 (c) 2002-24 Silas S. Brown. GPL v3+." +program_name = "gradint v3.08 (c) 2002-24 Silas S. Brown. GPL v3+." # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From 033aa838c02a9d3cd47caf6aeb257d9f159a74d8 Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Thu, 5 Dec 2024 14:24:46 +0000 Subject: [PATCH 26/30] experimental support for Coqui speech synthesis on GNU/Linux --- advanced.txt | 22 +++++++++++++++++++--- src/synth.py | 22 +++++++++++++++++++++- src/top.py | 2 +- 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/advanced.txt b/advanced.txt index 87533df..ca37321 100644 --- a/advanced.txt +++ b/advanced.txt @@ -202,7 +202,7 @@ systemVoice = "en" # - Festival Lite on Windows (if all else fails) : # put flite.exe in the gradint folder # -# - Linux: install Festival, or flite if you want a US accent +# - GNU/Linux: install Festival, or flite for US accent # # - S60: the phone's built-in speech can be used # @@ -212,6 +212,22 @@ systemVoice = "en" # older "Speech!" utility. These can be used only for # playing in real-time, not for generating files. +# Coqui voices are experimentally supported on GNU/Linux. +# Setup: pip install coqui-tts[server,zh,ja,ko] +# Then download the voices you want, e.g.: +# from TTS.api import TTS;langs = {} +# for m in TTS().list_models(): langs.setdefault(m.split('/')[1].split('-')[0],[]).append(m) +# +# TTS(langs["zh"][0]) +# TTS('tts_models/en/jenny/jenny') +# (If any model crashes during download, be sure to delete the +# result from ~/.local/share/tts before running Gradint. For +# example vocoder_models--ja--kokoro--hifigan_v1 may crash. +# I did say support for these voices is experimental.) +# Gradint detects voices that have been downloaded +# (but prefer_espeak overrides this). The Chinese +# voice does NOT support pinyin. + # You can also set extra_speech to a list of # (language prefix, command), for example: # extra_speech=[ ("la","say-latvian"),("de","say-german") ] @@ -351,7 +367,7 @@ lily_file = "C:\\Program Files\\NeoSpeech\\Lily16\\data-common\\userdict\\userdi # somewhere under C:\Program Files\VW\VT\Lily\M16-SAPI5\lib\ # but I don't know exactly) -# If you want to use SAPI under WINE in Linux +# If you want to use SAPI under WINE in GNU/Linux # then you can set ptts_program: ptts_program = None # (hint: run winecfg and set Windows version to Millenium (ME) @@ -760,7 +776,7 @@ gui_output_directory = "output" # in which case the first directory that EXISTS will be used # (or the last one on the list if all else fail). # Useful if the directory to your MP3 player only appears when -# it's plugged in for example. With Linux automounters you can +# it's plugged in for example. With GNU/Linux automounters # set "/media/*" as one of the directories, and it will expand to # whatever removable device is mounted IF there is only one. diff --git a/src/synth.py b/src/synth.py index 9f01bb3..9e98123 100644 --- a/src/synth.py +++ b/src/synth.py @@ -945,6 +945,26 @@ def finish_makefile(self): if self.theProcess: self.theProcess.close() self.theProcess = None +class CoquiSynth(Synth): + def __init__(self): + Synth.__init__(self) + self.synths = {} + def works_on_this_platform(self): + if not unix: return 0 # I'm unable to test elsewhere + self.base = os.environ.get("HOME","")+".local/share/tts" + if not isDirectory(self.base): return 0 # As voices require large downloads the first time they are used, we'll use only already-downloaded voices + def supports_language(self,lang): return any(a.startswith("tts_models--"+lang+"-") for a in os.listdir(self.base)) # TODO: might not want to use all downloaded models, or might not want to use for all input types (e.g. zh does not support pinyin) + def guess_length(self,lang,text): return quickGuess(len(text),6 if lang in ["zh"] else 12) # need better estimate + def makefile(self,lang,text): + text = ensure_unicode(text) + if lang=="zh": text += u"\u3002" # otherwise that model can glitch and repeat the last word of the phrase + if not lang in self.synths: + import torch;from TTS.api import TTS # shouldn't fault if models are downloaded to ~/.local/share/tts (unless uninstalled and not cleaned up...) + self.synths[lang]=TTS([a if a.startswith("tts_models--"+lang+"-") for a in os.listdir(self.base)][0].replace("--","/")).to("cuda" if torch.cuda.is_available() else "cpu") + fname = os.tempnam()+dotwav + self.synths[lang].tts_to_file(text,file_path=fname) + return fname + class GeneralSynth(Synth): def __init__(self): Synth.__init__(self) def supports_language(self,lang): @@ -991,7 +1011,7 @@ def makefile(self,lang,text): all_synth_classes.append(OSXSynth_Say) all_synth_classes.append(OSXSynth_OSAScript) # (prefer _Say if >=10.3 because it's faster) elif s.lower()=="sapi": all_synth_classes.append(PttsSynth) -all_synth_classes += [FestivalSynth,FliteSynth,OldRiscosSynth,S60Synth,AndroidSynth] +all_synth_classes += [CoquiSynth,FestivalSynth,FliteSynth,OldRiscosSynth,S60Synth,AndroidSynth] prefer_espeak = prefer_espeak.split() viable_synths = [] diff --git a/src/top.py b/src/top.py index 8f268bf..0cbedc1 100644 --- a/src/top.py +++ b/src/top.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # (Python 2 or Python 3, but more fully tested on 2) -program_name = "gradint v3.08 (c) 2002-24 Silas S. Brown. GPL v3+." +program_name = "gradint v3.09 (c) 2002-24 Silas S. Brown. GPL v3+." # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From 122ea179ef44b68b86936fd039900e80bbbbf700 Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Fri, 6 Dec 2024 00:09:47 +0000 Subject: [PATCH 27/30] fix Coqui support + Python 2 compatibility --- src/synth.py | 14 +++++++++----- src/top.py | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/synth.py b/src/synth.py index 9e98123..20fa9a1 100644 --- a/src/synth.py +++ b/src/synth.py @@ -932,13 +932,13 @@ def guess_length(self,lang,text): return quickGuess(len(text),12) # TODO need a if oss_sound_device: def play(self,lang,text): if not self.theProcess: self.startProcess() - self.theProcess.write("(Parameter.set 'Audio_Command \"play --device=%s \$FILE vol %.1f\")\n(tts_text \"%s\" nil)\n" % (oss_sound_device,5*soundVolume,text)) # (tts_text text nil) can be better than (SayText text) because it splits into multiple utterances if necessary + self.theProcess.write("(Parameter.set 'Audio_Command \"play --device=%s \\$FILE vol %.1f\")\n(tts_text \"%s\" nil)\n" % (oss_sound_device,5*soundVolume,text)) # (tts_text text nil) can be better than (SayText text) because it splits into multiple utterances if necessary self.theProcess.flush() # else send it via a file, because we haven't got code to give it to play to the other devices directly def makefile(self,lang,text): if not self.theProcess: self.startProcess() fname = os.tempnam()+dotwav - self.theProcess.write("(Parameter.set 'Audio_Command \"sox \$FILE %s vol 5\")\n(SayText \"%s\")\n" % (fname,text)) + self.theProcess.write("(Parameter.set 'Audio_Command \"sox \\$FILE %s vol 5\")\n(SayText \"%s\")\n" % (fname,text)) self.theProcess.flush() return fname def finish_makefile(self): @@ -951,8 +951,8 @@ def __init__(self): self.synths = {} def works_on_this_platform(self): if not unix: return 0 # I'm unable to test elsewhere - self.base = os.environ.get("HOME","")+".local/share/tts" - if not isDirectory(self.base): return 0 # As voices require large downloads the first time they are used, we'll use only already-downloaded voices + self.base = os.environ.get("HOME","")+"/.local/share/tts" + return isDirectory(self.base) # Voices require large downloads the first time they are used, so we'll use only already-downloaded voices def supports_language(self,lang): return any(a.startswith("tts_models--"+lang+"-") for a in os.listdir(self.base)) # TODO: might not want to use all downloaded models, or might not want to use for all input types (e.g. zh does not support pinyin) def guess_length(self,lang,text): return quickGuess(len(text),6 if lang in ["zh"] else 12) # need better estimate def makefile(self,lang,text): @@ -960,7 +960,11 @@ def makefile(self,lang,text): if lang=="zh": text += u"\u3002" # otherwise that model can glitch and repeat the last word of the phrase if not lang in self.synths: import torch;from TTS.api import TTS # shouldn't fault if models are downloaded to ~/.local/share/tts (unless uninstalled and not cleaned up...) - self.synths[lang]=TTS([a if a.startswith("tts_models--"+lang+"-") for a in os.listdir(self.base)][0].replace("--","/")).to("cuda" if torch.cuda.is_available() else "cpu") + # We can assume Python 3 by this point, but must still use syntax compatible with Python 2 + for a in sorted(os.listdir(self.base)): + if a.startswith("tts_models--"+lang+"-"): + self.synths[lang]=TTS(a.replace("--","/")).to(cond(torch.cuda.is_available(),"cuda","cpu")) + break fname = os.tempnam()+dotwav self.synths[lang].tts_to_file(text,file_path=fname) return fname diff --git a/src/top.py b/src/top.py index 0cbedc1..2f44760 100644 --- a/src/top.py +++ b/src/top.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # (Python 2 or Python 3, but more fully tested on 2) -program_name = "gradint v3.09 (c) 2002-24 Silas S. Brown. GPL v3+." +program_name = "gradint v3.091 (c) 2002-24 Silas S. Brown. GPL v3+." # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From 5f792aa13090fde79055c9ab28069e61ee47dfd3 Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Thu, 26 Dec 2024 21:30:42 +0000 Subject: [PATCH 28/30] fix PATH for some recent Macs; server support HTTP Range (improves iOS compatibility) + avoid downgrading protocol when linking to whole program --- .../Contents/MacOS/start-gradint | 2 +- server/gradint.cgi | 78 ++++++++++--------- 2 files changed, 43 insertions(+), 37 deletions(-) diff --git a/mac/start-gradint.app/Contents/MacOS/start-gradint b/mac/start-gradint.app/Contents/MacOS/start-gradint index e159ee9..c0f531b 100755 --- a/mac/start-gradint.app/Contents/MacOS/start-gradint +++ b/mac/start-gradint.app/Contents/MacOS/start-gradint @@ -1,5 +1,5 @@ #!/bin/bash -export PATH="$PATH:/usr/local/bin" # in case lame etc is there +export PATH="/usr/local/bin:$PATH" # for python3 override + in case lame etc is there cd "${BASH_SOURCE%/*}/../.." # needed on macOS 14, possibly 13 if sw_vers 2>/dev/null|grep ^ProductVersion.*1[2-9]; then # macOS 12+ if test $(python3 -c 'import tkinter,sys;print(sys.version_info[:3]>=(3,10,1))' 2>/dev/null) = "True"; then exec python3 gradint.py; fi diff --git a/server/gradint.cgi b/server/gradint.cgi index 9d5c165..332364d 100755 --- a/server/gradint.cgi +++ b/server/gradint.cgi @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # (either Python 2 or Python 3) -program_name = "gradint.cgi v1.36 (c) 2011,2015,2017-24 Silas S. Brown. GPL v3+" +program_name = "gradint.cgi v1.37 (c) 2011,2015,2017-24 Silas S. Brown. GPL v3+" # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -20,6 +20,8 @@ lib_path_add = "$HOME/gradint/lib" espeak_data_path = "$HOME/gradint" import os, os.path, sys, cgi, urllib, time, re +import tempfile, getpass +myTmp = tempfile.gettempdir()+os.sep+getpass.getuser()+"-gradint-cgi" try: from commands import getoutput # Python 2 except: from subprocess import getoutput # Python 3 try: from urllib import quote,quote_plus,unquote # Python 2 @@ -269,7 +271,7 @@ def htmlOut(body_u8,title_extra="",links=1): else: print(body_u8) print ('
') if links: - print ('This is Gradint Web edition. If you need recorded words or additional functions, please download the full version of Gradint.') + print ('This is Gradint Web edition. If you need recorded words or additional functions, please download the full version of Gradint.') # TODO @ low-priority: Android 3 print ('

'+program_name[:program_name.index("(")]+"using "+gradint.program_name[:gradint.program_name.index("(")]) print ("") @@ -279,45 +281,30 @@ def serveAudio(stream=0, filetype="mp3", inURL=1): # caller imports gradint (and sets justSynthesize or whatever) first if os.environ.get("HTTP_IF_MODIFIED_SINCE",""): print ("Status: 304 Not Modified\n\n") ; return + httpRange = re.match("bytes=([0-9]*)-([0-9]*)$",os.environ.get('HTTP_RANGE','')) # we MUST support Range: for some iOS players (Apple did not follow the HTTP standard of having a sensible fallback if servers respond with 200, and Apache will not do Range for us if we're CGI). Single Range should be sufficient. + if httpRange: httpRange = httpRange.groups() + if httpRange==('',''): httpRange = None # must spec one + if httpRange: + if not httpRange[0]: httpRange=[-int(httpRange[1]),None] + elif not httpRange[1]: httpRange=[int(httpRange[0]),None] + else: httpRange=[int(httpRange[0]),int(httpRange[1])+1] + print ("Status: 206 Partial Content") + stream = 0 if filetype=="mp3": print ("Content-type: audio/mpeg") else: print ("Content-type: audio/"+filetype) # ok for ogg, wav? if inURL: print ("Last-Modified: Sun, 06 Jul 2008 13:20:05 GMT") print ("Expires: Wed, 1 Dec 2036 23:59:59 GMT") # TODO: S2G - print ("Content-disposition: attachment; filename=gradint."+filetype+"\n") # helps with some browsers that can't really do streaming + print ("Content-disposition: attachment; filename=gradint."+filetype) # helps with some browsers that can't really do streaming gradint.out_type = filetype gradint.waitBeforeStart = 0 def mainOrSynth(): oldProgress = None ; rollback = False if not gradint.justSynthesize and 'h5a' in query: - # TODO: if os.environ.get('HTTP_RANGE','')=='bytes=0-1' then that'll be '\xff' for mp3 but would need to stop the web server from adding a Content-Length etc (flush stdout and wait indefinitely for server to terminate the cgi process??) try: oldProgress = open(gradint.progressFile,'rb').read() except: pass rollback = True - if 'lesson' in query: random.seed(query.getfirst('lesson')) # so clients that re-GET same lesson from partway through can work - if os.environ.get('HTTP_X_PLAYBACK_SESSION_ID',''): # seen on iOS: assumes the stream is a live broadcast and reconnecting to it continues where it left off. TODO: cache the mp3 output? (but don't delay the initial response) Recalculating for now with sox trim: - if os.path.exists(gradint.progressFile+'-ts'): - trimTo = time.time() - os.stat(gradint.progressFile+'-ts').st_mtime - if 15 < trimTo < gradint.maxLenOfLesson: # lower limit added due to Safari requesting whole lesson twice within the first few seconds and before the 1st one had finished generating: will result in 1st seconds missing if we trim - cmd = "sox "+(gradint.soundCollector.soxParams()+' - ')*2+" trim "+str(int(trimTo)) - if type("")==type(u""): # Python 3 - from subprocess import Popen,PIPE - global p # please don't gc it - p=Popen(cmd.split(),stdin=PIPE,stdout=gradint.soundCollector.o) - gradint.soundCollector.o = p.stdin - else: # Python 2 - cin,cout = os.popen2(cmd) - gradint.soundCollector.o,copyTo = cin,gradint.soundCollector.o - def copyStream(a,b): - while True: - try: x = a.read(1024) - except EOFError: break - b.write(x) - b.close() - import thread ; thread.start_new(copyStream,(cout,copyTo)) - else: open(gradint.progressFile+'-ts','w') # previous one was abandoned, restart - else: open(gradint.progressFile+'-ts','w') # create 1st one - # end of if HTTP_X_PLAYBACK_SESSION_ID + if "lesson" in query: random.seed(query.getfirst("lesson")) # so clients that re-GET same lesson from partway through can work try: gradint.main() except SystemExit: if not gradint.justSynthesize: @@ -333,18 +320,36 @@ def serveAudio(stream=0, filetype="mp3", inURL=1): if oldProgress: open(gradint.progressFile,'wb').write(oldProgress) # end of def mainOrSynth if stream: + print ("") sys.stdout.flush() gradint.write_to_stdout = 1 gradint.outputFile="-."+filetype ; gradint.setSoundCollector(gradint.SoundCollector()) mainOrSynth() else: - tempdir = getoutput("mktemp -d") gradint.write_to_stdout = 0 - gradint.outputFile=tempdir+"/serveThis."+filetype ; gradint.setSoundCollector(gradint.SoundCollector()) - mainOrSynth() - print ("Content-Length: "+repr(os.stat(tempdir+"/serveThis."+filetype).st_size)+"\n") + tempdir = tempfile.mkdtemp() + fn,fn2 = tempdir+"/I."+filetype, tempdir+"/O."+filetype + if httpRange and "lesson" in query: # try to cache it + try: os.mkdir(myTmp) + except: pass # exist ok + for f in os.listdir(myTmp): + if os.stat(myTmp+os.sep+f).st_mtime < time.time()-4000: + os.remove(myTmp+os.sep+f) + fn = gradint.outputPrefix+str(int(query.getfirst("lesson")))+"."+filetype # (don't be tricked into clobbering paths with non-int lesson IDs) + if not os.path.exists(fn): + gradint.outputFile=fn + gradint.setSoundCollector(gradint.SoundCollector()) + mainOrSynth() + if httpRange: + total = os.stat(fn).st_size + open(fn2,"wb").write(open(fn,"rb").read()[httpRange[0]:httpRange[1]]) + if httpRange[0]<0: httpRange[0] += total + if not httpRange[1]: httpRange[1] = total + print("Content-Range: bytes %d-%d/%d" % (httpRange[0],httpRange[1]-1,total)) + else: fn2 = fn + print ("Content-Length: "+repr(os.stat(fn2).st_size)+"\n") sys.stdout.flush() - os.system("cat "+tempdir+"/serveThis."+filetype) + os.system("cat "+fn2) # components already validated so no quoting required os.system("rm -r "+tempdir) def addWord(l1w,l2w,l1,l2,out=True): @@ -363,7 +368,7 @@ def addWord(l1w,l2w,l1,l2,out=True): def redirectHomeKeepCookie(dirID,extra=""): dirID = gradint.S(dirID) # just in case - print ("Location: "+cginame+"?random="+str(random.random())+"&id="+dirID[dirID.rindex("/")+1:]+extra+"\n") + print ("Location: "+cginame+"?random="+str(random.random())[2:]+"&id="+dirID[dirID.rindex("/")+1:]+extra+"\n") def langSelect(name,curLang): curLang = gradint.espeak_language_aliases.get(curLang,curLang) @@ -465,7 +470,7 @@ def listVocab(hasList): # main screen if data: hasList = "

"+"".join(["%s" % (num,gradint.secondLanguage,htmlize(dest,gradint.secondLanguage),gradint.firstLanguage,htmlize(src,gradint.firstLanguage),deleteLink(src,dest)) for num,src,dest in data])+"
"+localise("Your word list",1)+" ("+localise("click for audio",1)+")
"+localise("Repeats",1)+""+localise(gradint.secondLanguage,1)+""+localise(gradint.firstLanguage,1)+"
%d%s%s
" else: hasList="" else: hasList="" - if hasList: body += '

'+numSelect('new',range(2,10),gradint.maxNewWords)+' '+localise("new words in")+' '+numSelect('mins',[15,20,25,30],int(gradint.maxLenOfLesson/60))+' '+localise('mins')+"""
""" # when lesson ended, refresh with lFinish which saves progress (interrupts before then cancel it), but don't save progress if somehow got the ended event in 1st minute as that could be a browser issue + if hasList: body += '

'+numSelect('new',range(2,10),gradint.maxNewWords)+' '+localise("new words in")+' '+numSelect('mins',[15,20,25,30],int(gradint.maxLenOfLesson/60))+' '+localise('mins')+"""
""" # when lesson ended, refresh with lFinish which saves progress (interrupts before then cancel it), but don't save progress if somehow got the ended event in 1st minute as that could be a browser issue if "dictionary" in query: if query.getfirst("dictionary")=="1": body += '' # apparently it is -1, not -2; the redirect doesn't count as one (TODO are there any JS browsers that do count it as 2?) else: body += '

'+localise("Back to dictionary",1)+'' # TODO check for cross-site scripting @@ -504,11 +509,12 @@ def setup_userID(): open(dirName+'/'+userID+'-settings.txt','w') # TODO this could still be a race condition (but should be OK under normal circumstances) need_write = 1 print ('Set-Cookie: id=' + userID+'; expires=Wed, 1 Dec 2036 23:59:59 GMT') # TODO: S2G - userID = dirName+'/'+userID + userID0, userID = userID, dirName+os.sep+userID # already validated gradint.progressFileBackup=gradint.pickledProgressFile=None gradint.vocabFile = userID+"-vocab.txt" gradint.progressFile = userID+"-progress.txt" gradint.settingsFile = userID+"-settings.txt" + gradint.outputPrefix = myTmp+os.sep+userID0+"-" if need_write: gradint.updateSettingsFile(gradint.settingsFile,{'firstLanguage':gradint.firstLanguage,'secondLanguage':gradint.secondLanguage}) else: gradint.readSettings(gradint.settingsFile) gradint.auto_advancedPrompt=1 # prompt in L2 if we don't have L1 prompts on the server, what else can we do... From bf7912ad7cff27027ef3454a5f02e358bfdfa8ca Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Sat, 28 Dec 2024 09:54:00 +0000 Subject: [PATCH 29/30] minor Mac warning simplification --- .gitignore | 1 + src/play.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index b25c15b..9b5e2e8 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ *~ +__pycache__ diff --git a/src/play.py b/src/play.py index 80e9856..a5d85b3 100644 --- a/src/play.py +++ b/src/play.py @@ -148,8 +148,7 @@ def sox_check(): if macsound: if not gotSox and not os.system("mv sox-14.4.2 sox && rm sox.README"): gotSox,soxMp3 = sox_check() # see if that one works instead (NB must use os.system here: our system() has not yet been defined) if not gotSox and got_program("sox"): - if macsound: xtra=". (If you're on 10.8 Mountain Lion, try downloading a more recent sox binary from sox.sourceforge.net and putting it inside Gradint.app, but that will break compatibility with older PowerPC Macs.)" # TODO: ship TWO binaries? but we don't want the default gradint to get too big. See sox.README for more notes. - elif cygwin: xtra="" + if macsound or cygwin: xtra="" else: xtra=". Ubuntu users please install libsox-fmt-all." show_warning("SoX found but can't handle WAV, so you won't be able to write lessons to files for later"+xtra) else: gotSox = got_program("sox") From 70c0e93096e61369c6c0eeb4b1ac901f276547ae Mon Sep 17 00:00:00 2001 From: "Silas S. Brown" Date: Sat, 28 Dec 2024 10:31:31 +0000 Subject: [PATCH 30/30] omit pycache from gradint-build.7z --- Makefile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 6bed3d9..9d79987 100644 --- a/Makefile +++ b/Makefile @@ -238,7 +238,8 @@ gradint-build.7z: cp -r * /tmp/gradint-build00 rm -r /tmp/gradint-build00/LICENSE /tmp/gradint-build00/README.md /tmp/gradint-build00/charlearn mv /tmp/gradint-build00 gradint - cd gradint ; make clean ; rm -rf extras ; cd .. + make -C gradint clean + rm -rf gradint/extras 7za a gradint-build.7z gradint/ rm -rf gradint @@ -265,6 +266,7 @@ CD: $(Mac_Files) gradint.zip echo;echo;echo "Made CD directory. Can add gradint/samples, gradint/vocab.txt, gradint/espeak for Windows, gradint/espeak-.. for OSX, sox Win/Mac binaries, oggenc or whatever for Windows, etc." cleanup: - rm -f `find . -type f -name '*~' -o -name '*.pyc' -o -name DEADJOE` + find . -type f '(' -name '*~' -o -name '*.pyc' -o -name DEADJOE ')' -exec rm -vf '{}' ';' + rm -rvf __pycache__ # must be separate from find, as some find implementations exec before trying to descend and then error clean: cleanup - rm -f gradint.py $(All_Versions) src/defaults.py gradint-installer.command gradint.dmg + rm -rf gradint.py $(All_Versions) src/defaults.py gradint-installer.command gradint.dmg