geekbookapp.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""geekbookapp - the main program. The magic starts here!"""

import time
import os
import sys
import argparse
import logging
import gc
import platform
import subprocess

logging.basicConfig(format='%(asctime)s - %(filename)s - %(message)s')
logger = logging.getLogger('geekbook')
logger.setLevel('INFO')

PATH = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))  # __file__)))
sys.path.append(PATH)

from icecream import ic
import sys
ic.configureOutput(outputFunction=lambda *a: print(*a, file=sys.stderr))
ic.configureOutput(prefix='')

from engine.conf import PATH_TO_MD, PATH_TO_HTML, PATH_TO_IMG, PATH_TO_ORIG, AI_WRITER
from engine.page import Page
from engine.md_update import Md_update
from engine.make_index import Index
from engine.colors import bcolors
from engine.searcher import make_db
from engine.plugins import ia_writer
PINNED_NOTES = ['workflow.md']

class GeekbookError(Exception):
    pass


class MdFiles(object):
    """MdFiles manages the index of your md files (notes)"""
    path_to_watch = PATH_TO_MD

    def __init__(self):
        self.md_files = []
        self.get_filelist()
        self.sort_by_mtime()

    def get_filelist(self):
        """Get a raw index of all files in your notes folder, clean it and save the list as
        self.md_files"""
        self.md_files = os.listdir(self.path_to_watch)
        nfiles = []
        for f in self.md_files:
            if f.startswith('flycheck_') and f.endswith('.md'):
                continue
            if f.endswith('.md') and not f.startswith('.#'):
                if ' ' in f:
                    # print("""We don't handle names of you notes with spaces, please \
   # use `-`. e.g. geekbook-is-the-best.md Please rename your note and start this app again. Fix: %s """ % f)
                    continue
                nfiles.append(f)
        self.md_files = nfiles

    def sort_by_mtime(self):
        """Sort by mtime the list of md files"""
        self.md_files.sort(key=lambda x: os.stat(os.path.join(self.path_to_watch, x)).st_mtime)
        self.md_files.reverse()

    def get_files(self):
        """Get a list of your MD files.
        Update: alwasy get an updated list!"""
        self.get_filelist()
        self.sort_by_mtime()
        self.md_files = PINNED_NOTES  + self.md_files
        # remove duplicates, if case if the pinned note is alos the recent edit
        self.md_files = list(dict.fromkeys(self.md_files))
        return self.md_files


def exe(cmd):
    o = subprocess.Popen(
        cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out = o.stdout.read().decode()
    err = o.stderr.read().decode()
    return out, err


class App(object):
    """App class"""

    def __init__(self, args):
        self.args = args
        # fix #
        try:
            os.mkdir(PATH_TO_ORIG)
        except OSError:
            pass

    def start(self, update):
        """Start the App.
        """
        if not self.args.debug:
            # os.system('clear')
            print (bcolors.OKGREEN + "\n                 ________               __   __________               __    \n                /  _____/  ____   ____ |  | _\______   \ ____   ____ |  | __\n               /   \  ____/ __ \_/ __ \|  |/ /|    |  _//  _ \ /  _ \|  |/ /\n               \    \_\  \  ___/\  ___/|    < |    |   (  <_> |  <_> )    < \n                \______  /\___  >\___  >__|_ \|______  /\____/ \____/|__|_ \ \n                       \/     \/     \/     \/       \/                   \/ \n" + bcolors.ENDC)
        logger.info("G33kB00k3 is Running... [ok]")

        logger.info("root path: %s" % PATH)
        try:
            os.makedirs(PATH_TO_HTML)
        except OSError:
            pass
        logger.info("html path: <file://" + PATH_TO_HTML + 'index.html>')
        logger.info("imgs path: " + PATH_TO_IMG)

        logger.info('Ready to go! Please edit me: notes/')

        o = subprocess.Popen('cd "' + PATH_TO_MD + '" && cat *md | wc -l', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out = o.stdout.read().strip().decode()

        mf = MdFiles()
        logger.info('You have %i notes' % len(mf.get_files()) + ' with ' + out + ' lines! Congrats, keep noting!')

        index = Index()
        index.update(mf.get_files())

        # yappi.start()
        c = 0

        ipynb_mtime = {}

        while c < 10:  # for debugging
            # check for ipython

            # if not ipynb_mtime:
            # notebook_files = os.listdir(PATH_TO_MD)
            # for n in [n for n in notebook_files if n.endswith('.ipynb') and n.startswith('jupyter')]:
            # cmd = "jupyter nbconvert " + PATH_TO_MD + os.sep + n + " --to markdown"
            # print cmd
            # os.system(cmd)
            # ipynb_mtime[n] = os.path.getmtime(PATH_TO_MD + os.sep + n)
            # else:
            # notebook_files = os.listdir(PATH_TO_MD)
            # for n in [n for n in notebook_files if n.endswith('.ipynb')]:
            # if n in ipynb_mtime.keys():
            # mt = os.path.getmtime(PATH_TO_MD + os.sep + n)
            # if ipynb_mtime[n] < mt:
            # cmd = "jupyter nbconvert " + PATH_TO_MD + os.sep + n + " --to markdown"
            # print cmd
            # os.system(cmd)
            # ipynb_mtime[n] = mt
            # else:
            # mt = os.path.getmtime(PATH_TO_MD + os.sep + n)
            # cmd = "jupyter nbconvert " + PATH_TO_MD + os.sep + n + " --to markdown"
            # print cmd
            # os.system(cmd)
            # ipynb_mtime[n] = mt

            # see what's new - diff between to folders your notes and orig files that keep copy of our notes
            # grep -v removes things from your list, ~, # (and in mmagnus case org mode files)
            cmd = "diff -u -r \"" + PATH_TO_MD + "\" \"" + PATH_TO_ORIG + "\" " + \
                " | grep -v '\.org' | grep -v 'flycheck_' | grep -v '~$' | grep -v '#' | grep '\.md'".strip()
            out, err  = exe(cmd)
            if err:
                if 'No such file or directory' in err:
                    continue
                else:
                    raise Exception("Geekbook3 diff can't be completed")

            files_changed = []

            # pick all file names that are changed
            for l in out.split('\n'):
                # new notes
                if l.startswith('Only in ' + PATH_TO_MD):
                    files_changed.append(os.path.basename(l.split()[-1]))
                # changes notes
                if l.startswith('diff -u -r'):
                    files_changed.append(os.path.basename(l.split()[-1]))

            # if there are files change compile them
            for f in files_changed:
                m = Md_update(f)
                p = Page(f)
                if p.is_changed():
                    # if m is changed then (by using any of plugins working on markdown, run this
                    changed = m.compile()
                    if changed:  # only if something is changed in md
                        m.save()

                    p.compile()
                    p.save()
                    p.collect_to_pdf_save()
                    
                    index = Index()
                    index.update(mf.get_files())

                    # update search db if any of the files
                    # is changed
                    make_db()

            if update == -1:
                fn = '/Users/magnus/geekbook/to-pdf.txt'
                try:
                    update = len(open(fn).read().split('\n')) + 1
                    os.remove(fn) # remove file
                except FileNotFoundError:
                    sys.exit(0) # update = 0 # dont update
                    pass

            if update:
                i = 0
                for f in mf.get_files():
                    if f == 'imgs':
                        pass
                    else:
                        p = Page(f)
                        p.compile()
                        p.save()
                        p.to_pdf()

                        i += 1
                        if i > update: # do it for last x notes
                            sys.exit(0)                            

                sys.exit(0)

            # dev -d <file>
            if DEV:
                # update index
                index = Index()
                index.update(mf.get_files())

                # update this one picked note
                m = Md_update(args.debug)
                changed = m.compile()  # if changed MD
                if changed:
                    m.save()

                p = Page(args.debug)
                p.compile()
                p.save()
                p.to_pdf()
                
                sys.exit(0)

            gc.collect()
            time.sleep(1)  # if this is too big you have too wait for ii too long (!)
            # off c += 1

        # yappi.stop()
        # stats = yappi.get_func_stats()
        # stats.save('yappi.callgrind', type="callgrind")


def start_flask(args):
    if not args.noflask and not args.debug and not args.update:
        logger.info("Start off flask!")
        if args.public:
            os.system('python3 "' + PATH + os.sep + 'geekbook/engine/webserverflask.py" --public &')
        else:
            os.system('python3 "' + PATH + os.sep + 'geekbook/engine/webserverflask.py" &')


def start_gitweb():
    """Start git instaweb"""
    os.chdir(PATH_TO_MD)
    os.system('git instaweb')


def start_browser_with_index():
    """Detect the operative system in use and open the html file using the default browser.
    Works with Linux and macOS."""
    if platform.system() == "Linux":
        os.system('xdg-open http://127.0.0.1:5000/view/index.html')
    elif platform.system() == "Darwin":
        os.system('open http://127.0.0.1:5000/view/index.html')
    else:
        logger.info("Sorry, I cannot detect your system, you will have to open the file manually @")


def get_parser():
    """Get parser of arguments"""
    parser = argparse.ArgumentParser('geekbookapp.py')
    parser.add_argument('-d', '--debug', help='debug mode, run only for file,' +
                        'WARNING: use only name of the note, e.g. test.md, NOT notes/test.md')
    parser.add_argument('-u', '--update', help='updates of # last notes', type=int) #action='store_true')
    parser.add_argument('-s', '--silent', help='dont bring up the Internet Browser', action='store_true')
    parser.add_argument('-n', '--notebook',
                        help='updates all jupiter notebooks!', action='store_true')
    parser.add_argument('--noflask', help='dont run flask', action='store_true')
    parser.add_argument('--noupdatedb', help='dont update the db', action='store_true')
    parser.add_argument('--public', help='run as public server, edit engine.open_access to configure', action='store_true')
    return parser


def convert_jupyter_notebook_to_markdown():
    """

    #[mm] notes git:(master) ✗
    #    [NbConvertApp] Converting notebook testA.ipynb to markdown
    #[NbConvertApp] Support files will be in testA_files/
    #[NbConvertApp] Making directory testA_files
    #[NbConvertApp] Writing 2960 bytes to testA.md

    """
    if args.notebook:
        notebook_files = os.listdir(PATH_TO_MD)
        for n in [n for n in notebook_files if n.endswith('.ipynb')]:
            cmd = "jupyter nbconvert " + PATH_TO_MD + os.sep + n + " --to markdown"
            print(cmd)
            os.system(cmd)
        sys.exit(1)


# main
if __name__ == '__main__':
    parser = get_parser()
    args = parser.parse_args()

    # emacs & python debugging
    # args = parser.parse_args(['--debug', 'test.md', '-s'])
    # args = parser.parse_args(['-u'])

    app = App(args)
    if not args.noupdatedb:
        make_db()
    convert_jupyter_notebook_to_markdown()

    if not args.debug and not args.update:
        # kill old flask
        logger.info("Kill old flask... [ok]")
        os.system('kill -9 $(lsof -ti:5000)')
    start_flask(args)

    if args.debug:
        DEV = True
        UPDATE = False
    elif args.update:
        UPDATE = args.update #True
        DEV = False
    else:
        DEV = False
        UPDATE = False
        if not args.silent:
            start_gitweb()
            time.sleep(1)  # so it's time for the Flask to be opened in the browser
            start_browser_with_index()

    app.start(UPDATE)