dokuwiki2wikijs.py

#! /usr/bin/env python3

import os
from pathlib import Path
import sys
from zipfile import ZipFile
from os.path import basename
from shutil import rmtree, copyfile
import subprocess
import re
from glob import glob 
from datetime import datetime
from urllib.parse import unquote
from tqdm import tqdm
from argparse import ArgumentParser

time = datetime.now()
tmp_prefix = os.path.join('/tmp', f'dokuwiki2wikijs_{time:%Y-%m-%d_%H-%M}')

errors = []
errorfile = os.path.join(tmp_prefix, 'errors.txt')

barformat = "{percentage:.0f}%|{bar}{r_bar}"

# Wrapper for Pandoc to convert one file
def pandoc(infile):
    result = subprocess.run(
        ["pandoc", "-f", "dokuwiki", "-t", "markdown_mmd", "--wrap=none", infile], stdout=subprocess.PIPE)
    if len(result.stdout) == 0:
        raise ValueError('Pandoc returned no output for file `%s`. This usually means that Pandoc encountered a syntax error in the input file and crashed. Try running `pandoc -f dokuwiki -t markdown_mmd --wrap=none AFFECTED_FILE` on the affected file to see if there is a syntax error.' % infile)
    return result.stdout.decode('utf-8')


# Choose pagename for new page
# Remove old pagename and decrease all lower level headings by 1 if necessary
def page_headings(lines, pagename):
    # We surround the title with quotes to ensure Wiki.js can import it.
    # Wiki.js requires a string and some possible titles are not considered
    # strings (e.g. dates)
    title = ''
    if lines[0][0] == '#':
        title = lines[0].partition(' ')[2].strip()
        if not any(l.startswith('# ') for l in lines[1:]):
            lines = lines[1:]
            lines = [l[1:] if l.startswith('#') else l for l in lines]
    else:
        title = pagename

    return {
        "title": f'"{title}"'
    }, lines


def starts_with_text(line):
    # Empty line -> no
    if len(line.strip()) == 0:
        return False
    # Letters and quote -> yes
    if line[0].isalpha() or line[0] == '"':
        return True
    # Numeric list marker -> no
    if re.match(r'^[0-9]+\. ', line):
        return False
    return False


def unwrap_sentences(lines):
    # pandoc wraps markdown paragraphs however the input was formatted,
    # so unwrap it according to markdown/asciidoc conventions (one sentence
    # per line).
    result = []
    compacted_lines = []
    doing_compacting = False
    compacted_line = ""

    for line in lines:
        if not doing_compacting:
            if len(line) > 0 and line[-1] != '.':
                doing_compacting = True
                compacted_line = line
            else:
                compacted_lines.append(line)
        else:
            if starts_with_text(line):
                compacted_line = compacted_line + ' ' + line
            else:
                compacted_lines.append(compacted_line)
                compacted_lines.append(line)
                compacted_line = ""
                doing_compacting = False
    if compacted_line != '':
        compacted_lines.append(compacted_line)
    for line in compacted_lines:
        while ". " in line:
            line1, line = line.split('. ', 1)
            result.append(line1 + '.')
        result.append(line)
    return result


def find_next_link_start(line, pos):
    match = re.search(r"(\[\[)|(\{\{)", line[pos:])
    if match:
        return match.start()
    else:
        return -1


def convert_links(lines):
    for i, line in enumerate(lines):
        pos = find_next_link_start(line, 0)
        while pos != -1:
            pattern = r'(\[\[|\{\{)(?P<uri>[^\|]+?)(\|(?P<text>.+?)?)?(\]\]|\}\})'
            match = re.search(pattern, line[pos:])
            if match:
                text = match.group('text')
                uri = match.group('uri').rstrip('|')
                if not uri.startswith(('http', 'mailto', 'tel')):
                    uri = uri.replace(':', '/')
                if not text:
                    text = uri
                # Ensure internal uri starts at the root
                if not uri.startswith('http') and not uri.startswith('/'):
                    uri = f'/{uri}'
                uri = uri.lower()
                link = '[%s](%s)' % (text, uri)
                line = re.sub(pattern, link, line, count=1)
            pos = find_next_link_start(line, pos)
        lines[i] = line
    return lines


def wrap_kind(tag):
    words = tag.split(' ')
    if 'info' in words or 'notice' in words:
        return "{.is-info}"
    if 'important' in words or 'warning' in words or 'caution' in words:
        return "{.is-warning}"
    if 'alert' in words or 'danger' in words:
        return "{.is-danger}"
    if 'tip' in words or 'help' in words or 'todo' in words:
        return "{.is-danger}"
    if 'safety' in words or 'danger' in words:
        return "{.is-danger}"
    return ""


def convert_wrap(lines):
    kind = "{.is-info}"
    wrapping = False
    for i, line in enumerate(lines):
        # This might be a pandoc'ed markdown in which case tags are escaped
        if line.startswith("<WRAP") or line.startswith("\<WRAP"):
            tag, line = line.split('>', 1)
            kind = wrap_kind(tag)
            lines[i] = "> " + line
            wrapping = True
        if "</WRAP>" in line:
            lines[i] = lines[i].replace("</WRAP>", kind)
            wrapping = False
        if "\</WRAP\>" in line:
            lines[i] = lines[i].replace("\</WRAP\>", kind)
            wrapping = False
        if wrapping:
            lines[i] = "> "+line
    return lines


def add_metadata(lines, metadata):
    lines.insert(0, "---")
    for key, value in metadata.items():
        lines.insert(1, key+": "+value)
    lines.insert(len(metadata)+1, "---")


def ensure_path_exists(path):
    # dirname returns only name of directory (/a/b/c.txt -> /a/b)
    directory = os.path.dirname(path)
    if not os.path.exists(directory):
        os.makedirs(directory)

def remove_useless_tags(lines):
    new_lines = []
    for line in lines:
        line = line.replace('\\<sortable\\>', '')
        line = line.replace('\\</sortable\\>', '')
        new_lines.append(line)
    return new_lines


def convert_file(txtfile, title):
    try:
        lines = str(pandoc(txtfile)).split('\n')
    except ValueError as e:
        print(e)
        errors.append(txtfile)

    lines = remove_useless_tags(lines)
    lines = convert_wrap(lines)
    # lines = unwrap_sentences(lines)
    metadata, lines = page_headings(lines, title)
    add_metadata(lines, metadata)

    return lines


def temporary_file_for(pathname):
    parts = pathname.removeprefix(str(path)).split(os.sep)
    temporary = os.path.join(tmp_prefix, *parts[parts.index('data'):])
    return temporary


def collect_and_convert_all_pages():
    for folder, _, files in os.walk(os.path.join(path, "data", "pages")):
        txt_files = [file for file in files if file.endswith(".txt")]
        print(f'Converting {len(txt_files)} txt Files in {folder}...')
        for f in tqdm(txt_files, bar_format=barformat):
            filename_with_txt = os.path.join(folder, f)
            filename = temporary_file_for(
                # URL-decode filename
                unquote(filename_with_txt[:-4]))
            # basename returns only name of file (/a/b/c.txt -> c.txt)
            basename = os.path.basename(filename)
            ensure_path_exists(filename)

            # Ignore the sidebar(?)
            if basename == 'sidebar':
                continue

            # print(filename_with_txt+"("+basename+")... ", end="", flush=True)

            # this is where the magic happens
            lines = convert_file(filename_with_txt, basename)

            # rename to .md
            filename_with_md = os.path.join(tmp_prefix, f'{filename}.md')

            # rename homepage
            if basename == 'start':
                filename_with_md = os.path.join(folder, 'home.md')

            # write lines to .md-file
            with open(filename_with_md, "w", encoding="utf-8") as file:
                file.writelines('\n'.join(lines))


def collect_all_media():
    for folder, _, media_files in os.walk(os.path.join(path, "data", "media")):
    # Check if given path exists
        print(f'Converting {len(media_files)} Media Files in {folder}...')
        for f in tqdm(media_files, bar_format=barformat):
            media_file = os.path.join(folder, f)
            filename = temporary_file_for(
                # URL-decode filename
                unquote(media_file))
            filename = filename.replace('media', 'pages')
            ensure_path_exists(filename)
            copyfile(media_file, filename)


if __name__ == "__main__":
    parser = ArgumentParser()
    parser.add_argument("installation", help="Path to the dokuwiki installation")
    args = parser.parse_args()
    path = Path(args.installation)

    # Check if given path exists
    if not path.exists():
        print(f"'{path}' does not exist")
        sys.exit(1)

    # Check if given path is root of dokuwiki installation by looking for path/data/pages
    if not (path / "data" / "pages").exists():
        print(
            "The folder given as argument should be at the root of a dokuwiki installation or copy")
        sys.exit(-1)

    # search for existing temp dirs /temp/dokuwiki2wikijs_(.*)
    temp_dirs = glob('/tmp/dokuwiki2wikijs_*')
    if len(temp_dirs):
        # ask if old temp should be deleted
        print('Found old temporary directories:')
        for dir in temp_dirs:
            delete = input(f'Delete "{dir}"? (y/N) ')
            if delete.lower() == 'y':
                rmtree(dir, ignore_errors=True)
    rmtree(tmp_prefix, ignore_errors=True)
    os.makedirs(tmp_prefix)

    collect_and_convert_all_pages()
    collect_all_media()

    if len(errors):
        with open(errorfile, "w", encoding="utf-8") as file:
            file.writelines('\n'.join(errors))
        print(f'{len(errors)} errors encountered during conversion written to {errorfile}')

    print("done")