documentation: automatically generate documentation from all .py file…

…s in xDSL (#3758)

Excludes:

 - `__main__.py` files
 - `_`-prefixed files
- `xdsl/irdl/error.py` as it contains a function called `error` which
confuses the doc mechanism for some reason
- `xdsl/ir/*.py` as we've already got curated docs for these files

Still needs a bit of cleanup and to merge the code changes separately.

The big question for this PR is whether we want to commit these files
and, if not, how do we review them?

docs/mlir_interoperation.md → docs/guides/mlir_interoperation.md

@@ -1,6 +1,6 @@
 ## Generating executables through MLIR
 
-As mentioned previously, xDSL can interoperate with MLIR as its backend. As this
+xDSL can interoperate with MLIR as its backend. As this
 requires an installation, and therefore a compilation, of a lot of the LLVM
 project and MLIR, this functonality is not distributed with xDSL by default. To
 actually leverage from this functionality, first clone and build MLIR. Please

mkdocs.yml

@@ -2,14 +2,41 @@ site_name: xDSL
 
 theme:
   name: "material"
+  features:
+    - navigation.instant
+    - navigation.tracking
+    - navigation.tabs
+    - navigation.sections
+    - navigation.expand
 
 plugins:
 - search
-- mkdocstrings
+- gen-files:
+    scripts:
+    - scripts/gen_ref_pages.py
+- mkdocstrings:
+    handlers:
+      python:
+        options:
+          docstring_style: google
+          members_order: source
+          show_root_heading: true
+          show_root_full_path: false
+          show_signature_annotations: true
 
-
-# nav:
-#   - xDSL: index.md
-#   - API Reference:
-#     - API Reference: api/index.md
-#     - Core IR Data Structures: api/ir/index.md
+nav:
+  - xDSL: index.md
+  - API Reference:
+    - API Reference: api/index.md
+    - Core IR Data Structures:
+      - Core IR Data Structures: api/ir/index.md
+      - Attribute: api/ir/attribute.md
+      - SSAValue: api/ir/ssa_value.md
+      - IRNode: api/ir/ir_node.md
+      - Operation: api/ir/operation.md
+      - Block: api/ir/block.md
+      - Region: api/ir/region.md
+      - Dialect: api/ir/dialect.md
+  - Code Reference: reference/index.md
+  - Guides:
+    - MLIR Interoperation: guides/mlir_interoperation.md

pyproject.toml

@@ -34,8 +34,9 @@ dev = [
     "pyright==1.1.392.post0",
 ]
 docs = [
-    "mkdocs>=1.6.1",
+    "mkdocs-gen-files>=0.5.0",
     "mkdocs-material>=9.5.49",
+    "mkdocs>=1.6.1",
     "mkdocstrings[python]>=0.27.0",
 ]
 gui = ["textual==1.0.0", "pyclip==0.7"]

scripts/gen_ref_pages.py

@@ -0,0 +1,49 @@
+"""Generate the code reference pages and navigation."""
+
+from pathlib import Path
+
+import mkdocs_gen_files
+from mkdocs_gen_files.nav import Nav
+
+nav = Nav()
+
+root = Path(__file__).parent.parent
+src = root / "xdsl"
+
+for path in sorted(src.rglob("*.py")):
+    module_path = path.relative_to(src).with_suffix("")
+    doc_path = path.relative_to(src).with_suffix(".md")
+    full_doc_path = Path("reference", doc_path)
+
+    parts = tuple(module_path.parts)
+
+    if parts[-1] == "__init__":
+        parts = parts[:-1]
+        doc_path = doc_path.with_name("index.md")
+        full_doc_path = full_doc_path.with_name("index.md")
+    elif parts[-1] == "__main__":
+        continue
+    elif parts[-1].startswith("_"):
+        continue
+    if not parts:
+        continue
+
+    if "ir" == parts[0]:
+        # IR is documented separately
+        continue
+
+    ident = ".".join(parts)
+
+    if ident in ("irdl.error",):
+        # TODO: rename error function, treated as circular reference
+        continue
+
+    nav[parts] = doc_path.as_posix()
+
+    with mkdocs_gen_files.open(full_doc_path, "w") as fd:
+        fd.write(f"::: xdsl.{ident}")
+
+    mkdocs_gen_files.set_edit_path(full_doc_path, path.relative_to(root))
+
+with mkdocs_gen_files.open("reference/index.md", "w") as nav_file:
+    nav_file.writelines(nav.build_literate_nav())