
1 changed files with 674 additions and 0 deletions
@ -0,0 +1,674 @@
@@ -0,0 +1,674 @@
|
||||
#!/usr/bin/env python3 |
||||
|
||||
# Copyright 2020 David Robillard <d@drobilla.net> |
||||
# |
||||
# Permission to use, copy, modify, and/or distribute this software for any |
||||
# purpose with or without fee is hereby granted, provided that the above |
||||
# copyright notice and this permission notice appear in all copies. |
||||
# |
||||
# THIS SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES |
||||
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF |
||||
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR |
||||
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES |
||||
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN |
||||
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF |
||||
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. |
||||
|
||||
""" |
||||
Write Sphinx markup from Doxygen XML. |
||||
|
||||
Takes a path to a directory of XML generated by Doxygen, and emits a directory |
||||
with a reStructuredText file for every documented symbol. |
||||
""" |
||||
|
||||
import argparse |
||||
import os |
||||
import sys |
||||
import textwrap |
||||
import xml.etree.ElementTree |
||||
|
||||
__author__ = "David Robillard" |
||||
__date__ = "2020-11-18" |
||||
__email__ = "d@drobilla.net" |
||||
__license__ = "ISC" |
||||
__version__ = __date__.replace("-", ".") |
||||
|
||||
|
||||
def load_index(index_path): |
||||
""" |
||||
Load the index from XML. |
||||
|
||||
:returns: A dictionary from ID to skeleton records with basic information |
||||
for every documented entity. Some records have an ``xml_filename`` key |
||||
with the filename of a definition file. These files will be loaded later |
||||
to flesh out the records in the index. |
||||
""" |
||||
|
||||
root = xml.etree.ElementTree.parse(index_path).getroot() |
||||
index = {} |
||||
|
||||
for compound in root: |
||||
compound_id = compound.get("refid") |
||||
compound_kind = compound.get("kind") |
||||
compound_name = compound.find("name").text |
||||
if compound_kind in ["dir", "file", "page"]: |
||||
continue |
||||
|
||||
# Add record for compound (compounds appear only once in the index) |
||||
assert compound_id not in index |
||||
index[compound_id] = { |
||||
"kind": compound_kind, |
||||
"name": compound_name, |
||||
"xml_filename": compound_id + ".xml", |
||||
"children": [], |
||||
} |
||||
|
||||
name_prefix = ( |
||||
("%s::" % compound_name) if compound_kind == "namespace" else "" |
||||
) |
||||
|
||||
for child in compound.findall("member"): |
||||
if child.get("refid") in index: |
||||
assert compound_kind == "group" |
||||
continue |
||||
|
||||
# Everything has a kind and a name |
||||
child_record = { |
||||
"kind": child.get("kind"), |
||||
"name": name_prefix + child.find("name").text, |
||||
} |
||||
|
||||
if child.get("kind") == "enum": |
||||
# Enums are not compounds, but we want to resolve the parent of |
||||
# their values so they are not written as top level documents |
||||
child_record["children"] = [] |
||||
|
||||
if child.get("kind") == "enumvalue": |
||||
# Remove namespace prefix |
||||
child_record["name"] = child.find("name").text |
||||
|
||||
index[child.get("refid")] = child_record |
||||
|
||||
return index |
||||
|
||||
|
||||
def resolve_index(index, root): |
||||
""" |
||||
Walk a definition document and extend the index for linking. |
||||
|
||||
This does two things: sets the "parent" and "children" fields of all |
||||
applicable records, and sets the "strong" field of enums so that the |
||||
correct Sphinx role can be used when referring to them. |
||||
""" |
||||
|
||||
def add_child(index, parent_id, child_id): |
||||
parent = index[parent_id] |
||||
child = index[child_id] |
||||
|
||||
if child["kind"] == "enumvalue": |
||||
assert parent["kind"] == "enum" |
||||
assert "parent" not in child or child["parent"] == parent_id |
||||
child["parent"] = parent_id |
||||
|
||||
else: |
||||
if parent["kind"] in ["class", "struct", "union"]: |
||||
assert "parent" not in child or child["parent"] == parent_id |
||||
child["parent"] = parent_id |
||||
|
||||
if child_id not in parent["children"]: |
||||
parent["children"] += [child_id] |
||||
|
||||
compound = root.find("compounddef") |
||||
compound_kind = compound.get("kind") |
||||
|
||||
if compound_kind == "group": |
||||
for subgroup in compound.findall("innergroup"): |
||||
add_child(index, compound.get("id"), subgroup.get("refid")) |
||||
|
||||
for klass in compound.findall("innerclass"): |
||||
add_child(index, compound.get("id"), klass.get("refid")) |
||||
|
||||
for section in compound.findall("sectiondef"): |
||||
if section.get("kind").startswith("private"): |
||||
for member in section.findall("memberdef"): |
||||
if member.get("id") in index: |
||||
del index[member.get("id")] |
||||
else: |
||||
for member in section.findall("memberdef"): |
||||
member_id = member.get("id") |
||||
add_child(index, compound.get("id"), member_id) |
||||
|
||||
if member.get("kind") == "enum": |
||||
index[member_id]["strong"] = member.get("strong") == "yes" |
||||
for value in member.findall("enumvalue"): |
||||
add_child(index, member_id, value.get("id")) |
||||
|
||||
|
||||
def sphinx_role(record, lang): |
||||
""" |
||||
Return the Sphinx role used for a record. |
||||
|
||||
This is used for the description directive like ".. c:function::", and |
||||
links like ":c:func:`foo`. |
||||
""" |
||||
|
||||
kind = record["kind"] |
||||
|
||||
if kind in ["class", "function", "namespace", "struct", "union"]: |
||||
return lang + ":" + kind |
||||
|
||||
if kind == "define": |
||||
return "c:macro" |
||||
|
||||
if kind == "enum": |
||||
return lang + (":enum-class" if record["strong"] else ":enum") |
||||
|
||||
if kind == "typedef": |
||||
return lang + ":type" |
||||
|
||||
if kind == "enumvalue": |
||||
return lang + ":enumerator" |
||||
|
||||
if kind == "variable": |
||||
return lang + (":member" if "parent" in record else ":var") |
||||
|
||||
raise RuntimeError("No known role for kind '%s'" % kind) |
||||
|
||||
|
||||
def child_identifier(lang, parent_name, child_name): |
||||
""" |
||||
Return the identifier for an enum value or struct member. |
||||
|
||||
Sphinx, for some reason, uses a different syntax for this in C and C++. |
||||
""" |
||||
|
||||
separator = "::" if lang == "cpp" else "." |
||||
|
||||
return "%s%s%s" % (parent_name, separator, child_name) |
||||
|
||||
|
||||
def link_markup(index, lang, refid): |
||||
"""Return a Sphinx link for a Doxygen reference.""" |
||||
|
||||
record = index[refid] |
||||
kind, name = record["kind"], record["name"] |
||||
role = sphinx_role(record, lang) |
||||
|
||||
if kind in ["class", "enum", "struct", "typedef", "union"]: |
||||
return ":%s:`%s`" % (role, name) |
||||
|
||||
if kind == "function": |
||||
return ":%s:func:`%s`" % (lang, name) |
||||
|
||||
if kind == "enumvalue": |
||||
parent_name = index[record["parent"]]["name"] |
||||
return ":%s:`%s`" % (role, child_identifier(lang, parent_name, name)) |
||||
|
||||
if kind == "variable": |
||||
if "parent" not in record: |
||||
return ":%s:var:`%s`" % (lang, name) |
||||
|
||||
parent_name = index[record["parent"]]["name"] |
||||
return ":%s:`%s`" % (role, child_identifier(lang, parent_name, name)) |
||||
|
||||
raise RuntimeError("Unknown link target kind: %s" % kind) |
||||
|
||||
|
||||
def indent(markup, depth): |
||||
""" |
||||
Indent markup to a depth level. |
||||
|
||||
Like textwrap.indent() but takes an integer and works in reST indentation |
||||
levels for clarity." |
||||
""" |
||||
|
||||
return textwrap.indent(markup, " " * depth) |
||||
|
||||
|
||||
def heading(text, level): |
||||
""" |
||||
Return a ReST heading at a given level. |
||||
|
||||
Follows the style in the Python documentation guide, see |
||||
<https://devguide.python.org/documenting/#sections>. |
||||
""" |
||||
|
||||
assert 1 <= level <= 6 |
||||
|
||||
chars = ("#", "*", "=", "-", "^", '"') |
||||
line = chars[level] * len(text) |
||||
|
||||
return "%s\n%s\n%s\n\n" % (line if level < 3 else "", text, line) |
||||
|
||||
|
||||
def dox_to_rst(index, lang, node): |
||||
""" |
||||
Convert documentation commands (docCmdGroup) to Sphinx markup. |
||||
|
||||
This is used to convert the content of descriptions in the documentation. |
||||
It recursively parses all children tags and raises a RuntimeError if any |
||||
unknown tag is encountered. |
||||
""" |
||||
|
||||
def field_value(markup): |
||||
"""Return a value for a field as a single line or indented block.""" |
||||
if "\n" in markup.strip(): |
||||
return "\n" + indent(markup, 1) |
||||
|
||||
return " " + markup.strip() |
||||
|
||||
if node.tag == "computeroutput": |
||||
# assert len(node) == 0 FIXME |
||||
return "``%s``" % node.text |
||||
|
||||
if node.tag == "itemizedlist": |
||||
markup = "" |
||||
for item in node.findall("listitem"): |
||||
assert len(item) == 1 |
||||
markup += "\n- %s" % dox_to_rst(index, lang, item[0]) |
||||
|
||||
return markup |
||||
|
||||
if node.tag == "para": |
||||
markup = node.text if node.text is not None else "" |
||||
for child in node: |
||||
markup += dox_to_rst(index, lang, child) |
||||
markup += child.tail if child.tail is not None else "" |
||||
|
||||
return markup.strip() + "\n\n" |
||||
|
||||
if node.tag == "parameterlist": |
||||
markup = "" |
||||
for item in node.findall("parameteritem"): |
||||
name = item.find("parameternamelist/parametername") |
||||
description = item.find("parameterdescription") |
||||
assert len(description) == 1 |
||||
markup += "\n\n:param %s:%s" % ( |
||||
name.text, |
||||
field_value(dox_to_rst(index, lang, description[0])), |
||||
) |
||||
|
||||
return markup + "\n" |
||||
|
||||
if node.tag == "programlisting": |
||||
return "\n.. code-block:: %s\n\n%s" % ( |
||||
lang, |
||||
indent(plain_text(node), 1), |
||||
) |
||||
|
||||
if node.tag == "ref": |
||||
refid = node.get("refid") |
||||
if refid not in index: |
||||
sys.stderr.write("warning: Unresolved link: %s\n" % refid) |
||||
return node.text |
||||
|
||||
assert len(node) == 0 |
||||
assert len(link_markup(index, lang, refid)) > 0 |
||||
return link_markup(index, lang, refid) |
||||
|
||||
if node.tag == "simplesect": |
||||
assert len(node) == 1 |
||||
|
||||
if node.get("kind") == "return": |
||||
return "\n:returns:" + field_value( |
||||
dox_to_rst(index, lang, node[0]) |
||||
) |
||||
|
||||
if node.get("kind") == "see": |
||||
return dox_to_rst(index, lang, node[0]) |
||||
|
||||
raise RuntimeError("Unknown simplesect kind: %s" % node.get("kind")) |
||||
|
||||
if node.tag == "ulink": |
||||
return "`%s <%s>`_" % (node.text, node.get("url")) |
||||
|
||||
raise RuntimeError("Unknown documentation command: %s" % node.tag) |
||||
|
||||
|
||||
def description_markup(index, lang, node): |
||||
"""Return the markup for a brief or detailed description.""" |
||||
|
||||
assert node.tag == "briefdescription" or node.tag == "detaileddescription" |
||||
assert not (node.tag == "briefdescription" and len(node) > 1) |
||||
assert len(node.text.strip()) == 0 |
||||
|
||||
return "".join([dox_to_rst(index, lang, child) for child in node]) |
||||
|
||||
|
||||
def set_descriptions(index, lang, definition, record): |
||||
"""Set a record's brief/detailed descriptions from the XML definition.""" |
||||
|
||||
for tag in ["briefdescription", "detaileddescription"]: |
||||
node = definition.find(tag) |
||||
if node is not None: |
||||
record[tag] = description_markup(index, lang, node) |
||||
|
||||
|
||||
def set_template_params(node, record): |
||||
"""Set a record's template_params from the XML definition.""" |
||||
|
||||
template_param_list = node.find("templateparamlist") |
||||
if template_param_list is not None: |
||||
params = [] |
||||
for param in template_param_list.findall("param"): |
||||
if param.find("declname") is not None: |
||||
# Value parameter |
||||
type_text = plain_text(param.find("type")) |
||||
name_text = plain_text(param.find("declname")) |
||||
|
||||
params += ["%s %s" % (type_text, name_text)] |
||||
else: |
||||
# Type parameter |
||||
params += ["%s" % (plain_text(param.find("type")))] |
||||
|
||||
record["template_params"] = "%s" % ", ".join(params) |
||||
|
||||
|
||||
def plain_text(node): |
||||
""" |
||||
Return the plain text of a node with all tags ignored. |
||||
|
||||
This is needed where Doxygen may include refs but Sphinx needs plain text |
||||
because it parses things itself to generate links. |
||||
""" |
||||
|
||||
if node.tag == "sp": |
||||
markup = " " |
||||
elif node.text is not None: |
||||
markup = node.text |
||||
else: |
||||
markup = "" |
||||
|
||||
for child in node: |
||||
markup += plain_text(child) |
||||
markup += child.tail if child.tail is not None else "" |
||||
|
||||
return markup |
||||
|
||||
|
||||
def local_name(name): |
||||
"""Return a name with all namespace prefixes stripped.""" |
||||
|
||||
return name[name.rindex("::") + 2 :] if "::" in name else name |
||||
|
||||
|
||||
def read_definition_doc(index, lang, root): |
||||
"""Walk a definition document and update described records in the index.""" |
||||
|
||||
# Set descriptions for the compound itself |
||||
compound = root.find("compounddef") |
||||
compound_record = index[compound.get("id")] |
||||
set_descriptions(index, lang, compound, compound_record) |
||||
set_template_params(compound, compound_record) |
||||
|
||||
if compound.find("title") is not None: |
||||
compound_record["title"] = compound.find("title").text.strip() |
||||
|
||||
# Set documentation for all children |
||||
for section in compound.findall("sectiondef"): |
||||
if section.get("kind").startswith("private"): |
||||
continue |
||||
|
||||
for member in section.findall("memberdef"): |
||||
kind = member.get("kind") |
||||
record = index[member.get("id")] |
||||
set_descriptions(index, lang, member, record) |
||||
set_template_params(member, record) |
||||
|
||||
if compound.get("kind") in ["class", "struct", "union"]: |
||||
assert kind in ["function", "typedef", "variable"] |
||||
record["type"] = plain_text(member.find("type")) |
||||
|
||||
if kind == "enum": |
||||
for value in member.findall("enumvalue"): |
||||
set_descriptions( |
||||
index, lang, value, index[value.get("id")] |
||||
) |
||||
|
||||
elif kind == "function": |
||||
record["prototype"] = "%s %s%s" % ( |
||||
plain_text(member.find("type")), |
||||
member.find("name").text, |
||||
member.find("argsstring").text, |
||||
) |
||||
|
||||
elif kind == "typedef": |
||||
name = local_name(record["name"]) |
||||
args_text = member.find("argsstring").text |
||||
target_text = plain_text(member.find("type")) |
||||
if args_text is not None: # Function pointer |
||||
assert target_text[-2:] == "(*" and args_text[0] == ")" |
||||
record["type"] = target_text + args_text |
||||
record["definition"] = target_text + name + args_text |
||||
else: # Normal named typedef |
||||
assert target_text is not None |
||||
record["type"] = target_text |
||||
if member.find("definition").text.startswith("using"): |
||||
record["definition"] = "%s = %s" % ( |
||||
name, |
||||
target_text, |
||||
) |
||||
else: |
||||
record["definition"] = "%s %s" % ( |
||||
target_text, |
||||
name, |
||||
) |
||||
|
||||
elif kind == "variable": |
||||
record["definition"] = member.find("definition").text |
||||
|
||||
|
||||
def declaration_string(record): |
||||
""" |
||||
Return the string that describes a declaration. |
||||
|
||||
This is what follows the directive, and is in C/C++ syntax, except without |
||||
keywords like "typedef" and "using" as expected by Sphinx. For example, |
||||
"struct ThingImpl Thing" or "void run(int value)". |
||||
""" |
||||
|
||||
kind = record["kind"] |
||||
result = "" |
||||
|
||||
if "template_params" in record: |
||||
result = "template <%s> " % record["template_params"] |
||||
|
||||
if kind == "function": |
||||
result += record["prototype"] |
||||
elif kind == "typedef": |
||||
result += record["definition"] |
||||
elif kind == "variable": |
||||
if "parent" in record: |
||||
result += "%s %s" % (record["type"], local_name(record["name"])) |
||||
else: |
||||
result += record["definition"] |
||||
elif "type" in record: |
||||
result += "%s %s" % (record["type"], local_name(record["name"])) |
||||
else: |
||||
result += local_name(record["name"]) |
||||
|
||||
return result |
||||
|
||||
|
||||
def document_markup(index, lang, record): |
||||
"""Return the complete document that describes some documented entity.""" |
||||
|
||||
kind = record["kind"] |
||||
role = sphinx_role(record, lang) |
||||
name = record["name"] |
||||
markup = "" |
||||
|
||||
if name != local_name(name): |
||||
markup += ".. cpp:namespace:: %s\n\n" % name[0 : name.rindex("::")] |
||||
|
||||
# Write top-level directive |
||||
markup += ".. %s:: %s\n" % (role, declaration_string(record)) |
||||
|
||||
# Write main description blurb |
||||
markup += "\n" |
||||
markup += indent(record["briefdescription"], 1) |
||||
markup += indent(record["detaileddescription"], 1) |
||||
|
||||
assert ( |
||||
kind in ["class", "enum", "namespace", "struct", "union"] |
||||
or "children" not in record |
||||
) |
||||
|
||||
# Sphinx C++ namespaces work by setting a scope, they have no content |
||||
child_indent = 0 if kind == "namespace" else 1 |
||||
|
||||
# Write inline children if applicable |
||||
markup += "\n" |
||||
for child_id in record.get("children", []): |
||||
child_record = index[child_id] |
||||
child_role = sphinx_role(child_record, lang) |
||||
|
||||
child_header = ".. %s:: %s\n\n" % ( |
||||
child_role, |
||||
declaration_string(child_record), |
||||
) |
||||
|
||||
markup += "\n" |
||||
markup += indent(child_header, child_indent) |
||||
markup += indent(child_record["briefdescription"], child_indent + 1) |
||||
markup += indent(child_record["detaileddescription"], child_indent + 1) |
||||
markup += "\n" |
||||
|
||||
return markup |
||||
|
||||
|
||||
def symbol_filename(name): |
||||
"""Adapt the name of a symbol to be suitable for use as a filename.""" |
||||
|
||||
return name.replace("::", "__") |
||||
|
||||
|
||||
def emit_symbols(index, lang, symbol_dir, force): |
||||
"""Write a description file for every symbol documented in the index.""" |
||||
|
||||
for record in index.values(): |
||||
if ( |
||||
record["kind"] in ["group", "namespace"] |
||||
or "parent" in record |
||||
and index[record["parent"]]["kind"] != "group" |
||||
): |
||||
continue |
||||
|
||||
name = record["name"] |
||||
filename = os.path.join(symbol_dir, symbol_filename("%s.rst" % name)) |
||||
if not force and os.path.exists(filename): |
||||
raise FileExistsError("File already exists: '%s'" % filename) |
||||
|
||||
with open(filename, "w") as rst: |
||||
rst.write(heading(local_name(name), 3)) |
||||
rst.write(document_markup(index, lang, record)) |
||||
|
||||
|
||||
def emit_groups(index, output_dir, symbol_dir_name, force): |
||||
"""Write a description file for every group documented in the index.""" |
||||
|
||||
for record in index.values(): |
||||
if record["kind"] != "group": |
||||
continue |
||||
|
||||
name = record["name"] |
||||
filename = os.path.join(output_dir, "%s.rst" % name) |
||||
if not force and os.path.exists(filename): |
||||
raise FileExistsError("File already exists: '%s'" % filename) |
||||
|
||||
with open(filename, "w") as rst: |
||||
rst.write(heading(record["title"], 2)) |
||||
|
||||
# Get all child group and symbol names |
||||
group_names = [] |
||||
symbol_names = [] |
||||
for child_id in record["children"]: |
||||
child = index[child_id] |
||||
if child["kind"] == "group": |
||||
group_names += [child["name"]] |
||||
else: |
||||
symbol_names += [child["name"]] |
||||
|
||||
# Emit description (document body) |
||||
rst.write(record["briefdescription"] + "\n\n") |
||||
rst.write(record["detaileddescription"] + "\n\n") |
||||
|
||||
# Emit TOC |
||||
rst.write(".. toctree::\n") |
||||
|
||||
# Emit groups at the top of the TOC |
||||
for group_name in group_names: |
||||
rst.write("\n" + indent(group_name, 1)) |
||||
|
||||
# Emit symbols in sorted order |
||||
for symbol_name in sorted(symbol_names): |
||||
path = "/".join( |
||||
[symbol_dir_name, symbol_filename(symbol_name)] |
||||
) |
||||
rst.write("\n" + indent(path, 1)) |
||||
|
||||
rst.write("\n") |
||||
|
||||
|
||||
def run(index_xml_path, output_dir, symbol_dir_name, language, force): |
||||
"""Write a directory of Sphinx files from a Doxygen XML directory.""" |
||||
|
||||
# Build skeleton index from index.xml |
||||
xml_dir = os.path.dirname(index_xml_path) |
||||
index = load_index(index_xml_path) |
||||
|
||||
# Load all definition documents |
||||
definition_docs = [] |
||||
for record in index.values(): |
||||
if "xml_filename" in record: |
||||
xml_path = os.path.join(xml_dir, record["xml_filename"]) |
||||
definition_docs += [xml.etree.ElementTree.parse(xml_path)] |
||||
|
||||
# Do an initial pass of the definition documents to resolve the index |
||||
for root in definition_docs: |
||||
resolve_index(index, root) |
||||
|
||||
# Finally read the documentation from definition documents |
||||
for root in definition_docs: |
||||
read_definition_doc(index, language, root) |
||||
|
||||
# Emit output files |
||||
symbol_dir = os.path.join(output_dir, symbol_dir_name) |
||||
os.makedirs(symbol_dir, exist_ok=True) |
||||
emit_symbols(index, language, symbol_dir, force) |
||||
emit_groups(index, output_dir, symbol_dir_name, force) |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
ap = argparse.ArgumentParser( |
||||
usage="%(prog)s [OPTION]... XML_DIR OUTPUT_DIR", |
||||
description=__doc__, |
||||
formatter_class=argparse.RawDescriptionHelpFormatter, |
||||
) |
||||
|
||||
ap.add_argument( |
||||
"-f", |
||||
"--force", |
||||
action="store_true", |
||||
help="overwrite files", |
||||
) |
||||
|
||||
ap.add_argument( |
||||
"-l", |
||||
"--language", |
||||
default="c", |
||||
choices=["c", "cpp"], |
||||
help="language domain for output", |
||||
) |
||||
|
||||
ap.add_argument( |
||||
"-s", |
||||
"--symbol-dir-name", |
||||
default="symbols", |
||||
help="name for subdirectory of symbol documentation files", |
||||
) |
||||
|
||||
ap.add_argument("index_xml_path", help="path index.xml from Doxygen") |
||||
ap.add_argument("output_dir", help="output directory") |
||||
|
||||
run(**vars(ap.parse_args(sys.argv[1:]))) |
Loading…
Reference in new issue