Keep reference manifests parser-free
This commit is contained in:
parent
58c0196be5
commit
9cd7c4e424
4 changed files with 368 additions and 67 deletions
|
|
@ -136,7 +136,7 @@ class _SourceRecord:
|
|||
fingerprint: str
|
||||
text: str
|
||||
raw: bytes
|
||||
root: _TreeNode
|
||||
root: _TreeNode | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
|
@ -229,7 +229,7 @@ class JavaScriptReferenceAdapter:
|
|||
return _UNSUPPORTED_FACTS
|
||||
|
||||
def load_manifest(self) -> AdapterManifest:
|
||||
records = self._inventory()
|
||||
records = self._inventory(parse=False)
|
||||
path_sources = {record.source_path: record.source_id for record in records}
|
||||
sources = tuple(
|
||||
sorted(
|
||||
|
|
@ -390,7 +390,7 @@ class JavaScriptReferenceAdapter:
|
|||
)
|
||||
return result
|
||||
|
||||
def _inventory(self) -> tuple[_SourceRecord, ...]:
|
||||
def _inventory(self, *, parse: bool = True) -> tuple[_SourceRecord, ...]:
|
||||
paths: list[str] = []
|
||||
for source_root in self.source_roots:
|
||||
absolute_root = self.root.joinpath(*PurePosixPath(source_root).parts)
|
||||
|
|
@ -439,7 +439,7 @@ class JavaScriptReferenceAdapter:
|
|||
"Script source inventory exceeds the configured limit",
|
||||
maximum=self.max_sources,
|
||||
)
|
||||
records = tuple(self._read_source(path) for path in sorted(paths))
|
||||
records = tuple(self._read_source(path, parse=parse) for path in sorted(paths))
|
||||
modules = [record.module_name for record in records]
|
||||
if len(modules) != len(set(modules)):
|
||||
duplicates = sorted(module for module in set(modules) if modules.count(module) > 1)
|
||||
|
|
@ -450,7 +450,7 @@ class JavaScriptReferenceAdapter:
|
|||
)
|
||||
return records
|
||||
|
||||
def _read_source(self, relative: str) -> _SourceRecord:
|
||||
def _read_source(self, relative: str, *, parse: bool = True) -> _SourceRecord:
|
||||
path = PurePosixPath(relative)
|
||||
if path.is_absolute() or ".." in path.parts or path.suffix not in _SOURCE_SUFFIXES:
|
||||
raise DocForgeError("path_escape", "Script source path is unsafe")
|
||||
|
|
@ -543,14 +543,17 @@ class JavaScriptReferenceAdapter:
|
|||
source=relative,
|
||||
) from error
|
||||
language = self._language_for_suffix(path.suffix)
|
||||
parse_raw, root = self._parse(text, language)
|
||||
if root.has_error:
|
||||
raise DocForgeError(
|
||||
"invalid_script_source",
|
||||
f"{language.title()} source cannot be parsed by the reference adapter",
|
||||
source=relative,
|
||||
line=self._first_error_line(root) or 1,
|
||||
)
|
||||
parse_raw = text.encode("utf-8")
|
||||
root: _TreeNode | None = None
|
||||
if parse:
|
||||
parse_raw, root = self._parse(text, language)
|
||||
if root.has_error:
|
||||
raise DocForgeError(
|
||||
"invalid_script_source",
|
||||
f"{language.title()} source cannot be parsed by the reference adapter",
|
||||
source=relative,
|
||||
line=self._first_error_line(root) or 1,
|
||||
)
|
||||
return _SourceRecord(
|
||||
source_id=self._source_id(relative),
|
||||
source_path=relative,
|
||||
|
|
@ -682,31 +685,170 @@ class JavaScriptReferenceAdapter:
|
|||
|
||||
@classmethod
|
||||
def _static_specifiers(cls, record: _SourceRecord) -> tuple[str, ...]:
|
||||
result: set[str] = set()
|
||||
stack = [record.root]
|
||||
while stack:
|
||||
node = stack.pop()
|
||||
if node.type in {"import_statement", "export_statement"}:
|
||||
source = node.child_by_field_name("source")
|
||||
if source is not None:
|
||||
value = cls._string_literal(source, record.raw)
|
||||
if value is not None and value.startswith(("./", "../")):
|
||||
result.add(value)
|
||||
stack.extend(node.named_children)
|
||||
result = {
|
||||
value
|
||||
for value in cls._lexical_static_specifiers(record.text)
|
||||
if value.startswith(("./", "../"))
|
||||
}
|
||||
return tuple(sorted(result))
|
||||
|
||||
@staticmethod
|
||||
def _string_literal(node: _TreeNode, raw: bytes) -> str | None:
|
||||
value = raw[node.start_byte : node.end_byte].decode("utf-8")
|
||||
if (
|
||||
len(value) < 2
|
||||
or value[0] not in {'"', "'"}
|
||||
or value[-1] != value[0]
|
||||
or "\\" in value
|
||||
or "\x00" in value
|
||||
):
|
||||
return None
|
||||
return value[1:-1]
|
||||
def _script_tokens(source: str) -> tuple[tuple[str, str], ...]:
|
||||
"""Tokenize only enough JavaScript syntax to inventory static module specifiers."""
|
||||
|
||||
tokens: list[tuple[str, str]] = []
|
||||
position = 0
|
||||
length = len(source)
|
||||
while position < length:
|
||||
character = source[position]
|
||||
if character in " \t\f\v":
|
||||
position += 1
|
||||
continue
|
||||
if character in "\r\n":
|
||||
if character == "\r" and position + 1 < length and source[position + 1] == "\n":
|
||||
position += 1
|
||||
tokens.append(("newline", "\n"))
|
||||
position += 1
|
||||
continue
|
||||
if source.startswith("//", position):
|
||||
position += 2
|
||||
while position < length and source[position] not in "\r\n":
|
||||
position += 1
|
||||
continue
|
||||
if source.startswith("/*", position):
|
||||
position += 2
|
||||
while position < length and not source.startswith("*/", position):
|
||||
if source[position] in "\r\n":
|
||||
tokens.append(("newline", "\n"))
|
||||
if (
|
||||
source[position] == "\r"
|
||||
and position + 1 < length
|
||||
and source[position + 1] == "\n"
|
||||
):
|
||||
position += 1
|
||||
position += 1
|
||||
position = min(length, position + 2)
|
||||
continue
|
||||
if character in {'"', "'"}:
|
||||
quote = character
|
||||
position += 1
|
||||
value: list[str] = []
|
||||
escaped = False
|
||||
closed = False
|
||||
while position < length:
|
||||
current = source[position]
|
||||
if current == "\\":
|
||||
escaped = True
|
||||
position += 2
|
||||
continue
|
||||
if current == quote:
|
||||
position += 1
|
||||
closed = True
|
||||
break
|
||||
if current in "\r\n":
|
||||
break
|
||||
value.append(current)
|
||||
position += 1
|
||||
tokens.append(("string" if closed and not escaped else "opaque", "".join(value)))
|
||||
continue
|
||||
if character == "`":
|
||||
position += 1
|
||||
while position < length:
|
||||
current = source[position]
|
||||
if current == "\\":
|
||||
position += 2
|
||||
continue
|
||||
position += 1
|
||||
if current == "`":
|
||||
break
|
||||
tokens.append(("opaque", "template"))
|
||||
continue
|
||||
if character.isalpha() or character in {"_", "$"}:
|
||||
start = position
|
||||
position += 1
|
||||
while position < length and (
|
||||
source[position].isalnum() or source[position] in {"_", "$"}
|
||||
):
|
||||
position += 1
|
||||
tokens.append(("name", source[start:position]))
|
||||
continue
|
||||
tokens.append(("punct", character))
|
||||
position += 1
|
||||
return tuple(tokens)
|
||||
|
||||
@classmethod
|
||||
def _lexical_static_specifiers(cls, source: str) -> tuple[str, ...]:
|
||||
tokens = cls._script_tokens(source)
|
||||
result: set[str] = set()
|
||||
for position, (kind, value) in enumerate(tokens):
|
||||
if kind != "name" or value not in {"import", "export"}:
|
||||
continue
|
||||
if position and tokens[position - 1] == ("punct", "."):
|
||||
continue
|
||||
keyword = value
|
||||
index = position + 1
|
||||
depth = 0
|
||||
clause_seen = False
|
||||
line_break = False
|
||||
while index < len(tokens):
|
||||
token_kind, token_value = tokens[index]
|
||||
if token_kind == "newline" and depth == 0:
|
||||
line_break = True
|
||||
if keyword == "export" and not clause_seen:
|
||||
break
|
||||
index += 1
|
||||
continue
|
||||
if token_kind == "punct":
|
||||
if token_value in "([{":
|
||||
depth += 1
|
||||
clause_seen = True
|
||||
elif token_value in ")]}":
|
||||
depth = max(0, depth - 1)
|
||||
clause_seen = True
|
||||
elif token_value == ";" and depth == 0:
|
||||
break
|
||||
elif token_value in {"*", ","}:
|
||||
clause_seen = True
|
||||
elif token_kind == "string":
|
||||
if keyword == "import" and not clause_seen:
|
||||
result.add(token_value)
|
||||
break
|
||||
elif token_kind == "name":
|
||||
if line_break and token_value in {
|
||||
"class",
|
||||
"const",
|
||||
"export",
|
||||
"function",
|
||||
"import",
|
||||
"let",
|
||||
"return",
|
||||
"throw",
|
||||
"var",
|
||||
}:
|
||||
break
|
||||
if keyword == "export" and token_value in {
|
||||
"async",
|
||||
"class",
|
||||
"const",
|
||||
"default",
|
||||
"function",
|
||||
"let",
|
||||
"var",
|
||||
}:
|
||||
break
|
||||
if token_value == "from" and clause_seen:
|
||||
source_index = index + 1
|
||||
while source_index < len(tokens) and tokens[source_index][0] == "newline":
|
||||
source_index += 1
|
||||
if source_index < len(tokens) and tokens[source_index][0] == "string":
|
||||
result.add(tokens[source_index][1])
|
||||
break
|
||||
if keyword == "import" and token_value not in {"type"}:
|
||||
clause_seen = True
|
||||
if token_kind != "newline":
|
||||
line_break = False
|
||||
index += 1
|
||||
return tuple(sorted(result))
|
||||
|
||||
@staticmethod
|
||||
def _resolve_relative_specifier(
|
||||
|
|
@ -733,6 +875,12 @@ class JavaScriptReferenceAdapter:
|
|||
record: _SourceRecord,
|
||||
source: AdapterSource,
|
||||
) -> AdapterSourceProjection:
|
||||
if record.root is None:
|
||||
raise DocForgeError(
|
||||
"invalid_adapter",
|
||||
"Script extraction requires a parsed source record",
|
||||
source=record.source_path,
|
||||
)
|
||||
file_id = self._file_node_id(source.source_id)
|
||||
module_id = self._module_node_id(source.source_id)
|
||||
nodes: list[AdapterNode] = [
|
||||
|
|
@ -839,6 +987,13 @@ class JavaScriptReferenceAdapter:
|
|||
)
|
||||
|
||||
def _definitions(self, record: _SourceRecord) -> tuple[_Definition, ...]:
|
||||
root = record.root
|
||||
if root is None:
|
||||
raise DocForgeError(
|
||||
"invalid_adapter",
|
||||
"Script definition extraction requires a parsed source record",
|
||||
source=record.source_path,
|
||||
)
|
||||
definitions: list[_Definition] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
|
|
@ -859,7 +1014,7 @@ class JavaScriptReferenceAdapter:
|
|||
for child in node.named_children:
|
||||
visit(child, next_scopes)
|
||||
|
||||
visit(record.root, ())
|
||||
visit(root, ())
|
||||
ordered = tuple(sorted(definitions, key=lambda item: item.node_id))
|
||||
public_functions = {
|
||||
(function.qualified_name, function.line)
|
||||
|
|
|
|||
|
|
@ -91,7 +91,7 @@ class _SourceRecord:
|
|||
fingerprint: str
|
||||
raw: bytes
|
||||
text: str
|
||||
tree: ast.Module
|
||||
tree: ast.Module | None
|
||||
dependencies: tuple[str, ...] = ()
|
||||
|
||||
|
||||
|
|
@ -154,7 +154,7 @@ class PythonReferenceAdapter:
|
|||
return _UNSUPPORTED_FACTS
|
||||
|
||||
def load_manifest(self) -> AdapterManifest:
|
||||
records = self._inventory()
|
||||
records = self._inventory(parse=False)
|
||||
module_sources = {record.module_name: record.source_id for record in records}
|
||||
if len(module_sources) != len(records):
|
||||
raise DocForgeError(
|
||||
|
|
@ -325,7 +325,7 @@ class PythonReferenceAdapter:
|
|||
)
|
||||
return result
|
||||
|
||||
def _inventory(self) -> tuple[_SourceRecord, ...]:
|
||||
def _inventory(self, *, parse: bool = True) -> tuple[_SourceRecord, ...]:
|
||||
paths: list[str] = []
|
||||
for source_root in self.source_roots:
|
||||
absolute_root = self.root.joinpath(*PurePosixPath(source_root).parts)
|
||||
|
|
@ -374,7 +374,7 @@ class PythonReferenceAdapter:
|
|||
"Python source inventory exceeds the configured limit",
|
||||
maximum=self.max_sources,
|
||||
)
|
||||
records = tuple(self._read_source(path) for path in sorted(paths))
|
||||
records = tuple(self._read_source(path, parse=parse) for path in sorted(paths))
|
||||
modules = [record.module_name for record in records]
|
||||
if len(modules) != len(set(modules)):
|
||||
duplicates = sorted(module for module in set(modules) if modules.count(module) > 1)
|
||||
|
|
@ -385,7 +385,7 @@ class PythonReferenceAdapter:
|
|||
)
|
||||
return records
|
||||
|
||||
def _read_source(self, relative: str) -> _SourceRecord:
|
||||
def _read_source(self, relative: str, *, parse: bool = True) -> _SourceRecord:
|
||||
path = PurePosixPath(relative)
|
||||
if path.is_absolute() or ".." in path.parts or path.suffix != ".py":
|
||||
raise DocForgeError("path_escape", "Python source path is unsafe")
|
||||
|
|
@ -478,15 +478,17 @@ class PythonReferenceAdapter:
|
|||
source=relative,
|
||||
)
|
||||
text = self._decode_source(raw, relative)
|
||||
try:
|
||||
tree = ast.parse(text, filename=relative, type_comments=True)
|
||||
except SyntaxError as error:
|
||||
raise DocForgeError(
|
||||
"invalid_python_source",
|
||||
"Python source cannot be parsed by the reference adapter",
|
||||
source=relative,
|
||||
line=error.lineno,
|
||||
) from error
|
||||
tree: ast.Module | None = None
|
||||
if parse:
|
||||
try:
|
||||
tree = ast.parse(text, filename=relative, type_comments=True)
|
||||
except SyntaxError as error:
|
||||
raise DocForgeError(
|
||||
"invalid_python_source",
|
||||
"Python source cannot be parsed by the reference adapter",
|
||||
source=relative,
|
||||
line=error.lineno,
|
||||
) from error
|
||||
module_name = self._module_name(path, PurePosixPath(source_root))
|
||||
return _SourceRecord(
|
||||
source_id=self._source_id(relative),
|
||||
|
|
@ -585,24 +587,127 @@ class PythonReferenceAdapter:
|
|||
if record.source_path.endswith("/__init__.py")
|
||||
else record.module_name.rpartition(".")[0]
|
||||
)
|
||||
for node in ast.walk(record.tree):
|
||||
for level, module, imported_names in self._lexical_imports(record):
|
||||
candidates: list[str] = []
|
||||
if isinstance(node, ast.Import):
|
||||
candidates.extend(alias.name for alias in node.names)
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
base = self._import_from_base(package, node.level, node.module)
|
||||
if not imported_names and level == 0 and module is not None:
|
||||
candidates.append(module)
|
||||
else:
|
||||
base = self._import_from_base(package, level, module)
|
||||
if base:
|
||||
candidates.append(base)
|
||||
if node.module is None and base:
|
||||
candidates.extend(
|
||||
f"{base}.{alias.name}" for alias in node.names if alias.name != "*"
|
||||
)
|
||||
candidates.extend(f"{base}.{name}" for name in imported_names if name != "*")
|
||||
for module_name in candidates:
|
||||
target = module_sources.get(module_name)
|
||||
if target is not None and target != record.source_id:
|
||||
dependencies.add(target)
|
||||
return tuple(sorted(dependencies))
|
||||
|
||||
@staticmethod
|
||||
def _lexical_imports(
|
||||
record: _SourceRecord,
|
||||
) -> tuple[tuple[int, str | None, tuple[str, ...]], ...]:
|
||||
"""Inventory import dependencies without constructing a Python AST."""
|
||||
|
||||
try:
|
||||
tokens = tuple(tokenize.generate_tokens(io.StringIO(record.text).readline))
|
||||
except (IndentationError, tokenize.TokenError) as error:
|
||||
line = error.args[1][0] if len(error.args) > 1 else None
|
||||
raise DocForgeError(
|
||||
"invalid_python_source",
|
||||
"Python source cannot be tokenized by the reference adapter",
|
||||
source=record.source_path,
|
||||
line=line,
|
||||
) from error
|
||||
|
||||
statements: list[list[tokenize.TokenInfo]] = []
|
||||
current: list[tokenize.TokenInfo] = []
|
||||
nesting = 0
|
||||
ignored = {
|
||||
tokenize.ENCODING,
|
||||
tokenize.INDENT,
|
||||
tokenize.DEDENT,
|
||||
tokenize.NL,
|
||||
tokenize.COMMENT,
|
||||
}
|
||||
for token in tokens:
|
||||
if token.type in ignored:
|
||||
continue
|
||||
if token.type == tokenize.OP:
|
||||
if token.string in "([{":
|
||||
nesting += 1
|
||||
elif token.string in ")]}":
|
||||
nesting = max(0, nesting - 1)
|
||||
elif token.string == ";" and nesting == 0:
|
||||
if current:
|
||||
statements.append(current)
|
||||
current = []
|
||||
continue
|
||||
if token.type in {tokenize.NEWLINE, tokenize.ENDMARKER} and nesting == 0:
|
||||
if current:
|
||||
statements.append(current)
|
||||
current = []
|
||||
continue
|
||||
current.append(token)
|
||||
|
||||
imports: list[tuple[int, str | None, tuple[str, ...]]] = []
|
||||
for statement in statements:
|
||||
for position, token in enumerate(statement):
|
||||
if token.type != tokenize.NAME or token.string not in {"import", "from"}:
|
||||
continue
|
||||
if token.string == "import":
|
||||
names = PythonReferenceAdapter._imported_names(statement[position + 1 :])
|
||||
imports.extend((0, name, ()) for name in names)
|
||||
break
|
||||
import_position = next(
|
||||
(
|
||||
index
|
||||
for index in range(position + 1, len(statement))
|
||||
if statement[index].type == tokenize.NAME
|
||||
and statement[index].string == "import"
|
||||
),
|
||||
None,
|
||||
)
|
||||
if import_position is None:
|
||||
break
|
||||
prefix = statement[position + 1 : import_position]
|
||||
level = 0
|
||||
for item in prefix:
|
||||
if item.type == tokenize.NAME:
|
||||
break
|
||||
if item.type == tokenize.OP and set(item.string) == {"."}:
|
||||
level += len(item.string)
|
||||
module_parts = [item.string for item in prefix if item.type == tokenize.NAME]
|
||||
module = ".".join(module_parts) or None
|
||||
names = PythonReferenceAdapter._imported_names(statement[import_position + 1 :])
|
||||
imports.append((level, module, names))
|
||||
break
|
||||
return tuple(imports)
|
||||
|
||||
@staticmethod
|
||||
def _imported_names(tokens: Sequence[tokenize.TokenInfo]) -> tuple[str, ...]:
|
||||
names: list[str] = []
|
||||
current: list[str] = []
|
||||
skip_alias = False
|
||||
for token in tokens:
|
||||
if token.type == tokenize.NAME and token.string == "as":
|
||||
skip_alias = True
|
||||
continue
|
||||
if token.type == tokenize.OP and token.string == ",":
|
||||
if current:
|
||||
names.append(".".join(current))
|
||||
current = []
|
||||
skip_alias = False
|
||||
continue
|
||||
if token.type == tokenize.OP and token.string == "*":
|
||||
if not skip_alias:
|
||||
current.append("*")
|
||||
continue
|
||||
if token.type == tokenize.NAME and not skip_alias:
|
||||
current.append(token.string)
|
||||
if current:
|
||||
names.append(".".join(current))
|
||||
return tuple(name for name in names if name)
|
||||
|
||||
@staticmethod
|
||||
def _import_from_base(package: str, level: int, module: str | None) -> str:
|
||||
if level == 0:
|
||||
|
|
@ -621,6 +726,13 @@ class PythonReferenceAdapter:
|
|||
record: _SourceRecord,
|
||||
source: AdapterSource,
|
||||
) -> AdapterSourceProjection:
|
||||
tree = record.tree
|
||||
if tree is None:
|
||||
raise DocForgeError(
|
||||
"invalid_adapter",
|
||||
"Python extraction requires a parsed source record",
|
||||
source=record.source_path,
|
||||
)
|
||||
file_id = self._file_node_id(source.source_id)
|
||||
module_id = self._module_node_id(source.source_id)
|
||||
nodes: list[AdapterNode] = [
|
||||
|
|
@ -639,8 +751,7 @@ class PythonReferenceAdapter:
|
|||
kind="module",
|
||||
qualified_name=record.module_name,
|
||||
content=(
|
||||
ast.get_docstring(record.tree, clean=False)
|
||||
or f"Python module {record.module_name}."
|
||||
ast.get_docstring(tree, clean=False) or f"Python module {record.module_name}."
|
||||
),
|
||||
source_path=record.source_path,
|
||||
anchor="L1",
|
||||
|
|
@ -721,6 +832,13 @@ class PythonReferenceAdapter:
|
|||
record: _SourceRecord,
|
||||
module_node_id: str,
|
||||
) -> tuple[_Definition, ...]:
|
||||
tree = record.tree
|
||||
if tree is None:
|
||||
raise DocForgeError(
|
||||
"invalid_adapter",
|
||||
"Python definition extraction requires a parsed source record",
|
||||
source=record.source_path,
|
||||
)
|
||||
definitions: list[_Definition] = []
|
||||
qualified_names: set[str] = set()
|
||||
adapter = self
|
||||
|
|
@ -773,7 +891,7 @@ class PythonReferenceAdapter:
|
|||
def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:
|
||||
self._definition("function", node)
|
||||
|
||||
Collector().visit(record.tree)
|
||||
Collector().visit(tree)
|
||||
return tuple(sorted(definitions, key=lambda item: item.node_id))
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue