dilim/meson
1
0
Fork 0
mirror of https://github.com/mesonbuild/meson.git synced 2026-06-30 19:57:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

parser: revert to single StringNode type

Browse Source 
this will allow transforming string types in the formater
This commit is contained in:
Charles Brunet
2023-09-28 09:08:18 -04:00
committed by Dylan Baker
parent 728fcdaab3
commit bd4fd90730
13 changed files with 62 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
mesonbuild/ast/interpreter.py
View File

@@ -207,8 +207,8 @@ class AstInterpreter(InterpreterBase):
def method_call(self, node: BaseNode) -> bool:
return True
def evaluate_fstring(self, node: mparser.FormatStringNode) -> str:
assert isinstance(node, mparser.FormatStringNode)
def evaluate_fstring(self, node: mparser.StringNode) -> str:
assert isinstance(node, mparser.StringNode)
return node.value
def evaluate_arraystatement(self, cur: mparser.ArrayNode) -> TYPE_var:
@@ -231,7 +231,7 @@ class AstInterpreter(InterpreterBase):
def evaluate_dictstatement(self, node: mparser.DictNode) -> TYPE_nkwargs:
def resolve_key(node: mparser.BaseNode) -> str:
if isinstance(node, mparser.BaseStringNode):
if isinstance(node, mparser.StringNode):
return node.value
return '__AST_UNKNOWN__'
arguments, kwargs = self.reduce_arguments(node.args, key_resolver=resolve_key)

10
mesonbuild/ast/introspection.py
View File

@@ -16,7 +16,7 @@ from ..build import Executable, Jar, SharedLibrary, SharedModule, StaticLibrary
from ..compilers import detect_compiler_for
from ..interpreterbase import InvalidArguments, SubProject
from ..mesonlib import MachineChoice, OptionKey
from ..mparser import BaseNode, ArithmeticNode, ArrayNode, ElementaryNode, IdNode, FunctionNode, BaseStringNode
from ..mparser import BaseNode, ArithmeticNode, ArrayNode, ElementaryNode, IdNode, FunctionNode, StringNode
from .interpreter import AstInterpreter
if T.TYPE_CHECKING:
@@ -118,7 +118,7 @@ class IntrospectionInterpreter(AstInterpreter):
if not self.is_subproject() and 'subproject_dir' in kwargs:
spdirname = kwargs['subproject_dir']
if isinstance(spdirname, BaseStringNode):
if isinstance(spdirname, StringNode):
assert isinstance(spdirname.value, str)
self.subproject_dir = spdirname.value
if not self.is_subproject():
@@ -165,7 +165,7 @@ class IntrospectionInterpreter(AstInterpreter):
for l in self.flatten_args(raw_langs):
if isinstance(l, str):
langs.append(l)
elif isinstance(l, BaseStringNode):
elif isinstance(l, StringNode):
langs.append(l.value)
for lang in sorted(langs, key=compilers.sort_clink):
@@ -254,7 +254,7 @@ class IntrospectionInterpreter(AstInterpreter):
# Pop the first element if the function is a build target function
if isinstance(curr, FunctionNode) and curr.func_name.value in BUILD_TARGET_FUNCTIONS:
arg_nodes.pop(0)
elementary_nodes = [x for x in arg_nodes if isinstance(x, (str, BaseStringNode))]
elementary_nodes = [x for x in arg_nodes if isinstance(x, (str, StringNode))]
inqueue += [x for x in arg_nodes if isinstance(x, (FunctionNode, ArrayNode, IdNode, ArithmeticNode))]
if elementary_nodes:
res += [curr]
@@ -369,6 +369,6 @@ class IntrospectionInterpreter(AstInterpreter):
assert isinstance(kw, IdNode), 'for mypy'
if kw.value == 'subproject_dir':
# mypy does not understand "and isinstance"
if isinstance(val, BaseStringNode):
if isinstance(val, StringNode):
return val.value
return None

46
mesonbuild/ast/printer.py
View File

@@ -61,22 +61,13 @@ class AstPrinter(AstVisitor):
def visit_StringNode(self, node: mparser.StringNode) -> None:
assert isinstance(node.value, str)
self.append("'" + self.escape(node.value) + "'", node)
node.lineno = self.curr_line or node.lineno
def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None:
assert isinstance(node.value, str)
self.append("f'" + self.escape(node.value) + "'", node)
node.lineno = self.curr_line or node.lineno
def visit_MultilineStringNode(self, node: mparser.MultilineStringNode) -> None:
assert isinstance(node.value, str)
self.append("'''" + node.value + "'''", node)
node.lineno = self.curr_line or node.lineno
def visit_FormatMultilineStringNode(self, node: mparser.MultilineFormatStringNode) -> None:
assert isinstance(node.value, str)
self.append("f'''" + node.value + "'''", node)
if node.is_fstring:
self.append('f', node)
if node.is_multiline:
self.append("'''" + node.value + "'''", node)
else:
self.append("'" + self.escape(node.value) + "'", node)
node.lineno = self.curr_line or node.lineno
def visit_ContinueNode(self, node: mparser.ContinueNode) -> None:
@@ -258,22 +249,12 @@ class RawPrinter(FullAstVisitor):
def visit_StringNode(self, node: mparser.StringNode) -> None:
self.enter_node(node)
self.result += f"'{node.raw_value}'"
self.exit_node(node)
def visit_MultilineStringNode(self, node: mparser.MultilineStringNode) -> None:
self.enter_node(node)
self.result += f"'''{node.value}'''"
self.exit_node(node)
def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None:
self.enter_node(node)
self.result += f"f'{node.raw_value}'"
self.exit_node(node)
def visit_MultilineFormatStringNode(self, node: mparser.MultilineFormatStringNode) -> None:
self.enter_node(node)
self.result += f"f'''{node.value}'''"
if node.is_fstring:
self.result += 'f'
if node.is_multiline:
self.result += f"'''{node.value}'''"
else:
self.result += f"'{node.raw_value}'"
self.exit_node(node)
def visit_ContinueNode(self, node: mparser.ContinueNode) -> None:
@@ -342,9 +323,6 @@ class AstJSONPrinter(AstVisitor):
def visit_StringNode(self, node: mparser.StringNode) -> None:
self.gen_ElementaryNode(node)
def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None:
self.gen_ElementaryNode(node)
def visit_ArrayNode(self, node: mparser.ArrayNode) -> None:
self._accept('args', node.args)
self.setbase(node)

9
mesonbuild/ast/visitor.py
View File

@@ -30,15 +30,6 @@ class AstVisitor:
def visit_StringNode(self, node: mparser.StringNode) -> None:
self.visit_default_func(node)
def visit_FormatStringNode(self, node: mparser.FormatStringNode) -> None:
self.visit_default_func(node)
def visit_MultilineStringNode(self, node: mparser.MultilineStringNode) -> None:
self.visit_default_func(node)
def visit_FormatMultilineStringNode(self, node: mparser.MultilineFormatStringNode) -> None:
self.visit_default_func(node)
def visit_ContinueNode(self, node: mparser.ContinueNode) -> None:
self.visit_default_func(node)

2
mesonbuild/coredata.py
View File

@@ -1142,7 +1142,7 @@ class MachineFileParser():
return section
def _evaluate_statement(self, node: mparser.BaseNode) -> T.Union[str, bool, int, T.List[str]]:
if isinstance(node, (mparser.BaseStringNode)):
if isinstance(node, (mparser.StringNode)):
return node.value
elif isinstance(node, mparser.BooleanNode):
return node.value

2
mesonbuild/interpreter/interpreter.py
View File

@@ -529,7 +529,7 @@ class Interpreter(InterpreterBase, HoldableObject):
assert isinstance(kw, mparser.IdNode), 'for mypy'
if kw.value == 'meson_version':
# mypy does not understand "and isinstance"
if isinstance(val, mparser.BaseStringNode):
if isinstance(val, mparser.StringNode):
self.handle_meson_version(val.value, val)
def get_build_def_files(self) -> mesonlib.OrderedSet[str]:

4
mesonbuild/interpreterbase/helpers.py
View File

@@ -15,7 +15,7 @@ if T.TYPE_CHECKING:
from .baseobjects import TYPE_var, TYPE_kwargs, SubProject
def flatten(args: T.Union['TYPE_var', T.List['TYPE_var']]) -> T.List['TYPE_var']:
if isinstance(args, mparser.BaseStringNode):
if isinstance(args, mparser.StringNode):
assert isinstance(args.value, str)
return [args.value]
if not isinstance(args, collections.abc.Sequence):
@@ -25,7 +25,7 @@ def flatten(args: T.Union['TYPE_var', T.List['TYPE_var']]) -> T.List['TYPE_var']
if isinstance(a, list):
rest = flatten(a)
result = result + rest
elif isinstance(a, mparser.BaseStringNode):
elif isinstance(a, mparser.StringNode):
result.append(a.value)
else:
result.append(a)

17
mesonbuild/interpreterbase/interpreterbase.py
View File

@@ -198,11 +198,12 @@ class InterpreterBase:
self.assignment(cur)
elif isinstance(cur, mparser.MethodNode):
return self.method_call(cur)
elif isinstance(cur, mparser.BaseStringNode):
if isinstance(cur, mparser.MultilineFormatStringNode):
return self.evaluate_multiline_fstring(cur)
elif isinstance(cur, mparser.FormatStringNode):
return self.evaluate_fstring(cur)
elif isinstance(cur, mparser.StringNode):
if cur.is_fstring:
if cur.is_multiline:
return self.evaluate_multiline_fstring(cur)
else:
return self.evaluate_fstring(cur)
else:
return self._holderify(cur.value)
elif isinstance(cur, mparser.BooleanNode):
@@ -256,7 +257,7 @@ class InterpreterBase:
@FeatureNew('dict', '0.47.0')
def evaluate_dictstatement(self, cur: mparser.DictNode) -> InterpreterObject:
def resolve_key(key: mparser.BaseNode) -> str:
if not isinstance(key, mparser.BaseStringNode):
if not isinstance(key, mparser.StringNode):
FeatureNew.single_use('Dictionary entry using non literal key', '0.53.0', self.subproject)
key_holder = self.evaluate_statement(key)
if key_holder is None:
@@ -424,11 +425,11 @@ class InterpreterBase:
return self.evaluate_statement(node.falseblock)
@FeatureNew('multiline format strings', '0.63.0')
def evaluate_multiline_fstring(self, node: mparser.MultilineFormatStringNode) -> InterpreterObject:
def evaluate_multiline_fstring(self, node: mparser.StringNode) -> InterpreterObject:
return self.evaluate_fstring(node)
@FeatureNew('format strings', '0.58.0')
def evaluate_fstring(self, node: T.Union[mparser.FormatStringNode, mparser.MultilineFormatStringNode]) -> InterpreterObject:
def evaluate_fstring(self, node: mparser.StringNode) -> InterpreterObject:
def replace(match: T.Match[str]) -> str:
var = str(match.group(1))
try:

4
mesonbuild/mintro.py
View File

@@ -26,7 +26,7 @@ from .dependencies import Dependency
from . import environment
from .interpreterbase import ObjectHolder
from .mesonlib import OptionKey
from .mparser import FunctionNode, ArrayNode, ArgumentNode, BaseStringNode
from .mparser import FunctionNode, ArrayNode, ArgumentNode, StringNode
if T.TYPE_CHECKING:
import argparse
@@ -185,7 +185,7 @@ def list_targets_from_source(intr: IntrospectionInterpreter) -> T.List[T.Dict[st
elif isinstance(n, ArgumentNode):
args = n.arguments
for j in args:
if isinstance(j, BaseStringNode):
if isinstance(j, StringNode):
assert isinstance(j.value, str)
res += [Path(j.value)]
elif isinstance(j, str):

34
mesonbuild/mparser.py
View File

@@ -298,31 +298,25 @@ class NumberNode(ElementaryNode[int]):
self.value = int(token.value, base=0)
self.bytespan = token.bytespan
class BaseStringNode(ElementaryNode[str]):
pass
@dataclass(unsafe_hash=True)
class StringNode(BaseStringNode):
class StringNode(ElementaryNode[str]):
raw_value: str = field(hash=False)
is_multiline: bool
is_fstring: bool
def __init__(self, token: Token[str], escape: bool = True):
super().__init__(token)
self.value = ESCAPE_SEQUENCE_SINGLE_RE.sub(decode_match, token.value) if escape else token.value
self.is_multiline = 'multiline' in token.tid
self.is_fstring = 'fstring' in token.tid
self.raw_value = token.value
class FormatStringNode(StringNode):
pass
if escape and not self.is_multiline:
self.value = self.escape()
@dataclass(unsafe_hash=True)
class MultilineStringNode(BaseStringNode):
def __init__(self, token: Token[str]):
super().__init__(token)
self.value = token.value
class MultilineFormatStringNode(MultilineStringNode):
pass
def escape(self) -> str:
return ESCAPE_SEQUENCE_SINGLE_RE.sub(decode_match, self.raw_value)
class ContinueNode(ElementaryNode):
pass
@@ -930,14 +924,8 @@ class Parser:
return self.create_node(IdNode, t)
if self.accept('number'):
return self.create_node(NumberNode, t)
if self.accept('string'):
if self.accept_any(('string', 'fstring', 'multiline_string', 'multiline_fstring')):
return self.create_node(StringNode, t)
if self.accept('fstring'):
return self.create_node(FormatStringNode, t)
if self.accept('multiline_string'):
return self.create_node(MultilineStringNode, t)
if self.accept('multiline_fstring'):
return self.create_node(MultilineFormatStringNode, t)
return EmptyNode(self.current.lineno, self.current.colno, self.current.filename)
def key_values(self) -> ArgumentNode:

5
mesonbuild/optinterpreter.py
View File

@@ -105,15 +105,14 @@ class OptionInterpreter:
return arg
if isinstance(arg, mparser.ParenthesizedNode):
return self.reduce_single(arg.inner)
elif isinstance(arg, (mparser.BaseStringNode, mparser.BooleanNode,
mparser.NumberNode)):
elif isinstance(arg, (mparser.StringNode, mparser.BooleanNode, mparser.NumberNode)):
return arg.value
elif isinstance(arg, mparser.ArrayNode):
return [self.reduce_single(curarg) for curarg in arg.args.arguments]
elif isinstance(arg, mparser.DictNode):
d = {}
for k, v in arg.args.kwargs.items():
if not isinstance(k, mparser.BaseStringNode):
if not isinstance(k, mparser.StringNode):
raise OptionException('Dictionary keys must be a string literal')
d[k.value] = self.reduce_single(v)
return d

28
mesonbuild/rewriter.py
View File

@@ -13,7 +13,7 @@ from .ast import IntrospectionInterpreter, BUILD_TARGET_FUNCTIONS, AstConditionL
from mesonbuild.mesonlib import MesonException, setup_vsenv
from . import mlog, environment
from functools import wraps
from .mparser import Token, ArrayNode, ArgumentNode, AssignmentNode, BaseStringNode, BooleanNode, ElementaryNode, IdNode, FunctionNode, StringNode, SymbolNode
from .mparser import Token, ArrayNode, ArgumentNode, AssignmentNode, StringNode, BooleanNode, ElementaryNode, IdNode, FunctionNode, SymbolNode
import json, os, re, sys
import typing as T
@@ -142,7 +142,7 @@ class MTypeStr(MTypeBase):
def new_node(cls, value=None):
if value is None:
value = ''
return StringNode(Token('', '', 0, 0, 0, None, str(value)))
return StringNode(Token('string', '', 0, 0, 0, None, str(value)))
@classmethod
def supported_nodes(cls):
@@ -259,17 +259,17 @@ class MTypeStrList(MTypeList):
@classmethod
def _new_element_node(cls, value):
return StringNode(Token('', '', 0, 0, 0, None, str(value)))
return StringNode(Token('string', '', 0, 0, 0, None, str(value)))
@staticmethod
def _check_is_equal(node, value) -> bool:
if isinstance(node, BaseStringNode):
if isinstance(node, StringNode):
return node.value == value
return False
@staticmethod
def _check_regex_matches(node, regex: str) -> bool:
if isinstance(node, BaseStringNode):
if isinstance(node, StringNode):
return re.match(regex, node.value) is not None
return False
@@ -293,7 +293,7 @@ class MTypeIDList(MTypeList):
@staticmethod
def _check_regex_matches(node, regex: str) -> bool:
if isinstance(node, BaseStringNode):
if isinstance(node, StringNode):
return re.match(regex, node.value) is not None
return False
@@ -657,7 +657,7 @@ class Rewriter:
src_list = []
for i in target['sources']:
for j in arg_list_from_node(i):
if isinstance(j, BaseStringNode):
if isinstance(j, StringNode):
src_list += [j.value]
# Generate the new String nodes
@@ -691,7 +691,7 @@ class Rewriter:
def find_node(src):
for i in target['sources']:
for j in arg_list_from_node(i):
if isinstance(j, BaseStringNode):
if isinstance(j, StringNode):
if j.value == src:
return i, j
return None, None
@@ -750,7 +750,7 @@ class Rewriter:
extra_files_list = []
for i in target['extra_files']:
for j in arg_list_from_node(i):
if isinstance(j, BaseStringNode):
if isinstance(j, StringNode):
extra_files_list += [j.value]
# Generate the new String nodes
@@ -781,7 +781,7 @@ class Rewriter:
def find_node(src):
for i in target['extra_files']:
for j in arg_list_from_node(i):
if isinstance(j, BaseStringNode):
if isinstance(j, StringNode):
if j.value == src:
return i, j
return None, None
@@ -850,12 +850,12 @@ class Rewriter:
src_list = []
for i in target['sources']:
for j in arg_list_from_node(i):
if isinstance(j, BaseStringNode):
if isinstance(j, StringNode):
src_list += [j.value]
extra_files_list = []
for i in target['extra_files']:
for j in arg_list_from_node(i):
if isinstance(j, BaseStringNode):
if isinstance(j, StringNode):
extra_files_list += [j.value]
test_data = {
'name': target['name'],
@@ -870,8 +870,8 @@ class Rewriter:
alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)]
path_sorter = lambda key: ([(key.count('/') <= idx, alphanum_key(x)) for idx, x in enumerate(key.split('/'))])
unknown = [x for x in i.arguments if not isinstance(x, BaseStringNode)]
sources = [x for x in i.arguments if isinstance(x, BaseStringNode)]
unknown = [x for x in i.arguments if not isinstance(x, StringNode)]
sources = [x for x in i.arguments if isinstance(x, StringNode)]
sources = sorted(sources, key=lambda x: path_sorter(x.value))
i.arguments = unknown + sources

1
unittests/allplatformstests.py
View File

@@ -3583,7 +3583,6 @@ class AllPlatformTests(BasePlatformTests):
'IdNode': [('value', None, str)],
'NumberNode': [('value', None, int)],
'StringNode': [('value', None, str)],
'FormatStringNode': [('value', None, str)],
'ContinueNode': [],
'BreakNode': [],
'ArgumentNode': [('positional', accept_node_list), ('kwargs', accept_kwargs)],