forked from RustPython/RustPython
-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathlib_updater.py
More file actions
executable file
·352 lines (275 loc) · 10.4 KB
/
lib_updater.py
File metadata and controls
executable file
·352 lines (275 loc) · 10.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
#!/usr/bin/env python
__doc__ = """
This tool helps with updating test files from CPython.
Examples
--------
To move the patches found in `Lib/test/foo.py` to ` ~/cpython/Lib/test/foo.py` then write the contents back to `Lib/test/foo.py`
>>> ./{fname} --from Lib/test/foo.py --to ~/cpython/Lib/test/foo.py -o Lib/test/foo.py
You can run the same command without `-o` to override the `--from` path:
>>> ./{fname} --from Lib/test/foo.py --to ~/cpython/Lib/test/foo.py
To get a baseline of patches, you can alter the patches file with your favorite tool/script/etc and then reapply it with:
>>> ./{fname} --from Lib/test/foo.py --show-patches -o my_patches.json
(By default the output is set to print to stdout).
When you want to apply your own patches:
>>> ./{fname} -p my_patches.json --to Lib/test/foo.py
""".format(fname=__import__("os").path.basename(__file__))
import argparse
import ast
import collections
import enum
import json
import pathlib
import re
import sys
import textwrap
import typing
if typing.TYPE_CHECKING:
from collections.abc import Iterator
type Patches = dict[str, dict[str, list["PatchSpec"]]]
DEFAULT_INDENT = " " * 4
COMMENT = "TODO: RUSTPYTHON"
UT = "unittest"
@enum.unique
class UtMethod(enum.StrEnum):
"""
UnitTest Method.
"""
def _generate_next_value_(name, start, count, last_values) -> str:
return name[0].lower() + name[1:]
def has_args(self) -> bool:
return self != self.ExpectedFailure
def has_cond(self) -> bool:
return self.endswith(("If", "Unless"))
ExpectedFailure = enum.auto()
ExpectedFailureIf = enum.auto()
ExpectedFailureIfWindows = enum.auto()
Skip = enum.auto()
SkipIf = enum.auto()
SkipUnless = enum.auto()
class PatchSpec(typing.NamedTuple):
"""
Attributes
----------
ut_method : UtMethod
unittest method.
cond : str, optional
`ut_method` condition. Relevant only for some of `ut_method` types.
reason : str, optional
Reason for why the test is patched in this way.
"""
ut_method: UtMethod
cond: str | None = None
reason: str = ""
@property
def _reason(self) -> str:
return f"{COMMENT}; {self.reason}".strip(" ;")
@property
def _attr_node(self) -> ast.Attribute:
return ast.Attribute(value=ast.Name(id=UT), attr=self.ut_method)
def as_ast_node(self) -> ast.Attribute | ast.Call:
if not self.ut_method.has_args():
return self._attr_node
args = []
if self.cond:
args.append(ast.parse(self.cond).body[0].value)
args.append(ast.Constant(value=self._reason))
return ast.Call(func=self._attr_node, args=args, keywords=[])
def as_decorator(self) -> str:
unparsed = ast.unparse(self.as_ast_node())
if not self.ut_method.has_args():
unparsed = f"{unparsed} # {self._reason}"
return f"@{unparsed}"
class PatchEntry(typing.NamedTuple):
"""
Stores patch metadata.
Attributes
----------
parent_class : str
Parent class of test.
test_name : str
Test name.
spec : PatchSpec
Patch spec.
"""
parent_class: str
test_name: str
spec: PatchSpec
@classmethod
def iter_patch_entires(
cls, tree: ast.Module, lines: list[str]
) -> "Iterator[typing.Self]":
for cls_node, fn_node in iter_tests(tree):
parent_class = cls_node.name
for dec_node in fn_node.decorator_list:
if not isinstance(dec_node, (ast.Attribute, ast.Call)):
continue
attr_node = (
dec_node if isinstance(dec_node, ast.Attribute) else dec_node.func
)
if (
isinstance(attr_node, ast.Name)
or getattr(attr_node.value, "id", None) != UT
):
continue
cond = None
try:
ut_method = UtMethod(attr_node.attr)
except ValueError:
continue
# If our ut_method has args then,
# we need to search for a constant that contains our `COMMENT`.
# Otherwise we need to search it in the raw source code :/
if ut_method.has_args():
reason = next(
(
node.value
for node in ast.walk(dec_node)
if isinstance(node, ast.Constant)
and isinstance(node.value, str)
and COMMENT in node.value
),
None,
)
# If we didn't find a constant containing <COMMENT>,
# then we didn't put this decorator
if not reason:
continue
if ut_method.has_cond():
cond = ast.unparse(dec_node.args[0])
else:
# Search first on decorator line, then in the line before
for line in lines[dec_node.lineno - 1 : dec_node.lineno - 3 : -1]:
if found := re.search(rf"{COMMENT}.?(.*)", line):
reason = found.group()
break
else:
# Didn't find our `COMMENT` :)
continue
reason = reason.removeprefix(COMMENT).strip(";:, ")
spec = PatchSpec(ut_method, cond, reason)
yield cls(parent_class, fn_node.name, spec)
def iter_tests(
tree: ast.Module,
) -> "Iterator[tuple[ast.ClassDef, ast.FunctionDef | ast.AsyncFunctionDef]]":
for key, nodes in ast.iter_fields(tree):
if key != "body":
continue
for cls_node in nodes:
if not isinstance(cls_node, ast.ClassDef):
continue
for fn_node in cls_node.body:
if not isinstance(fn_node, (ast.FunctionDef, ast.AsyncFunctionDef)):
continue
yield (cls_node, fn_node)
def iter_patches(contents: str) -> "Iterator[PatchEntry]":
lines = contents.splitlines()
tree = ast.parse(contents)
yield from PatchEntry.iter_patch_entires(tree, lines)
def build_patch_dict(it: "Iterator[PatchEntry]") -> Patches:
patches = collections.defaultdict(lambda: collections.defaultdict(list))
for entry in it:
patches[entry.parent_class][entry.test_name].append(entry.spec)
return {k: dict(v) for k, v in patches.items()}
def iter_patch_lines(tree: ast.Module, patches: Patches) -> "Iterator[tuple[int, str]]":
cache = {} # Used in phase 2. Stores the end line location of a class name.
# Phase 1: Iterate and mark existing tests
for cls_node, fn_node in iter_tests(tree):
cache[cls_node.name] = cls_node.end_lineno
specs = patches.get(cls_node.name, {}).pop(fn_node.name, None)
if not specs:
continue
lineno = min(
(dec_node.lineno for dec_node in fn_node.decorator_list),
default=fn_node.lineno,
)
indent = " " * fn_node.col_offset
patch_lines = "\n".join(spec.as_decorator() for spec in specs)
yield (lineno - 1, textwrap.indent(patch_lines, indent))
# Phase 2: Iterate and mark inhereted tests
for cls_name, tests in patches.items():
lineno = cache.get(cls_name)
if not lineno:
print(f"WARNING: {cls_name} does not exist in remote file", file=sys.stderr)
continue
for test_name, specs in tests.items():
decorators = "\n".join(spec.as_decorator() for spec in specs)
patch_lines = f"""
{decorators}
def {test_name}(self):
{DEFAULT_INDENT}return super().{test_name}()
""".rstrip()
yield (lineno, textwrap.indent(patch_lines, DEFAULT_INDENT))
def apply_patches(contents: str, patches: Patches) -> str:
tree = ast.parse(contents)
lines = contents.splitlines()
modifications = list(iter_patch_lines(tree, patches))
# Going in reverse to not distrupt the line offset
for lineno, patch in sorted(modifications, reverse=True):
lines.insert(lineno, patch)
joined = "\n".join(lines)
return f"{joined}\n"
def write_output(data: str, dest: str) -> None:
if dest == "-":
print(data, end="")
return
with open(dest, "w") as fd:
fd.write(data)
def build_argparse() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
)
patches_group = parser.add_mutually_exclusive_group(required=True)
patches_group.add_argument(
"-p",
"--patches",
help="File path to file containing patches in a JSON format",
type=pathlib.Path,
)
patches_group.add_argument(
"--from",
help="File to gather patches from",
dest="gather_from",
type=pathlib.Path,
)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument(
"--to",
help="File to apply patches to",
type=pathlib.Path,
)
group.add_argument(
"--show-patches", action="store_true", help="Show the patches and exit"
)
parser.add_argument(
"-o", "--output", default="-", help="Output file. Set to '-' for stdout"
)
return parser
if __name__ == "__main__":
parser = build_argparse()
args = parser.parse_args()
if args.patches:
patches = {
cls_name: {
test_name: [
PatchSpec(**spec)._replace(ut_method=UtMethod(spec["ut_method"]))
for spec in specs
]
for test_name, specs in tests.items()
}
for cls_name, tests in json.loads(args.patches.read_text()).items()
}
else:
patches = build_patch_dict(iter_patches(args.gather_from.read_text()))
if args.show_patches:
patches = {
cls_name: {
test_name: [spec._asdict() for spec in specs]
for test_name, specs in tests.items()
}
for cls_name, tests in patches.items()
}
output = json.dumps(patches, indent=4) + "\n"
write_output(output, args.output)
sys.exit(0)
patched = apply_patches(args.to.read_text(), patches)
write_output(patched, args.output)