Coverage for C:\leo.repo\leo-editor\leo\plugins\importers\cython.py : 60%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#@+leo-ver=5-thin
2#@+node:ekr.20200619141135.1: * @file ../plugins/importers/cython.py
3"""@auto importer for cython."""
4import re
5from typing import Any, Dict, List
6from leo.core import leoGlobals as g
7from leo.plugins.importers import linescanner
8Importer = linescanner.Importer
9Target = linescanner.Target
10#@+others
11#@+node:ekr.20200619141201.2: ** class Cython_Importer(Importer)
12class Cython_Importer(Importer):
13 """A class to store and update scanning state."""
15 starts_pattern = re.compile(r'\s*(class|def|cdef|cpdef)\s+')
16 # Matches lines that apparently start a class or def.
17 class_pat = re.compile(r'\s*class\s+(\w+)\s*(\([\w.]+\))?')
18 def_pat = re.compile(r'\s*(cdef|cpdef|def)\s+(\w+)')
19 trace = False
20 #@+others
21 #@+node:ekr.20200619144343.1: *3* cy_i.ctor
22 def __init__(self, importCommands, **kwargs):
23 """Cython_Importer.ctor."""
24 super().__init__(
25 importCommands,
26 language='cython',
27 state_class=Cython_ScanState,
28 strict=True,
29 )
30 self.put_decorators = self.c.config.getBool('put-cython-decorators-in-imported-headlines')
31 #@+node:ekr.20200619141201.3: *3* cy_i.clean_headline
32 def clean_headline(self, s, p=None):
33 """Return a cleaned up headline s."""
34 if p: # Called from clean_all_headlines:
35 return self.get_decorator(p) + p.h
36 # Handle def, cdef, cpdef.
37 m = re.match(r'\s*(cpdef|cdef|def)\s+(\w+)', s)
38 if m:
39 return m.group(2)
40 # Handle classes.
41 m = re.match(r'\s*class\s+(\w+)\s*(\([\w.]+\))?', s)
42 if m:
43 return 'class %s%s' % (m.group(1), m.group(2) or '')
44 return s.strip()
46 #@+node:vitalije.20211207173723.1: *3* check
47 def check(self, unused_s, parent):
48 """
49 Cython_Importer.check: override Importer.check.
51 Return True if perfect import checks pass, making additional allowances
52 for underindented comment lines.
54 Raise AssertionError if the checks fail while unit testing.
55 """
56 if g.app.suppressImportChecks:
57 g.app.suppressImportChecks = False
58 return True
59 s1 = g.toUnicode(self.file_s, self.encoding)
60 s2 = self.trial_write()
61 # Regularize the lines first.
62 lines1 = g.splitLines(s1.rstrip() + '\n')
63 lines2 = g.splitLines(s2.rstrip() + '\n')
64 # #2327: Ignore blank lines and lws in comment lines.
65 test_lines1 = self.strip_blank_and_comment_lines(lines1)
66 test_lines2 = self.strip_blank_and_comment_lines(lines2)
67 # #2327: Report all remaining mismatches.
68 ok = test_lines1 == test_lines2
69 if not ok:
70 self.show_failure(lines1, lines2, g.shortFileName(self.root.h))
71 return ok
72 #@+node:vitalije.20211207173805.1: *3* strip_blank_and_comment_lines
73 def strip_blank_and_comment_lines(self, lines):
74 """Strip all blank lines and strip lws from comment lines."""
76 def strip(s):
77 return s.strip() if s.isspace() else s.lstrip() if s.strip().startswith('#') else s
79 return [strip(z) for z in lines]
80 #@+node:vitalije.20211207173901.1: *3* get_decorator
81 decorator_pat = re.compile(r'\s*@\s*([\w\.]+)')
83 def get_decorator(self, p):
84 if g.unitTesting or self.put_decorators:
85 for s in self.get_lines(p):
86 if not s.isspace():
87 m = self.decorator_pat.match(s)
88 if m:
89 s = s.strip()
90 if s.endswith('('):
91 s = s[:-1].strip()
92 return s + ' '
93 return ''
94 return ''
95 #@+node:vitalije.20211207173935.1: *3* find_class
96 def find_class(self, parent):
97 """
98 Find the start and end of a class/def in a node.
99 Return (kind, i, j), where kind in (None, 'class', 'def')
100 """
101 # Called from Leo's core to implement two minor commands.
102 prev_state = Cython_ScanState()
103 target = Target(parent, prev_state)
104 stack = [target]
105 lines = g.splitlines(parent.b)
106 index = 0
107 for i, line in enumerate(lines):
108 new_state = self.scan_line(line, prev_state)
109 if self.prev_state.context or self.ws_pattern.match(line): # type:ignore
110 pass
111 else:
112 m = self.class_or_def_pattern.match(line)
113 if m:
114 return self.skip_block(i, index, lines, new_state, stack)
115 prev_state = new_state
116 return None, -1, -1
117 #@+node:vitalije.20211207174005.1: *3* skip_block
118 def skip_block(self, i, index, lines, prev_state, stack):
119 """
120 Find the end of a class/def starting at index
121 on line i of lines.
122 Return (kind, i, j), where kind in (None, 'class', 'def').
123 """
124 index1 = index
125 line = lines[i]
126 kind = 'class' if line.strip().startswith('class') else 'def'
127 top = stack[-1] ### new
128 i += 1
129 while i < len(lines):
130 line = lines[i]
131 index += len(line)
132 new_state = self.scan_line(line, prev_state)
133 ### if self.ends_block(line, new_state, prev_state, stack):
134 if new_state.indent < top.state.indent:
135 return kind, index1, index
136 prev_state = new_state
137 i += 1
138 return None, -1, -1
139 #@+node:vitalije.20211207174043.1: *3* gen_lines
140 class_or_def_pattern = re.compile(r'\s*(class|cdef|cpdef|def)\s+')
142 def gen_lines(self, lines, parent):
143 """
144 Non-recursively parse all lines of s into parent, creating descendant
145 nodes as needed.
146 """
147 assert self.root == parent, (self.root, parent)
148 # Init the state.
149 self.new_state = Cython_ScanState()
150 assert self.new_state.indent == 0
151 self.vnode_info = {
152 # Keys are vnodes, values are inner dicts.
153 parent.v: {
154 '@others': True,
155 'indent': 0, # None denotes a to-be-defined value.
156 'kind': 'outer',
157 'lines': ['@others\n'], # The post pass adds @language and @tabwidth directives.
158 }
159 }
160 if g.unitTesting:
161 g.vnode_info = self.vnode_info # A hack.
162 # Create a Declarations node.
163 p = self.start_python_block('org', 'Declarations', parent)
164 #
165 # The main importer loop. Don't worry about the speed of this loop.
166 for line in lines:
167 # Update the state, remembering the previous state.
168 self.prev_state = self.new_state
169 self.new_state = self.scan_line(line, self.prev_state)
170 # Handle the line.
171 if self.prev_state.context:
172 # A line with a string or docstring.
173 self.add_line(p, line, tag='string')
174 elif self.ws_pattern.match(line):
175 # A blank or comment line.
176 self.add_line(p, line, tag='whitespace')
177 else:
178 # The leading whitespace of all other lines are significant.
179 m = self.class_or_def_pattern.match(line)
180 kind = m.group(1) if m else 'normal'
181 if kind in ('cdef', 'cpdef'):
182 kind = 'def'
183 p = self.end_previous_blocks(kind, line, p)
184 if m:
185 assert kind in ('class', 'def'), repr(kind)
186 if kind == 'class':
187 p = self.do_class(line, p)
188 else:
189 p = self.do_def(line, p)
190 else:
191 p = self.do_normal_line(line,p)
192 #@+node:vitalije.20211207174130.1: *3* do_class
193 def do_class(self, line, parent):
195 d = self.vnode_info [parent.v]
196 parent_kind = d ['kind']
197 if parent_kind in ('outer', 'org', 'class'):
198 # Create a new parent.
199 self.gen_python_ref(line, parent)
200 p = self.start_python_block('class', line, parent)
201 else:
202 # Don't change parent.
203 p = parent
204 self.add_line(p, line, tag='class')
205 return p
206 #@+node:vitalije.20211207174137.1: *3* do_def
207 def do_def(self, line, parent):
209 new_indent = self.new_state.indent
210 d = self.vnode_info
211 parent_indent = d [parent.v] ['indent']
212 parent_kind = d [parent.v] ['kind']
213 if parent_kind in ('outer', 'class'):
214 # Create a new parent.
215 self.gen_python_ref(line, parent)
216 p = self.start_python_block('def', line, parent)
217 self.add_line(p, line, tag='def')
218 return p
219 # For 'org' parents, look at the grand parent kind.
220 if parent_kind == 'org':
221 grand_kind = d [parent.parent().v] ['kind']
222 if grand_kind == 'class' and new_indent <= parent_indent:
223 self.gen_python_ref(line, parent)
224 p = parent.parent()
225 p = self.start_python_block('def', line, p)
226 self.add_line(p, line, tag='def')
227 return p
228 # The default: don't change parent.
229 self.add_line(parent, line, tag='def')
230 return parent
231 #@+node:vitalije.20211207174148.1: *3* do_normal_line
232 def do_normal_line(self, line, p):
234 new_indent = self.new_state.indent
235 d = self.vnode_info [p.v]
236 parent_indent = d ['indent']
237 parent_kind = d ['kind']
238 if parent_kind == 'outer':
239 # Create an organizer node, regardless of indentation.
240 p = self.start_python_block('org', line, p)
241 elif parent_kind == 'class' and new_indent < parent_indent:
242 # Create an organizer node.
243 self.gen_python_ref(line, p)
244 p = self.start_python_block('org', line, p)
245 self.add_line(p, line, tag='normal')
246 return p
247 #@+node:vitalije.20211207174159.1: *3* end_previous_blocks
248 def end_previous_blocks(self, kind, line, p):
249 """
250 End blocks that are incompatible with the new line.
251 - kind: The kind of the incoming line: 'class', 'def' or 'normal'.
252 - new_indent: The indentation of the incoming line.
254 Return p, a parent that will either contain the new line or will be the
255 parent of a new child of parent.
256 """
257 new_indent = self.new_state.indent
258 while p:
259 d = self.vnode_info [p.v]
260 parent_indent, parent_kind = d ['indent'], d ['kind']
261 if parent_kind == 'outer':
262 return p
263 if new_indent > parent_indent:
264 return p
265 if new_indent < parent_indent:
266 p = p.parent()
267 continue
268 assert new_indent == parent_indent, (new_indent, parent_indent)
269 if kind == 'normal':
270 # Don't change parent, whatever it is.
271 return p
272 if new_indent == 0:
273 # Continue until we get to the outer level.
274 if parent_kind == 'outer':
275 return p
276 p = p.parent()
277 continue
278 # The context-dependent cases...
279 assert new_indent > 0 and new_indent == parent_indent, (new_indent, parent_indent)
280 assert kind in ('class', 'def')
281 if kind == 'class':
282 # Allow nested classes.
283 return p.parent() if new_indent < parent_indent else p
284 assert kind == 'def', repr(kind)
285 if parent_kind in ('class', 'outer'):
286 return p
287 d2 = self.vnode_info [p.parent().v]
288 grand_kind = d2 ['kind']
289 if parent_kind == 'def' and grand_kind in ('class', 'outer'):
290 return p.parent()
291 return p
292 assert False, 'No parent'
293 #@+node:vitalije.20211207174206.1: *3* gen_python_ref
294 def gen_python_ref(self, line, p):
295 """Generate the at-others directive and set p's at-others flag"""
296 d = self.vnode_info [p.v]
297 if d ['@others']:
298 return
299 d ['@others'] = True
300 indent_ws = self.get_str_lws(line)
301 ref_line = f"{indent_ws}@others\n"
302 self.add_line(p, ref_line, tag='@others')
303 #@+node:vitalije.20211207174213.1: *3* start_python_block
304 def start_python_block(self, kind, line, parent):
305 """
306 Create, p as the last child of parent and initialize the p.v._import_* ivars.
308 Return p.
309 """
310 assert kind in ('org', 'class', 'def'), g.callers()
311 # Create a new node p.
312 p = parent.insertAsLastChild()
313 v = p.v
314 # Set p.h.
315 p.h = self.clean_headline(line, p=None).strip()
316 if kind == 'org':
317 p.h = f"Organizer: {p.h}"
318 #
319 # Compute the indentation at p.
320 parent_info = self.vnode_info.get(parent.v)
321 assert parent_info, (parent.h, g.callers())
322 parent_indent = parent_info.get('indent')
323 ### Dubious: prevents proper handling of strangely-indented code.
324 indent = parent_indent + 4 if kind == 'class' else parent_indent
325 # Update vnode_info for p.v
326 assert not v in self.vnode_info, (p.h, g.callers())
327 self.vnode_info [v] = {
328 '@others': False,
329 'indent': indent,
330 'kind': kind,
331 'lines': [],
332 }
333 return p
334 #@+node:vitalije.20211207174318.1: *3* adjust_all_decorator_lines
335 def adjust_all_decorator_lines(self, parent):
336 """Move decorator lines (only) to the next sibling node."""
337 g.trace(parent.h)
338 for p in parent.self_and_subtree():
339 for child in p.children():
340 if child.hasNext():
341 self.adjust_decorator_lines(child)
342 #@+node:vitalije.20211207174323.1: *4* adjust_decorator_lines
343 def adjust_decorator_lines(self, p):
344 """Move decorator lines from the end of p.b to the start of p.next().b."""
345 ### To do
346 #@+node:vitalije.20211207174343.1: *3* promote_first_child
347 def promote_first_child(self, parent):
348 """Move a smallish first child to the start of parent."""
349 #@+node:vitalije.20211207174354.1: *3* create_child_node
350 def create_child_node(self, parent, line, headline):
351 """Create a child node of parent."""
352 assert False, g.callers()
353 #@+node:vitalije.20211207174358.1: *3* cut_stack
354 def cut_stack(self, new_state, stack):
355 """Cut back the stack until stack[-1] matches new_state."""
356 assert False, g.callers()
357 #@+node:vitalije.20211207174403.1: *3* trace_status
358 def trace_status(self, line, new_state, prev_state, stack, top):
359 """Do-nothing override of Import.trace_status."""
360 assert False, g.callers()
361 #@+node:vitalije.20211207174409.1: *3* add_line
362 heading_printed = False
364 def add_line(self, p, s, tag=None):
365 """Append the line s to p.v._import_lines."""
366 assert s and isinstance(s, str), (repr(s), g.callers())
367 if self.trace:
368 h = p.h
369 if h.startswith('@'):
370 h_parts = p.h.split('.')
371 h = h_parts[-1]
372 if not self.heading_printed:
373 self.heading_printed = True
374 g.trace(f"{'tag or caller ':>20} {' '*8+'top node':30} line")
375 g.trace(f"{'-' * 13 + ' ':>20} {' '*8+'-' * 8:30} {'-' * 4}")
376 if tag:
377 kind = self.vnode_info [p.v] ['kind']
378 tag = f"{kind:>5}:{tag:<10}"
379 g.trace(f"{(tag or g.caller()):>20} {h[:30]!r:30} {s!r}")
380 self.vnode_info [p.v] ['lines'].append(s)
381 #@+node:vitalije.20211207174416.1: *3* common_lws
382 def common_lws(self, lines):
383 """
384 Override Importer.common_lws.
386 Return the lws (a string) common to all lines.
388 We must unindent the class/def line fully.
389 It would be wrong to examine the indentation of other lines.
390 """
391 return self.get_str_lws(lines[0]) if lines else ''
392 #@+node:vitalije.20211207174422.1: *3* clean_all_headlines
393 def clean_all_headlines(self, parent):
394 """
395 Clean all headlines in parent's tree by calling the language-specific
396 clean_headline method.
397 """
398 for p in parent.subtree():
399 # Important: i.gen_ref does not know p when it calls
400 # self.clean_headline.
401 h = self.clean_headline(p.h, p=p)
402 if h and h != p.h:
403 p.h = h
404 #@+node:vitalije.20211207174429.1: *3* find_tail
405 def find_tail(self, p):
406 """
407 Find the tail (trailing unindented) lines.
408 return head, tail
409 """
410 lines = self.get_lines(p) [:]
411 tail = []
412 # First, find all potentially tail lines, including blank lines.
413 while lines:
414 line = lines.pop()
415 if line.lstrip() == line or not line.strip():
416 tail.append(line)
417 else:
418 break
419 # Next, remove leading blank lines from the tail.
420 while tail:
421 line = tail[-1]
422 if line.strip():
423 break
424 else:
425 tail.pop(0)
426 if 0:
427 g.printObj(lines, tag=f"lines: find_tail: {p.h}")
428 g.printObj(tail, tag=f"tail: find_tail: {p.h}")
429 #@+node:vitalije.20211207174436.1: *3* promote_last_lines
430 def promote_last_lines(self, parent):
431 """A do-nothing override."""
432 #@+node:vitalije.20211207174441.1: *3* promote_trailing_underindented_lines
433 def promote_trailing_underindented_lines(self, parent):
434 """A do-nothing override."""
435 #@+node:vitalije.20211207174501.1: *3* get_new_dict
436 #@@nobeautify
438 def get_new_dict(self, context):
439 """
440 Return a *general* state dictionary for the given context.
441 Subclasses may override...
442 """
443 comment, block1, block2 = self.single_comment, self.block1, self.block2
445 def add_key(d, key, data):
446 aList = d.get(key,[])
447 aList.append(data)
448 d[key] = aList
450 d: Dict[str, List[Any]]
452 if context:
453 d = {
454 # key kind pattern ends?
455 '\\': [('len+1', '\\',None),],
456 '"':[
457 ('len', '"""', context == '"""'),
458 ('len', '"', context == '"'),
459 ],
460 "'":[
461 ('len', "'''", context == "'''"),
462 ('len', "'", context == "'"),
463 ],
464 }
465 if block1 and block2:
466 add_key(d, block2[0], ('len', block1, True))
467 else:
468 # Not in any context.
469 d = {
470 # key kind pattern new-ctx deltas
471 '\\': [('len+1','\\', context, None),],
472 '#': [('all', '#', context, None),],
473 '"':[
474 # order matters.
475 ('len', '"""', '"""', None),
476 ('len', '"', '"', None),
477 ],
478 "'":[
479 # order matters.
480 ('len', "'''", "'''", None),
481 ('len', "'", "'", None),
482 ],
483 '{': [('len', '{', context, (1,0,0)),],
484 '}': [('len', '}', context, (-1,0,0)),],
485 '(': [('len', '(', context, (0,1,0)),],
486 ')': [('len', ')', context, (0,-1,0)),],
487 '[': [('len', '[', context, (0,0,1)),],
488 ']': [('len', ']', context, (0,0,-1)),],
489 }
490 if comment:
491 add_key(d, comment[0], ('all', comment, '', None))
492 if block1 and block2:
493 add_key(d, block1[0], ('len', block1, block1, None))
494 return d
495 #@-others
496#@+node:vitalije.20211207174609.1: ** class Cython_State
497class Cython_ScanState:
498 """A class representing the state of the python line-oriented scan."""
500 def __init__(self, d=None):
501 """Cython_ScanState ctor."""
502 if d:
503 indent = d.get('indent')
504 prev = d.get('prev')
505 self.indent = prev.indent if prev.bs_nl else indent
506 self.context = prev.context
507 self.curlies = prev.curlies
508 self.parens = prev.parens
509 self.squares = prev.squares
510 else:
511 self.bs_nl = False
512 self.context = ''
513 self.curlies = self.parens = self.squares = 0
514 self.indent = 0
516 def __repr__(self):
517 """Py_State.__repr__"""
518 return self.short_description()
520 __str__ = __repr__
522 def short_description(self): # pylint: disable=no-else-return
523 bsnl = 'bs-nl' if self.bs_nl else ''
524 context = f"{self.context} " if self.context else ''
525 indent = self.indent
526 curlies = f"{{{self.curlies}}}" if self.curlies else ''
527 parens = f"({self.parens})" if self.parens else ''
528 squares = f"[{self.squares}]" if self.squares else ''
529 return f"{context}indent:{indent}{curlies}{parens}{squares}{bsnl}"
530 def level(self):
531 """Python_ScanState.level."""
532 return self.indent
533 def in_context(self):
534 """True if in a special context."""
535 return (
536 self.context or
537 self.curlies > 0 or
538 self.parens > 0 or
539 self.squares > 0 or
540 self.bs_nl
541 )
542 def update(self, data):
543 """
544 Update the state using the 6-tuple returned by i.scan_line.
545 Return i = data[1]
546 """
547 context, i, delta_c, delta_p, delta_s, bs_nl = data
548 self.bs_nl = bs_nl
549 self.context = context
550 self.curlies += delta_c
551 self.parens += delta_p
552 self.squares += delta_s
553 return i
554#@+node:ekr.20211121065103.1: ** class CythonTarget
555class CythonTarget:
556 """
557 A class describing a target node p.
558 state is used to cut back the stack.
559 """
560 # Same as the legacy PythonTarget class, except for the class name.
562 def __init__(self, p, state):
563 self.at_others_flag = False # True: @others has been generated for this target.
564 self.kind = 'None' # in ('None', 'class', 'def')
565 self.p = p
566 self.state = state
568 def __repr__(self):
569 return 'CythonTarget: %s kind: %s @others: %s p: %s' % (
570 self.state,
571 self.kind,
572 int(self.at_others_flag),
573 g.shortFileName(self.p.h),
574 )
575#@-others
576importer_dict = {
577 'func': Cython_Importer.do_import(),
578 'extensions': ['.pyx',],
579}
580#@@language python
581#@@tabwidth -4
582#@-leo