Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1#@+leo-ver=5-thin 

2#@+node:ekr.20200619141135.1: * @file ../plugins/importers/cython.py 

3"""@auto importer for cython.""" 

4import re 

5from typing import Any, Dict, List 

6from leo.core import leoGlobals as g 

7from leo.plugins.importers import linescanner 

8Importer = linescanner.Importer 

9Target = linescanner.Target 

10#@+others 

11#@+node:ekr.20200619141201.2: ** class Cython_Importer(Importer) 

12class Cython_Importer(Importer): 

13 """A class to store and update scanning state.""" 

14 

15 starts_pattern = re.compile(r'\s*(class|def|cdef|cpdef)\s+') 

16 # Matches lines that apparently start a class or def. 

17 class_pat = re.compile(r'\s*class\s+(\w+)\s*(\([\w.]+\))?') 

18 def_pat = re.compile(r'\s*(cdef|cpdef|def)\s+(\w+)') 

19 trace = False 

20 #@+others 

21 #@+node:ekr.20200619144343.1: *3* cy_i.ctor 

22 def __init__(self, importCommands, **kwargs): 

23 """Cython_Importer.ctor.""" 

24 super().__init__( 

25 importCommands, 

26 language='cython', 

27 state_class=Cython_ScanState, 

28 strict=True, 

29 ) 

30 self.put_decorators = self.c.config.getBool('put-cython-decorators-in-imported-headlines') 

31 #@+node:ekr.20200619141201.3: *3* cy_i.clean_headline 

32 def clean_headline(self, s, p=None): 

33 """Return a cleaned up headline s.""" 

34 if p: # Called from clean_all_headlines: 

35 return self.get_decorator(p) + p.h 

36 # Handle def, cdef, cpdef. 

37 m = re.match(r'\s*(cpdef|cdef|def)\s+(\w+)', s) 

38 if m: 

39 return m.group(2) 

40 # Handle classes. 

41 m = re.match(r'\s*class\s+(\w+)\s*(\([\w.]+\))?', s) 

42 if m: 

43 return 'class %s%s' % (m.group(1), m.group(2) or '') 

44 return s.strip() 

45 

46 #@+node:vitalije.20211207173723.1: *3* check 

47 def check(self, unused_s, parent): 

48 """ 

49 Cython_Importer.check: override Importer.check. 

50  

51 Return True if perfect import checks pass, making additional allowances 

52 for underindented comment lines. 

53  

54 Raise AssertionError if the checks fail while unit testing. 

55 """ 

56 if g.app.suppressImportChecks: 

57 g.app.suppressImportChecks = False 

58 return True 

59 s1 = g.toUnicode(self.file_s, self.encoding) 

60 s2 = self.trial_write() 

61 # Regularize the lines first. 

62 lines1 = g.splitLines(s1.rstrip() + '\n') 

63 lines2 = g.splitLines(s2.rstrip() + '\n') 

64 # #2327: Ignore blank lines and lws in comment lines. 

65 test_lines1 = self.strip_blank_and_comment_lines(lines1) 

66 test_lines2 = self.strip_blank_and_comment_lines(lines2) 

67 # #2327: Report all remaining mismatches. 

68 ok = test_lines1 == test_lines2 

69 if not ok: 

70 self.show_failure(lines1, lines2, g.shortFileName(self.root.h)) 

71 return ok 

72 #@+node:vitalije.20211207173805.1: *3* strip_blank_and_comment_lines 

73 def strip_blank_and_comment_lines(self, lines): 

74 """Strip all blank lines and strip lws from comment lines.""" 

75 

76 def strip(s): 

77 return s.strip() if s.isspace() else s.lstrip() if s.strip().startswith('#') else s 

78 

79 return [strip(z) for z in lines] 

80 #@+node:vitalije.20211207173901.1: *3* get_decorator 

81 decorator_pat = re.compile(r'\s*@\s*([\w\.]+)') 

82 

83 def get_decorator(self, p): 

84 if g.unitTesting or self.put_decorators: 

85 for s in self.get_lines(p): 

86 if not s.isspace(): 

87 m = self.decorator_pat.match(s) 

88 if m: 

89 s = s.strip() 

90 if s.endswith('('): 

91 s = s[:-1].strip() 

92 return s + ' ' 

93 return '' 

94 return '' 

95 #@+node:vitalije.20211207173935.1: *3* find_class 

96 def find_class(self, parent): 

97 """ 

98 Find the start and end of a class/def in a node. 

99 Return (kind, i, j), where kind in (None, 'class', 'def') 

100 """ 

101 # Called from Leo's core to implement two minor commands. 

102 prev_state = Cython_ScanState() 

103 target = Target(parent, prev_state) 

104 stack = [target] 

105 lines = g.splitlines(parent.b) 

106 index = 0 

107 for i, line in enumerate(lines): 

108 new_state = self.scan_line(line, prev_state) 

109 if self.prev_state.context or self.ws_pattern.match(line): # type:ignore 

110 pass 

111 else: 

112 m = self.class_or_def_pattern.match(line) 

113 if m: 

114 return self.skip_block(i, index, lines, new_state, stack) 

115 prev_state = new_state 

116 return None, -1, -1 

117 #@+node:vitalije.20211207174005.1: *3* skip_block 

118 def skip_block(self, i, index, lines, prev_state, stack): 

119 """ 

120 Find the end of a class/def starting at index 

121 on line i of lines. 

122 Return (kind, i, j), where kind in (None, 'class', 'def'). 

123 """ 

124 index1 = index 

125 line = lines[i] 

126 kind = 'class' if line.strip().startswith('class') else 'def' 

127 top = stack[-1] ### new 

128 i += 1 

129 while i < len(lines): 

130 line = lines[i] 

131 index += len(line) 

132 new_state = self.scan_line(line, prev_state) 

133 ### if self.ends_block(line, new_state, prev_state, stack): 

134 if new_state.indent < top.state.indent: 

135 return kind, index1, index 

136 prev_state = new_state 

137 i += 1 

138 return None, -1, -1 

139 #@+node:vitalije.20211207174043.1: *3* gen_lines 

140 class_or_def_pattern = re.compile(r'\s*(class|cdef|cpdef|def)\s+') 

141 

142 def gen_lines(self, lines, parent): 

143 """ 

144 Non-recursively parse all lines of s into parent, creating descendant 

145 nodes as needed. 

146 """ 

147 assert self.root == parent, (self.root, parent) 

148 # Init the state. 

149 self.new_state = Cython_ScanState() 

150 assert self.new_state.indent == 0 

151 self.vnode_info = { 

152 # Keys are vnodes, values are inner dicts. 

153 parent.v: { 

154 '@others': True, 

155 'indent': 0, # None denotes a to-be-defined value. 

156 'kind': 'outer', 

157 'lines': ['@others\n'], # The post pass adds @language and @tabwidth directives. 

158 } 

159 } 

160 if g.unitTesting: 

161 g.vnode_info = self.vnode_info # A hack. 

162 # Create a Declarations node. 

163 p = self.start_python_block('org', 'Declarations', parent) 

164 # 

165 # The main importer loop. Don't worry about the speed of this loop. 

166 for line in lines: 

167 # Update the state, remembering the previous state. 

168 self.prev_state = self.new_state 

169 self.new_state = self.scan_line(line, self.prev_state) 

170 # Handle the line. 

171 if self.prev_state.context: 

172 # A line with a string or docstring. 

173 self.add_line(p, line, tag='string') 

174 elif self.ws_pattern.match(line): 

175 # A blank or comment line. 

176 self.add_line(p, line, tag='whitespace') 

177 else: 

178 # The leading whitespace of all other lines are significant. 

179 m = self.class_or_def_pattern.match(line) 

180 kind = m.group(1) if m else 'normal' 

181 if kind in ('cdef', 'cpdef'): 

182 kind = 'def' 

183 p = self.end_previous_blocks(kind, line, p) 

184 if m: 

185 assert kind in ('class', 'def'), repr(kind) 

186 if kind == 'class': 

187 p = self.do_class(line, p) 

188 else: 

189 p = self.do_def(line, p) 

190 else: 

191 p = self.do_normal_line(line,p) 

192 #@+node:vitalije.20211207174130.1: *3* do_class 

193 def do_class(self, line, parent): 

194 

195 d = self.vnode_info [parent.v] 

196 parent_kind = d ['kind'] 

197 if parent_kind in ('outer', 'org', 'class'): 

198 # Create a new parent. 

199 self.gen_python_ref(line, parent) 

200 p = self.start_python_block('class', line, parent) 

201 else: 

202 # Don't change parent. 

203 p = parent 

204 self.add_line(p, line, tag='class') 

205 return p 

206 #@+node:vitalije.20211207174137.1: *3* do_def 

207 def do_def(self, line, parent): 

208 

209 new_indent = self.new_state.indent 

210 d = self.vnode_info 

211 parent_indent = d [parent.v] ['indent'] 

212 parent_kind = d [parent.v] ['kind'] 

213 if parent_kind in ('outer', 'class'): 

214 # Create a new parent. 

215 self.gen_python_ref(line, parent) 

216 p = self.start_python_block('def', line, parent) 

217 self.add_line(p, line, tag='def') 

218 return p 

219 # For 'org' parents, look at the grand parent kind. 

220 if parent_kind == 'org': 

221 grand_kind = d [parent.parent().v] ['kind'] 

222 if grand_kind == 'class' and new_indent <= parent_indent: 

223 self.gen_python_ref(line, parent) 

224 p = parent.parent() 

225 p = self.start_python_block('def', line, p) 

226 self.add_line(p, line, tag='def') 

227 return p 

228 # The default: don't change parent. 

229 self.add_line(parent, line, tag='def') 

230 return parent 

231 #@+node:vitalije.20211207174148.1: *3* do_normal_line 

232 def do_normal_line(self, line, p): 

233 

234 new_indent = self.new_state.indent 

235 d = self.vnode_info [p.v] 

236 parent_indent = d ['indent'] 

237 parent_kind = d ['kind'] 

238 if parent_kind == 'outer': 

239 # Create an organizer node, regardless of indentation. 

240 p = self.start_python_block('org', line, p) 

241 elif parent_kind == 'class' and new_indent < parent_indent: 

242 # Create an organizer node. 

243 self.gen_python_ref(line, p) 

244 p = self.start_python_block('org', line, p) 

245 self.add_line(p, line, tag='normal') 

246 return p 

247 #@+node:vitalije.20211207174159.1: *3* end_previous_blocks 

248 def end_previous_blocks(self, kind, line, p): 

249 """ 

250 End blocks that are incompatible with the new line. 

251 - kind: The kind of the incoming line: 'class', 'def' or 'normal'. 

252 - new_indent: The indentation of the incoming line. 

253  

254 Return p, a parent that will either contain the new line or will be the 

255 parent of a new child of parent. 

256 """ 

257 new_indent = self.new_state.indent 

258 while p: 

259 d = self.vnode_info [p.v] 

260 parent_indent, parent_kind = d ['indent'], d ['kind'] 

261 if parent_kind == 'outer': 

262 return p 

263 if new_indent > parent_indent: 

264 return p 

265 if new_indent < parent_indent: 

266 p = p.parent() 

267 continue 

268 assert new_indent == parent_indent, (new_indent, parent_indent) 

269 if kind == 'normal': 

270 # Don't change parent, whatever it is. 

271 return p 

272 if new_indent == 0: 

273 # Continue until we get to the outer level. 

274 if parent_kind == 'outer': 

275 return p 

276 p = p.parent() 

277 continue 

278 # The context-dependent cases... 

279 assert new_indent > 0 and new_indent == parent_indent, (new_indent, parent_indent) 

280 assert kind in ('class', 'def') 

281 if kind == 'class': 

282 # Allow nested classes. 

283 return p.parent() if new_indent < parent_indent else p 

284 assert kind == 'def', repr(kind) 

285 if parent_kind in ('class', 'outer'): 

286 return p 

287 d2 = self.vnode_info [p.parent().v] 

288 grand_kind = d2 ['kind'] 

289 if parent_kind == 'def' and grand_kind in ('class', 'outer'): 

290 return p.parent() 

291 return p 

292 assert False, 'No parent' 

293 #@+node:vitalije.20211207174206.1: *3* gen_python_ref 

294 def gen_python_ref(self, line, p): 

295 """Generate the at-others directive and set p's at-others flag""" 

296 d = self.vnode_info [p.v] 

297 if d ['@others']: 

298 return 

299 d ['@others'] = True 

300 indent_ws = self.get_str_lws(line) 

301 ref_line = f"{indent_ws}@others\n" 

302 self.add_line(p, ref_line, tag='@others') 

303 #@+node:vitalije.20211207174213.1: *3* start_python_block 

304 def start_python_block(self, kind, line, parent): 

305 """ 

306 Create, p as the last child of parent and initialize the p.v._import_* ivars. 

307  

308 Return p. 

309 """ 

310 assert kind in ('org', 'class', 'def'), g.callers() 

311 # Create a new node p. 

312 p = parent.insertAsLastChild() 

313 v = p.v 

314 # Set p.h. 

315 p.h = self.clean_headline(line, p=None).strip() 

316 if kind == 'org': 

317 p.h = f"Organizer: {p.h}" 

318 # 

319 # Compute the indentation at p. 

320 parent_info = self.vnode_info.get(parent.v) 

321 assert parent_info, (parent.h, g.callers()) 

322 parent_indent = parent_info.get('indent') 

323 ### Dubious: prevents proper handling of strangely-indented code. 

324 indent = parent_indent + 4 if kind == 'class' else parent_indent 

325 # Update vnode_info for p.v 

326 assert not v in self.vnode_info, (p.h, g.callers()) 

327 self.vnode_info [v] = { 

328 '@others': False, 

329 'indent': indent, 

330 'kind': kind, 

331 'lines': [], 

332 } 

333 return p 

334 #@+node:vitalije.20211207174318.1: *3* adjust_all_decorator_lines 

335 def adjust_all_decorator_lines(self, parent): 

336 """Move decorator lines (only) to the next sibling node.""" 

337 g.trace(parent.h) 

338 for p in parent.self_and_subtree(): 

339 for child in p.children(): 

340 if child.hasNext(): 

341 self.adjust_decorator_lines(child) 

342 #@+node:vitalije.20211207174323.1: *4* adjust_decorator_lines 

343 def adjust_decorator_lines(self, p): 

344 """Move decorator lines from the end of p.b to the start of p.next().b.""" 

345 ### To do 

346 #@+node:vitalije.20211207174343.1: *3* promote_first_child 

347 def promote_first_child(self, parent): 

348 """Move a smallish first child to the start of parent.""" 

349 #@+node:vitalije.20211207174354.1: *3* create_child_node 

350 def create_child_node(self, parent, line, headline): 

351 """Create a child node of parent.""" 

352 assert False, g.callers() 

353 #@+node:vitalije.20211207174358.1: *3* cut_stack 

354 def cut_stack(self, new_state, stack): 

355 """Cut back the stack until stack[-1] matches new_state.""" 

356 assert False, g.callers() 

357 #@+node:vitalije.20211207174403.1: *3* trace_status 

358 def trace_status(self, line, new_state, prev_state, stack, top): 

359 """Do-nothing override of Import.trace_status.""" 

360 assert False, g.callers() 

361 #@+node:vitalije.20211207174409.1: *3* add_line 

362 heading_printed = False 

363 

364 def add_line(self, p, s, tag=None): 

365 """Append the line s to p.v._import_lines.""" 

366 assert s and isinstance(s, str), (repr(s), g.callers()) 

367 if self.trace: 

368 h = p.h 

369 if h.startswith('@'): 

370 h_parts = p.h.split('.') 

371 h = h_parts[-1] 

372 if not self.heading_printed: 

373 self.heading_printed = True 

374 g.trace(f"{'tag or caller ':>20} {' '*8+'top node':30} line") 

375 g.trace(f"{'-' * 13 + ' ':>20} {' '*8+'-' * 8:30} {'-' * 4}") 

376 if tag: 

377 kind = self.vnode_info [p.v] ['kind'] 

378 tag = f"{kind:>5}:{tag:<10}" 

379 g.trace(f"{(tag or g.caller()):>20} {h[:30]!r:30} {s!r}") 

380 self.vnode_info [p.v] ['lines'].append(s) 

381 #@+node:vitalije.20211207174416.1: *3* common_lws 

382 def common_lws(self, lines): 

383 """ 

384 Override Importer.common_lws. 

385  

386 Return the lws (a string) common to all lines. 

387  

388 We must unindent the class/def line fully. 

389 It would be wrong to examine the indentation of other lines. 

390 """ 

391 return self.get_str_lws(lines[0]) if lines else '' 

392 #@+node:vitalije.20211207174422.1: *3* clean_all_headlines 

393 def clean_all_headlines(self, parent): 

394 """ 

395 Clean all headlines in parent's tree by calling the language-specific 

396 clean_headline method. 

397 """ 

398 for p in parent.subtree(): 

399 # Important: i.gen_ref does not know p when it calls 

400 # self.clean_headline. 

401 h = self.clean_headline(p.h, p=p) 

402 if h and h != p.h: 

403 p.h = h 

404 #@+node:vitalije.20211207174429.1: *3* find_tail 

405 def find_tail(self, p): 

406 """ 

407 Find the tail (trailing unindented) lines. 

408 return head, tail 

409 """ 

410 lines = self.get_lines(p) [:] 

411 tail = [] 

412 # First, find all potentially tail lines, including blank lines. 

413 while lines: 

414 line = lines.pop() 

415 if line.lstrip() == line or not line.strip(): 

416 tail.append(line) 

417 else: 

418 break 

419 # Next, remove leading blank lines from the tail. 

420 while tail: 

421 line = tail[-1] 

422 if line.strip(): 

423 break 

424 else: 

425 tail.pop(0) 

426 if 0: 

427 g.printObj(lines, tag=f"lines: find_tail: {p.h}") 

428 g.printObj(tail, tag=f"tail: find_tail: {p.h}") 

429 #@+node:vitalije.20211207174436.1: *3* promote_last_lines 

430 def promote_last_lines(self, parent): 

431 """A do-nothing override.""" 

432 #@+node:vitalije.20211207174441.1: *3* promote_trailing_underindented_lines 

433 def promote_trailing_underindented_lines(self, parent): 

434 """A do-nothing override.""" 

435 #@+node:vitalije.20211207174501.1: *3* get_new_dict 

436 #@@nobeautify 

437 

438 def get_new_dict(self, context): 

439 """ 

440 Return a *general* state dictionary for the given context. 

441 Subclasses may override... 

442 """ 

443 comment, block1, block2 = self.single_comment, self.block1, self.block2 

444 

445 def add_key(d, key, data): 

446 aList = d.get(key,[]) 

447 aList.append(data) 

448 d[key] = aList 

449 

450 d: Dict[str, List[Any]] 

451 

452 if context: 

453 d = { 

454 # key kind pattern ends? 

455 '\\': [('len+1', '\\',None),], 

456 '"':[ 

457 ('len', '"""', context == '"""'), 

458 ('len', '"', context == '"'), 

459 ], 

460 "'":[ 

461 ('len', "'''", context == "'''"), 

462 ('len', "'", context == "'"), 

463 ], 

464 } 

465 if block1 and block2: 

466 add_key(d, block2[0], ('len', block1, True)) 

467 else: 

468 # Not in any context. 

469 d = { 

470 # key kind pattern new-ctx deltas 

471 '\\': [('len+1','\\', context, None),], 

472 '#': [('all', '#', context, None),], 

473 '"':[ 

474 # order matters. 

475 ('len', '"""', '"""', None), 

476 ('len', '"', '"', None), 

477 ], 

478 "'":[ 

479 # order matters. 

480 ('len', "'''", "'''", None), 

481 ('len', "'", "'", None), 

482 ], 

483 '{': [('len', '{', context, (1,0,0)),], 

484 '}': [('len', '}', context, (-1,0,0)),], 

485 '(': [('len', '(', context, (0,1,0)),], 

486 ')': [('len', ')', context, (0,-1,0)),], 

487 '[': [('len', '[', context, (0,0,1)),], 

488 ']': [('len', ']', context, (0,0,-1)),], 

489 } 

490 if comment: 

491 add_key(d, comment[0], ('all', comment, '', None)) 

492 if block1 and block2: 

493 add_key(d, block1[0], ('len', block1, block1, None)) 

494 return d 

495 #@-others 

496#@+node:vitalije.20211207174609.1: ** class Cython_State 

497class Cython_ScanState: 

498 """A class representing the state of the python line-oriented scan.""" 

499 

500 def __init__(self, d=None): 

501 """Cython_ScanState ctor.""" 

502 if d: 

503 indent = d.get('indent') 

504 prev = d.get('prev') 

505 self.indent = prev.indent if prev.bs_nl else indent 

506 self.context = prev.context 

507 self.curlies = prev.curlies 

508 self.parens = prev.parens 

509 self.squares = prev.squares 

510 else: 

511 self.bs_nl = False 

512 self.context = '' 

513 self.curlies = self.parens = self.squares = 0 

514 self.indent = 0 

515 

516 def __repr__(self): 

517 """Py_State.__repr__""" 

518 return self.short_description() 

519 

520 __str__ = __repr__ 

521 

522 def short_description(self): # pylint: disable=no-else-return 

523 bsnl = 'bs-nl' if self.bs_nl else '' 

524 context = f"{self.context} " if self.context else '' 

525 indent = self.indent 

526 curlies = f"{{{self.curlies}}}" if self.curlies else '' 

527 parens = f"({self.parens})" if self.parens else '' 

528 squares = f"[{self.squares}]" if self.squares else '' 

529 return f"{context}indent:{indent}{curlies}{parens}{squares}{bsnl}" 

530 def level(self): 

531 """Python_ScanState.level.""" 

532 return self.indent 

533 def in_context(self): 

534 """True if in a special context.""" 

535 return ( 

536 self.context or 

537 self.curlies > 0 or 

538 self.parens > 0 or 

539 self.squares > 0 or 

540 self.bs_nl 

541 ) 

542 def update(self, data): 

543 """ 

544 Update the state using the 6-tuple returned by i.scan_line. 

545 Return i = data[1] 

546 """ 

547 context, i, delta_c, delta_p, delta_s, bs_nl = data 

548 self.bs_nl = bs_nl 

549 self.context = context 

550 self.curlies += delta_c 

551 self.parens += delta_p 

552 self.squares += delta_s 

553 return i 

554#@+node:ekr.20211121065103.1: ** class CythonTarget 

555class CythonTarget: 

556 """ 

557 A class describing a target node p. 

558 state is used to cut back the stack. 

559 """ 

560 # Same as the legacy PythonTarget class, except for the class name. 

561 

562 def __init__(self, p, state): 

563 self.at_others_flag = False # True: @others has been generated for this target. 

564 self.kind = 'None' # in ('None', 'class', 'def') 

565 self.p = p 

566 self.state = state 

567 

568 def __repr__(self): 

569 return 'CythonTarget: %s kind: %s @others: %s p: %s' % ( 

570 self.state, 

571 self.kind, 

572 int(self.at_others_flag), 

573 g.shortFileName(self.p.h), 

574 ) 

575#@-others 

576importer_dict = { 

577 'func': Cython_Importer.do_import(), 

578 'extensions': ['.pyx',], 

579} 

580#@@language python 

581#@@tabwidth -4 

582#@-leo