Coverage for C:\leo.repo\leo-editor\leo\core\leoImport.py : 13%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2#@+leo-ver=5-thin
3#@+node:ekr.20031218072017.3206: * @file leoImport.py
4#@@first
5#@+<< imports >>
6#@+node:ekr.20091224155043.6539: ** << imports >> (leoImport)
7import csv
8import io
9import json
10import os
11import re
12import textwrap
13import time
14from typing import Any, List
15import urllib
16#
17# Third-party imports.
18try:
19 import docutils
20 import docutils.core
21 assert docutils
22 assert docutils.core
23except ImportError:
24 # print('leoImport.py: can not import docutils')
25 docutils = None # type:ignore
26try:
27 import lxml
28except ImportError:
29 lxml = None
30#
31# Leo imports...
32from leo.core import leoGlobals as g
33from leo.core import leoNodes
34#
35# Abbreviation.
36StringIO = io.StringIO
37#@-<< imports >>
38#@+others
39#@+node:ekr.20160503145550.1: ** class FreeMindImporter
40class FreeMindImporter:
41 """Importer class for FreeMind (.mmap) files."""
43 def __init__(self, c):
44 """ctor for FreeMind Importer class."""
45 self.c = c
46 self.count = 0
47 self.d = {}
48 #@+others
49 #@+node:ekr.20170222084048.1: *3* freemind.add_children
50 def add_children(self, parent, element):
51 """
52 parent is the parent position, element is the parent element.
53 Recursively add all the child elements as descendants of parent_p.
54 """
55 p = parent.insertAsLastChild()
56 attrib_text = element.attrib.get('text', '').strip()
57 tag = element.tag if isinstance(element.tag, str) else ''
58 text = element.text or ''
59 if not tag:
60 text = text.strip()
61 p.h = attrib_text or tag or 'Comment'
62 p.b = text if text.strip() else ''
63 for child in element:
64 self.add_children(p, child)
65 #@+node:ekr.20160503125844.1: *3* freemind.create_outline
66 def create_outline(self, path):
67 """Create a tree of nodes from a FreeMind file."""
68 c = self.c
69 junk, fileName = g.os_path_split(path)
70 undoData = c.undoer.beforeInsertNode(c.p)
71 try:
72 self.import_file(path)
73 c.undoer.afterInsertNode(c.p, 'Import', undoData)
74 except Exception:
75 g.es_print('Exception importing FreeMind file', g.shortFileName(path))
76 g.es_exception()
77 return c.p
78 #@+node:ekr.20160503191518.4: *3* freemind.import_file
79 def import_file(self, path):
80 """The main line of the FreeMindImporter class."""
81 c = self.c
82 sfn = g.shortFileName(path)
83 if g.os_path_exists(path):
84 htmltree = lxml.html.parse(path)
85 root = htmltree.getroot()
86 body = root.findall('body')[0]
87 if body is None:
88 g.error(f"no body in: {sfn}")
89 else:
90 root_p = c.lastTopLevel().insertAfter()
91 root_p.h = g.shortFileName(path)
92 for child in body:
93 if child != body:
94 self.add_children(root_p, child)
95 c.selectPosition(root_p)
96 c.redraw()
97 else:
98 g.error(f"file not found: {sfn}")
99 #@+node:ekr.20160503145113.1: *3* freemind.import_files
100 def import_files(self, files):
101 """Import a list of FreeMind (.mmap) files."""
102 c = self.c
103 if files:
104 self.tab_width = c.getTabWidth(c.p)
105 for fileName in files:
106 g.setGlobalOpenDir(fileName)
107 p = self.create_outline(fileName)
108 p.contract()
109 p.setDirty()
110 c.setChanged()
111 c.redraw(p)
112 #@+node:ekr.20160504043823.1: *3* freemind.prompt_for_files
113 def prompt_for_files(self):
114 """Prompt for a list of FreeMind (.mm.html) files and import them."""
115 if not lxml:
116 g.trace("FreeMind importer requires lxml")
117 return
118 c = self.c
119 types = [
120 ("FreeMind files", "*.mm.html"),
121 ("All files", "*"),
122 ]
123 names = g.app.gui.runOpenFileDialog(c,
124 title="Import FreeMind File",
125 filetypes=types,
126 defaultextension=".html",
127 multiple=True)
128 c.bringToFront()
129 if names:
130 g.chdir(names[0])
131 self.import_files(names)
132 #@-others
133#@+node:ekr.20160504144241.1: ** class JSON_Import_Helper
134class JSON_Import_Helper:
135 """
136 A class that helps client scripts import .json files.
138 Client scripts supply data describing how to create Leo outlines from
139 the .json data.
140 """
142 def __init__(self, c):
143 """ctor for the JSON_Import_Helper class."""
144 self.c = c
145 self.vnodes_dict = {}
146 #@+others
147 #@+node:ekr.20160504144353.1: *3* json.create_nodes (generalize)
148 def create_nodes(self, parent, parent_d):
149 """Create the tree of nodes rooted in parent."""
150 d = self.gnx_dict
151 for child_gnx in parent_d.get('children'):
152 d2 = d.get(child_gnx)
153 if child_gnx in self.vnodes_dict:
154 # It's a clone.
155 v = self.vnodes_dict.get(child_gnx)
156 n = parent.numberOfChildren()
157 child = leoNodes.Position(v)
158 child._linkAsNthChild(parent, n)
159 # Don't create children again.
160 else:
161 child = parent.insertAsLastChild()
162 child.h = d2.get('h') or '<**no h**>'
163 child.b = d2.get('b') or ''
164 if d2.get('gnx'):
165 child.v.fileIndex = gnx = d2.get('gnx') # 2021/06/23: found by mypy.
166 self.vnodes_dict[gnx] = child.v
167 if d2.get('ua'):
168 child.u = d2.get('ua')
169 self.create_nodes(child, d2)
170 #@+node:ekr.20160504144241.2: *3* json.create_outline (generalize)
171 def create_outline(self, path):
172 c = self.c
173 junk, fileName = g.os_path_split(path)
174 undoData = c.undoer.beforeInsertNode(c.p)
175 # Create the top-level headline.
176 p = c.lastTopLevel().insertAfter()
177 fn = g.shortFileName(path).strip()
178 if fn.endswith('.json'):
179 fn = fn[:-5]
180 p.h = fn
181 self.scan(path, p)
182 c.undoer.afterInsertNode(p, 'Import', undoData)
183 return p
184 #@+node:ekr.20160504144314.1: *3* json.scan (generalize)
185 def scan(self, s, parent):
186 """Create an outline from a MindMap (.csv) file."""
187 c, d, self.gnx_dict = self.c, json.loads(s), {}
188 for d2 in d.get('nodes', []):
189 gnx = d2.get('gnx')
190 self.gnx_dict[gnx] = d2
191 top_d = d.get('top')
192 if top_d:
193 # Don't set parent.h or parent.gnx or parent.v.u.
194 parent.b = top_d.get('b') or ''
195 self.create_nodes(parent, top_d)
196 c.redraw()
197 return bool(top_d)
198 #@-others
199#@+node:ekr.20071127175948: ** class LeoImportCommands
200class LeoImportCommands:
201 """
202 A class implementing all of Leo's import/export code. This class
203 uses **importers** in the leo/plugins/importers folder.
205 For more information, see leo/plugins/importers/howto.txt.
206 """
207 #@+others
208 #@+node:ekr.20031218072017.3207: *3* ic.__init__& ic.reload_settings
209 def __init__(self, c):
210 """ctor for LeoImportCommands class."""
211 self.c = c
212 self.encoding = 'utf-8'
213 self.errors = 0
214 self.fileName = None # The original file name, say x.cpp
215 self.fileType = None # ".py", ".c", etc.
216 self.methodName = None # x, as in < < x methods > > =
217 self.output_newline = g.getOutputNewline(c=c) # Value of @bool output_newline
218 self.tab_width = c.tab_width
219 self.treeType = "@file" # None or "@file"
220 self.verbose = True # Leo 6.6
221 self.webType = "@noweb" # "cweb" or "noweb"
222 self.web_st = [] # noweb symbol table.
223 self.reload_settings()
225 def reload_settings(self):
226 pass
228 reloadSettings = reload_settings
229 #@+node:ekr.20031218072017.3289: *3* ic.Export
230 #@+node:ekr.20031218072017.3290: *4* ic.convertCodePartToWeb & helpers
231 def convertCodePartToWeb(self, s, i, p, result):
232 """
233 # Headlines not containing a section reference are ignored in noweb
234 and generate index index in cweb.
235 """
236 ic = self
237 nl = ic.output_newline
238 head_ref = ic.getHeadRef(p)
239 file_name = ic.getFileName(p)
240 if g.match_word(s, i, "@root"):
241 i = g.skip_line(s, i)
242 ic.appendRefToFileName(file_name, result)
243 elif g.match_word(s, i, "@c") or g.match_word(s, i, "@code"):
244 i = g.skip_line(s, i)
245 ic.appendHeadRef(p, file_name, head_ref, result)
246 elif g.match_word(p.h, 0, "@file"):
247 # Only do this if nothing else matches.
248 ic.appendRefToFileName(file_name, result)
249 i = g.skip_line(s, i) # 4/28/02
250 else:
251 ic.appendHeadRef(p, file_name, head_ref, result)
252 i, result = ic.copyPart(s, i, result)
253 return i, result.strip() + nl
254 #@+at %defs a b c
255 #@+node:ekr.20140630085837.16720: *5* ic.appendHeadRef
256 def appendHeadRef(self, p, file_name, head_ref, result):
257 ic = self
258 nl = ic.output_newline
259 if ic.webType == "cweb":
260 if head_ref:
261 escaped_head_ref = head_ref.replace("@", "@@")
262 result += "@<" + escaped_head_ref + "@>=" + nl
263 else:
264 result += "@^" + p.h.strip() + "@>" + nl
265 # Convert the headline to an index entry.
266 result += "@c" + nl
267 # @c denotes a new section.
268 else:
269 if head_ref:
270 pass
271 elif p == ic.c.p:
272 head_ref = file_name or "*"
273 else:
274 head_ref = "@others"
275 # 2019/09/12
276 result += (g.angleBrackets(head_ref) + "=" + nl)
277 #@+node:ekr.20140630085837.16719: *5* ic.appendRefToFileName
278 def appendRefToFileName(self, file_name, result):
279 ic = self
280 nl = ic.output_newline
281 if ic.webType == "cweb":
282 if not file_name:
283 result += "@<root@>=" + nl
284 else:
285 result += "@(" + file_name + "@>" + nl
286 # @(...@> denotes a file.
287 else:
288 if not file_name:
289 file_name = "*"
290 # 2019/09/12.
291 lt = "<<"
292 rt = ">>"
293 result += (lt + file_name + rt + "=" + nl)
294 #@+node:ekr.20140630085837.16721: *5* ic.getHeadRef
295 def getHeadRef(self, p):
296 """
297 Look for either noweb or cweb brackets.
298 Return everything between those brackets.
299 """
300 h = p.h.strip()
301 if g.match(h, 0, "<<"):
302 i = h.find(">>", 2)
303 elif g.match(h, 0, "<@"):
304 i = h.find("@>", 2)
305 else:
306 return h
307 return h[2:i].strip()
308 #@+node:ekr.20031218072017.3292: *5* ic.getFileName
309 def getFileName(self, p):
310 """Return the file name from an @file or @root node."""
311 h = p.h.strip()
312 if g.match(h, 0, "@file") or g.match(h, 0, "@root"):
313 line = h[5:].strip()
314 # set j & k so line[j:k] is the file name.
315 if g.match(line, 0, "<"):
316 j, k = 1, line.find(">", 1)
317 elif g.match(line, 0, '"'):
318 j, k = 1, line.find('"', 1)
319 else:
320 j, k = 0, line.find(" ", 0)
321 if k == -1:
322 k = len(line)
323 file_name = line[j:k].strip()
324 else:
325 file_name = ''
326 return file_name
327 #@+node:ekr.20031218072017.3296: *4* ic.convertDocPartToWeb (handle @ %def)
328 def convertDocPartToWeb(self, s, i, result):
329 nl = self.output_newline
330 if g.match_word(s, i, "@doc"):
331 i = g.skip_line(s, i)
332 elif g.match(s, i, "@ ") or g.match(s, i, "@\t") or g.match(s, i, "@*"):
333 i += 2
334 elif g.match(s, i, "@\n"):
335 i += 1
336 i = g.skip_ws_and_nl(s, i)
337 i, result2 = self.copyPart(s, i, "")
338 if result2:
339 # Break lines after periods.
340 result2 = result2.replace(". ", "." + nl)
341 result2 = result2.replace(". ", "." + nl)
342 result += nl + "@" + nl + result2.strip() + nl + nl
343 else:
344 # All nodes should start with '@', even if the doc part is empty.
345 result += nl + "@ " if self.webType == "cweb" else nl + "@" + nl
346 return i, result
347 #@+node:ekr.20031218072017.3297: *4* ic.convertVnodeToWeb
348 def convertVnodeToWeb(self, v):
349 """
350 This code converts a VNode to noweb text as follows:
352 Convert @doc to @
353 Convert @root or @code to < < name > >=, assuming the headline contains < < name > >
354 Ignore other directives
355 Format doc parts so they fit in pagewidth columns.
356 Output code parts as is.
357 """
358 c = self.c
359 if not v or not c:
360 return ""
361 startInCode = not c.config.at_root_bodies_start_in_doc_mode
362 nl = self.output_newline
363 docstart = nl + "@ " if self.webType == "cweb" else nl + "@" + nl
364 s = v.b
365 lb = "@<" if self.webType == "cweb" else "<<"
366 i, result, docSeen = 0, "", False
367 while i < len(s):
368 progress = i
369 i = g.skip_ws_and_nl(s, i)
370 if self.isDocStart(s, i) or g.match_word(s, i, "@doc"):
371 i, result = self.convertDocPartToWeb(s, i, result)
372 docSeen = True
373 elif (
374 g.match_word(s, i, "@code") or
375 g.match_word(s, i, "@root") or
376 g.match_word(s, i, "@c") or
377 g.match(s, i, lb)
378 ):
379 if not docSeen:
380 docSeen = True
381 result += docstart
382 i, result = self.convertCodePartToWeb(s, i, v, result)
383 elif self.treeType == "@file" or startInCode:
384 if not docSeen:
385 docSeen = True
386 result += docstart
387 i, result = self.convertCodePartToWeb(s, i, v, result)
388 else:
389 i, result = self.convertDocPartToWeb(s, i, result)
390 docSeen = True
391 assert progress < i
392 result = result.strip()
393 if result:
394 result += nl
395 return result
396 #@+node:ekr.20031218072017.3299: *4* ic.copyPart
397 # Copies characters to result until the end of the present section is seen.
399 def copyPart(self, s, i, result):
401 lb = "@<" if self.webType == "cweb" else "<<"
402 rb = "@>" if self.webType == "cweb" else ">>"
403 theType = self.webType
404 while i < len(s):
405 progress = j = i # We should be at the start of a line here.
406 i = g.skip_nl(s, i)
407 i = g.skip_ws(s, i)
408 if self.isDocStart(s, i):
409 return i, result
410 if (g.match_word(s, i, "@doc") or
411 g.match_word(s, i, "@c") or
412 g.match_word(s, i, "@root") or
413 g.match_word(s, i, "@code") # 2/25/03
414 ):
415 return i, result
416 # 2019/09/12
417 lt = "<<"
418 rt = ">>="
419 if g.match(s, i, lt) and g.find_on_line(s, i, rt) > -1:
420 return i, result
421 # Copy the entire line, escaping '@' and
422 # Converting @others to < < @ others > >
423 i = g.skip_line(s, j)
424 line = s[j:i]
425 if theType == "cweb":
426 line = line.replace("@", "@@")
427 else:
428 j = g.skip_ws(line, 0)
429 if g.match(line, j, "@others"):
430 line = line.replace("@others", lb + "@others" + rb)
431 elif g.match(line, 0, "@"):
432 # Special case: do not escape @ %defs.
433 k = g.skip_ws(line, 1)
434 if not g.match(line, k, "%defs"):
435 line = "@" + line
436 result += line
437 assert progress < i
438 return i, result.rstrip()
439 #@+node:ekr.20031218072017.1462: *4* ic.exportHeadlines
440 def exportHeadlines(self, fileName):
441 p = self.c.p
442 nl = self.output_newline
443 if not p:
444 return
445 self.setEncoding()
446 firstLevel = p.level()
447 try:
448 with open(fileName, 'w') as theFile:
449 for p in p.self_and_subtree(copy=False):
450 head = p.moreHead(firstLevel, useVerticalBar=True)
451 theFile.write(head + nl)
452 except IOError:
453 g.warning("can not open", fileName)
454 #@+node:ekr.20031218072017.1147: *4* ic.flattenOutline
455 def flattenOutline(self, fileName):
456 """
457 A helper for the flatten-outline command.
459 Export the selected outline to an external file.
460 The outline is represented in MORE format.
461 """
462 c = self.c
463 nl = self.output_newline
464 p = c.p
465 if not p:
466 return
467 self.setEncoding()
468 firstLevel = p.level()
469 try:
470 theFile = open(fileName, 'wb')
471 # Fix crasher: open in 'wb' mode.
472 except IOError:
473 g.warning("can not open", fileName)
474 return
475 for p in p.self_and_subtree(copy=False):
476 s = p.moreHead(firstLevel) + nl
477 s = g.toEncodedString(s, encoding=self.encoding, reportErrors=True)
478 theFile.write(s)
479 s = p.moreBody() + nl # Inserts escapes.
480 if s.strip():
481 s = g.toEncodedString(s, self.encoding, reportErrors=True)
482 theFile.write(s)
483 theFile.close()
484 #@+node:ekr.20031218072017.1148: *4* ic.outlineToWeb
485 def outlineToWeb(self, fileName, webType):
486 c = self.c
487 nl = self.output_newline
488 current = c.p
489 if not current:
490 return
491 self.setEncoding()
492 self.webType = webType
493 try:
494 theFile = open(fileName, 'w')
495 except IOError:
496 g.warning("can not open", fileName)
497 return
498 self.treeType = "@file"
499 # Set self.treeType to @root if p or an ancestor is an @root node.
500 for p in current.parents():
501 flag, junk = g.is_special(p.b, "@root")
502 if flag:
503 self.treeType = "@root"
504 break
505 for p in current.self_and_subtree(copy=False):
506 s = self.convertVnodeToWeb(p)
507 if s:
508 theFile.write(s)
509 if s[-1] != '\n':
510 theFile.write(nl)
511 theFile.close()
512 #@+node:ekr.20031218072017.3300: *4* ic.removeSentinelsCommand
513 def removeSentinelsCommand(self, paths, toString=False):
514 c = self.c
515 self.setEncoding()
516 for fileName in paths:
517 g.setGlobalOpenDir(fileName)
518 path, self.fileName = g.os_path_split(fileName)
519 s, e = g.readFileIntoString(fileName, self.encoding)
520 if s is None:
521 return None
522 if e:
523 self.encoding = e
524 #@+<< set delims from the header line >>
525 #@+node:ekr.20031218072017.3302: *5* << set delims from the header line >>
526 # Skip any non @+leo lines.
527 i = 0
528 while i < len(s) and g.find_on_line(s, i, "@+leo") == -1:
529 i = g.skip_line(s, i)
530 # Get the comment delims from the @+leo sentinel line.
531 at = self.c.atFileCommands
532 j = g.skip_line(s, i)
533 line = s[i:j]
534 valid, junk, start_delim, end_delim, junk = at.parseLeoSentinel(line)
535 if not valid:
536 if not toString:
537 g.es("invalid @+leo sentinel in", fileName)
538 return None
539 if end_delim:
540 line_delim = None
541 else:
542 line_delim, start_delim = start_delim, None
543 #@-<< set delims from the header line >>
544 s = self.removeSentinelLines(s, line_delim, start_delim, end_delim)
545 ext = c.config.remove_sentinels_extension
546 if not ext:
547 ext = ".txt"
548 if ext[0] == '.':
549 newFileName = g.os_path_finalize_join(path, fileName + ext) # 1341
550 else:
551 head, ext2 = g.os_path_splitext(fileName)
552 newFileName = g.os_path_finalize_join(path, head + ext + ext2) # 1341
553 if toString:
554 return s
555 #@+<< Write s into newFileName >>
556 #@+node:ekr.20031218072017.1149: *5* << Write s into newFileName >> (remove-sentinels)
557 # Remove sentinels command.
558 try:
559 with open(newFileName, 'w') as theFile:
560 theFile.write(s)
561 if not g.unitTesting:
562 g.es("created:", newFileName)
563 except Exception:
564 g.es("exception creating:", newFileName)
565 g.print_exception()
566 #@-<< Write s into newFileName >>
567 return None
568 #@+node:ekr.20031218072017.3303: *4* ic.removeSentinelLines
569 # This does not handle @nonl properly, but that no longer matters.
571 def removeSentinelLines(self, s, line_delim, start_delim, unused_end_delim):
572 """Properly remove all sentinle lines in s."""
573 delim = (line_delim or start_delim or '') + '@'
574 verbatim = delim + 'verbatim'
575 verbatimFlag = False
576 result = []
577 for line in g.splitLines(s):
578 i = g.skip_ws(line, 0)
579 if not verbatimFlag and g.match(line, i, delim):
580 if g.match(line, i, verbatim):
581 # Force the next line to be in the result.
582 verbatimFlag = True
583 else:
584 result.append(line)
585 verbatimFlag = False
586 return ''.join(result)
587 #@+node:ekr.20031218072017.1464: *4* ic.weave
588 def weave(self, filename):
589 p = self.c.p
590 nl = self.output_newline
591 if not p:
592 return
593 self.setEncoding()
594 try:
595 with open(filename, 'w', encoding=self.encoding) as f:
596 for p in p.self_and_subtree():
597 s = p.b
598 s2 = s.strip()
599 if s2:
600 f.write("-" * 60)
601 f.write(nl)
602 #@+<< write the context of p to f >>
603 #@+node:ekr.20031218072017.1465: *5* << write the context of p to f >> (weave)
604 # write the headlines of p, p's parent and p's grandparent.
605 context = []
606 p2 = p.copy()
607 i = 0
608 while i < 3:
609 i += 1
610 if not p2:
611 break
612 context.append(p2.h)
613 p2.moveToParent()
614 context.reverse()
615 indent = ""
616 for line in context:
617 f.write(indent)
618 indent += '\t'
619 f.write(line)
620 f.write(nl)
621 #@-<< write the context of p to f >>
622 f.write("-" * 60)
623 f.write(nl)
624 f.write(s.rstrip() + nl)
625 except Exception:
626 g.es("exception opening:", filename)
627 g.print_exception()
628 #@+node:ekr.20031218072017.3209: *3* ic.Import
629 #@+node:ekr.20031218072017.3210: *4* ic.createOutline & helpers
630 def createOutline(self, parent, ext=None, s=None):
631 """
632 Create an outline by importing a file, reading the file with the
633 given encoding if string s is None.
635 ext, The file extension to be used, or None.
636 fileName: A string or None. The name of the file to be read.
637 parent: The parent position of the created outline.
638 s: A string or None. The file's contents.
639 """
640 c = self.c
641 p = parent.copy()
642 self.treeType = '@file'
643 # Fix #352.
644 fileName = g.fullPath(c, parent)
645 if g.is_binary_external_file(fileName):
646 return self.import_binary_file(fileName, parent)
647 # Init ivars.
648 self.setEncoding(
649 p=parent,
650 default=c.config.default_at_auto_file_encoding,
651 )
652 ext, s = self.init_import(ext, fileName, s)
653 if s is None:
654 return None
655 # Get the so-called scanning func.
656 func = self.dispatch(ext, p)
657 # Func is a callback. It must have a c argument.
658 # Call the scanning function.
659 if g.unitTesting:
660 assert func or ext in ('.txt', '.w', '.xxx'), (repr(func), ext, p.h)
661 if func and not c.config.getBool('suppress-import-parsing', default=False):
662 s = g.toUnicode(s, encoding=self.encoding)
663 s = s.replace('\r', '')
664 # func is a factory that instantiates the importer class.
665 ok = func(c=c, parent=p, s=s)
666 else:
667 # Just copy the file to the parent node.
668 s = g.toUnicode(s, encoding=self.encoding)
669 s = s.replace('\r', '')
670 ok = self.scanUnknownFileType(s, p, ext)
671 if g.unitTesting:
672 return p if ok else None
673 # #488894: unsettling dialog when saving Leo file
674 # #889175: Remember the full fileName.
675 c.atFileCommands.rememberReadPath(fileName, p)
676 p.contract()
677 w = c.frame.body.wrapper
678 w.setInsertPoint(0)
679 w.seeInsertPoint()
680 return p
681 #@+node:ekr.20140724064952.18038: *5* ic.dispatch & helpers
682 def dispatch(self, ext, p):
683 """Return the correct scanner function for p, an @auto node."""
684 # Match the @auto type first, then the file extension.
685 c = self.c
686 return g.app.scanner_for_at_auto(c, p) or g.app.scanner_for_ext(c, ext)
687 #@+node:ekr.20170405191106.1: *5* ic.import_binary_file
688 def import_binary_file(self, fileName, parent):
690 # Fix bug 1185409 importing binary files puts binary content in body editor.
691 # Create an @url node.
692 c = self.c
693 if parent:
694 p = parent.insertAsLastChild()
695 else:
696 p = c.lastTopLevel().insertAfter()
697 p.h = f"@url file://{fileName}"
698 return p
699 #@+node:ekr.20140724175458.18052: *5* ic.init_import
700 def init_import(self, ext, fileName, s):
701 """
702 Init ivars imports and read the file into s.
703 Return ext, s.
704 """
705 junk, self.fileName = g.os_path_split(fileName)
706 self.methodName, self.fileType = g.os_path_splitext(self.fileName)
707 if not ext:
708 ext = self.fileType
709 ext = ext.lower()
710 if not s:
711 # Set the kind for error messages in readFileIntoString.
712 s, e = g.readFileIntoString(fileName, encoding=self.encoding)
713 if s is None:
714 return None, None
715 if e:
716 self.encoding = e
717 return ext, s
718 #@+node:ekr.20070713075352: *5* ic.scanUnknownFileType & helper
719 def scanUnknownFileType(self, s, p, ext):
720 """Scan the text of an unknown file type."""
721 body = ''
722 if ext in ('.html', '.htm'):
723 body += '@language html\n'
724 elif ext in ('.txt', '.text'):
725 body += '@nocolor\n'
726 else:
727 language = self.languageForExtension(ext)
728 if language:
729 body += f"@language {language}\n"
730 self.setBodyString(p, body + s)
731 for p in p.self_and_subtree():
732 p.clearDirty()
733 return True
734 #@+node:ekr.20080811174246.1: *6* ic.languageForExtension
735 def languageForExtension(self, ext):
736 """Return the language corresponding to the extension ext."""
737 unknown = 'unknown_language'
738 if ext.startswith('.'):
739 ext = ext[1:]
740 if ext:
741 z = g.app.extra_extension_dict.get(ext)
742 if z not in (None, 'none', 'None'):
743 language = z
744 else:
745 language = g.app.extension_dict.get(ext)
746 if language in (None, 'none', 'None'):
747 language = unknown
748 else:
749 language = unknown
750 # Return the language even if there is no colorizer mode for it.
751 return language
752 #@+node:ekr.20070806111212: *4* ic.readAtAutoNodes
753 def readAtAutoNodes(self):
754 c, p = self.c, self.c.p
755 after = p.nodeAfterTree()
756 found = False
757 while p and p != after:
758 if p.isAtAutoNode():
759 if p.isAtIgnoreNode():
760 g.warning('ignoring', p.h)
761 p.moveToThreadNext()
762 else:
763 c.atFileCommands.readOneAtAutoNode(p)
764 found = True
765 p.moveToNodeAfterTree()
766 else:
767 p.moveToThreadNext()
768 if not g.unitTesting:
769 message = 'finished' if found else 'no @auto nodes in the selected tree'
770 g.blue(message)
771 c.redraw()
772 #@+node:ekr.20031218072017.1810: *4* ic.importDerivedFiles
773 def importDerivedFiles(self, parent=None, paths=None, command='Import'):
774 """
775 Import one or more external files.
776 This is not a command. It must *not* have an event arg.
777 command is None when importing from the command line.
778 """
779 at, c, u = self.c.atFileCommands, self.c, self.c.undoer
780 current = c.p or c.rootPosition()
781 self.tab_width = c.getTabWidth(current)
782 if not paths:
783 return None
784 # Initial open from command line is not undoable.
785 if command:
786 u.beforeChangeGroup(current, command)
787 for fileName in paths:
788 fileName = fileName.replace('\\', '/') # 2011/10/09.
789 g.setGlobalOpenDir(fileName)
790 isThin = at.scanHeaderForThin(fileName)
791 if command:
792 undoData = u.beforeInsertNode(parent)
793 p = parent.insertAfter()
794 if isThin:
795 # Create @file node, not a deprecated @thin node.
796 p.initHeadString("@file " + fileName)
797 at.read(p)
798 else:
799 p.initHeadString("Imported @file " + fileName)
800 at.read(p)
801 p.contract()
802 p.setDirty() # 2011/10/09: tell why the file is dirty!
803 if command:
804 u.afterInsertNode(p, command, undoData)
805 current.expand()
806 c.setChanged()
807 if command:
808 u.afterChangeGroup(p, command)
809 c.redraw(current)
810 return p
811 #@+node:ekr.20031218072017.3212: *4* ic.importFilesCommand
812 def importFilesCommand(self,
813 files=None,
814 parent=None,
815 shortFn=False,
816 treeType=None,
817 verbose=True, # Legacy value.
818 ):
819 # Not a command. It must *not* have an event arg.
820 c, u = self.c, self.c.undoer
821 if not c or not c.p or not files:
822 return
823 self.tab_width = c.getTabWidth(c.p)
824 self.treeType = treeType or '@file'
825 self.verbose = verbose
826 if not parent:
827 g.trace('===== no parent', g.callers())
828 return
829 for fn in files or []:
830 # Report exceptions here, not in the caller.
831 try:
832 g.setGlobalOpenDir(fn)
833 # Leo 5.6: Handle undo here, not in createOutline.
834 undoData = u.beforeInsertNode(parent)
835 p = parent.insertAsLastChild()
836 p.h = f"{treeType} {fn}"
837 u.afterInsertNode(p, 'Import', undoData)
838 p = self.createOutline(parent=p)
839 if p: # createOutline may fail.
840 if self.verbose and not g.unitTesting:
841 g.blue("imported", g.shortFileName(fn) if shortFn else fn)
842 p.contract()
843 p.setDirty()
844 c.setChanged()
845 except Exception:
846 g.es_print('Exception importing', fn)
847 g.es_exception()
848 c.validateOutline()
849 parent.expand()
850 #@+node:ekr.20160503125237.1: *4* ic.importFreeMind
851 def importFreeMind(self, files):
852 """
853 Import a list of .mm.html files exported from FreeMind:
854 http://freemind.sourceforge.net/wiki/index.php/Main_Page
855 """
856 FreeMindImporter(self.c).import_files(files)
857 #@+node:ekr.20160503125219.1: *4* ic.importMindMap
858 def importMindMap(self, files):
859 """
860 Import a list of .csv files exported from MindJet:
861 https://www.mindjet.com/
862 """
863 MindMapImporter(self.c).import_files(files)
864 #@+node:ekr.20031218072017.3224: *4* ic.importWebCommand & helpers
865 def importWebCommand(self, files, webType):
866 c, current = self.c, self.c.p
867 if current is None:
868 return
869 if not files:
870 return
871 self.tab_width = c.getTabWidth(current) # New in 4.3.
872 self.webType = webType
873 for fileName in files:
874 g.setGlobalOpenDir(fileName)
875 p = self.createOutlineFromWeb(fileName, current)
876 p.contract()
877 p.setDirty()
878 c.setChanged()
879 c.redraw(current)
880 #@+node:ekr.20031218072017.3225: *5* createOutlineFromWeb
881 def createOutlineFromWeb(self, path, parent):
882 c = self.c
883 u = c.undoer
884 junk, fileName = g.os_path_split(path)
885 undoData = u.beforeInsertNode(parent)
886 # Create the top-level headline.
887 p = parent.insertAsLastChild()
888 p.initHeadString(fileName)
889 if self.webType == "cweb":
890 self.setBodyString(p, "@ignore\n@language cweb")
891 # Scan the file, creating one section for each function definition.
892 self.scanWebFile(path, p)
893 u.afterInsertNode(p, 'Import', undoData)
894 return p
895 #@+node:ekr.20031218072017.3227: *5* findFunctionDef
896 def findFunctionDef(self, s, i):
897 # Look at the next non-blank line for a function name.
898 i = g.skip_ws_and_nl(s, i)
899 k = g.skip_line(s, i)
900 name = None
901 while i < k:
902 if g.is_c_id(s[i]):
903 j = i
904 i = g.skip_c_id(s, i)
905 name = s[j:i]
906 elif s[i] == '(':
907 if name:
908 return name
909 break
910 else:
911 i += 1
912 return None
913 #@+node:ekr.20031218072017.3228: *5* scanBodyForHeadline
914 #@+at This method returns the proper headline text.
915 # 1. If s contains a section def, return the section ref.
916 # 2. cweb only: if s contains @c, return the function name following the @c.
917 # 3. cweb only: if s contains @d name, returns @d name.
918 # 4. Otherwise, returns "@"
919 #@@c
921 def scanBodyForHeadline(self, s):
922 if self.webType == "cweb":
923 #@+<< scan cweb body for headline >>
924 #@+node:ekr.20031218072017.3229: *6* << scan cweb body for headline >>
925 i = 0
926 while i < len(s):
927 i = g.skip_ws_and_nl(s, i)
928 # Allow constructs such as @ @c, or @ @<.
929 if self.isDocStart(s, i):
930 i += 2
931 i = g.skip_ws(s, i)
932 if g.match(s, i, "@d") or g.match(s, i, "@f"):
933 # Look for a macro name.
934 directive = s[i : i + 2]
935 i = g.skip_ws(s, i + 2) # skip the @d or @f
936 if i < len(s) and g.is_c_id(s[i]):
937 j = i
938 g.skip_c_id(s, i)
939 return s[j:i]
940 return directive
941 if g.match(s, i, "@c") or g.match(s, i, "@p"):
942 # Look for a function def.
943 name = self.findFunctionDef(s, i + 2)
944 return name if name else "outer function"
945 if g.match(s, i, "@<"):
946 # Look for a section def.
947 # A small bug: the section def must end on this line.
948 j = i
949 k = g.find_on_line(s, i, "@>")
950 if k > -1 and (g.match(s, k + 2, "+=") or g.match(s, k + 2, "=")):
951 return s[j : k + 2] # return the section ref.
952 i = g.skip_line(s, i)
953 #@-<< scan cweb body for headline >>
954 else:
955 #@+<< scan noweb body for headline >>
956 #@+node:ekr.20031218072017.3230: *6* << scan noweb body for headline >>
957 i = 0
958 while i < len(s):
959 i = g.skip_ws_and_nl(s, i)
960 if g.match(s, i, "<<"):
961 k = g.find_on_line(s, i, ">>=")
962 if k > -1:
963 ref = s[i : k + 2]
964 name = s[i + 2 : k].strip()
965 if name != "@others":
966 return ref
967 else:
968 name = self.findFunctionDef(s, i)
969 if name:
970 return name
971 i = g.skip_line(s, i)
972 #@-<< scan noweb body for headline >>
973 return "@" # default.
974 #@+node:ekr.20031218072017.3231: *5* scanWebFile (handles limbo)
975 def scanWebFile(self, fileName, parent):
976 theType = self.webType
977 lb = "@<" if theType == "cweb" else "<<"
978 rb = "@>" if theType == "cweb" else ">>"
979 s, e = g.readFileIntoString(fileName)
980 if s is None:
981 return
982 #@+<< Create a symbol table of all section names >>
983 #@+node:ekr.20031218072017.3232: *6* << Create a symbol table of all section names >>
984 i = 0
985 self.web_st = []
986 while i < len(s):
987 progress = i
988 i = g.skip_ws_and_nl(s, i)
989 if self.isDocStart(s, i):
990 if theType == "cweb":
991 i += 2
992 else:
993 i = g.skip_line(s, i)
994 elif theType == "cweb" and g.match(s, i, "@@"):
995 i += 2
996 elif g.match(s, i, lb):
997 i += 2
998 j = i
999 k = g.find_on_line(s, j, rb)
1000 if k > -1:
1001 self.cstEnter(s[j:k])
1002 else:
1003 i += 1
1004 assert i > progress
1005 #@-<< Create a symbol table of all section names >>
1006 #@+<< Create nodes for limbo text and the root section >>
1007 #@+node:ekr.20031218072017.3233: *6* << Create nodes for limbo text and the root section >>
1008 i = 0
1009 while i < len(s):
1010 progress = i
1011 i = g.skip_ws_and_nl(s, i)
1012 if self.isModuleStart(s, i) or g.match(s, i, lb):
1013 break
1014 else: i = g.skip_line(s, i)
1015 assert i > progress
1016 j = g.skip_ws(s, 0)
1017 if j < i:
1018 self.createHeadline(parent, "@ " + s[j:i], "Limbo")
1019 j = i
1020 if g.match(s, i, lb):
1021 while i < len(s):
1022 progress = i
1023 i = g.skip_ws_and_nl(s, i)
1024 if self.isModuleStart(s, i):
1025 break
1026 else: i = g.skip_line(s, i)
1027 assert i > progress
1028 self.createHeadline(parent, s[j:i], g.angleBrackets(" @ "))
1030 #@-<< Create nodes for limbo text and the root section >>
1031 while i < len(s):
1032 outer_progress = i
1033 #@+<< Create a node for the next module >>
1034 #@+node:ekr.20031218072017.3234: *6* << Create a node for the next module >>
1035 if theType == "cweb":
1036 assert self.isModuleStart(s, i)
1037 start = i
1038 if self.isDocStart(s, i):
1039 i += 2
1040 while i < len(s):
1041 progress = i
1042 i = g.skip_ws_and_nl(s, i)
1043 if self.isModuleStart(s, i):
1044 break
1045 else:
1046 i = g.skip_line(s, i)
1047 assert i > progress
1048 #@+<< Handle cweb @d, @f, @c and @p directives >>
1049 #@+node:ekr.20031218072017.3235: *7* << Handle cweb @d, @f, @c and @p directives >>
1050 if g.match(s, i, "@d") or g.match(s, i, "@f"):
1051 i += 2
1052 i = g.skip_line(s, i)
1053 # Place all @d and @f directives in the same node.
1054 while i < len(s):
1055 progress = i
1056 i = g.skip_ws_and_nl(s, i)
1057 if g.match(s, i, "@d") or g.match(s, i, "@f"):
1058 i = g.skip_line(s, i)
1059 else:
1060 break
1061 assert i > progress
1062 i = g.skip_ws_and_nl(s, i)
1063 while i < len(s) and not self.isModuleStart(s, i):
1064 progress = i
1065 i = g.skip_line(s, i)
1066 i = g.skip_ws_and_nl(s, i)
1067 assert i > progress
1068 if g.match(s, i, "@c") or g.match(s, i, "@p"):
1069 i += 2
1070 while i < len(s):
1071 progress = i
1072 i = g.skip_line(s, i)
1073 i = g.skip_ws_and_nl(s, i)
1074 if self.isModuleStart(s, i):
1075 break
1076 assert i > progress
1077 #@-<< Handle cweb @d, @f, @c and @p directives >>
1078 else:
1079 assert self.isDocStart(s, i)
1080 start = i
1081 i = g.skip_line(s, i)
1082 while i < len(s):
1083 progress = i
1084 i = g.skip_ws_and_nl(s, i)
1085 if self.isDocStart(s, i):
1086 break
1087 else:
1088 i = g.skip_line(s, i)
1089 assert i > progress
1090 body = s[start:i]
1091 body = self.massageWebBody(body)
1092 headline = self.scanBodyForHeadline(body)
1093 self.createHeadline(parent, body, headline)
1094 #@-<< Create a node for the next module >>
1095 assert i > outer_progress
1096 #@+node:ekr.20031218072017.3236: *5* Symbol table
1097 #@+node:ekr.20031218072017.3237: *6* cstCanonicalize
1098 # We canonicalize strings before looking them up,
1099 # but strings are entered in the form they are first encountered.
1101 def cstCanonicalize(self, s, lower=True):
1102 if lower:
1103 s = s.lower()
1104 s = s.replace("\t", " ").replace("\r", "")
1105 s = s.replace("\n", " ").replace(" ", " ")
1106 return s.strip()
1107 #@+node:ekr.20031218072017.3238: *6* cstDump
1108 def cstDump(self):
1109 s = "Web Symbol Table...\n\n"
1110 for name in sorted(self.web_st):
1111 s += name + "\n"
1112 return s
1113 #@+node:ekr.20031218072017.3239: *6* cstEnter
1114 # We only enter the section name into the symbol table if the ... convention is not used.
1116 def cstEnter(self, s):
1117 # Don't enter names that end in "..."
1118 s = s.rstrip()
1119 if s.endswith("..."):
1120 return
1121 # Put the section name in the symbol table, retaining capitalization.
1122 lower = self.cstCanonicalize(s, True) # do lower
1123 upper = self.cstCanonicalize(s, False) # don't lower.
1124 for name in self.web_st:
1125 if name.lower() == lower:
1126 return
1127 self.web_st.append(upper)
1128 #@+node:ekr.20031218072017.3240: *6* cstLookup
1129 # This method returns a string if the indicated string is a prefix of an entry in the web_st.
1131 def cstLookup(self, target):
1132 # Do nothing if the ... convention is not used.
1133 target = target.strip()
1134 if not target.endswith("..."):
1135 return target
1136 # Canonicalize the target name, and remove the trailing "..."
1137 ctarget = target[:-3]
1138 ctarget = self.cstCanonicalize(ctarget).strip()
1139 found = False
1140 result = target
1141 for s in self.web_st:
1142 cs = self.cstCanonicalize(s)
1143 if cs[: len(ctarget)] == ctarget:
1144 if found:
1145 g.es('', f"****** {target}", 'is also a prefix of', s)
1146 else:
1147 found = True
1148 result = s
1149 # g.es("replacing",target,"with",s)
1150 return result
1151 #@+node:ekr.20140531104908.18833: *3* ic.parse_body
1152 def parse_body(self, p):
1153 """
1154 Parse p.b as source code, creating a tree of descendant nodes.
1155 This is essentially an import of p.b.
1156 """
1157 if not p:
1158 return
1159 c, d, ic = self.c, g.app.language_extension_dict, self
1160 if p.hasChildren():
1161 g.es_print('can not run parse-body: node has children:', p.h)
1162 return
1163 language = g.scanForAtLanguage(c, p)
1164 self.treeType = '@file'
1165 ext = '.' + d.get(language)
1166 parser = g.app.classDispatchDict.get(ext)
1167 # Fix bug 151: parse-body creates "None declarations"
1168 if p.isAnyAtFileNode():
1169 fn = p.anyAtFileNodeName()
1170 ic.methodName, ic.fileType = g.os_path_splitext(fn)
1171 else:
1172 fileType = d.get(language, 'py')
1173 ic.methodName, ic.fileType = p.h, fileType
1174 if not parser:
1175 g.es_print(f"parse-body: no parser for @language {language or 'None'}")
1176 return
1177 bunch = c.undoer.beforeChangeTree(p)
1178 s = p.b
1179 p.b = ''
1180 try:
1181 parser(c, s, p) # 2357.
1182 c.undoer.afterChangeTree(p, 'parse-body', bunch)
1183 p.expand()
1184 c.selectPosition(p)
1185 c.redraw()
1186 except Exception:
1187 g.es_exception()
1188 p.b = s
1189 #@+node:ekr.20031218072017.3305: *3* ic.Utilities
1190 #@+node:ekr.20090122201952.4: *4* ic.appendStringToBody & setBodyString (leoImport)
1191 def appendStringToBody(self, p, s):
1192 """Similar to c.appendStringToBody,
1193 but does not recolor the text or redraw the screen."""
1194 if s:
1195 p.b = p.b + g.toUnicode(s, self.encoding)
1197 def setBodyString(self, p, s):
1198 """
1199 Similar to c.setBodyString, but does not recolor the text or
1200 redraw the screen.
1201 """
1202 c, v = self.c, p.v
1203 if not c or not p:
1204 return
1205 s = g.toUnicode(s, self.encoding)
1206 if c.p and p.v == c.p.v:
1207 w = c.frame.body.wrapper
1208 i = len(s)
1209 w.setAllText(s)
1210 w.setSelectionRange(i, i, insert=i)
1211 # Keep the body text up-to-date.
1212 if v.b != s:
1213 v.setBodyString(s)
1214 v.setSelection(0, 0)
1215 p.setDirty()
1216 if not c.isChanged():
1217 c.setChanged()
1218 #@+node:ekr.20031218072017.3306: *4* ic.createHeadline
1219 def createHeadline(self, parent, body, headline):
1220 """Create a new VNode as the last child of parent position."""
1221 p = parent.insertAsLastChild()
1222 p.initHeadString(headline)
1223 if body:
1224 self.setBodyString(p, body)
1225 return p
1226 #@+node:ekr.20031218072017.3307: *4* ic.error
1227 def error(self, s):
1228 g.es('', s)
1229 #@+node:ekr.20031218072017.3309: *4* ic.isDocStart & isModuleStart
1230 # The start of a document part or module in a noweb or cweb file.
1231 # Exporters may have to test for @doc as well.
1233 def isDocStart(self, s, i):
1234 if not g.match(s, i, "@"):
1235 return False
1236 j = g.skip_ws(s, i + 1)
1237 if g.match(s, j, "%defs"):
1238 return False
1239 if self.webType == "cweb" and g.match(s, i, "@*"):
1240 return True
1241 return g.match(s, i, "@ ") or g.match(s, i, "@\t") or g.match(s, i, "@\n")
1243 def isModuleStart(self, s, i):
1244 if self.isDocStart(s, i):
1245 return True
1246 return self.webType == "cweb" and (
1247 g.match(s, i, "@c") or g.match(s, i, "@p") or
1248 g.match(s, i, "@d") or g.match(s, i, "@f"))
1249 #@+node:ekr.20031218072017.3312: *4* ic.massageWebBody
1250 def massageWebBody(self, s):
1251 theType = self.webType
1252 lb = "@<" if theType == "cweb" else "<<"
1253 rb = "@>" if theType == "cweb" else ">>"
1254 #@+<< Remove most newlines from @space and @* sections >>
1255 #@+node:ekr.20031218072017.3313: *5* << Remove most newlines from @space and @* sections >>
1256 i = 0
1257 while i < len(s):
1258 progress = i
1259 i = g.skip_ws_and_nl(s, i)
1260 if self.isDocStart(s, i):
1261 # Scan to end of the doc part.
1262 if g.match(s, i, "@ %def"):
1263 # Don't remove the newline following %def
1264 i = g.skip_line(s, i)
1265 start = end = i
1266 else:
1267 start = end = i
1268 i += 2
1269 while i < len(s):
1270 progress2 = i
1271 i = g.skip_ws_and_nl(s, i)
1272 if self.isModuleStart(s, i) or g.match(s, i, lb):
1273 end = i
1274 break
1275 elif theType == "cweb":
1276 i += 1
1277 else:
1278 i = g.skip_to_end_of_line(s, i)
1279 assert i > progress2
1280 # Remove newlines from start to end.
1281 doc = s[start:end]
1282 doc = doc.replace("\n", " ")
1283 doc = doc.replace("\r", "")
1284 doc = doc.strip()
1285 if doc:
1286 if doc == "@":
1287 doc = "@ " if self.webType == "cweb" else "@\n"
1288 else:
1289 doc += "\n\n"
1290 s = s[:start] + doc + s[end:]
1291 i = start + len(doc)
1292 else: i = g.skip_line(s, i)
1293 assert i > progress
1294 #@-<< Remove most newlines from @space and @* sections >>
1295 #@+<< Replace abbreviated names with full names >>
1296 #@+node:ekr.20031218072017.3314: *5* << Replace abbreviated names with full names >>
1297 i = 0
1298 while i < len(s):
1299 progress = i
1300 if g.match(s, i, lb):
1301 i += 2
1302 j = i
1303 k = g.find_on_line(s, j, rb)
1304 if k > -1:
1305 name = s[j:k]
1306 name2 = self.cstLookup(name)
1307 if name != name2:
1308 # Replace name by name2 in s.
1309 s = s[:j] + name2 + s[k:]
1310 i = j + len(name2)
1311 i = g.skip_line(s, i)
1312 assert i > progress
1313 #@-<< Replace abbreviated names with full names >>
1314 s = s.rstrip()
1315 return s
1316 #@+node:ekr.20031218072017.1463: *4* ic.setEncoding
1317 def setEncoding(self, p=None, default=None):
1318 c = self.c
1319 encoding = g.getEncodingAt(p or c.p) or default
1320 if encoding and g.isValidEncoding(encoding):
1321 self.encoding = encoding
1322 elif default:
1323 self.encoding = default
1324 else:
1325 self.encoding = 'utf-8'
1326 #@-others
1327#@+node:ekr.20160503144404.1: ** class MindMapImporter
1328class MindMapImporter:
1329 """Mind Map Importer class."""
1331 def __init__(self, c):
1332 """ctor for MindMapImporter class."""
1333 self.c = c
1334 #@+others
1335 #@+node:ekr.20160503130209.1: *3* mindmap.create_outline
1336 def create_outline(self, path):
1337 c = self.c
1338 junk, fileName = g.os_path_split(path)
1339 undoData = c.undoer.beforeInsertNode(c.p)
1340 # Create the top-level headline.
1341 p = c.lastTopLevel().insertAfter()
1342 fn = g.shortFileName(path).strip()
1343 if fn.endswith('.csv'):
1344 fn = fn[:-4]
1345 p.h = fn
1346 try:
1347 self.scan(path, p)
1348 except Exception:
1349 g.es_print('Invalid MindJet file:', fn)
1350 c.undoer.afterInsertNode(p, 'Import', undoData)
1351 return p
1352 #@+node:ekr.20160503144647.1: *3* mindmap.import_files
1353 def import_files(self, files):
1354 """Import a list of MindMap (.csv) files."""
1355 c = self.c
1356 if files:
1357 self.tab_width = c.getTabWidth(c.p)
1358 for fileName in files:
1359 g.setGlobalOpenDir(fileName)
1360 p = self.create_outline(fileName)
1361 p.contract()
1362 p.setDirty()
1363 c.setChanged()
1364 c.redraw(p)
1365 #@+node:ekr.20160504043243.1: *3* mindmap.prompt_for_files
1366 def prompt_for_files(self):
1367 """Prompt for a list of MindJet (.csv) files and import them."""
1368 c = self.c
1369 types = [
1370 ("MindJet files", "*.csv"),
1371 ("All files", "*"),
1372 ]
1373 names = g.app.gui.runOpenFileDialog(c,
1374 title="Import MindJet File",
1375 filetypes=types,
1376 defaultextension=".csv",
1377 multiple=True)
1378 c.bringToFront()
1379 if names:
1380 g.chdir(names[0])
1381 self.import_files(names)
1382 #@+node:ekr.20160503130256.1: *3* mindmap.scan & helpers
1383 def scan(self, path, target):
1384 """Create an outline from a MindMap (.csv) file."""
1385 c = self.c
1386 f = open(path)
1387 reader = csv.reader(f)
1388 max_chars_in_header = 80
1389 n1 = n = target.level()
1390 p = target.copy()
1391 for row in list(reader)[1:]:
1392 new_level = self.csv_level(row) + n1
1393 self.csv_string(row)
1394 if new_level > n:
1395 p = p.insertAsLastChild().copy()
1396 p.b = self.csv_string(row)
1397 n = n + 1
1398 elif new_level == n:
1399 p = p.insertAfter().copy()
1400 p.b = self.csv_string(row)
1401 elif new_level < n:
1402 for item in p.parents():
1403 if item.level() == new_level - 1:
1404 p = item.copy()
1405 break
1406 p = p.insertAsLastChild().copy()
1407 p.b = self.csv_string(row)
1408 n = p.level()
1409 for p in target.unique_subtree():
1410 if len(p.b.splitlines()) == 1:
1411 if len(p.b.splitlines()[0]) < max_chars_in_header:
1412 p.h = p.b.splitlines()[0]
1413 p.b = ""
1414 else:
1415 p.h = "@node_with_long_text"
1416 else:
1417 p.h = "@node_with_long_text"
1418 c.redraw()
1419 f.close()
1420 #@+node:ekr.20160503130810.4: *4* mindmap.csv_level
1421 def csv_level(self, row):
1422 """Return the level of the given row."""
1423 count = 0
1424 while count <= len(row):
1425 if row[count]:
1426 return count + 1
1427 count = count + 1
1428 return -1
1429 #@+node:ekr.20160503130810.5: *4* mindmap.csv_string
1430 def csv_string(self, row):
1431 """Return the string for the given csv row."""
1432 count = 0
1433 while count <= len(row):
1434 if row[count]:
1435 return row[count]
1436 count = count + 1
1437 return None
1438 #@-others
1439#@+node:ekr.20161006100941.1: ** class MORE_Importer
1440class MORE_Importer:
1441 """Class to import MORE files."""
1443 def __init__(self, c):
1444 """ctor for MORE_Importer class."""
1445 self.c = c
1446 #@+others
1447 #@+node:ekr.20161006101111.1: *3* MORE.prompt_for_files
1448 def prompt_for_files(self):
1449 """Prompt for a list of MORE files and import them."""
1450 c = self.c
1451 types = [
1452 ("All files", "*"),
1453 ]
1454 names = g.app.gui.runOpenFileDialog(c,
1455 title="Import MORE Files",
1456 filetypes=types,
1457 # defaultextension=".txt",
1458 multiple=True)
1459 c.bringToFront()
1460 if names:
1461 g.chdir(names[0])
1462 self.import_files(names)
1463 #@+node:ekr.20161006101218.1: *3* MORE.import_files
1464 def import_files(self, files):
1465 """Import a list of MORE (.csv) files."""
1466 c = self.c
1467 if files:
1468 changed = False
1469 self.tab_width = c.getTabWidth(c.p)
1470 for fileName in files:
1471 g.setGlobalOpenDir(fileName)
1472 p = self.import_file(fileName)
1473 if p:
1474 p.contract()
1475 p.setDirty()
1476 c.setChanged()
1477 changed = True
1478 if changed:
1479 c.redraw(p)
1480 #@+node:ekr.20161006101347.1: *3* MORE.import_file
1481 def import_file(self, fileName): # Not a command, so no event arg.
1482 c = self.c
1483 u = c.undoer
1484 ic = c.importCommands
1485 if not c.p:
1486 return None
1487 ic.setEncoding()
1488 g.setGlobalOpenDir(fileName)
1489 s, e = g.readFileIntoString(fileName)
1490 if s is None:
1491 return None
1492 s = s.replace('\r', '') # Fixes bug 626101.
1493 lines = g.splitLines(s)
1494 # Convert the string to an outline and insert it after the current node.
1495 if self.check_lines(lines):
1496 last = c.lastTopLevel()
1497 undoData = u.beforeInsertNode(c.p)
1498 root = last.insertAfter()
1499 root.h = fileName
1500 p = self.import_lines(lines, root)
1501 if p:
1502 c.endEditing()
1503 c.validateOutline()
1504 p.setDirty()
1505 c.setChanged()
1506 u.afterInsertNode(root, 'Import MORE File', undoData)
1507 c.selectPosition(root)
1508 c.redraw()
1509 return root
1510 if not g.unitTesting:
1511 g.es("not a valid MORE file", fileName)
1512 return None
1513 #@+node:ekr.20031218072017.3215: *3* MORE.import_lines
1514 def import_lines(self, strings, first_p):
1515 c = self.c
1516 if not strings:
1517 return None
1518 if not self.check_lines(strings):
1519 return None
1520 firstLevel, junk = self.headlineLevel(strings[0])
1521 lastLevel = -1
1522 theRoot = last_p = None
1523 index = 0
1524 while index < len(strings):
1525 progress = index
1526 s = strings[index]
1527 level, junk = self.headlineLevel(s)
1528 level -= firstLevel
1529 if level >= 0:
1530 #@+<< Link a new position p into the outline >>
1531 #@+node:ekr.20031218072017.3216: *4* << Link a new position p into the outline >>
1532 assert level >= 0
1533 if not last_p:
1534 theRoot = p = first_p.insertAsLastChild() # 2016/10/06.
1535 elif level == lastLevel:
1536 p = last_p.insertAfter()
1537 elif level == lastLevel + 1:
1538 p = last_p.insertAsNthChild(0)
1539 else:
1540 assert level < lastLevel
1541 while level < lastLevel:
1542 lastLevel -= 1
1543 last_p = last_p.parent()
1544 assert last_p
1545 assert lastLevel >= 0
1546 p = last_p.insertAfter()
1547 last_p = p
1548 lastLevel = level
1549 #@-<< Link a new position p into the outline >>
1550 #@+<< Set the headline string, skipping over the leader >>
1551 #@+node:ekr.20031218072017.3217: *4* << Set the headline string, skipping over the leader >>
1552 j = 0
1553 while g.match(s, j, '\t') or g.match(s, j, ' '):
1554 j += 1
1555 if g.match(s, j, "+ ") or g.match(s, j, "- "):
1556 j += 2
1557 p.initHeadString(s[j:])
1558 #@-<< Set the headline string, skipping over the leader >>
1559 #@+<< Count the number of following body lines >>
1560 #@+node:ekr.20031218072017.3218: *4* << Count the number of following body lines >>
1561 bodyLines = 0
1562 index += 1 # Skip the headline.
1563 while index < len(strings):
1564 s = strings[index]
1565 level, junk = self.headlineLevel(s)
1566 level -= firstLevel
1567 if level >= 0:
1568 break
1569 # Remove first backslash of the body line.
1570 if g.match(s, 0, '\\'):
1571 strings[index] = s[1:]
1572 bodyLines += 1
1573 index += 1
1574 #@-<< Count the number of following body lines >>
1575 #@+<< Add the lines to the body text of p >>
1576 #@+node:ekr.20031218072017.3219: *4* << Add the lines to the body text of p >>
1577 if bodyLines > 0:
1578 body = ""
1579 n = index - bodyLines
1580 while n < index:
1581 body += strings[n].rstrip()
1582 if n != index - 1:
1583 body += "\n"
1584 n += 1
1585 p.setBodyString(body)
1586 #@-<< Add the lines to the body text of p >>
1587 p.setDirty()
1588 else: index += 1
1589 assert progress < index
1590 if theRoot:
1591 theRoot.setDirty()
1592 c.setChanged()
1593 c.redraw()
1594 return theRoot
1595 #@+node:ekr.20031218072017.3222: *3* MORE.headlineLevel
1596 def headlineLevel(self, s):
1597 """return the headline level of s,or -1 if the string is not a MORE headline."""
1598 level = 0
1599 i = 0
1600 while i < len(s) and s[i] in ' \t': # 2016/10/06: allow blanks or tabs.
1601 level += 1
1602 i += 1
1603 plusFlag = g.match(s, i, "+")
1604 if g.match(s, i, "+ ") or g.match(s, i, "- "):
1605 return level, plusFlag
1606 return -1, plusFlag
1607 #@+node:ekr.20031218072017.3223: *3* MORE.check & check_lines
1608 def check(self, s):
1609 s = s.replace("\r", "")
1610 strings = g.splitLines(s)
1611 return self.check_lines(strings)
1613 def check_lines(self, strings):
1615 if not strings:
1616 return False
1617 level1, plusFlag = self.headlineLevel(strings[0])
1618 if level1 == -1:
1619 return False
1620 # Check the level of all headlines.
1621 lastLevel = level1
1622 for s in strings:
1623 level, newFlag = self.headlineLevel(s)
1624 if level == -1:
1625 return True # A body line.
1626 if level < level1 or level > lastLevel + 1:
1627 return False # improper level.
1628 if level > lastLevel and not plusFlag:
1629 return False # parent of this node has no children.
1630 if level == lastLevel and plusFlag:
1631 return False # last node has missing child.
1632 lastLevel = level
1633 plusFlag = newFlag
1634 return True
1635 #@-others
1636#@+node:ekr.20130823083943.12596: ** class RecursiveImportController
1637class RecursiveImportController:
1638 """Recursively import all python files in a directory and clean the result."""
1639 #@+others
1640 #@+node:ekr.20130823083943.12615: *3* ric.ctor
1641 def __init__(self, c, kind,
1642 add_context=None, # Override setting only if True/False
1643 add_file_context=None, # Override setting only if True/False
1644 add_path=True,
1645 recursive=True,
1646 safe_at_file=True,
1647 theTypes=None,
1648 ignore_pattern=None,
1649 verbose=True, # legacy value.
1650 ):
1651 """Ctor for RecursiveImportController class."""
1652 self.c = c
1653 self.add_path = add_path
1654 self.file_pattern = re.compile(r'^(@@|@)(auto|clean|edit|file|nosent)')
1655 self.ignore_pattern = ignore_pattern or re.compile(r'\.git|node_modules')
1656 self.kind = kind # in ('@auto', '@clean', '@edit', '@file', '@nosent')
1657 self.recursive = recursive
1658 self.root = None
1659 self.safe_at_file = safe_at_file
1660 self.theTypes = theTypes
1661 self.verbose = verbose
1662 # #1605:
1664 def set_bool(setting, val):
1665 if val not in (True, False):
1666 return
1667 c.config.set(None, 'bool', setting, val, warn=True)
1669 set_bool('add-context-to-headlines', add_context)
1670 set_bool('add-file-context-to-headlines', add_file_context)
1671 #@+node:ekr.20130823083943.12613: *3* ric.run & helpers
1672 def run(self, dir_):
1673 """
1674 Import all files whose extension matches self.theTypes in dir_.
1675 In fact, dir_ can be a path to a single file.
1676 """
1677 if self.kind not in ('@auto', '@clean', '@edit', '@file', '@nosent'):
1678 g.es('bad kind param', self.kind, color='red')
1679 try:
1680 c = self.c
1681 p1 = self.root = c.p
1682 t1 = time.time()
1683 g.app.disable_redraw = True
1684 bunch = c.undoer.beforeChangeTree(p1)
1685 # Leo 5.6: Always create a new last top-level node.
1686 last = c.lastTopLevel()
1687 parent = last.insertAfter()
1688 parent.v.h = 'imported files'
1689 # Leo 5.6: Special case for a single file.
1690 self.n_files = 0
1691 if g.os_path_isfile(dir_):
1692 if self.verbose:
1693 g.es_print('\nimporting file:', dir_)
1694 self.import_one_file(dir_, parent)
1695 else:
1696 self.import_dir(dir_, parent)
1697 self.post_process(parent, dir_)
1698 # Fix # 1033.
1699 c.undoer.afterChangeTree(p1, 'recursive-import', bunch)
1700 except Exception:
1701 g.es_print('Exception in recursive import')
1702 g.es_exception()
1703 finally:
1704 g.app.disable_redraw = False
1705 for p2 in parent.self_and_subtree(copy=False):
1706 p2.contract()
1707 c.redraw(parent)
1708 t2 = time.time()
1709 n = len(list(parent.self_and_subtree()))
1710 g.es_print(
1711 f"imported {n} node{g.plural(n)} "
1712 f"in {self.n_files} file{g.plural(self.n_files)} "
1713 f"in {t2 - t1:2.2f} seconds")
1714 #@+node:ekr.20130823083943.12597: *4* ric.import_dir
1715 def import_dir(self, dir_, parent):
1716 """Import selected files from dir_, a directory."""
1717 if g.os_path_isfile(dir_):
1718 files = [dir_]
1719 else:
1720 if self.verbose:
1721 g.es_print('importing directory:', dir_)
1722 files = os.listdir(dir_)
1723 dirs, files2 = [], []
1724 for path in files:
1725 try:
1726 # Fix #408. Catch path exceptions.
1727 # The idea here is to keep going on small errors.
1728 path = g.os_path_join(dir_, path)
1729 if g.os_path_isfile(path):
1730 name, ext = g.os_path_splitext(path)
1731 if ext in self.theTypes:
1732 files2.append(path)
1733 elif self.recursive:
1734 if not self.ignore_pattern.search(path):
1735 dirs.append(path)
1736 except OSError:
1737 g.es_print('Exception computing', path)
1738 g.es_exception()
1739 if files or dirs:
1740 assert parent and parent.v != self.root.v, g.callers()
1741 parent = parent.insertAsLastChild()
1742 parent.v.h = dir_
1743 if files2:
1744 for f in files2:
1745 if not self.ignore_pattern.search(f):
1746 self.import_one_file(f, parent=parent)
1747 if dirs:
1748 assert self.recursive
1749 for dir_ in sorted(dirs):
1750 self.import_dir(dir_, parent)
1751 #@+node:ekr.20170404103953.1: *4* ric.import_one_file
1752 def import_one_file(self, path, parent):
1753 """Import one file to the last top-level node."""
1754 c = self.c
1755 self.n_files += 1
1756 assert parent and parent.v != self.root.v, g.callers()
1757 if self.kind == '@edit':
1758 p = parent.insertAsLastChild()
1759 p.v.h = '@edit ' + path.replace('\\', '/') # 2021/02/19: bug fix: add @edit.
1760 s, e = g.readFileIntoString(path, kind=self.kind)
1761 p.v.b = s
1762 return
1763 # #1484: Use this for @auto as well.
1764 c.importCommands.importFilesCommand(
1765 files=[path],
1766 parent=parent,
1767 shortFn=True,
1768 treeType='@file', # '@auto','@clean','@nosent' cause problems.
1769 verbose=self.verbose, # Leo 6.6.
1770 )
1771 p = parent.lastChild()
1772 p.h = self.kind + p.h[5:]
1773 # Bug fix 2017/10/27: honor the requested kind.
1774 if self.safe_at_file:
1775 p.v.h = '@' + p.v.h
1776 #@+node:ekr.20130823083943.12607: *4* ric.post_process & helpers
1777 def post_process(self, p, prefix):
1778 """
1779 Traverse p's tree, replacing all nodes that start with prefix
1780 by the smallest equivalent @path or @file node.
1781 """
1782 self.fix_back_slashes(p)
1783 prefix = prefix.replace('\\', '/')
1784 if self.kind not in ('@auto', '@edit'):
1785 self.remove_empty_nodes(p)
1786 if p.firstChild():
1787 self.minimize_headlines(p.firstChild(), prefix)
1788 self.clear_dirty_bits(p)
1789 self.add_class_names(p)
1790 #@+node:ekr.20180524100258.1: *5* ric.add_class_names
1791 def add_class_names(self, p):
1792 """Add class names to headlines for all descendant nodes."""
1793 # pylint: disable=no-else-continue
1794 after, class_name = None, None
1795 class_paren_pattern = re.compile(r'(.*)\(.*\)\.(.*)')
1796 paren_pattern = re.compile(r'(.*)\(.*\.py\)')
1797 for p in p.self_and_subtree(copy=False):
1798 # Part 1: update the status.
1799 m = self.file_pattern.match(p.h)
1800 if m:
1801 # prefix = m.group(1)
1802 # fn = g.shortFileName(p.h[len(prefix):].strip())
1803 after, class_name = None, None
1804 continue
1805 elif p.h.startswith('@path '):
1806 after, class_name = None, None
1807 elif p.h.startswith('class '):
1808 class_name = p.h[5:].strip()
1809 if class_name:
1810 after = p.nodeAfterTree()
1811 continue
1812 elif p == after:
1813 after, class_name = None, None
1814 # Part 2: update the headline.
1815 if class_name:
1816 if p.h.startswith(class_name):
1817 m = class_paren_pattern.match(p.h)
1818 if m:
1819 p.h = f"{m.group(1)}.{m.group(2)}".rstrip()
1820 else:
1821 p.h = f"{class_name}.{p.h}"
1822 else:
1823 m = paren_pattern.match(p.h)
1824 if m:
1825 p.h = m.group(1).rstrip()
1826 # elif fn:
1827 # tag = ' (%s)' % fn
1828 # if not p.h.endswith(tag):
1829 # p.h += tag
1830 #@+node:ekr.20130823083943.12608: *5* ric.clear_dirty_bits
1831 def clear_dirty_bits(self, p):
1832 c = self.c
1833 c.clearChanged() # Clears *all* dirty bits.
1834 for p in p.self_and_subtree(copy=False):
1835 p.clearDirty()
1836 #@+node:ekr.20130823083943.12609: *5* ric.dump_headlines
1837 def dump_headlines(self, p):
1838 # show all headlines.
1839 for p in p.self_and_subtree(copy=False):
1840 print(p.h)
1841 #@+node:ekr.20130823083943.12610: *5* ric.fix_back_slashes
1842 def fix_back_slashes(self, p):
1843 """Convert backslash to slash in all headlines."""
1844 for p in p.self_and_subtree(copy=False):
1845 s = p.h.replace('\\', '/')
1846 if s != p.h:
1847 p.v.h = s
1848 #@+node:ekr.20130823083943.12611: *5* ric.minimize_headlines & helper
1849 def minimize_headlines(self, p, prefix):
1850 """Create @path nodes to minimize the paths required in descendant nodes."""
1851 if prefix and not prefix.endswith('/'):
1852 prefix = prefix + '/'
1853 m = self.file_pattern.match(p.h)
1854 if m:
1855 # It's an @file node of some kind. Strip off the prefix.
1856 kind = m.group(0)
1857 path = p.h[len(kind) :].strip()
1858 stripped = self.strip_prefix(path, prefix)
1859 p.h = f"{kind} {stripped or path}"
1860 # Put the *full* @path directive in the body.
1861 if self.add_path and prefix:
1862 tail = g.os_path_dirname(stripped).rstrip('/')
1863 p.b = f"@path {prefix}{tail}\n{p.b}"
1864 else:
1865 # p.h is a path.
1866 path = p.h
1867 stripped = self.strip_prefix(path, prefix)
1868 p.h = f"@path {stripped or path}"
1869 for p in p.children():
1870 self.minimize_headlines(p, prefix + stripped)
1871 #@+node:ekr.20170404134052.1: *6* ric.strip_prefix
1872 def strip_prefix(self, path, prefix):
1873 """Strip the prefix from the path and return the result."""
1874 if path.startswith(prefix):
1875 return path[len(prefix) :]
1876 return '' # A signal.
1877 #@+node:ekr.20130823083943.12612: *5* ric.remove_empty_nodes
1878 def remove_empty_nodes(self, p):
1879 """Remove empty nodes. Not called for @auto or @edit trees."""
1880 c = self.c
1881 aList = [
1882 p2 for p2 in p.self_and_subtree()
1883 if not p2.b and not p2.hasChildren()]
1884 if aList:
1885 c.deletePositionsInList(aList) # Don't redraw.
1886 #@-others
1887#@+node:ekr.20161006071801.1: ** class TabImporter
1888class TabImporter:
1889 """
1890 A class to import a file whose outline levels are indicated by
1891 leading tabs or blanks (but not both).
1892 """
1894 def __init__(self, c, separate=True):
1895 """Ctor for the TabImporter class."""
1896 self.c = c
1897 self.root = None
1898 self.separate = separate
1899 self.stack = []
1900 #@+others
1901 #@+node:ekr.20161006071801.2: *3* tabbed.check
1902 def check(self, lines, warn=True):
1903 """Return False and warn if lines contains mixed leading tabs/blanks."""
1904 blanks, tabs = 0, 0
1905 for s in lines:
1906 lws = self.lws(s)
1907 if '\t' in lws:
1908 tabs += 1
1909 if ' ' in lws:
1910 blanks += 1
1911 if tabs and blanks:
1912 if warn:
1913 g.es_print('intermixed leading blanks and tabs.')
1914 return False
1915 return True
1916 #@+node:ekr.20161006071801.3: *3* tabbed.dump_stack
1917 def dump_stack(self):
1918 """Dump the stack, containing (level, p) tuples."""
1919 g.trace('==========')
1920 for i, data in enumerate(self.stack):
1921 level, p = data
1922 print(f"{i:2} {level} {p.h!r}")
1923 #@+node:ekr.20161006073129.1: *3* tabbed.import_files
1924 def import_files(self, files):
1925 """Import a list of tab-delimited files."""
1926 c, u = self.c, self.c.undoer
1927 if files:
1928 p = None
1929 for fn in files:
1930 try:
1931 g.setGlobalOpenDir(fn)
1932 s = open(fn).read()
1933 s = s.replace('\r', '')
1934 except Exception:
1935 continue
1936 if s.strip() and self.check(g.splitLines(s)):
1937 undoData = u.beforeInsertNode(c.p)
1938 last = c.lastTopLevel()
1939 self.root = p = last.insertAfter()
1940 self.scan(s)
1941 p.h = g.shortFileName(fn)
1942 p.contract()
1943 p.setDirty()
1944 u.afterInsertNode(p, 'Import Tabbed File', undoData)
1945 if p:
1946 c.setChanged()
1947 c.redraw(p)
1948 #@+node:ekr.20161006071801.4: *3* tabbed.lws
1949 def lws(self, s):
1950 """Return the length of the leading whitespace of s."""
1951 for i, ch in enumerate(s):
1952 if ch not in ' \t':
1953 return s[:i]
1954 return s
1955 #@+node:ekr.20161006072958.1: *3* tabbed.prompt_for_files
1956 def prompt_for_files(self):
1957 """Prompt for a list of FreeMind (.mm.html) files and import them."""
1958 c = self.c
1959 types = [
1960 ("All files", "*"),
1961 ]
1962 names = g.app.gui.runOpenFileDialog(c,
1963 title="Import Tabbed File",
1964 filetypes=types,
1965 defaultextension=".html",
1966 multiple=True)
1967 c.bringToFront()
1968 if names:
1969 g.chdir(names[0])
1970 self.import_files(names)
1971 #@+node:ekr.20161006071801.5: *3* tabbed.scan
1972 def scan(self, s1, fn=None, root=None):
1973 """Create the outline corresponding to s1."""
1974 c = self.c
1975 # Self.root can be None if we are called from a script or unit test.
1976 if not self.root:
1977 last = root if root else c.lastTopLevel()
1978 # For unit testing.
1979 self.root = last.insertAfter()
1980 if fn:
1981 self.root.h = fn
1982 lines = g.splitLines(s1)
1983 self.stack = []
1984 # Redo the checks in case we are called from a script.
1985 if s1.strip() and self.check(lines):
1986 for s in lines:
1987 if s.strip() or not self.separate:
1988 self.scan_helper(s)
1989 return self.root
1990 #@+node:ekr.20161006071801.6: *3* tabbed.scan_helper
1991 def scan_helper(self, s):
1992 """Update the stack as necessary and return level."""
1993 root, separate, stack = self.root, self.separate, self.stack
1994 if stack:
1995 level, parent = stack[-1]
1996 else:
1997 level, parent = 0, None
1998 lws = len(self.lws(s))
1999 h = s.strip()
2000 if lws == level:
2001 if separate or not parent:
2002 # Replace the top of the stack with a new entry.
2003 if stack:
2004 stack.pop()
2005 grand_parent = stack[-1][1] if stack else root
2006 parent = grand_parent.insertAsLastChild() # lws == level
2007 parent.h = h
2008 stack.append((level, parent),)
2009 elif not parent.h:
2010 parent.h = h
2011 elif lws > level:
2012 # Create a new parent.
2013 level = lws
2014 parent = parent.insertAsLastChild()
2015 parent.h = h
2016 stack.append((level, parent),)
2017 else:
2018 # Find the previous parent.
2019 while stack:
2020 level2, parent2 = stack.pop()
2021 if level2 == lws:
2022 grand_parent = stack[-1][1] if stack else root
2023 parent = grand_parent.insertAsLastChild() # lws < level
2024 parent.h = h
2025 level = lws
2026 stack.append((level, parent),)
2027 break
2028 else:
2029 level = 0
2030 parent = root.insertAsLastChild()
2031 parent.h = h
2032 stack = [(0, parent),]
2033 assert parent and parent == stack[-1][1]
2034 # An important invariant.
2035 assert level == stack[-1][0], (level, stack[-1][0])
2036 if not separate:
2037 parent.b = parent.b + self.undent(level, s)
2038 return level
2039 #@+node:ekr.20161006071801.7: *3* tabbed.undent
2040 def undent(self, level, s):
2041 """Unindent all lines of p.b by level."""
2042 if level <= 0:
2043 return s
2044 if s.strip():
2045 lines = g.splitLines(s)
2046 ch = lines[0][0]
2047 assert ch in ' \t', repr(ch)
2048 # Check that all lines start with the proper lws.
2049 lws = ch * level
2050 for s in lines:
2051 if not s.startswith(lws):
2052 g.trace(f"bad indentation: {s!r}")
2053 return s
2054 return ''.join([z[len(lws) :] for z in lines])
2055 return ''
2056 #@-others
2057#@+node:ekr.20200310060123.1: ** class ToDoImporter
2058class ToDoImporter:
2060 def __init__(self, c):
2061 self.c = c
2063 #@+others
2064 #@+node:ekr.20200310103606.1: *3* todo_i.get_tasks_from_file
2065 def get_tasks_from_file(self, path):
2066 """Return the tasks from the given path."""
2067 tag = 'import-todo-text-files'
2068 if not os.path.exists(path):
2069 print(f"{tag}: file not found: {path}")
2070 return []
2071 try:
2072 with open(path, 'r') as f:
2073 contents = f.read()
2074 tasks = self.parse_file_contents(contents)
2075 return tasks
2076 except Exception:
2077 print(f"unexpected exception in {tag}")
2078 g.es_exception()
2079 return []
2080 #@+node:ekr.20200310101028.1: *3* todo_i.import_files
2081 def import_files(self, files):
2082 """
2083 Import all todo.txt files in the given list of file names.
2085 Return a dict: keys are full paths, values are lists of ToDoTasks"
2086 """
2087 d, tag = {}, 'import-todo-text-files'
2088 for path in files:
2089 try:
2090 with open(path, 'r') as f:
2091 contents = f.read()
2092 tasks = self.parse_file_contents(contents)
2093 d[path] = tasks
2094 except Exception:
2095 print(f"unexpected exception in {tag}")
2096 g.es_exception()
2097 return d
2098 #@+node:ekr.20200310062758.1: *3* todo_i.parse_file_contents
2099 # Patterns...
2100 mark_s = r'([x]\ )'
2101 priority_s = r'(\([A-Z]\)\ )'
2102 date_s = r'([0-9]{4}-[0-9]{2}-[0-9]{2}\ )'
2103 task_s = r'\s*(.+)'
2104 line_s = fr"^{mark_s}?{priority_s}?{date_s}?{date_s}?{task_s}$"
2105 line_pat = re.compile(line_s)
2107 def parse_file_contents(self, s):
2108 """
2109 Parse the contents of a file.
2110 Return a list of ToDoTask objects.
2111 """
2112 trace = False
2113 tasks = []
2114 for line in g.splitLines(s):
2115 if not line.strip():
2116 continue
2117 if trace:
2118 print(f"task: {line.rstrip()!s}")
2119 m = self.line_pat.match(line)
2120 if not m:
2121 print(f"invalid task: {line.rstrip()!s}")
2122 continue
2123 # Groups 1, 2 and 5 are context independent.
2124 completed = m.group(1)
2125 priority = m.group(2)
2126 task_s = m.group(5)
2127 if not task_s:
2128 print(f"invalid task: {line.rstrip()!s}")
2129 continue
2130 # Groups 3 and 4 are context dependent.
2131 if m.group(3) and m.group(4):
2132 complete_date = m.group(3)
2133 start_date = m.group(4)
2134 elif completed:
2135 complete_date = m.group(3)
2136 start_date = ''
2137 else:
2138 start_date = m.group(3) or ''
2139 complete_date = ''
2140 if completed and not complete_date:
2141 print(f"no completion date: {line.rstrip()!s}")
2142 tasks.append(ToDoTask(
2143 bool(completed), priority, start_date, complete_date, task_s))
2144 return tasks
2145 #@+node:ekr.20200310100919.1: *3* todo_i.prompt_for_files
2146 def prompt_for_files(self):
2147 """
2148 Prompt for a list of todo.text files and import them.
2150 Return a python dict. Keys are full paths; values are lists of ToDoTask objects.
2151 """
2152 c = self.c
2153 types = [
2154 ("Text files", "*.txt"),
2155 ("All files", "*"),
2156 ]
2157 names = g.app.gui.runOpenFileDialog(c,
2158 title="Import todo.txt File",
2159 filetypes=types,
2160 defaultextension=".txt",
2161 multiple=True,
2162 )
2163 c.bringToFront()
2164 if not names:
2165 return {}
2166 g.chdir(names[0])
2167 d = self.import_files(names)
2168 for key in sorted(d):
2169 tasks = d.get(key)
2170 print(f"tasks in {g.shortFileName(key)}...\n")
2171 for task in tasks:
2172 print(f" {task}")
2173 return d
2174 #@-others
2175#@+node:ekr.20200310063208.1: ** class ToDoTask
2176class ToDoTask:
2177 """A class representing the components of a task line."""
2179 def __init__(self, completed, priority, start_date, complete_date, task_s):
2180 self.completed = completed
2181 self.priority = priority and priority[1] or ''
2182 self.start_date = start_date and start_date.rstrip() or ''
2183 self.complete_date = complete_date and complete_date.rstrip() or ''
2184 self.task_s = task_s.strip()
2185 # Parse tags into separate dictionaries.
2186 self.projects = []
2187 self.contexts = []
2188 self.key_vals = []
2189 self.parse_task()
2191 #@+others
2192 #@+node:ekr.20200310075514.1: *3* task.__repr__ & __str__
2193 def __repr__(self):
2194 start_s = self.start_date if self.start_date else ''
2195 end_s = self.complete_date if self.complete_date else ''
2196 mark_s = '[X]' if self.completed else '[ ]'
2197 result = [
2198 f"Task: "
2199 f"{mark_s} "
2200 f"{self.priority:1} "
2201 f"start: {start_s:10} "
2202 f"end: {end_s:10} "
2203 f"{self.task_s}"
2204 ]
2205 for ivar in ('contexts', 'projects', 'key_vals'):
2206 aList = getattr(self, ivar, None)
2207 if aList:
2208 result.append(f"{' '*13}{ivar}: {aList}")
2209 return '\n'.join(result)
2211 __str__ = __repr__
2212 #@+node:ekr.20200310063138.1: *3* task.parse_task
2213 # Patterns...
2214 project_pat = re.compile(r'(\+\S+)')
2215 context_pat = re.compile(r'(@\S+)')
2216 key_val_pat = re.compile(r'((\S+):(\S+))') # Might be a false match.
2218 def parse_task(self):
2220 trace = False and not g.unitTesting
2221 s = self.task_s
2222 table = (
2223 ('context', self.context_pat, self.contexts),
2224 ('project', self.project_pat, self.projects),
2225 ('key:val', self.key_val_pat, self.key_vals),
2226 )
2227 for kind, pat, aList in table:
2228 for m in re.finditer(pat, s):
2229 pat_s = repr(pat).replace("re.compile('", "").replace("')", "")
2230 pat_s = pat_s.replace(r'\\', '\\')
2231 # Check for false key:val match:
2232 if pat == self.key_val_pat:
2233 key, value = m.group(2), m.group(3)
2234 if ':' in key or ':' in value:
2235 break
2236 tag = m.group(1)
2237 # Add the tag.
2238 if tag in aList:
2239 if trace:
2240 g.trace('Duplicate tag:', tag)
2241 else:
2242 if trace:
2243 g.trace(f"Add {kind} tag: {tag!s}")
2244 aList.append(tag)
2245 # Remove the tag from the task.
2246 s = re.sub(pat, "", s)
2247 if s != self.task_s:
2248 self.task_s = s.strip()
2249 #@-others
2250#@+node:ekr.20141210051628.26: ** class ZimImportController
2251class ZimImportController:
2252 """
2253 A class to import Zim folders and files: http://zim-wiki.org/
2254 First use Zim to export your project to rst files.
2256 Original script by Davy Cottet.
2258 User options:
2259 @int rst_level = 0
2260 @string rst_type
2261 @string zim_node_name
2262 @string path_to_zim
2264 """
2265 #@+others
2266 #@+node:ekr.20141210051628.31: *3* zic.__init__ & zic.reloadSettings
2267 def __init__(self, c):
2268 """Ctor for ZimImportController class."""
2269 self.c = c
2270 self.pathToZim = c.config.getString('path-to-zim')
2271 self.rstLevel = c.config.getInt('zim-rst-level') or 0
2272 self.rstType = c.config.getString('zim-rst-type') or 'rst'
2273 self.zimNodeName = c.config.getString('zim-node-name') or 'Imported Zim Tree'
2274 #@+node:ekr.20141210051628.28: *3* zic.parseZimIndex
2275 def parseZimIndex(self):
2276 """
2277 Parse Zim wiki index.rst and return a list of tuples (level, name, path) or None.
2278 """
2279 # c = self.c
2280 pathToZim = g.os_path_abspath(self.pathToZim)
2281 pathToIndex = g.os_path_join(pathToZim, 'index.rst')
2282 if not g.os_path_exists(pathToIndex):
2283 g.es(f"not found: {pathToIndex}", color='red')
2284 return None
2285 index = open(pathToIndex).read()
2286 parse = re.findall(r'(\t*)-\s`(.+)\s<(.+)>`_', index)
2287 if not parse:
2288 g.es(f"invalid index: {pathToIndex}", color='red')
2289 return None
2290 results = []
2291 for result in parse:
2292 level = len(result[0])
2293 name = result[1].decode('utf-8')
2294 unquote = urllib.parse.unquote
2295 # mypy: error: "str" has no attribute "decode"; maybe "encode"? [attr-defined]
2296 path = [g.os_path_abspath(g.os_path_join(
2297 pathToZim, unquote(result[2]).decode('utf-8')))] # type:ignore
2298 results.append((level, name, path))
2299 return results
2300 #@+node:ekr.20141210051628.29: *3* zic.rstToLastChild
2301 def rstToLastChild(self, p, name, rst):
2302 """Import an rst file as a last child of pos node with the specified name"""
2303 c = self.c
2304 c.importCommands.importFilesCommand(
2305 files=rst,
2306 parent=p,
2307 treeType='@rst',
2308 )
2309 rstNode = p.getLastChild()
2310 rstNode.h = name
2311 return rstNode
2312 #@+node:davy.20141212140940.1: *3* zic.clean
2313 def clean(self, zimNode, rstType):
2314 """Clean useless nodes"""
2315 warning = 'Warning: this node is ignored when writing this file'
2316 for p in zimNode.subtree_iter():
2317 # looking for useless bodies
2318 if p.hasFirstChild() and warning in p.b:
2319 child = p.getFirstChild()
2320 fmt = "@rst-no-head %s declarations"
2321 table = (
2322 fmt % p.h.replace(' ', '_'),
2323 fmt % p.h.replace(rstType, '').strip().replace(' ', '_'),
2324 )
2325 # Replace content with @rest-no-head first child (without title head) and delete it
2326 if child.h in table:
2327 p.b = '\n'.join(child.b.split('\n')[3:])
2328 child.doDelete()
2329 # Replace content of empty body parent node with first child with same name
2330 elif p.h == child.h or (f"{rstType} {child.h}" == p.h):
2331 if not child.hasFirstChild():
2332 p.b = child.b
2333 child.doDelete()
2334 elif not child.hasNext():
2335 p.b = child.b
2336 child.copyTreeFromSelfTo(p)
2337 child.doDelete()
2338 else:
2339 child.h = 'Introduction'
2340 elif p.hasFirstChild(
2341 ) and p.h.startswith("@rst-no-head") and not p.b.strip():
2342 child = p.getFirstChild()
2343 p_no_head = p.h.replace("@rst-no-head", "").strip()
2344 # Replace empty @rst-no-head by its same named chidren
2345 if child.h.strip() == p_no_head and not child.hasFirstChild():
2346 p.h = p_no_head
2347 p.b = child.b
2348 child.doDelete()
2349 elif p.h.startswith("@rst-no-head"):
2350 lines = p.b.split('\n')
2351 p.h = lines[1]
2352 p.b = '\n'.join(lines[3:])
2353 #@+node:ekr.20141210051628.30: *3* zic.run
2354 def run(self):
2355 """Create the zim node as the last top-level node."""
2356 c = self.c
2357 # Make sure a path is given.
2358 if not self.pathToZim:
2359 g.es('Missing setting: @string path_to_zim', color='red')
2360 return
2361 root = c.rootPosition()
2362 while root.hasNext():
2363 root.moveToNext()
2364 zimNode = root.insertAfter()
2365 zimNode.h = self.zimNodeName
2366 # Parse the index file
2367 files = self.parseZimIndex()
2368 if files:
2369 # Do the import
2370 rstNodes = {'0': zimNode,}
2371 for level, name, rst in files:
2372 if level == self.rstLevel:
2373 name = f"{self.rstType} {name}"
2374 rstNodes[
2375 str(
2376 level + 1)] = self.rstToLastChild(rstNodes[str(level)], name, rst)
2377 # Clean nodes
2378 g.es('Start cleaning process. Please wait...', color='blue')
2379 self.clean(zimNode, self.rstType)
2380 g.es('Done', color='blue')
2381 # Select zimNode
2382 c.selectPosition(zimNode)
2383 c.redraw()
2384 #@-others
2385#@+node:ekr.20200424152850.1: ** class LegacyExternalFileImporter
2386class LegacyExternalFileImporter:
2387 """
2388 A class to import external files written by versions of Leo earlier
2389 than 5.0.
2390 """
2391 # Sentinels to ignore, without the leading comment delim.
2392 ignore = ('@+at', '@-at', '@+leo', '@-leo', '@nonl', '@nl', '@-others')
2394 def __init__(self, c):
2395 self.c = c
2397 #@+others
2398 #@+node:ekr.20200424093946.1: *3* class Node
2399 class Node:
2401 def __init__(self, h, level):
2402 """Hold node data."""
2403 self.h = h.strip()
2404 self.level = level
2405 self.lines = []
2406 #@+node:ekr.20200424092652.1: *3* legacy.add
2407 def add(self, line, stack):
2408 """Add a line to the present node."""
2409 if stack:
2410 node = stack[-1]
2411 node.lines.append(line)
2412 else:
2413 print('orphan line: ', repr(line))
2414 #@+node:ekr.20200424160847.1: *3* legacy.compute_delim1
2415 def compute_delim1(self, path):
2416 """Return the opening comment delim for the given file."""
2417 junk, ext = os.path.splitext(path)
2418 if not ext:
2419 return None
2420 language = g.app.extension_dict.get(ext[1:])
2421 if not language:
2422 return None
2423 delim1, delim2, delim3 = g.set_delims_from_language(language)
2424 g.trace(language, delim1 or delim2)
2425 return delim1 or delim2
2426 #@+node:ekr.20200424153139.1: *3* legacy.import_file
2427 def import_file(self, path):
2428 """Import one legacy external file."""
2429 c = self.c
2430 root_h = g.shortFileName(path)
2431 delim1 = self.compute_delim1(path)
2432 if not delim1:
2433 g.es_print('unknown file extension:', color='red')
2434 g.es_print(path)
2435 return
2436 # Read the file into s.
2437 with open(path, 'r') as f:
2438 s = f.read()
2439 # Do nothing if the file is a newer external file.
2440 if delim1 + '@+leo-ver=4' not in s:
2441 g.es_print('not a legacy external file:', color='red')
2442 g.es_print(path)
2443 return
2444 # Compute the local ignore list for this file.
2445 ignore = tuple(delim1 + z for z in self.ignore)
2446 # Handle each line of the file.
2447 nodes: List[Any] = [] # An list of Nodes, in file order.
2448 stack: List[Any] = [] # A stack of Nodes.
2449 for line in g.splitLines(s):
2450 s = line.lstrip()
2451 lws = line[: len(line) - len(line.lstrip())]
2452 if s.startswith(delim1 + '@@'):
2453 self.add(lws + s[2:], stack)
2454 elif s.startswith(ignore):
2455 # Ignore these. Use comments instead of @doc bodies.
2456 pass
2457 elif (
2458 s.startswith(delim1 + '@+others') or
2459 s.startswith(delim1 + '@' + lws + '@+others')
2460 ):
2461 self.add(lws + '@others\n', stack)
2462 elif s.startswith(delim1 + '@<<'):
2463 n = len(delim1 + '@<<')
2464 self.add(lws + '<<' + s[n:].rstrip() + '\n', stack)
2465 elif s.startswith(delim1 + '@+node:'):
2466 # Compute the headline.
2467 if stack:
2468 h = s[8:]
2469 i = h.find(':')
2470 h = h[i + 1 :] if ':' in h else h
2471 else:
2472 h = root_h
2473 # Create a node and push it.
2474 node = self.Node(h, len(stack))
2475 nodes.append(node)
2476 stack.append(node)
2477 elif s.startswith(delim1 + '@-node'):
2478 # End the node.
2479 stack.pop()
2480 elif s.startswith(delim1 + '@'):
2481 print('oops:', repr(s))
2482 else:
2483 self.add(line, stack)
2484 if stack:
2485 print('Unbalanced node sentinels')
2486 # Generate nodes.
2487 last = c.lastTopLevel()
2488 root = last.insertAfter()
2489 root.h = f"imported file: {root_h}"
2490 stack = [root]
2491 for node in nodes:
2492 b = textwrap.dedent(''.join(node.lines))
2493 level = node.level
2494 if level == 0:
2495 root.h = root_h
2496 root.b = b
2497 else:
2498 parent = stack[level - 1]
2499 p = parent.insertAsLastChild()
2500 p.b = b
2501 p.h = node.h
2502 # Good for debugging.
2503 # p.h = f"{level} {node.h}"
2504 stack = stack[:level] + [p]
2505 c.selectPosition(root)
2506 root.expand() # c.expandAllSubheads()
2507 c.redraw()
2508 #@+node:ekr.20200424154553.1: *3* legacy.import_files
2509 def import_files(self, paths):
2510 """Import zero or more files."""
2511 for path in paths:
2512 if os.path.exists(path):
2513 self.import_file(path)
2514 else:
2515 g.es_print(f"not found: {path!r}")
2516 #@+node:ekr.20200424154416.1: *3* legacy.prompt_for_files
2517 def prompt_for_files(self):
2518 """Prompt for a list of legacy external .py files and import them."""
2519 c = self.c
2520 types = [
2521 ("Legacy external files", "*.py"),
2522 ("All files", "*"),
2523 ]
2524 paths = g.app.gui.runOpenFileDialog(c,
2525 title="Import Legacy External Files",
2526 filetypes=types,
2527 defaultextension=".py",
2528 multiple=True)
2529 c.bringToFront()
2530 if paths:
2531 g.chdir(paths[0])
2532 self.import_files(paths)
2533 #@-others
2534#@+node:ekr.20101103093942.5938: ** Commands (leoImport)
2535#@+node:ekr.20160504050255.1: *3* @g.command(import-free-mind-files)
2536@g.command('import-free-mind-files')
2537def import_free_mind_files(event):
2538 """Prompt for free-mind files and import them."""
2539 c = event.get('c')
2540 if c:
2541 FreeMindImporter(c).prompt_for_files()
2543#@+node:ekr.20200424154303.1: *3* @g.command(import-legacy-external-file)
2544@g.command('import-legacy-external-files')
2545def import_legacy_external_files(event):
2546 """Prompt for legacy external files and import them."""
2547 c = event.get('c')
2548 if c:
2549 LegacyExternalFileImporter(c).prompt_for_files()
2550#@+node:ekr.20160504050325.1: *3* @g.command(import-mind-map-files
2551@g.command('import-mind-jet-files')
2552def import_mind_jet_files(event):
2553 """Prompt for mind-jet files and import them."""
2554 c = event.get('c')
2555 if c:
2556 MindMapImporter(c).prompt_for_files()
2557#@+node:ekr.20161006100854.1: *3* @g.command(import-MORE-files)
2558@g.command('import-MORE-files')
2559def import_MORE_files_command(event):
2560 """Prompt for MORE files and import them."""
2561 c = event.get('c')
2562 if c:
2563 MORE_Importer(c).prompt_for_files()
2564#@+node:ekr.20161006072227.1: *3* @g.command(import-tabbed-files)
2565@g.command('import-tabbed-files')
2566def import_tabbed_files_command(event):
2567 """Prompt for tabbed files and import them."""
2568 c = event.get('c')
2569 if c:
2570 TabImporter(c).prompt_for_files()
2571#@+node:ekr.20200310095703.1: *3* @g.command(import-todo-text-files)
2572@g.command('import-todo-text-files')
2573def import_todo_text_files(event):
2574 """Prompt for free-mind files and import them."""
2575 c = event.get('c')
2576 if c:
2577 ToDoImporter(c).prompt_for_files()
2578#@+node:ekr.20141210051628.33: *3* @g.command(import-zim-folder)
2579@g.command('import-zim-folder')
2580def import_zim_command(event):
2581 """
2582 Import a zim folder, http://zim-wiki.org/, as the last top-level node of the outline.
2584 First use Zim to export your project to rst files.
2586 This command requires the following Leo settings::
2588 @int rst_level = 0
2589 @string rst_type
2590 @string zim_node_name
2591 @string path_to_zim
2592 """
2593 c = event.get('c')
2594 if c:
2595 ZimImportController(c).run()
2596#@+node:ekr.20120429125741.10057: *3* @g.command(parse-body)
2597@g.command('parse-body')
2598def parse_body_command(event):
2599 """The parse-body command."""
2600 c = event.get('c')
2601 if c and c.p:
2602 c.importCommands.parse_body(c.p)
2603#@-others
2604#@@language python
2605#@@tabwidth -4
2606#@@pagewidth 70
2607#@@encoding utf-8
2608#@-leo