Coverage for C:\leo.repo\leo-editor\leo\core\leoAtFile.py : 94%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2#@+leo-ver=5-thin
3#@+node:ekr.20150323150718.1: * @file leoAtFile.py
4#@@first
5"""Classes to read and write @file nodes."""
6#@+<< imports >>
7#@+node:ekr.20041005105605.2: ** << imports >> (leoAtFile.py)
8import io
9import os
10import re
11import sys
12import tabnanny
13import time
14import tokenize
15from typing import List
16from leo.core import leoGlobals as g
17from leo.core import leoNodes
18#@-<< imports >>
19#@+others
20#@+node:ekr.20150509194251.1: ** cmd (decorator)
21def cmd(name): # pragma: no cover
22 """Command decorator for the AtFileCommands class."""
23 return g.new_cmd_decorator(name, ['c', 'atFileCommands',])
24#@+node:ekr.20160514120655.1: ** class AtFile
25class AtFile:
26 """A class implementing the atFile subcommander."""
27 #@+<< define class constants >>
28 #@+node:ekr.20131224053735.16380: *3* << define class constants >>
29 #@@nobeautify
31 # directives...
32 noDirective = 1 # not an at-directive.
33 allDirective = 2 # at-all (4.2)
34 docDirective = 3 # @doc.
35 atDirective = 4 # @<space> or @<newline>
36 codeDirective = 5 # @code
37 cDirective = 6 # @c<space> or @c<newline>
38 othersDirective = 7 # at-others
39 miscDirective = 8 # All other directives
40 startVerbatim = 9 # @verbatim Not a real directive. Used to issue warnings.
41 #@-<< define class constants >>
42 #@+others
43 #@+node:ekr.20041005105605.7: *3* at.Birth & init
44 #@+node:ekr.20041005105605.8: *4* at.ctor & helpers
45 # Note: g.getScript also call the at.__init__ and at.finishCreate().
47 def __init__(self, c):
48 """ctor for atFile class."""
49 # **Warning**: all these ivars must **also** be inited in initCommonIvars.
50 self.c = c
51 self.encoding = 'utf-8' # 2014/08/13
52 self.fileCommands = c.fileCommands
53 self.errors = 0 # Make sure at.error() works even when not inited.
54 # #2276: allow different section delims.
55 self.section_delim1 = '<<'
56 self.section_delim2 = '>>'
57 # **Only** at.writeAll manages these flags.
58 self.unchangedFiles = 0
59 # promptForDangerousWrite sets cancelFlag and yesToAll only if canCancelFlag is True.
60 self.canCancelFlag = False
61 self.cancelFlag = False
62 self.yesToAll = False
63 # User options: set in reloadSettings.
64 self.checkPythonCodeOnWrite = False
65 self.runPyFlakesOnWrite = False
66 self.underindentEscapeString = '\\-'
67 self.reloadSettings()
68 #@+node:ekr.20171113152939.1: *5* at.reloadSettings
69 def reloadSettings(self):
70 """AtFile.reloadSettings"""
71 c = self.c
72 self.checkPythonCodeOnWrite = c.config.getBool(
73 'check-python-code-on-write', default=True)
74 self.runPyFlakesOnWrite = c.config.getBool(
75 'run-pyflakes-on-write', default=False)
76 self.underindentEscapeString = c.config.getString(
77 'underindent-escape-string') or '\\-'
78 #@+node:ekr.20041005105605.10: *4* at.initCommonIvars
79 def initCommonIvars(self):
80 """
81 Init ivars common to both reading and writing.
83 The defaults set here may be changed later.
84 """
85 at = self
86 c = at.c
87 at.at_auto_encoding = c.config.default_at_auto_file_encoding
88 at.encoding = c.config.default_derived_file_encoding
89 at.endSentinelComment = ""
90 at.errors = 0
91 at.inCode = True
92 at.indent = 0 # The unit of indentation is spaces, not tabs.
93 at.language = None
94 at.output_newline = g.getOutputNewline(c=c)
95 at.page_width = None
96 at.root = None # The root (a position) of tree being read or written.
97 at.startSentinelComment = ""
98 at.startSentinelComment = ""
99 at.tab_width = c.tab_width or -4
100 at.writing_to_shadow_directory = False
101 #@+node:ekr.20041005105605.13: *4* at.initReadIvars
102 def initReadIvars(self, root, fileName):
103 at = self
104 at.initCommonIvars()
105 at.bom_encoding = None # The encoding implied by any BOM (set by g.stripBOM)
106 at.cloneSibCount = 0 # n > 1: Make sure n cloned sibs exists at next @+node sentinel
107 at.correctedLines = 0 # For perfect import.
108 at.docOut = [] # The doc part being accumulated.
109 at.done = False # True when @-leo seen.
110 at.fromString = False
111 at.importRootSeen = False
112 at.indentStack = []
113 at.lastLines = [] # The lines after @-leo
114 at.leadingWs = ""
115 at.lineNumber = 0 # New in Leo 4.4.8.
116 at.out = None
117 at.outStack = []
118 at.read_i = 0
119 at.read_lines = []
120 at.readVersion = '' # "5" for new-style thin files.
121 at.readVersion5 = False # Synonym for at.readVersion >= '5'
122 at.root = root
123 at.rootSeen = False
124 at.targetFileName = fileName # For at.writeError only.
125 at.v = None
126 at.vStack = [] # Stack of at.v values.
127 at.thinChildIndexStack = [] # number of siblings at this level.
128 at.thinNodeStack = [] # Entries are vnodes.
129 at.updateWarningGiven = False
130 #@+node:ekr.20041005105605.15: *4* at.initWriteIvars
131 def initWriteIvars(self, root):
132 """
133 Compute default values of all write-related ivars.
134 Return the finalized name of the output file.
135 """
136 at, c = self, self.c
137 if not c and c.config:
138 return None # pragma: no cover
139 make_dirs = c.config.create_nonexistent_directories
140 assert root
141 self.initCommonIvars()
142 assert at.checkPythonCodeOnWrite is not None
143 assert at.underindentEscapeString is not None
144 #
145 # Copy args
146 at.root = root
147 at.sentinels = True
148 #
149 # Override initCommonIvars.
150 if g.unitTesting:
151 at.output_newline = '\n'
152 #
153 # Set other ivars.
154 at.force_newlines_in_at_nosent_bodies = c.config.getBool(
155 'force-newlines-in-at-nosent-bodies')
156 # For at.putBody only.
157 at.outputList = []
158 # For stream output.
159 at.scanAllDirectives(root)
160 # Sets the following ivars:
161 # at.encoding
162 # at.explicitLineEnding
163 # at.language
164 # at.output_newline
165 # at.page_width
166 # at.tab_width
167 #
168 # Overrides of at.scanAllDirectives...
169 if at.language == 'python':
170 # Encoding directive overrides everything else.
171 encoding = g.getPythonEncodingFromString(root.b)
172 if encoding:
173 at.encoding = encoding
174 #
175 # Clean root.v.
176 if not at.errors and at.root:
177 at.root.v._p_changed = True
178 #
179 # #1907: Compute the file name and create directories as needed.
180 targetFileName = g.os_path_realpath(g.fullPath(c, root))
181 at.targetFileName = targetFileName # For at.writeError only.
182 #
183 # targetFileName can be empty for unit tests & @command nodes.
184 if not targetFileName: # pragma: no cover
185 targetFileName = root.h if g.unitTesting else None
186 at.targetFileName = targetFileName # For at.writeError only.
187 return targetFileName
188 #
189 # #2276: scan for section delims
190 at.scanRootForSectionDelims(root)
191 #
192 # Do nothing more if the file already exists.
193 if os.path.exists(targetFileName):
194 return targetFileName
195 #
196 # Create directories if enabled.
197 root_dir = g.os_path_dirname(targetFileName)
198 if make_dirs and root_dir: # pragma: no cover
199 ok = g.makeAllNonExistentDirectories(root_dir)
200 if not ok:
201 g.error(f"Error creating directories: {root_dir}")
202 return None
203 #
204 # Return the target file name, regardless of future problems.
205 return targetFileName
206 #@+node:ekr.20041005105605.17: *3* at.Reading
207 #@+node:ekr.20041005105605.18: *4* at.Reading (top level)
208 #@+node:ekr.20070919133659: *5* at.checkExternalFile
209 @cmd('check-external-file')
210 def checkExternalFile(self, event=None): # pragma: no cover
211 """Make sure an external file written by Leo may be read properly."""
212 c, p = self.c, self.c.p
213 if not p.isAtFileNode() and not p.isAtThinFileNode():
214 g.red('Please select an @thin or @file node')
215 return
216 fn = g.fullPath(c, p) # #1910.
217 if not g.os_path_exists(fn):
218 g.red(f"file not found: {fn}")
219 return
220 s, e = g.readFileIntoString(fn)
221 if s is None:
222 g.red(f"empty file: {fn}")
223 return
224 #
225 # Create a dummy, unconnected, VNode as the root.
226 root_v = leoNodes.VNode(context=c)
227 root = leoNodes.Position(root_v)
228 FastAtRead(c, gnx2vnode={}).read_into_root(s, fn, root)
229 #@+node:ekr.20041005105605.19: *5* at.openFileForReading & helper
230 def openFileForReading(self, fromString=False):
231 """
232 Open the file given by at.root.
233 This will be the private file for @shadow nodes.
234 """
235 at, c = self, self.c
236 is_at_shadow = self.root.isAtShadowFileNode()
237 if fromString: # pragma: no cover
238 if is_at_shadow: # pragma: no cover
239 return at.error(
240 'can not call at.read from string for @shadow files')
241 at.initReadLine(fromString)
242 return None, None
243 #
244 # Not from a string. Carefully read the file.
245 # Returns full path, including file name.
246 fn = g.fullPath(c, at.root)
247 # Remember the full path to this node.
248 at.setPathUa(at.root, fn)
249 if is_at_shadow: # pragma: no cover
250 fn = at.openAtShadowFileForReading(fn)
251 if not fn:
252 return None, None
253 assert fn
254 try:
255 # Sets at.encoding, regularizes whitespace and calls at.initReadLines.
256 s = at.readFileToUnicode(fn)
257 # #1466.
258 if s is None: # pragma: no cover
259 # The error has been given.
260 at._file_bytes = g.toEncodedString('')
261 return None, None
262 at.warnOnReadOnlyFile(fn)
263 except Exception:
264 at.error(f"unexpected exception opening: '@file {fn}'")
265 at._file_bytes = g.toEncodedString('')
266 fn, s = None, None
267 return fn, s
268 #@+node:ekr.20150204165040.4: *6* at.openAtShadowFileForReading
269 def openAtShadowFileForReading(self, fn): # pragma: no cover
270 """Open an @shadow for reading and return shadow_fn."""
271 at = self
272 x = at.c.shadowController
273 # readOneAtShadowNode should already have checked these.
274 shadow_fn = x.shadowPathName(fn)
275 shadow_exists = (g.os_path_exists(shadow_fn) and g.os_path_isfile(shadow_fn))
276 if not shadow_exists:
277 g.trace('can not happen: no private file',
278 shadow_fn, g.callers())
279 at.error(f"can not happen: private file does not exist: {shadow_fn}")
280 return None
281 # This method is the gateway to the shadow algorithm.
282 x.updatePublicAndPrivateFiles(at.root, fn, shadow_fn)
283 return shadow_fn
284 #@+node:ekr.20041005105605.21: *5* at.read & helpers
285 def read(self, root, fromString=None):
286 """Read an @thin or @file tree."""
287 at, c = self, self.c
288 fileName = g.fullPath(c, root) # #1341. #1889.
289 if not fileName: # pragma: no cover
290 at.error("Missing file name. Restoring @file tree from .leo file.")
291 return False
292 # Fix bug 760531: always mark the root as read, even if there was an error.
293 # Fix bug 889175: Remember the full fileName.
294 at.rememberReadPath(g.fullPath(c, root), root)
295 at.initReadIvars(root, fileName)
296 at.fromString = fromString
297 if at.errors:
298 return False # pragma: no cover
299 fileName, file_s = at.openFileForReading(fromString=fromString)
300 # #1798:
301 if file_s is None:
302 return False # pragma: no cover
303 #
304 # Set the time stamp.
305 if fileName:
306 c.setFileTimeStamp(fileName)
307 elif not fileName and not fromString and not file_s: # pragma: no cover
308 return False
309 root.clearVisitedInTree()
310 at.scanAllDirectives(root)
311 # Sets the following ivars:
312 # at.encoding: **changed later** by readOpenFile/at.scanHeader.
313 # at.explicitLineEnding
314 # at.language
315 # at.output_newline
316 # at.page_width
317 # at.tab_width
318 gnx2vnode = c.fileCommands.gnxDict
319 contents = fromString or file_s
320 FastAtRead(c, gnx2vnode).read_into_root(contents, fileName, root)
321 root.clearDirty()
322 return True
323 #@+node:ekr.20071105164407: *6* at.deleteUnvisitedNodes
324 def deleteUnvisitedNodes(self, root): # pragma: no cover
325 """
326 Delete unvisited nodes in root's subtree, not including root.
328 Before Leo 5.6: Move unvisited node to be children of the 'Resurrected
329 Nodes'.
330 """
331 at, c = self, self.c
332 # Find the unvisited nodes.
333 aList = [z for z in root.subtree() if not z.isVisited()]
334 if aList:
335 at.c.deletePositionsInList(aList)
336 c.redraw()
338 #@+node:ekr.20041005105605.26: *5* at.readAll & helpers
339 def readAll(self, root):
340 """Scan positions, looking for @<file> nodes to read."""
341 at, c = self, self.c
342 old_changed = c.changed
343 t1 = time.time()
344 c.init_error_dialogs()
345 files = at.findFilesToRead(root, all=True)
346 for p in files:
347 at.readFileAtPosition(p)
348 for p in files:
349 p.v.clearDirty()
350 if not g.unitTesting and files:
351 t2 = time.time()
352 g.es(f"read {len(files)} files in {t2 - t1:2.2f} seconds")
353 c.changed = old_changed
354 c.raise_error_dialogs()
355 #@+node:ekr.20190108054317.1: *6* at.findFilesToRead
356 def findFilesToRead(self, root, all): # pragma: no cover
358 c = self.c
359 p = root.copy()
360 scanned_nodes = set()
361 files = []
362 after = None if all else p.nodeAfterTree()
363 while p and p != after:
364 data = (p.gnx, g.fullPath(c, p))
365 # skip clones referring to exactly the same paths.
366 if data in scanned_nodes:
367 p.moveToNodeAfterTree()
368 continue
369 scanned_nodes.add(data)
370 if not p.h.startswith('@'):
371 p.moveToThreadNext()
372 elif p.isAtIgnoreNode():
373 if p.isAnyAtFileNode():
374 c.ignored_at_file_nodes.append(p.h)
375 p.moveToNodeAfterTree()
376 elif (
377 p.isAtThinFileNode() or
378 p.isAtAutoNode() or
379 p.isAtEditNode() or
380 p.isAtShadowFileNode() or
381 p.isAtFileNode() or
382 p.isAtCleanNode() # 1134.
383 ):
384 files.append(p.copy())
385 p.moveToNodeAfterTree()
386 elif p.isAtAsisFileNode() or p.isAtNoSentFileNode():
387 # Note (see #1081): @asis and @nosent can *not* be updated automatically.
388 # Doing so using refresh-from-disk will delete all child nodes.
389 p.moveToNodeAfterTree()
390 else:
391 p.moveToThreadNext()
392 return files
393 #@+node:ekr.20190108054803.1: *6* at.readFileAtPosition
394 def readFileAtPosition(self, p): # pragma: no cover
395 """Read the @<file> node at p."""
396 at, c, fileName = self, self.c, p.anyAtFileNodeName()
397 if p.isAtThinFileNode() or p.isAtFileNode():
398 at.read(p)
399 elif p.isAtAutoNode():
400 at.readOneAtAutoNode(p)
401 elif p.isAtEditNode():
402 at.readOneAtEditNode(fileName, p)
403 elif p.isAtShadowFileNode():
404 at.readOneAtShadowNode(fileName, p)
405 elif p.isAtAsisFileNode() or p.isAtNoSentFileNode():
406 at.rememberReadPath(g.fullPath(c, p), p)
407 elif p.isAtCleanNode():
408 at.readOneAtCleanNode(p)
409 #@+node:ekr.20220121052056.1: *5* at.readAllSelected
410 def readAllSelected(self, root):
411 """Read all @<file> nodes in root's tree."""
412 at, c = self, self.c
413 old_changed = c.changed
414 t1 = time.time()
415 c.init_error_dialogs()
416 files = at.findFilesToRead(root, all=False)
417 for p in files:
418 at.readFileAtPosition(p)
419 for p in files:
420 p.v.clearDirty()
421 if not g.unitTesting: # pragma: no cover
422 if files:
423 t2 = time.time()
424 g.es(f"read {len(files)} files in {t2 - t1:2.2f} seconds")
425 else:
426 g.es("no @<file> nodes in the selected tree")
427 c.changed = old_changed
428 c.raise_error_dialogs()
429 #@+node:ekr.20080801071227.7: *5* at.readAtShadowNodes
430 def readAtShadowNodes(self, p): # pragma: no cover
431 """Read all @shadow nodes in the p's tree."""
432 at = self
433 after = p.nodeAfterTree()
434 p = p.copy() # Don't change p in the caller.
435 while p and p != after: # Don't use iterator.
436 if p.isAtShadowFileNode():
437 fileName = p.atShadowFileNodeName()
438 at.readOneAtShadowNode(fileName, p)
439 p.moveToNodeAfterTree()
440 else:
441 p.moveToThreadNext()
442 #@+node:ekr.20070909100252: *5* at.readOneAtAutoNode
443 def readOneAtAutoNode(self, p): # pragma: no cover
444 """Read an @auto file into p. Return the *new* position."""
445 at, c, ic = self, self.c, self.c.importCommands
446 fileName = g.fullPath(c, p) # #1521, #1341, #1914.
447 if not g.os_path_exists(fileName):
448 g.error(f"not found: {p.h!r}", nodeLink=p.get_UNL())
449 return p
450 # Remember that we have seen the @auto node.
451 # Fix bug 889175: Remember the full fileName.
452 at.rememberReadPath(fileName, p)
453 # if not g.unitTesting: g.es("reading:", p.h)
454 try:
455 # For #451: return p.
456 old_p = p.copy()
457 at.scanAllDirectives(p)
458 p.v.b = '' # Required for @auto API checks.
459 p.v._deleteAllChildren()
460 p = ic.createOutline(parent=p.copy())
461 # Do *not* select a position here.
462 # That would improperly expand nodes.
463 # c.selectPosition(p)
464 except Exception:
465 p = old_p
466 ic.errors += 1
467 g.es_print('Unexpected exception importing', fileName)
468 g.es_exception()
469 if ic.errors:
470 g.error(f"errors inhibited read @auto {fileName}")
471 elif c.persistenceController:
472 c.persistenceController.update_after_read_foreign_file(p)
473 # Finish.
474 if ic.errors or not g.os_path_exists(fileName):
475 p.clearDirty()
476 else:
477 g.doHook('after-auto', c=c, p=p)
478 return p
479 #@+node:ekr.20090225080846.3: *5* at.readOneAtEditNode
480 def readOneAtEditNode(self, fn, p): # pragma: no cover
481 at = self
482 c = at.c
483 ic = c.importCommands
484 # #1521
485 fn = g.fullPath(c, p)
486 junk, ext = g.os_path_splitext(fn)
487 # Fix bug 889175: Remember the full fileName.
488 at.rememberReadPath(fn, p)
489 # if not g.unitTesting: g.es("reading: @edit %s" % (g.shortFileName(fn)))
490 s, e = g.readFileIntoString(fn, kind='@edit')
491 if s is None:
492 return
493 encoding = 'utf-8' if e is None else e
494 # Delete all children.
495 while p.hasChildren():
496 p.firstChild().doDelete()
497 head = ''
498 ext = ext.lower()
499 if ext in ('.html', '.htm'):
500 head = '@language html\n'
501 elif ext in ('.txt', '.text'):
502 head = '@nocolor\n'
503 else:
504 language = ic.languageForExtension(ext)
505 if language and language != 'unknown_language':
506 head = f"@language {language}\n"
507 else:
508 head = '@nocolor\n'
509 p.b = head + g.toUnicode(s, encoding=encoding, reportErrors=True)
510 g.doHook('after-edit', p=p)
511 #@+node:ekr.20190201104956.1: *5* at.readOneAtAsisNode
512 def readOneAtAsisNode(self, fn, p): # pragma: no cover
513 """Read one @asis node. Used only by refresh-from-disk"""
514 at, c = self, self.c
515 # #1521 & #1341.
516 fn = g.fullPath(c, p)
517 junk, ext = g.os_path_splitext(fn)
518 # Remember the full fileName.
519 at.rememberReadPath(fn, p)
520 # if not g.unitTesting: g.es("reading: @asis %s" % (g.shortFileName(fn)))
521 s, e = g.readFileIntoString(fn, kind='@edit')
522 if s is None:
523 return
524 encoding = 'utf-8' if e is None else e
525 # Delete all children.
526 while p.hasChildren():
527 p.firstChild().doDelete()
528 old_body = p.b
529 p.b = g.toUnicode(s, encoding=encoding, reportErrors=True)
530 if not c.isChanged() and p.b != old_body:
531 c.setChanged()
532 #@+node:ekr.20150204165040.5: *5* at.readOneAtCleanNode & helpers
533 def readOneAtCleanNode(self, root): # pragma: no cover
534 """Update the @clean/@nosent node at root."""
535 at, c, x = self, self.c, self.c.shadowController
536 fileName = g.fullPath(c, root)
537 if not g.os_path_exists(fileName):
538 g.es_print(f"not found: {fileName}", color='red', nodeLink=root.get_UNL())
539 return False
540 at.rememberReadPath(fileName, root)
541 at.initReadIvars(root, fileName)
542 # Must be called before at.scanAllDirectives.
543 at.scanAllDirectives(root)
544 # Sets at.startSentinelComment/endSentinelComment.
545 new_public_lines = at.read_at_clean_lines(fileName)
546 old_private_lines = self.write_at_clean_sentinels(root)
547 marker = x.markerFromFileLines(old_private_lines, fileName)
548 old_public_lines, junk = x.separate_sentinels(old_private_lines, marker)
549 if old_public_lines:
550 new_private_lines = x.propagate_changed_lines(
551 new_public_lines, old_private_lines, marker, p=root)
552 else:
553 new_private_lines = []
554 root.b = ''.join(new_public_lines)
555 return True
556 if new_private_lines == old_private_lines:
557 return True
558 if not g.unitTesting:
559 g.es("updating:", root.h)
560 root.clearVisitedInTree()
561 gnx2vnode = at.fileCommands.gnxDict
562 contents = ''.join(new_private_lines)
563 FastAtRead(c, gnx2vnode).read_into_root(contents, fileName, root)
564 return True # Errors not detected.
565 #@+node:ekr.20150204165040.7: *6* at.dump_lines
566 def dump(self, lines, tag): # pragma: no cover
567 """Dump all lines."""
568 print(f"***** {tag} lines...\n")
569 for s in lines:
570 print(s.rstrip())
571 #@+node:ekr.20150204165040.8: *6* at.read_at_clean_lines
572 def read_at_clean_lines(self, fn): # pragma: no cover
573 """Return all lines of the @clean/@nosent file at fn."""
574 at = self
575 s = at.openFileHelper(fn)
576 # Use the standard helper. Better error reporting.
577 # Important: uses 'rb' to open the file.
578 # #1798.
579 if s is None:
580 s = ''
581 else:
582 s = g.toUnicode(s, encoding=at.encoding)
583 s = s.replace('\r\n', '\n')
584 # Suppress meaningless "node changed" messages.
585 return g.splitLines(s)
586 #@+node:ekr.20150204165040.9: *6* at.write_at_clean_sentinels
587 def write_at_clean_sentinels(self, root): # pragma: no cover
588 """
589 Return all lines of the @clean tree as if it were
590 written as an @file node.
591 """
592 at = self
593 result = at.atFileToString(root, sentinels=True)
594 s = g.toUnicode(result, encoding=at.encoding)
595 return g.splitLines(s)
596 #@+node:ekr.20080711093251.7: *5* at.readOneAtShadowNode & helper
597 def readOneAtShadowNode(self, fn, p): # pragma: no cover
599 at, c = self, self.c
600 x = c.shadowController
601 if not fn == p.atShadowFileNodeName():
602 at.error(
603 f"can not happen: fn: {fn} != atShadowNodeName: "
604 f"{p.atShadowFileNodeName()}")
605 return
606 fn = g.fullPath(c, p) # #1521 & #1341.
607 # #889175: Remember the full fileName.
608 at.rememberReadPath(fn, p)
609 shadow_fn = x.shadowPathName(fn)
610 shadow_exists = g.os_path_exists(shadow_fn) and g.os_path_isfile(shadow_fn)
611 # Delete all children.
612 while p.hasChildren():
613 p.firstChild().doDelete()
614 if shadow_exists:
615 at.read(p)
616 else:
617 ok = at.importAtShadowNode(p)
618 if ok:
619 # Create the private file automatically.
620 at.writeOneAtShadowNode(p)
621 #@+node:ekr.20080712080505.1: *6* at.importAtShadowNode
622 def importAtShadowNode(self, p): # pragma: no cover
623 c, ic = self.c, self.c.importCommands
624 fn = g.fullPath(c, p) # #1521, #1341, #1914.
625 if not g.os_path_exists(fn):
626 g.error(f"not found: {p.h!r}", nodeLink=p.get_UNL())
627 return p
628 # Delete all the child nodes.
629 while p.hasChildren():
630 p.firstChild().doDelete()
631 # Import the outline, exactly as @auto does.
632 ic.createOutline(parent=p.copy())
633 if ic.errors:
634 g.error('errors inhibited read @shadow', fn)
635 if ic.errors or not g.os_path_exists(fn):
636 p.clearDirty()
637 return ic.errors == 0
638 #@+node:ekr.20180622110112.1: *4* at.fast_read_into_root
639 def fast_read_into_root(self, c, contents, gnx2vnode, path, root): # pragma: no cover
640 """A convenience wrapper for FastAtRead.read_into_root()"""
641 return FastAtRead(c, gnx2vnode).read_into_root(contents, path, root)
642 #@+node:ekr.20041005105605.116: *4* at.Reading utils...
643 #@+node:ekr.20041005105605.119: *5* at.createImportedNode
644 def createImportedNode(self, root, headline): # pragma: no cover
645 at = self
646 if at.importRootSeen:
647 p = root.insertAsLastChild()
648 p.initHeadString(headline)
649 else:
650 # Put the text into the already-existing root node.
651 p = root
652 at.importRootSeen = True
653 p.v.setVisited() # Suppress warning about unvisited node.
654 return p
655 #@+node:ekr.20130911110233.11286: *5* at.initReadLine
656 def initReadLine(self, s):
657 """Init the ivars so that at.readLine will read all of s."""
658 at = self
659 at.read_i = 0
660 at.read_lines = g.splitLines(s)
661 at._file_bytes = g.toEncodedString(s)
662 #@+node:ekr.20041005105605.120: *5* at.parseLeoSentinel
663 def parseLeoSentinel(self, s):
664 """
665 Parse the sentinel line s.
666 If the sentinel is valid, set at.encoding, at.readVersion, at.readVersion5.
667 """
668 at, c = self, self.c
669 # Set defaults.
670 encoding = c.config.default_derived_file_encoding
671 readVersion, readVersion5 = None, None
672 new_df, start, end, isThin = False, '', '', False
673 # Example: \*@+leo-ver=5-thin-encoding=utf-8,.*/
674 pattern = re.compile(
675 r'(.+)@\+leo(-ver=([0123456789]+))?(-thin)?(-encoding=(.*)(\.))?(.*)')
676 # The old code weirdly allowed '.' in version numbers.
677 # group 1: opening delim
678 # group 2: -ver=
679 # group 3: version number
680 # group(4): -thin
681 # group(5): -encoding=utf-8,.
682 # group(6): utf-8,
683 # group(7): .
684 # group(8): closing delim.
685 m = pattern.match(s)
686 valid = bool(m)
687 if valid:
688 start = m.group(1) # start delim
689 valid = bool(start)
690 if valid:
691 new_df = bool(m.group(2)) # -ver=
692 if new_df:
693 # Set the version number.
694 if m.group(3):
695 readVersion = m.group(3)
696 readVersion5 = readVersion >= '5'
697 else:
698 valid = False # pragma: no cover
699 if valid:
700 # set isThin
701 isThin = bool(m.group(4))
702 if valid and m.group(5):
703 # set encoding.
704 encoding = m.group(6)
705 if encoding and encoding.endswith(','):
706 # Leo 4.2 or after.
707 encoding = encoding[:-1]
708 if not g.isValidEncoding(encoding): # pragma: no cover
709 g.es_print("bad encoding in derived file:", encoding)
710 valid = False
711 if valid:
712 end = m.group(8) # closing delim
713 if valid:
714 at.encoding = encoding
715 at.readVersion = readVersion
716 at.readVersion5 = readVersion5
717 return valid, new_df, start, end, isThin
718 #@+node:ekr.20130911110233.11284: *5* at.readFileToUnicode & helpers
719 def readFileToUnicode(self, fileName): # pragma: no cover
720 """
721 Carefully sets at.encoding, then uses at.encoding to convert the file
722 to a unicode string.
724 Sets at.encoding as follows:
725 1. Use the BOM, if present. This unambiguously determines the encoding.
726 2. Use the -encoding= field in the @+leo header, if present and valid.
727 3. Otherwise, uses existing value of at.encoding, which comes from:
728 A. An @encoding directive, found by at.scanAllDirectives.
729 B. The value of c.config.default_derived_file_encoding.
731 Returns the string, or None on failure.
732 """
733 at = self
734 s = at.openFileHelper(fileName)
735 # Catches all exceptions.
736 # #1798.
737 if s is None:
738 return None
739 e, s = g.stripBOM(s)
740 if e:
741 # The BOM determines the encoding unambiguously.
742 s = g.toUnicode(s, encoding=e)
743 else:
744 # Get the encoding from the header, or the default encoding.
745 s_temp = g.toUnicode(s, 'ascii', reportErrors=False)
746 e = at.getEncodingFromHeader(fileName, s_temp)
747 s = g.toUnicode(s, encoding=e)
748 s = s.replace('\r\n', '\n')
749 at.encoding = e
750 at.initReadLine(s)
751 return s
752 #@+node:ekr.20130911110233.11285: *6* at.openFileHelper
753 def openFileHelper(self, fileName):
754 """Open a file, reporting all exceptions."""
755 at = self
756 # #1798: return None as a flag on any error.
757 s = None
758 try:
759 with open(fileName, 'rb') as f:
760 s = f.read()
761 except IOError:
762 at.error(f"can not open {fileName}")
763 except Exception:
764 at.error(f"Exception reading {fileName}")
765 g.es_exception()
766 return s
767 #@+node:ekr.20130911110233.11287: *6* at.getEncodingFromHeader
768 def getEncodingFromHeader(self, fileName, s):
769 """
770 Return the encoding given in the @+leo sentinel, if the sentinel is
771 present, or the previous value of at.encoding otherwise.
772 """
773 at = self
774 if at.errors: # pragma: no cover
775 g.trace('can not happen: at.errors > 0', g.callers())
776 e = at.encoding
777 if g.unitTesting:
778 assert False, g.callers()
779 else:
780 at.initReadLine(s)
781 old_encoding = at.encoding
782 assert old_encoding
783 at.encoding = None
784 # Execute scanHeader merely to set at.encoding.
785 at.scanHeader(fileName, giveErrors=False)
786 e = at.encoding or old_encoding
787 assert e
788 return e
789 #@+node:ekr.20041005105605.128: *5* at.readLine
790 def readLine(self):
791 """
792 Read one line from file using the present encoding.
793 Returns at.read_lines[at.read_i++]
794 """
795 # This is an old interface, now used only by at.scanHeader.
796 # For now, it's not worth replacing.
797 at = self
798 if at.read_i < len(at.read_lines):
799 s = at.read_lines[at.read_i]
800 at.read_i += 1
801 return s
802 # Not an error.
803 return '' # pragma: no cover
804 #@+node:ekr.20041005105605.129: *5* at.scanHeader
805 def scanHeader(self, fileName, giveErrors=True):
806 """
807 Scan the @+leo sentinel, using the old readLine interface.
809 Sets self.encoding, and self.start/endSentinelComment.
811 Returns (firstLines,new_df,isThinDerivedFile) where:
812 firstLines contains all @first lines,
813 new_df is True if we are reading a new-format derived file.
814 isThinDerivedFile is True if the file is an @thin file.
815 """
816 at = self
817 new_df, isThinDerivedFile = False, False
818 firstLines: List[str] = [] # The lines before @+leo.
819 s = self.scanFirstLines(firstLines)
820 valid = len(s) > 0
821 if valid:
822 valid, new_df, start, end, isThinDerivedFile = at.parseLeoSentinel(s)
823 if valid:
824 at.startSentinelComment = start
825 at.endSentinelComment = end
826 elif giveErrors: # pragma: no cover
827 at.error(f"No @+leo sentinel in: {fileName}")
828 g.trace(g.callers())
829 return firstLines, new_df, isThinDerivedFile
830 #@+node:ekr.20041005105605.130: *6* at.scanFirstLines
831 def scanFirstLines(self, firstLines): # pragma: no cover
832 """
833 Append all lines before the @+leo line to firstLines.
835 Empty lines are ignored because empty @first directives are
836 ignored.
838 We can not call sentinelKind here because that depends on the comment
839 delimiters we set here.
840 """
841 at = self
842 s = at.readLine()
843 while s and s.find("@+leo") == -1:
844 firstLines.append(s)
845 s = at.readLine()
846 return s
847 #@+node:ekr.20050103163224: *5* at.scanHeaderForThin (import code)
848 def scanHeaderForThin(self, fileName): # pragma: no cover
849 """
850 Return true if the derived file is a thin file.
852 This is a kludgy method used only by the import code."""
853 at = self
854 at.readFileToUnicode(fileName)
855 # Sets at.encoding, regularizes whitespace and calls at.initReadLines.
856 junk, junk, isThin = at.scanHeader(None)
857 # scanHeader uses at.readline instead of its args.
858 # scanHeader also sets at.encoding.
859 return isThin
860 #@+node:ekr.20041005105605.132: *3* at.Writing
861 #@+node:ekr.20041005105605.133: *4* Writing (top level)
862 #@+node:ekr.20190111153551.1: *5* at.commands
863 #@+node:ekr.20070806105859: *6* at.writeAtAutoNodes
864 @cmd('write-at-auto-nodes')
865 def writeAtAutoNodes(self, event=None): # pragma: no cover
866 """Write all @auto nodes in the selected outline."""
867 at, c, p = self, self.c, self.c.p
868 c.init_error_dialogs()
869 after, found = p.nodeAfterTree(), False
870 while p and p != after:
871 if p.isAtAutoNode() and not p.isAtIgnoreNode():
872 ok = at.writeOneAtAutoNode(p)
873 if ok:
874 found = True
875 p.moveToNodeAfterTree()
876 else:
877 p.moveToThreadNext()
878 else:
879 p.moveToThreadNext()
880 if g.unitTesting:
881 return
882 if found:
883 g.es("finished")
884 else:
885 g.es("no @auto nodes in the selected tree")
886 c.raise_error_dialogs(kind='write')
888 #@+node:ekr.20220120072251.1: *6* at.writeDirtyAtAutoNodes
889 @cmd('write-dirty-at-auto-nodes') # pragma: no cover
890 def writeDirtyAtAutoNodes(self, event=None):
891 """Write all dirty @auto nodes in the selected outline."""
892 at, c, p = self, self.c, self.c.p
893 c.init_error_dialogs()
894 after, found = p.nodeAfterTree(), False
895 while p and p != after:
896 if p.isAtAutoNode() and not p.isAtIgnoreNode() and p.isDirty():
897 ok = at.writeOneAtAutoNode(p)
898 if ok:
899 found = True
900 p.moveToNodeAfterTree()
901 else:
902 p.moveToThreadNext()
903 else:
904 p.moveToThreadNext()
905 if g.unitTesting:
906 return
907 if found:
908 g.es("finished")
909 else:
910 g.es("no dirty @auto nodes in the selected tree")
911 c.raise_error_dialogs(kind='write')
912 #@+node:ekr.20080711093251.3: *6* at.writeAtShadowNodes
913 @cmd('write-at-shadow-nodes')
914 def writeAtShadowNodes(self, event=None): # pragma: no cover
915 """Write all @shadow nodes in the selected outline."""
916 at, c, p = self, self.c, self.c.p
917 c.init_error_dialogs()
918 after, found = p.nodeAfterTree(), False
919 while p and p != after:
920 if p.atShadowFileNodeName() and not p.isAtIgnoreNode():
921 ok = at.writeOneAtShadowNode(p)
922 if ok:
923 found = True
924 g.blue(f"wrote {p.atShadowFileNodeName()}")
925 p.moveToNodeAfterTree()
926 else:
927 p.moveToThreadNext()
928 else:
929 p.moveToThreadNext()
930 if g.unitTesting:
931 return found
932 if found:
933 g.es("finished")
934 else:
935 g.es("no @shadow nodes in the selected tree")
936 c.raise_error_dialogs(kind='write')
937 return found
939 #@+node:ekr.20220120072917.1: *6* at.writeDirtyAtShadowNodes
940 @cmd('write-dirty-at-shadow-nodes')
941 def writeDirtyAtShadowNodes(self, event=None): # pragma: no cover
942 """Write all @shadow nodes in the selected outline."""
943 at, c, p = self, self.c, self.c.p
944 c.init_error_dialogs()
945 after, found = p.nodeAfterTree(), False
946 while p and p != after:
947 if p.atShadowFileNodeName() and not p.isAtIgnoreNode() and p.isDirty():
948 ok = at.writeOneAtShadowNode(p)
949 if ok:
950 found = True
951 g.blue(f"wrote {p.atShadowFileNodeName()}")
952 p.moveToNodeAfterTree()
953 else:
954 p.moveToThreadNext()
955 else:
956 p.moveToThreadNext()
957 if g.unitTesting:
958 return found
959 if found:
960 g.es("finished")
961 else:
962 g.es("no dirty @shadow nodes in the selected tree")
963 c.raise_error_dialogs(kind='write')
964 return found
966 #@+node:ekr.20041005105605.157: *5* at.putFile
967 def putFile(self, root, fromString='', sentinels=True):
968 """Write the contents of the file to the output stream."""
969 at = self
970 s = fromString if fromString else root.v.b
971 root.clearAllVisitedInTree()
972 at.putAtFirstLines(s)
973 at.putOpenLeoSentinel("@+leo-ver=5")
974 at.putInitialComment()
975 at.putOpenNodeSentinel(root)
976 at.putBody(root, fromString=fromString)
977 # The -leo sentinel is required to handle @last.
978 at.putSentinel("@-leo")
979 root.setVisited()
980 at.putAtLastLines(s)
981 #@+node:ekr.20041005105605.147: *5* at.writeAll & helpers
982 def writeAll(self, all=False, dirty=False):
983 """Write @file nodes in all or part of the outline"""
984 at = self
985 # This is the *only* place where these are set.
986 # promptForDangerousWrite sets cancelFlag only if canCancelFlag is True.
987 at.unchangedFiles = 0
988 at.canCancelFlag = True
989 at.cancelFlag = False
990 at.yesToAll = False
991 files, root = at.findFilesToWrite(all)
992 for p in files:
993 try:
994 at.writeAllHelper(p, root)
995 except Exception:
996 at.internalWriteError(p)
997 # Make *sure* these flags are cleared for other commands.
998 at.canCancelFlag = False
999 at.cancelFlag = False
1000 at.yesToAll = False
1001 # Say the command is finished.
1002 at.reportEndOfWrite(files, all, dirty)
1003 # #2338: Never call at.saveOutlineIfPossible().
1004 #@+node:ekr.20190108052043.1: *6* at.findFilesToWrite
1005 def findFilesToWrite(self, force): # pragma: no cover
1006 """
1007 Return a list of files to write.
1008 We must do this in a prepass, so as to avoid errors later.
1009 """
1010 trace = 'save' in g.app.debug and not g.unitTesting
1011 if trace:
1012 g.trace(f"writing *{'selected' if force else 'all'}* files")
1013 c = self.c
1014 if force:
1015 # The Write @<file> Nodes command.
1016 # Write all nodes in the selected tree.
1017 root = c.p
1018 p = c.p
1019 after = p.nodeAfterTree()
1020 else:
1021 # Write dirty nodes in the entire outline.
1022 root = c.rootPosition()
1023 p = c.rootPosition()
1024 after = None
1025 seen = set()
1026 files = []
1027 while p and p != after:
1028 if p.isAtIgnoreNode() and not p.isAtAsisFileNode():
1029 # Honor @ignore in *body* text, but *not* in @asis nodes.
1030 if p.isAnyAtFileNode():
1031 c.ignored_at_file_nodes.append(p.h)
1032 p.moveToNodeAfterTree()
1033 elif p.isAnyAtFileNode():
1034 data = p.v, g.fullPath(c, p)
1035 if data in seen:
1036 if trace and force:
1037 g.trace('Already seen', p.h)
1038 else:
1039 seen.add(data)
1040 files.append(p.copy())
1041 # Don't scan nested trees???
1042 p.moveToNodeAfterTree()
1043 else:
1044 p.moveToThreadNext()
1045 # When scanning *all* nodes, we only actually write dirty nodes.
1046 if not force:
1047 files = [z for z in files if z.isDirty()]
1048 if trace:
1049 g.printObj([z.h for z in files], tag='Files to be saved')
1050 return files, root
1051 #@+node:ekr.20190108053115.1: *6* at.internalWriteError
1052 def internalWriteError(self, p): # pragma: no cover
1053 """
1054 Fix bug 1260415: https://bugs.launchpad.net/leo-editor/+bug/1260415
1055 Give a more urgent, more specific, more helpful message.
1056 """
1057 g.es_exception()
1058 g.es(f"Internal error writing: {p.h}", color='red')
1059 g.es('Please report this error to:', color='blue')
1060 g.es('https://groups.google.com/forum/#!forum/leo-editor', color='blue')
1061 g.es('Warning: changes to this file will be lost', color='red')
1062 g.es('unless you can save the file successfully.', color='red')
1063 #@+node:ekr.20190108112519.1: *6* at.reportEndOfWrite
1064 def reportEndOfWrite(self, files, all, dirty): # pragma: no cover
1066 at = self
1067 if g.unitTesting:
1068 return
1069 if files:
1070 n = at.unchangedFiles
1071 g.es(f"finished: {n} unchanged file{g.plural(n)}")
1072 elif all:
1073 g.warning("no @<file> nodes in the selected tree")
1074 elif dirty:
1075 g.es("no dirty @<file> nodes in the selected tree")
1076 #@+node:ekr.20041005105605.149: *6* at.writeAllHelper & helper
1077 def writeAllHelper(self, p, root):
1078 """
1079 Write one file for at.writeAll.
1081 Do *not* write @auto files unless p == root.
1083 This prevents the write-all command from needlessly updating
1084 the @persistence data, thereby annoyingly changing the .leo file.
1085 """
1086 at = self
1087 at.root = root
1088 if p.isAtIgnoreNode(): # pragma: no cover
1089 # Should have been handled in findFilesToWrite.
1090 g.trace(f"Can not happen: {p.h} is an @ignore node")
1091 return
1092 try:
1093 at.writePathChanged(p)
1094 except IOError:
1095 return
1096 table = (
1097 (p.isAtAsisFileNode, at.asisWrite),
1098 (p.isAtAutoNode, at.writeOneAtAutoNode),
1099 (p.isAtCleanNode, at.writeOneAtCleanNode),
1100 (p.isAtEditNode, at.writeOneAtEditNode),
1101 (p.isAtFileNode, at.writeOneAtFileNode),
1102 (p.isAtNoSentFileNode, at.writeOneAtNosentNode),
1103 (p.isAtShadowFileNode, at.writeOneAtShadowNode),
1104 (p.isAtThinFileNode, at.writeOneAtFileNode),
1105 )
1106 for pred, func in table:
1107 if pred():
1108 func(p) # type:ignore
1109 break
1110 else: # pragma: no cover
1111 g.trace(f"Can not happen: {p.h}")
1112 return
1113 #
1114 # Clear the dirty bits in all descendant nodes.
1115 # The persistence data may still have to be written.
1116 for p2 in p.self_and_subtree(copy=False):
1117 p2.v.clearDirty()
1118 #@+node:ekr.20190108105509.1: *7* at.writePathChanged
1119 def writePathChanged(self, p): # pragma: no cover
1120 """
1121 raise IOError if p's path has changed *and* user forbids the write.
1122 """
1123 at, c = self, self.c
1124 #
1125 # Suppress this message during save-as and save-to commands.
1126 if c.ignoreChangedPaths:
1127 return # pragma: no cover
1128 oldPath = g.os_path_normcase(at.getPathUa(p))
1129 newPath = g.os_path_normcase(g.fullPath(c, p))
1130 try: # #1367: samefile can throw an exception.
1131 changed = oldPath and not os.path.samefile(oldPath, newPath)
1132 except Exception:
1133 changed = True
1134 if not changed:
1135 return
1136 ok = at.promptForDangerousWrite(
1137 fileName=None,
1138 message=(
1139 f"{g.tr('path changed for %s' % (p.h))}\n"
1140 f"{g.tr('write this file anyway?')}"
1141 ),
1142 )
1143 if not ok:
1144 raise IOError
1145 at.setPathUa(p, newPath) # Remember that we have changed paths.
1146 #@+node:ekr.20190109172025.1: *5* at.writeAtAutoContents
1147 def writeAtAutoContents(self, fileName, root): # pragma: no cover
1148 """Common helper for atAutoToString and writeOneAtAutoNode."""
1149 at, c = self, self.c
1150 # Dispatch the proper writer.
1151 junk, ext = g.os_path_splitext(fileName)
1152 writer = at.dispatch(ext, root)
1153 if writer:
1154 at.outputList = []
1155 writer(root)
1156 return '' if at.errors else ''.join(at.outputList)
1157 if root.isAtAutoRstNode():
1158 # An escape hatch: fall back to the theRst writer
1159 # if there is no rst writer plugin.
1160 at.outputFile = outputFile = io.StringIO()
1161 ok = c.rstCommands.writeAtAutoFile(root, fileName, outputFile)
1162 return outputFile.close() if ok else None
1163 # leo 5.6: allow undefined section references in all @auto files.
1164 ivar = 'allow_undefined_refs'
1165 try:
1166 setattr(at, ivar, True)
1167 at.outputList = []
1168 at.putFile(root, sentinels=False)
1169 return '' if at.errors else ''.join(at.outputList)
1170 except Exception:
1171 return None
1172 finally:
1173 if hasattr(at, ivar):
1174 delattr(at, ivar)
1175 #@+node:ekr.20190111153522.1: *5* at.writeX...
1176 #@+node:ekr.20041005105605.154: *6* at.asisWrite & helper
1177 def asisWrite(self, root): # pragma: no cover
1178 at, c = self, self.c
1179 try:
1180 c.endEditing()
1181 c.init_error_dialogs()
1182 fileName = at.initWriteIvars(root)
1183 # #1450.
1184 if not fileName or not at.precheck(fileName, root):
1185 at.addToOrphanList(root)
1186 return
1187 at.outputList = []
1188 for p in root.self_and_subtree(copy=False):
1189 at.writeAsisNode(p)
1190 if not at.errors:
1191 contents = ''.join(at.outputList)
1192 at.replaceFile(contents, at.encoding, fileName, root)
1193 except Exception:
1194 at.writeException(fileName, root)
1196 silentWrite = asisWrite # Compatibility with old scripts.
1197 #@+node:ekr.20170331141933.1: *7* at.writeAsisNode
1198 def writeAsisNode(self, p): # pragma: no cover
1199 """Write the p's node to an @asis file."""
1200 at = self
1202 def put(s):
1203 """Append s to self.output_list."""
1204 # #1480: Avoid calling at.os().
1205 s = g.toUnicode(s, at.encoding, reportErrors=True)
1206 at.outputList.append(s)
1208 # Write the headline only if it starts with '@@'.
1210 s = p.h
1211 if g.match(s, 0, "@@"):
1212 s = s[2:]
1213 if s:
1214 put('\n') # Experimental.
1215 put(s)
1216 put('\n')
1217 # Write the body.
1218 s = p.b
1219 if s:
1220 put(s)
1221 #@+node:ekr.20041005105605.151: *6* at.writeMissing & helper
1222 def writeMissing(self, p): # pragma: no cover
1223 at, c = self, self.c
1224 writtenFiles = False
1225 c.init_error_dialogs()
1226 # #1450.
1227 at.initWriteIvars(root=p.copy())
1228 p = p.copy()
1229 after = p.nodeAfterTree()
1230 while p and p != after: # Don't use iterator.
1231 if (
1232 p.isAtAsisFileNode() or (p.isAnyAtFileNode() and not p.isAtIgnoreNode())
1233 ):
1234 fileName = p.anyAtFileNodeName()
1235 if fileName:
1236 fileName = g.fullPath(c, p) # #1914.
1237 if at.precheck(fileName, p):
1238 at.writeMissingNode(p)
1239 writtenFiles = True
1240 else:
1241 at.addToOrphanList(p)
1242 p.moveToNodeAfterTree()
1243 elif p.isAtIgnoreNode():
1244 p.moveToNodeAfterTree()
1245 else:
1246 p.moveToThreadNext()
1247 if not g.unitTesting:
1248 if writtenFiles > 0:
1249 g.es("finished")
1250 else:
1251 g.es("no @file node in the selected tree")
1252 c.raise_error_dialogs(kind='write')
1253 #@+node:ekr.20041005105605.152: *7* at.writeMissingNode
1254 def writeMissingNode(self, p): # pragma: no cover
1256 at = self
1257 table = (
1258 (p.isAtAsisFileNode, at.asisWrite),
1259 (p.isAtAutoNode, at.writeOneAtAutoNode),
1260 (p.isAtCleanNode, at.writeOneAtCleanNode),
1261 (p.isAtEditNode, at.writeOneAtEditNode),
1262 (p.isAtFileNode, at.writeOneAtFileNode),
1263 (p.isAtNoSentFileNode, at.writeOneAtNosentNode),
1264 (p.isAtShadowFileNode, at.writeOneAtShadowNode),
1265 (p.isAtThinFileNode, at.writeOneAtFileNode),
1266 )
1267 for pred, func in table:
1268 if pred():
1269 func(p) # type:ignore
1270 return
1271 g.trace(f"Can not happen unknown @<file> kind: {p.h}")
1272 #@+node:ekr.20070806141607: *6* at.writeOneAtAutoNode & helpers
1273 def writeOneAtAutoNode(self, p): # pragma: no cover
1274 """
1275 Write p, an @auto node.
1276 File indices *must* have already been assigned.
1277 Return True if the node was written successfully.
1278 """
1279 at, c = self, self.c
1280 root = p.copy()
1281 try:
1282 c.endEditing()
1283 if not p.atAutoNodeName():
1284 return False
1285 fileName = at.initWriteIvars(root)
1286 at.sentinels = False
1287 # #1450.
1288 if not fileName or not at.precheck(fileName, root):
1289 at.addToOrphanList(root)
1290 return False
1291 if c.persistenceController:
1292 c.persistenceController.update_before_write_foreign_file(root)
1293 contents = at.writeAtAutoContents(fileName, root)
1294 if contents is None:
1295 g.es("not written:", fileName)
1296 at.addToOrphanList(root)
1297 return False
1298 at.replaceFile(contents, at.encoding, fileName, root,
1299 ignoreBlankLines=root.isAtAutoRstNode())
1300 return True
1301 except Exception:
1302 at.writeException(fileName, root)
1303 return False
1304 #@+node:ekr.20140728040812.17993: *7* at.dispatch & helpers
1305 def dispatch(self, ext, p): # pragma: no cover
1306 """Return the correct writer function for p, an @auto node."""
1307 at = self
1308 # Match @auto type before matching extension.
1309 return at.writer_for_at_auto(p) or at.writer_for_ext(ext)
1310 #@+node:ekr.20140728040812.17995: *8* at.writer_for_at_auto
1311 def writer_for_at_auto(self, root): # pragma: no cover
1312 """A factory returning a writer function for the given kind of @auto directive."""
1313 at = self
1314 d = g.app.atAutoWritersDict
1315 for key in d:
1316 aClass = d.get(key)
1317 if aClass and g.match_word(root.h, 0, key):
1319 def writer_for_at_auto_cb(root):
1320 # pylint: disable=cell-var-from-loop
1321 try:
1322 writer = aClass(at.c)
1323 s = writer.write(root)
1324 return s
1325 except Exception:
1326 g.es_exception()
1327 return None
1329 return writer_for_at_auto_cb
1330 return None
1331 #@+node:ekr.20140728040812.17997: *8* at.writer_for_ext
1332 def writer_for_ext(self, ext): # pragma: no cover
1333 """A factory returning a writer function for the given file extension."""
1334 at = self
1335 d = g.app.writersDispatchDict
1336 aClass = d.get(ext)
1337 if aClass:
1339 def writer_for_ext_cb(root):
1340 try:
1341 return aClass(at.c).write(root)
1342 except Exception:
1343 g.es_exception()
1344 return None
1346 return writer_for_ext_cb
1348 return None
1349 #@+node:ekr.20210501064359.1: *6* at.writeOneAtCleanNode
1350 def writeOneAtCleanNode(self, root): # pragma: no cover
1351 """Write one @clean file..
1352 root is the position of an @clean node.
1353 """
1354 at, c = self, self.c
1355 try:
1356 c.endEditing()
1357 fileName = at.initWriteIvars(root)
1358 at.sentinels = False
1359 if not fileName or not at.precheck(fileName, root):
1360 return
1361 at.outputList = []
1362 at.putFile(root, sentinels=False)
1363 at.warnAboutOrphandAndIgnoredNodes()
1364 if at.errors:
1365 g.es("not written:", g.shortFileName(fileName))
1366 at.addToOrphanList(root)
1367 else:
1368 contents = ''.join(at.outputList)
1369 at.replaceFile(contents, at.encoding, fileName, root)
1370 except Exception:
1371 at.writeException(fileName, root)
1372 #@+node:ekr.20090225080846.5: *6* at.writeOneAtEditNode
1373 def writeOneAtEditNode(self, p): # pragma: no cover
1374 """Write one @edit node."""
1375 at, c = self, self.c
1376 root = p.copy()
1377 try:
1378 c.endEditing()
1379 c.init_error_dialogs()
1380 if not p.atEditNodeName():
1381 return False
1382 if p.hasChildren():
1383 g.error('@edit nodes must not have children')
1384 g.es('To save your work, convert @edit to @auto, @file or @clean')
1385 return False
1386 fileName = at.initWriteIvars(root)
1387 at.sentinels = False
1388 # #1450.
1389 if not fileName or not at.precheck(fileName, root):
1390 at.addToOrphanList(root)
1391 return False
1392 contents = ''.join([s for s in g.splitLines(p.b)
1393 if at.directiveKind4(s, 0) == at.noDirective])
1394 at.replaceFile(contents, at.encoding, fileName, root)
1395 c.raise_error_dialogs(kind='write')
1396 return True
1397 except Exception:
1398 at.writeException(fileName, root)
1399 return False
1400 #@+node:ekr.20210501075610.1: *6* at.writeOneAtFileNode
1401 def writeOneAtFileNode(self, root): # pragma: no cover
1402 """Write @file or @thin file."""
1403 at, c = self, self.c
1404 try:
1405 c.endEditing()
1406 fileName = at.initWriteIvars(root)
1407 at.sentinels = True
1408 if not fileName or not at.precheck(fileName, root):
1409 # Raise dialog warning of data loss.
1410 at.addToOrphanList(root)
1411 return
1412 at.outputList = []
1413 at.putFile(root, sentinels=True)
1414 at.warnAboutOrphandAndIgnoredNodes()
1415 if at.errors:
1416 g.es("not written:", g.shortFileName(fileName))
1417 at.addToOrphanList(root)
1418 else:
1419 contents = ''.join(at.outputList)
1420 at.replaceFile(contents, at.encoding, fileName, root)
1421 except Exception:
1422 at.writeException(fileName, root)
1423 #@+node:ekr.20210501065352.1: *6* at.writeOneAtNosentNode
1424 def writeOneAtNosentNode(self, root): # pragma: no cover
1425 """Write one @nosent node.
1426 root is the position of an @<file> node.
1427 sentinels will be False for @clean and @nosent nodes.
1428 """
1429 at, c = self, self.c
1430 try:
1431 c.endEditing()
1432 fileName = at.initWriteIvars(root)
1433 at.sentinels = False
1434 if not fileName or not at.precheck(fileName, root):
1435 return
1436 at.outputList = []
1437 at.putFile(root, sentinels=False)
1438 at.warnAboutOrphandAndIgnoredNodes()
1439 if at.errors:
1440 g.es("not written:", g.shortFileName(fileName))
1441 at.addToOrphanList(root)
1442 else:
1443 contents = ''.join(at.outputList)
1444 at.replaceFile(contents, at.encoding, fileName, root)
1445 except Exception:
1446 at.writeException(fileName, root)
1447 #@+node:ekr.20080711093251.5: *6* at.writeOneAtShadowNode & helper
1448 def writeOneAtShadowNode(self, p, testing=False): # pragma: no cover
1449 """
1450 Write p, an @shadow node.
1451 File indices *must* have already been assigned.
1453 testing: set by unit tests to suppress the call to at.precheck.
1454 Testing is not the same as g.unitTesting.
1455 """
1456 at, c = self, self.c
1457 root = p.copy()
1458 x = c.shadowController
1459 try:
1460 c.endEditing() # Capture the current headline.
1461 fn = p.atShadowFileNodeName()
1462 assert fn, p.h
1463 self.adjustTargetLanguage(fn)
1464 # A hack to support unknown extensions. May set c.target_language.
1465 full_path = g.fullPath(c, p)
1466 at.initWriteIvars(root)
1467 # Force python sentinels to suppress an error message.
1468 # The actual sentinels will be set below.
1469 at.endSentinelComment = None
1470 at.startSentinelComment = "#"
1471 # Make sure we can compute the shadow directory.
1472 private_fn = x.shadowPathName(full_path)
1473 if not private_fn:
1474 return False
1475 if not testing and not at.precheck(full_path, root):
1476 return False
1477 #
1478 # Bug fix: Leo 4.5.1:
1479 # use x.markerFromFileName to force the delim to match
1480 # what is used in x.propegate changes.
1481 marker = x.markerFromFileName(full_path)
1482 at.startSentinelComment, at.endSentinelComment = marker.getDelims()
1483 if g.unitTesting:
1484 ivars_dict = g.getIvarsDict(at)
1485 #
1486 # Write the public and private files to strings.
1488 def put(sentinels):
1489 at.outputList = []
1490 at.sentinels = sentinels
1491 at.putFile(root, sentinels=sentinels)
1492 return '' if at.errors else ''.join(at.outputList)
1494 at.public_s = put(False)
1495 at.private_s = put(True)
1496 at.warnAboutOrphandAndIgnoredNodes()
1497 if g.unitTesting:
1498 exceptions = ('public_s', 'private_s', 'sentinels', 'outputList')
1499 assert g.checkUnchangedIvars(
1500 at, ivars_dict, exceptions), 'writeOneAtShadowNode'
1501 if not at.errors:
1502 # Write the public and private files.
1503 x.makeShadowDirectory(full_path)
1504 # makeShadowDirectory takes a *public* file name.
1505 x.replaceFileWithString(at.encoding, private_fn, at.private_s)
1506 x.replaceFileWithString(at.encoding, full_path, at.public_s)
1507 at.checkPythonCode(contents=at.private_s, fileName=full_path, root=root)
1508 if at.errors:
1509 g.error("not written:", full_path)
1510 at.addToOrphanList(root)
1511 else:
1512 root.clearDirty()
1513 return not at.errors
1514 except Exception:
1515 at.writeException(full_path, root)
1516 return False
1517 #@+node:ekr.20080819075811.13: *7* at.adjustTargetLanguage
1518 def adjustTargetLanguage(self, fn): # pragma: no cover
1519 """Use the language implied by fn's extension if
1520 there is a conflict between it and c.target_language."""
1521 at = self
1522 c = at.c
1523 junk, ext = g.os_path_splitext(fn)
1524 if ext:
1525 if ext.startswith('.'):
1526 ext = ext[1:]
1527 language = g.app.extension_dict.get(ext)
1528 if language:
1529 c.target_language = language
1530 else:
1531 # An unknown language.
1532 # Use the default language, **not** 'unknown_language'
1533 pass
1534 #@+node:ekr.20190111153506.1: *5* at.XToString
1535 #@+node:ekr.20190109160056.1: *6* at.atAsisToString
1536 def atAsisToString(self, root): # pragma: no cover
1537 """Write the @asis node to a string."""
1538 at, c = self, self.c
1539 try:
1540 c.endEditing()
1541 fileName = at.initWriteIvars(root)
1542 at.outputList = []
1543 for p in root.self_and_subtree(copy=False):
1544 at.writeAsisNode(p)
1545 return '' if at.errors else ''.join(at.outputList)
1546 except Exception:
1547 at.writeException(fileName, root)
1548 return ''
1549 #@+node:ekr.20190109160056.2: *6* at.atAutoToString
1550 def atAutoToString(self, root): # pragma: no cover
1551 """Write the root @auto node to a string, and return it."""
1552 at, c = self, self.c
1553 try:
1554 c.endEditing()
1555 fileName = at.initWriteIvars(root)
1556 at.sentinels = False
1557 # #1450.
1558 if not fileName:
1559 at.addToOrphanList(root)
1560 return ''
1561 return at.writeAtAutoContents(fileName, root) or ''
1562 except Exception:
1563 at.writeException(fileName, root)
1564 return ''
1565 #@+node:ekr.20190109160056.3: *6* at.atEditToString
1566 def atEditToString(self, root): # pragma: no cover
1567 """Write one @edit node."""
1568 at, c = self, self.c
1569 try:
1570 c.endEditing()
1571 if root.hasChildren():
1572 g.error('@edit nodes must not have children')
1573 g.es('To save your work, convert @edit to @auto, @file or @clean')
1574 return False
1575 fileName = at.initWriteIvars(root)
1576 at.sentinels = False
1577 # #1450.
1578 if not fileName:
1579 at.addToOrphanList(root)
1580 return ''
1581 contents = ''.join([
1582 s for s in g.splitLines(root.b)
1583 if at.directiveKind4(s, 0) == at.noDirective])
1584 return contents
1585 except Exception:
1586 at.writeException(fileName, root)
1587 return ''
1588 #@+node:ekr.20190109142026.1: *6* at.atFileToString
1589 def atFileToString(self, root, sentinels=True): # pragma: no cover
1590 """Write an external file to a string, and return its contents."""
1591 at, c = self, self.c
1592 try:
1593 c.endEditing()
1594 at.initWriteIvars(root)
1595 at.sentinels = sentinels
1596 at.outputList = []
1597 at.putFile(root, sentinels=sentinels)
1598 assert root == at.root, 'write'
1599 contents = '' if at.errors else ''.join(at.outputList)
1600 return contents
1601 except Exception:
1602 at.exception("exception preprocessing script")
1603 root.v._p_changed = True
1604 return ''
1605 #@+node:ekr.20050506084734: *6* at.stringToString
1606 def stringToString(self, root, s, forcePythonSentinels=True, sentinels=True): # pragma: no cover
1607 """
1608 Write an external file from a string.
1610 This is at.write specialized for scripting.
1611 """
1612 at, c = self, self.c
1613 try:
1614 c.endEditing()
1615 at.initWriteIvars(root)
1616 if forcePythonSentinels:
1617 at.endSentinelComment = None
1618 at.startSentinelComment = "#"
1619 at.language = "python"
1620 at.sentinels = sentinels
1621 at.outputList = []
1622 at.putFile(root, fromString=s, sentinels=sentinels)
1623 contents = '' if at.errors else ''.join(at.outputList)
1624 # Major bug: failure to clear this wipes out headlines!
1625 # Sometimes this causes slight problems...
1626 if root:
1627 root.v._p_changed = True
1628 return contents
1629 except Exception:
1630 at.exception("exception preprocessing script")
1631 return ''
1632 #@+node:ekr.20041005105605.160: *4* Writing helpers
1633 #@+node:ekr.20041005105605.161: *5* at.putBody & helper
1634 def putBody(self, p, fromString=''):
1635 """
1636 Generate the body enclosed in sentinel lines.
1637 Return True if the body contains an @others line.
1638 """
1639 at = self
1640 #
1641 # New in 4.3 b2: get s from fromString if possible.
1642 s = fromString if fromString else p.b
1643 p.v.setVisited()
1644 # Make sure v is never expanded again.
1645 # Suppress orphans check.
1646 #
1647 # #1048 & #1037: regularize most trailing whitespace.
1648 if s and (at.sentinels or at.force_newlines_in_at_nosent_bodies):
1649 if not s.endswith('\n'):
1650 s = s + '\n'
1653 class Status:
1654 at_comment_seen=False
1655 at_delims_seen=False
1656 at_warning_given=False
1657 has_at_others=False
1658 in_code=True
1661 i = 0
1662 status = Status()
1663 while i < len(s):
1664 next_i = g.skip_line(s, i)
1665 assert next_i > i, 'putBody'
1666 kind = at.directiveKind4(s, i)
1667 at.putLine(i, kind, p, s, status)
1668 i = next_i
1669 if not status.in_code:
1670 at.putEndDocLine()
1671 return status.has_at_others
1672 #@+node:ekr.20041005105605.163: *6* at.putLine
1673 def putLine(self, i, kind, p, s, status):
1674 """Put the line at s[i:] of the given kind, updating the status."""
1675 at = self
1676 if kind == at.noDirective:
1677 if status.in_code:
1678 # Important: the so-called "name" must include brackets.
1679 name, n1, n2 = at.findSectionName(s, i, p)
1680 if name:
1681 at.putRefLine(s, i, n1, n2, name, p)
1682 else:
1683 at.putCodeLine(s, i)
1684 else:
1685 at.putDocLine(s, i)
1686 elif kind in (at.docDirective, at.atDirective):
1687 if not status.in_code:
1688 # Bug fix 12/31/04: handle adjacent doc parts.
1689 at.putEndDocLine()
1690 at.putStartDocLine(s, i, kind)
1691 status.in_code = False
1692 elif kind in (at.cDirective, at.codeDirective):
1693 # Only @c and @code end a doc part.
1694 if not status.in_code:
1695 at.putEndDocLine()
1696 at.putDirective(s, i, p)
1697 status.in_code = True
1698 elif kind == at.allDirective:
1699 if status.in_code:
1700 if p == self.root:
1701 at.putAtAllLine(s, i, p)
1702 else:
1703 at.error(f"@all not valid in: {p.h}") # pragma: no cover
1704 else:
1705 at.putDocLine(s, i)
1706 elif kind == at.othersDirective:
1707 if status.in_code:
1708 if status.has_at_others:
1709 at.error(f"multiple @others in: {p.h}") # pragma: no cover
1710 else:
1711 at.putAtOthersLine(s, i, p)
1712 status.has_at_others = True
1713 else:
1714 at.putDocLine(s, i)
1715 elif kind == at.startVerbatim: # pragma: no cover
1716 # Fix bug 778204: @verbatim not a valid Leo directive.
1717 if g.unitTesting:
1718 # A hack: unit tests for @shadow use @verbatim as a kind of directive.
1719 pass
1720 else:
1721 at.error(f"@verbatim is not a Leo directive: {p.h}")
1722 elif kind == at.miscDirective:
1723 # Fix bug 583878: Leo should warn about @comment/@delims clashes.
1724 if g.match_word(s, i, '@comment'):
1725 status.at_comment_seen = True
1726 elif g.match_word(s, i, '@delims'):
1727 status.at_delims_seen = True
1728 if (
1729 status.at_comment_seen and
1730 status.at_delims_seen and not
1731 status.at_warning_given
1732 ): # pragma: no cover
1733 status.at_warning_given = True
1734 at.error(f"@comment and @delims in node {p.h}")
1735 at.putDirective(s, i, p)
1736 else:
1737 at.error(f"putBody: can not happen: unknown directive kind: {kind}") # pragma: no cover
1738 #@+node:ekr.20041005105605.164: *5* writing code lines...
1739 #@+node:ekr.20041005105605.165: *6* at: @all
1740 #@+node:ekr.20041005105605.166: *7* at.putAtAllLine
1741 def putAtAllLine(self, s, i, p):
1742 """Put the expansion of @all."""
1743 at = self
1744 j, delta = g.skip_leading_ws_with_indent(s, i, at.tab_width)
1745 k = g.skip_to_end_of_line(s, i)
1746 at.putLeadInSentinel(s, i, j)
1747 at.indent += delta
1748 at.putSentinel("@+" + s[j + 1 : k].strip())
1749 # s[j:k] starts with '@all'
1750 for child in p.children():
1751 at.putAtAllChild(child)
1752 at.putSentinel("@-all")
1753 at.indent -= delta
1754 #@+node:ekr.20041005105605.167: *7* at.putAtAllBody
1755 def putAtAllBody(self, p):
1756 """ Generate the body enclosed in sentinel lines."""
1757 at = self
1758 s = p.b
1759 p.v.setVisited()
1760 # Make sure v is never expanded again.
1761 # Suppress orphans check.
1762 if at.sentinels and s and s[-1] != '\n':
1763 s = s + '\n'
1764 i = 0
1765 # Leo 6.6. This code never changes at.in_code status!
1766 while i < len(s):
1767 next_i = g.skip_line(s, i)
1768 assert next_i > i
1769 at.putCodeLine(s, i)
1770 i = next_i
1771 #@+node:ekr.20041005105605.169: *7* at.putAtAllChild
1772 def putAtAllChild(self, p):
1773 """
1774 This code puts only the first of two or more cloned siblings, preceding
1775 the clone with an @clone n sentinel.
1777 This is a debatable choice: the cloned tree appears only once in the
1778 external file. This should be benign; the text created by @all is
1779 likely to be used only for recreating the outline in Leo. The
1780 representation in the derived file doesn't matter much.
1781 """
1782 at = self
1783 at.putOpenNodeSentinel(p, inAtAll=True)
1784 # Suppress warnings about @file nodes.
1785 at.putAtAllBody(p)
1786 for child in p.children():
1787 at.putAtAllChild(child) # pragma: no cover (recursive call)
1788 #@+node:ekr.20041005105605.170: *6* at: @others
1789 #@+node:ekr.20041005105605.173: *7* at.putAtOthersLine & helper
1790 def putAtOthersLine(self, s, i, p):
1791 """Put the expansion of @others."""
1792 at = self
1793 j, delta = g.skip_leading_ws_with_indent(s, i, at.tab_width)
1794 k = g.skip_to_end_of_line(s, i)
1795 at.putLeadInSentinel(s, i, j)
1796 at.indent += delta
1797 # s[j:k] starts with '@others'
1798 # Never write lws in new sentinels.
1799 at.putSentinel("@+" + s[j + 1 : k].strip())
1800 for child in p.children():
1801 p = child.copy()
1802 after = p.nodeAfterTree()
1803 while p and p != after:
1804 if at.validInAtOthers(p):
1805 at.putOpenNodeSentinel(p)
1806 at_others_flag = at.putBody(p)
1807 if at_others_flag:
1808 p.moveToNodeAfterTree()
1809 else:
1810 p.moveToThreadNext()
1811 else:
1812 p.moveToNodeAfterTree()
1813 # This is the same in both old and new sentinels.
1814 at.putSentinel("@-others")
1815 at.indent -= delta
1816 #@+node:ekr.20041005105605.171: *8* at.validInAtOthers
1817 def validInAtOthers(self, p):
1818 """
1819 Return True if p should be included in the expansion of the @others
1820 directive in the body text of p's parent.
1821 """
1822 at = self
1823 i = g.skip_ws(p.h, 0)
1824 isSection, junk = at.isSectionName(p.h, i)
1825 if isSection:
1826 return False # A section definition node.
1827 if at.sentinels:
1828 # @ignore must not stop expansion here!
1829 return True
1830 if p.isAtIgnoreNode(): # pragma: no cover
1831 g.error('did not write @ignore node', p.v.h)
1832 return False
1833 return True
1834 #@+node:ekr.20041005105605.199: *6* at.findSectionName
1835 def findSectionName(self, s, i, p):
1836 """
1837 Return n1, n2 representing a section name.
1839 Return the reference, *including* brackes.
1840 """
1841 at = self
1843 def is_space(i1, i2):
1844 """A replacement for s[i1 : i2] that doesn't create any substring."""
1845 return i == j or all(s[z] in ' \t\n' for z in range(i1, i2))
1847 end = s.find('\n', i)
1848 j = len(s) if end == -1 else end
1849 # Careful: don't look beyond the end of the line!
1850 if end == -1:
1851 n1 = s.find(at.section_delim1, i)
1852 n2 = s.find(at.section_delim2, i)
1853 else:
1854 n1 = s.find(at.section_delim1, i, end)
1855 n2 = s.find(at.section_delim2, i, end)
1856 n3 = n2 + len(at.section_delim2)
1857 if -1 < n1 < n2: # A *possible* section reference.
1858 if is_space(i, n1) and is_space(n3, j): # A *real* section reference.
1859 return s[n1 : n3], n1, n3
1860 # An apparent section reference.
1861 if 'sections' in g.app.debug and not g.unitTesting: # pragma: no cover
1862 i1, i2 = g.getLine(s, i)
1863 g.es_print('Ignoring apparent section reference:', color='red')
1864 g.es_print('Node: ', p.h)
1865 g.es_print('Line: ', s[i1 : i2].rstrip())
1866 return None, 0, 0
1867 #@+node:ekr.20041005105605.174: *6* at.putCodeLine
1868 def putCodeLine(self, s, i):
1869 """Put a normal code line."""
1870 at = self
1871 # Put @verbatim sentinel if required.
1872 k = g.skip_ws(s, i)
1873 if g.match(s, k, self.startSentinelComment + '@'):
1874 self.putSentinel('@verbatim')
1875 j = g.skip_line(s, i)
1876 line = s[i:j]
1877 # Don't put any whitespace in otherwise blank lines.
1878 if len(line) > 1: # Preserve *anything* the user puts on the line!!!
1879 at.putIndent(at.indent, line)
1880 if line[-1:] == '\n':
1881 at.os(line[:-1])
1882 at.onl()
1883 else:
1884 at.os(line)
1885 elif line and line[-1] == '\n':
1886 at.onl()
1887 elif line:
1888 at.os(line) # Bug fix: 2013/09/16
1889 else:
1890 g.trace('Can not happen: completely empty line') # pragma: no cover
1891 #@+node:ekr.20041005105605.176: *6* at.putRefLine
1892 def putRefLine(self, s, i, n1, n2, name, p):
1893 """
1894 Put a line containing one or more references.
1896 Important: the so-called name *must* include brackets.
1897 """
1898 at = self
1899 ref = g.findReference(name, p)
1900 if ref:
1901 junk, delta = g.skip_leading_ws_with_indent(s, i, at.tab_width)
1902 at.putLeadInSentinel(s, i, n1)
1903 at.indent += delta
1904 at.putSentinel("@+" + name)
1905 at.putOpenNodeSentinel(ref)
1906 at.putBody(ref)
1907 at.putSentinel("@-" + name)
1908 at.indent -= delta
1909 return
1910 if hasattr(at, 'allow_undefined_refs'): # pragma: no cover
1911 p.v.setVisited() # #2311
1912 # Allow apparent section reference: just write the line.
1913 at.putCodeLine(s, i)
1914 else: # pragma: no cover
1915 # Do give this error even if unit testing.
1916 at.writeError(
1917 f"undefined section: {g.truncate(name, 60)}\n"
1918 f" referenced from: {g.truncate(p.h, 60)}")
1919 #@+node:ekr.20041005105605.180: *5* writing doc lines...
1920 #@+node:ekr.20041005105605.181: *6* at.putBlankDocLine
1921 def putBlankDocLine(self):
1922 at = self
1923 if not at.endSentinelComment:
1924 at.putIndent(at.indent)
1925 at.os(at.startSentinelComment)
1926 # #1496: Retire the @doc convention.
1927 # Remove the blank.
1928 # at.oblank()
1929 at.onl()
1930 #@+node:ekr.20041005105605.183: *6* at.putDocLine
1931 def putDocLine(self, s, i):
1932 """Handle one line of a doc part."""
1933 at = self
1934 j = g.skip_line(s, i)
1935 s = s[i:j]
1936 #
1937 # #1496: Retire the @doc convention:
1938 # Strip all trailing ws here.
1939 if not s.strip():
1940 # A blank line.
1941 at.putBlankDocLine()
1942 return
1943 # Write the line as it is.
1944 at.putIndent(at.indent)
1945 if not at.endSentinelComment:
1946 at.os(at.startSentinelComment)
1947 # #1496: Retire the @doc convention.
1948 # Leave this blank. The line is not blank.
1949 at.oblank()
1950 at.os(s)
1951 if not s.endswith('\n'):
1952 at.onl() # pragma: no cover
1953 #@+node:ekr.20041005105605.185: *6* at.putEndDocLine
1954 def putEndDocLine(self):
1955 """Write the conclusion of a doc part."""
1956 at = self
1957 # Put the closing delimiter if we are using block comments.
1958 if at.endSentinelComment:
1959 at.putIndent(at.indent)
1960 at.os(at.endSentinelComment)
1961 at.onl() # Note: no trailing whitespace.
1962 #@+node:ekr.20041005105605.182: *6* at.putStartDocLine
1963 def putStartDocLine(self, s, i, kind):
1964 """Write the start of a doc part."""
1965 at = self
1966 sentinel = "@+doc" if kind == at.docDirective else "@+at"
1967 directive = "@doc" if kind == at.docDirective else "@"
1968 # Put whatever follows the directive in the sentinel.
1969 # Skip past the directive.
1970 i += len(directive)
1971 j = g.skip_to_end_of_line(s, i)
1972 follow = s[i:j]
1973 # Put the opening @+doc or @-doc sentinel, including whatever follows the directive.
1974 at.putSentinel(sentinel + follow)
1975 # Put the opening comment if we are using block comments.
1976 if at.endSentinelComment:
1977 at.putIndent(at.indent)
1978 at.os(at.startSentinelComment)
1979 at.onl()
1980 #@+node:ekr.20041005105605.187: *4* Writing sentinels...
1981 #@+node:ekr.20041005105605.188: *5* at.nodeSentinelText & helper
1982 def nodeSentinelText(self, p):
1983 """Return the text of a @+node or @-node sentinel for p."""
1984 at = self
1985 h = at.removeCommentDelims(p)
1986 if getattr(at, 'at_shadow_test_hack', False): # pragma: no cover
1987 # A hack for @shadow unit testing.
1988 # see AtShadowTestCase.makePrivateLines.
1989 return h
1990 gnx = p.v.fileIndex
1991 level = 1 + p.level() - self.root.level()
1992 if level > 2:
1993 return f"{gnx}: *{level}* {h}"
1994 return f"{gnx}: {'*' * level} {h}"
1995 #@+node:ekr.20041005105605.189: *6* at.removeCommentDelims
1996 def removeCommentDelims(self, p):
1997 """
1998 If the present @language/@comment settings do not specify a single-line comment
1999 we remove all block comment delims from h. This prevents headline text from
2000 interfering with the parsing of node sentinels.
2001 """
2002 at = self
2003 start = at.startSentinelComment
2004 end = at.endSentinelComment
2005 h = p.h
2006 if end:
2007 h = h.replace(start, "")
2008 h = h.replace(end, "")
2009 return h
2010 #@+node:ekr.20041005105605.190: *5* at.putLeadInSentinel
2011 def putLeadInSentinel(self, s, i, j):
2012 """
2013 Set at.leadingWs as needed for @+others and @+<< sentinels.
2015 i points at the start of a line.
2016 j points at @others or a section reference.
2017 """
2018 at = self
2019 at.leadingWs = "" # Set the default.
2020 if i == j:
2021 return # The @others or ref starts a line.
2022 k = g.skip_ws(s, i)
2023 if j == k:
2024 # Remember the leading whitespace, including its spelling.
2025 at.leadingWs = s[i:j]
2026 else:
2027 self.putIndent(at.indent) # 1/29/04: fix bug reported by Dan Winkler.
2028 at.os(s[i:j])
2029 at.onl_sent()
2030 #@+node:ekr.20041005105605.192: *5* at.putOpenLeoSentinel 4.x
2031 def putOpenLeoSentinel(self, s):
2032 """Write @+leo sentinel."""
2033 at = self
2034 if at.sentinels or hasattr(at, 'force_sentinels'):
2035 s = s + "-thin"
2036 encoding = at.encoding.lower()
2037 if encoding != "utf-8": # pragma: no cover
2038 # New in 4.2: encoding fields end in ",."
2039 s = s + f"-encoding={encoding},."
2040 at.putSentinel(s)
2041 #@+node:ekr.20041005105605.193: *5* at.putOpenNodeSentinel
2042 def putOpenNodeSentinel(self, p, inAtAll=False):
2043 """Write @+node sentinel for p."""
2044 # Note: lineNumbers.py overrides this method.
2045 at = self
2046 if not inAtAll and p.isAtFileNode() and p != at.root: # pragma: no cover
2047 at.writeError("@file not valid in: " + p.h)
2048 return
2049 s = at.nodeSentinelText(p)
2050 at.putSentinel("@+node:" + s)
2051 # Leo 4.7: we never write tnodeLists.
2052 #@+node:ekr.20041005105605.194: *5* at.putSentinel (applies cweb hack) 4.x
2053 def putSentinel(self, s):
2054 """
2055 Write a sentinel whose text is s, applying the CWEB hack if needed.
2057 This method outputs all sentinels.
2058 """
2059 at = self
2060 if at.sentinels or hasattr(at, 'force_sentinels'):
2061 at.putIndent(at.indent)
2062 at.os(at.startSentinelComment)
2063 # #2194. The following would follow the black convention,
2064 # but doing so is a dubious idea.
2065 # at.os(' ')
2066 # Apply the cweb hack to s:
2067 # If the opening comment delim ends in '@',
2068 # double all '@' signs except the first.
2069 start = at.startSentinelComment
2070 if start and start[-1] == '@':
2071 s = s.replace('@', '@@')[1:]
2072 at.os(s)
2073 if at.endSentinelComment:
2074 at.os(at.endSentinelComment)
2075 at.onl()
2076 #@+node:ekr.20041005105605.196: *4* Writing utils...
2077 #@+node:ekr.20181024134823.1: *5* at.addToOrphanList
2078 def addToOrphanList(self, root): # pragma: no cover
2079 """Mark the root as erroneous for c.raise_error_dialogs()."""
2080 c = self.c
2081 # Fix #1050:
2082 root.setOrphan()
2083 c.orphan_at_file_nodes.append(root.h)
2084 #@+node:ekr.20220120210617.1: *5* at.checkPyflakes
2085 def checkPyflakes(self, contents, fileName, root):
2086 at = self
2087 ok = True
2088 if g.unitTesting or not at.runPyFlakesOnWrite:
2089 return ok
2090 if not contents or not fileName or not fileName.endswith('.py'):
2091 return ok
2092 ok = self.runPyflakes(root)
2093 if not ok:
2094 g.app.syntax_error_files.append(g.shortFileName(fileName))
2095 return ok
2096 #@+node:ekr.20090514111518.5661: *5* at.checkPythonCode & helpers
2097 def checkPythonCode(self, contents, fileName, root): # pragma: no cover
2098 """Perform python-related checks on root."""
2099 at = self
2100 if g.unitTesting or not contents or not fileName or not fileName.endswith('.py'):
2101 return
2102 ok = True
2103 if at.checkPythonCodeOnWrite:
2104 ok = at.checkPythonSyntax(root, contents)
2105 if ok and at.runPyFlakesOnWrite:
2106 ok = self.runPyflakes(root)
2107 if not ok:
2108 g.app.syntax_error_files.append(g.shortFileName(fileName))
2109 #@+node:ekr.20090514111518.5663: *6* at.checkPythonSyntax
2110 def checkPythonSyntax(self, p, body):
2111 at = self
2112 try:
2113 body = body.replace('\r', '')
2114 fn = f"<node: {p.h}>"
2115 compile(body + '\n', fn, 'exec')
2116 return True
2117 except SyntaxError:
2118 if not g.unitTesting:
2119 at.syntaxError(p, body)
2120 except Exception:
2121 g.trace("unexpected exception")
2122 g.es_exception()
2123 return False
2124 #@+node:ekr.20090514111518.5666: *7* at.syntaxError (leoAtFile)
2125 def syntaxError(self, p, body): # pragma: no cover
2126 """Report a syntax error."""
2127 g.error(f"Syntax error in: {p.h}")
2128 typ, val, tb = sys.exc_info()
2129 message = hasattr(val, 'message') and val.message
2130 if message:
2131 g.es_print(message)
2132 if val is None:
2133 return
2134 lines = g.splitLines(body)
2135 n = val.lineno
2136 offset = val.offset or 0
2137 if n is None:
2138 return
2139 i = val.lineno - 1
2140 for j in range(max(0, i - 2), min(i + 2, len(lines) - 1)):
2141 line = lines[j].rstrip()
2142 if j == i:
2143 unl = p.get_UNL()
2144 g.es_print(f"{j+1:5}:* {line}", nodeLink=f"{unl}::-{j+1:d}") # Global line.
2145 g.es_print(' ' * (7 + offset) + '^')
2146 else:
2147 g.es_print(f"{j+1:5}: {line}")
2148 #@+node:ekr.20161021084954.1: *6* at.runPyflakes
2149 def runPyflakes(self, root): # pragma: no cover
2150 """Run pyflakes on the selected node."""
2151 try:
2152 from leo.commands import checkerCommands
2153 if checkerCommands.pyflakes:
2154 x = checkerCommands.PyflakesCommand(self.c)
2155 ok = x.run(root)
2156 return ok
2157 return True # Suppress error if pyflakes can not be imported.
2158 except Exception:
2159 g.es_exception()
2160 return True # Pretend all is well
2161 #@+node:ekr.20041005105605.198: *5* at.directiveKind4 (write logic)
2162 # These patterns exclude constructs such as @encoding.setter or @encoding(whatever)
2163 # However, they must allow @language python, @nocolor-node, etc.
2165 at_directive_kind_pattern = re.compile(r'\s*@([\w-]+)\s*')
2167 def directiveKind4(self, s, i):
2168 """
2169 Return the kind of at-directive or noDirective.
2171 Potential simplifications:
2172 - Using strings instead of constants.
2173 - Using additional regex's to recognize directives.
2174 """
2175 at = self
2176 n = len(s)
2177 if i >= n or s[i] != '@':
2178 j = g.skip_ws(s, i)
2179 if g.match_word(s, j, "@others"):
2180 return at.othersDirective
2181 if g.match_word(s, j, "@all"):
2182 return at.allDirective
2183 return at.noDirective
2184 table = (
2185 ("@all", at.allDirective),
2186 ("@c", at.cDirective),
2187 ("@code", at.codeDirective),
2188 ("@doc", at.docDirective),
2189 ("@others", at.othersDirective),
2190 ("@verbatim", at.startVerbatim))
2191 # ("@end_raw", at.endRawDirective), # #2276.
2192 # ("@raw", at.rawDirective), # #2276
2193 # Rewritten 6/8/2005.
2194 if i + 1 >= n or s[i + 1] in (' ', '\t', '\n'):
2195 # Bare '@' not recognized in cweb mode.
2196 return at.noDirective if at.language == "cweb" else at.atDirective
2197 if not s[i + 1].isalpha():
2198 return at.noDirective # Bug fix: do NOT return miscDirective here!
2199 if at.language == "cweb" and g.match_word(s, i, '@c'):
2200 return at.noDirective
2201 # When the language is elixir, @doc followed by a space and string delimiter
2202 # needs to be treated as plain text; the following does not enforce the
2203 # 'string delimiter' part of that. An @doc followed by something other than
2204 # a space will fall through to usual Leo @doc processing.
2205 if at.language == "elixir" and g.match_word(s, i, '@doc '):
2206 return at.noDirective
2207 for name, directive in table:
2208 if g.match_word(s, i, name):
2209 return directive
2210 # Support for add_directives plugin.
2211 # Use regex to properly distinguish between Leo directives
2212 # and python decorators.
2213 s2 = s[i:]
2214 m = self.at_directive_kind_pattern.match(s2)
2215 if m:
2216 word = m.group(1)
2217 if word not in g.globalDirectiveList:
2218 return at.noDirective
2219 s3 = s2[m.end(1) :]
2220 if s3 and s3[0] in ".(":
2221 return at.noDirective
2222 return at.miscDirective
2223 # An unusual case.
2224 return at.noDirective # pragma: no cover
2225 #@+node:ekr.20041005105605.200: *5* at.isSectionName
2226 # returns (flag, end). end is the index of the character after the section name.
2228 def isSectionName(self, s, i): # pragma: no cover
2230 at = self
2231 # Allow leading periods.
2232 while i < len(s) and s[i] == '.':
2233 i += 1
2234 if not g.match(s, i, at.section_delim1):
2235 return False, -1
2236 i = g.find_on_line(s, i, at.section_delim2)
2237 if i > -1:
2238 return True, i + len(at.section_delim2)
2239 return False, -1
2240 #@+node:ekr.20190111112442.1: *5* at.isWritable
2241 def isWritable(self, path): # pragma: no cover
2242 """Return True if the path is writable."""
2243 try:
2244 # os.access() may not exist on all platforms.
2245 ok = os.access(path, os.W_OK)
2246 except AttributeError:
2247 return True
2248 if not ok:
2249 g.es('read only:', repr(path), color='red')
2250 return ok
2251 #@+node:ekr.20041005105605.201: *5* at.os and allies
2252 #@+node:ekr.20041005105605.202: *6* at.oblank, oblanks & otabs
2253 def oblank(self):
2254 self.os(' ')
2256 def oblanks(self, n): # pragma: no cover
2257 self.os(' ' * abs(n))
2259 def otabs(self, n): # pragma: no cover
2260 self.os('\t' * abs(n))
2261 #@+node:ekr.20041005105605.203: *6* at.onl & onl_sent
2262 def onl(self):
2263 """Write a newline to the output stream."""
2264 self.os('\n') # **not** self.output_newline
2266 def onl_sent(self):
2267 """Write a newline to the output stream, provided we are outputting sentinels."""
2268 if self.sentinels:
2269 self.onl()
2270 #@+node:ekr.20041005105605.204: *6* at.os
2271 def os(self, s):
2272 """
2273 Append a string to at.outputList.
2275 All output produced by leoAtFile module goes here.
2276 """
2277 at = self
2278 if s.startswith(self.underindentEscapeString): # pragma: no cover
2279 try:
2280 junk, s = at.parseUnderindentTag(s)
2281 except Exception:
2282 at.exception("exception writing:" + s)
2283 return
2284 s = g.toUnicode(s, at.encoding)
2285 at.outputList.append(s)
2286 #@+node:ekr.20041005105605.205: *5* at.outputStringWithLineEndings
2287 def outputStringWithLineEndings(self, s): # pragma: no cover
2288 """
2289 Write the string s as-is except that we replace '\n' with the proper line ending.
2291 Calling self.onl() runs afoul of queued newlines.
2292 """
2293 at = self
2294 s = g.toUnicode(s, at.encoding)
2295 s = s.replace('\n', at.output_newline)
2296 self.os(s)
2297 #@+node:ekr.20190111045822.1: *5* at.precheck (calls shouldPrompt...)
2298 def precheck(self, fileName, root): # pragma: no cover
2299 """
2300 Check whether a dirty, potentially dangerous, file should be written.
2302 Return True if so. Return False *and* issue a warning otherwise.
2303 """
2304 at = self
2305 #
2306 # #1450: First, check that the directory exists.
2307 theDir = g.os_path_dirname(fileName)
2308 if theDir and not g.os_path_exists(theDir):
2309 at.error(f"Directory not found:\n{theDir}")
2310 return False
2311 #
2312 # Now check the file.
2313 if not at.shouldPromptForDangerousWrite(fileName, root):
2314 # Fix bug 889175: Remember the full fileName.
2315 at.rememberReadPath(fileName, root)
2316 return True
2317 #
2318 # Prompt if the write would overwrite the existing file.
2319 ok = self.promptForDangerousWrite(fileName)
2320 if ok:
2321 # Fix bug 889175: Remember the full fileName.
2322 at.rememberReadPath(fileName, root)
2323 return True
2324 #
2325 # Fix #1031: do not add @ignore here!
2326 g.es("not written:", fileName)
2327 return False
2328 #@+node:ekr.20050506090446.1: *5* at.putAtFirstLines
2329 def putAtFirstLines(self, s):
2330 """
2331 Write any @firstlines from string s.
2332 These lines are converted to @verbatim lines,
2333 so the read logic simply ignores lines preceding the @+leo sentinel.
2334 """
2335 at = self
2336 tag = "@first"
2337 i = 0
2338 while g.match(s, i, tag):
2339 i += len(tag)
2340 i = g.skip_ws(s, i)
2341 j = i
2342 i = g.skip_to_end_of_line(s, i)
2343 # Write @first line, whether empty or not
2344 line = s[j:i]
2345 at.os(line)
2346 at.onl()
2347 i = g.skip_nl(s, i)
2348 #@+node:ekr.20050506090955: *5* at.putAtLastLines
2349 def putAtLastLines(self, s):
2350 """
2351 Write any @last lines from string s.
2352 These lines are converted to @verbatim lines,
2353 so the read logic simply ignores lines following the @-leo sentinel.
2354 """
2355 at = self
2356 tag = "@last"
2357 # Use g.splitLines to preserve trailing newlines.
2358 lines = g.splitLines(s)
2359 n = len(lines)
2360 j = k = n - 1
2361 # Scan backwards for @last directives.
2362 while j >= 0:
2363 line = lines[j]
2364 if g.match(line, 0, tag):
2365 j -= 1
2366 elif not line.strip():
2367 j -= 1
2368 else:
2369 break # pragma: no cover (coverage bug)
2370 # Write the @last lines.
2371 for line in lines[j + 1 : k + 1]:
2372 if g.match(line, 0, tag):
2373 i = len(tag)
2374 i = g.skip_ws(line, i)
2375 at.os(line[i:])
2376 #@+node:ekr.20041005105605.206: *5* at.putDirective & helper
2377 def putDirective(self, s, i, p):
2378 r"""
2379 Output a sentinel a directive or reference s.
2381 It is important for PHP and other situations that \@first and \@last
2382 directives get translated to verbatim lines that do *not* include what
2383 follows the @first & @last directives.
2384 """
2385 at = self
2386 k = i
2387 j = g.skip_to_end_of_line(s, i)
2388 directive = s[i:j]
2389 if g.match_word(s, k, "@delims"):
2390 at.putDelims(directive, s, k)
2391 elif g.match_word(s, k, "@language"):
2392 self.putSentinel("@" + directive)
2393 elif g.match_word(s, k, "@comment"):
2394 self.putSentinel("@" + directive)
2395 elif g.match_word(s, k, "@last"):
2396 # #1307.
2397 if p.isAtCleanNode(): # pragma: no cover
2398 at.error(f"ignoring @last directive in {p.h!r}")
2399 g.es_print('@last is not valid in @clean nodes')
2400 # #1297.
2401 elif g.app.inScript or g.unitTesting or p.isAnyAtFileNode():
2402 self.putSentinel("@@last")
2403 # Convert to an verbatim line _without_ anything else.
2404 else:
2405 at.error(f"ignoring @last directive in {p.h!r}") # pragma: no cover
2406 elif g.match_word(s, k, "@first"):
2407 # #1307.
2408 if p.isAtCleanNode(): # pragma: no cover
2409 at.error(f"ignoring @first directive in {p.h!r}")
2410 g.es_print('@first is not valid in @clean nodes')
2411 # #1297.
2412 elif g.app.inScript or g.unitTesting or p.isAnyAtFileNode():
2413 self.putSentinel("@@first")
2414 # Convert to an verbatim line _without_ anything else.
2415 else:
2416 at.error(f"ignoring @first directive in {p.h!r}") # pragma: no cover
2417 else:
2418 self.putSentinel("@" + directive)
2419 i = g.skip_line(s, k)
2420 return i
2421 #@+node:ekr.20041005105605.207: *6* at.putDelims
2422 def putDelims(self, directive, s, k):
2423 """Put an @delims directive."""
2424 at = self
2425 # Put a space to protect the last delim.
2426 at.putSentinel(directive + " ") # 10/23/02: put @delims, not @@delims
2427 # Skip the keyword and whitespace.
2428 j = i = g.skip_ws(s, k + len("@delims"))
2429 # Get the first delim.
2430 while i < len(s) and not g.is_ws(s[i]) and not g.is_nl(s, i):
2431 i += 1
2432 if j < i:
2433 at.startSentinelComment = s[j:i]
2434 # Get the optional second delim.
2435 j = i = g.skip_ws(s, i)
2436 while i < len(s) and not g.is_ws(s[i]) and not g.is_nl(s, i):
2437 i += 1
2438 at.endSentinelComment = s[j:i] if j < i else ""
2439 else:
2440 at.writeError("Bad @delims directive") # pragma: no cover
2441 #@+node:ekr.20041005105605.210: *5* at.putIndent
2442 def putIndent(self, n, s=''): # pragma: no cover
2443 """Put tabs and spaces corresponding to n spaces,
2444 assuming that we are at the start of a line.
2446 Remove extra blanks if the line starts with the underindentEscapeString"""
2447 tag = self.underindentEscapeString
2448 if s.startswith(tag):
2449 n2, s2 = self.parseUnderindentTag(s)
2450 if n2 >= n:
2451 return
2452 if n > 0:
2453 n -= n2
2454 else:
2455 n += n2
2456 if n > 0:
2457 w = self.tab_width
2458 if w > 1:
2459 q, r = divmod(n, w)
2460 self.otabs(q)
2461 self.oblanks(r)
2462 else:
2463 self.oblanks(n)
2464 #@+node:ekr.20041005105605.211: *5* at.putInitialComment
2465 def putInitialComment(self): # pragma: no cover
2466 c = self.c
2467 s2 = c.config.output_initial_comment
2468 if s2:
2469 lines = s2.split("\\n")
2470 for line in lines:
2471 line = line.replace("@date", time.asctime())
2472 if line:
2473 self.putSentinel("@comment " + line)
2474 #@+node:ekr.20190111172114.1: *5* at.replaceFile & helpers
2475 def replaceFile(self, contents, encoding, fileName, root, ignoreBlankLines=False):
2476 """
2477 Write or create the given file from the contents.
2478 Return True if the original file was changed.
2479 """
2480 at, c = self, self.c
2481 if root:
2482 root.clearDirty()
2483 #
2484 # Create the timestamp (only for messages).
2485 if c.config.getBool('log-show-save-time', default=False): # pragma: no cover
2486 format = c.config.getString('log-timestamp-format') or "%H:%M:%S"
2487 timestamp = time.strftime(format) + ' '
2488 else:
2489 timestamp = ''
2490 #
2491 # Adjust the contents.
2492 assert isinstance(contents, str), g.callers()
2493 if at.output_newline != '\n': # pragma: no cover
2494 contents = contents.replace('\r', '').replace('\n', at.output_newline)
2495 #
2496 # If file does not exist, create it from the contents.
2497 fileName = g.os_path_realpath(fileName)
2498 sfn = g.shortFileName(fileName)
2499 if not g.os_path_exists(fileName):
2500 ok = g.writeFile(contents, encoding, fileName)
2501 if ok:
2502 c.setFileTimeStamp(fileName)
2503 if not g.unitTesting:
2504 g.es(f"{timestamp}created: {fileName}") # pragma: no cover
2505 if root:
2506 # Fix bug 889175: Remember the full fileName.
2507 at.rememberReadPath(fileName, root)
2508 at.checkPythonCode(contents, fileName, root)
2509 else:
2510 at.addToOrphanList(root) # pragma: no cover
2511 # No original file to change. Return value tested by a unit test.
2512 return False # No change to original file.
2513 #
2514 # Compare the old and new contents.
2515 old_contents = g.readFileIntoUnicodeString(fileName,
2516 encoding=at.encoding, silent=True)
2517 if not old_contents:
2518 old_contents = ''
2519 unchanged = (
2520 contents == old_contents
2521 or (not at.explicitLineEnding and at.compareIgnoringLineEndings(old_contents, contents))
2522 or ignoreBlankLines and at.compareIgnoringBlankLines(old_contents, contents))
2523 if unchanged:
2524 at.unchangedFiles += 1
2525 if not g.unitTesting and c.config.getBool(
2526 'report-unchanged-files', default=True):
2527 g.es(f"{timestamp}unchanged: {sfn}") # pragma: no cover
2528 # Leo 5.6: Check unchanged files.
2529 at.checkPyflakes(contents, fileName, root)
2530 return False # No change to original file.
2531 #
2532 # Warn if we are only adjusting the line endings.
2533 if at.explicitLineEnding: # pragma: no cover
2534 ok = (
2535 at.compareIgnoringLineEndings(old_contents, contents) or
2536 ignoreBlankLines and at.compareIgnoringLineEndings(
2537 old_contents, contents))
2538 if not ok:
2539 g.warning("correcting line endings in:", fileName)
2540 #
2541 # Write a changed file.
2542 ok = g.writeFile(contents, encoding, fileName)
2543 if ok:
2544 c.setFileTimeStamp(fileName)
2545 if not g.unitTesting:
2546 g.es(f"{timestamp}wrote: {sfn}") # pragma: no cover
2547 else: # pragma: no cover
2548 g.error('error writing', sfn)
2549 g.es('not written:', sfn)
2550 at.addToOrphanList(root)
2551 at.checkPythonCode(contents, fileName, root)
2552 # Check *after* writing the file.
2553 return ok
2554 #@+node:ekr.20190114061452.27: *6* at.compareIgnoringBlankLines
2555 def compareIgnoringBlankLines(self, s1, s2): # pragma: no cover
2556 """Compare two strings, ignoring blank lines."""
2557 assert isinstance(s1, str), g.callers()
2558 assert isinstance(s2, str), g.callers()
2559 if s1 == s2:
2560 return True
2561 s1 = g.removeBlankLines(s1)
2562 s2 = g.removeBlankLines(s2)
2563 return s1 == s2
2564 #@+node:ekr.20190114061452.28: *6* at.compareIgnoringLineEndings
2565 def compareIgnoringLineEndings(self, s1, s2): # pragma: no cover
2566 """Compare two strings, ignoring line endings."""
2567 assert isinstance(s1, str), (repr(s1), g.callers())
2568 assert isinstance(s2, str), (repr(s2), g.callers())
2569 if s1 == s2:
2570 return True
2571 # Wrong: equivalent to ignoreBlankLines!
2572 # s1 = s1.replace('\n','').replace('\r','')
2573 # s2 = s2.replace('\n','').replace('\r','')
2574 s1 = s1.replace('\r', '')
2575 s2 = s2.replace('\r', '')
2576 return s1 == s2
2577 #@+node:ekr.20211029052041.1: *5* at.scanRootForSectionDelims
2578 def scanRootForSectionDelims(self, root):
2579 """
2580 Scan root.b for an "@section-delims" directive.
2581 Set section_delim1 and section_delim2 ivars.
2582 """
2583 at = self
2584 # Set defaults.
2585 at.section_delim1 = '<<'
2586 at.section_delim2 = '>>'
2587 # Scan root.b.
2588 lines = []
2589 for s in g.splitLines(root.b):
2590 m = g.g_section_delims_pat.match(s)
2591 if m:
2592 lines.append(s)
2593 at.section_delim1 = m.group(1)
2594 at.section_delim2 = m.group(2)
2595 # Disallow multiple directives.
2596 if len(lines) > 1: # pragma: no cover
2597 at.error(f"Multiple @section-delims directives in {root.h}")
2598 g.es_print('using default delims')
2599 at.section_delim1 = '<<'
2600 at.section_delim2 = '>>'
2601 #@+node:ekr.20090514111518.5665: *5* at.tabNannyNode
2602 def tabNannyNode(self, p, body):
2603 try:
2604 readline = g.ReadLinesClass(body).next
2605 tabnanny.process_tokens(tokenize.generate_tokens(readline))
2606 except IndentationError:
2607 if g.unitTesting:
2608 raise
2609 junk2, msg, junk = sys.exc_info()
2610 g.error("IndentationError in", p.h)
2611 g.es('', str(msg))
2612 except tokenize.TokenError:
2613 if g.unitTesting:
2614 raise
2615 junk3, msg, junk = sys.exc_info()
2616 g.error("TokenError in", p.h)
2617 g.es('', str(msg))
2618 except tabnanny.NannyNag:
2619 if g.unitTesting:
2620 raise
2621 junk4, nag, junk = sys.exc_info()
2622 badline = nag.get_lineno()
2623 line = nag.get_line()
2624 message = nag.get_msg()
2625 g.error("indentation error in", p.h, "line", badline)
2626 g.es(message)
2627 line2 = repr(str(line))[1:-1]
2628 g.es("offending line:\n", line2)
2629 except Exception:
2630 g.trace("unexpected exception")
2631 g.es_exception()
2632 raise
2633 #@+node:ekr.20041005105605.216: *5* at.warnAboutOrpanAndIgnoredNodes
2634 # Called from putFile.
2636 def warnAboutOrphandAndIgnoredNodes(self): # pragma: no cover
2637 # Always warn, even when language=="cweb"
2638 at, root = self, self.root
2639 if at.errors:
2640 return # No need to repeat this.
2641 for p in root.self_and_subtree(copy=False):
2642 if not p.v.isVisited():
2643 at.writeError("Orphan node: " + p.h)
2644 if p.hasParent():
2645 g.blue("parent node:", p.parent().h)
2646 p = root.copy()
2647 after = p.nodeAfterTree()
2648 while p and p != after:
2649 if p.isAtAllNode():
2650 p.moveToNodeAfterTree()
2651 else:
2652 # #1050: test orphan bit.
2653 if p.isOrphan():
2654 at.writeError("Orphan node: " + p.h)
2655 if p.hasParent():
2656 g.blue("parent node:", p.parent().h)
2657 p.moveToThreadNext()
2658 #@+node:ekr.20041005105605.217: *5* at.writeError
2659 def writeError(self, message): # pragma: no cover
2660 """Issue an error while writing an @<file> node."""
2661 at = self
2662 if at.errors == 0:
2663 fn = at.targetFileName or 'unnamed file'
2664 g.es_error(f"errors writing: {fn}")
2665 at.error(message)
2666 at.addToOrphanList(at.root)
2667 #@+node:ekr.20041005105605.218: *5* at.writeException
2668 def writeException(self, fileName, root): # pragma: no cover
2669 at = self
2670 g.error("exception writing:", fileName)
2671 g.es_exception()
2672 if getattr(at, 'outputFile', None):
2673 at.outputFile.flush()
2674 at.outputFile.close()
2675 at.outputFile = None
2676 at.remove(fileName)
2677 at.addToOrphanList(root)
2678 #@+node:ekr.20041005105605.219: *3* at.Utilites
2679 #@+node:ekr.20041005105605.220: *4* at.error & printError
2680 def error(self, *args): # pragma: no cover
2681 at = self
2682 at.printError(*args)
2683 at.errors += 1
2685 def printError(self, *args): # pragma: no cover
2686 """Print an error message that may contain non-ascii characters."""
2687 at = self
2688 if at.errors:
2689 g.error(*args)
2690 else:
2691 g.warning(*args)
2692 #@+node:ekr.20041005105605.221: *4* at.exception
2693 def exception(self, message): # pragma: no cover
2694 self.error(message)
2695 g.es_exception()
2696 #@+node:ekr.20050104131929: *4* at.file operations...
2697 # Error checking versions of corresponding functions in Python's os module.
2698 #@+node:ekr.20050104131820: *5* at.chmod
2699 def chmod(self, fileName, mode): # pragma: no cover
2700 # Do _not_ call self.error here.
2701 if mode is None:
2702 return
2703 try:
2704 os.chmod(fileName, mode)
2705 except Exception:
2706 g.es("exception in os.chmod", fileName)
2707 g.es_exception()
2709 #@+node:ekr.20050104132018: *5* at.remove
2710 def remove(self, fileName): # pragma: no cover
2711 if not fileName:
2712 g.trace('No file name', g.callers())
2713 return False
2714 try:
2715 os.remove(fileName)
2716 return True
2717 except Exception:
2718 if not g.unitTesting:
2719 self.error(f"exception removing: {fileName}")
2720 g.es_exception()
2721 return False
2722 #@+node:ekr.20050104132026: *5* at.stat
2723 def stat(self, fileName): # pragma: no cover
2724 """Return the access mode of named file, removing any setuid, setgid, and sticky bits."""
2725 # Do _not_ call self.error here.
2726 try:
2727 mode = (os.stat(fileName))[0] & (7 * 8 * 8 + 7 * 8 + 7) # 0777
2728 except Exception:
2729 mode = None
2730 return mode
2732 #@+node:ekr.20090530055015.6023: *4* at.get/setPathUa
2733 def getPathUa(self, p):
2734 if hasattr(p.v, 'tempAttributes'):
2735 d = p.v.tempAttributes.get('read-path', {})
2736 return d.get('path')
2737 return ''
2739 def setPathUa(self, p, path):
2740 if not hasattr(p.v, 'tempAttributes'):
2741 p.v.tempAttributes = {}
2742 d = p.v.tempAttributes.get('read-path', {})
2743 d['path'] = path
2744 p.v.tempAttributes['read-path'] = d
2745 #@+node:ekr.20081216090156.4: *4* at.parseUnderindentTag
2746 # Important: this is part of the *write* logic.
2747 # It is called from at.os and at.putIndent.
2749 def parseUnderindentTag(self, s): # pragma: no cover
2750 tag = self.underindentEscapeString
2751 s2 = s[len(tag) :]
2752 # To be valid, the escape must be followed by at least one digit.
2753 i = 0
2754 while i < len(s2) and s2[i].isdigit():
2755 i += 1
2756 if i > 0:
2757 n = int(s2[:i])
2758 # Bug fix: 2012/06/05: remove any period following the count.
2759 # This is a new convention.
2760 if i < len(s2) and s2[i] == '.':
2761 i += 1
2762 return n, s2[i:]
2763 return 0, s
2764 #@+node:ekr.20090712050729.6017: *4* at.promptForDangerousWrite
2765 def promptForDangerousWrite(self, fileName, message=None): # pragma: no cover
2766 """Raise a dialog asking the user whether to overwrite an existing file."""
2767 at, c, root = self, self.c, self.root
2768 if at.cancelFlag:
2769 assert at.canCancelFlag
2770 return False
2771 if at.yesToAll:
2772 assert at.canCancelFlag
2773 return True
2774 if root and root.h.startswith('@auto-rst'):
2775 # Fix bug 50: body text lost switching @file to @auto-rst
2776 # Refuse to convert any @<file> node to @auto-rst.
2777 d = root.v.at_read if hasattr(root.v, 'at_read') else {}
2778 aList = sorted(d.get(fileName, []))
2779 for h in aList:
2780 if not h.startswith('@auto-rst'):
2781 g.es('can not convert @file to @auto-rst!', color='red')
2782 g.es('reverting to:', h)
2783 root.h = h
2784 c.redraw()
2785 return False
2786 if message is None:
2787 message = (
2788 f"{g.splitLongFileName(fileName)}\n"
2789 f"{g.tr('already exists.')}\n"
2790 f"{g.tr('Overwrite this file?')}")
2791 result = g.app.gui.runAskYesNoCancelDialog(c,
2792 title='Overwrite existing file?',
2793 yesToAllMessage="Yes To &All",
2794 message=message,
2795 cancelMessage="&Cancel (No To All)",
2796 )
2797 if at.canCancelFlag:
2798 # We are in the writeAll logic so these flags can be set.
2799 if result == 'cancel':
2800 at.cancelFlag = True
2801 elif result == 'yes-to-all':
2802 at.yesToAll = True
2803 return result in ('yes', 'yes-to-all')
2804 #@+node:ekr.20120112084820.10001: *4* at.rememberReadPath
2805 def rememberReadPath(self, fn, p):
2806 """
2807 Remember the files that have been read *and*
2808 the full headline (@<file> type) that caused the read.
2809 """
2810 v = p.v
2811 # Fix bug #50: body text lost switching @file to @auto-rst
2812 if not hasattr(v, 'at_read'):
2813 v.at_read = {} # pragma: no cover
2814 d = v.at_read
2815 aSet = d.get(fn, set())
2816 aSet.add(p.h)
2817 d[fn] = aSet
2818 #@+node:ekr.20080923070954.4: *4* at.scanAllDirectives
2819 def scanAllDirectives(self, p):
2820 """
2821 Scan p and p's ancestors looking for directives,
2822 setting corresponding AtFile ivars.
2823 """
2824 at, c = self, self.c
2825 d = c.scanAllDirectives(p)
2826 #
2827 # Language & delims: Tricky.
2828 lang_dict = d.get('lang-dict') or {}
2829 delims, language = None, None
2830 if lang_dict:
2831 # There was an @delims or @language directive.
2832 language = lang_dict.get('language')
2833 delims = lang_dict.get('delims')
2834 if not language:
2835 # No language directive. Look for @<file> nodes.
2836 # Do *not* used.get('language')!
2837 language = g.getLanguageFromAncestorAtFileNode(p) or 'python'
2838 at.language = language
2839 if not delims:
2840 delims = g.set_delims_from_language(language)
2841 #
2842 # Previously, setting delims was sometimes skipped, depending on kwargs.
2843 #@+<< Set comment strings from delims >>
2844 #@+node:ekr.20080923070954.13: *5* << Set comment strings from delims >> (at.scanAllDirectives)
2845 delim1, delim2, delim3 = delims
2846 # Use single-line comments if we have a choice.
2847 # delim1,delim2,delim3 now correspond to line,start,end
2848 if delim1:
2849 at.startSentinelComment = delim1
2850 at.endSentinelComment = "" # Must not be None.
2851 elif delim2 and delim3:
2852 at.startSentinelComment = delim2
2853 at.endSentinelComment = delim3
2854 else: # pragma: no cover
2855 #
2856 # Emergency!
2857 #
2858 # Issue an error only if at.language has been set.
2859 # This suppresses a message from the markdown importer.
2860 if not g.unitTesting and at.language:
2861 g.trace(repr(at.language), g.callers())
2862 g.es_print("unknown language: using Python comment delimiters")
2863 g.es_print("c.target_language:", c.target_language)
2864 at.startSentinelComment = "#" # This should never happen!
2865 at.endSentinelComment = ""
2866 #@-<< Set comment strings from delims >>
2867 #
2868 # Easy cases
2869 at.encoding = d.get('encoding') or c.config.default_derived_file_encoding
2870 lineending = d.get('lineending')
2871 at.explicitLineEnding = bool(lineending)
2872 at.output_newline = lineending or g.getOutputNewline(c=c)
2873 at.page_width = d.get('pagewidth') or c.page_width
2874 at.tab_width = d.get('tabwidth') or c.tab_width
2875 return {
2876 "encoding": at.encoding,
2877 "language": at.language,
2878 "lineending": at.output_newline,
2879 "pagewidth": at.page_width,
2880 "path": d.get('path'),
2881 "tabwidth": at.tab_width,
2882 }
2883 #@+node:ekr.20120110174009.9965: *4* at.shouldPromptForDangerousWrite
2884 def shouldPromptForDangerousWrite(self, fn, p): # pragma: no cover
2885 """
2886 Return True if Leo should warn the user that p is an @<file> node that
2887 was not read during startup. Writing that file might cause data loss.
2889 See #50: https://github.com/leo-editor/leo-editor/issues/50
2890 """
2891 trace = 'save' in g.app.debug
2892 sfn = g.shortFileName(fn)
2893 c = self.c
2894 efc = g.app.externalFilesController
2895 if p.isAtNoSentFileNode():
2896 # #1450.
2897 # No danger of overwriting a file.
2898 # It was never read.
2899 return False
2900 if not g.os_path_exists(fn):
2901 # No danger of overwriting fn.
2902 if trace:
2903 g.trace('Return False: does not exist:', sfn)
2904 return False
2905 # #1347: Prompt if the external file is newer.
2906 if efc:
2907 # Like c.checkFileTimeStamp.
2908 if c.sqlite_connection and c.mFileName == fn:
2909 # sqlite database file is never actually overwriten by Leo,
2910 # so do *not* check its timestamp.
2911 pass
2912 elif efc.has_changed(fn):
2913 if trace:
2914 g.trace('Return True: changed:', sfn)
2915 return True
2916 if hasattr(p.v, 'at_read'):
2917 # Fix bug #50: body text lost switching @file to @auto-rst
2918 d = p.v.at_read
2919 for k in d:
2920 # Fix bug # #1469: make sure k still exists.
2921 if (
2922 os.path.exists(k) and os.path.samefile(k, fn)
2923 and p.h in d.get(k, set())
2924 ):
2925 d[fn] = d[k]
2926 if trace:
2927 g.trace('Return False: in p.v.at_read:', sfn)
2928 return False
2929 aSet = d.get(fn, set())
2930 if trace:
2931 g.trace(f"Return {p.h not in aSet()}: p.h not in aSet(): {sfn}")
2932 return p.h not in aSet
2933 if trace:
2934 g.trace('Return True: never read:', sfn)
2935 return True # The file was never read.
2936 #@+node:ekr.20041005105605.20: *4* at.warnOnReadOnlyFile
2937 def warnOnReadOnlyFile(self, fn):
2938 # os.access() may not exist on all platforms.
2939 try:
2940 read_only = not os.access(fn, os.W_OK)
2941 except AttributeError:
2942 read_only = False
2943 if read_only:
2944 g.error("read only:", fn) # pragma: no cover
2945 #@-others
2946atFile = AtFile # compatibility
2947#@+node:ekr.20180602102448.1: ** class FastAtRead
2948class FastAtRead:
2949 """
2950 Read an exteral file, created from an @file tree.
2951 This is Vitalije's code, edited by EKR.
2952 """
2954 #@+others
2955 #@+node:ekr.20211030193146.1: *3* fast_at.__init__
2956 def __init__(self, c, gnx2vnode):
2958 self.c = c
2959 assert gnx2vnode is not None
2960 self.gnx2vnode = gnx2vnode # The global fc.gnxDict. Keys are gnx's, values are vnodes.
2961 self.path = None
2962 self.root = None
2963 # compiled patterns...
2964 self.after_pat = None
2965 self.all_pat = None
2966 self.code_pat = None
2967 self.comment_pat = None
2968 self.delims_pat = None
2969 self.doc_pat = None
2970 self.first_pat = None
2971 self.last_pat = None
2972 self.node_start_pat = None
2973 self.others_pat = None
2974 self.ref_pat = None
2975 self.section_delims_pat = None
2976 #@+node:ekr.20180602103135.3: *3* fast_at.get_patterns
2977 #@@nobeautify
2979 def get_patterns(self, comment_delims):
2980 """Create regex patterns for the given comment delims."""
2981 # This must be a function, because of @comments & @delims.
2982 comment_delim_start, comment_delim_end = comment_delims
2983 delim1 = re.escape(comment_delim_start)
2984 delim2 = re.escape(comment_delim_end or '')
2985 ref = g.angleBrackets(r'(.*)')
2986 table = (
2987 # These patterns must be mutually exclusive.
2988 ('after', fr'^\s*{delim1}@afterref{delim2}$'), # @afterref
2989 ('all', fr'^(\s*){delim1}@(\+|-)all\b(.*){delim2}$'), # @all
2990 ('code', fr'^\s*{delim1}@@c(ode)?{delim2}$'), # @c and @code
2991 ('comment', fr'^\s*{delim1}@@comment(.*){delim2}'), # @comment
2992 ('delims', fr'^\s*{delim1}@delims(.*){delim2}'), # @delims
2993 ('doc', fr'^\s*{delim1}@\+(at|doc)?(\s.*?)?{delim2}\n'), # @doc or @
2994 ('first', fr'^\s*{delim1}@@first{delim2}$'), # @first
2995 ('last', fr'^\s*{delim1}@@last{delim2}$'), # @last
2996 # @node
2997 ('node_start', fr'^(\s*){delim1}@\+node:([^:]+): \*(\d+)?(\*?) (.*){delim2}$'),
2998 ('others', fr'^(\s*){delim1}@(\+|-)others\b(.*){delim2}$'), # @others
2999 ('ref', fr'^(\s*){delim1}@(\+|-){ref}\s*{delim2}$'), # section ref
3000 # @section-delims
3001 ('section_delims', fr'^\s*{delim1}@@section-delims[ \t]+([^ \w\n\t]+)[ \t]+([^ \w\n\t]+)[ \t]*{delim2}$'),
3002 )
3003 # Set the ivars.
3004 for (name, pattern) in table:
3005 ivar = f"{name}_pat"
3006 assert hasattr(self, ivar), ivar
3007 setattr(self, ivar, re.compile(pattern))
3008 #@+node:ekr.20180602103135.2: *3* fast_at.scan_header
3009 header_pattern = re.compile(
3010 r'''
3011 ^(.+)@\+leo
3012 (-ver=(\d+))?
3013 (-thin)?
3014 (-encoding=(.*)(\.))?
3015 (.*)$''',
3016 re.VERBOSE,
3017 )
3019 def scan_header(self, lines):
3020 """
3021 Scan for the header line, which follows any @first lines.
3022 Return (delims, first_lines, i+1) or None
3023 """
3024 first_lines: List[str] = []
3025 i = 0 # To keep some versions of pylint happy.
3026 for i, line in enumerate(lines):
3027 m = self.header_pattern.match(line)
3028 if m:
3029 delims = m.group(1), m.group(8) or ''
3030 return delims, first_lines, i + 1
3031 first_lines.append(line)
3032 return None # pragma: no cover (defensive)
3033 #@+node:ekr.20180602103135.8: *3* fast_at.scan_lines
3034 def scan_lines(self, comment_delims, first_lines, lines, path, start):
3035 """Scan all lines of the file, creating vnodes."""
3036 #@+<< init scan_lines >>
3037 #@+node:ekr.20180602103135.9: *4* << init scan_lines >>
3038 #
3039 # Simple vars...
3040 afterref = False # True: the next line follows @afterref.
3041 clone_v = None # The root of the clone tree.
3042 comment_delim1, comment_delim2 = comment_delims # The start/end *comment* delims.
3043 doc_skip = (comment_delim1 + '\n', comment_delim2 + '\n') # To handle doc parts.
3044 first_i = 0 # Index into first array.
3045 in_doc = False # True: in @doc parts.
3046 is_cweb = comment_delim1 == '@q@' and comment_delim2 == '@>' # True: cweb hack in effect.
3047 indent = 0 # The current indentation.
3048 level_stack = [] # Entries are (vnode, in_clone_tree)
3049 n_last_lines = 0 # The number of @@last directives seen.
3050 root_gnx_adjusted = False # True: suppress final checks.
3051 # #1065 so reads will not create spurious child nodes.
3052 root_seen = False # False: The next +@node sentinel denotes the root, regardless of gnx.
3053 section_delim1 = '<<'
3054 section_delim2 = '>>'
3055 section_reference_seen = False
3056 sentinel = comment_delim1 + '@' # Faster than a regex!
3057 # The stack is updated when at+others, at+<section>, or at+all is seen.
3058 stack = [] # Entries are (gnx, indent, body)
3059 # The spelling of at-verbatim sentinel
3060 verbatim_line = comment_delim1 + '@verbatim' + comment_delim2 + '\n'
3061 verbatim = False # True: the next line must be added without change.
3062 #
3063 # Init the parent vnode.
3064 #
3065 root_gnx = gnx = self.root.gnx
3066 context = self.c
3067 parent_v = self.root.v
3068 root_v = parent_v # Does not change.
3069 level_stack.append((root_v, False),)
3070 #
3071 # Init the gnx dict last.
3072 #
3073 gnx2vnode = self.gnx2vnode # Keys are gnx's, values are vnodes.
3074 gnx2body = {} # Keys are gnxs, values are list of body lines.
3075 gnx2vnode[gnx] = parent_v # Add gnx to the keys
3076 # Add gnx to the keys.
3077 # Body is the list of lines presently being accumulated.
3078 gnx2body[gnx] = body = first_lines
3079 #
3080 # Set the patterns
3081 self.get_patterns(comment_delims)
3082 #@-<< init scan_lines >>
3083 i = 0 # To keep pylint happy.
3084 for i, line in enumerate(lines[start:]):
3085 # Strip the line only once.
3086 strip_line = line.strip()
3087 if afterref:
3088 #@+<< handle afterref line>>
3089 #@+node:ekr.20211102052251.1: *4* << handle afterref line >>
3090 if body: # a List of lines.
3091 body[-1] = body[-1].rstrip() + line
3092 else:
3093 body = [line] # pragma: no cover
3094 afterref = False
3095 #@-<< handle afterref line>>
3096 continue
3097 if verbatim:
3098 #@+<< handle verbatim line >>
3099 #@+node:ekr.20211102052518.1: *4* << handle verbatim line >>
3100 # Previous line was verbatim *sentinel*. Append this line as it is.
3101 body.append(line)
3102 verbatim = False
3103 #@-<< handle verbatim line >>
3104 continue
3105 if line == verbatim_line: # <delim>@verbatim.
3106 verbatim = True
3107 continue
3108 #@+<< finalize line >>
3109 #@+node:ekr.20180602103135.10: *4* << finalize line >>
3110 # Undo the cweb hack.
3111 if is_cweb and line.startswith(sentinel):
3112 line = line[: len(sentinel)] + line[len(sentinel) :].replace('@@', '@')
3113 # Adjust indentation.
3114 if indent and line[:indent].isspace() and len(line) > indent:
3115 line = line[indent:]
3116 #@-<< finalize line >>
3117 if not in_doc and not strip_line.startswith(sentinel): # Faster than a regex!
3118 body.append(line)
3119 continue
3120 # These three sections might clear in_doc.
3121 #@+<< handle @others >>
3122 #@+node:ekr.20180602103135.14: *4* << handle @others >>
3123 m = self.others_pat.match(line)
3124 if m:
3125 in_doc = False
3126 if m.group(2) == '+': # opening sentinel
3127 body.append(f"{m.group(1)}@others{m.group(3) or ''}\n")
3128 stack.append((gnx, indent, body))
3129 indent += m.end(1) # adjust current identation
3130 else: # closing sentinel.
3131 # m.group(2) is '-' because the pattern matched.
3132 gnx, indent, body = stack.pop()
3133 continue
3134 #@-<< handle @others >>
3135 #@+<< handle section refs >>
3136 #@+node:ekr.20180602103135.18: *4* << handle section refs >>
3137 # Note: scan_header sets *comment* delims, not *section* delims.
3138 # This section coordinates with the section that handles @section-delims.
3139 m = self.ref_pat.match(line)
3140 if m:
3141 in_doc = False
3142 if m.group(2) == '+':
3143 # Any later @section-delims directive is a serious error.
3144 # This kind of error should have been caught by Leo's atFile write logic.
3145 section_reference_seen = True
3146 # open sentinel.
3147 body.append(m.group(1) + section_delim1 + m.group(3) + section_delim2 + '\n')
3148 stack.append((gnx, indent, body))
3149 indent += m.end(1)
3150 elif stack:
3151 # m.group(2) is '-' because the pattern matched.
3152 gnx, indent, body = stack.pop() # #1232: Only if the stack exists.
3153 continue # 2021/10/29: *always* continue.
3154 #@-<< handle section refs >>
3155 #@+<< handle node_start >>
3156 #@+node:ekr.20180602103135.19: *4* << handle node_start >>
3157 m = self.node_start_pat.match(line)
3158 if m:
3159 in_doc = False
3160 gnx, head = m.group(2), m.group(5)
3161 level = int(m.group(3)) if m.group(3) else 1 + len(m.group(4))
3162 # m.group(3) is the level number, m.group(4) is the number of stars.
3163 v = gnx2vnode.get(gnx)
3164 #
3165 # Case 1: The root @file node. Don't change the headline.
3166 if not root_seen and not v and not g.unitTesting:
3167 # Don't warn about a gnx mismatch in the root.
3168 root_gnx_adjusted = True # pragma: no cover
3169 if not root_seen:
3170 # Fix #1064: The node represents the root, regardless of the gnx!
3171 root_seen = True
3172 clone_v = None
3173 gnx2body[gnx] = body = []
3174 # This case can happen, but not in unit tests.
3175 if not v: # pragma: no cover
3176 # Fix #1064.
3177 v = root_v
3178 # This message is annoying when using git-diff.
3179 # if gnx != root_gnx:
3180 # g.es_print("using gnx from external file: %s" % (v.h), color='blue')
3181 gnx2vnode[gnx] = v
3182 v.fileIndex = gnx
3183 v.children = []
3184 continue
3185 #
3186 # Case 2: We are scanning the descendants of a clone.
3187 parent_v, clone_v = level_stack[level - 2]
3188 if v and clone_v:
3189 # The last version of the body and headline wins..
3190 gnx2body[gnx] = body = []
3191 v._headString = head
3192 # Update the level_stack.
3193 level_stack = level_stack[: level - 1]
3194 level_stack.append((v, clone_v),)
3195 # Always clear the children!
3196 v.children = []
3197 parent_v.children.append(v)
3198 continue
3199 #
3200 # Case 3: we are not already scanning the descendants of a clone.
3201 if v:
3202 # The *start* of a clone tree. Reset the children.
3203 clone_v = v
3204 v.children = []
3205 else:
3206 # Make a new vnode.
3207 v = leoNodes.VNode(context=context, gnx=gnx)
3208 #
3209 # The last version of the body and headline wins.
3210 gnx2vnode[gnx] = v
3211 gnx2body[gnx] = body = []
3212 v._headString = head
3213 #
3214 # Update the stack.
3215 level_stack = level_stack[: level - 1]
3216 level_stack.append((v, clone_v),)
3217 #
3218 # Update the links.
3219 assert v != root_v
3220 parent_v.children.append(v)
3221 v.parents.append(parent_v)
3222 continue
3223 #@-<< handle node_start >>
3224 if in_doc:
3225 #@+<< handle @c or @code >>
3226 #@+node:ekr.20211031033532.1: *4* << handle @c or @code >>
3227 # When delim_end exists the doc block:
3228 # - begins with the opening delim, alone on its own line
3229 # - ends with the closing delim, alone on its own line.
3230 # Both of these lines should be skipped.
3231 #
3232 # #1496: Retire the @doc convention.
3233 # An empty line is no longer a sentinel.
3234 if comment_delim2 and line in doc_skip:
3235 # doc_skip is (comment_delim1 + '\n', delim_end + '\n')
3236 continue
3237 #
3238 # Check for @c or @code.
3239 m = self.code_pat.match(line)
3240 if m:
3241 in_doc = False
3242 body.append('@code\n' if m.group(1) else '@c\n')
3243 continue
3244 #@-<< handle @c or @code >>
3245 else:
3246 #@+<< handle @ or @doc >>
3247 #@+node:ekr.20211031033754.1: *4* << handle @ or @doc >>
3248 m = self.doc_pat.match(line)
3249 if m:
3250 # @+at or @+doc?
3251 doc = '@doc' if m.group(1) == 'doc' else '@'
3252 doc2 = m.group(2) or '' # Trailing text.
3253 if doc2:
3254 body.append(f"{doc}{doc2}\n")
3255 else:
3256 body.append(doc + '\n')
3257 # Enter @doc mode.
3258 in_doc = True
3259 continue
3260 #@-<< handle @ or @doc >>
3261 if line.startswith(comment_delim1 + '@-leo'): # Faster than a regex!
3262 # The @-leo sentinel adds *nothing* to the text.
3263 i += 1
3264 break
3265 # Order doesn't matter.
3266 #@+<< handle @all >>
3267 #@+node:ekr.20180602103135.13: *4* << handle @all >>
3268 m = self.all_pat.match(line)
3269 if m:
3270 # @all tells Leo's *write* code not to check for undefined sections.
3271 # Here, in the read code, we merely need to add it to the body.
3272 # Pushing and popping the stack may not be necessary, but it can't hurt.
3273 if m.group(2) == '+': # opening sentinel
3274 body.append(f"{m.group(1)}@all{m.group(3) or ''}\n")
3275 stack.append((gnx, indent, body))
3276 else: # closing sentinel.
3277 # m.group(2) is '-' because the pattern matched.
3278 gnx, indent, body = stack.pop()
3279 gnx2body[gnx] = body
3280 continue
3281 #@-<< handle @all >>
3282 #@+<< handle afterref >>
3283 #@+node:ekr.20180603063102.1: *4* << handle afterref >>
3284 m = self.after_pat.match(line)
3285 if m:
3286 afterref = True
3287 continue
3288 #@-<< handle afterref >>
3289 #@+<< handle @first and @last >>
3290 #@+node:ekr.20180606053919.1: *4* << handle @first and @last >>
3291 m = self.first_pat.match(line)
3292 if m:
3293 # pylint: disable=no-else-continue
3294 if 0 <= first_i < len(first_lines):
3295 body.append('@first ' + first_lines[first_i])
3296 first_i += 1
3297 continue
3298 else: # pragma: no cover
3299 g.trace(f"\ntoo many @first lines: {path}")
3300 print('@first is valid only at the start of @<file> nodes\n')
3301 g.printObj(first_lines, tag='first_lines')
3302 g.printObj(lines[start : i + 2], tag='lines[start:i+2]')
3303 continue
3304 m = self.last_pat.match(line)
3305 if m:
3306 # Just increment the count of the expected last lines.
3307 # We'll fill in the @last line directives after we see the @-leo directive.
3308 n_last_lines += 1
3309 continue
3310 #@-<< handle @first and @last >>
3311 #@+<< handle @comment >>
3312 #@+node:ekr.20180621050901.1: *4* << handle @comment >>
3313 # http://leoeditor.com/directives.html#part-4-dangerous-directives
3314 m = self.comment_pat.match(line)
3315 if m:
3316 # <1, 2 or 3 comment delims>
3317 delims = m.group(1).strip()
3318 # Whatever happens, retain the @delims line.
3319 body.append(f"@comment {delims}\n")
3320 delim1, delim2, delim3 = g.set_delims_from_string(delims)
3321 # delim1 is always the single-line delimiter.
3322 if delim1:
3323 comment_delim1, comment_delim2 = delim1, ''
3324 else:
3325 comment_delim1, comment_delim2 = delim2, delim3
3326 #
3327 # Within these delimiters:
3328 # - double underscores represent a newline.
3329 # - underscores represent a significant space,
3330 comment_delim1 = comment_delim1.replace('__', '\n').replace('_', ' ')
3331 comment_delim2 = comment_delim2.replace('__', '\n').replace('_', ' ')
3332 # Recalculate all delim-related values
3333 doc_skip = (comment_delim1 + '\n', comment_delim2 + '\n')
3334 is_cweb = comment_delim1 == '@q@' and comment_delim2 == '@>'
3335 sentinel = comment_delim1 + '@'
3336 #
3337 # Recalculate the patterns.
3338 comment_delims = comment_delim1, comment_delim2
3339 self.get_patterns(comment_delims)
3340 continue
3341 #@-<< handle @comment >>
3342 #@+<< handle @delims >>
3343 #@+node:ekr.20180608104836.1: *4* << handle @delims >>
3344 m = self.delims_pat.match(line)
3345 if m:
3346 # Get 1 or 2 comment delims
3347 # Whatever happens, retain the original @delims line.
3348 delims = m.group(1).strip()
3349 body.append(f"@delims {delims}\n")
3350 #
3351 # Parse the delims.
3352 self.delims_pat = re.compile(r'^([^ ]+)\s*([^ ]+)?')
3353 m2 = self.delims_pat.match(delims)
3354 if not m2: # pragma: no cover
3355 g.trace(f"Ignoring invalid @delims: {line!r}")
3356 continue
3357 comment_delim1 = m2.group(1)
3358 comment_delim2 = m2.group(2) or ''
3359 #
3360 # Within these delimiters:
3361 # - double underscores represent a newline.
3362 # - underscores represent a significant space,
3363 comment_delim1 = comment_delim1.replace('__', '\n').replace('_', ' ')
3364 comment_delim2 = comment_delim2.replace('__', '\n').replace('_', ' ')
3365 # Recalculate all delim-related values
3366 doc_skip = (comment_delim1 + '\n', comment_delim2 + '\n')
3367 is_cweb = comment_delim1 == '@q@' and comment_delim2 == '@>'
3368 sentinel = comment_delim1 + '@'
3369 #
3370 # Recalculate the patterns
3371 comment_delims = comment_delim1, comment_delim2
3372 self.get_patterns(comment_delims)
3373 continue
3374 #@-<< handle @delims >>
3375 #@+<< handle @section-delims >>
3376 #@+node:ekr.20211030033211.1: *4* << handle @section-delims >>
3377 m = self.section_delims_pat.match(line)
3378 if m:
3379 if section_reference_seen: # pragma: no cover
3380 # This is a serious error.
3381 # This kind of error should have been caught by Leo's atFile write logic.
3382 g.es_print('section-delims seen after a section reference', color='red')
3383 else:
3384 # Carefully update the section reference pattern!
3385 section_delim1 = d1 = re.escape(m.group(1))
3386 section_delim2 = d2 = re.escape(m.group(2) or '')
3387 self.ref_pat = re.compile(fr'^(\s*){comment_delim1}@(\+|-){d1}(.*){d2}\s*{comment_delim2}$')
3388 body.append(f"@section-delims {m.group(1)} {m.group(2)}\n")
3389 continue
3390 #@-<< handle @section-delims >>
3391 # These sections must be last, in this order.
3392 #@+<< handle remaining @@ lines >>
3393 #@+node:ekr.20180603135602.1: *4* << handle remaining @@ lines >>
3394 # @first, @last, @delims and @comment generate @@ sentinels,
3395 # So this must follow all of those.
3396 if line.startswith(comment_delim1 + '@@'):
3397 ii = len(comment_delim1) + 1 # on second '@'
3398 jj = line.rfind(comment_delim2) if comment_delim2 else -1
3399 body.append(line[ii:jj] + '\n')
3400 continue
3401 #@-<< handle remaining @@ lines >>
3402 if in_doc:
3403 #@+<< handle remaining @doc lines >>
3404 #@+node:ekr.20180606054325.1: *4* << handle remaining @doc lines >>
3405 if comment_delim2:
3406 # doc lines are unchanged.
3407 body.append(line)
3408 continue
3409 # Doc lines start with start_delim + one blank.
3410 # #1496: Retire the @doc convention.
3411 # #2194: Strip lws.
3412 tail = line.lstrip()[len(comment_delim1) + 1 :]
3413 if tail.strip():
3414 body.append(tail)
3415 else:
3416 body.append('\n')
3417 continue
3418 #@-<< handle remaining @doc lines >>
3419 #@+<< handle remaining @ lines >>
3420 #@+node:ekr.20180602103135.17: *4* << handle remaining @ lines >>
3421 # Handle an apparent sentinel line.
3422 # This *can* happen after the git-diff or refresh-from-disk commands.
3423 #
3424 if 1: # pragma: no cover (defensive)
3425 # This assert verifies the short-circuit test.
3426 assert strip_line.startswith(sentinel), (repr(sentinel), repr(line))
3427 # A useful trace.
3428 g.trace(
3429 f"{g.shortFileName(self.path)}: "
3430 f"warning: inserting unexpected line: {line.rstrip()!r}"
3431 )
3432 # #2213: *Do* insert the line, with a warning.
3433 body.append(line)
3434 #@-<< handle remaining @ lines >>
3435 else:
3436 # No @-leo sentinel!
3437 return # pragma: no cover
3438 #@+<< final checks >>
3439 #@+node:ekr.20211104054823.1: *4* << final checks >>
3440 if g.unitTesting:
3441 # Unit tests must use the proper value for root.gnx.
3442 assert not root_gnx_adjusted
3443 assert not stack, stack
3444 assert root_gnx == gnx, (root_gnx, gnx)
3445 elif root_gnx_adjusted: # pragma: no cover
3446 pass # Don't check!
3447 elif stack: # pragma: no cover
3448 g.error('scan_lines: Stack should be empty')
3449 g.printObj(stack, tag='stack')
3450 elif root_gnx != gnx: # pragma: no cover
3451 g.error('scan_lines: gnx error')
3452 g.es_print(f"root_gnx: {root_gnx} != gnx: {gnx}")
3453 #@-<< final checks >>
3454 #@+<< insert @last lines >>
3455 #@+node:ekr.20211103101453.1: *4* << insert @last lines >>
3456 tail_lines = lines[start + i :]
3457 if tail_lines:
3458 # Convert the trailing lines to @last directives.
3459 last_lines = [f"@last {z.rstrip()}\n" for z in tail_lines]
3460 # Add the lines to the dictionary of lines.
3461 gnx2body[gnx] = gnx2body[gnx] + last_lines
3462 # Warn if there is an unexpected number of last lines.
3463 if n_last_lines != len(last_lines): # pragma: no cover
3464 n1 = n_last_lines
3465 n2 = len(last_lines)
3466 g.trace(f"Expected {n1} trailing line{g.plural(n1)}, got {n2}")
3467 #@-<< insert @last lines >>
3468 #@+<< post pass: set all body text>>
3469 #@+node:ekr.20211104054426.1: *4* << post pass: set all body text>>
3470 # Set the body text.
3471 assert root_v.gnx in gnx2vnode, root_v
3472 assert root_v.gnx in gnx2body, root_v
3473 for key in gnx2body:
3474 body = gnx2body.get(key)
3475 v = gnx2vnode.get(key)
3476 assert v, (key, v)
3477 v._bodyString = g.toUnicode(''.join(body))
3478 #@-<< post pass: set all body text>>
3479 #@+node:ekr.20180603170614.1: *3* fast_at.read_into_root
3480 def read_into_root(self, contents, path, root):
3481 """
3482 Parse the file's contents, creating a tree of vnodes
3483 anchored in root.v.
3484 """
3485 trace = False
3486 t1 = time.process_time()
3487 self.path = path
3488 self.root = root
3489 sfn = g.shortFileName(path)
3490 contents = contents.replace('\r', '')
3491 lines = g.splitLines(contents)
3492 data = self.scan_header(lines)
3493 if not data: # pragma: no cover
3494 g.trace(f"Invalid external file: {sfn}")
3495 return False
3496 # Clear all children.
3497 # Previously, this had been done in readOpenFile.
3498 root.v._deleteAllChildren()
3499 comment_delims, first_lines, start_i = data
3500 self.scan_lines(comment_delims, first_lines, lines, path, start_i)
3501 if trace:
3502 t2 = time.process_time()
3503 g.trace(f"{t2 - t1:5.2f} sec. {path}")
3504 return True
3505 #@-others
3506#@-others
3507#@@language python
3508#@@tabwidth -4
3509#@@pagewidth 60
3511#@-leo