Coverage for C:\leo.repo\leo-editor\leo\core\leoShadow.py : 67%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2#@+leo-ver=5-thin
3#@+node:ekr.20080708094444.1: * @file leoShadow.py
4#@@first
5#@+<< docstring >>
6#@+node:ekr.20080708094444.78: ** << docstring >>
7"""
8leoShadow.py
10This code allows users to use Leo with files which contain no sentinels
11and still have information flow in both directions between outlines and
12derived files.
14Private files contain sentinels: they live in the Leo-shadow subdirectory.
15Public files contain no sentinels: they live in the parent (main) directory.
17When Leo first reads an @shadow we create a file without sentinels in the regular directory.
19The slightly hard thing to do is to pick up changes from the file without
20sentinels, and put them into the file with sentinels.
22Settings:
23- @string shadow_subdir (default: .leo_shadow): name of the shadow directory.
25- @string shadow_prefix (default: x): prefix of shadow files.
26 This prefix allows the shadow file and the original file to have different names.
27 This is useful for name-based tools like py.test.
28"""
29#@-<< docstring >>
30#@+<< imports >>
31#@+node:ekr.20080708094444.52: ** << imports >> (leoShadow)
32import difflib
33import os
34import pprint
35from typing import List
36from leo.core import leoGlobals as g
37#@-<< imports >>
38#@+others
39#@+node:ekr.20080708094444.80: ** class ShadowController
40class ShadowController:
41 """A class to manage @shadow files"""
42 #@+others
43 #@+node:ekr.20080708094444.79: *3* x.ctor & x.reloadSettings
44 def __init__(self, c, trace=False, trace_writers=False):
45 """Ctor for ShadowController class."""
46 self.c = c
47 # Opcode dispatch dict.
48 self.dispatch_dict = {
49 'delete': self.op_delete,
50 'equal': self.op_equal,
51 'insert': self.op_insert,
52 'replace': self.op_replace,
53 }
54 # File encoding.
55 self.encoding = c.config.default_derived_file_encoding
56 # Configuration: set in reloadSettings.
57 self.shadow_subdir = None
58 self.shadow_prefix = None
59 self.shadow_in_home_dir = None
60 self.shadow_subdir = None
61 # Error handling...
62 self.errors = 0
63 self.last_error = '' # The last error message, regardless of whether it was actually shown.
64 self.trace = False
65 # Support for goto-line.
66 self.line_mapping = []
67 self.reloadSettings()
69 def reloadSettings(self):
70 """ShadowController.reloadSettings."""
71 c = self.c
72 self.shadow_subdir = c.config.getString('shadow-subdir') or '.leo_shadow'
73 self.shadow_prefix = c.config.getString('shadow-prefix') or ''
74 self.shadow_in_home_dir = c.config.getBool('shadow-in-home-dir', default=False)
75 self.shadow_subdir = g.os_path_normpath(self.shadow_subdir)
76 #@+node:ekr.20080711063656.1: *3* x.File utils
77 #@+node:ekr.20080711063656.7: *4* x.baseDirName
78 def baseDirName(self):
79 c = self.c
80 filename = c.fileName()
81 if filename:
82 return g.os_path_dirname(g.os_path_finalize(filename)) # 1341
83 print('')
84 self.error('Can not compute shadow path: .leo file has not been saved')
85 return None
86 #@+node:ekr.20080711063656.4: *4* x.dirName and pathName
87 def dirName(self, filename):
88 """Return the directory for filename."""
89 x = self
90 return g.os_path_dirname(x.pathName(filename))
92 def pathName(self, filename):
93 """Return the full path name of filename."""
94 x = self
95 theDir = x.baseDirName()
96 return theDir and g.os_path_finalize_join(theDir, filename) # 1341
97 #@+node:ekr.20080712080505.3: *4* x.isSignificantPublicFile
98 def isSignificantPublicFile(self, fn):
99 """
100 This tells the AtFile.read logic whether to import a public file
101 or use an existing public file.
102 """
103 return (
104 g.os_path_exists(fn) and
105 g.os_path_isfile(fn) and
106 g.os_path_getsize(fn) > 10
107 )
108 #@+node:ekr.20080710082231.19: *4* x.makeShadowDirectory
109 def makeShadowDirectory(self, fn):
110 """Make a shadow directory for the **public** fn."""
111 x = self
112 path = x.shadowDirName(fn)
113 if not g.os_path_exists(path):
114 # Force the creation of the directories.
115 g.makeAllNonExistentDirectories(path)
116 return g.os_path_exists(path) and g.os_path_isdir(path)
117 #@+node:ekr.20080713091247.1: *4* x.replaceFileWithString
118 def replaceFileWithString(self, encoding, fileName, s):
119 """
120 Replace the file with s if s is different from theFile's contents.
122 Return True if theFile was changed.
123 """
124 x, c = self, self.c
125 exists = g.os_path_exists(fileName)
126 if exists:
127 # Read the file. Return if it is the same.
128 s2, e = g.readFileIntoString(fileName)
129 if s2 is None:
130 return False
131 if s == s2:
132 report = c.config.getBool('report-unchanged-files', default=True)
133 if report and not g.unitTesting:
134 g.es('unchanged:', fileName)
135 return False
136 # Issue warning if directory does not exist.
137 theDir = g.os_path_dirname(fileName)
138 if theDir and not g.os_path_exists(theDir):
139 if not g.unitTesting:
140 x.error(f"not written: {fileName} directory not found")
141 return False
142 # Replace the file.
143 try:
144 with open(fileName, 'wb') as f:
145 # Fix bug 1243847: unicode error when saving @shadow nodes.
146 f.write(g.toEncodedString(s, encoding=encoding))
147 c.setFileTimeStamp(fileName)
148 # Fix #1053. This is an *ancient* bug.
149 if not g.unitTesting:
150 kind = 'wrote' if exists else 'created'
151 g.es(f"{kind:>6}: {fileName}")
152 return True
153 except IOError:
154 x.error(f"unexpected exception writing file: {fileName}")
155 g.es_exception()
156 return False
157 #@+node:ekr.20080711063656.6: *4* x.shadowDirName and shadowPathName
158 def shadowDirName(self, filename):
159 """Return the directory for the shadow file corresponding to filename."""
160 x = self
161 return g.os_path_dirname(x.shadowPathName(filename))
163 def shadowPathName(self, filename):
164 """Return the full path name of filename, resolved using c.fileName()"""
165 x = self
166 c = x.c
167 baseDir = x.baseDirName()
168 fileDir = g.os_path_dirname(filename)
169 # 2011/01/26: bogomil: redirect shadow dir
170 if self.shadow_in_home_dir:
171 # Each .leo file has a separate shadow_cache in base dir
172 fname = "_".join(
173 [os.path.splitext(os.path.basename(c.mFileName))[0], "shadow_cache"])
174 # On Windows incorporate the drive letter to the private file path
175 if os.name == "nt":
176 fileDir = fileDir.replace(':', '%')
177 # build the chache path as a subdir of the base dir
178 fileDir = "/".join([baseDir, fname, fileDir])
179 return baseDir and g.os_path_finalize_join( # 1341
180 baseDir,
181 fileDir, # Bug fix: honor any directories specified in filename.
182 x.shadow_subdir,
183 x.shadow_prefix + g.shortFileName(filename))
184 #@+node:ekr.20080708192807.1: *3* x.Propagation
185 #@+node:ekr.20080708094444.35: *4* x.check_output
186 def check_output(self):
187 """Check that we produced a valid output."""
188 x = self
189 lines1 = x.b
190 junk, sents1 = x.separate_sentinels(x.old_sent_lines, x.marker)
191 lines2, sents2 = x.separate_sentinels(x.results, x.marker)
192 ok = lines1 == lines2 and sents1 == sents2
193 if g.unitTesting:
194 # The unit test will report the error.
195 return ok
196 if lines1 != lines2:
197 g.trace()
198 d = difflib.Differ()
199 aList = list(d.compare(lines2, x.b))
200 pprint.pprint(aList)
201 if sents1 != sents2:
202 x.show_error(
203 lines1=sents1,
204 lines2=sents2,
205 message="Sentinals not preserved!",
206 lines1_message="old sentinels",
207 lines2_message="new sentinels")
208 return ok
209 #@+node:ekr.20080708094444.38: *4* x.propagate_changed_lines (main algorithm) & helpers
210 def propagate_changed_lines(
211 self, new_public_lines, old_private_lines, marker, p=None):
212 #@+<< docstring >>
213 #@+node:ekr.20150207044400.9: *5* << docstring >>
214 """
215 The Mulder update algorithm, revised by EKR.
217 Use the diff between the old and new public lines to insperse sentinels
218 from old_private_lines into the result.
220 The algorithm never deletes or rearranges sentinels. However, verbatim
221 sentinels may be inserted or deleted as needed.
222 """
223 #@-<< docstring >>
224 x = self
225 x.init_ivars(new_public_lines, old_private_lines, marker)
226 sm = difflib.SequenceMatcher(None, x.a, x.b)
227 # Ensure leading sentinels are put first.
228 x.put_sentinels(0)
229 x.sentinels[0] = []
230 for tag, ai, aj, bi, bj in sm.get_opcodes():
231 f = x.dispatch_dict.get(tag, x.op_bad)
232 f(tag, ai, aj, bi, bj)
233 # Put the trailing sentinels & check the result.
234 x.results.extend(x.trailing_sentinels)
235 # check_output is likely to be more buggy than the code under test.
236 # x.check_output()
237 return x.results
238 #@+node:ekr.20150207111757.180: *5* x.dump_args
239 def dump_args(self):
240 """Dump the argument lines."""
241 x = self
242 table = (
243 (x.old_sent_lines, 'old private lines'),
244 (x.a, 'old public lines'),
245 (x.b, 'new public lines'),
246 )
247 for lines, title in table:
248 x.dump_lines(lines, title)
249 g.pr()
250 #@+node:ekr.20150207111757.178: *5* x.dump_lines
251 def dump_lines(self, lines, title):
252 """Dump the given lines."""
253 print(f"\n{title}...\n")
254 for i, line in enumerate(lines):
255 g.pr(f"{i:4} {line!r}")
256 #@+node:ekr.20150209044257.6: *5* x.init_data
257 def init_data(self):
258 """
259 Init x.sentinels and x.trailing_sentinels arrays.
260 Return the list of non-sentinel lines in x.old_sent_lines.
261 """
262 x = self
263 lines = x.old_sent_lines
264 sentinels: List[str] = []
265 # The sentinels preceding each non-sentinel line,
266 # not including @verbatim sentinels.
267 new_lines = []
268 # A list of all non-sentinel lines found. Should match x.a.
269 x.sentinels = []
270 # A list of lists of sentinels preceding each line.
271 i = 0
272 while i < len(lines):
273 line = lines[i]
274 i += 1
275 if x.marker.isVerbatimSentinel(line):
276 # Do *not* include the @verbatim sentinel.
277 if i < len(lines):
278 line = lines[i]
279 i += 1
280 x.sentinels.append(sentinels)
281 sentinels = []
282 new_lines.append(line)
283 else:
284 x.verbatim_error()
285 elif x.marker.isSentinel(line):
286 sentinels.append(line)
287 else:
288 x.sentinels.append(sentinels)
289 sentinels = []
290 new_lines.append(line)
291 x.trailing_sentinels = sentinels
292 return new_lines
293 #@+node:ekr.20080708094444.40: *5* x.init_ivars
294 def init_ivars(self, new_public_lines, old_private_lines, marker):
295 """Init all ivars used by propagate_changed_lines & its helpers."""
296 x = self
297 x.delim1, x.delim2 = marker.getDelims()
298 x.marker = marker
299 x.old_sent_lines = old_private_lines
300 x.results = []
301 x.verbatim_line = f"{x.delim1}@verbatim{x.delim2}\n"
302 old_public_lines = x.init_data()
303 x.b = x.preprocess(new_public_lines)
304 x.a = x.preprocess(old_public_lines)
305 #@+node:ekr.20150207044400.16: *5* x.op_bad
306 def op_bad(self, tag, ai, aj, bi, bj):
307 """Report an unexpected opcode."""
308 x = self
309 x.error(f"unknown SequenceMatcher opcode: {tag!r}")
310 #@+node:ekr.20150207044400.12: *5* x.op_delete
311 def op_delete(self, tag, ai, aj, bi, bj):
312 """Handle the 'delete' opcode."""
313 x = self
314 for i in range(ai, aj):
315 x.put_sentinels(i)
316 #@+node:ekr.20150207044400.13: *5* x.op_equal
317 def op_equal(self, tag, ai, aj, bi, bj):
318 """Handle the 'equal' opcode."""
319 x = self
320 assert aj - ai == bj - bi and x.a[ai:aj] == x.b[bi:bj]
321 for i in range(ai, aj):
322 x.put_sentinels(i)
323 x.put_plain_line(x.a[i])
324 # works because x.lines[ai:aj] == x.lines[bi:bj]
325 #@+node:ekr.20150207044400.14: *5* x.op_insert
326 def op_insert(self, tag, ai, aj, bi, bj):
327 """Handle the 'insert' opcode."""
328 x = self
329 for i in range(bi, bj):
330 x.put_plain_line(x.b[i])
331 # Prefer to put sentinels after inserted nodes.
332 # Requires a call to x.put_sentinels(0) before the main loop.
333 #@+node:ekr.20150207044400.15: *5* x.op_replace
334 def op_replace(self, tag, ai, aj, bi, bj):
335 """Handle the 'replace' opcode."""
336 x = self
337 if 1:
338 # Intersperse sentinels and lines.
339 b_lines = x.b[bi:bj]
340 for i in range(ai, aj):
341 x.put_sentinels(i)
342 if b_lines:
343 x.put_plain_line(b_lines.pop(0))
344 # Put any trailing lines.
345 while b_lines:
346 x.put_plain_line(b_lines.pop(0))
347 else:
348 # Feasible. Causes the present unit tests to fail.
349 for i in range(ai, aj):
350 x.put_sentinels(i)
351 for i in range(bi, bj):
352 x.put_plain_line(x.b[i])
353 #@+node:ekr.20150208060128.7: *5* x.preprocess
354 def preprocess(self, lines):
355 """
356 Preprocess public lines, adding newlines as needed.
357 This happens before the diff.
358 """
359 result = []
360 for line in lines:
361 if not line.endswith('\n'):
362 line = line + '\n'
363 result.append(line)
364 return result
365 #@+node:ekr.20150208223018.4: *5* x.put_plain_line
366 def put_plain_line(self, line):
367 """Put a plain line to x.results, inserting verbatim lines if necessary."""
368 x = self
369 if x.marker.isSentinel(line):
370 x.results.append(x.verbatim_line)
371 if x.trace:
372 print(f"put {repr(x.verbatim_line)}")
373 x.results.append(line)
374 if x.trace:
375 print(f"put {line!r}")
376 #@+node:ekr.20150209044257.8: *5* x.put_sentinels
377 def put_sentinels(self, i):
378 """Put all the sentinels to the results"""
379 x = self
380 if 0 <= i < len(x.sentinels):
381 sentinels = x.sentinels[i]
382 if x.trace:
383 g.trace(f"{i:3} {sentinels}")
384 x.results.extend(sentinels)
385 #@+node:ekr.20080708094444.36: *4* x.propagate_changes
386 def propagate_changes(self, old_public_file, old_private_file):
387 """
388 Propagate the changes from the public file (without_sentinels)
389 to the private file (with_sentinels)
390 """
391 x, at = self, self.c.atFileCommands
392 at.errors = 0
393 self.encoding = at.encoding
394 s = at.readFileToUnicode(old_private_file)
395 # Sets at.encoding and inits at.readLines.
396 old_private_lines = g.splitLines(s or '') # #1466.
397 s = at.readFileToUnicode(old_public_file)
398 if at.encoding != self.encoding:
399 g.trace(f"can not happen: encoding mismatch: {at.encoding} {self.encoding}")
400 at.encoding = self.encoding
401 old_public_lines = g.splitLines(s)
402 if 0:
403 g.trace(f"\nprivate lines...{old_private_file}")
404 for s in old_private_lines:
405 g.trace(type(s), isinstance(s, str), repr(s))
406 g.trace(f"\npublic lines...{old_public_file}")
407 for s in old_public_lines:
408 g.trace(type(s), isinstance(s, str), repr(s))
409 marker = x.markerFromFileLines(old_private_lines, old_private_file)
410 new_private_lines = x.propagate_changed_lines(
411 old_public_lines, old_private_lines, marker)
412 # Important bug fix: Never create the private file here!
413 fn = old_private_file
414 exists = g.os_path_exists(fn)
415 different = new_private_lines != old_private_lines
416 copy = exists and different
417 # 2010/01/07: check at.errors also.
418 if copy and x.errors == 0 and at.errors == 0:
419 s = ''.join(new_private_lines)
420 x.replaceFileWithString(at.encoding, fn, s)
421 return copy
422 #@+node:bwmulder.20041231170726: *4* x.updatePublicAndPrivateFiles
423 def updatePublicAndPrivateFiles(self, root, fn, shadow_fn):
424 """handle crucial @shadow read logic.
426 This will be called only if the public and private files both exist."""
427 x = self
428 if x.isSignificantPublicFile(fn):
429 # Update the private shadow file from the public file.
430 written = x.propagate_changes(fn, shadow_fn)
431 if written:
432 x.message(f"updated private {shadow_fn} from public {fn}")
433 else:
434 pass
435 # Don't write *anything*.
436 # if 0: # This causes considerable problems.
437 # # Create the public file from the private shadow file.
438 # x.copy_file_removing_sentinels(shadow_fn,fn)
439 # x.message("created public %s from private %s " % (fn, shadow_fn))
440 #@+node:ekr.20080708094444.89: *3* x.Utils...
441 #@+node:ekr.20080708094444.85: *4* x.error & message & verbatim_error
442 def error(self, s, silent=False):
443 x = self
444 if not silent:
445 g.error(s)
446 # For unit testing.
447 x.last_error = s
448 x.errors += 1
450 def message(self, s):
451 g.es_print(s, color='orange')
453 def verbatim_error(self):
454 x = self
455 x.error('file syntax error: nothing follows verbatim sentinel')
456 g.trace(g.callers())
457 #@+node:ekr.20090529125512.6122: *4* x.markerFromFileLines & helper
458 def markerFromFileLines(self, lines, fn):
459 """Return the sentinel delimiter comment to be used for filename."""
460 at, x = self.c.atFileCommands, self
461 s = x.findLeoLine(lines)
462 ok, junk, start, end, junk = at.parseLeoSentinel(s)
463 if end:
464 delims = '', start, end
465 else:
466 delims = start, '', ''
467 return x.Marker(delims)
468 #@+node:ekr.20090529125512.6125: *5* x.findLeoLine
469 def findLeoLine(self, lines):
470 """Return the @+leo line, or ''."""
471 for line in lines:
472 i = line.find('@+leo')
473 if i != -1:
474 return line
475 return ''
476 #@+node:ekr.20080708094444.9: *4* x.markerFromFileName
477 def markerFromFileName(self, filename):
478 """Return the sentinel delimiter comment to be used for filename."""
479 x = self
480 if not filename:
481 return None
482 root, ext = g.os_path_splitext(filename)
483 if ext == '.tmp':
484 root, ext = os.path.splitext(root)
485 delims = g.comment_delims_from_extension(filename)
486 marker = x.Marker(delims)
487 return marker
488 #@+node:ekr.20080708094444.29: *4* x.separate_sentinels
489 def separate_sentinels(self, lines, marker):
490 """
491 Separates regular lines from sentinel lines.
492 Do not return @verbatim sentinels.
494 Returns (regular_lines, sentinel_lines)
495 """
496 x = self
497 regular_lines = []
498 sentinel_lines = []
499 i = 0
500 while i < len(lines):
501 line = lines[i]
502 if marker.isVerbatimSentinel(line):
503 # Add the plain line that *looks* like a sentinel,
504 # but *not* the preceding @verbatim sentinel itself.
505 # Adding the actual sentinel would spoil the sentinel test when
506 # the user adds or deletes a line requiring an @verbatim line.
507 i += 1
508 if i < len(lines):
509 line = lines[i]
510 regular_lines.append(line)
511 else:
512 x.verbatim_error()
513 elif marker.isSentinel(line):
514 sentinel_lines.append(line)
515 else:
516 regular_lines.append(line)
517 i += 1
518 return regular_lines, sentinel_lines
519 #@+node:ekr.20080708094444.33: *4* x.show_error & helper
520 def show_error(self, lines1, lines2, message, lines1_message, lines2_message):
521 x = self
522 banner1 = '=' * 30
523 banner2 = '-' * 30
524 g.es_print(f"{banner1}\n{message}\n{banner1}\n{lines1_message}\n{banner2}")
525 x.show_error_lines(lines1, 'shadow_errors.tmp1')
526 g.es_print(f"\n{banner1}\n{lines2_message}\n{banner1}")
527 x.show_error_lines(lines2, 'shadow_errors.tmp2')
528 g.es_print('\n@shadow did not pick up the external changes correctly')
529 # g.es_print('Please check shadow.tmp1 and shadow.tmp2 for differences')
530 #@+node:ekr.20080822065427.4: *5* x.show_error_lines
531 def show_error_lines(self, lines, fileName):
532 for i, line in enumerate(lines):
533 g.es_print(f"{i:3} {line!r}")
534 if False: # Only for major debugging.
535 try:
536 f1 = open(fileName, "w")
537 for s in lines:
538 f1.write(repr(s))
539 f1.close()
540 except IOError:
541 g.es_exception()
542 g.es_print('can not open', fileName)
543 #@+node:ekr.20090529061522.5727: *3* class x.Marker
544 class Marker:
545 """A class representing comment delims in @shadow files."""
546 #@+others
547 #@+node:ekr.20090529061522.6257: *4* ctor & repr
548 def __init__(self, delims):
549 """Ctor for Marker class."""
550 delim1, delim2, delim3 = delims
551 self.delim1 = delim1 # Single-line comment delim.
552 self.delim2 = delim2 # Block comment starting delim.
553 self.delim3 = delim3 # Block comment ending delim.
554 if not delim1 and not delim2:
555 # if g.unitTesting:
556 # assert False,repr(delims)
557 self.delim1 = g.app.language_delims_dict.get('unknown_language')
559 def __repr__(self):
560 if self.delim1:
561 delims = self.delim1
562 else:
563 delims = f"{self.delim2} {self.delim3}"
564 return f"<Marker: delims: {delims!r}>"
565 #@+node:ekr.20090529061522.6258: *4* getDelims
566 def getDelims(self):
567 """Return the pair of delims to be used in sentinel lines."""
568 if self.delim1:
569 return self.delim1, ''
570 return self.delim2, self.delim3
571 #@+node:ekr.20090529061522.6259: *4* isSentinel
572 def isSentinel(self, s, suffix=''):
573 """Return True is line s contains a valid sentinel comment."""
574 s = s.strip()
575 if self.delim1 and s.startswith(self.delim1):
576 return s.startswith(self.delim1 + '@' + suffix)
577 if self.delim2:
578 return s.startswith(
579 self.delim2 + '@' + suffix) and s.endswith(self.delim3)
580 return False
581 #@+node:ekr.20090529061522.6260: *4* isVerbatimSentinel
582 def isVerbatimSentinel(self, s):
583 """Return True if s is an @verbatim sentinel."""
584 return self.isSentinel(s, suffix='verbatim')
585 #@-others
586 #@-others
587#@-others
588#@@language python
589#@@tabwidth -4
590#@@pagewidth 70
591#@-leo