Coverage for C:\leo.repo\leo-editor\leo\core\leoPersistence.py : 51%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2#@+leo-ver=5-thin
3#@+node:ekr.20140821055201.18331: * @file leoPersistence.py
4#@@first
5"""Support for persistent clones, gnx's and uA's using @persistence trees."""
6import binascii
7import pickle
8from leo.core import leoGlobals as g
9#@+others
10#@+node:ekr.20140711111623.17886: ** Commands (leoPersistence.py)
12@g.command('clean-persistence')
13def view_pack_command(event):
14 """Remove all @data nodes that do not correspond to an existing foreign file."""
15 c = event.get('c')
16 if c and c.persistenceController:
17 c.persistenceController.clean()
18#@+node:ekr.20140711111623.17790: ** class PersistenceDataController
19class PersistenceDataController:
20 #@+<< docstring >>
21 #@+node:ekr.20140711111623.17791: *3* << docstring >> (class persistenceController)
22 """
23 A class to handle persistence in **foreign files**, that is,
24 files created by @auto, @org-mode or @vim-outline node.
26 All required data are held in nodes having the following structure::
28 - @persistence
29 - @data <headline of foreign node>
30 - @gnxs
31 body text: pairs of lines: gnx:<gnx><newline>unl:<unl>
32 - @uas
33 @ua <gnx>
34 body text: the pickled uA
35 """
36 #@-<< docstring >>
37 #@+others
38 #@+node:ekr.20141023154408.3: *3* pd.ctor
39 def __init__(self, c):
40 """Ctor for persistenceController class."""
41 self.c = c
42 self.at_persistence = None
43 # The position of the @position node.
44 #@+node:ekr.20140711111623.17793: *3* pd.Entry points
45 #@+node:ekr.20140718153519.17731: *4* pd.clean
46 def clean(self):
47 """Remove all @data nodes that do not correspond to an existing foreign file."""
48 c = self.c
49 at_persistence = self.has_at_persistence_node()
50 if not at_persistence:
51 return
52 foreign_list = [
53 p.h.strip() for p in c.all_unique_positions()
54 if self.is_foreign_file(p)]
55 delete_list = []
56 tag = '@data:'
57 for child in at_persistence.children():
58 if child.h.startswith(tag):
59 name = child.h[len(tag) :].strip()
60 if name not in foreign_list:
61 delete_list.append(child.copy())
62 if delete_list:
63 at_persistence.setDirty()
64 c.setChanged()
65 for p in delete_list:
66 g.es_print('deleting:', p.h)
67 c.deletePositionsInList(delete_list)
68 c.redraw()
69 #@+node:ekr.20140711111623.17804: *4* pd.update_before_write_foreign_file & helpers
70 def update_before_write_foreign_file(self, root):
71 """
72 Update the @data node for root, a foreign node.
73 Create @gnxs nodes and @uas trees as needed.
74 """
75 # Delete all children of the @data node.
76 self.at_persistence = self.find_at_persistence_node()
77 if not self.at_persistence:
78 return None
79 # was return at_data # for at-file-to-at-auto command.
80 at_data = self.find_at_data_node(root)
81 self.delete_at_data_children(at_data, root)
82 # Create the data for the @gnxs and @uas trees.
83 aList, seen = [], []
84 for p in root.subtree():
85 gnx = p.v.gnx
86 assert gnx
87 if gnx not in seen:
88 seen.append(gnx)
89 aList.append(p.copy())
90 # Create the @gnxs node
91 at_gnxs = self.find_at_gnxs_node(root)
92 at_gnxs.b = ''.join(
93 [f"gnx: {p.v.gnx}\nunl: {self.relative_unl(p, root)}\n"
94 for p in aList])
95 # Create the @uas tree.
96 uas = [p for p in aList if p.v.u]
97 if uas:
98 at_uas = self.find_at_uas_node(root)
99 if at_uas.hasChildren():
100 at_uas.v._deleteAllChildren()
101 for p in uas:
102 p2 = at_uas.insertAsLastChild()
103 p2.h = '@ua:' + p.v.gnx
104 p2.b = f"unl:{self.relative_unl(p, root)}\nua:{self.pickle(p)}"
105 # This is no longer necessary because of at.saveOutlineIfPossible.
106 # Explain why the .leo file has become dirty.
107 # g.es_print(f"updated: @data:{root.h} ")
108 return at_data # For at-file-to-at-auto command.
109 #@+node:ekr.20140716021139.17773: *5* pd.delete_at_data_children
110 def delete_at_data_children(self, at_data, root):
111 """Delete all children of the @data node"""
112 if at_data.hasChildren():
113 at_data.v._deleteAllChildren()
114 #@+node:ekr.20140711111623.17807: *4* pd.update_after_read_foreign_file & helpers
115 def update_after_read_foreign_file(self, root):
116 """Restore gnx's, uAs and clone links using @gnxs nodes and @uas trees."""
117 self.at_persistence = self.find_at_persistence_node()
118 if not self.at_persistence:
119 return
120 if not root:
121 return
122 if not self.is_foreign_file(root):
123 return
124 # Create clone links from @gnxs node
125 at_gnxs = self.has_at_gnxs_node(root)
126 if at_gnxs:
127 self.restore_gnxs(at_gnxs, root)
128 # Create uas from @uas tree.
129 at_uas = self.has_at_uas_node(root)
130 if at_uas:
131 self.create_uas(at_uas, root)
132 #@+node:ekr.20140711111623.17810: *5* pd.restore_gnxs & helpers
133 def restore_gnxs(self, at_gnxs, root):
134 """
135 Recreate gnx's and clone links from an @gnxs node.
136 @gnxs nodes contain pairs of lines:
137 gnx:<gnx>
138 unl:<unl>
139 """
140 lines = g.splitLines(at_gnxs.b)
141 gnxs = [s[4:].strip() for s in lines if s.startswith('gnx:')]
142 unls = [s[4:].strip() for s in lines if s.startswith('unl:')]
143 if len(gnxs) == len(unls):
144 d = self.create_outer_gnx_dict(root)
145 for gnx, unl in zip(gnxs, unls):
146 self.restore_gnx(d, gnx, root, unl)
147 else:
148 g.trace('bad @gnxs contents', gnxs, unls)
149 #@+node:ekr.20141021083702.18341: *6* pd.create_outer_gnx_dict
150 def create_outer_gnx_dict(self, root):
151 """
152 Return a dict whose keys are gnx's and whose values are positions
153 **outside** of root's tree.
154 """
155 c, d = self.c, {}
156 p = c.rootPosition()
157 while p:
158 if p.v == root.v:
159 p.moveToNodeAfterTree()
160 else:
161 gnx = p.v.fileIndex
162 d[gnx] = p.copy()
163 p.moveToThreadNext()
164 return d
165 #@+node:ekr.20140711111623.17809: *6* pd.restore_gnx
166 def restore_gnx(self, d, gnx, root, unl):
167 """
168 d is an *outer* gnx dict, associating nodes *outside* the tree with positions.
169 Let p1 be the position of the node *within* root's tree corresponding to unl.
170 Let p2 be the position of any node *outside* root's tree with the given gnx.
171 - Set p1.v.fileIndex = gnx.
172 - If p2 exists, relink p1 so it is a clone of p2.
173 """
174 p1 = self.find_position_for_relative_unl(root, unl)
175 if not p1:
176 return
177 p2 = d.get(gnx)
178 if p2:
179 if p1.h == p2.h and p1.b == p2.b:
180 p1._relinkAsCloneOf(p2)
181 # Warning: p1 *no longer exists* here.
182 # _relinkAsClone does *not* set p1.v = p2.v.
183 else:
184 g.es_print('mismatch in cloned node', p1.h)
185 else:
186 # Fix #526: A major bug: this was not set!
187 p1.v.fileIndex = gnx
188 g.app.nodeIndices.updateLastIndex(g.toUnicode(gnx))
189 #@+node:ekr.20140711111623.17892: *5* pd.create_uas
190 def create_uas(self, at_uas, root):
191 """Recreate uA's from the @ua nodes in the @uas tree."""
192 # Create an *inner* gnx dict.
193 # Keys are gnx's, values are positions *within* root's tree.
194 d = {}
195 for p in root.self_and_subtree(copy=False):
196 d[p.v.gnx] = p.copy()
197 # Recreate the uA's for the gnx's given by each @ua node.
198 for at_ua in at_uas.children():
199 h, b = at_ua.h, at_ua.b
200 gnx = h[4:].strip()
201 if b and gnx and g.match_word(h, 0, '@ua'):
202 p = d.get(gnx)
203 if p:
204 # Handle all recent variants of the node.
205 lines = g.splitLines(b)
206 if b.startswith('unl:') and len(lines) == 2:
207 # pylint: disable=unbalanced-tuple-unpacking
208 unl, ua = lines
209 else:
210 unl, ua = None, b
211 if ua.startswith('ua:'):
212 ua = ua[3:]
213 if ua:
214 ua = self.unpickle(ua)
215 p.v.u = ua
216 else:
217 g.trace('Can not unpickle uA in',
218 p.h, repr(unl), type(ua), ua[:40])
219 #@+node:ekr.20140712105818.16750: *3* pd.Helpers
220 #@+node:ekr.20140711111623.17845: *4* pd.at_data_body
221 # Note: the unl of p relative to p is simply p.h,
222 # so it is pointless to add that to @data nodes.
224 def at_data_body(self, p):
225 """Return the body text for p's @data node."""
226 return f"gnx: {p.v.gnx}\n"
227 #@+node:ekr.20140712105644.16744: *4* pd.expected_headline
228 def expected_headline(self, p):
229 """Return the expected imported headline for p."""
230 return getattr(p.v, '_imported_headline', p.h)
231 #@+node:ekr.20140711111623.17854: *4* pd.find...
232 # The find commands create the node if not found.
233 #@+node:ekr.20140711111623.17856: *5* pd.find_at_data_node & helper
234 def find_at_data_node(self, root):
235 """
236 Return the @data node for root, a foreign node.
237 Create the node if it does not exist.
238 """
239 self.at_persistence = self.find_at_persistence_node()
240 if not self.at_persistence:
241 return None
242 p = self.has_at_data_node(root)
243 if p:
244 return p
245 p = self.at_persistence.insertAsLastChild()
246 if not p: # #2103
247 return None
248 p.h = '@data:' + root.h
249 p.b = self.at_data_body(root)
250 return p
251 #@+node:ekr.20140711111623.17857: *5* pd.find_at_gnxs_node
252 def find_at_gnxs_node(self, root):
253 """
254 Find the @gnxs node for root, a foreign node.
255 Create the @gnxs node if it does not exist.
256 """
257 h = '@gnxs'
258 if not self.at_persistence:
259 return None
260 data = self.find_at_data_node(root)
261 p = g.findNodeInTree(self.c, data, h)
262 if p:
263 return p
264 p = data.insertAsLastChild()
265 if p: # #2103
266 p.h = h
267 return p
268 #@+node:ekr.20140711111623.17863: *5* pd.find_at_persistence_node
269 def find_at_persistence_node(self):
270 """
271 Find the first @persistence node in the outline.
272 If it does not exist, create it as the *last* top-level node,
273 so that no existing positions become invalid.
274 """
275 c, h = self.c, '@persistence'
276 p = g.findNodeAnywhere(c, h)
277 if p:
278 return p
279 if c.config.getBool('create-at-persistence-nodes-automatically'):
280 last = c.rootPosition()
281 while last.hasNext():
282 last.moveToNext()
283 p = last.insertAfter()
284 if p: # #2103
285 p.h = h
286 g.es_print(f"created {h} node", color='red')
287 return p
288 #@+node:ekr.20140711111623.17891: *5* pd.find_at_uas_node
289 def find_at_uas_node(self, root):
290 """
291 Find the @uas node for root, a foreign node.
292 Create the @uas node if it does not exist.
293 """
294 h = '@uas'
295 if not self.at_persistence:
296 return None
297 auto_view = self.find_at_data_node(root)
298 p = g.findNodeInTree(self.c, auto_view, h)
299 if p:
300 return p
301 p = auto_view.insertAsLastChild()
302 if p: # #2103
303 p.h = h
304 return p
305 #@+node:ekr.20140711111623.17861: *5* pd.find_position_for_relative_unl & helpers
306 def find_position_for_relative_unl(self, root, unl):
307 """
308 Given a unl relative to root, return the node whose
309 unl matches the longest suffix of the given unl.
310 """
311 unl_list = unl.split('-->')
312 if not unl_list or len(unl_list) == 1 and not unl_list[0]:
313 return root
314 return self.find_exact_match(root, unl_list)
315 # return self.find_best_match(root, unl_list)
316 #@+node:ekr.20140716021139.17764: *6* pd.find_best_match
317 def find_best_match(self, root, unl_list):
318 """Find the best partial matches of the tail in root's tree."""
319 tail = unl_list[-1]
320 matches = []
321 for p in root.self_and_subtree(copy=False):
322 if p.h == tail: # A match
323 # Compute the partial unl.
324 parents = 0
325 for parent2 in p.parents():
326 if parent2 == root:
327 break
328 elif parents + 2 > len(unl_list):
329 break
330 elif parent2.h != unl_list[-2 - parents]:
331 break
332 else:
333 parents += 1
334 matches.append((parents, p.copy()),)
335 if matches:
336 # Take the match with the greatest number of parents.
338 def key(aTuple):
339 return aTuple[0]
341 n, p = list(sorted(matches, key=key))[-1]
342 return p
343 return None
344 #@+node:ekr.20140716021139.17765: *6* pd.find_exact_match
345 def find_exact_match(self, root, unl_list):
346 """
347 Find an exact match of the unl_list in root's tree.
348 The root does not appear in the unl_list.
349 """
350 # full_unl = '-->'.join(unl_list)
351 parent = root
352 for unl in unl_list:
353 for child in parent.children():
354 if child.h.strip() == unl.strip():
355 parent = child
356 break
357 else:
358 return None
359 return parent
360 #@+node:ekr.20140711111623.17862: *5* pd.find_representative_node
361 def find_representative_node(self, root, target):
362 """
363 root is a foreign node. target is a gnxs node within root's tree.
365 Return a node *outside* of root's tree that is cloned to target,
366 preferring nodes outside any @<file> tree.
367 Never return any node in any @persistence tree.
368 """
369 assert target
370 assert root
371 # Pass 1: accept only nodes outside any @file tree.
372 p = self.c.rootPosition()
373 while p:
374 if p.h.startswith('@persistence'):
375 p.moveToNodeAfterTree()
376 elif p.isAnyAtFileNode():
377 p.moveToNodeAfterTree()
378 elif p.v == target.v:
379 return p
380 else:
381 p.moveToThreadNext()
382 # Pass 2: accept any node outside the root tree.
383 p = self.c.rootPosition()
384 while p:
385 if p.h.startswith('@persistence'):
386 p.moveToNodeAfterTree()
387 elif p == root:
388 p.moveToNodeAfterTree()
389 elif p.v == target.v:
390 return p
391 else:
392 p.moveToThreadNext()
393 g.trace('no representative node for:', target, 'parent:', target.parent())
394 return None
395 #@+node:ekr.20140712105818.16751: *4* pd.foreign_file_name
396 def foreign_file_name(self, p):
397 """Return the file name for p, a foreign file node."""
398 for tag in ('@auto', '@org-mode', '@vim-outline'):
399 if g.match_word(p.h, 0, tag):
400 return p.h[len(tag) :].strip()
401 return None
402 #@+node:ekr.20140711111623.17864: *4* pd.has...
403 # The has commands return None if the node does not exist.
404 #@+node:ekr.20140711111623.17865: *5* pd.has_at_data_node
405 def has_at_data_node(self, root):
406 """
407 Return the @data node corresponding to root, a foreign node.
408 Return None if no such node exists.
409 """
410 # if g.unitTesting:
411 # pass
412 if not self.at_persistence:
413 return None
414 if not self.is_at_auto_node(root):
415 return None
416 # Find a direct child of the @persistence nodes with matching headline and body.
417 s = self.at_data_body(root)
418 for p in self.at_persistence.children():
419 if p.b == s:
420 return p
421 return None
422 #@+node:ekr.20140711111623.17890: *5* pd.has_at_gnxs_node
423 def has_at_gnxs_node(self, root):
424 """
425 Find the @gnxs node for an @data node with the given unl.
426 Return None if it does not exist.
427 """
428 if self.at_persistence:
429 p = self.has_at_data_node(root)
430 return p and g.findNodeInTree(self.c, p, '@gnxs')
431 return None
432 #@+node:ekr.20140711111623.17894: *5* pd.has_at_uas_node
433 def has_at_uas_node(self, root):
434 """
435 Find the @uas node for an @data node with the given unl.
436 Return None if it does not exist.
437 """
438 if self.at_persistence:
439 p = self.has_at_data_node(root)
440 return p and g.findNodeInTree(self.c, p, '@uas')
441 return None
442 #@+node:ekr.20140711111623.17869: *5* pd.has_at_persistence_node
443 def has_at_persistence_node(self):
444 """Return the @persistence node or None if it does not exist."""
445 return g.findNodeAnywhere(self.c, '@persistence')
446 #@+node:ekr.20140711111623.17870: *4* pd.is...
447 #@+node:ekr.20140711111623.17871: *5* pd.is_at_auto_node
448 def is_at_auto_node(self, p):
449 """
450 Return True if p is *any* kind of @auto node,
451 including @auto-otl and @auto-rst.
452 """
453 return p.isAtAutoNode()
454 # The safe way: it tracks changes to p.isAtAutoNode.
455 #@+node:ekr.20140711111623.17897: *5* pd.is_at_file_node
456 def is_at_file_node(self, p):
457 """Return True if p is an @file node."""
458 return g.match_word(p.h, 0, '@file')
459 #@+node:ekr.20140711111623.17872: *5* pd.is_cloned_outside_parent_tree
460 def is_cloned_outside_parent_tree(self, p):
461 """Return True if a clone of p exists outside the tree of p.parent()."""
462 return len(list(set(p.v.parents))) > 1
463 #@+node:ekr.20140712105644.16745: *5* pd.is_foreign_file
464 def is_foreign_file(self, p):
465 return (
466 self.is_at_auto_node(p) or
467 g.match_word(p.h, 0, '@org-mode') or
468 g.match_word(p.h, 0, '@vim-outline'))
469 #@+node:ekr.20140713135856.17745: *4* pd.Pickling
470 #@+node:ekr.20140713062552.17737: *5* pd.pickle
471 def pickle(self, p):
472 """Pickle val and return the hexlified result."""
473 try:
474 ua = p.v.u
475 s = pickle.dumps(ua, protocol=1)
476 s2 = binascii.hexlify(s)
477 s3 = g.toUnicode(s2, 'utf-8')
478 return s3
479 except pickle.PicklingError:
480 g.warning("ignoring non-pickleable value", ua, "in", p.h)
481 return ''
482 except Exception:
483 g.error("pd.pickle: unexpected exception in", p.h)
484 g.es_exception()
485 return ''
486 #@+node:ekr.20140713135856.17744: *5* pd.unpickle
487 def unpickle(self, s):
488 """Unhexlify and unpickle string s into p."""
489 try:
490 bin = binascii.unhexlify(g.toEncodedString(s))
491 # Throws TypeError if s is not a hex string.
492 return pickle.loads(bin)
493 except Exception:
494 g.es_exception()
495 return None
496 #@+node:ekr.20140711111623.17879: *4* pd.unls...
497 #@+node:ekr.20140711111623.17881: *5* pd.drop_unl_parent/tail
498 def drop_unl_parent(self, unl):
499 """Drop the penultimate part of the unl."""
500 aList = unl.split('-->')
501 return '-->'.join(aList[:-2] + aList[-1:])
503 def drop_unl_tail(self, unl):
504 """Drop the last part of the unl."""
505 return '-->'.join(unl.split('-->')[:-1])
506 #@+node:ekr.20140711111623.17883: *5* pd.relative_unl
507 def relative_unl(self, p, root):
508 """Return the unl of p relative to the root position."""
509 result = []
510 for p in p.self_and_parents(copy=False):
511 if p == root:
512 break
513 else:
514 result.append(self.expected_headline(p))
515 return '-->'.join(reversed(result))
516 #@+node:ekr.20140711111623.17896: *5* pd.unl
517 def unl(self, p):
518 """Return the unl corresponding to the given position."""
519 return '-->'.join(reversed(
520 [self.expected_headline(p2) for p2 in p.self_and_parents(copy=False)]))
521 #@+node:ekr.20140711111623.17885: *5* pd.unl_tail
522 def unl_tail(self, unl):
523 """Return the last part of a unl."""
524 return unl.split('-->')[:-1][0]
525 #@-others
526#@-others
527#@@language python
528#@@tabwidth -4
529#@@pagewidth 70
530#@-leo