Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# -*- coding: utf-8 -*- 

2#@+leo-ver=5-thin 

3#@+node:ekr.20140821055201.18331: * @file leoPersistence.py 

4#@@first 

5"""Support for persistent clones, gnx's and uA's using @persistence trees.""" 

6import binascii 

7import pickle 

8from leo.core import leoGlobals as g 

9#@+others 

10#@+node:ekr.20140711111623.17886: ** Commands (leoPersistence.py) 

11 

12@g.command('clean-persistence') 

13def view_pack_command(event): 

14 """Remove all @data nodes that do not correspond to an existing foreign file.""" 

15 c = event.get('c') 

16 if c and c.persistenceController: 

17 c.persistenceController.clean() 

18#@+node:ekr.20140711111623.17790: ** class PersistenceDataController 

19class PersistenceDataController: 

20 #@+<< docstring >> 

21 #@+node:ekr.20140711111623.17791: *3* << docstring >> (class persistenceController) 

22 """ 

23 A class to handle persistence in **foreign files**, that is, 

24 files created by @auto, @org-mode or @vim-outline node. 

25 

26 All required data are held in nodes having the following structure:: 

27 

28 - @persistence 

29 - @data <headline of foreign node> 

30 - @gnxs 

31 body text: pairs of lines: gnx:<gnx><newline>unl:<unl> 

32 - @uas 

33 @ua <gnx> 

34 body text: the pickled uA 

35 """ 

36 #@-<< docstring >> 

37 #@+others 

38 #@+node:ekr.20141023154408.3: *3* pd.ctor 

39 def __init__(self, c): 

40 """Ctor for persistenceController class.""" 

41 self.c = c 

42 self.at_persistence = None 

43 # The position of the @position node. 

44 #@+node:ekr.20140711111623.17793: *3* pd.Entry points 

45 #@+node:ekr.20140718153519.17731: *4* pd.clean 

46 def clean(self): 

47 """Remove all @data nodes that do not correspond to an existing foreign file.""" 

48 c = self.c 

49 at_persistence = self.has_at_persistence_node() 

50 if not at_persistence: 

51 return 

52 foreign_list = [ 

53 p.h.strip() for p in c.all_unique_positions() 

54 if self.is_foreign_file(p)] 

55 delete_list = [] 

56 tag = '@data:' 

57 for child in at_persistence.children(): 

58 if child.h.startswith(tag): 

59 name = child.h[len(tag) :].strip() 

60 if name not in foreign_list: 

61 delete_list.append(child.copy()) 

62 if delete_list: 

63 at_persistence.setDirty() 

64 c.setChanged() 

65 for p in delete_list: 

66 g.es_print('deleting:', p.h) 

67 c.deletePositionsInList(delete_list) 

68 c.redraw() 

69 #@+node:ekr.20140711111623.17804: *4* pd.update_before_write_foreign_file & helpers 

70 def update_before_write_foreign_file(self, root): 

71 """ 

72 Update the @data node for root, a foreign node. 

73 Create @gnxs nodes and @uas trees as needed. 

74 """ 

75 # Delete all children of the @data node. 

76 self.at_persistence = self.find_at_persistence_node() 

77 if not self.at_persistence: 

78 return None 

79 # was return at_data # for at-file-to-at-auto command. 

80 at_data = self.find_at_data_node(root) 

81 self.delete_at_data_children(at_data, root) 

82 # Create the data for the @gnxs and @uas trees. 

83 aList, seen = [], [] 

84 for p in root.subtree(): 

85 gnx = p.v.gnx 

86 assert gnx 

87 if gnx not in seen: 

88 seen.append(gnx) 

89 aList.append(p.copy()) 

90 # Create the @gnxs node 

91 at_gnxs = self.find_at_gnxs_node(root) 

92 at_gnxs.b = ''.join( 

93 [f"gnx: {p.v.gnx}\nunl: {self.relative_unl(p, root)}\n" 

94 for p in aList]) 

95 # Create the @uas tree. 

96 uas = [p for p in aList if p.v.u] 

97 if uas: 

98 at_uas = self.find_at_uas_node(root) 

99 if at_uas.hasChildren(): 

100 at_uas.v._deleteAllChildren() 

101 for p in uas: 

102 p2 = at_uas.insertAsLastChild() 

103 p2.h = '@ua:' + p.v.gnx 

104 p2.b = f"unl:{self.relative_unl(p, root)}\nua:{self.pickle(p)}" 

105 # This is no longer necessary because of at.saveOutlineIfPossible. 

106 # Explain why the .leo file has become dirty. 

107 # g.es_print(f"updated: @data:{root.h} ") 

108 return at_data # For at-file-to-at-auto command. 

109 #@+node:ekr.20140716021139.17773: *5* pd.delete_at_data_children 

110 def delete_at_data_children(self, at_data, root): 

111 """Delete all children of the @data node""" 

112 if at_data.hasChildren(): 

113 at_data.v._deleteAllChildren() 

114 #@+node:ekr.20140711111623.17807: *4* pd.update_after_read_foreign_file & helpers 

115 def update_after_read_foreign_file(self, root): 

116 """Restore gnx's, uAs and clone links using @gnxs nodes and @uas trees.""" 

117 self.at_persistence = self.find_at_persistence_node() 

118 if not self.at_persistence: 

119 return 

120 if not root: 

121 return 

122 if not self.is_foreign_file(root): 

123 return 

124 # Create clone links from @gnxs node 

125 at_gnxs = self.has_at_gnxs_node(root) 

126 if at_gnxs: 

127 self.restore_gnxs(at_gnxs, root) 

128 # Create uas from @uas tree. 

129 at_uas = self.has_at_uas_node(root) 

130 if at_uas: 

131 self.create_uas(at_uas, root) 

132 #@+node:ekr.20140711111623.17810: *5* pd.restore_gnxs & helpers 

133 def restore_gnxs(self, at_gnxs, root): 

134 """ 

135 Recreate gnx's and clone links from an @gnxs node. 

136 @gnxs nodes contain pairs of lines: 

137 gnx:<gnx> 

138 unl:<unl> 

139 """ 

140 lines = g.splitLines(at_gnxs.b) 

141 gnxs = [s[4:].strip() for s in lines if s.startswith('gnx:')] 

142 unls = [s[4:].strip() for s in lines if s.startswith('unl:')] 

143 if len(gnxs) == len(unls): 

144 d = self.create_outer_gnx_dict(root) 

145 for gnx, unl in zip(gnxs, unls): 

146 self.restore_gnx(d, gnx, root, unl) 

147 else: 

148 g.trace('bad @gnxs contents', gnxs, unls) 

149 #@+node:ekr.20141021083702.18341: *6* pd.create_outer_gnx_dict 

150 def create_outer_gnx_dict(self, root): 

151 """ 

152 Return a dict whose keys are gnx's and whose values are positions 

153 **outside** of root's tree. 

154 """ 

155 c, d = self.c, {} 

156 p = c.rootPosition() 

157 while p: 

158 if p.v == root.v: 

159 p.moveToNodeAfterTree() 

160 else: 

161 gnx = p.v.fileIndex 

162 d[gnx] = p.copy() 

163 p.moveToThreadNext() 

164 return d 

165 #@+node:ekr.20140711111623.17809: *6* pd.restore_gnx 

166 def restore_gnx(self, d, gnx, root, unl): 

167 """ 

168 d is an *outer* gnx dict, associating nodes *outside* the tree with positions. 

169 Let p1 be the position of the node *within* root's tree corresponding to unl. 

170 Let p2 be the position of any node *outside* root's tree with the given gnx. 

171 - Set p1.v.fileIndex = gnx. 

172 - If p2 exists, relink p1 so it is a clone of p2. 

173 """ 

174 p1 = self.find_position_for_relative_unl(root, unl) 

175 if not p1: 

176 return 

177 p2 = d.get(gnx) 

178 if p2: 

179 if p1.h == p2.h and p1.b == p2.b: 

180 p1._relinkAsCloneOf(p2) 

181 # Warning: p1 *no longer exists* here. 

182 # _relinkAsClone does *not* set p1.v = p2.v. 

183 else: 

184 g.es_print('mismatch in cloned node', p1.h) 

185 else: 

186 # Fix #526: A major bug: this was not set! 

187 p1.v.fileIndex = gnx 

188 g.app.nodeIndices.updateLastIndex(g.toUnicode(gnx)) 

189 #@+node:ekr.20140711111623.17892: *5* pd.create_uas 

190 def create_uas(self, at_uas, root): 

191 """Recreate uA's from the @ua nodes in the @uas tree.""" 

192 # Create an *inner* gnx dict. 

193 # Keys are gnx's, values are positions *within* root's tree. 

194 d = {} 

195 for p in root.self_and_subtree(copy=False): 

196 d[p.v.gnx] = p.copy() 

197 # Recreate the uA's for the gnx's given by each @ua node. 

198 for at_ua in at_uas.children(): 

199 h, b = at_ua.h, at_ua.b 

200 gnx = h[4:].strip() 

201 if b and gnx and g.match_word(h, 0, '@ua'): 

202 p = d.get(gnx) 

203 if p: 

204 # Handle all recent variants of the node. 

205 lines = g.splitLines(b) 

206 if b.startswith('unl:') and len(lines) == 2: 

207 # pylint: disable=unbalanced-tuple-unpacking 

208 unl, ua = lines 

209 else: 

210 unl, ua = None, b 

211 if ua.startswith('ua:'): 

212 ua = ua[3:] 

213 if ua: 

214 ua = self.unpickle(ua) 

215 p.v.u = ua 

216 else: 

217 g.trace('Can not unpickle uA in', 

218 p.h, repr(unl), type(ua), ua[:40]) 

219 #@+node:ekr.20140712105818.16750: *3* pd.Helpers 

220 #@+node:ekr.20140711111623.17845: *4* pd.at_data_body 

221 # Note: the unl of p relative to p is simply p.h, 

222 # so it is pointless to add that to @data nodes. 

223 

224 def at_data_body(self, p): 

225 """Return the body text for p's @data node.""" 

226 return f"gnx: {p.v.gnx}\n" 

227 #@+node:ekr.20140712105644.16744: *4* pd.expected_headline 

228 def expected_headline(self, p): 

229 """Return the expected imported headline for p.""" 

230 return getattr(p.v, '_imported_headline', p.h) 

231 #@+node:ekr.20140711111623.17854: *4* pd.find... 

232 # The find commands create the node if not found. 

233 #@+node:ekr.20140711111623.17856: *5* pd.find_at_data_node & helper 

234 def find_at_data_node(self, root): 

235 """ 

236 Return the @data node for root, a foreign node. 

237 Create the node if it does not exist. 

238 """ 

239 self.at_persistence = self.find_at_persistence_node() 

240 if not self.at_persistence: 

241 return None 

242 p = self.has_at_data_node(root) 

243 if p: 

244 return p 

245 p = self.at_persistence.insertAsLastChild() 

246 if not p: # #2103 

247 return None 

248 p.h = '@data:' + root.h 

249 p.b = self.at_data_body(root) 

250 return p 

251 #@+node:ekr.20140711111623.17857: *5* pd.find_at_gnxs_node 

252 def find_at_gnxs_node(self, root): 

253 """ 

254 Find the @gnxs node for root, a foreign node. 

255 Create the @gnxs node if it does not exist. 

256 """ 

257 h = '@gnxs' 

258 if not self.at_persistence: 

259 return None 

260 data = self.find_at_data_node(root) 

261 p = g.findNodeInTree(self.c, data, h) 

262 if p: 

263 return p 

264 p = data.insertAsLastChild() 

265 if p: # #2103 

266 p.h = h 

267 return p 

268 #@+node:ekr.20140711111623.17863: *5* pd.find_at_persistence_node 

269 def find_at_persistence_node(self): 

270 """ 

271 Find the first @persistence node in the outline. 

272 If it does not exist, create it as the *last* top-level node, 

273 so that no existing positions become invalid. 

274 """ 

275 c, h = self.c, '@persistence' 

276 p = g.findNodeAnywhere(c, h) 

277 if p: 

278 return p 

279 if c.config.getBool('create-at-persistence-nodes-automatically'): 

280 last = c.rootPosition() 

281 while last.hasNext(): 

282 last.moveToNext() 

283 p = last.insertAfter() 

284 if p: # #2103 

285 p.h = h 

286 g.es_print(f"created {h} node", color='red') 

287 return p 

288 #@+node:ekr.20140711111623.17891: *5* pd.find_at_uas_node 

289 def find_at_uas_node(self, root): 

290 """ 

291 Find the @uas node for root, a foreign node. 

292 Create the @uas node if it does not exist. 

293 """ 

294 h = '@uas' 

295 if not self.at_persistence: 

296 return None 

297 auto_view = self.find_at_data_node(root) 

298 p = g.findNodeInTree(self.c, auto_view, h) 

299 if p: 

300 return p 

301 p = auto_view.insertAsLastChild() 

302 if p: # #2103 

303 p.h = h 

304 return p 

305 #@+node:ekr.20140711111623.17861: *5* pd.find_position_for_relative_unl & helpers 

306 def find_position_for_relative_unl(self, root, unl): 

307 """ 

308 Given a unl relative to root, return the node whose 

309 unl matches the longest suffix of the given unl. 

310 """ 

311 unl_list = unl.split('-->') 

312 if not unl_list or len(unl_list) == 1 and not unl_list[0]: 

313 return root 

314 return self.find_exact_match(root, unl_list) 

315 # return self.find_best_match(root, unl_list) 

316 #@+node:ekr.20140716021139.17764: *6* pd.find_best_match 

317 def find_best_match(self, root, unl_list): 

318 """Find the best partial matches of the tail in root's tree.""" 

319 tail = unl_list[-1] 

320 matches = [] 

321 for p in root.self_and_subtree(copy=False): 

322 if p.h == tail: # A match 

323 # Compute the partial unl. 

324 parents = 0 

325 for parent2 in p.parents(): 

326 if parent2 == root: 

327 break 

328 elif parents + 2 > len(unl_list): 

329 break 

330 elif parent2.h != unl_list[-2 - parents]: 

331 break 

332 else: 

333 parents += 1 

334 matches.append((parents, p.copy()),) 

335 if matches: 

336 # Take the match with the greatest number of parents. 

337 

338 def key(aTuple): 

339 return aTuple[0] 

340 

341 n, p = list(sorted(matches, key=key))[-1] 

342 return p 

343 return None 

344 #@+node:ekr.20140716021139.17765: *6* pd.find_exact_match 

345 def find_exact_match(self, root, unl_list): 

346 """ 

347 Find an exact match of the unl_list in root's tree. 

348 The root does not appear in the unl_list. 

349 """ 

350 # full_unl = '-->'.join(unl_list) 

351 parent = root 

352 for unl in unl_list: 

353 for child in parent.children(): 

354 if child.h.strip() == unl.strip(): 

355 parent = child 

356 break 

357 else: 

358 return None 

359 return parent 

360 #@+node:ekr.20140711111623.17862: *5* pd.find_representative_node 

361 def find_representative_node(self, root, target): 

362 """ 

363 root is a foreign node. target is a gnxs node within root's tree. 

364 

365 Return a node *outside* of root's tree that is cloned to target, 

366 preferring nodes outside any @<file> tree. 

367 Never return any node in any @persistence tree. 

368 """ 

369 assert target 

370 assert root 

371 # Pass 1: accept only nodes outside any @file tree. 

372 p = self.c.rootPosition() 

373 while p: 

374 if p.h.startswith('@persistence'): 

375 p.moveToNodeAfterTree() 

376 elif p.isAnyAtFileNode(): 

377 p.moveToNodeAfterTree() 

378 elif p.v == target.v: 

379 return p 

380 else: 

381 p.moveToThreadNext() 

382 # Pass 2: accept any node outside the root tree. 

383 p = self.c.rootPosition() 

384 while p: 

385 if p.h.startswith('@persistence'): 

386 p.moveToNodeAfterTree() 

387 elif p == root: 

388 p.moveToNodeAfterTree() 

389 elif p.v == target.v: 

390 return p 

391 else: 

392 p.moveToThreadNext() 

393 g.trace('no representative node for:', target, 'parent:', target.parent()) 

394 return None 

395 #@+node:ekr.20140712105818.16751: *4* pd.foreign_file_name 

396 def foreign_file_name(self, p): 

397 """Return the file name for p, a foreign file node.""" 

398 for tag in ('@auto', '@org-mode', '@vim-outline'): 

399 if g.match_word(p.h, 0, tag): 

400 return p.h[len(tag) :].strip() 

401 return None 

402 #@+node:ekr.20140711111623.17864: *4* pd.has... 

403 # The has commands return None if the node does not exist. 

404 #@+node:ekr.20140711111623.17865: *5* pd.has_at_data_node 

405 def has_at_data_node(self, root): 

406 """ 

407 Return the @data node corresponding to root, a foreign node. 

408 Return None if no such node exists. 

409 """ 

410 # if g.unitTesting: 

411 # pass 

412 if not self.at_persistence: 

413 return None 

414 if not self.is_at_auto_node(root): 

415 return None 

416 # Find a direct child of the @persistence nodes with matching headline and body. 

417 s = self.at_data_body(root) 

418 for p in self.at_persistence.children(): 

419 if p.b == s: 

420 return p 

421 return None 

422 #@+node:ekr.20140711111623.17890: *5* pd.has_at_gnxs_node 

423 def has_at_gnxs_node(self, root): 

424 """ 

425 Find the @gnxs node for an @data node with the given unl. 

426 Return None if it does not exist. 

427 """ 

428 if self.at_persistence: 

429 p = self.has_at_data_node(root) 

430 return p and g.findNodeInTree(self.c, p, '@gnxs') 

431 return None 

432 #@+node:ekr.20140711111623.17894: *5* pd.has_at_uas_node 

433 def has_at_uas_node(self, root): 

434 """ 

435 Find the @uas node for an @data node with the given unl. 

436 Return None if it does not exist. 

437 """ 

438 if self.at_persistence: 

439 p = self.has_at_data_node(root) 

440 return p and g.findNodeInTree(self.c, p, '@uas') 

441 return None 

442 #@+node:ekr.20140711111623.17869: *5* pd.has_at_persistence_node 

443 def has_at_persistence_node(self): 

444 """Return the @persistence node or None if it does not exist.""" 

445 return g.findNodeAnywhere(self.c, '@persistence') 

446 #@+node:ekr.20140711111623.17870: *4* pd.is... 

447 #@+node:ekr.20140711111623.17871: *5* pd.is_at_auto_node 

448 def is_at_auto_node(self, p): 

449 """ 

450 Return True if p is *any* kind of @auto node, 

451 including @auto-otl and @auto-rst. 

452 """ 

453 return p.isAtAutoNode() 

454 # The safe way: it tracks changes to p.isAtAutoNode. 

455 #@+node:ekr.20140711111623.17897: *5* pd.is_at_file_node 

456 def is_at_file_node(self, p): 

457 """Return True if p is an @file node.""" 

458 return g.match_word(p.h, 0, '@file') 

459 #@+node:ekr.20140711111623.17872: *5* pd.is_cloned_outside_parent_tree 

460 def is_cloned_outside_parent_tree(self, p): 

461 """Return True if a clone of p exists outside the tree of p.parent().""" 

462 return len(list(set(p.v.parents))) > 1 

463 #@+node:ekr.20140712105644.16745: *5* pd.is_foreign_file 

464 def is_foreign_file(self, p): 

465 return ( 

466 self.is_at_auto_node(p) or 

467 g.match_word(p.h, 0, '@org-mode') or 

468 g.match_word(p.h, 0, '@vim-outline')) 

469 #@+node:ekr.20140713135856.17745: *4* pd.Pickling 

470 #@+node:ekr.20140713062552.17737: *5* pd.pickle 

471 def pickle(self, p): 

472 """Pickle val and return the hexlified result.""" 

473 try: 

474 ua = p.v.u 

475 s = pickle.dumps(ua, protocol=1) 

476 s2 = binascii.hexlify(s) 

477 s3 = g.toUnicode(s2, 'utf-8') 

478 return s3 

479 except pickle.PicklingError: 

480 g.warning("ignoring non-pickleable value", ua, "in", p.h) 

481 return '' 

482 except Exception: 

483 g.error("pd.pickle: unexpected exception in", p.h) 

484 g.es_exception() 

485 return '' 

486 #@+node:ekr.20140713135856.17744: *5* pd.unpickle 

487 def unpickle(self, s): 

488 """Unhexlify and unpickle string s into p.""" 

489 try: 

490 bin = binascii.unhexlify(g.toEncodedString(s)) 

491 # Throws TypeError if s is not a hex string. 

492 return pickle.loads(bin) 

493 except Exception: 

494 g.es_exception() 

495 return None 

496 #@+node:ekr.20140711111623.17879: *4* pd.unls... 

497 #@+node:ekr.20140711111623.17881: *5* pd.drop_unl_parent/tail 

498 def drop_unl_parent(self, unl): 

499 """Drop the penultimate part of the unl.""" 

500 aList = unl.split('-->') 

501 return '-->'.join(aList[:-2] + aList[-1:]) 

502 

503 def drop_unl_tail(self, unl): 

504 """Drop the last part of the unl.""" 

505 return '-->'.join(unl.split('-->')[:-1]) 

506 #@+node:ekr.20140711111623.17883: *5* pd.relative_unl 

507 def relative_unl(self, p, root): 

508 """Return the unl of p relative to the root position.""" 

509 result = [] 

510 for p in p.self_and_parents(copy=False): 

511 if p == root: 

512 break 

513 else: 

514 result.append(self.expected_headline(p)) 

515 return '-->'.join(reversed(result)) 

516 #@+node:ekr.20140711111623.17896: *5* pd.unl 

517 def unl(self, p): 

518 """Return the unl corresponding to the given position.""" 

519 return '-->'.join(reversed( 

520 [self.expected_headline(p2) for p2 in p.self_and_parents(copy=False)])) 

521 #@+node:ekr.20140711111623.17885: *5* pd.unl_tail 

522 def unl_tail(self, unl): 

523 """Return the last part of a unl.""" 

524 return unl.split('-->')[:-1][0] 

525 #@-others 

526#@-others 

527#@@language python 

528#@@tabwidth -4 

529#@@pagewidth 70 

530#@-leo