pyMez.Code.Utils.HelpUtils module
The HelpUtils module has tools for interacting with help files. It uses pdoc for auto-generated help and nbcovert to change ipynb based examples to html. There is an error when certain extensions are activated in jupyter for nbconvert that is solved by changing the imports in three modules see (https://github.com/jupyter/nbconvert/pull/370/commits/f01e44daca69f349bfdcf24aa397aa8edc7b2b53)
#----------------------------------------------------------------------------- # Name: HelpUtils # Purpose: Provides general utilities for help based functions # Author: Aric Sanders # Created: 3/11/2016 # License: MIT License #----------------------------------------------------------------------------- """ The HelpUtils module has tools for interacting with help files. It uses pdoc for auto-generated help and nbcovert to change ipynb based examples to html. There is an error when certain extensions are activated in jupyter for nbconvert that is solved by changing the imports in three modules see (https://github.com/jupyter/nbconvert/pull/370/commits/f01e44daca69f349bfdcf24aa397aa8edc7b2b53)""" #----------------------------------------------------------------------------- # Standard Imports import os import inspect import sys import shutil #----------------------------------------------------------------------------- # Third Party Imports sys.path.append(os.path.join(os.path.dirname( __file__ ), '..','..')) try: import pdoc except: print("Could not import pdoc, add it to the python path or install it. pip install pdoc") try: from Code.Utils.Alias import * METHOD_ALIASES=1 except: print("The module pyMez.Code.Utils.Alias was not found") METHOD_ALIASES=0 pass try: from Code.Utils.Names import auto_name,change_extension DEFAULT_FILE_NAME=None except: print("The function auto_name in pyMez.Code.Utils.Names was not found") print("Setting Default file name to New_Data_Table.txt") DEFAULT_FILE_NAME='New_Data_Table.txt' pass #----------------------------------------------------------------------------- # Module Constants TESTS_DIRECTORY=os.path.join(os.path.dirname(os.path.realpath(__file__)),'Tests') PYMEASURE_ROOT=os.path.join(os.path.dirname(os.path.realpath(__file__)),'..','..') DOCUMENTATION_DIRECTORY=os.path.join(PYMEASURE_ROOT,"Documentation") INDEX_HTML_PREFIX="""<html> <head> <style> body { background: #fff; font-family: "Source Sans Pro", "Helvetica Neueue", Helvetica, sans; font-weight: 300; font-size: 16px; line-height: 1.6em; background: #fff; color: #000; } li {font-family: "Ubuntu Mono", "Cousine", "DejaVu Sans Mono", monospace;} h1 { font-size: 2.5em; line-height: 1.1em; margin: 0 0 .50em 0; font-weight: 300; } </style> </head> <body> <a href="./pyMez_Documentation.html">Documentation Home</a> | <a href="./pyMez/index.html">API Documentation Home</a> | <a href="./Examples/html/Examples_Home.html">Examples Home</a> | <a href="./Reference_Index.html">Index </a> <h1>Index of All Functions and Classes in pyMez</h1> <ol>""" INDEX_HTML_POSTFIX="""</ol></body></html>""" IMPORT_HMTL_PREFIX="""<html> <head> <style> body { background: #fff; font-family: "Source Sans Pro", "Helvetica Neueue", Helvetica, sans; font-weight: 300; font-size: 16px; line-height: 1.6em; background: #fff; color: #000; } li {font-family: "Ubuntu Mono", "Cousine", "DejaVu Sans Mono", monospace;} h1 { font-size: 2.5em; line-height: 1.1em; margin: 0 0 .50em 0; font-weight: 300; } </style> </head> <body> <h1>All Imports</h1> <ol>""" #----------------------------------------------------------------------------- # Module Functions def return_help(object): """Returns an html help page autogenerated by the pdoc module for any live object""" module=inspect.getmodule(object).__name__ html_text=pdoc.html(module_name=module,allsubmodules=True) return html_text def create_help_page(module,output_format='html',file_path=None): """Uses the pdoc module to create a html autogenerated help file for the specified module. If file_path is not specified it auto names it and saves it in the current working directory.""" if re.search('htm',output_format,re.IGNORECASE): html_text=pdoc.html(module_name=module,allsubmodules=True) if file_path is None: file_path=auto_name(module.replace('.','_'),'Help',directory=None,extension='html') out_file=open(file_path,'w') out_file.write(html_text) out_file.close() def create_examples_page(ipynb_path): """Given a jupyter notebook uses nbconvert to output an html file at the specified path.""" os.system("jupyter nbconvert --to html %s"%ipynb_path) #----------------------------------------------------------------------------- # Module Classes #----------------------------------------------------------------------------- # Module Scripts def test_create_help_page(module='pyMez.Code.DataHandlers.GeneralModels'): "Tests the create help page function, it seems pretty slow" os.chdir(TESTS_DIRECTORY) create_help_page(module) def test_create_examples_page(ipynb_path='Development_Stack_Installation_Example_20160130_01.ipynb'): """Tests the create_examples_page function, really nb convert on the command line""" os.chdir(TESTS_DIRECTORY) create_examples_page(ipynb_path) def autogenerate_api_documentation_script(): """Autogenerates the api help files. It requires that pdoc is installed and that the pdoc script is in the Documentation folder under pyMez. If the folder exists it first deletes the folder and then creates a new one. The folder permissions must be set properly to allow the script to delete the old API help. Uses the mako templates found in Documentation/templates""" os.chdir(DOCUMENTATION_DIRECTORY) template_directory=os.path.join(DOCUMENTATION_DIRECTORY,"templates") try: shutil.rmtree(os.path.join(DOCUMENTATION_DIRECTORY,"pyMez")) except: print("Could not delete existing API documentation") pass os.system("python pdoc --html --overwrite --template-dir {0} pyMez".format(template_directory)) def create_index_html_script(top_directory): """create_index returns an html page with links to the autogenerated help page for all functions and classes in .py files in the directory structure. Uses the module constants INDEX_HTML_PREFIX and INDEX_HTML_POSTFIX to wrap the resulting html elements. """ classes_and_functions=[] class_pattern=re.compile('class (?P<name>\w+)\(') function_pattern=re.compile('def (?P<name>\w+)\(') links_dictionary={} link_template="<li><a href='{0}'>{1}</a></li>\n" for directory, dirnames, file_names in os.walk(top_directory): clean_directory=directory.split('..\\')[-1].replace("\\","/") for file_name in file_names: extension=file_name.split(".")[-1] if re.match('py',extension,re.IGNORECASE): in_file=open(os.path.join(directory,file_name),'r') for line in in_file: if re.search(class_pattern,line): reference_file="pyMez/"+change_extension(os.path.join(clean_directory,file_name),'m')+".html" reference_id = reference_file.split(".")[0] reference_file="./"+reference_file.replace("\\","/") name=re.search(class_pattern,line).groupdict()['name'] reference_id = reference_id.replace("\\", "/") reference_id=reference_id.replace("/", ".")+"."+name reference=reference_file+"#"+reference_id classes_and_functions.append(name) links_dictionary[name]=link_template.format(reference, name) elif re.match(function_pattern,line): reference_file="pyMez/"+change_extension(os.path.join(clean_directory,file_name),'m')+".html" reference_id = reference_file.split(".")[0] reference_file="./"+reference_file.replace("\\","/") name=re.search(function_pattern,line).groupdict()['name'] classes_and_functions.append(name) reference_id = reference_id.replace("\\", "/") reference_id=reference_id.replace("/", ".")+"."+name reference=reference_file+"#"+reference_id links_dictionary[name]=link_template.format(reference, name) #print("{0} is {1}".format('classes_and_functions',classes_and_functions)) #print("{0} is {1}".format('links_string',links_string)) # now arrange all in alphabetical order sorted_keys=sorted(links_dictionary.keys(),key=str.lower) links_string="" for key in sorted_keys: links_string=links_string+links_dictionary[key] links_string = INDEX_HTML_PREFIX + links_string + INDEX_HTML_POSTFIX out_file=open(os.path.join(DOCUMENTATION_DIRECTORY,"Reference_Index.html"),"w") out_file.write(links_string) out_file.close() def create_imports_html_script(top_directory,output_directory=None): """Reads all .py files under top directory and creates an html page that contains all of the imports and the file that imports them.""" if output_directory is None: output_directory=top_directory imports=[] import_pattern_1=re.compile('import (?P<name>[\w|\.]+\n)') import_pattern_2=re.compile('from (?P<name>\[w|\.]+) import \w+\n') for directory, dirnames, file_names in os.walk(top_directory): clean_directory=directory.split('..\\')[-1].replace("\\","/") for file_name in file_names: extension=file_name.split(".")[-1] if re.match('py',extension,re.IGNORECASE): in_file=open(os.path.join(directory,file_name),'r') for line in in_file: if re.search(import_pattern_1,line) and not re.match(import_pattern_2,line): name=re.search(import_pattern_1,line).groupdict()['name'] imports.append("{0} in {1}".format(name,file_name)) elif re.match(import_pattern_2,line): name=re.search(import_pattern_2,line).groupdict()['name'] imports.append("{0} in {1}".format(name, file_name)) unique_imports=sorted(list(set(imports))) html_string="" for import_name in unique_imports: html_string=html_string+"<li>{0}</li>".format(import_name) html_string=IMPORT_HMTL_PREFIX+html_string+INDEX_HTML_POSTFIX out_file=open(os.path.join(output_directory,"pyMez_Imports.html"),"w") out_file.write(html_string) out_file.close() def create_modules_html_script(top_directory,output_directory=None,output_name="pyMez_Modules.html"): """Creates an html page with all python modules under top_directory""" if output_directory is None: output_directory=top_directory modules=[] for directory, dirnames, file_names in os.walk(top_directory): for file_name in file_names: extension=file_name.split(".")[-1] if re.match('py(?!c)',extension,re.IGNORECASE): modules.append(file_name) html_string="" for module_name in modules: html_string=html_string+"<li>{0}</li>".format(module_name) html_string=IMPORT_HMTL_PREFIX.replace("All Imports","All Modules")+html_string+INDEX_HTML_POSTFIX out_file=open(os.path.join(output_directory,output_name),"w") out_file.write(html_string) out_file.close() def create_examples_html_script(jupyter_examples_directory): """Uses nbconvert to change all .ipynb to .html then moves all files that are not .ipynb, to a directory at the same level called html. It deletes out any files that are generated in the original directory""" html_examples_directory=os.path.join(jupyter_examples_directory,"..","html") os.chdir(jupyter_examples_directory) os.system("jupyter nbconvert *.ipynb --to html") shutil.rmtree(html_examples_directory) shutil.copytree(jupyter_examples_directory,html_examples_directory) os.chdir(html_examples_directory) file_names=os.listdir(html_examples_directory) # now clean out any .ipynb files in html for file_name in file_names: if re.search('.ipynb',file_name): try: os.remove(file_name) except: pass # now delete any generated .html files in jupyter os.chdir(jupyter_examples_directory) file_names = os.listdir(jupyter_examples_directory) for file_name in file_names: file_root=file_name.split(".")[0] if re.search('.html',file_name): ipynb_name=file_root+".ipynb" if ipynb_name in file_names: try: os.remove(file_name) except: pass def change_links_examples_script(top_directory): """Changes all the links with extensions .ipynb to .html, designed to be run after nbconvert has changed all of the .ipynb files to .html so that the new files will now link to .html versions of themselves instead of .ipynb""" file_names=os.listdir(top_directory) for file_name in file_names: extension=file_name.split(".")[-1] if re.search('htm',extension,re.IGNORECASE): in_file=open(os.path.join(top_directory,file_name),'r') content=in_file.read() new_content=content.replace('.ipynb','.html') in_file.close() out_file=open(os.path.join(top_directory,file_name),'w') out_file.write(new_content) out_file.close() def add_navigation_script(top_directory): """Adds navigation links to the body tag of all html files in top_directory, designed to add navigation to all files in Documentation/Examples/html""" navigation_html="""<a href="../../pyMez_Documentation.html">Documentation Home </a> | <a href="../../pyMez/index.html">API Documentation </a> | <a href="./Examples_Home.html">Examples</a> | <a href="../../Reference_Index.html">Index </a>""" file_names=os.listdir(top_directory) for file_name in file_names: extension=file_name.split(".")[-1] if re.search('htm',extension,re.IGNORECASE): in_file=open(os.path.join(top_directory,file_name),'r') lines=[] for line in in_file: if re.search("\<body\>",line,re.IGNORECASE): line=line+navigation_html lines.append(line) else: lines.append(line) in_file.close() out_file=open(os.path.join(top_directory,file_name),'w') out_file.writelines(lines) out_file.close() def test_create_index_html(top_directory=PYMEASURE_ROOT): "Tests the create_index_html_script by applying it to pyMez itself. " create_index_html_script(top_directory=top_directory) #----------------------------------------------------------------------------- # Module Runner if __name__ == '__main__': #test_create_help_page() #test_create_help_page('pyMez') #test_create_help_page('pyMez.Code.DataHandlers.NISTModels') #test_create_help_page('pyMez.Code.DataHandlers') #test_create_examples_page() test_create_examples_page(os.path.join(DOCUMENTATION_DIRECTORY,"pyMez_Documentation.ipynb")) autogenerate_api_documentation_script() test_create_index_html() #create_imports_html_script(PYMEASURE_ROOT,DOCUMENTATION_DIRECTORY) create_examples_html_script(os.path.join(DOCUMENTATION_DIRECTORY,"Examples","jupyter")) change_links_examples_script(os.path.join(DOCUMENTATION_DIRECTORY, "Examples", "html")) add_navigation_script(os.path.join(DOCUMENTATION_DIRECTORY, "Examples", "html")) #create_modules_html_script(PYMEASURE_ROOT)
Functions
Adds navigation links to the body tag of all html files in top_directory, designed to add navigation to all files in Documentation/Examples/html
def autogenerate_api_documentation_script(
)
Autogenerates the api help files. It requires that pdoc is installed and that the pdoc script is in the Documentation folder under pyMez. If the folder exists it first deletes the folder and then creates a new one. The folder permissions must be set properly to allow the script to delete the old API help. Uses the mako templates found in Documentation/templates
def autogenerate_api_documentation_script(): """Autogenerates the api help files. It requires that pdoc is installed and that the pdoc script is in the Documentation folder under pyMez. If the folder exists it first deletes the folder and then creates a new one. The folder permissions must be set properly to allow the script to delete the old API help. Uses the mako templates found in Documentation/templates""" os.chdir(DOCUMENTATION_DIRECTORY) template_directory=os.path.join(DOCUMENTATION_DIRECTORY,"templates") try: shutil.rmtree(os.path.join(DOCUMENTATION_DIRECTORY,"pyMez")) except: print("Could not delete existing API documentation") pass os.system("python pdoc --html --overwrite --template-dir {0} pyMez".format(template_directory))
def change_links_examples_script(
top_directory)
Changes all the links with extensions .ipynb to .html, designed to be run after nbconvert has changed all of the .ipynb files to .html so that the new files will now link to .html versions of themselves instead of .ipynb
def change_links_examples_script(top_directory): """Changes all the links with extensions .ipynb to .html, designed to be run after nbconvert has changed all of the .ipynb files to .html so that the new files will now link to .html versions of themselves instead of .ipynb""" file_names=os.listdir(top_directory) for file_name in file_names: extension=file_name.split(".")[-1] if re.search('htm',extension,re.IGNORECASE): in_file=open(os.path.join(top_directory,file_name),'r') content=in_file.read() new_content=content.replace('.ipynb','.html') in_file.close() out_file=open(os.path.join(top_directory,file_name),'w') out_file.write(new_content) out_file.close()
def create_examples_html_script(
jupyter_examples_directory)
Uses nbconvert to change all .ipynb to .html then moves all files that are not .ipynb, to a directory at the same level called html. It deletes out any files that are generated in the original directory
def create_examples_html_script(jupyter_examples_directory): """Uses nbconvert to change all .ipynb to .html then moves all files that are not .ipynb, to a directory at the same level called html. It deletes out any files that are generated in the original directory""" html_examples_directory=os.path.join(jupyter_examples_directory,"..","html") os.chdir(jupyter_examples_directory) os.system("jupyter nbconvert *.ipynb --to html") shutil.rmtree(html_examples_directory) shutil.copytree(jupyter_examples_directory,html_examples_directory) os.chdir(html_examples_directory) file_names=os.listdir(html_examples_directory) # now clean out any .ipynb files in html for file_name in file_names: if re.search('.ipynb',file_name): try: os.remove(file_name) except: pass # now delete any generated .html files in jupyter os.chdir(jupyter_examples_directory) file_names = os.listdir(jupyter_examples_directory) for file_name in file_names: file_root=file_name.split(".")[0] if re.search('.html',file_name): ipynb_name=file_root+".ipynb" if ipynb_name in file_names: try: os.remove(file_name) except: pass
def create_examples_page(
ipynb_path)
Given a jupyter notebook uses nbconvert to output an html file at the specified path.
def create_examples_page(ipynb_path): """Given a jupyter notebook uses nbconvert to output an html file at the specified path.""" os.system("jupyter nbconvert --to html %s"%ipynb_path)
def create_help_page(
module, output_format='html', file_path=None)
Uses the pdoc module to create a html autogenerated help file for the specified module. If file_path is not specified it auto names it and saves it in the current working directory.
def create_help_page(module,output_format='html',file_path=None): """Uses the pdoc module to create a html autogenerated help file for the specified module. If file_path is not specified it auto names it and saves it in the current working directory.""" if re.search('htm',output_format,re.IGNORECASE): html_text=pdoc.html(module_name=module,allsubmodules=True) if file_path is None: file_path=auto_name(module.replace('.','_'),'Help',directory=None,extension='html') out_file=open(file_path,'w') out_file.write(html_text) out_file.close()
def create_imports_html_script(
top_directory, output_directory=None)
Reads all .py files under top directory and creates an html page that contains all of the imports and the file that imports them.
def create_imports_html_script(top_directory,output_directory=None): """Reads all .py files under top directory and creates an html page that contains all of the imports and the file that imports them.""" if output_directory is None: output_directory=top_directory imports=[] import_pattern_1=re.compile('import (?P<name>[\w|\.]+\n)') import_pattern_2=re.compile('from (?P<name>\[w|\.]+) import \w+\n') for directory, dirnames, file_names in os.walk(top_directory): clean_directory=directory.split('..\\')[-1].replace("\\","/") for file_name in file_names: extension=file_name.split(".")[-1] if re.match('py',extension,re.IGNORECASE): in_file=open(os.path.join(directory,file_name),'r') for line in in_file: if re.search(import_pattern_1,line) and not re.match(import_pattern_2,line): name=re.search(import_pattern_1,line).groupdict()['name'] imports.append("{0} in {1}".format(name,file_name)) elif re.match(import_pattern_2,line): name=re.search(import_pattern_2,line).groupdict()['name'] imports.append("{0} in {1}".format(name, file_name)) unique_imports=sorted(list(set(imports))) html_string="" for import_name in unique_imports: html_string=html_string+"<li>{0}</li>".format(import_name) html_string=IMPORT_HMTL_PREFIX+html_string+INDEX_HTML_POSTFIX out_file=open(os.path.join(output_directory,"pyMez_Imports.html"),"w") out_file.write(html_string) out_file.close()
def create_index_html_script(
top_directory)
create_index returns an html page with links to the autogenerated help page for all functions and classes in .py files in the directory structure. Uses the module constants INDEX_HTML_PREFIX and INDEX_HTML_POSTFIX to wrap the resulting html elements.
def create_index_html_script(top_directory): """create_index returns an html page with links to the autogenerated help page for all functions and classes in .py files in the directory structure. Uses the module constants INDEX_HTML_PREFIX and INDEX_HTML_POSTFIX to wrap the resulting html elements. """ classes_and_functions=[] class_pattern=re.compile('class (?P<name>\w+)\(') function_pattern=re.compile('def (?P<name>\w+)\(') links_dictionary={} link_template="<li><a href='{0}'>{1}</a></li>\n" for directory, dirnames, file_names in os.walk(top_directory): clean_directory=directory.split('..\\')[-1].replace("\\","/") for file_name in file_names: extension=file_name.split(".")[-1] if re.match('py',extension,re.IGNORECASE): in_file=open(os.path.join(directory,file_name),'r') for line in in_file: if re.search(class_pattern,line): reference_file="pyMez/"+change_extension(os.path.join(clean_directory,file_name),'m')+".html" reference_id = reference_file.split(".")[0] reference_file="./"+reference_file.replace("\\","/") name=re.search(class_pattern,line).groupdict()['name'] reference_id = reference_id.replace("\\", "/") reference_id=reference_id.replace("/", ".")+"."+name reference=reference_file+"#"+reference_id classes_and_functions.append(name) links_dictionary[name]=link_template.format(reference, name) elif re.match(function_pattern,line): reference_file="pyMez/"+change_extension(os.path.join(clean_directory,file_name),'m')+".html" reference_id = reference_file.split(".")[0] reference_file="./"+reference_file.replace("\\","/") name=re.search(function_pattern,line).groupdict()['name'] classes_and_functions.append(name) reference_id = reference_id.replace("\\", "/") reference_id=reference_id.replace("/", ".")+"."+name reference=reference_file+"#"+reference_id links_dictionary[name]=link_template.format(reference, name) #print("{0} is {1}".format('classes_and_functions',classes_and_functions)) #print("{0} is {1}".format('links_string',links_string)) # now arrange all in alphabetical order sorted_keys=sorted(links_dictionary.keys(),key=str.lower) links_string="" for key in sorted_keys: links_string=links_string+links_dictionary[key] links_string = INDEX_HTML_PREFIX + links_string + INDEX_HTML_POSTFIX out_file=open(os.path.join(DOCUMENTATION_DIRECTORY,"Reference_Index.html"),"w") out_file.write(links_string) out_file.close()
def create_modules_html_script(
top_directory, output_directory=None, output_name='pyMez_Modules.html')
Creates an html page with all python modules under top_directory
def create_modules_html_script(top_directory,output_directory=None,output_name="pyMez_Modules.html"): """Creates an html page with all python modules under top_directory""" if output_directory is None: output_directory=top_directory modules=[] for directory, dirnames, file_names in os.walk(top_directory): for file_name in file_names: extension=file_name.split(".")[-1] if re.match('py(?!c)',extension,re.IGNORECASE): modules.append(file_name) html_string="" for module_name in modules: html_string=html_string+"<li>{0}</li>".format(module_name) html_string=IMPORT_HMTL_PREFIX.replace("All Imports","All Modules")+html_string+INDEX_HTML_POSTFIX out_file=open(os.path.join(output_directory,output_name),"w") out_file.write(html_string) out_file.close()
def return_help(
object)
Returns an html help page autogenerated by the pdoc module for any live object
def return_help(object): """Returns an html help page autogenerated by the pdoc module for any live object""" module=inspect.getmodule(object).__name__ html_text=pdoc.html(module_name=module,allsubmodules=True) return html_text
def test_create_examples_page(
ipynb_path='Development_Stack_Installation_Example_20160130_01.ipynb')
Tests the create_examples_page function, really nb convert on the command line
def test_create_examples_page(ipynb_path='Development_Stack_Installation_Example_20160130_01.ipynb'): """Tests the create_examples_page function, really nb convert on the command line""" os.chdir(TESTS_DIRECTORY) create_examples_page(ipynb_path)
def test_create_help_page(
module='pyMez.Code.DataHandlers.GeneralModels')
Tests the create help page function, it seems pretty slow
def test_create_help_page(module='pyMez.Code.DataHandlers.GeneralModels'): "Tests the create help page function, it seems pretty slow" os.chdir(TESTS_DIRECTORY) create_help_page(module)
def test_create_index_html(
top_directory='C:\\ProgramData\\Anaconda2\\lib\\site-packages\\pyMez\\Code\\Utils\\..\\..')
Tests the create_index_html_script by applying it to pyMez itself.
def test_create_index_html(top_directory=PYMEASURE_ROOT): "Tests the create_index_html_script by applying it to pyMez itself. " create_index_html_script(top_directory=top_directory)
Module variables
var DEFAULT_FILE_NAME
var DOCUMENTATION_DIRECTORY
var IMPORT_HMTL_PREFIX
var INDEX_HTML_POSTFIX
var INDEX_HTML_PREFIX
var METHOD_ALIASES
var PYMEASURE_ROOT
var TESTS_DIRECTORY