Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Second attempt for easily accessible dependencies #1961

Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions src/sas/qtgui/MainWindow/GuiManager.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
from sas.qtgui.MainWindow.AboutBox import AboutBox
from sas.qtgui.MainWindow.WelcomePanel import WelcomePanel
from sas.qtgui.MainWindow.CategoryManager import CategoryManager
from sas.qtgui.MainWindow.PackageGatherer import PackageGatherer

from sas.qtgui.MainWindow.DataManager import DataManager

Expand Down Expand Up @@ -456,6 +457,7 @@ def quitApplication(self):

# Exit if yes
if reply == QMessageBox.Yes:

# save the paths etc.
self.saveCustomConfig()
reactor.callFromThread(reactor.stop)
Expand Down Expand Up @@ -484,6 +486,18 @@ def checkUpdate(self):
except ValueError as ex:
logging.info("Failed to connect to www.sasview.org:", ex)

def log_installed_packages(self):
"""
Log version number of locally installed python packages
"""
PackageGatherer().log_installed_modules()

def log_imported_packages(self):
"""
Log version number of python packages imported in this instance of SasView.
"""
PackageGatherer().log_imported_modules()

def processVersion(self, version_info):
"""
Call-back method for the process of checking for updates.
Expand Down Expand Up @@ -644,6 +658,8 @@ def addTriggers(self):
self._workspace.actionAbout.triggered.connect(self.actionAbout)
self._workspace.actionWelcomeWidget.triggered.connect(self.actionWelcome)
self._workspace.actionCheck_for_update.triggered.connect(self.actionCheck_for_update)
self._workspace.actionLog_installed_packages.triggered.connect(self.actionLog_installed_packages)
self._workspace.actionLog_imported_packages.triggered.connect(self.actionLog_imported_packages)

self.communicate.sendDataToGridSignal.connect(self.showBatchOutput)
self.communicate.resultPlotUpdateSignal.connect(self.showFitResults)
Expand Down Expand Up @@ -1165,6 +1181,18 @@ def actionCheck_for_update(self):
"""
self.checkUpdate()

def actionLog_installed_packages(self):
"""
Log version number of locally installed python packages
"""
self.log_installed_packages()

def actionLog_imported_packages(self):
"""
Log version number of python packages imported in this instance of SasView.
"""
self.log_imported_packages()

def updateTheoryFromPerspective(self, index):
"""
Catch the theory update signal from a perspective
Expand Down
247 changes: 247 additions & 0 deletions src/sas/qtgui/MainWindow/PackageGatherer.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,247 @@
import sys
import logging
import subprocess
import pkg_resources
import json
import pathlib

import sas



class PackageGatherer:
"""
A class used to gather packages/modules used by SasView and their current installed version

Methods
-------
log_installed_modules
Log version number of locally installed python packages

log_imported_modules
Log version number of python packages imported in this instance of SasView.

get_imported_modules
Get a dictionary of imported module version numbers

remove_duplicate_modules
Strip duplicate instances of each module

format_unattainable_modules_list
Format module names in the unattainable_modules list
"""


def log_installed_modules(self):
""" Log version number of locally installed python packages

Use pip list to create a dictionary of installed modules as the keys, with their respective version numbers
as the values. Only packages available through pip will be included.

Returns
-------
None
"""
installed_modules = {'python': sys.version}

# Get python modules installed locally
installed_modules_json = json.loads(subprocess.check_output("pip list -l --format=json"))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fails for installed application.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Pip fails?

for mod in installed_modules_json:
installed_modules[mod['name']] = mod['version']

logging.info(f"Installed modules\n"
f"{installed_modules}")


def log_imported_modules(self):
""" Log version number of python packages imported in this instance of SasView.

Use the get_imported_modules method to to create a dictionary of installed modules as the keys, with their
respective version numbers as the values. There may be some packages whose version number is unattainable.

Returns
-------
None
"""
imported_modules = self.get_imported_modules()

logging.info(f"Imported modules\n"
f"{imported_modules}")


def get_imported_modules(self):
""" Get a dictionary of imported module version numbers

Use a variaty of method, for example a module.version call, to attempt to get the module version of each
module that has been imported in this instance of running SasView. The sys.modules command lists the
imported modules. A list of modules whose version number cannot be found is also included.

Returns
-------
module_versions_dict : dict
A dictionary with the module names as the key, with their respective version numbers as the value.
"""
module_versions_dict = {'python': sys.version, 'SasView': sas.sasview.__version__}
unattainable_modules = []
# Generate a list of standard modules by looking at the local python library
standard_lib = [path.stem.split('.')[0] for path in pathlib.Path(pathlib.__file__).parent.absolute().glob('*')]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can combine the standard lib packages, the builtin packages and sas into a single set. This simplifies the test in the loop:

standard_path = pathlib.Path(pathlib.__file__).parent.absolute()
standard_lib = {path.stem.split('.')[0] for path in standard_path.glob('*')} 
standard_lib.update(sys.builtin_module_names)
standard_lib.add('sas') # include 'sas' as standard lib

Note that this will not do what you want in the shipped application because pathlib may be in a zip file instead of on the disk. It may even fail in the application since pathlib.__file__ might not be defined. This is why I suggested hard-coding the list rather than building it on the fly.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi, so I've included your standard_path code in a try except block, with a hard coded list of bultin modules in the event that pathlib.file causes an error.

Could someone check if pathlib.file does indedd fail in the shipped application, as I would not know how to check.

If it does fail, then the list of builtin modules will probably have to be checked each time the python version SasView uses is updated.

May be worth having this list in an external file, making the job of updating the list easier?


for mod in sys.modules.keys():

# A built in python module or a local file, which have no version, only the python/SasView version
if mod in sys.builtin_module_names or mod.split('.')[0] in standard_lib or mod.split('.')[0] == 'sas':
continue

# Modules that require specific methods to get the version number
if "PyQt5" in mod:
try:
from PyQt5.QtCore import QT_VERSION_STR, PYQT_VERSION_STR
except:
unattainable_modules.append(["Qt", "PyQt"])
else:
module_versions_dict["Qt"] = QT_VERSION_STR
module_versions_dict["PyQt"] = PYQT_VERSION_STR
continue

# Different attempts of retrieving the modules version
try:
module_versions_dict[mod] = __import__(mod).__version__
except AttributeError:
# Module has no __version__ attribute
pass
except Exception as x:
# Unable to access module
logging.error(f"{x} when attempting to access {mod} version using .__version__")
pass
else:
continue

try:
module_versions_dict[mod] = __import__(mod).version
except AttributeError:
# Module has no .version attribute
pass
except Exception as x:
# Unable to access module
logging.error(f"{x} when attempting to access {mod} version using .version")
pass
else:
continue
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If version exists it might be a function or a module or a string. See my comment from before so I don't have to rewrite it. Or just punt and say that anything that isn't __version__ is a special case and address them as they appear.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same comment as above re: exception handling.

And to repeat, this will probably not do what you want.

Example

>>> h5py.version
<module 'h5py.version' from '/Users/pkienzle/conda/envs/py38/lib/python3.8/site-packages/h5py/version.py'>

Though in this case h5py.__version__ exists so you won't get here.

If you insist on keeping this clause at least check if isinstance(package.version, str).


# Unreliable, so last option
try:
module_versions_dict[mod] = pkg_resources.get_distribution(mod).version
except:
# Modules that cannot be found by pkg_resources
pass
else:
continue

# Below is code that calculates the version of each module using pip, however it is
# very time consuming, and only is useful for a handful of modules
# try:
# pip_module_show = str(subprocess.check_output(f"pip show {mod}"), 'utf-8')
# show_list = pip_module_show.replace("\r\n", ",").split(",")
# for sec in show_list:
# if sec.startswith("Version"):
# module_versions_dict[mod] = sec.split(":")[1].strip()
# else:
# # Unalbe to get version for this specific module
# pass
# except Exception as x:
# # Module not available through pip
# logging.error(f"{x} when attempting to get the version of {mod} through pip")
# pass
# else:
# continue

# Modules version number could not be attained by any of the previous methods
unattainable_modules.append(mod)

# Clean up
module_versions_dict = self.remove_duplicate_modules(module_versions_dict)
unattainable_modules = self.format_unattainable_modules_list(module_versions_dict, unattainable_modules)

# Modules whose version number could not be found
module_versions_dict["Can't get version number for following modules"] = unattainable_modules

return module_versions_dict



def remove_duplicate_modules(self, modules_dict):
""" Strip duplicate instances of each module

Multiple instances of one module can be keys of the dictionary of module version numbers generated by the
method get_imported_modules. This is because if an individual class is imported from a module, then each class
would be listed in sys.modules. For example the command from PyQt5.QtWidgets import QMainWindow, QMdiArea
lead to both QMainWindow and QMdiArea being keys, when in reality they are both part of PyQt5. This method
save the first instance of each module, unless the version numbers are different.

Parameters
----------
modules_dict : dict
A dictionary with the module names as the key, with their respective version numbers as the value.

Returns
-------
output_dict : dict
A reduced / cleaneddictionary with the module names as the key, with their respective version numbers
as the value.
"""

output_dict = dict()

for mod in modules_dict.keys():
parent_module = mod.split('.')[0]

# Save one instance of each module
if parent_module not in output_dict.keys():
output_dict[parent_module] = modules_dict[mod]
else:
# Modules versions are not the same
if output_dict[parent_module] != modules_dict[mod]:
output_dict[f"{parent_module}_from_{mod}"] = modules_dict[mod]
pass

return output_dict


def format_unattainable_modules_list(self, modules_dict, unattainable_modules):
"""Format module names in the unattainable_modules list

The unattainable_modules is a list of modules whose version number could not be found. This method rename each
module in the unattainable_modules to it's parent modules name, remove modules that already have a version
number and remove duplicate modules from the unattainable_modules list. Entries may appear in the
unattainable_modules if they are a class in a module, and the version number could not be ascertained from
the class.

Parameters
----------
modules_dict : dict
A dictionary with the module names as the key, with their respective version numbers as the value.
unattainable_modules : list
A list of modules whose version number could not be found.

Returns
-------
output_list : list
A reduced / clean list of modules whose version number could not be found

"""

output_list = list()

for mod in unattainable_modules:
parent_module = mod.split('.')[0]
# Version number exists for this module
if parent_module in modules_dict.keys():
pass
# Module is already in output_list
elif parent_module in output_list:
pass
# Append module to output_list
else:
output_list.append(mod)

return output_list
13 changes: 13 additions & 0 deletions src/sas/qtgui/MainWindow/UI/MainWindowUI.ui
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,9 @@
<addaction name="actionWelcomeWidget"/>
<addaction name="separator"/>
<addaction name="actionCheck_for_update"/>
<addaction name="separator"/>
<addaction name="actionLog_installed_packages"/>
<addaction name="actionLog_imported_packages"/>
</widget>
<addaction name="menu_File"/>
<addaction name="menuEdit"/>
Expand Down Expand Up @@ -514,6 +517,16 @@
<string>Check for update</string>
</property>
</action>
<action name="actionLog_installed_packages">
<property name="text">
<string>Log installed packages</string>
</property>
</action>
<action name="actionLog_imported_packages">
<property name="text">
<string>Log imported packages</string>
</property>
</action>
<action name="actionExcel">
<property name="text">
<string>Excel</string>
Expand Down