From 5e77abcb9951afd4ddfe3720a19752cc489e6e85 Mon Sep 17 00:00:00 2001 From: Tristan Gerrish Date: Mon, 7 Feb 2022 17:36:34 +0000 Subject: [PATCH 01/14] Commit for posterity --- .../Compute/InstallPythonEnvironment.cs | 1 + .../InstallToolkitPythonEnvironment.cs | 3 ++ Python_Engine/Create/PythonEnvironment.cs | 24 ++++++++++++ Python_Engine/Python/Python_Toolkit.json | 37 +++++++++++++++++++ 4 files changed, 65 insertions(+) create mode 100644 Python_Engine/Python/Python_Toolkit.json diff --git a/Python_Engine/Compute/InstallPythonEnvironment.cs b/Python_Engine/Compute/InstallPythonEnvironment.cs index 33063a1..d835176 100644 --- a/Python_Engine/Compute/InstallPythonEnvironment.cs +++ b/Python_Engine/Compute/InstallPythonEnvironment.cs @@ -46,6 +46,7 @@ public static PythonEnvironment InstallPythonEnvironment(bool run = false, bool new PythonPackage(){ Name="pymongo", Version="3.12.1" }, new PythonPackage(){ Name="SQLAlchemy", Version="1.4.27" }, new PythonPackage(){ Name="pyodbc", Version="4.0.32" }, + new PythonPackage(){ Name="jupyterlab", Version="3.2.9" }, }; PythonEnvironment pythonEnvironment = Create.PythonEnvironment(Query.ToolkitName(), version, packages); diff --git a/Python_Engine/Compute/InstallToolkitPythonEnvironment.cs b/Python_Engine/Compute/InstallToolkitPythonEnvironment.cs index 9c5bdad..7da43d6 100644 --- a/Python_Engine/Compute/InstallToolkitPythonEnvironment.cs +++ b/Python_Engine/Compute/InstallToolkitPythonEnvironment.cs @@ -44,6 +44,9 @@ public static PythonEnvironment InstallToolkitPythonEnvironment(this PythonEnvir if (!run) return null; + // check that base environment exists, and if not - create it! + //PythonEnvironment basePythonEnvironment = new PythonEnvironment("Python_Toolkit", ); + // load existing environment if it matches the requested environment PythonEnvironment existingEnvironment = Query.LoadPythonEnvironment(pythonEnvironment.Name); if (existingEnvironment.IsInstalled()) diff --git a/Python_Engine/Create/PythonEnvironment.cs b/Python_Engine/Create/PythonEnvironment.cs index ef313b0..e2a73c0 100644 --- a/Python_Engine/Create/PythonEnvironment.cs +++ b/Python_Engine/Create/PythonEnvironment.cs @@ -29,6 +29,8 @@ using System.IO; using System.Linq; using System; +using static System.Environment; +using BH.Adapter.File; namespace BH.Engine.Python { @@ -60,6 +62,28 @@ public static PythonEnvironment PythonEnvironment(string name, PythonVersion ver Packages = packages, }; } + + [Description("Create a BHoM Python environment from an environment.json config file.")] + [Input("config", "The path to the environment.json config file.")] + [Output("pythonEnvironment", "A BHoM PythonEnvironment object.")] + public static PythonEnvironment PythonEnvironment(string configJSON) + { + // load the json + string jsonStr = System.IO.File.ReadAllText(configJSON); + BH.oM.Base.CustomObject obj = (BH.oM.Base.CustomObject)BH.Engine.Serialiser.Convert.FromJson(jsonStr); + string version = (string)BH.Engine.Base.Query.PropertyValue(obj, "version"); + List> packages = (List>)BH.Engine.Base.Query.PropertyValue(obj, "packages"); + + // convert to the correct dtypes + // TODO - Convert teh string repr of the version into the Enum Python version + // TODO - Convert the package dicts into the PythonPackage objects + + // populate the PythonEnvironment + PythonEnvironment env = new PythonEnvironment(); + + return env; + + } } } diff --git a/Python_Engine/Python/Python_Toolkit.json b/Python_Engine/Python/Python_Toolkit.json new file mode 100644 index 0000000..6bece2e --- /dev/null +++ b/Python_Engine/Python/Python_Toolkit.json @@ -0,0 +1,37 @@ +{ + "version": "3.9.7", + "packages": [ + { + "name": "pandas", + "version": "1.3.4" + }, + { + "name": "tables", + "version": "3.7.0" + }, + { + "name": "numpy", + "version": "1.21.3" + }, + { + "name": "matplotlib", + "version": "3.4.3" + }, + { + "name": "pymongo", + "version": "3.12.1" + }, + { + "name": "SQLAlchemy", + "version": "1.4.27" + }, + { + "name": "pyodbc", + "version": "4.0.32" + }, + { + "name": "jupyterlab", + "version": "3.2.9" + } + ] +} From 648431b6a4911fef6de35cbdeaa98ebbc8070e75 Mon Sep 17 00:00:00 2001 From: Tristan Gerrish Date: Thu, 10 Feb 2022 17:01:47 +0000 Subject: [PATCH 02/14] Made environment configurable via JSON --- .../Compute/InstallPythonEnvironment.cs | 20 +++-------- Python_Engine/Create/PythonEnvironment.cs | 30 +++++++++++----- Python_Engine/Python/Python_Toolkit.json | 36 +++++++++---------- Python_Engine/Python_Engine.csproj | 23 +++++++++--- Python_Engine/Query/EmbeddableURL.cs | 5 +++ Python_oM/Enums/PythonVersion.cs | 11 +++++- Python_oM/Python_oM.csproj | 4 +-- 7 files changed, 79 insertions(+), 50 deletions(-) diff --git a/Python_Engine/Compute/InstallPythonEnvironment.cs b/Python_Engine/Compute/InstallPythonEnvironment.cs index d835176..22e66b9 100644 --- a/Python_Engine/Compute/InstallPythonEnvironment.cs +++ b/Python_Engine/Compute/InstallPythonEnvironment.cs @@ -33,25 +33,13 @@ public static partial class Compute [Description("Install the default BHoM Python_Toolkit environment.")] [Input("run", "Set to True to run the PythonEnvironment installer.")] [Input("force", "If the environment already exists recreate it rather than re-using it.")] + [Input("configJSON", "Path to a config JSON containing Python environment configuration.")] [Output("pythonEnvironment", "The default BHoM Python_Toolkit environment object.")] - public static PythonEnvironment InstallPythonEnvironment(bool run = false, bool force = false) + public static PythonEnvironment InstallPythonEnvironment(bool run = false, bool force = false, string configJSON = @"C:\ProgramData\BHoM\Settings\Python\Python_Toolkit.json") { - oM.Python.Enums.PythonVersion version = oM.Python.Enums.PythonVersion.v3_9_7; + PythonEnvironment pythonEnvironment = Create.PythonEnvironment(configJSON); - List packages = new List() - { - new PythonPackage(){ Name="pandas", Version="1.3.4" }, - new PythonPackage(){ Name="numpy", Version="1.21.3" }, - new PythonPackage(){ Name="matplotlib", Version="3.4.3" }, - new PythonPackage(){ Name="pymongo", Version="3.12.1" }, - new PythonPackage(){ Name="SQLAlchemy", Version="1.4.27" }, - new PythonPackage(){ Name="pyodbc", Version="4.0.32" }, - new PythonPackage(){ Name="jupyterlab", Version="3.2.9" }, - }; - - PythonEnvironment pythonEnvironment = Create.PythonEnvironment(Query.ToolkitName(), version, packages); - - return Python.Compute.InstallToolkitPythonEnvironment(pythonEnvironment, force, run); + return pythonEnvironment.InstallToolkitPythonEnvironment(force, run); } } } diff --git a/Python_Engine/Create/PythonEnvironment.cs b/Python_Engine/Create/PythonEnvironment.cs index e2a73c0..9b24566 100644 --- a/Python_Engine/Create/PythonEnvironment.cs +++ b/Python_Engine/Create/PythonEnvironment.cs @@ -68,21 +68,35 @@ public static PythonEnvironment PythonEnvironment(string name, PythonVersion ver [Output("pythonEnvironment", "A BHoM PythonEnvironment object.")] public static PythonEnvironment PythonEnvironment(string configJSON) { + string environmentName = System.IO.Path.GetFileNameWithoutExtension(configJSON); + // load the json string jsonStr = System.IO.File.ReadAllText(configJSON); BH.oM.Base.CustomObject obj = (BH.oM.Base.CustomObject)BH.Engine.Serialiser.Convert.FromJson(jsonStr); - string version = (string)BH.Engine.Base.Query.PropertyValue(obj, "version"); - List> packages = (List>)BH.Engine.Base.Query.PropertyValue(obj, "packages"); - // convert to the correct dtypes - // TODO - Convert teh string repr of the version into the Enum Python version - // TODO - Convert the package dicts into the PythonPackage objects + string versionString = (string)BH.Engine.Base.Query.PropertyValue(obj.CustomData, "Version"); + PythonVersion environmentVersion = (PythonVersion)Enum.Parse(typeof(PythonVersion), $"v{versionString.Replace(".", "_")}"); - // populate the PythonEnvironment - PythonEnvironment env = new PythonEnvironment(); + List pkgObjects = (List)BH.Engine.Base.Query.PropertyValue(obj.CustomData, "Packages"); + List environmentPackages = new List(); + foreach (object pkgObject in pkgObjects) + { + string pkgName = (string)BH.Engine.Base.Query.PropertyValue((BH.oM.Base.CustomObject)pkgObject, "Name"); + string pkgVersion = (string)BH.Engine.Base.Query.PropertyValue((BH.oM.Base.CustomObject)pkgObject, "Version"); + environmentPackages.Add( + new PythonPackage() { Name = pkgName, Version = pkgVersion } + ); + } - return env; + // populate the PythonEnvironment + PythonEnvironment pythonEnvironment = new PythonEnvironment() + { + Name = environmentName, + Version = environmentVersion, + Packages = environmentPackages + }; + return pythonEnvironment; } } } diff --git a/Python_Engine/Python/Python_Toolkit.json b/Python_Engine/Python/Python_Toolkit.json index 6bece2e..1a13eb2 100644 --- a/Python_Engine/Python/Python_Toolkit.json +++ b/Python_Engine/Python/Python_Toolkit.json @@ -1,37 +1,37 @@ { - "version": "3.9.7", - "packages": [ + "Version": "3.9.10", + "Packages": [ { - "name": "pandas", - "version": "1.3.4" + "Name": "pandas", + "Version": "1.4.0" }, { - "name": "tables", - "version": "3.7.0" + "Name": "tables", + "Version": "3.7.0" }, { - "name": "numpy", - "version": "1.21.3" + "Name": "numpy", + "Version": "1.22.2" }, { - "name": "matplotlib", - "version": "3.4.3" + "Name": "matplotlib", + "Version": "3.5.1" }, { - "name": "pymongo", - "version": "3.12.1" + "Name": "pymongo", + "Version": "4.0.1" }, { - "name": "SQLAlchemy", - "version": "1.4.27" + "Name": "SQLAlchemy", + "Version": "1.4.31" }, { - "name": "pyodbc", - "version": "4.0.32" + "Name": "pyodbc", + "Version": "4.0.32" }, { - "name": "jupyterlab", - "version": "3.2.9" + "Name": "jupyterlab", + "Version": "3.2.9" } ] } diff --git a/Python_Engine/Python_Engine.csproj b/Python_Engine/Python_Engine.csproj index 5cfab15..a30f3fd 100644 --- a/Python_Engine/Python_Engine.csproj +++ b/Python_Engine/Python_Engine.csproj @@ -1,4 +1,4 @@ - + @@ -36,11 +36,17 @@ C:\ProgramData\BHoM\Assemblies\BHoM.dll False + + ..\..\..\..\..\..\..\ProgramData\BHoM\Assemblies\BHoM_Adapter.dll + C:\ProgramData\BHoM\Assemblies\BHoM_Engine.dll False False + + ..\..\..\..\..\..\..\ProgramData\BHoM\Assemblies\File_Adapter.dll + False ..\libs\Python.Runtime.dll @@ -91,8 +97,8 @@ - + @@ -113,6 +119,7 @@ + @@ -142,13 +149,19 @@ xcopy "$(TargetDir)Python.Runtime.dll" "C:\\ProgramData\\BHoM\\Assemblies" /Y :: create infrastructure necessary to support Python environments if not exist "C:\ProgramData\BHoM\Extensions\PythonEnvironments" mkdir "C:\ProgramData\BHoM\Extensions\PythonEnvironments" if not exist "C:\ProgramData\BHoM\Extensions\PythonCode" mkdir "C:\ProgramData\BHoM\Extensions\PythonCode" -if not exist "C:\ProgramData\BHoM\Extensions\PythonCode\__init__.py" type nul>"C:\ProgramData\BHoM\Extensions\PythonCode\__init__.py" +if not exist "C:\ProgramData\BHoM\Extensions\PythonCode\__init__.py" type nul > "C:\ProgramData\BHoM\Extensions\PythonCode\__init__.py" -:: remove any existing within target directory for current toolkits Python code +:: remove any old versions within target directory for current toolkits Python code if exist "C:\ProgramData\BHoM\Extensions\PythonCode\$(SolutionName)" rmdir "C:\ProgramData\BHoM\Extensions\PythonCode\$(SolutionName)" /S /Q :: move Python code over to toolkit extensions folder, redirecting output to temp log file to silence it robocopy "$(ProjectDir)Python" "C:\ProgramData\BHoM\Extensions\PythonCode\$(SolutionName)" /s /purge /xf "*.pyc" /xd "\__pycache__\" > output.log + +:: copy local Python config JSON to BHoM/Settings/Python +if not exist "C:\ProgramData\BHoM\Settings\Python" mkdir "C:\ProgramData\BHoM\Settings\Python" +robocopy "$(ProjectDir)Python" "C:\ProgramData\BHoM\Settings\Python" "$(SolutionName).json" > output.log + +:: remove temporary log file del output.log - + \ No newline at end of file diff --git a/Python_Engine/Query/EmbeddableURL.cs b/Python_Engine/Query/EmbeddableURL.cs index cf2c872..a84aa69 100644 --- a/Python_Engine/Query/EmbeddableURL.cs +++ b/Python_Engine/Query/EmbeddableURL.cs @@ -73,7 +73,12 @@ public static string EmbeddableURL(this PythonVersion version) { PythonVersion.v3_9_5, "https://www.python.org/ftp/python/3.9.5/python-3.9.5-embed-amd64.zip" }, { PythonVersion.v3_9_6, "https://www.python.org/ftp/python/3.9.6/python-3.9.6-embed-amd64.zip" }, { PythonVersion.v3_9_7, "https://www.python.org/ftp/python/3.9.7/python-3.9.7-embed-amd64.zip" }, + { PythonVersion.v3_9_8, "https://www.python.org/ftp/python/3.9.8/python-3.9.8-embed-amd64.zip" }, + { PythonVersion.v3_9_9, "https://www.python.org/ftp/python/3.9.9/python-3.9.9-embed-amd64.zip" }, + { PythonVersion.v3_9_10, "https://www.python.org/ftp/python/3.9.10/python-3.9.10-embed-amd64.zip" }, { PythonVersion.v3_10_0, "https://www.python.org/ftp/python/3.10.0/python-3.10.0-embed-amd64.zip" }, + { PythonVersion.v3_10_1, "https://www.python.org/ftp/python/3.10.1/python-3.10.1-embed-amd64.zip" }, + { PythonVersion.v3_10_2, "https://www.python.org/ftp/python/3.10.2/python-3.10.2-embed-amd64.zip" }, }; return versions[version]; diff --git a/Python_oM/Enums/PythonVersion.cs b/Python_oM/Enums/PythonVersion.cs index 55d2d38..7ca1173 100644 --- a/Python_oM/Enums/PythonVersion.cs +++ b/Python_oM/Enums/PythonVersion.cs @@ -100,8 +100,17 @@ public enum PythonVersion v3_9_6, [Description("3.9.7")] v3_9_7, + [Description("3.9.8")] + v3_9_8, + [Description("3.9.9")] + v3_9_9, + [Description("3.9.10")] + v3_9_10, [Description("3.10.0")] v3_10_0, + [Description("3.10.1")] + v3_10_1, + [Description("3.10.2")] + v3_10_2, } } - diff --git a/Python_oM/Python_oM.csproj b/Python_oM/Python_oM.csproj index 61f7c27..20e6a23 100644 --- a/Python_oM/Python_oM.csproj +++ b/Python_oM/Python_oM.csproj @@ -1,4 +1,4 @@ - + @@ -63,4 +63,4 @@ xcopy "$(TargetDir)$(TargetFileName)" "C:\ProgramData\BHoM\Assemblies" /Y - + \ No newline at end of file From 47d0fcf9208ad1a87e25214cdf86bd7436ee5459 Mon Sep 17 00:00:00 2001 From: Tristan Gerrish Date: Thu, 10 Feb 2022 17:41:26 +0000 Subject: [PATCH 03/14] Simplified envirtonment creation --- .../Compute/InstallPythonEnvironment.cs | 111 +++++++++++++- .../InstallToolkitPythonEnvironment.cs | 139 ------------------ Python_Engine/Python_Engine.csproj | 1 - 3 files changed, 109 insertions(+), 142 deletions(-) delete mode 100644 Python_Engine/Compute/InstallToolkitPythonEnvironment.cs diff --git a/Python_Engine/Compute/InstallPythonEnvironment.cs b/Python_Engine/Compute/InstallPythonEnvironment.cs index 22e66b9..ad8ae73 100644 --- a/Python_Engine/Compute/InstallPythonEnvironment.cs +++ b/Python_Engine/Compute/InstallPythonEnvironment.cs @@ -25,11 +25,115 @@ using System.Collections.Generic; using System.ComponentModel; +using System.IO; +using System; +using System.Linq; namespace BH.Engine.Python { public static partial class Compute { + [Description("Install the BHoM Python environment.")] + [Input("pythonEnvironment", "A BHoM Python environment.")] + [Input("force", "If the environment already exists recreate it rather than re-using it.")] + [Input("run", "Set to True to run the PythonEnvironment installer.")] + [Output("pythonEnvironment", "A BHoM PythonEnvironment object.")] + public static PythonEnvironment InstallPythonEnvironment(this PythonEnvironment pythonEnvironment, bool force = false, bool run = false) + { + if (!run) + { + BH.Engine.Base.Compute.RecordNote($"This component will install a Python environment for {pythonEnvironment.Name} if it doesn not exist, or return the eixtsing environment if it does."); + return null; + } + + // load existing environment if it matches the requested environment + PythonEnvironment existingEnvironment = Query.LoadPythonEnvironment(pythonEnvironment.Name); + if (existingEnvironment.IsInstalled()) + { + if (!force) + { + if (existingEnvironment.Version != pythonEnvironment.Version) + { + BH.Engine.Base.Compute.RecordError($"An environment exists with the given name, but its Python version does not match the given version. To overwrite this existing environment set \"force\" to True."); + return null; + } + + // check that the existing environment contains the packages requested + List existingPackages = existingEnvironment.Packages; + foreach (PythonPackage pkg in pythonEnvironment.Packages) + { + if (!existingPackages.PackageInList(pkg)) + { + BH.Engine.Base.Compute.RecordError($"An environment exists with the given name, but it doesn't contain {pkg.GetString()}. To overwrite this existing environment set \"force\" to True."); + return null; + } + } + return existingEnvironment; + } + else + { + existingEnvironment.RemoveEnvironment(true); + } + } + + // create the new environment directory + string environmentDirectory = pythonEnvironment.EnvironmentDirectory(); + System.IO.Directory.CreateDirectory(environmentDirectory); + + // download the target version of Python + string embeddablePythonZip = Compute.DownloadFile(pythonEnvironment.Version.EmbeddableURL()); + + // extract embeddable python into environment directory + System.IO.Compression.ZipFile.ExtractToDirectory(embeddablePythonZip, environmentDirectory); + + // run the pip installer + string pipInstallerPy = Compute.DownloadFile("https://bootstrap.pypa.io/get-pip.py"); + string installPipCommand = $"{pythonEnvironment.PythonExecutable()} {pipInstallerPy} && exit"; + if (!Compute.RunCommandBool(installPipCommand, hideWindows: true)) + { + BH.Engine.Base.Compute.RecordError($"Pip installation did not work using the command {installPipCommand}."); + return null; + } + + // remove _pth file + List pthFiles = System.IO.Directory.GetFiles(environmentDirectory, "*.*", SearchOption.TopDirectoryOnly).Where(s => s.EndsWith("._pth")).ToList(); + foreach (string pthFile in pthFiles) + { + try + { + File.Delete(pthFile); + } + catch (Exception e) + { + BH.Engine.Base.Compute.RecordError($"{pthFile} not found to be deleted: {e}."); + return null; + } + } + + // move PYD and DLL files to DLLs directory + string libDirectory = System.IO.Directory.CreateDirectory(Path.Combine(environmentDirectory, "DLLs")).FullName; + List libFiles = System.IO.Directory.GetFiles(environmentDirectory, "*.*", SearchOption.TopDirectoryOnly).Where(s => (s.EndsWith(".dll") || s.EndsWith(".pyd")) && !Path.GetFileName(s).Contains("python") && !Path.GetFileName(s).Contains("vcruntime")).ToList(); + foreach (string libFile in libFiles) + { + string newLibFile = Path.Combine(libDirectory, Path.GetFileName(libFile)); + try + { + File.Move(libFile, newLibFile); + } + catch (Exception e) + { + BH.Engine.Base.Compute.RecordError($"{libFile} not capable of being moved to {newLibFile}: {e}."); + return null; + } + } + + // install packages using Pip + pythonEnvironment = pythonEnvironment.InstallPythonPackages(pythonEnvironment.Packages, force: true, run: true); + + return pythonEnvironment; + + } + [Description("Install the default BHoM Python_Toolkit environment.")] [Input("run", "Set to True to run the PythonEnvironment installer.")] [Input("force", "If the environment already exists recreate it rather than re-using it.")] @@ -37,9 +141,12 @@ public static partial class Compute [Output("pythonEnvironment", "The default BHoM Python_Toolkit environment object.")] public static PythonEnvironment InstallPythonEnvironment(bool run = false, bool force = false, string configJSON = @"C:\ProgramData\BHoM\Settings\Python\Python_Toolkit.json") { - PythonEnvironment pythonEnvironment = Create.PythonEnvironment(configJSON); + // Install base Python environment if it doesnt already exist + PythonEnvironment basePythonEnvironment = Create.PythonEnvironment(@"C:\ProgramData\BHoM\Settings\Python\Python_Toolkit.json"); + basePythonEnvironment.InstallPythonEnvironment(force, run); - return pythonEnvironment.InstallToolkitPythonEnvironment(force, run); + PythonEnvironment toolkitPythonEnvironment = Create.PythonEnvironment(configJSON); + return toolkitPythonEnvironment.InstallPythonEnvironment(force, run); } } } diff --git a/Python_Engine/Compute/InstallToolkitPythonEnvironment.cs b/Python_Engine/Compute/InstallToolkitPythonEnvironment.cs deleted file mode 100644 index 7da43d6..0000000 --- a/Python_Engine/Compute/InstallToolkitPythonEnvironment.cs +++ /dev/null @@ -1,139 +0,0 @@ -/* - * This file is part of the Buildings and Habitats object Model (BHoM) - * Copyright (c) 2015 - 2022, the respective contributors. All rights reserved. - * - * Each contributor holds copyright over their respective contributions. - * The project versioning (Git) records all such contribution source information. - * - * - * The BHoM is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3.0 of the License, or - * (at your option) any later version. - * - * The BHoM is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this code. If not, see . - */ - -using BH.oM.Python; -using BH.oM.Base.Attributes; -using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.IO; -using System.Linq; - -namespace BH.Engine.Python -{ - public static partial class Compute - { - [Description("Install the BHoM Python environment.")] - [Input("pythonEnvironment", "A BHoM Python environment.")] - [Input("force", "If the environment already exists recreate it rather than re-using it.")] - [Input("run", "Set to True to run the PythonEnvironment installer.")] - [Output("pythonEnvironment", "A BHoM PythonEnvironment object.")] - public static PythonEnvironment InstallToolkitPythonEnvironment(this PythonEnvironment pythonEnvironment, bool force = false, bool run = false) - { - BH.Engine.Base.Compute.RecordNote($"This component will install a Python environment for {pythonEnvironment.Name}."); - - if (!run) - return null; - - // check that base environment exists, and if not - create it! - //PythonEnvironment basePythonEnvironment = new PythonEnvironment("Python_Toolkit", ); - - // load existing environment if it matches the requested environment - PythonEnvironment existingEnvironment = Query.LoadPythonEnvironment(pythonEnvironment.Name); - if (existingEnvironment.IsInstalled()) - { - if (!force) - { - if (existingEnvironment.Version != pythonEnvironment.Version) - { - BH.Engine.Base.Compute.RecordError($"An environment exists with the given name, but its Python version does not match the given version. To overwrite this existing environment set \"force\" to True."); - return null; - } - - // check that the existing environment contains the packages requested - List existingPackages = existingEnvironment.Packages; - foreach (PythonPackage pkg in pythonEnvironment.Packages) - { - if (!existingPackages.PackageInList(pkg)) - { - BH.Engine.Base.Compute.RecordError($"An environment exists with the given name, but it doesn't contain {pkg.GetString()}. To overwrite this existing environment set \"force\" to True."); - return null; - } - } - return existingEnvironment; - } - else - { - existingEnvironment.RemoveEnvironment(true); - } - } - - // create the new environment directory - string environmentDirectory = pythonEnvironment.EnvironmentDirectory(); - System.IO.Directory.CreateDirectory(environmentDirectory); - - // download the target version of Python - string embeddablePythonZip = Compute.DownloadFile(pythonEnvironment.Version.EmbeddableURL()); - - // extract embeddable python into environment directory - System.IO.Compression.ZipFile.ExtractToDirectory(embeddablePythonZip, environmentDirectory); - - // run the pip installer - string pipInstallerPy = Compute.DownloadFile("https://bootstrap.pypa.io/get-pip.py"); - string installPipCommand = $"{pythonEnvironment.PythonExecutable()} {pipInstallerPy} && exit"; - if (!Compute.RunCommandBool(installPipCommand, hideWindows: true)) - { - BH.Engine.Base.Compute.RecordError($"Pip installation did not work using the command {installPipCommand}."); - return null; - } - - // remove _pth file - List pthFiles = System.IO.Directory.GetFiles(environmentDirectory, "*.*", SearchOption.TopDirectoryOnly).Where(s => s.EndsWith("._pth")).ToList(); - foreach (string pthFile in pthFiles) - { - try - { - File.Delete(pthFile); - } - catch (Exception e) - { - BH.Engine.Base.Compute.RecordError($"{pthFile} not found to be deleted: {e}."); - return null; - } - } - - // move PYD and DLL files to DLLs directory - string libDirectory = System.IO.Directory.CreateDirectory(Path.Combine(environmentDirectory, "DLLs")).FullName; - List libFiles = System.IO.Directory.GetFiles(environmentDirectory, "*.*", SearchOption.TopDirectoryOnly).Where(s => (s.EndsWith(".dll") || s.EndsWith(".pyd")) && !Path.GetFileName(s).Contains("python") && !Path.GetFileName(s).Contains("vcruntime")).ToList(); - foreach (string libFile in libFiles) - { - string newLibFile = Path.Combine(libDirectory, Path.GetFileName(libFile)); - try - { - File.Move(libFile, newLibFile); - } - catch (Exception e) - { - BH.Engine.Base.Compute.RecordError($"{libFile} not capable of being moved to {newLibFile}: {e}."); - return null; - } - } - - // install packages using Pip - pythonEnvironment = pythonEnvironment.InstallPythonPackages(pythonEnvironment.Packages, force: true, run: true); - - return pythonEnvironment; - - } - } -} - diff --git a/Python_Engine/Python_Engine.csproj b/Python_Engine/Python_Engine.csproj index a30f3fd..fef59d3 100644 --- a/Python_Engine/Python_Engine.csproj +++ b/Python_Engine/Python_Engine.csproj @@ -80,7 +80,6 @@ - From 1fb34c24641e28c80f7db79d13e1fdeaedb8809f Mon Sep 17 00:00:00 2001 From: Tristan Gerrish Date: Fri, 11 Feb 2022 10:46:06 +0000 Subject: [PATCH 04/14] Minor update --- Python_Engine/Create/PythonRequirementsTxt.cs | 64 +++++++++++++++++++ Python_Engine/Python_Engine.csproj | 1 + 2 files changed, 65 insertions(+) create mode 100644 Python_Engine/Create/PythonRequirementsTxt.cs diff --git a/Python_Engine/Create/PythonRequirementsTxt.cs b/Python_Engine/Create/PythonRequirementsTxt.cs new file mode 100644 index 0000000..9ede062 --- /dev/null +++ b/Python_Engine/Create/PythonRequirementsTxt.cs @@ -0,0 +1,64 @@ +/* + * This file is part of the Buildings and Habitats object Model (BHoM) + * Copyright (c) 2015 - 2022, the respective contributors. All rights reserved. + * + * Each contributor holds copyright over their respective contributions. + * The project versioning (Git) records all such contribution source information. + * + * + * The BHoM is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3.0 of the License, or + * (at your option) any later version. + * + * The BHoM is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this code. If not, see . + */ + +using BH.oM.Python; +using BH.oM.Base.Attributes; + +using System.ComponentModel; +using System.Linq; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BH.Engine.Python +{ + public static partial class Create + { + [Description("Create a requirements.txt file for use in creating a Python envrionment outside of BHoM.")] + [Input("pythonEnvironment", " A BHoM PythonEnvironment object.")] + [Input("directory", "The directory in which the requirements.txt file should be written.")] + [Output("file", "The created requirements.txt file.")] + public static string PythonRequirementsTxt(this PythonEnvironment pythonEnvironment, string directory) + { + return pythonEnvironment.Packages.PythonRequirementsTxt(directory); + } + + [Description("Create a requirements.txt file for use in creating a Python envrionment outside of BHoM.")] + [Input("packages", " A list of Python packages to incude in the requirements.txt file.")] + [Input("directory", "The directory in which the requirements.txt file should be written.")] + [Output("file", "The created requirements.txt file.")] + public static string PythonRequirementsTxt(this List packages, string directory) + { + string requirementsFile = $"{directory}\\requirements.txt"; + + StringBuilder sb = new StringBuilder(); + foreach (PythonPackage pkg in packages) + { + sb.AppendLine($"{pkg.Name}=={pkg.Version}"); + } + System.IO.File.WriteAllText(requirementsFile, sb.ToString()); + + return requirementsFile; + } + } +} + diff --git a/Python_Engine/Python_Engine.csproj b/Python_Engine/Python_Engine.csproj index fef59d3..2ad20ed 100644 --- a/Python_Engine/Python_Engine.csproj +++ b/Python_Engine/Python_Engine.csproj @@ -90,6 +90,7 @@ + From 1968ba10dfc4539ce69f3e78a4125f0259d254d2 Mon Sep 17 00:00:00 2001 From: Tristan Gerrish Date: Fri, 11 Feb 2022 11:14:28 +0000 Subject: [PATCH 05/14] Minor update --- Python_Engine/Query/EmbeddableURL.cs | 9 --------- Python_oM/Enums/PythonVersion.cs | 18 ------------------ 2 files changed, 27 deletions(-) diff --git a/Python_Engine/Query/EmbeddableURL.cs b/Python_Engine/Query/EmbeddableURL.cs index a84aa69..0f605f5 100644 --- a/Python_Engine/Query/EmbeddableURL.cs +++ b/Python_Engine/Query/EmbeddableURL.cs @@ -37,15 +37,6 @@ public static string EmbeddableURL(this PythonVersion version) { Dictionary versions = new Dictionary() { - { PythonVersion.v3_6_0, "https://www.python.org/ftp/python/3.6.0/python-3.6.0-embed-amd64.zip" }, - { PythonVersion.v3_6_1, "https://www.python.org/ftp/python/3.6.1/python-3.6.1-embed-amd64.zip" }, - { PythonVersion.v3_6_2, "https://www.python.org/ftp/python/3.6.2/python-3.6.2-embed-amd64.zip" }, - { PythonVersion.v3_6_3, "https://www.python.org/ftp/python/3.6.3/python-3.6.3-embed-amd64.zip" }, - { PythonVersion.v3_6_4, "https://www.python.org/ftp/python/3.6.4/python-3.6.4-embed-amd64.zip" }, - { PythonVersion.v3_6_5, "https://www.python.org/ftp/python/3.6.5/python-3.6.5-embed-amd64.zip" }, - { PythonVersion.v3_6_6, "https://www.python.org/ftp/python/3.6.6/python-3.6.6-embed-amd64.zip" }, - { PythonVersion.v3_6_7, "https://www.python.org/ftp/python/3.6.7/python-3.6.7-embed-amd64.zip" }, - { PythonVersion.v3_6_8, "https://www.python.org/ftp/python/3.6.8/python-3.6.8-embed-amd64.zip" }, { PythonVersion.v3_7_0, "https://www.python.org/ftp/python/3.7.0/python-3.7.0-embed-amd64.zip" }, { PythonVersion.v3_7_1, "https://www.python.org/ftp/python/3.7.1/python-3.7.1-embed-amd64.zip" }, { PythonVersion.v3_7_3, "https://www.python.org/ftp/python/3.7.3/python-3.7.3-embed-amd64.zip" }, diff --git a/Python_oM/Enums/PythonVersion.cs b/Python_oM/Enums/PythonVersion.cs index 7ca1173..3a5de0c 100644 --- a/Python_oM/Enums/PythonVersion.cs +++ b/Python_oM/Enums/PythonVersion.cs @@ -28,24 +28,6 @@ public enum PythonVersion { [Description("Undefined")] Undefined, - [Description("3.6.0")] - v3_6_0, - [Description("3.6.1")] - v3_6_1, - [Description("3.6.2")] - v3_6_2, - [Description("3.6.3")] - v3_6_3, - [Description("3.6.4")] - v3_6_4, - [Description("3.6.5")] - v3_6_5, - [Description("3.6.6")] - v3_6_6, - [Description("3.6.7")] - v3_6_7, - [Description("3.6.8")] - v3_6_8, [Description("3.7.0")] v3_7_0, [Description("3.7.1")] From 182a00b5801a57db3b11365c0fd87fb54d9f0122 Mon Sep 17 00:00:00 2001 From: Tristan Gerrish <10939984+tg359@users.noreply.github.com> Date: Fri, 11 Feb 2022 15:20:01 +0000 Subject: [PATCH 06/14] Update Python_Engine.csproj Fixed HintPath --- Python_Engine/Python_Engine.csproj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python_Engine/Python_Engine.csproj b/Python_Engine/Python_Engine.csproj index 2ad20ed..8515b3b 100644 --- a/Python_Engine/Python_Engine.csproj +++ b/Python_Engine/Python_Engine.csproj @@ -37,7 +37,7 @@ False - ..\..\..\..\..\..\..\ProgramData\BHoM\Assemblies\BHoM_Adapter.dll + C:\ProgramData\BHoM\Assemblies\BHoM_Adapter.dll C:\ProgramData\BHoM\Assemblies\BHoM_Engine.dll @@ -45,7 +45,7 @@ False - ..\..\..\..\..\..\..\ProgramData\BHoM\Assemblies\File_Adapter.dll + C:\ProgramData\BHoM\Assemblies\File_Adapter.dll False @@ -164,4 +164,4 @@ robocopy "$(ProjectDir)Python" "C:\ProgramData\BHoM\Settings\Python" "$(Solution :: remove temporary log file del output.log - \ No newline at end of file + From c11857234f8aabafdccdae4682adcc09e9c361af Mon Sep 17 00:00:00 2001 From: Tristan Gerrish <10939984+tg359@users.noreply.github.com> Date: Fri, 11 Feb 2022 15:22:55 +0000 Subject: [PATCH 07/14] Update PythonEnvironment.cs Removed auto-imported "using" in header --- Python_Engine/Create/PythonEnvironment.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Python_Engine/Create/PythonEnvironment.cs b/Python_Engine/Create/PythonEnvironment.cs index 9b24566..ec2de73 100644 --- a/Python_Engine/Create/PythonEnvironment.cs +++ b/Python_Engine/Create/PythonEnvironment.cs @@ -29,7 +29,6 @@ using System.IO; using System.Linq; using System; -using static System.Environment; using BH.Adapter.File; namespace BH.Engine.Python From 162bd669e25772ba124526522a3f079de653375a Mon Sep 17 00:00:00 2001 From: Fraser Greenroyd Date: Wed, 16 Feb 2022 10:05:45 +0000 Subject: [PATCH 08/14] Update Python_Engine/Compute/InstallPythonEnvironment.cs --- Python_Engine/Compute/InstallPythonEnvironment.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Python_Engine/Compute/InstallPythonEnvironment.cs b/Python_Engine/Compute/InstallPythonEnvironment.cs index ad8ae73..819a6d6 100644 --- a/Python_Engine/Compute/InstallPythonEnvironment.cs +++ b/Python_Engine/Compute/InstallPythonEnvironment.cs @@ -139,6 +139,7 @@ public static PythonEnvironment InstallPythonEnvironment(this PythonEnvironment [Input("force", "If the environment already exists recreate it rather than re-using it.")] [Input("configJSON", "Path to a config JSON containing Python environment configuration.")] [Output("pythonEnvironment", "The default BHoM Python_Toolkit environment object.")] + [PreviousVersion("5.1", "BH.Engine.Python.Compute.InstallPythonEnvironment(System.Boolean, System.Boolean)")] public static PythonEnvironment InstallPythonEnvironment(bool run = false, bool force = false, string configJSON = @"C:\ProgramData\BHoM\Settings\Python\Python_Toolkit.json") { // Install base Python environment if it doesnt already exist From 1d81417019eb1bc4f2781996b0f71b161b25dff7 Mon Sep 17 00:00:00 2001 From: Fraser Greenroyd Date: Wed, 16 Feb 2022 10:05:49 +0000 Subject: [PATCH 09/14] Update Python_Engine/Compute/InstallPythonEnvironment.cs --- Python_Engine/Compute/InstallPythonEnvironment.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Python_Engine/Compute/InstallPythonEnvironment.cs b/Python_Engine/Compute/InstallPythonEnvironment.cs index 819a6d6..ff92f81 100644 --- a/Python_Engine/Compute/InstallPythonEnvironment.cs +++ b/Python_Engine/Compute/InstallPythonEnvironment.cs @@ -38,6 +38,7 @@ public static partial class Compute [Input("force", "If the environment already exists recreate it rather than re-using it.")] [Input("run", "Set to True to run the PythonEnvironment installer.")] [Output("pythonEnvironment", "A BHoM PythonEnvironment object.")] + [PreviousVersion("5.1", "BH.Engine.Python.Compute.InstallToolkitPythonEnvironment(BH.oM.Python.PythonEnvironment, System.Boolean, System.Boolean)")] public static PythonEnvironment InstallPythonEnvironment(this PythonEnvironment pythonEnvironment, bool force = false, bool run = false) { if (!run) From 338eb2d0311f35ec42d75d38c42e76feefd1ca59 Mon Sep 17 00:00:00 2001 From: Fraser Greenroyd Date: Wed, 16 Feb 2022 10:05:53 +0000 Subject: [PATCH 10/14] Update Python_Engine/Create/PythonEnvironment.cs --- Python_Engine/Create/PythonEnvironment.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python_Engine/Create/PythonEnvironment.cs b/Python_Engine/Create/PythonEnvironment.cs index ec2de73..642ad1c 100644 --- a/Python_Engine/Create/PythonEnvironment.cs +++ b/Python_Engine/Create/PythonEnvironment.cs @@ -63,7 +63,7 @@ public static PythonEnvironment PythonEnvironment(string name, PythonVersion ver } [Description("Create a BHoM Python environment from an environment.json config file.")] - [Input("config", "The path to the environment.json config file.")] + [Input("configJSON", "The path to the environment.json config file.")] [Output("pythonEnvironment", "A BHoM PythonEnvironment object.")] public static PythonEnvironment PythonEnvironment(string configJSON) { From af77033e010df0b8d835030ec4769587d7cd1753 Mon Sep 17 00:00:00 2001 From: Tristan Gerrish Date: Wed, 16 Feb 2022 10:08:28 +0000 Subject: [PATCH 11/14] Moved file to proper namespace --- Python_Engine/{Create => Compute}/PythonRequirementsTxt.cs | 2 +- Python_Engine/Python_Engine.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename Python_Engine/{Create => Compute}/PythonRequirementsTxt.cs (98%) diff --git a/Python_Engine/Create/PythonRequirementsTxt.cs b/Python_Engine/Compute/PythonRequirementsTxt.cs similarity index 98% rename from Python_Engine/Create/PythonRequirementsTxt.cs rename to Python_Engine/Compute/PythonRequirementsTxt.cs index 9ede062..38747be 100644 --- a/Python_Engine/Create/PythonRequirementsTxt.cs +++ b/Python_Engine/Compute/PythonRequirementsTxt.cs @@ -31,7 +31,7 @@ namespace BH.Engine.Python { - public static partial class Create + public static partial class Compute { [Description("Create a requirements.txt file for use in creating a Python envrionment outside of BHoM.")] [Input("pythonEnvironment", " A BHoM PythonEnvironment object.")] diff --git a/Python_Engine/Python_Engine.csproj b/Python_Engine/Python_Engine.csproj index 8515b3b..d1d54e0 100644 --- a/Python_Engine/Python_Engine.csproj +++ b/Python_Engine/Python_Engine.csproj @@ -90,7 +90,7 @@ - + From 9dd9b4f6546352df6b86f9f4eb760e154b885e22 Mon Sep 17 00:00:00 2001 From: Tristan Gerrish Date: Wed, 16 Feb 2022 10:12:10 +0000 Subject: [PATCH 12/14] Updated dependencies.txt --- dependencies.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dependencies.txt b/dependencies.txt index 77b41ce..2be71ce 100644 --- a/dependencies.txt +++ b/dependencies.txt @@ -1,2 +1,4 @@ BHoM/BHoM -BHoM/BHoM_Engine \ No newline at end of file +BHoM/BHoM_Engine +BHoM/BHoM_Adapter +BHoM/File_Toolkit \ No newline at end of file From b96d120d60303859c1cda993c28f46317322aa79 Mon Sep 17 00:00:00 2001 From: BHoMBot Date: Wed, 16 Feb 2022 10:20:48 +0000 Subject: [PATCH 13/14] Update CSProject compliance --- Python_Engine/Python_Engine.csproj | 6 +++++- Python_oM/Python_oM.csproj | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Python_Engine/Python_Engine.csproj b/Python_Engine/Python_Engine.csproj index d1d54e0..b52ddde 100644 --- a/Python_Engine/Python_Engine.csproj +++ b/Python_Engine/Python_Engine.csproj @@ -1,4 +1,4 @@ - + @@ -38,6 +38,8 @@ C:\ProgramData\BHoM\Assemblies\BHoM_Adapter.dll + False + False C:\ProgramData\BHoM\Assemblies\BHoM_Engine.dll @@ -46,6 +48,8 @@ C:\ProgramData\BHoM\Assemblies\File_Adapter.dll + False + False False diff --git a/Python_oM/Python_oM.csproj b/Python_oM/Python_oM.csproj index 20e6a23..61f7c27 100644 --- a/Python_oM/Python_oM.csproj +++ b/Python_oM/Python_oM.csproj @@ -1,4 +1,4 @@ - + @@ -63,4 +63,4 @@ xcopy "$(TargetDir)$(TargetFileName)" "C:\ProgramData\BHoM\Assemblies" /Y - \ No newline at end of file + From e02ba751bdaa22abe550cf29bd0575e9ae10c82e Mon Sep 17 00:00:00 2001 From: Fraser Greenroyd Date: Mon, 21 Feb 2022 09:44:23 +0000 Subject: [PATCH 14/14] Remove reference to File_Toolkit --- Python_Engine/Create/PythonEnvironment.cs | 1 - Python_Engine/Python_Engine.csproj | 14 ++------------ dependencies.txt | 4 +--- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/Python_Engine/Create/PythonEnvironment.cs b/Python_Engine/Create/PythonEnvironment.cs index 642ad1c..3fce837 100644 --- a/Python_Engine/Create/PythonEnvironment.cs +++ b/Python_Engine/Create/PythonEnvironment.cs @@ -29,7 +29,6 @@ using System.IO; using System.Linq; using System; -using BH.Adapter.File; namespace BH.Engine.Python { diff --git a/Python_Engine/Python_Engine.csproj b/Python_Engine/Python_Engine.csproj index b52ddde..e7c6050 100644 --- a/Python_Engine/Python_Engine.csproj +++ b/Python_Engine/Python_Engine.csproj @@ -1,4 +1,4 @@ - + @@ -36,21 +36,11 @@ C:\ProgramData\BHoM\Assemblies\BHoM.dll False - - C:\ProgramData\BHoM\Assemblies\BHoM_Adapter.dll - False - False - C:\ProgramData\BHoM\Assemblies\BHoM_Engine.dll False False - - C:\ProgramData\BHoM\Assemblies\File_Adapter.dll - False - False - False ..\libs\Python.Runtime.dll @@ -168,4 +158,4 @@ robocopy "$(ProjectDir)Python" "C:\ProgramData\BHoM\Settings\Python" "$(Solution :: remove temporary log file del output.log - + \ No newline at end of file diff --git a/dependencies.txt b/dependencies.txt index 2be71ce..77b41ce 100644 --- a/dependencies.txt +++ b/dependencies.txt @@ -1,4 +1,2 @@ BHoM/BHoM -BHoM/BHoM_Engine -BHoM/BHoM_Adapter -BHoM/File_Toolkit \ No newline at end of file +BHoM/BHoM_Engine \ No newline at end of file