From 231c1fd44e72b91127a76846e3fe50192eeaa2e6 Mon Sep 17 00:00:00 2001 From: himanshumahajan138 Date: Mon, 2 Dec 2024 01:01:15 +0530 Subject: [PATCH] Fixes: #3928 ; First Test Phase --- example/package.mill | 1 + .../basic/2-custom-build-logic/foo/src/foo.py | 4 +- .../module/1-common-config/build.mill | 75 +++++++++++++++++++ .../custom-resources/MyOtherResources.txt | 1 + .../module/1-common-config/custom-src/foo2.py | 48 ++++++++++++ .../1-common-config/resources/MyResource.txt | 1 + .../module/1-common-config/src/foo.py | 15 ++++ .../module/2-custom-tasks/build.mill | 57 ++++++++++++++ .../module/2-custom-tasks/src/foo.py | 22 ++++++ .../module/3-override-tasks/build.mill | 68 +++++++++++++++++ .../4-compilation-execution-flags/build.mill | 15 ++++ .../4-compilation-execution-flags/src/foo.py | 8 ++ .../pythonlib/module/5-resources/build.mill | 29 +++++++ .../module/5-resources/foo/resources/file.txt | 1 + .../module/5-resources/foo/src/foo.py | 11 +++ .../foo/test/other-files/other-file.txt | 1 + .../foo/test/resources/test-file-a.txt | 1 + .../foo/test/resources/test-file-b.txt | 1 + .../module/5-resources/foo/test/src/test.py | 36 +++++++++ .../src/mill/pythonlib/PythonModule.scala | 17 ++++- 20 files changed, 407 insertions(+), 5 deletions(-) create mode 100644 example/pythonlib/module/1-common-config/build.mill create mode 100644 example/pythonlib/module/1-common-config/custom-resources/MyOtherResources.txt create mode 100644 example/pythonlib/module/1-common-config/custom-src/foo2.py create mode 100644 example/pythonlib/module/1-common-config/resources/MyResource.txt create mode 100644 example/pythonlib/module/1-common-config/src/foo.py create mode 100644 example/pythonlib/module/2-custom-tasks/build.mill create mode 100644 example/pythonlib/module/2-custom-tasks/src/foo.py create mode 100644 example/pythonlib/module/3-override-tasks/build.mill create mode 100644 example/pythonlib/module/4-compilation-execution-flags/build.mill create mode 100644 example/pythonlib/module/4-compilation-execution-flags/src/foo.py create mode 100644 example/pythonlib/module/5-resources/build.mill create mode 100644 example/pythonlib/module/5-resources/foo/resources/file.txt create mode 100644 example/pythonlib/module/5-resources/foo/src/foo.py create mode 100644 example/pythonlib/module/5-resources/foo/test/other-files/other-file.txt create mode 100644 example/pythonlib/module/5-resources/foo/test/resources/test-file-a.txt create mode 100644 example/pythonlib/module/5-resources/foo/test/resources/test-file-b.txt create mode 100644 example/pythonlib/module/5-resources/foo/test/src/test.py diff --git a/example/package.mill b/example/package.mill index 7b43280b76f..9de96bd1671 100644 --- a/example/package.mill +++ b/example/package.mill @@ -67,6 +67,7 @@ object `package` extends RootModule with Module { object pythonlib extends Module { object basic extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "basic")) object dependencies extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "dependencies")) + object module extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "module")) } object cli extends Module{ diff --git a/example/pythonlib/basic/2-custom-build-logic/foo/src/foo.py b/example/pythonlib/basic/2-custom-build-logic/foo/src/foo.py index 3997e6c2095..8bb0aafa948 100644 --- a/example/pythonlib/basic/2-custom-build-logic/foo/src/foo.py +++ b/example/pythonlib/basic/2-custom-build-logic/foo/src/foo.py @@ -2,8 +2,8 @@ def line_count() -> int: - with importlib.resources.open_text("resources", "line-count.txt") as file: - return int(file.readline().strip()) + resource_content = (importlib.resources.files("resources").joinpath("line-count.txt").read_text()) + return int(resource_content.strip()) if __name__ == "__main__": diff --git a/example/pythonlib/module/1-common-config/build.mill b/example/pythonlib/module/1-common-config/build.mill new file mode 100644 index 00000000000..9f05ab06877 --- /dev/null +++ b/example/pythonlib/module/1-common-config/build.mill @@ -0,0 +1,75 @@ +package build +import mill._, pythonlib._ + +object `package` extends RootModule with PythonModule { + // You can have arbitrary numbers of third-party libraries + def pythonDeps = Seq("MarkupSafe==3.0.2", "Jinja2==3.1.4") + + // choose a main Script to run if there are multiple present + def mainScript = Task.Source { millSourcePath / "custom-src" / "foo2.py" } + + // TODO: we have to choose whether to include `millSourcePath` by default in PYTHONPATH or not. + // def sources = Task.Sources { + // super.sources() ++ Seq(PathRef(millSourcePath)) + // } + + // def resources = Task.Sources { + // super.resources() ++ Seq(PathRef(millSourcePath)) + // } + + def generatedSources: T[Seq[PathRef]] = Task { + val destPath = Task.dest / "generatedSources" + os.makeDir.all(destPath) + for (name <- Seq("A", "B", "C")) os.write( + destPath / s"foo$name.py", + s""" + |class Foo$name: + | value = "hello $name" + """.stripMargin + ) + + Seq(PathRef(destPath)) + } + + def forkEnv: T[Map[String, String]] = Map("MY_CUSTOM_ENV" -> "my-env-value") + + // Additional Python options e.g. to Turn On Warnings + // we can use -Werror to treat warnings as errors + def pythonOptions: T[Seq[String]] = Seq("-Wall") + +} + +/** Usage + +> ./mill run +... +Foo2.value:

hello2

+Foo.value:

hello

+FooA.value: hello A +FooB.value: hello B +FooC.value: hello C +MyResource: My Resource Contents +MyOtherResource: My Other Resource Contents +MY_CUSTOM_ENV: my-env-value +... + +> ./mill show bundle +".../out/bundle.dest/bundle.pex" + +> ./out/bundle.dest/bundle.pex +... +Foo2.value:

hello2

+Foo.value:

hello

+FooA.value: hello A +FooB.value: hello B +FooC.value: hello C +MyResource: My Resource Contents +MyOtherResource: My Other Resource Contents +... + +> sed -i.bak 's/import os/import os, warnings; warnings.warn("This is a test warning!")/g' custom-src/foo2.py + +> ./mill run +...UserWarning: This is a test warning!... + +*/ diff --git a/example/pythonlib/module/1-common-config/custom-resources/MyOtherResources.txt b/example/pythonlib/module/1-common-config/custom-resources/MyOtherResources.txt new file mode 100644 index 00000000000..8c584b77885 --- /dev/null +++ b/example/pythonlib/module/1-common-config/custom-resources/MyOtherResources.txt @@ -0,0 +1 @@ +My Other Resource Contents \ No newline at end of file diff --git a/example/pythonlib/module/1-common-config/custom-src/foo2.py b/example/pythonlib/module/1-common-config/custom-src/foo2.py new file mode 100644 index 00000000000..db06adf1307 --- /dev/null +++ b/example/pythonlib/module/1-common-config/custom-src/foo2.py @@ -0,0 +1,48 @@ +import os +import importlib.resources +from jinja2 import Template +from foo import Foo # type: ignore +from fooA import FooA # type: ignore +from fooB import FooB # type: ignore +from fooC import FooC # type: ignore + + +class Foo2: + + def value(self, text: str): + """Generates an HTML template with dynamic content.""" + template = Template("

{{ text }}

") + return template.render(text=text) + + def read_resource(self, package: str, resource_name: str) -> str: + """Reads the content of a resource file.""" + try: + resource_content = ( + importlib.resources.files(package).joinpath(resource_name).read_text() + ) + return resource_content.strip() + except FileNotFoundError: + return f"Resource '{resource_name}' not found." + + def main(self): + # Output for value() + print(f"Foo2.value: {self.value('hello2')}") + print(f"Foo.value: {Foo().value('hello')}") + print(f"FooA.value: {FooA.value}") + print(f"FooB.value: {FooB.value}") + print(f"FooC.value: {FooC.value}") + + # Reading resources + print(f"MyResource: {self.read_resource('resources', 'MyResource.txt')}") + print( + f"MyOtherResource: {self.read_resource('custom-resources', 'MyOtherResources.txt')}" + ) + + # Accessing environment variable + my_custom_env = os.environ.get("MY_CUSTOM_ENV") + if my_custom_env: + print(f"MY_CUSTOM_ENV: {my_custom_env}") + + +if __name__ == "__main__": + Foo2().main() diff --git a/example/pythonlib/module/1-common-config/resources/MyResource.txt b/example/pythonlib/module/1-common-config/resources/MyResource.txt new file mode 100644 index 00000000000..697537075e4 --- /dev/null +++ b/example/pythonlib/module/1-common-config/resources/MyResource.txt @@ -0,0 +1 @@ +My Resource Contents \ No newline at end of file diff --git a/example/pythonlib/module/1-common-config/src/foo.py b/example/pythonlib/module/1-common-config/src/foo.py new file mode 100644 index 00000000000..31cd9f883cf --- /dev/null +++ b/example/pythonlib/module/1-common-config/src/foo.py @@ -0,0 +1,15 @@ +from jinja2 import Template + + +class Foo: + def value(self, text: str): + """Generates an HTML template with dynamic content.""" + template = Template("

{{ text }}

") + return template.render(text=text) + + def main(self): + print(f"Foo.value: {self.value('hello')}") + + +if __name__ == "__main__": + Foo().main() diff --git a/example/pythonlib/module/2-custom-tasks/build.mill b/example/pythonlib/module/2-custom-tasks/build.mill new file mode 100644 index 00000000000..c14ff2f3eae --- /dev/null +++ b/example/pythonlib/module/2-custom-tasks/build.mill @@ -0,0 +1,57 @@ +package build +import mill._, pythonlib._ + +object `package` extends RootModule with PythonModule { + + def pythonDeps = Seq("argparse==1.4.0", "jinja2==3.1.4") + + def mainScript = Task.Source { millSourcePath / "src" / "foo.py" } + + def generatedSources: T[Seq[PathRef]] = Task { + val destPath = Task.dest / "generatedSources" + os.makeDir.all(destPath) + + val prettyPythonDeps = pythonDeps().map { dep => + val parts = dep.split("==") + s"""("${parts(0)}", "${parts(1)}")""" + }.mkString(", ") + + os.write( + destPath / s"myDeps.py", + s""" + |class MyDeps: + | value = [${prettyPythonDeps}] + """.stripMargin + ) + + Seq(PathRef(destPath)) + } + + def lineCount: T[Int] = Task { + sources() + .flatMap(pathRef => os.walk(pathRef.path)) + .filter(_.ext == "py") + .map(os.read.lines(_).size) + .sum + } + + def forkEnv: T[Map[String, String]] = Map("MY_LINE_COUNT" -> s"${lineCount()}") + + def printLineCount() = Task.Command { println(lineCount()) } + +} + +/** Usage + +> ./mill run --text hello +text: hello +MyDeps.value: [('argparse', '1.4.0'), ('jinja2', '3.1.4')] +My_Line_Count: 22 + +> ./mill show lineCount +22 + +> ./mill printLineCount +22 + +*/ diff --git a/example/pythonlib/module/2-custom-tasks/src/foo.py b/example/pythonlib/module/2-custom-tasks/src/foo.py new file mode 100644 index 00000000000..63fdf0bd0e8 --- /dev/null +++ b/example/pythonlib/module/2-custom-tasks/src/foo.py @@ -0,0 +1,22 @@ +import argparse +import os +from myDeps import MyDeps # type: ignore + + +class Foo: + def main(self, text: str) -> None: + print("text: ", text) + print("MyDeps.value: ", MyDeps.value) + print("My_Line_Count: ", os.environ.get("MY_LINE_COUNT")) + +if __name__ == '__main__': + # Create the argument parser + parser = argparse.ArgumentParser(description="Process text argument") + + # Add argument for text + parser.add_argument("--text", type=str, required=True, help="Text for printing") + + # Parse the arguments + args = parser.parse_args() + + Foo().main(args.text) diff --git a/example/pythonlib/module/3-override-tasks/build.mill b/example/pythonlib/module/3-override-tasks/build.mill new file mode 100644 index 00000000000..78cc87d0c06 --- /dev/null +++ b/example/pythonlib/module/3-override-tasks/build.mill @@ -0,0 +1,68 @@ +package build +import mill._, pythonlib._ + +object foo extends PythonModule { + def sources = Task.Sources { + val destPath = Task.dest / "src" + os.makeDir.all(destPath) + + os.write( + destPath / "foo.py", + s""" + |class Foo: + | def main(self) -> None: + | print("Hello World") + | + |if __name__ == '__main__': + | Foo().main() + """.stripMargin + ) + Seq(PathRef(destPath)) + } + + def mainScript = Task.Source { sources().head.path / "foo.py" } + + def typeCheck = Task { + println("Type Checking...") + super.typeCheck() + } + + def run(args: mill.define.Args) = Task.Command { + typeCheck() + println("Running..." + args.value.mkString(" ")) + super.run(args)() + } +} + +object foo2 extends PythonModule { + def generatedSources = Task { + val destPath = Task.dest / "src" + os.makeDir.all(destPath) + os.write(destPath / "foo.py", """...""") + Seq(PathRef(destPath)) + } + + def mainScript = Task.Source { generatedSources().head.path / "foo.py" } + +} + +object foo3 extends PythonModule { + def sources = Task { + val destPath = Task.dest / "src" + os.makeDir.all(destPath) + os.write(destPath / "foo.py", """...""") + super.sources() ++ Seq(PathRef(destPath)) + } + def mainScript = Task.Source { sources().head.path / "foo.py" } + +} + +/** Usage + +> ./mill foo.run +Type Checking... +Success: no issues found in 1 source file +Running... +Hello World + +*/ diff --git a/example/pythonlib/module/4-compilation-execution-flags/build.mill b/example/pythonlib/module/4-compilation-execution-flags/build.mill new file mode 100644 index 00000000000..8e2186ae943 --- /dev/null +++ b/example/pythonlib/module/4-compilation-execution-flags/build.mill @@ -0,0 +1,15 @@ +package build +import mill._, pythonlib._ + +object `package` extends RootModule with PythonModule { + def mainScript = Task.Source { millSourcePath / "src" / "foo.py" } + def pythonOptions = Seq("-Wall", "-Xdev") + def forkEnv = Map("MY_ENV_VAR" -> "HELLO MILL!") +} + +/** Usage + +> ./mill run +HELLO MILL! + +*/ diff --git a/example/pythonlib/module/4-compilation-execution-flags/src/foo.py b/example/pythonlib/module/4-compilation-execution-flags/src/foo.py new file mode 100644 index 00000000000..e808407c58e --- /dev/null +++ b/example/pythonlib/module/4-compilation-execution-flags/src/foo.py @@ -0,0 +1,8 @@ +import os + +class Foo: + def main(self) -> None: + print(os.environ.get("MY_ENV_VAR")) + +if __name__ == '__main__': + Foo().main() \ No newline at end of file diff --git a/example/pythonlib/module/5-resources/build.mill b/example/pythonlib/module/5-resources/build.mill new file mode 100644 index 00000000000..2f6cf4b3f49 --- /dev/null +++ b/example/pythonlib/module/5-resources/build.mill @@ -0,0 +1,29 @@ +package build +import mill._, pythonlib._ + +object foo extends PythonModule { + + def mainScript = Task.Source { millSourcePath / "src" / "foo.py" } + + object test extends PythonTests with TestModule.Unittest { + def otherFiles = Task.Source(millSourcePath / "other-files") + + def forkEnv: T[Map[String, String]] = + super.forkEnv() ++ Map("OTHER_FILES_DIR" -> otherFiles().path.toString) + } +} + +/** Usage + +> ./mill foo.run +Hello World Resource File + +> ./mill foo.test +... +test_all (test.TestScript.test_all) ... ok +...Ran 1 test... +... +OK +... + +*/ diff --git a/example/pythonlib/module/5-resources/foo/resources/file.txt b/example/pythonlib/module/5-resources/foo/resources/file.txt new file mode 100644 index 00000000000..c3e18f6e79b --- /dev/null +++ b/example/pythonlib/module/5-resources/foo/resources/file.txt @@ -0,0 +1 @@ +Hello World Resource File \ No newline at end of file diff --git a/example/pythonlib/module/5-resources/foo/src/foo.py b/example/pythonlib/module/5-resources/foo/src/foo.py new file mode 100644 index 00000000000..45679b04e9b --- /dev/null +++ b/example/pythonlib/module/5-resources/foo/src/foo.py @@ -0,0 +1,11 @@ +import importlib.resources + +class Foo: + def PythonPathResourceText(self, package, resourceName: str) -> None: + resource_content = ( + importlib.resources.files(package).joinpath(resourceName).read_text() + ) + return resource_content.strip() + +if __name__ == "__main__": + print(Foo().PythonPathResourceText("resources", "file.txt")) \ No newline at end of file diff --git a/example/pythonlib/module/5-resources/foo/test/other-files/other-file.txt b/example/pythonlib/module/5-resources/foo/test/other-files/other-file.txt new file mode 100644 index 00000000000..194f575e5c8 --- /dev/null +++ b/example/pythonlib/module/5-resources/foo/test/other-files/other-file.txt @@ -0,0 +1 @@ +Other Hello World File \ No newline at end of file diff --git a/example/pythonlib/module/5-resources/foo/test/resources/test-file-a.txt b/example/pythonlib/module/5-resources/foo/test/resources/test-file-a.txt new file mode 100644 index 00000000000..9e67fcfe4be --- /dev/null +++ b/example/pythonlib/module/5-resources/foo/test/resources/test-file-a.txt @@ -0,0 +1 @@ +Test Hello World Resource File A \ No newline at end of file diff --git a/example/pythonlib/module/5-resources/foo/test/resources/test-file-b.txt b/example/pythonlib/module/5-resources/foo/test/resources/test-file-b.txt new file mode 100644 index 00000000000..289e86183e8 --- /dev/null +++ b/example/pythonlib/module/5-resources/foo/test/resources/test-file-b.txt @@ -0,0 +1 @@ +Test Hello World Resource File B \ No newline at end of file diff --git a/example/pythonlib/module/5-resources/foo/test/src/test.py b/example/pythonlib/module/5-resources/foo/test/src/test.py new file mode 100644 index 00000000000..6ffe05ce4a6 --- /dev/null +++ b/example/pythonlib/module/5-resources/foo/test/src/test.py @@ -0,0 +1,36 @@ +import os +from pathlib import Path +import importlib.resources +import unittest +from foo import Foo # type: ignore + + +class TestScript(unittest.TestCase): + def test_all(self) -> None: + appClasspathResourceText = Foo().PythonPathResourceText("resources", "file.txt") + self.assertEqual(appClasspathResourceText, "Hello World Resource File") + + testClasspathResourceText = ( + importlib.resources.files("resources") + .joinpath("test-file-a.txt") + .read_text() + .strip() + ) + self.assertEqual(testClasspathResourceText, "Test Hello World Resource File A") + + testFileResourceFile = next( + Path(os.getenv("MILL_TEST_RESOURCE_DIR")).rglob("test-file-b.txt"), None + ) + with open(testFileResourceFile, "r", encoding="utf-8") as file: + testFileResourceText = file.readline() + self.assertEqual(testFileResourceText, "Test Hello World Resource File B") + + with open( + Path(os.getenv("OTHER_FILES_DIR"), "other-file.txt"), "r", encoding="utf-8" + ) as file: + otherFileText = file.readline() + self.assertEqual(otherFileText, "Other Hello World File") + + +if __name__ == "__main__": + unittest.main() diff --git a/pythonlib/src/mill/pythonlib/PythonModule.scala b/pythonlib/src/mill/pythonlib/PythonModule.scala index 219673a3254..1b2b4120395 100644 --- a/pythonlib/src/mill/pythonlib/PythonModule.scala +++ b/pythonlib/src/mill/pythonlib/PythonModule.scala @@ -75,6 +75,9 @@ trait PythonModule extends PipModule with TaskModule { outer => */ def unmanagedPythonPath: T[Seq[PathRef]] = Task { Seq.empty[PathRef] } + def generatedSources: T[Seq[PathRef]] = Task { Seq.empty[PathRef] } + + // TODO: we have to choose whether to include `millSourcePath` by default in PYTHONPATH or not. /** * The directories used to construct the PYTHONPATH for this module, used for * execution, excluding upstream modules. @@ -83,7 +86,9 @@ trait PythonModule extends PipModule with TaskModule { outer => * directories. */ def localPythonPath: T[Seq[PathRef]] = Task { - sources() ++ resources() ++ unmanagedPythonPath() + Seq( + PathRef(millSourcePath) + ) ++ sources() ++ resources() ++ generatedSources() ++ unmanagedPythonPath() } /** @@ -95,17 +100,22 @@ trait PythonModule extends PipModule with TaskModule { outer => localPythonPath() ++ upstream } + def forkEnv: T[Map[String, String]] = Task { Map.empty[String, String] } + + def pythonOptions: T[Seq[String]] = Task { Seq.empty[String] } + // TODO: right now, any task that calls this helper will have its own python // cache. This is slow. Look into sharing the cache between tasks. def runner: Task[PythonModule.Runner] = Task.Anon { new PythonModule.RunnerImpl( command0 = pythonExe().path.toString, + options = pythonOptions(), env0 = Map( "PYTHONPATH" -> transitivePythonPath().map(_.path).mkString(java.io.File.pathSeparator), "PYTHONPYCACHEPREFIX" -> (T.dest / "cache").toString, if (Task.log.colored) { "FORCE_COLOR" -> "1" } else { "NO_COLOR" -> "1" } - ), + ) ++ forkEnv(), workingDir0 = Task.workspace ) } @@ -194,6 +204,7 @@ object PythonModule { private class RunnerImpl( command0: String, + options: Seq[String], env0: Map[String, String], workingDir0: os.Path ) extends Runner { @@ -204,7 +215,7 @@ object PythonModule { workingDir: os.Path = null )(implicit ctx: Ctx): Unit = Jvm.runSubprocess( - commandArgs = Seq(Option(command).getOrElse(command0)) ++ args.value, + commandArgs = Seq(Option(command).getOrElse(command0)) ++ options ++ args.value, envArgs = Option(env).getOrElse(env0), workingDir = Option(workingDir).getOrElse(workingDir0) )