Fixed: #3928 [Module Examples] Add First Class Python Support

docs/modules/ROOT/nav.adoc

@@ -32,6 +32,7 @@
 ** xref:android/java.adoc[]
 ** xref:android/kotlin.adoc[]
 ** xref:pythonlib/intro.adoc[]
+*** xref:pythonlib/module-config.adoc[]
 *** xref:pythonlib/dependencies.adoc[]
 *** xref:pythonlib/publishing.adoc[]
 * xref:comparisons/why-mill.adoc[]

docs/modules/ROOT/pages/pythonlib/module-config.adoc

@@ -0,0 +1,36 @@
+= Python Module Configuration
+:page-aliases: Python_Module_Config.adoc
+
+include::partial$gtag-config.adoc[]
+
+:language: Python
+:language-small: python
+
+This page goes into more detail about the various configuration options
+for `PythonModule`.
+
+// TODO: How to include PythonModule Link ?
+// Many of the APIs covered here are listed in the Scaladoc:
+
+// * {mill-doc-url}/api/latest/mill/pythonlib/PythonModule.html[mill.pythonlib.PythonModule]
+
+
+== Common Configuration Overrides
+
+include::partial$example/pythonlib/module/1-common-config.adoc[]
+
+== Custom Tasks
+
+include::partial$example/pythonlib/module/2-custom-tasks.adoc[]
+
+== Overriding Tasks
+
+include::partial$example/pythonlib/module/3-override-tasks.adoc[]
+
+== Compilation & Execution Flags
+
+include::partial$example/pythonlib/module/4-compilation-execution-flags.adoc[]
+
+== PythonPath and Filesystem Resources
+
+include::partial$example/pythonlib/module/5-resources.adoc[]

example/package.mill

@@ -68,6 +68,7 @@ object `package` extends RootModule with Module {
     object basic extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "basic"))
     object dependencies extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "dependencies"))
     object publishing extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "publishing"))
+    object module extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "module"))
   }
 
   object cli extends Module{

example/pythonlib/module/1-common-config/build.mill

@@ -0,0 +1,99 @@
+// This example shows some of the common tasks you may want to override on a
+// `PythonModule`: specifying the `mainScript`, adding additional
+// sources/resources, generating sources, and setting typecheck/run options.
+// Also we have support for `forkenv` and  `pythonOptions` to allow user to
+// add variables for environment and options for python respectively.
+
+package build
+import mill._, pythonlib._
+
+object foo extends PythonModule {
+  // You can have arbitrary numbers of third-party libraries
+  def pythonDeps = Seq("MarkupSafe==3.0.2", "Jinja2==3.1.4")
+
+  // choose a main Script to run if there are multiple present
+  def mainScript = Task.Source { millSourcePath / "custom-src" / "foo2.py" }
+
+  // TODO: should we include only one(it is sufficient) or both tasks(for demo purpose)?
+  // Add (or replace) source folders for the module to use
+  def sources = Task.Sources {
+    super.sources() ++ Seq(PathRef(millSourcePath))
+  }
+
+  // Add (or replace) resource folders for the module to use
+  def resources = Task.Sources {
+    super.resources() ++ Seq(PathRef(millSourcePath))
+  }
+
+  // Generate sources at build time
+  def generatedSources: T[Seq[PathRef]] = Task {
+    val destPath = Task.dest / "generatedSources"
+    os.makeDir.all(destPath)
+    for (name <- Seq("A", "B", "C")) os.write(
+      destPath / s"foo$name.py",
+      s"""
+         |class Foo$name:
+         |    value = "hello $name"
+      """.stripMargin
+    )
+
+    Seq(PathRef(destPath))
+  }
+  // Pass additional environmental variables when `.run` is called.
+  def forkEnv: T[Map[String, String]] = Map("MY_CUSTOM_ENV" -> "my-env-value")
+
+  // Additional Python options e.g. to Turn On Warnings and ignore import and resource warnings
+  // we can use -Werror to treat warnings as errors
+  def pythonOptions: T[Seq[String]] =
+    Seq("-Wall", "-Wignore::ImportWarning", "-Wignore::ResourceWarning")
+
+}
+
+// Note the use of `millSourcePath`, `Task.dest`, and `PathRef` when preforming
+// various filesystem operations:
+//
+// 1. `millSourcePath`: Base path of the module. For the root module, it's the repo root.
+//    For inner modules, it's the module path (e.g., `foo/bar/qux` for `foo.bar.qux`). Can be overridden if needed.
+//
+// 2. `Task.dest`: Destination folder in the `out/` folder for task output.
+//    Prevents filesystem conflicts and serves as temporary storage or output for tasks.
+//
+// 3. `PathRef`: Represents the contents of a file or folder, not just its path,
+//    ensuring downstream tasks properly invalidate when contents change.
+//
+// Typical Usage is given below:
+//
+/** Usage
+
+> ./mill foo.run
+...
+Foo2.value: <h1>hello2</h1>
+Foo.value: <h1>hello</h1>
+FooA.value: hello A
+FooB.value: hello B
+FooC.value: hello C
+MyResource: My Resource Contents
+MyOtherResource: My Other Resource Contents
+MY_CUSTOM_ENV: my-env-value
+...
+
+> ./mill show foo.bundle
+".../out/foo/bundle.dest/bundle.pex"
+
+> out/foo/bundle.dest/bundle.pex
+...
+Foo2.value: <h1>hello2</h1>
+Foo.value: <h1>hello</h1>
+FooA.value: hello A
+FooB.value: hello B
+FooC.value: hello C
+MyResource: My Resource Contents
+MyOtherResource: My Other Resource Contents
+...
+
+> sed -i.bak 's/import os/import os, warnings; warnings.warn("This is a test warning!")/g' foo/custom-src/foo2.py
+
+> ./mill foo.run
+...UserWarning: This is a test warning!...
+
+*/

example/pythonlib/module/1-common-config/foo/custom-resources/MyOtherResources.txt

@@ -0,0 +1 @@
+My Other Resource Contents

example/pythonlib/module/1-common-config/foo/custom-src/foo2.py

@@ -0,0 +1,48 @@
+import os
+import importlib.resources
+from jinja2 import Template
+from foo import Foo  # type: ignore
+from fooA import FooA  # type: ignore
+from fooB import FooB  # type: ignore
+from fooC import FooC  # type: ignore
+
+
+class Foo2:
+
+    def value(self, text: str):
+        """Generates an HTML template with dynamic content."""
+        template = Template("<h1>{{ text }}</h1>")
+        return template.render(text=text)
+
+    def read_resource(self, package: str, resource_name: str) -> str:
+        """Reads the content of a resource file."""
+        try:
+            resource_content = (
+                importlib.resources.files(package).joinpath(resource_name).read_text()
+            )
+            return resource_content.strip()
+        except FileNotFoundError:
+            return f"Resource '{resource_name}' not found."
+
+    def main(self):
+        # Output for value()
+        print(f"Foo2.value: {self.value('hello2')}")
+        print(f"Foo.value: {Foo().value('hello')}")
+        print(f"FooA.value: {FooA.value}")
+        print(f"FooB.value: {FooB.value}")
+        print(f"FooC.value: {FooC.value}")
+
+        # Reading resources
+        print(f"MyResource: {self.read_resource('resources', 'MyResource.txt')}")
+        print(
+            f"MyOtherResource: {self.read_resource('custom-resources', 'MyOtherResources.txt')}"
+        )
+
+        # Accessing environment variable
+        my_custom_env = os.environ.get("MY_CUSTOM_ENV")
+        if my_custom_env:
+            print(f"MY_CUSTOM_ENV: {my_custom_env}")
+
+
+if __name__ == "__main__":
+    Foo2().main()

example/pythonlib/module/1-common-config/foo/resources/MyResource.txt

@@ -0,0 +1 @@
+My Resource Contents

example/pythonlib/module/1-common-config/foo/src/foo.py

@@ -0,0 +1,15 @@
+from jinja2 import Template
+
+
+class Foo:
+    def value(self, text: str):
+        """Generates an HTML template with dynamic content."""
+        template = Template("<h1>{{ text }}</h1>")
+        return template.render(text=text)
+
+    def main(self):
+        print(f"Foo.value: {self.value('hello')}")
+
+
+if __name__ == "__main__":
+    Foo().main()

example/pythonlib/module/2-custom-tasks/build.mill

@@ -0,0 +1,128 @@
+// This example shows how to define task that depend on other tasks:
+//
+// 1. For `generatedSources`, we override the task and make it depend
+//    directly on `pythonDeps` to generate its source files. In this example,
+//    to include the list of dependencies as tuples in the `value` variable.
+//
+// 2. For `lineCount`, we define a brand new task that depends on `sources`.
+//    That lets us access the line count at runtime using `MY_LINE_COUNT`
+//    env variable defined in `forkEnv` and print it when the program runs
+//
+package build
+import mill._, pythonlib._
+
+object foo extends PythonModule {
+
+  def pythonDeps = Seq("argparse==1.4.0", "jinja2==3.1.4")
+
+  def mainScript = Task.Source { millSourcePath / "src" / "foo.py" }
+
+  def generatedSources: T[Seq[PathRef]] = Task {
+    val destPath = Task.dest / "generatedSources"
+    os.makeDir.all(destPath)
+
+    val prettyPythonDeps = pythonDeps().map { dep =>
+      val parts = dep.split("==")
+      s"""("${parts(0)}", "${parts(1)}")"""
+    }.mkString(", ")
+
+    os.write(
+      destPath / s"myDeps.py",
+      s"""
+         |class MyDeps:
+         |    value = [${prettyPythonDeps}]
+			""".stripMargin
+    )
+
+    Seq(PathRef(destPath))
+  }
+
+  def lineCount: T[Int] = Task {
+    sources()
+      .flatMap(pathRef => os.walk(pathRef.path))
+      .filter(_.ext == "py")
+      .map(os.read.lines(_).size)
+      .sum
+  }
+
+  def forkEnv: T[Map[String, String]] = Map("MY_LINE_COUNT" -> s"${lineCount()}")
+
+  def printLineCount() = Task.Command { println(lineCount()) }
+
+}
+
+// The above build defines the customizations to the Mill task graph shown below,
+// with the boxes representing tasks defined or overriden above and the un-boxed
+// labels representing existing Mill tasks:
+//
+// ```graphviz
+// digraph G {
+//   rankdir=LR
+//   node [shape=box width=0 height=0]
+//   pythonDeps -> generatedSources -> compile -> "..."
+//
+//   sources -> lineCount -> forkEnv -> "..." -> run
+//   lineCount -> printLineCount
+//
+//   sources [color=white]
+//   run [color=white]
+//
+//   compile [color=white]
+//   "..." [color=white]
+// }
+// ```
+//
+// Mill lets you define new cached Tasks using the `Task {...}` syntax,
+// depending on existing Tasks e.g. `foo.sources` via the `foo.sources()`
+// syntax to extract their current value, as shown in `lineCount` above. The
+// return-type of a Task has to be JSON-serializable (using
+// https://github.com/lihaoyi/upickle[uPickle], one of Mill's xref:fundamentals/bundled-libraries.adoc[Bundled Libraries])
+// and the Task is cached when first run until its inputs change (in this case, if
+// someone edits the `foo.sources` files which live in `foo/src`). Cached Tasks
+// cannot take parameters.
+//
+// Note that depending on a task requires use of parentheses after the task
+// name, e.g. `pythonDeps()`, `sources()` and `lineCount()`. This converts the
+// task of type `T[V]` into a value of type `V` you can make use in your task
+// implementation.
+//
+// This example can be run as follows:
+//
+/** Usage
+
+> ./mill foo.run --text hello
+text:  hello
+MyDeps.value:  [('argparse', '1.4.0'), ('jinja2', '3.1.4')]
+My_Line_Count:  22
+
+> ./mill show foo.lineCount
+22
+
+> ./mill foo.printLineCount
+22
+
+*/
+
+// Custom tasks can contain arbitrary code. Whether you want to download files
+// using `requests.get`, generate sources to feed into a Python Interpreter, or
+// create some custom pex bundle with the files you want , all of these
+// can simply be custom tasks with your code running in the `Task {...}` block.
+//
+// You can create arbitrarily long chains of dependent tasks, and Mill will
+// handle the re-evaluation and caching of the tasks' output for you.
+// Mill also provides you a `Task.dest` folder for you to use as scratch space or
+// to store files you want to return:
+//
+// * Any files a task creates should live
+//   within `Task.dest`
+//
+// * Any files a task modifies should be copied into
+//   `Task.dest` before being modified.
+//
+// * Any files that a task returns should be returned as a `PathRef` to a path
+//   within `Task.dest`
+//
+// That ensures that the files belonging to a particular task all live in one place,
+// avoiding file-name conflicts, preventing race conditions when tasks evaluate
+// in parallel, and letting Mill automatically invalidate the files when
+// the task's inputs change.

example/pythonlib/module/2-custom-tasks/foo/src/foo.py

@@ -0,0 +1,22 @@
+import argparse
+import os
+from myDeps import MyDeps # type: ignore
+
+
+class Foo:
+    def main(self, text: str) -> None:
+        print("text: ", text)
+        print("MyDeps.value: ", MyDeps.value)
+        print("My_Line_Count: ", os.environ.get("MY_LINE_COUNT"))
+
+if __name__  == '__main__':
+    # Create the argument parser
+    parser = argparse.ArgumentParser(description="Process text argument")
+
+    # Add argument for text
+    parser.add_argument("--text", type=str, required=True, help="Text for printing")
+
+    # Parse the arguments
+    args = parser.parse_args()
+
+    Foo().main(args.text)