From dde43403f810553b23b35e001b68d06eaf6a4826 Mon Sep 17 00:00:00 2001 From: Erkan Ozgur Yilmaz Date: Tue, 5 Nov 2024 09:46:52 +0000 Subject: [PATCH 1/2] * [#90] Updated the documentation theme to `furo`. * [#90] Updated the whole documentation with contemporary Stalker concepts and information. --- CHANGELOG.rst | 23 +- Makefile | 14 +- README.rst | 233 +---- docs/source/conf.py | 20 +- docs/source/contents.rst | 3 +- docs/source/design.rst | 339 ++----- docs/source/index.rst | 2 - docs/source/inheritance_diagram.rst | 1 - docs/source/summary.rst | 1 - docs/source/task_review_workflow.rst | 154 ++- docs/source/todo.rst | 2 +- docs/source/tutorial.rst | 882 +----------------- docs/source/tutorial/asset_management.rst | 259 +++++ docs/source/tutorial/basics.rst | 190 ++++ docs/source/tutorial/collaboration.rst | 30 + docs/source/tutorial/conclusion.rst | 39 + docs/source/tutorial/creating_simple_data.rst | 176 ++++ docs/source/tutorial/extending_som.rst | 6 + docs/source/tutorial/pipeline.rst | 62 ++ .../tutorial/query_update_delete_data.rst | 75 ++ docs/source/tutorial/scheduling.rst | 114 +++ .../tutorial/task_and_resource_management.rst | 53 ++ .../{ => tutorial}/tutorial_files/tutorial.py | 0 requirements-dev.txt | 4 + src/stalker/db/setup.py | 2 +- src/stalker/models/version.py | 2 +- 26 files changed, 1278 insertions(+), 1408 deletions(-) create mode 100644 docs/source/tutorial/asset_management.rst create mode 100644 docs/source/tutorial/basics.rst create mode 100644 docs/source/tutorial/collaboration.rst create mode 100644 docs/source/tutorial/conclusion.rst create mode 100644 docs/source/tutorial/creating_simple_data.rst create mode 100644 docs/source/tutorial/extending_som.rst create mode 100644 docs/source/tutorial/pipeline.rst create mode 100644 docs/source/tutorial/query_update_delete_data.rst create mode 100644 docs/source/tutorial/scheduling.rst create mode 100644 docs/source/tutorial/task_and_resource_management.rst rename docs/source/{ => tutorial}/tutorial_files/tutorial.py (100%) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 07624502..c2bea37b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -476,9 +476,9 @@ testing the library. * **Fix:** Fixed ``Task.path`` to always return a path with forward slashes. * **New:** Introducing ``EntityGroups`` that lets one to group a bunch of - ``SimpleEntity``s together, it can be used in grouping tasks even if they are - in different places on the project task hierarchy or even in different - projects. + ``SimpleEntity`` instances together, it can be used in grouping tasks even if + they are in different places on the project task hierarchy or even in + different projects. * **Update:** ``Task.percent_complete`` is now correctly calculated for a ``Duration`` based task by using the ``Task.start`` and ``Task.end`` @@ -530,8 +530,8 @@ testing the library. variables for each repository in the database. * **New:** Added a new ``after_insert`` which listens ``Repository`` instance - ``insert``s to automatically add environment variables for the newly inserted - repositories. + ``insert`` instances to automatically add environment variables for the newly + inserted repositories. * **Update:** ``Repository.make_relative()`` now handles paths with environment variables. @@ -550,7 +550,8 @@ testing the library. ``linux_path`` on Linux or setting the ``windows_path`` on Windows or setting the ``osx_path`` on OSX will update the environment variable. -* **New:** Added ``Task.good`` attribute to easily connect tasks to ``Good``s. +* **New:** Added ``Task.good`` attribute to easily connect tasks to ``Good`` + instances. * **New:** Added new methods to ``Repository`` to help managing paths: @@ -562,10 +563,12 @@ testing the library. * ``Repository.env_var`` a new property that returns the related environment variable name of a repo instance. This is an instance property:: - # with default settings - repo = Repository(...) - repo.env_var # should print something like "REPO131" which will be used - # in paths as "$REPO131" + .. code=block:: python + + # with default settings + repo = Repository(...) + repo.env_var # should print something like "REPO131" which will be used + # in paths as "$REPO131" * **Fix:** Fixed ``User.company_role`` attribute which is a relationship to the ``ClienUser`` to cascade ``all, delete-orphan`` to prevent diff --git a/Makefile b/Makefile index 7554cab2..ef5489f1 100644 --- a/Makefile +++ b/Makefile @@ -38,9 +38,10 @@ install: clean: FORCE @printf "\n\033[36m--- $@: Clean ---\033[0m\n" -rm -rf .pytest_cache - -rm -rf $(VIRTUALENV_DIR) -rm -rf dist -rm -rf build + -rm -rf docs/build + -rm -rf docs/source/generated/* clean-all: clean @printf "\n\033[36m--- $@: Clean All---\033[0m\n" @@ -82,5 +83,16 @@ tests: echo -e "\n\033[36m--- $@: Using python interpretter '`which python`' ---\033[0m\n"; \ PYTHONPATH=src pytest -v -n auto -W ignore --color=yes --cov=src --cov-report term --cov-fail-under 99 tests; + +# sphinx-build \ +# {tty:--color} \ +# -b html \ +# doc/sphinx/source doc/sphinx/build \ +# {posargs} +.PHONY: docs +docs: + cd docs && $(MAKE) html + + # https://www.gnu.org/software/make/manual/html_node/Force-Targets.html FORCE: diff --git a/README.rst b/README.rst index 3436e9d0..520ebdc5 100644 --- a/README.rst +++ b/README.rst @@ -20,49 +20,47 @@ :target: https://pypi.python.org/pypi/stalker :alt: Wheel Support - - ===== About ===== Stalker is an Open Source Production Asset Management (ProdAM) Library designed -specifically for Animation and VFX Studios but can be used for any kind of -projects. Stalker is licensed under LGPL v3. +specifically for Animation and VFX Studios. But it can be used for any kind of +projects from any other industry. Stalker is licensed under LGPL v3. Features ======== Stalker has the following features: - * Designed for **Animation and VFX Studios**. - * Platform independent. - * Default installation handles nearly all the asset and project management - needs of an animation and vfx studio. - * Customizable with configuration scripts. + * Designed for **Animation and VFX Studios** (but not limited to). + * OS independent, can work simultaneously with **Windows**, **macOS** and + **Linux**. + * Supplies excellent **Project Management** capabilities, i.e. scheduling and + tracking tasks, milestones and deadlines (via **TaskJuggler**). + * Powerful **Asset management** capabilities, allows tracking of asset + references in shots, scenes, sequences and projects. * Customizable object model (**Stalker Object Model - SOM**). * Uses **TaskJuggler** as the project planing and tracking backend. * Mainly developed for **PostgreSQL** in mind but **SQLite3** is also supported. * Can be connected to all the major 3D animation packages like **Maya, - Houdini, Nuke, Fusion, Softimage, Blender** etc. and any application that - has a Python API. And with applications like **Adobe Photoshop** which does - not have a direct Python API but supports ``win32com`` or ``comtypes``. - * Mainly developed for **Python 3.0+** and **Python 2.7** is fully supported. - * Developed with **TDD** practices. + Houdini, Nuke, Fusion, DaVinci Resolve, Blender** etc. and any application + that has a Python API, and for **Adobe Suite** applications like + **Adobe Photoshop** through ``win32com`` or ``comtypes`` libraries. + * Developed with religious **TDD** practices. -Stalker is build over these other OpenSource projects: +Stalker is mainly build over the following OpenSource libraries: * Python * SQLAlchemy and Alembic * Jinja2 * TaskJuggler -Stalker as a library has no graphical UI, it is a python library that gives you -the ability to build your pipeline on top of it. There are other python -packages like the Open Source Pyramid Web Application `Stalker Pyramid`_ and -the Open Source pipeline library `Anima`_ which has PyQt/PySide/PySide2 UIs for -applications like Maya, Nuke, Houdini, Fusion, Photoshop etc. +As Stalker is a Python library and doesn't supply any graphical UI you can use +other tools like `Stalker Pyramid`_ which is a Pyramid Web Application and +`Anima`_ which has PyQt/PySide UIs for applications like Houdini, Maya, +Blender, Nuke, Fusion, DaVinci Resolve, Photoshop and many more. .. _`Stalker Pyramid`: https://github.com/eoyilmaz/stalker_pyramid .. _`Anima`: https://github.com/eoyilmaz/anima @@ -70,199 +68,52 @@ applications like Maya, Nuke, Houdini, Fusion, Photoshop etc. Installation ============ -Use:: +Simply use: - pip install stalker +.. code-block:: shell + pip install stalker Examples ======== Let's play with **Stalker**. -Initialize the database and fill with some default data: - -.. code:: python - - from stalker import db - db.setup() - db.init() - -Create a ``User``: - -.. code:: python - - from stalker.db.session import DBSession - from stalker import User - me = User( - name='Erkan Ozgur Yilmaz', - login='erkanozgur', - email='my_email@gmail.com', - password='secretpass' - ) - - # Save the user to database - DBSession.save(me) - -Create a ``Repository`` for project files to be saved under: - -.. code:: python - - from stalker import Repository - repo = Repository( - name='Commercial Projects Repository', - windows_path='Z:/Projects', - linux_path='/mnt/Z/Projects', - macos_path='/Volumes/Z/Projects' - ) - -Create a ``FilenameTemplate`` (to be used as file naming convention): - -.. code:: python - - from stalker import FilenameTemplate - - task_template = FilenameTemplate( - name='Standard Task Filename Template', - target_entity_type='Task', # This is for files saved for Tasks - path='{{project.repository.path}}/{{project.code}}/' - '{%- for parent_task in parent_tasks -%}' - '{{parent_task.nice_name}}/' - '{%- endfor -%}', # This is Jinja2 template code - filename='{{version.nice_name}}_v{{"%03d"|format(version.version_number)}}' - ) - -Create a ``Structure`` that uses this template: - -.. code:: python - - from stalker import Structure - standard_folder_structure = Structure( - name='Standard Project Folder Structure', - templates=[task_template], - custom_template='{{project.code}}/References' # If you need extra folders - ) - -Now create a ``Project`` that uses this structure and will be placed under the -repository: - -.. code:: python +Because Stalker uses SQLAlchemy, it is very easy to retrieve complex data. +Let's say that you want to query all the Shot Lighting tasks where a specific +asset is referenced: - from stalker import Project - new_project = Project( - name='Test Project', - code='TP', - structure=standard_folder_structure, - repositories=[repo], # if you have more than one repository you can do it - ) - -Define the project resolution: - -.. code:: python - - from stalker import ImageFormat - hd1080 = ImageFormat( - name='1080p', - width=1920, - height=1080 - ) - -Set the project resolution: - -.. code:: python - - new_project.image_format = hd1080 - - # Save the project and all the other data it is connected to it - DBSession.save(new_project) - -Create Assets, Shots and other Tasks: - -.. code:: python - - from stalker import Task, Asset, Shot, Type - - # define Character asset type - char_type = Type(name='Character', code='CHAR', target_entity_type='Asset') - - character1 = Asset( - name='Character 1', - code='CHAR1', - type=char_type, - project=new_project - ) - - # Save the Asset - DBSession.save(character1) - - model = Task( - name='Model', - parent=character1 - ) - - rigging = Task( - name='Rig', - parent=character1, - depends_on=[model], # For project management, define that Rig can not start - # before Model ends. - ) - - # Save the new tasks - DBSession.save([model, rigging]) - - # A shot and some tasks for it - shot = Shot( - name='SH001', - code='SH001', - project=new_project - ) - - # Save the Shot - DBSession.save(shot) +.. code-block:: python - animation = Task( - name='Animation', - parent=shot, - ) + from stalker import Asset, Shot, Version - lighting = Task( - name='Lighting', - parent=shot, - depends_on=[animation], # Lighting can not start before Animation ends, - schedule_timing=1, - schedule_unit='d', # The task expected to take 1 day to complete - resources=[me] - ) - DBSession.save([animation, lighting]) + my_asset = Asset.query.filter_by(name="My Asset").first() + refs = Version.query.filter_by(name="Lighting").filter(Version.inputs.contains(my_asset)).all() -Let's create versions for the Animation task. +Let's say you want to get all the tasks assigned to you in a specific Project: .. code-block:: python - from stalker import Version + from stalker import Project, Task, User - new_version = Version(task=animation) - new_version.update_paths() # to render the naming convention template - new_version.extension = '.ma' # let's say that we have created under Maya + me = User.query.filter_by(name="Erkan Ozgur Yilmaz").first() + my_project = Project.query.filter_by(name="My Project").first() + query = Task.query.filter_by(project=my_project).filter(Task.resources.contains(me)) + my_tasks = query.all() -Let's check how the version path is rendered: +You can further query let's say your WIP tasks by adding more criteria to the ``query`` +object: .. code-block:: python - assert new_version.absolute_full_path == \ - "Z:/Projects/TP/SH001/Animation/SH001_Animation_Main_v001.ma" - assert new_version.version_number == 1 - -Create a new version and check that the version number increased automatically: - -.. code-block:: python + from stalker import Status - new_version2 = Version(task=animation) - new_version2.update_paths() # to render the naming convention template - new_version2.extension = '.ma' # let's say that we have created under Maya + wip = Status.query.filter_by(code="WIP").first() + query = query.filter_by(status=wip) + my_wip_tasks = query.all() - assert new_version2.version_number == 2 +and that's the way to get complex data in Stalker. -See more detailed example in `API Tutorial`_. +See more detailed examples in `API Tutorial`_. .. _API Tutorial: https://pythonhosted.org/stalker/tutorial.html diff --git a/docs/source/conf.py b/docs/source/conf.py index 98f40146..241ea411 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -51,8 +51,8 @@ # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # -# source_suffix = ['.rst', '.md'] -source_suffix = ".rst" +source_suffix = ['.rst', '.md'] +# source_suffix = ".rst" # The encoding of source files. # @@ -87,7 +87,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: @@ -143,7 +143,7 @@ # html_theme = 'scrolls' # html_theme = 'agogo' # html_theme = 'sphinxdoc' -html_theme = "pyramid" +html_theme = "furo" # html_theme = 'alabaster' # Theme options are theme-specific and customize the look and feel of a theme @@ -444,7 +444,9 @@ # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {"https://docs.python.org/": None} +intersphinx_mapping = { + "python": ("https://docs.python.org/", None), +} autosummary_generate = True autodoc_member_order = "bysource" @@ -458,8 +460,8 @@ def setup(app): indextemplate="pair: %s; configuration value", ) - # this next two lines are for Sphinx 1.2 to work - import sqlalchemy.ext.declarative.api - from stalker.db.declarative import Base + # # this next two lines are for Sphinx 1.2 to work + # import sqlalchemy.ext.declarative.api + # from stalker.db.declarative import Base - sqlalchemy.ext.declarative.api.Base = Base + # sqlalchemy.ext.declarative.api.Base = Base diff --git a/docs/source/contents.rst b/docs/source/contents.rst index b5a3b46c..f7cda74d 100644 --- a/docs/source/contents.rst +++ b/docs/source/contents.rst @@ -5,7 +5,7 @@ Table of Contents .. toctree:: :maxdepth: 3 - + about.rst installation.rst tutorial.rst @@ -16,3 +16,4 @@ Table of Contents roadmap.rst changelog.rst todo.rst + summary.rst \ No newline at end of file diff --git a/docs/source/design.rst b/docs/source/design.rst index a72c4e2b..50e01400 100644 --- a/docs/source/design.rst +++ b/docs/source/design.rst @@ -4,85 +4,71 @@ Design ====== -The design of Stalker is mentioned in the following sections. +This document explores Stalker, an open-source Python library designed for +production asset management. Introduction ============ -Stalker is an Open Source Production Asset Management Library. Although it is -designed VFX and Animation studios in mind, its flexible Project Management -muscles will allow it to be used in a wide variety of fields. +While primarily designed for VFX and animation studios, Stalker's flexible +architecture makes it adaptable to various industries. -An Asset Management Systems' duty is to hold the data which are created by the -users of the system in an organised manner, and let them quickly reach and find -their files. A Production Asset Management Systems' duty is, in addition to the -asset management systems', also handle the production steps or tasks and -allow the users of the system to collaborate. If more information about this -subject is needed, there are great books about Digital Asset Management (DAM) -Systems. +An Asset Management (AM) System is responsible for organizing and storing data +created by users, ensuring easy accessibility. A Production Asset Management +(ProdAM) System extends the functionality of an AMS by managing production +steps, tasks, and enabling collaboration among team members. -The usage of an asset management system in an animation/vfx studio is a must -for the sake of the studio itself. Even the benefits of the system becomes -silly to be mentioned when compared to the lack of even a simple system to -organise stuff. +Implementing an ProdAM System in an animation or VFX studio is crucial for +maintaining order and efficiency. The benefits of a well-organized system far +outweigh the initial setup effort. -Every studio outside establishes and develops their own asset management -system. Stalker will try to be the framework that these proprietary asset -management systems will be build over. Thus reducing the work repeated on every -big projects' start. +Many studios develop their own custom ProdAM solutions. Stalker aims to provide +a solid foundation for these systems, reducing the need for redundant +development efforts. + +Stalker focuses on organizing assets and tasks within projects, +streamlining workflows. It goes beyond basic asset management by incorporating +production steps and collaboration tools. Concepts ======== -There are a couple of design concepts those needs to be clarified before any -further explanation of Stalker. - -Stalker on itself basically is the **Model** in an **MTV** system (where the -`Stalker Pyramid`_ is the *Template* and *View*). So it defines the data and -the interaction of the data with itself. +There are a few key design concepts to understand before diving deeper into +Stalker. -Because the idea behind Stalker was to build an open source library that any -studio using it can build their own pipeline on top of it, it is designed to -stay simple and solid at the same time. So the UI and other stuff is ripped off -from the original Stalker package and moved to another Pyramid web application -called `Stalker Pyramid`_. - -.. _`Stalker Pyramid`: https://pypi.python.org/pypi/stalker_pyramid +Essentially, Stalker serves as the **Model** component in an **MTV** +(Model-Template-View) architecture. `Stalker Pyramid`_ provides the +*Template* and *View* components, defining the presentation layer and user +interface. Stalker itself focuses on defining the data structures and their +interactions. Stalker Object Model (SOM) -------------------------- -Stalker has a very robust object model, which is called -**Stalker Object Model** or **SOM**. The idea behind SOM is to create a class -hierarchy which is both usable right out of the box and also expandable by the -studios' developers. SOM is actually a little bit more complex than a basic -possible model, it is designed in this way just to be able to create a simple -pipeline to be able to build the system over it. +Stalker's robust object model, the Stalker Object Model (SOM), provides a +flexible framework for building production pipelines. SOM is designed to be +both usable out-of-the-box and extensible to meet specific studio needs. -Lets look at how a simple studio works and try to create our asset management +Lets look at how a studio simply works and try to create our asset management concepts around it. -An animation/vfx studios duty is to complete a :class:`.Project`. A project, -generally is about to create a :class:`.Sequence` of :class:`.Shot`\ s which -are a series of images those at the end converts to a movie. So a sequence in -general contains Shots. And Shots can use :class:`.Asset`\ s. So basically to -complete a project the studio should complete the shots and assets needed by -those shots. +An animation of VFX studio's primary goal is to complete a :class:`.Project`. +This project involves creating a series of :class:`.Sequences`, each composed +of individual :class:`.Shot`\ s. These shots, in turn, often rely on reusable +:class:`.Asset`\ s. -Furthermore all the Projects, Sequences, Shots or Assets are divided in to -different :class:`.Task`\ s those need to be done sequentially or in parallel -to complete that project. +To break down the work into manageable chunks, Projects, Sequences, Shots, and +Assets are further divided into :class:`.Task`\ s. These tasks often represent +specific pipeline steps like modeling, look development, rigging, animation, +lighting, and so on. -A Task relates to a work, a work is a quantity of time spent or going to be -spend for that specific task. The time spent on the course of completion of a -Task can be recorded with :class:`.TimeLog`\ s. TimeLogs show the total time -spent by an artist for a certain Task. So it holds information about how much -**effort** has been spent to complete a Task. +These tasks can be assigned to specific :class:`User`\ s and require a certain +amount of **effort** to complete. This effort is tracked using +:class:`.TimeLog`\ s. -During the completion of the Task or at the end of the work a **User** creates -:class:`.Versions` for that particular Task. Versions are the different -incarnations or the progress of the resultant product, and it is connected to -files in the fileserver or in Stalkers term the :class:`.Repository`. +As work progresses on a task, :class:`.Version`\ s are created to represent +different iterations or revisions of the output. These versions are linked to +files stored in a :class:`.Repository`. All the names those shown in bold fonts are a class in SOM. and there are a series of other classes to accommodate the needs of a :class:`.Studio`. @@ -91,217 +77,76 @@ The inheritance diagram of the classes in the SOM is shown below: .. include:: inheritance_diagram.rst -Stalker is a configurable and expandable and most importantly it is an -open source system. All of these features allows the system to have a flexible -structure. +Stalker is a highly configurable and open-source system. This flexibility +allows for various customization options. -There are two levels of expansion, the first level is the simplest one, by just -adding different statuses, different types or these kind of things in -which Stalker's current design is ready to. This is explained in `How To -Customize Stalker`_. +There are two main approaches to extending Stalker: -The second level of expansion is achieved by expanding the SOM. Expanding the -SOM includes creating new classes and database tables, and updating the old -ones which are already coming with Stalker. These expansion schemes are -further explained in `How To Extend SOM`_. + 1. **Simple Customization:** This involves adding or modifying existing + entities like statuses, types, or other predefined elements. The current + Stalker design is well-suited for this level of customization. More + details can be found in the `How to Customize Stalker`_ section. + + 2. **Extending the SOM:** This involves creating new classes and database + tables, or modifying existing ones. This approach is more complex but + allows for significant customization of Stalker's core functionality. + Refer to the `How To Extend SOM`_ section for further guidance. Features -------- - 1. Developed purely in Python (2.6 and over) using TDD (Test Driven - Development) practices - - 2. SQLAlchemy for the database back-end and ORM +Stalker boasts a robust feature set designed to streamline your production +pipeline: - 3. Uses Jinja2 as the template system for the file and folder naming - convention, it is possible to use templates like: + 1. **Pure Python:** Built entirely on Python 3.8 and above (continuously + tested with Python 3.8, 3.9, 3.10, 3.11, 3.12, 3.13), utilizing rigorous + Test Driven Development (TDD) practices for exceptional code quality (test + coverage is 99.7%). - {repository.path}/{project.code}/Assets/{asset.type.name}/{asset.code}/ - {asset.name}_{asset.type.name}_v{version.version_number}.{version.extension} + 2. **SQLAlchemy Integration:** Leverages SQLAlchemy for its database backend + and Object-Relational Mapping (ORM) capabilities, ensuring efficient data + management. - 4. File and folders and file sequences can be uploaded to the server as - assets, and the server decides where to place the folder or file by using - the template system. + 3. **Jinja2 Templates:** Employs Jinja2 for flexible file and folder naming + conventions. For a structured naming scheme it is possible to define + templates like: - 5. The event system gives full control for every CRUDL (create/insert, read, - update, delete and list) by giving step like before insert, after insert - call-backs. - - 6. The messaging system allows the users collaborate efficiently. + {repository.path}/{project.code}/Assets/{asset.type.name}/{asset.code}/{asset.name}_{asset.type.name}_v{version.version_number:03d}.{version.extension} - 7. Has an embedded Ticket system. + 5. **Review Workflow:** Stalker incorporates a comprehensive task review + workflow and a robust task status management system to ensure efficient and + quality production. - 8. Uses TaskJuggler as the task management backend and supports basic Task - attributes. + 6. **Automated File Placement:** Upload files, folders, and even file + sequences as versions. Stalker utilizes the defined templates to + automatically determine their placement on the server, promoting + organization. - 9. Has a predefined workflow for task statuses called Task Status Workflow - which manages the statuses of a Task during the project completion. + 7. **Fine-Grained Event System:** Gain complete control over the CRUDL + (Create, Read, Update, Delete, List) lifecycle. Define custom callbacks to + execute before or after specific operations, enabling tailored behavior. + + 8. **Embedded Ticketing System:** Streamline issue tracking and project + discussions with a built-in ticketing system. + + 9. **TaskJuggler Integration:** Integrate with TaskJuggler for enhanced task + management capabilities, supporting basic task attributes. + + 9. **Predefined Task Statuses:** Manage task progress efficiently with a + pre-defined Task Status Workflow, providing a structured approach to + tracking task completion stages. For usage examples see :ref:`tutorial_toplevel`\ . How To Customize Stalker ======================== -This part explains the customization of Stalker. - +Upcoming! This part will explain the customization of Stalker. How To Extend SOM ================= -This part explains how to extend Stalker Object Model or SOM. - -Creating Data -============= - -There are some examples here, to create simple data. - -Creating a Project ------------------- - -To create a Project, we need: - -1. A Repository -2. A Structure object to define the file structure of the Project: -3. FilenameTemplates for Task, Asset, Shot, Sequence types, to define the - placement of the Versions created for them. -4. An ImageFormat to define the output size of the project. -5. A StatusList with enough Statuses that will define the desired Project - Statuses. Stalker doesn't have a Project Status Workflow, yet! so define - yours. -6. If desired we can also add a Type for the Project to distinguish commercials - from Feature Film projects. -7. We need to create a user as the lead for the project. - -Here is the code:: - - from stalker import (db, Repository, Structure, FilenameTemplate, StatusList, - Status, Task, User) - - # first setup the database connection (assuming that you have a config.py - # defined, so we do not need to supply a database address) - db.setup() - - # initialize the database just for the first time - db.init() # run this only for the first time, subsequent runs will not - # create any errors, but it is unnecessary - - # re-use Statuses NEW, WIP and CMPL from default statuses - status_new = Status.query.filter_by(code='NEW').first() - status_wip = Status.query.filter_by(code='WIP').first() - status_cmpl = Status.query.filter_by(code='CMPL').first() - - # and create a new one - status_on_air = Status(name='On Air', code='OA') - - # status list for project - project_status_list = StatusList( - name='Project Statuses', - target_entity_type='Project', - statuses=[ - status_new, - status_wip, - status_cmpl, - status_on_air - ], - ) - - image_format_hd = ImageFormat( - name="HD", - width=1920, - height=1080, - ) - - commercial_type = Type( - name='Commercial', - code='COMM', - target_entity_type='Project' - ) - - repo = Repository( - name='Commercials Repo', - linux_path='/mnt/T/Commercials/', - windows_path='T:/Commercials/', - macos_path='/Volumes/T/Commercials/' - ) - - commercial_structure = Structure( - name='Commercial Project Structure', - code='' - ) - - lead = User( - name='Erkan Ozgur Yilmaz', - login='eoyilmaz', - email='eoyilmaz@stalker.com', - password='secret' - ) - - # lets create the Project - proj1 = Project( - name='Test Project', - code='TP', - description="This is the first project", - lead=lead, - image_format=image_format_hd, - fps=25, - type=commercial_type, - structure=commercial_structure, - repository=repo, - status_list=project_status_list, - status=status_new - ) - - # just add the project to the database - from stalker.db.session import DBSession - DBSession.add(proj1) - - # and commit the data to database - DBSession.commit() - -It may seem too much for just creating a Project, but it is for the first time -only. For a second project, we can use the previous Repository, Structure, -Lead, StatusList etc. - -Create a Task -------------- - -Because we have a project now lets create a task for this project:: - - # connect to the database if you have not done yet - db.setup() - - # create a new user as the resource for the task - resource1 = User( - name='User1', - login='user1', - email='user@users.com', - password='secret' - ) - - # now create the task - task1 = Task( - name='Task1', - description="This is our first Task, and it is about, creating " - "something fancy", - resources=[resource1], - schedule_timing=1, - schedule_unit='d', - schedule_model='effort', - project=proj1 - ) - # we do not need to supply a StatusList for the Task, statuses for tasks are - # created by default when we called db.init() in previous example - - # add it to the database - DBSession.add(task1) - - # and commit - DBSession.commit() - -Now we have created a simple Task and assigned it to the resource1. Lets check -the status of the Task:: - - print(task1.status) - # this should print something like - # stating that our task is ready to start working on. +Upcoming! This part will explain how to extend Stalker Object Model or SOM. + +.. _`Stalker Pyramid`: https://pypi.python.org/pypi/stalker_pyramid diff --git a/docs/source/index.rst b/docs/source/index.rst index 8943f09d..91701f94 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -8,8 +8,6 @@ Stalker Documentation .. include:: contents.rst -.. include:: summary.rst - Indices and tables ------------------ diff --git a/docs/source/inheritance_diagram.rst b/docs/source/inheritance_diagram.rst index 4d87e4b5..5c4a9a99 100644 --- a/docs/source/inheritance_diagram.rst +++ b/docs/source/inheritance_diagram.rst @@ -6,7 +6,6 @@ Inheritance Diagram .. inheritance-diagram:: stalker.exceptions.CircularDependencyError stalker.exceptions.DependencyViolationError - stalker.exceptions.DBError stalker.exceptions.LoginError stalker.exceptions.OverBookedError stalker.exceptions.StatusError diff --git a/docs/source/summary.rst b/docs/source/summary.rst index 8da65272..5bd1e06a 100644 --- a/docs/source/summary.rst +++ b/docs/source/summary.rst @@ -11,7 +11,6 @@ Summary stalker.db.setup stalker.exceptions stalker.exceptions.CircularDependencyError - stalker.exceptions.DBError stalker.exceptions.LoginError stalker.exceptions.OverBookedError stalker.exceptions.StatusError diff --git a/docs/source/task_review_workflow.rst b/docs/source/task_review_workflow.rst index edf2093a..e329c4b3 100644 --- a/docs/source/task_review_workflow.rst +++ b/docs/source/task_review_workflow.rst @@ -7,63 +7,61 @@ Task Review Workflow Introduction ============ -All tasks created in Stalker has a purpose and has an aim. It is the duty of -the task resources to accomplish that task and it is the responsibles' duty to -check if the :class:`.Task` is accomplished correctly. With the ``Task Review -Workflow`` Stalker presents a way of reviewing a task. - -So lets start describing the workflow. +All tasks in Stalker have a specific purpose and goal. :class:`.Task` resources +are responsible for completing these tasks, while task responsible ensure their +correct execution. The Task Review Workflow provides a mechanism for reviewing +tasks withing Stalker. The Workflow ============ -When a resource of a task spent his/her time reserved for a task and thinks -that this task is complete or even he/she still has time but needs some -direction, the resource (or any resource of that particular task) can request a -review. - -When that happens, Stalker creates a :class:`.Review` instance and attaches it -to the task. - -A :class:`.Review` instance holds the status of the review (starting from NEW), -and if some revision is requested it will also hold the description of the -revision, the extra time that the reviewer has given for the revision etc. and -the desired states of all the tasks depending on the reviewed tasks. - -Lets think that a particular Task has only one responsible, and one resource. -Lets assume the resource has decided to request a review. When it is happened -Stalker creates a Review instance and assigns it to the responsible of that -Task. Then the responsible is responsible for reviewing the Task. It the -responsible thinks that the task is finished then he/she sets the status of the -Review to Approved (APP) (by calling :meth:`.Review.approve()`) or if he/she -thinks that still some work needs to be done then he/she sets the status of the -Review to Request Revision (RREV) (by calling :meth:`.Review.request_revision) -and gives some time for the requested work and decides if the tasks depending -to the reviewed task should continue working or should be set on hold (if the -reviewed task was initially a task with status complete). - -Lets think that there are multiple responsible for a particular task. Then -when the resource request a review, then Stalker will create a Review instance -for each of the responsible. And even if one of the responsible has requested a -revision then the task will not be considered as completed. And when more than -one responsible request a revision, then the total amount of timing for the -revisions will be added to the task and the resource will continue to work. - -Depending Tasks +A task resource can request a review from the task's responsible at any stage +of task completion, including for supervisory purposes. + +When a review request is made, Stalker creates a :class:`.Review` instance +associated with the :class:`.Task`. This :class:`.Review` instance tracks the +review's status (initially set to `NEW`), any requested revisions (including +description, additional time allowances, and desired task statuses). + +Single Responsible and Resource Scenario +======================================== + +Consider a task with a single responsible and a single resource. When the +resource requests a review, Stalker creates a :class:`.Review` instance +assigned to the responsible. The responsible then review the task and can: + +- **Approve**: If the task is complete, the responsible can approve the review + by calling the :meth:`.Review.approve()` method. + +- **Request Revision**: If additional work is required, the responsible can + request a revision by calling the :meth:`.Review.request_revision()` method. + This involves specifying the necessary revisions, additional time. + +Multiple Responsible Scenario +============================= + +If multiple responsible are assigned to a task, a Review instance is created +for each of them when a review is requested. The task is considered incomplete +until all responsible have approved the review. If multiple responsible request +revisions, the total revision time is added to the task, and the resource +continues working. + +Dependent Tasks =============== -If a revision request has been made to a completed (CMPL) task with other tasks -depending on it, there are a couple of different scenarios to follow. +When a revision is requested for a completed task with dependent tasks, different +scenarios arise: + +**Scenario A: Dependent Tasks Are All in Ready-To-Start (RTS) Status* + +If there are no dependent tasks or none have started (all in `RTS`), the +dependent tasks are set to `Waiting For Dependency (WFD)` to prevent work until +the original task is completed. -Scenario A: There are no dependent tasks to the revised task or none of the -dependent tasks have started yet (all in RTS status). Then according to the -reviewers will the tasks can be set to Dependency Has Revision (DREV) which -allows the resources to continue to work or set to Waiting-For-Dependency -(WFD) which prevents the resources to work on the task. +**Scenario B: Started or Completed Dependent Tasks** -Scenario B: There are dependent tasks and some of them has started or -completed. Again according to the reviewers will the statuses of the tasks will -follow the following table: +If there are dependent tasks and some have started or completed, +their status is updated based on the following table: +----------------+--------------+ | Initial Status | Final Status | @@ -87,10 +85,8 @@ follow the following table: | CMPL | DREV | +----------------+--------------+ -When the revised task approved again and set its status to CMPL, then the -dependent task statuses will be set to their normal statuses again. The -following table shows the statuses that the tasks will have depending on their -time_logs attribute after the depending task is set to CMPL: +Once the revised tasks is approved and set back to `CMPL`, dependent tasks are +restored to their original statuses based on their time logs: +-----------------+------+------+-----+----+------+ | | DREV | PREV | WFD | OH | STOP | @@ -108,10 +104,10 @@ stored to CMPL status because there were revisions to the depending task so there should be some work to be done to update this task, so it is restored as WIP. -The following workflow diagram shows the status workflow, and it is a good idea -to study this to become familiar with the task statuses used in Stalker. +The following workflow diagram illustrates the task status transitions, and it +is a good idea to familiarize yourself with the task statuses used in Stalker. -.. image:: ../../../docs/source/_static/images/Task_Status_Workflow.png +.. image:: _static/images/Task_Status_Workflow.png :width: 637 px :height: 381 px :align: center @@ -119,31 +115,21 @@ to study this to become familiar with the task statuses used in Stalker. Revision Counter ================ -Both :class:`.Task` instances and :class:`.Review` instances have an attribute -called ``review_number``. Each Review with the same review_number considered in -the same set of review. It is only possible to have multiple Review instances -with the same review_number value if their :attr:`.reviewer` attribute are -different. - -The :attr:`.Task.review_number` starts from 0 and this represents the base or -initial revision and it is increased by 1 when one of the resources request a -review (by calling :meth:`.Task.request_review()`). - -A newly created Review instance will have a review_number which is equal to -the value of the Task.review_number + 1 at the time it is created. But it -never will or should be 0 cause this represents the base or initial revision. - -So, a Task with review_number 0 has no review yet. A Task with review number is -set to 2 has two sets of reviews. - -The best way to create revisions is to use :meth:`.Task.request_review()`. This -will ensure that there are enough :class:`.Review` instances created for each -responsible and the review_number attribute of both ends are correctly set. -And the return value of that method should be a list of Review instances. - -Each of the responsible should use the supplied methods ( -:meth:`.Review.approve` or :meth:`.Review.request_revision`) of the Review -instances according to their reviews. So by using those actions, the -responsible users can both set the status to an appropriate value and if -they're requesting a revision they also can to set the extra timing info -they've given for the revision. +Both :class:`.Task` and :class:`.Review` instances have ``review_number`` +attribute. Reviews with the same ``review_number`` belong to the same review +set. Multiple :class:`.Review` instances with the same +:attr:`Review.review_number` can exist if they have different reviewers. + +- The :attr:`.Task.review_number` starts at 0 for the initial revision and + increments with each review requests. So a :class:`.Task` with + ``review_number`` is 0 has no reviews yet. + +- A newly created :class:`.Review` instance has a ``review_number`` one higher +than the :attr:`.Task.review_number` at the time of creation. + +To create revisions effectively, use the :meth:`.Task.request_review()` method. +This ensures correct :class:`.Review` instance creation per reviewer and +correct ``review_number`` assignment and will return the newly created +:class:`.Review` instances as a list. Each responsible should use the +:meth:`.Review.approve()` or :meth:`.Review.request_revision()` methods to set +the appropriate status and additional revision information. diff --git a/docs/source/todo.rst b/docs/source/todo.rst index 297c39c3..a9a16b60 100644 --- a/docs/source/todo.rst +++ b/docs/source/todo.rst @@ -1,3 +1,3 @@ .. _todo_toplevel: -.. include:: source/../../../TODO +.. include:: source/../../../TODO.rst diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index c2970abb..a0b15703 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -4,867 +4,33 @@ API Tutorial ============ -Introduction -============ - -Using Stalker along with Python is all about interacting with a database by -using the Stalker Object Model (SOM). Stalker uses the powerful `SQLAlchemy -ORM`_. - -.. _SQLAlchemy ORM: http://www.sqlalchemy.org/docs/orm/tutorial.html - -This tutorial section let you familiarise with the Stalker Python API and -Stalker Object Model (SOM). If you used SQLAlchemy before you will feel at -home and if you aren't you will see that it is fun dealing with databases with -SOM. - -Part I - Basics -=============== - -Lets say that we just installed Stalker (as you are right now) and want to use -Stalker in our first project. - -The first thing we are going to learn about is how to connect to the database -so we can enter information about our studio and the projects. - -We are going to use a helper script to connect to the default database. Use the -following command to connect to the database:: - - from stalker import db - db.setup({"sqlalchemy.url": "sqlite:///"}) - -This will create an in-memory SQLite3 database, which is useless other than -testing purposes. To be able to get more out of Stalker we should give a proper -database information. The most basic setup is to use a file based SQLite3 -database:: - - db.setup({"sqlalchemy.url": "sqlite:///C:/studio.db"}) # assumed Windows +.. _tutorial_contents: -or:: +Table of Contents +================= - db.setup({"sqlalchemy.url": "sqlite:////home/ozgur/studio.db"}) # under linux or macos - -.. :: - This command will do the following: - 1. setup the database connection, by creating an `engine`_ - 2. create the SQLite3 database file if doesn't exist - 3. create a `session`_ instance - 4. do the `mapping`_ +.. toctree:: + :maxdepth: 3 - .. _session: http://www.sqlalchemy.org/docs/orm/session.html - .. _engine: http://www.sqlalchemy.org/docs/core/engines.html - .. _mapping: http://www.sqlalchemy.org/docs/orm/mapper_config.html - -.. note:: - Although with Stalker v0.2.18 the SQLite3 support is dropped, Stalker can - still work with an SQLite3 database. But the suggested database backend is - PostgreSQL (preferably PostgreSQL 9.5). - -Then if this is the first time you are connecting to the database, then you -should initialize the database to create some default data:: - - db.init() - -This will create some very important default data required for Stalker to work -properly. Although it will not break anything to call ``db.init()`` multiple -times it is needed only once (so you don't need to call it again when you close -your python shell and open up a new and fresh one). - -Lets continue by creating a :class:`.Studio` for our self:: - - from stalker import Studio - my_studio = Studio( - name='My Great Studio' - ) - -For now don't care what a Studio is about. It will be explained later on this -tutorial. - -Lets continue by creating a **User** for ourselves in the database. The first -thing we need to do is to import the :class:`.User` class in to the current -namespace:: - - from stalker import User - -then create the :class:`.User` object:: - - me = User( - name="Erkan Ozgur Yilmaz", - login="eoyilmaz", - email="some_email_address@gmail.com", - password="secret", - description="This is me" - ) - -Now we have just created a user which represents us. - -Lets create a new :class:`.Department` to define your department:: - - from stalker import Department - tds_department = Department( - name="TDs", - description="This is the TDs department" - ) - -Now add your user to the department:: - - tds_department.users.append(me) - -or we can do it by using the User instance:: - - me.departments.append(tds_department) - -Even if you didn't do the latter, when you run:: - - print(me.departments) - # you should get something like - # [] - -We have successfully created a :class:`.User` and a :class:`.Department` and we -assigned the user as one of the member of the **TDs Department**. - -Because we didn't tell Stalker to commit the changes, no data has been saved to -the database yet. So lets send it the data to the database:: - - from stalker.db.session import DBSession - DBSession.add(my_studio) - DBSession.add(me) - DBSession.add(tds_department) - DBSession.commit() - -As you see we have used the ``DBSession`` object to send (commit) the data to -the database. These information are stored in the database right now. - -Lets try to get something back from the database by querying all the -departments, then getting the second one (the first department is always the -"admins" which is created by default) and getting its first members name:: - - all_departments = Department.query.all() - print(all_departments) - # This should print something like - # [, ] - # "admins" department is created by default - - admins = all_departments[0] - tds = all_departments[1] - - all_users = tds.users # Department.users is a synonym for Department.members - # they are essentially the same attribute - print(all_users[0]) - # this should print - # - -Part II/A - Creating Simple Data -================================ - -Lets say that we have this new commercial project coming and you want to start -using Stalker with it. So we need to create a :class:`.Project` object to hold -data about it. - -A project instance needs to have a suitable :class:`.StatusList` (see -:ref:`status_and_status_lists_toplevel`) and a :class:`.Repository` instance:: - - # we will reuse the Statuses created by default (in db.init()) - from stalker import Status - - status_new = Status.query.filter_by(code='NEW').first() - status_wip = Status.query.filter_by(code='WIP').first() - status_cmpl = Status.query.filter_by(code='CMPL').first() - -.. note:: - When the Stalker database is first initialized (with ``db.init()``) a set of - :class:`.Status`\ es for :class:`.Task`\ s, :class:`.Asset`\ s, - :class:`.Shot`\ s, :class:`.Sequence`\ s and :class:`.Ticket`\ s are created - along with a :class:`.StatusList` for each of the data types. Up to this - point in the tutorial we have used those Statuses (new, wip and cmpl) - that are created by default. - -For now we have just created generic statuses. These :class:`.Status` instances -can be used with any kind of **statusable** objects. The idea behind is to -define the statuses only once, and use them in mixtures suitable for different -type of objects. So you can define all the possible Statuses for your entities, -then you can create a list of them for specific type of objects. - -Lets create a :class:`.StatusList` suitable for :class:`.Project` instances:: - - # a status list which is suitable for Project instances - from stalker import StatusList, Project - - project_statuses = StatusList( - name="Project Status List", - statuses=[ - status_new, - status_wip, - status_cmpl - ], - target_entity_type='Project' # you can also use Project which is the - # class itself - ) - -So we defined a status list which is suitable for :class:`.Project` instances. -As you see we didn't used all the generic Statuses in our ``project_statuses`` -because for a Project object we thought that these statuses are enough. - -.. :: - We also need to specify the type of the project, which is *commercial* in our - case:: - - from stalker import Type - commercial_project_type = Type( - name="Commercial Project", - target_entity_type=Project - ) - - class:`~stalker.models.type.Type`\ s are generic entities that is accepted by - any kind of entity created in Stalker. So in Stalker you can define a type - for anything. But a couple of them, like the - :class:`~stalker.models.project.Project` class, needs the type to be defined - in the creation of the instance. - -And finally, the :class:`.Repository`. The Repository (or Repo if you like) is -a path in our file server, where we place files and which is visible to all the -workstations/render farmers:: - - from stalker import Repository - - # and the repository itself - commercial_repo = Repository( - name="Commercial Repository", - code="CR" - ) - -.. versionadded:: 0.2.24 - - Starting with Stalker version 0.2.24 :class:`.Repository` instances have - :attr:`stalker.models.repository.Repository.code` attribute to help - generating universal paths (both across OSes and different installations of - Stalker). - - -:class:`.Repository` class will be explained in detail in upcoming sections. - -So:: - - new_project = Project( - name="Fancy Commercial", - code='FC', - status_list=project_statuses, - repositories=[commercial_repo], - ) - -So we have created our project now. - -Lets enter more information about this new project:: - - import tzlocal - import datetime - from stalker import ImageFormat - - new_project.description = \ - """The commercial is about this fancy product. The - client want us to have a shiny look with their - product bla bla bla...""" - - new_project.image_format = ImageFormat( - name="HD 1080", - width=1920, - height=1080 - ) - - new_project.fps = 25 - local_tz = tzlocal.get_localzone() - new_project.end = datetime.datetime(2014, 5, 15, tzinfo=local_tz) - new_project.users.append(me) - -Lets save all the new data to the database:: - - DBSession.add(new_project) - DBSession.commit() - -As you see, even though we have created multiple objects (new_project, -statuses, status lists etc.) we've just added the ``new_project`` object to the -database, but don't worry all the related objects will be added to the -database. - -.. note:: - Starting with Stalker v0.2.18 all the datetime information needs to have - timezone information (we've used the local timezone in the example). - - -A Project generally is group of :class:`.Task`\ s that needs to be completed. A -:class:`.Task` in Stalker is a type of entity where we define the total amount -of effort need to be done (or the duration or the length of the task, see -:class:`.Task` class documentation) to consider that Task as completed. All of -the tasks (leaf tasks in fact, coming next) has ``resources`` which defines the -:class:`.User`\ s who need to work on that task and complete it. These are all -explained in :class:`.Task` class documentation. - -For now you just need to now that :class:`.Asset`\ s, :class:`.Shot`\ s and -:class:`.Sequence`\ s in Stalker are derived from :class:`.Task` and they are -in fact other type of Tasks or a specialized version of Tasks. - -So lets create a :class:`.Sequence`:: - - from stalker import Sequence - - seq1 = Sequence( - name="Sequence 1", - code="SEQ1", - project=new_project, - ) - -And a Sequence generally has :class:`.Shot`\ s:: - - from stalker import Shot - - sh001 = Shot( - name='SH001', - code='SH001', - project=new_project, - sequences=[seq1] - ) - sh002 = Shot( - code='SH002', - project=new_project, - sequences=[seq1] - ) - sh003 = Shot( - code='SH003', - project=new_project, - sequences=[seq1] - ) - -send them to the database:: - - DBsession.add_all([sh001, sh002, sh003]) - DBsession.commit() - -.. note:: - Even though, in this tutorial we have created :class:`.Shot`\ s with one - :class:`.Sequence` instance, it is not needed. You can create - :class:`.Shot`\ s without any :class:`.Sequence` instance needed. - - For small projects like commercials, you may skip creating a Sequence at - all. - - For bigger projects, like feature films, it is a very good idea to use - Sequences and then group the Shots under them. - - But again, a Shot can be connected to multiple sequences, which is useful if - your shot, let say, is a kind of flashback and you will use this shot again - without changing it at all, then this feature becomes handy. - -Part II/B - Querying, Updating and Deleting Data -================================================ - -So far we just created some simple data. What about updating them. Let say that -we created a new shot with wrong info:: - - sh004 = Shot( - code='SH004', - project=new_project, - sequences=[seq1] - ) - DBSession.add(sh004) - DBSession.commit() - -and you figured out that you have created and committed a wrong info and you -want to correct it:: - - sh004.code = "SH005" - DBsession.commit() - -later on lets say you wanted to get the shot back from database:: - - # first find the data - wrong_shot = Shot.query.filter_by(code="SH005").first() - - # now update it - wrong_shot.code = "SH004" - - # commit the changes to the database - DBsession.commit() - -and let say that you decided to delete the data:: - - DBsession.delete(wrong_shot) - DBsession.commit() - -If you don't close your python session, your variable are still going to -contain the data but they do not exist in the database anymore:: - - wrong_shot = Shot.query.filter_by(code="SH005").first() - print(wrong_shot) - # should print None - -for more info about update and delete options (like cascades) in SQLAlchemy -please see the `SQLAlchemy documentation`_. - -.. _SQLAlchemy documentation: http://www.sqlalchemy.org/docs/orm/session.html - -Part III - Pipeline -=================== - -Up until now, we skipped a lot of stuff here to take little steps every time. -Even tough we have created users, departments, projects, sequences and shots, -Stalker still doesn't know much about our studio. For example, it doesn't have -any information about the **pipeline** that we are following and what steps we -do to complete those shots, thus to complete the project. - -In Stalker, pipeline is managed by :class:`~stalker.models.task.Task`\ s. So -you create Tasks for Shots and then you can create dependencies between tasks. - -So lets create a couple of tasks for one of the shots we have created before:: - - from stalker import Task - - previs = Task( - name="Previs", - parent=sh001 - ) - - matchmove = Task( - name="Matchmove", - parent=sh001 - ) - - anim = Task( - name="Animation", - parent=sh001 - ) - - lighting = Task( - name="Lighting", - parent=sh001 - ) - - comp = Task( - name="comp", - parent=sh001 - ) - -Now create the dependencies between them:: - - comp.depends_on = [lighting] - lighting.depends_on = [anim] - anim.depends_on = [previs, matchmove] - -Stalker uses this dependency relation in scheduling these tasks. That is by -appending "lighting" task as one of the dependencies of comp, Stalker now know -that lighting should be completed to let the resource of the comp task start -working. The "Task Scheduling" will be explained in detail later on in this -tutorial. + tutorial/basics.rst + tutorial/creating_simple_data.rst + tutorial/query_update_delete_data.rst + tutorial/pipeline.rst + tutorial/task_and_resource_management.rst + tutorial/scheduling.rst + task_review_workflow.rst + tutorial/asset_management.rst + tutorial/collaboration.rst + tutorial/extending_som.rst + tutorial/conclusion.rst -Part IV - Task & Resource Management -==================================== - -Now we have a couple of Shots with couple of tasks inside it but we didn't -assign the tasks to anybody to let them complete this job. - -Lets assign all this stuff to our self (for now :) ):: - - previs.resources = [me] - previs.schedule_timing = 10 - previs.schedule_unit = 'd' - - matchmove.resources = [me] - matchmove.schedule_timing = 2 - matchmove.schedule_unit = 'd' - - anim.resources = [me] - anim.schedule_timing = 5 - anim.schedule_unit = 'd' - - lighting.resources = [me] - lighting.schedule_timing = 3 - lighting.schedule_unit = 'd' - - comp.resources = [me] - comp.schedule_timing = 6 - comp.schedule_unit = 'h' - -Now Stalker knows the hierarchy of the tasks and how much effort is needed to -complete this tasks. Stalker will use this information to solve the Scheduling -problem, and will tell you when to start and complete this tasks. - -Lets commit the changes again:: - - DBsession.commit() - -If you noticed, this time we didn't add anything to the session, cause we have -added the ``sh001`` in a previous commit, and because all the objects are -attached to this shot object in some way, all the changes has been tracked and -added to the database. - -Part V - Scheduling -=================== - -In previous sections of this tutorial we have created a :class:`.Shot` and then -created a couple of :class:`.Task`\ s to this shot and then assigned our self -as the resource of these tasks. - -Stalker knows enough about our little project now, but we don't know where to -start the project from. That is which task should we start from. - -In Stalker, defining the start and end dates of a Task (also of an Asset, Shot -and Sequence) is called "Scheduling". Stalker, with the help of `TaskJuggler`_, -can solve this problem and define when the resource should work on a specific -task. - -.. warning:: - You should have `TaskJuggler`_ installed in your system, and you should have - configured your Stalker installation to be able to find the ``tj3`` - executable. - - On a linux system this should be fairly straight forward, just install - `TaskJuggler`_ and stalker will be able to use it. - - But for other OSes, like macOS and Windows, you should create an environment - variable called ``STALKER_PATH`` and then place a file called ``config.py`` - inside the folder that this path is pointing at. And then add the following - to this ``config.py``:: - - tj_command = 'C:\\Path\\to\\tj3.exe' - - The default value for ``tj_command`` config variable is - ``/usr/local/bin/tj3``, so if on a Linux or macOS system when you run:: - - which tj3 - - is returning this value (``/usr/local/bin/tj3``) you don't need to setup - anything. - - .. _TaskJuggler: http://www.taskjuggler.org/ - -So, lets schedule our project by using the :class:`.Studio` instance that we -have created at the beginning of this tutorial:: - - from stalker import TaskJugglerScheduler - - my_studio.scheduler = TaskJugglerScheduler() - my_studio.duration = datetime.timedelta(days=365) # we are setting the - my_studio.schedule(scheduled_by=me) # duration to 1 year just - # to be sure that TJ3 - # will not complain - # about the project is not - # fitting in to the time - # frame. - DBsession.commit() # to reflect the change - -This should take a little while depending on your projects size (around 1-2 -seconds for this tutorial, but around ~15 min for a project with 15000+ tasks). - -When it is finished all of your tasks now have their ``computed_start`` and -``computed_end`` values filled with proper data. Now check the start and end -values:: - - print(previs.computed_start) # 2014-04-02 16:00:00 - print(previs.computed_end) # 2014-04-15 15:00:00 - - print(matchmove.computed_start) # 2014-04-15 15:00:00 - print(matchmove.computed_end) # 2014-04-17 13:00:00 - - print(anim.computed_start) # 2014-04-17 13:00:00 - print(anim.computed_end) # 2014-04-23 17:00:00 - - print(lighting.computed_start) # 2014-04-23 17:00:00 - print(lighting.computed_end) # 2014-04-24 11:00:00 - - print(comp.computed_start) # 2014-04-24 11:00:00 - print(comp.computed_end) # 2014-04-24 17:00:00 - -The dates are probably going to be different in your computer. But as you see -Stalker has computed the start and end date values for each of the tasks. They -are simply following one other, this is because we have entered only one -resource for each of the task. - -You should know that "Scheduling" is a huge concept and it is greatly explained -in `TaskJuggler`_ documentation. - -For a last thing you can check the ``to_tjp`` values of each data we have -created for now, so try running:: - - print(my_studio.to_tjp) - print(me.to_tjp) - print(comp.to_tjp) - print(new_project.to_tjp) - -If you are familiar with TaskJuggler, you should recognize the output of each -``to_tjp`` variable. So essentially Stalker is mapping all of its data to a -TaskJuggler compatible string. A very small part of TaskJuggler directives are -currently supported. But it is enough to schedule very complex projects with -complex dependency relation and Task hierarchies. And with every new version of -Stalker the supported TaskJuggler directives are expanded. - -Part VI - Asset Management -========================== - -Now we have created a lot of things but other then storing all the data in the -database, we didn't do much. Stalker still doesn't have information about a lot -of things. For example, it doesn't know how to handle your asset versions -(:class:`.Version`) namely it doesn't know how to store your data that you are -going to create while completing these tasks. - -So what we need to define is a place in our file structure. It doesn't need to -be a network shared directory but if you are not working alone than it means -that everyone needs to reach your data and the simplest way to do this is to -place your files in a network share, there are other alternatives like storing -your files locally and sharing your revisions with a Software Configuration -Management (SCM) system, Stalker doesn't support the latter right now. - -We are going to see the first alternative, which uses a network share in our -fileserver, and this network share is called a :class:`.Repository` in Stalker. - -A repository is a file path, preferably a path which is mapped or mounted to -the same path on every computer in your studio (also you can use ``autofs`` -with an OpenLDAP server in which you can synchronize all off the mount points -on all of your workstations and render slaves at once). - -In Stalker, you can have several repositories, let say one for Commercials and -another one for each big Movie projects. - -You can define repositories and assign projects to those repositories. - -We have already created a repository while creating our first project. But the -repository has missing information. A Repository object shows the path that we -create our projects into. Lets enter the paths for all the major operating -systems:: - - commercial_repo.linux_path = "/mnt/M/commercials" - commercial_repo.macos_path = "/Volumes/M/commercials" - commercial_repo.windows_path = "M:/commercials" # you can use reverse - # slashes (\\) if you want - -And if you ask for the path to a repository object it will always give the -correct answer according to your operating system:: - - print(commercial_repo.path) - # under Windows outputs: - # M:/commercials - # - # in Linux and variants: - # /mnt/M/commercials - # - # and in macOS: - # /Volumes/M/commercials - -.. note:: - Stalker always uses forward slashes no matter what operating system you are - using. It is like that even if you define your paths with reverse slashes - (\\). - -Assigning this repository to our project is not enough, Stalker still doesn't -know about the directory structure of this project. To explain the project -structure to Stalker we use a :class:`.Structure` instance:: - - from stalker import Structure - - commercial_project_structure = Structure( - name="Commercial Projects Structure" - ) - - # now assign this structure to our project - new_project.structure = commercial_project_structure - -.. versionadded:: 0.2.13 - - Starting with Stalker version 0.2.13 :class:`.Project` instances can have - **multiple** :class:`.Repository` instances attached. So you can create - complex templates where you can for example store published versions on a - different server/network share or you can setup so the outputs of a version - (like the rendered files) are stored on a different server, and etc. - - *The following examples are updated in a simple way and examples showing* - *the advantage of having multiple repositories will be added on later* - *versions.* - -Now we have created a very simple structure instance, but we still need to -create :class:`.FilenameTemplate` instances for Tasks which then will be used -by the :class:`.Version` instances to generate a consistent and meaningful path -and filename:: - - from stalker import FilenameTemplate - - task_template = FilenameTemplate( - name='Task Template for Commercials', - target_entity_type='Task', - path='$REPO{{project.repository.id}}/{{project.code}}/{%- for p in parent_tasks -%}{{p.nice_name}}/{%- endfor -%}', - filename='{{version.nice_name}}_v{{"%03d"|format(version.version_number)}}' - ) - - # and append it to our project structure - commercial_project_structure.templates.append(task_template) - - # commit to database - DBsession.commit() # no need to add anything, project is already on db - -By defining a :class:`.FilenameTemplate` instance we have essentially told -Stalker how to store :class:`.Version` instances created for :class:`.Task` -entities in our :class:`.Repository`. - -The data entered both to the ``path`` and ``filename`` arguments are `Jinja2`_ -directives. The :class:`.Version` class knows how to render these templates -while calculating its ``path`` and ``filename`` attributes. - -Also, if you noticed we have used an environment variable "$REPO" along with -the id of the first repository in the project "{{project.repository.id}}" -(attention! ``project.repository`` always shows the first repository in the -project), this is a new feature introduced with Stalker version 0.2.13. Stalker -creates environment variables on runtime for each of the repository whenever a -repository is created and inserted in to the DB or it will create environment -variables for already existing repositories upon a successful database -connection. - -Lets create a :class:`.Version` instance for one of our tasks:: - - from stalker import Version - - vers1 = Version( - task=comp - ) - - # we need to update the paths - vers1.update_paths() - - # check the path and filename - print(vers1.path) # '$REPO33/FC/SH001/comp' - print(vers1.filename) # 'SH001_comp_Main_v001' - print(vers1.full_path) # '$REPO33/FC/SH001/comp/SH001_comp_Main_v001' - - # now the absolute values, values with repository root - # because I'm running this code in a Linux laptop, my results are using the - # linux path of the repository - print(vers1.absolute_path) # '/mnt/M/commercials/FC/SH001/comp' - print(vers1.absolute_full_path) # '/mnt/M/commercials/FC/SH001/comp/SH001_comp_Main_v001' - - # check the version_number - print(vers1.version_number) # 1 - - # commit to database - DBsession.commit() - -As you see, the :class:`.Version` instance magically knows where to place -itself and what to use as the filename. Thanks to Stalker it is now easy to -create version files where you don't have weird file names (ex: -'Shot1_comp_Final', 'Shot1_comp_Final_revised', -'Shot1_comp_Final_revised_Final', 'Shot1_comp_Final_revised_Final_real_final' -and the list goes on, we all know those filenames don't we :) ). - -With Stalker the filename and path always follows strict rules. - -Also by using the :attr:`.Version.is_published` attribute you can define which -of the versions are usable and which are versions that you are still working -on:: - - vers1.is_published = False # I still work on this version, this is not a - # usable one - -Lets create another version for the same task and see what happens:: - - # be sure that you've committed the previous version to the database - # to let Stalker now what number to give for the next version - vers2 = Version(task=comp) - vers2.update_paths() # this call probably will disappear in next version of - # Stalker, so Stalker will automatically update the - # paths on Version.__init__() - - print(vers2.version_number) # 2 - print(vers2.filename) # 'SH001_comp_Main_v002' - - # before creating a new version commit this one to db - DBsession.commit() - - # now create a new version - vers3 = Version(task=comp) - vers3.update_paths() - - print(vers3.version_number) # 3 - print(vers3.filename) # 'SH001_comp_Main_v002' - -Isn't that nice, Stalker increments the version number automatically. - -Also you can query all the versions of a specific task by:: - - # using pure Python - vers_from_python = comp.versions # [, - # , - # ] - - # or using a query - vers_from_query = Version.query.filter_by(task=comp).all() - - # again returns - # [, - # , - # ] - - assert vers_from_python == vers_from_query - -.. _Jinja2: http://jinja.pocoo.org/ - -.. note:: - Stalker stores :attr:`.Version.path` and :attr:`.Version.filename` - attributes in the database, so the values does not contain any OS specific - path. It will only show the OS specific path on - :attr:`.Version.absolute_path` and on :attr:`.Version.absolute_full_path` - attributes by joining the :attr:`.Repository.path` with the path values from - database momentarily. - -You can also setup your project structure to have default directories:: - - commercial_project_structure.custom_template = """ - Temp - References - References/Movies - References/Images - """ - -When the above template is executed each line will refer to a directory. - -Part VII - Collaboration (not completed) -======================================== - -We came a lot from the start, but what is the use of an Production Asset -Management System if we can not communicate with our colleagues. - -In Stalker you can communicate with others in the system, by: - - * Leaving a :class:`.Note` to anything created in - Stalker (except you can not create a :class:`.Note` to another - :class:`.Note` and to a :class:`.Tag`). - * Sending a :class:`.Message` directly to them or to a group of users. (Not - implemented yet). - * Anyone can create a :class:`.Ticket` for a :class:`.Project`. - * You can create wiki :class:`.Page`\ s per :class:`.Project`. - -Part VIII - Extending SOM (coming) -================================== - -This part will be covered soon - -Conclusion -========== - -In this tutorial, you have nearly learned a quarter of what Stalker supplies as -a Python library. - -Stalker is a very flexible and powerful Production Asset Management system. As -of writing this tutorial it has been developed for the last 5 years (4 years -with the only developer being yours truly and for another 1 year where his wife -is also attended to the project) and it is currently been used in production of -a feature movie. - -But it is only a Python library so it doesn't supply any graphical user -interface. - -There are other projects, namely `Stalker Pyramid`_ and `Anima`_ that is using -Stalker in their back ends. `Stalker Pyramid`_ is an `Pyramid`_ based Web -application and `Anima`_ is a pipeline library. +Introduction +============ -You can clone their repositories to see how PyQt4 and PySide UIs are created -with Stalker (in Anima) and how it is used as the database model for a Web -application in `Stalker Pyramid`_. +Stalker leverages the powerful `SQLAlchemy ORM`_ to facilitate interaction with +databases using the Stalker Object Model (SOM). This tutorial introduces you to +the Stalker Python API and SOM. If you're familiar with SQLAlchemy, you'll find +the transition smooth. Otherwise, SOM offers a user-friendly way to manage +databases. -.. _Stalker Pyramid: https://www.github.com/eoyilmaz/stalker_pyramid -.. _Anima: https://github.com/eoyilmaz/anima -.. _Pyramid: http://www.pylonsproject.org/ +.. _SQLAlchemy ORM: http://www.sqlalchemy.org/docs/orm/tutorial.html diff --git a/docs/source/tutorial/asset_management.rst b/docs/source/tutorial/asset_management.rst new file mode 100644 index 00000000..a9a3cf2f --- /dev/null +++ b/docs/source/tutorial/asset_management.rst @@ -0,0 +1,259 @@ +.. _tutorial_asset_management_toplevel: + +Asset Management +================ + +Now that we've created projects, tasks and resources, it's time to manage the +files generated during production. + +File Storage and Repository setup +--------------------------------- + +Contrary to a Source Code Management (SCM) System where revisions to a file is +handled incrementally, Stalker handles file versions all together. Meaning +that, all the versions that are created for individual tasks +(:class:`.Version`) are individual files stored in a shared location accessible +to everyone in your studio. This location is called a :class:`.Repository` in +Stalker. + +Defining Repository Paths +------------------------- + +A repository can be a network share or a locally mounted directory. You can +define multiple repositories for different project types or needs. We've +already created a repository while creating our first project. But the +repository has missing information. Here's how to define paths for a commercial +project repository:: + +.. code-block:: python + + commercial_repo.linux_path = "/mnt/M/commercials" + commercial_repo.macos_path = "/Volumes/M/commercials" + commercial_repo.windows_path = "M:/commercials" + # Stalker automatically corrects backslashes (\) to forward slashes (/) + +And if you ask for the path to a repository object, it will always return the +correct path according to your operating system: + +.. code-block:: python + + print(commercial_repo.path) + +You'll get the appropriate path based on your OS: + +* **Windows:** M:/commercials +* **Linux:** /mnt/M/commercials +* **macOS:** /Volumes/M/commercials + +This ensures consistent file path handling across different platforms. + +.. note:: + + Stalker consistently uses forward slashes (/) in path definitions, regardless + of the operating system. This applies even if you initially specify paths + with backward slashes (\\). + + +Assigning Repository to Project +------------------------------- + +Connecting a repository to your project lets Stalker know where project files +are stored. However, it still needs information about the project's specific +directory structure. + +Defining Project Structure +-------------------------- + +A :class:`.Structure` object defines the directory hierarchy for your project +within the repository. We create a structure named "Commercial Projects +Structure" and assign it to our project: + +.. code-block:: python + + from stalker import Structure + + commercial_project_structure = Structure( + name="Commercial Projects Structure" + ) + + # now assign this structure to our project + new_project.structure = commercial_project_structure + +.. versionadded:: 0.2.13 + + Starting with Stalker version 0.2.13, :class:`.Project` instances can be + associated with multiple :class:`.Repository` instances. This allows for + more flexible file management, such as storing published versions on a + separate server or directing rendered outputs to a different location. + + While the following examples are simplified, future versions will showcase + the full potential of multiple repositories. + +Creating Filename Templates +--------------------------- + +Next we create :class:`.FilenameTemplate` instances. These templates define how +filenames and paths will be generated for :class:`.Version` instances +associated with tasks. + +Here, we create a :class:`.FilenameTemplate` named "Task Template for +Commercials" that uses `Jinja2`_ syntax for the ``path`` and ``filename`` +arguments. The :class:`.Version` class knows how to render these templates +while calculating its ``path`` and ``filename`` attributes: + +.. code-block:: python + + from stalker import FilenameTemplate + + task_template = FilenameTemplate( + name='Task Template for Commercials', + target_entity_type='Task', + path='$REPO{{project.repository.code}}/{{project.code}}/{%- for p in parent_tasks -%}{{p.nice_name}}/{%- endfor -%}', + filename='{{version.nice_name}}_v{{"%03d"|format(version.version_number)}}' + ) + + # Append the template to the project structure + commercial_project_structure.templates.append(task_template) + + # No need to add anything as the project is already in the database + DBsession.commit() + +Explanation of the Template: + +* ``$REPO{{project.repository.code}}``: This references the first repository + assigned to the project. Importantly, this uses an environment variable + ``$REPO``. Stalker dynamically creates environment variables for each + repository upon database connection or creation, simplifying path definitions + within templates. + +* ``{{project.code}}``: This represent the project code and it is guaranteed to + be file system safe. + +* ``{%- for p in parent_tasks -%}{{p.nice_name}}/{%- endfor -%}``: This loop + iterates over parent tasks, creating subdirectories for each. + +* ``{{version.nice_name}}_v{{"%03d"|format(version.version_number)}}``: This + defines the filename format with version number padding. + +Creating and Managing Versions +------------------------------ + +Now, let's create a :class:`.Version`` instance for the "comp" task: + +.. code-block:: python + + from stalker import Version + + vers1 = Version(task=comp) + + # Update paths using the template + vers1.update_paths() + + print(vers1.path) # '$REPO33/FC/SH001/comp' + print(vers1.filename) # 'SH001_comp_Main_v001' + print(vers1.full_path) # '$REPO33/FC/SH001/comp/SH001_comp_Main_v001' + + # Absolute paths with repository root based on your OS + print(vers1.absolute_path) # '/mnt/M/commercials/FC/SH001/comp' + print(vers1.absolute_full_path) # '/mnt/M/commercials/FC/SH001/comp/SH001_comp_Main_v001' + + # Get version number (automatically incremented) + print(vers1.version_number) # 1 + + # commit to database + DBsession.commit() + +Stalker automatically generates a consistent path and filename for the version +based on the template. + +Stalker eliminates the need for those cumbersome and confusing file naming +conventions like ``Shot1_comp_Final``, ``Shot1_comp_Final_revised``, +``Shot1_comp_Final_revised_Final``, +``Shot1_comp_Final_revised_Final_real_final`` ...and the list goes on, +we've all experienced the frustration of such naming conventions, haven't we +😊.. It ensures a consistent and organized file structure, making asset +management significantly more efficient. + +The :attr:`.Version.is_published` attribute within the :class:`.Version` class +helps differentiate between finalized and in-progress versions. Setting +:attr:`.is_published` to ``True`` flags a version as ready for use or review. + +.. code-block:: python + + vers1.is_published = False # This version is still being worked on + +Automatic Version Numbering +--------------------------- + +Stalker automatically increments version numbers for each new version created +for the same task. This ensures you always have the latest iteration readily +identified. + +.. code-block:: python + + vers2 = Version(task=comp) + vers2.update_paths() + + print(vers2.version_number) # Output: 2 + print(vers2.filename) # Output: 'SH001_comp_Main_v002' + + vers3 = Version(task=comp) + vers3.update_paths() + + print(vers3.version_number) # Output: 3 + print(vers3.filename) # Output: 'SH001_comp_Main_v003' + +Querying Versions +----------------- + +You can retrieve all versions associated with a specific task using either +using the :attr:`.Task.versions` attribute or by doing a database query. + +.. code-block:: python + + # using pure Python + vers_from_python = comp.versions + # [, + # , + # ] + + # # Using SQLAlchemy query + vers_from_query = Version.query.filter_by(task=comp).all() + + # again returns + # [, + # , + # ] + + # Both methods return a list of Version objects + assert vers_from_python == vers_from_query + +.. _Jinja2: http://jinja.pocoo.org/ + +.. note:: + + Stalker stores :attr:`.Version.path` and :att:`.Version.filename` attribute + values within the database, independent of your operating system. The + :attr:`.Version.absolute_path` and :attr:`.Version.absolute_full_path` + attributes provide OS-specific paths by combining the + :attr:`.Repository.path` with the database values momentarily. + +You can define default directories within your project structure using custom templates. + +.. code-block:: python + + commercial_project_structure.custom_template = """ + Temp + References + References/Movies + References/Images + """ + +When executed, this template will generate the following directory structure: + +.. code-block:: shell + + Temp + References + Movies + Images diff --git a/docs/source/tutorial/basics.rst b/docs/source/tutorial/basics.rst new file mode 100644 index 00000000..e0962408 --- /dev/null +++ b/docs/source/tutorial/basics.rst @@ -0,0 +1,190 @@ +.. _tutorial_basics_toplevel: + +Basics +====== + +Imagine you've just installed Stalker and want to integrate it into your first +project. The first step involves connecting to the database to store +information about your studio and projects. + +Connecting to the Database +-------------------------- + +A helper function is provided for connecting to the default database. Use the +following command: + +.. code-block:: python + + from stalker.db.setup import setup + setup({"sqlalchemy.url": "sqlite:///"}) + +This creates an in-memory SQLite3 database, suitable only for testing. For +practical use, consider a file-based SQLite3 database: + +.. code-block:: python + + # Windows + setup({"sqlalchemy.url": "sqlite:///C:/studio.db"}) + + # Linux or macOS + setup({"sqlalchemy.url": "sqlite:////home/ozgur/studio.db"}) + + +This command will do the following: + +1. **Database Connection:** Creates an `engine`_ to establish the connection. +2. **Database Creation:** Creates the SQLite3 database file if doesn't exist. +3. **Session Creation:** Creates a `session`_ instance for interacting with the + database. +4. **Mapping:** Defines how SOM classes `map`_ to database tables (see + SQLAlchemy documentation for details). + +.. _session: http://www.sqlalchemy.org/docs/orm/session.html +.. _engine: http://www.sqlalchemy.org/docs/core/engines.html +.. _map: http://www.sqlalchemy.org/docs/orm/mapper_config.html + +.. note:: + + While SQLite3 support was officially dropped in Stalker v0.2.18, it's still + possible to use SQLite3 databases with Stalker. However, PostgreSQL (versions + 14 to 17) is the recommended database backend. + +Database Initialization +----------------------- + +On your initial connection, use `db.init()` to create essential default data +for Stalker to function properly: + +.. code-block:: python + + db.init() + +This is a one-time operation; subsequent calls to `db.init()` won't break +anything, but they're unnecessary. + +Creating a Studio +----------------- + +Let's create a :class:`.Studio` object to represent your studio: + +.. code-block:: python + + from stalker import Studio + my_studio = Studio( + name='My Great Studio' + ) + +We'll explain the concept of :class:`.Studio` later in the tutorial. + +Creating a User +--------------- + +Now, let's create a :class:`.User` object representing yourself in the +database: + +1. Import the :class:`.User` class: + + .. code-block:: python + + from stalker import User + +2. Create the :class:`.User` object: + + .. code-block:: python + + me = User( + name="Erkan Ozgur Yilmaz", + login="eoyilmaz", + email="some_email_address@gmail.com", + password="secret", + description="This is me" + ) + +This creates a user object that represents you. + +Creating and Assigning a Department +----------------------------------- + +1. Import the :class:`.Department` class: + + .. code-block:: python + + from stalker import Department + +2. Create a :class:`.Department` object: + + .. code-block:: python + + tds_department = Department( + name="TDs", + description="This is the TDs department" + ) + +3. Assign yourself to the department: + +There are two ways to do this: + +* Using the :class:`.Department` object: + + .. code-block:: python + + tds_department.users.append(me) + +* Using the :class:`.User` object: + + .. code-block:: python + + me.departments.append(tds_department) + +Both methods achieve the same result. + +Verifying Department Assignment +------------------------------- + +You can verify the assignment by printing the :attr:`.User.departments` for +your user: + +.. code-block:: python + + print(me.departments) + # Output: [] + +Saving Data to the Database +--------------------------- + +So far, the data hasn't been saved to the database yet. To commit the changes, +use the :class:`.DBSession` object: + +.. code-block:: python + + from stalker.db.session import DBSession + + DBSession.add(my_studio) + DBSession.add(me) + DBSession.add(tds_department) + DBSession.commit() + +Retrieving Data +--------------- + +Let's retrieve data from the database. Here, we'll fetch all departments, get +the second one (excluding the default `admins` department), and print the name +of its first member: + +.. code-block:: python + + all_departments = Department.query.all() + print(all_departments) + # Output: [, ] + # "admins" department is created by default + + admins = all_departments[0] + tds = all_departments[1] + + all_users = tds.users # Department.users is a "synonym" for Department.members + # they are essentially the same attribute + + print(all_users[0]) + # Output: + +This retrieves and prints the information. diff --git a/docs/source/tutorial/collaboration.rst b/docs/source/tutorial/collaboration.rst new file mode 100644 index 00000000..5204b739 --- /dev/null +++ b/docs/source/tutorial/collaboration.rst @@ -0,0 +1,30 @@ +.. _tutorial_collaboration_toplevel: + +Collaboration in Stalker +======================== + +While we've covered the core functionalities of Stalker, effective +collaboration is essential in any production pipeline. Stalker provides several +tools to facilitate communication and knowledge sharing among team members. + +Note System: + + You can leave :class:`Note`\ s on any Stalker :class:`.Entity` (except + other :class:`.Note`\ s and :class:`.Tag`\ s). This allows you to add + comments, reminders, or specific instructions to tasks, assets, versions, + and other objects. + +Messaging System: + + A direct messaging system (currently under development) will allow you to + send private messages to individuals or groups of users. + +Ticket System: + + Create and track tickets for specific projects to report issues, request + features, or discuss project-related matters. + +Wiki Pages: + + Create and maintain project-specific wiki pages to document procedures, + best practices, and other important information. diff --git a/docs/source/tutorial/conclusion.rst b/docs/source/tutorial/conclusion.rst new file mode 100644 index 00000000..0d41dc9d --- /dev/null +++ b/docs/source/tutorial/conclusion.rst @@ -0,0 +1,39 @@ +.. _tutorial_toplevel: + +Conclusion +========== + +In this tutorial, you have nearly learned a quarter of what Stalker supplies as +a Python library. + +Stalker provides a robust framework for production asset management, serving +the needs of both large and small studios. Its 15-year development history (as +of 2024) and use in major feature films and countless commercials is a +testament to its effectiveness. + +While Stalker itself lacks a graphical user interface (GUI), its power extends +beyond raw code. Here are some additional tools that leverage Stalker's core +functionality: + + +`Stalker Pyramid`_: + + A web application built using the `Pyramid`_ framework, utilizing Stalker + as its database model. This allows for user-friendly web-based interaction + with project data. + +`Anima`_: + + A pipeline library that incorporates Stalker, showcasing how its + functionalities can be integrated into a pipeline management system. + Notably, Anima demonstrates the creation of Qt UIs using Stalker. + +For a deeper dive into how Stalker interacts with UIs and web applications, +consider exploring the repositories of `Stalker Pyramid`_ and `Anima`_. + +By understanding how Stalker integrates with these tools, you can unlock its +full potential for streamlining your production workflows. + +.. _Stalker Pyramid: https://www.github.com/eoyilmaz/stalker_pyramid +.. _Anima: https://github.com/eoyilmaz/anima +.. _Pyramid: https://trypyramid.com/ diff --git a/docs/source/tutorial/creating_simple_data.rst b/docs/source/tutorial/creating_simple_data.rst new file mode 100644 index 00000000..fecfe531 --- /dev/null +++ b/docs/source/tutorial/creating_simple_data.rst @@ -0,0 +1,176 @@ +.. _tutorial_creating_simple_data_toplevel: + +Creating Simple Data +==================== + +Let's imagine you're starting a new commercial project and want to use Stalker. +The first step is to create a :class:`.Project` object to store project +information. + +Project setup +------------- + +.. versionadded:: 0.2.24.2 + + Starting with Stalker v0.2.24.2, you no longer need to manually create + :class:`.StatusList` instances for :class:`.Project` objects. The + :func:`stalker.db.setup.init()` function will automatically create them + during database initialization. + +.. note:: + When the Stalker database is first initialized (with ``db.setup.init()``), a + set of default :class:`.Status` instances for :class:`.Task`, + :class:`.Asset`, :class:`.Shot`, :class:`.Sequence` and :class:`.Ticket` + classes are created, along with their respective :class:`.StatusList` + instances. + +Creating a Repository +--------------------- + +To create a project, we first need to create a :class:`.Repository`. +The Repository (or Repo) is a directory on your file server, where project +files are stored and accessible to all workstations and render farm computers: + +.. code-block:: python + + from stalker import Repository + + commercial_repo = Repository( + name="Commercial Repository", + code="CR" + ) + +.. versionadded:: 0.2.24 + + Starting with Stalker version 0.2.24 :class:`.Repository` instances have a + :attr:`stalker.models.repository.Repository.code` attribute to help + generate universal paths across operating systems and Stalker installations. + +:class:`.Repository` class will be explained in detail in upcoming sections. + +Creating a Project +------------------ + +Now, let's create the project: + +.. code-block:: python + + new_project = Project( + name="Fancy Commercial", + code='FC', + repositories=[commercial_repo], + ) + +Adding Project Details +---------------------- + +Let's add more details to the project: + +.. code-block:: python + + import tzlocal + import datetime + from stalker import ImageFormat + + new_project.description = ( + "The commercial is about this fancy product. The " + "client want us to have a shiny look with their " + "product bla bla bla..." + ) + + new_project.image_format = ImageFormat( + name="HD 1080", + width=1920, + height=1080 + ) + + new_project.fps = 25 + local_tz = tzlocal.get_localzone() + new_project.end = datetime.datetime(2024, 5, 15, tzinfo=local_tz) + new_project.users.append(me) + +Saving the Project +------------------ + +To save the proejct and its associated data to the database: + +.. code-block:: python + + DBSession.add(new_project) + DBSession.commit() + +Even though we've created multiple objects (project, repository etc.), we only +need to add the ``new_project`` object to the database. Stalker will handle the +relationships and save the related objects automatically. + +.. note:: + + Starting with Stalker v0.2.18, all the datetime information must include + timezone information. In the example, we've used the local timezone. + +Creating Sequences and Shots +---------------------------- + +A :class:`.Project` is typically composed of :class:`.Task` instances, which +represent units of work that need to be completed. A :class:`.Task` in Stalker +defines the total `effort` required to be considered finished. Tasks can also +be `duration` or `length` based, in which case they define the required time +to be considered finished. Leaf tasks, the final tasks in a task hierarchy, +are assigned to specific :class:`.User` instances who are responsible for +completing them. More details about :class:`.Task` and its attributes can be +found in the :class:`.Task` class documentation. :class:`.Asset`, +:class:`.Shot` and :class:`.Sequences` are specialized types of Tasks. + +Let's create a :class:`.Sequence`: + +.. code-block:: python + + from stalker import Sequence + + seq1 = Sequence( + name="Sequence 1", + code="SEQ1", + project=new_project, + ) + +And some :class:`.Shot`\ s withing the sequence: + +.. code-block:: python + + from stalker import Shot + + sh001 = Shot( + name='SH001', + code='SH001', + project=new_project, + sequences=[seq1] + ) + sh002 = Shot( + code='SH002', + project=new_project, + sequences=[seq1] + ) + sh003 = Shot( + code='SH003', + project=new_project, + sequences=[seq1] + ) + +Save the changes to the database: + +.. code-block:: python + + DBsession.add_all([sh001, sh002, sh003]) + DBsession.commit() + +.. note:: + + * While we've created :class:`.Shot` objects with a :class:`.Sequence` + instance, it's not strictly necessary. You can create :class:`.Shot` + objects without assigning them to a Sequence. + + * For smaller projects like commercials, you might skip creating sequences + altogether. + + * For larger projects like feature films, using sequences to group shots is + recommended. diff --git a/docs/source/tutorial/extending_som.rst b/docs/source/tutorial/extending_som.rst new file mode 100644 index 00000000..54096d2c --- /dev/null +++ b/docs/source/tutorial/extending_som.rst @@ -0,0 +1,6 @@ +.. _tutorial_extending_som_toplevel: + +Extending SOM (coming) +====================== + +This part will be covered soon diff --git a/docs/source/tutorial/pipeline.rst b/docs/source/tutorial/pipeline.rst new file mode 100644 index 00000000..602dc91b --- /dev/null +++ b/docs/source/tutorial/pipeline.rst @@ -0,0 +1,62 @@ +.. _tutorial_pipeline_toplevel: + +Pipeline +======== + +So far, we've covered the basics of creating data in Stalker. However. to fully +utilize Stalker's power, we need to define our studio's **pipeline**. This +involves creating tasks and establishing dependencies between them. + +Creating Tasks +-------------- + +Let's create some :class:`.Task`\ s for one of the shots we created earlier: + +.. code-block:: python + + from stalker import Task + + previs = Task( + name="Previs", + parent=sh001 + ) + + matchmove = Task( + name="Matchmove", + parent=sh001 + ) + + anim = Task( + name="Animation", + parent=sh001 + ) + + lighting = Task( + name="Lighting", + parent=sh001 + ) + + comp = Task( + name="Comp", + parent=sh001 + ) + +Defining Dependencies +--------------------- + +Now, let's define the dependencies between these tasks: + +.. code-block:: python + + comp.depends_on = [lighting] + lighting.depends_on = [anim] + anim.depends_on = [previs, matchmove] + +By establishing these dependencies, we're telling Stalker that certain tasks +need to be completed before others can begin. For example, the "Comp" task +depends on the "Lighting" task, meaning the "Lighting" task must be finished +before the "Comp" task can start. Stalker uses these dependencies to schedule +tasks effectively. + +We'll delve deeper into task scheduling and other pipeline-related concepts +later in this tutorial. diff --git a/docs/source/tutorial/query_update_delete_data.rst b/docs/source/tutorial/query_update_delete_data.rst new file mode 100644 index 00000000..6685a5fd --- /dev/null +++ b/docs/source/tutorial/query_update_delete_data.rst @@ -0,0 +1,75 @@ +.. _tutorial_query_update_delete_data_toplevel: + +Querying, Updating and Deleting Data +==================================== + +Now that you've created some data, let's explore how to update and delete it. + +Updating Data +------------- + +Imagine you created a shot with incorrect information: + +.. code-block:: python + + sh004 = Shot( + code='SH004', + project=new_project, + sequences=[seq1] + ) + DBSession.add(sh004) + DBSession.commit() + +Later, you realize you need to fix the code: + +.. code-block:: python + + sh004.code = "SH005" + DBsession.commit() + +Retrieving Data +--------------- + +To retrieve a shot from the database, you can use a query: + +.. code-block:: python + + wrong_shot = Shot.query.filter_by(code="SH005").first() + +This retrieves the first shot with the code "SH005". + +Updating Retrieved Data +----------------------- + +If you need to modify the retrieve data: + +.. code-block:: python + + wrong_shot.code = "SH004" # Correct the code + DBsession.commit() # Save the changes + +Deleting Data +------------- + +To delete data, use the :meth:`DBSession.delete()` method: + +.. code-block:: python + + DBsession.delete(wrong_shot) + DBsession.commit() + +After deleting data, you program variables might still hold references to the +deleted objects, but those objects no longer exist in the database. + +.. code-block:: python + + wrong_shot = Shot.query.filter_by(code="SH005").first() + print(wrong_shot) # This will print None + +For More information +-------------------- + +For advanced update and delete options (like cascades) in SQLAlchemy, refer to +the official `SQLAlchemy documentation`_. + +.. _SQLAlchemy documentation: http://www.sqlalchemy.org/docs/orm/session.html diff --git a/docs/source/tutorial/scheduling.rst b/docs/source/tutorial/scheduling.rst new file mode 100644 index 00000000..a31b2f9e --- /dev/null +++ b/docs/source/tutorial/scheduling.rst @@ -0,0 +1,114 @@ +.. _tutorial_scheduling_toplevel: + +Scheduling +========== + +Now that we've defined tasks, resources, and dependencies, let's schedule our +project! + + +TaskJuggler Integration +----------------------- + +Stalker utilizes `TaskJuggler`_ to solve scheduling problems and determine when +resources should work on specific tasks. + +.. warning:: + + * Ensure you have `TaskJuggler`_ installed on your system. + + * Configure Stalker to locate the ``tj3`` executable: + + * **Linux:** This is usually straightforward under Linux, just install + `TaskJuggler`_ and Stalker will be able to use it. + + * **macOS & Windows:** Create a ``STALKER_PATH`` environment variable + pointing to a folder containing a ``config.py`` file. Add the following + line to ``config.py``: + + .. code-block:: python + + tj_command = r"C:\Path\to\tj3.exe" + + The default value for ``tj_command`` is ``/usr/local/bin/tj3``. If you run + ``which tj3`` on Linux or macOS and it returns this value, no additional + setup is needed. + + .. _TaskJuggler: http://www.taskjuggler.org/ + +Scheduling Your Project +----------------------- + +Let's schedule our project using the :class:`.Studio` instance that we've +created at the beginning of this tutorial: + +.. code-block:: python + + from stalker import TaskJugglerScheduler + + my_studio.scheduler = TaskJugglerScheduler() + # Set a large duration (e.g., 1 year) to avoid TaskJuggler complaining the + # project is not fitting into the time frame. + my_studio.duration = datetime.timedelta(days=365) + my_studio.schedule(scheduled_by=me) + DBsession.commit() # Save changes + +This process might take a few seconds for small project or long for larger +ones. + +Viewing Scheduled Dates +----------------------- + +Once completed, each task will have its ``computed_start`` and ``computed_end`` +values populated: + +.. code-block:: python + + for task in [previs, matchmove, anim, lighting, comp]: + print("{:16s} {} -> {}".format( + task.name, + task.computed_start, + task.computed_end + )) + +Outputs: + +.. code-block:: shell + + Previs 2024-04-02 16:00 -> 2024-04-15 15:00 + Matchmove 2024-04-15 15:00 -> 2024-04-17 13:00 + Animation 2024-04-17 13:00 -> 2024-04-23 17:00 + Lighting 2024-04-23 17:00 -> 2024-04-24 11:00 + Comp 2024-04-24 11:00 -> 2024-04-24 17:00 + +Understanding the Output +------------------------ + +The output will display start and end dates for each task, reflecting the +dependencies. In this example, since each task has only one assigned resource +(you), they follow one another. + +Further Explorations +-------------------- + +Scheduling is complex topic. For in-depth information, refer to the +`TaskJuggler`_ documentation. + + +TaskJuggler Project Representation +---------------------------------- + +You can check the ``to_tjp`` values of the data objects: + +.. code-block:: python + + print(my_studio.to_tjp) + print(me.to_tjp) + print(comp.to_tjp) + print(new_project.to_tjp) + +If you're familiar with TaskJuggler, you'll recognize the output format. +Stalker maps its data to TaskJuggler-compatible strings. Although, Stalker is +currently supporting a subset of directives, it is enough for scheduling +complex projects with intricate dependencies and hierarchies. Support for +additional TaskJuggler directives will grow with future Stalker versions. diff --git a/docs/source/tutorial/task_and_resource_management.rst b/docs/source/tutorial/task_and_resource_management.rst new file mode 100644 index 00000000..e53a7c26 --- /dev/null +++ b/docs/source/tutorial/task_and_resource_management.rst @@ -0,0 +1,53 @@ +.. _tutorial_task_resource_management_toplevel: + +Task and Resource Management +============================ + +Now that we have created shots and tasks, we need to assign resources (users) +to these tasks to complete the work. + +Let's assign ourselves to all the tasks: + +.. code-block:: python + + previs.resources = [me] + previs.schedule_timing = 10 + previs.schedule_unit = 'd' + + matchmove.resources = [me] + matchmove.schedule_timing = 2 + matchmove.schedule_unit = 'd' + + anim.resources = [me] + anim.schedule_timing = 5 + anim.schedule_unit = 'd' + + lighting.resources = [me] + lighting.schedule_timing = 3 + lighting.schedule_unit = 'd' + + comp.resources = [me] + comp.schedule_timing = 6 + comp.schedule_unit = 'h' + +Here, we've assigned ourselves as the resource for each task and specified the +estimated time to complete the task using ``schedule_timing`` and +``schedule_unit`` attributes. + +Saving Changes +-------------- + +To save these changes to the database: + +.. code-block:: python + + DBsession.commit() + +Note that we didn't explicitly add any new object to the session. Since all the +tasks are related to the ``sh001`` shot, which is already tracked by the +session, SQLAlchemy will automatically track and save the changes to the +database. + +With this information, Stalker can now schedule these tasks, taking info +account dependencies and resource availability. This will help you plan and +manage your project more efficiently. diff --git a/docs/source/tutorial_files/tutorial.py b/docs/source/tutorial/tutorial_files/tutorial.py similarity index 100% rename from docs/source/tutorial_files/tutorial.py rename to docs/source/tutorial/tutorial_files/tutorial.py diff --git a/requirements-dev.txt b/requirements-dev.txt index 756ad78a..9f38ec69 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,10 +10,14 @@ flake8-mutable flake8-pep3101 flake8-spellcheck flake8-pyproject +furo pyglet pytest pytest-cov pytest-github-actions-annotate-failures pytest-xdist +sphinx +sphinx-autoapi +# sphinx-findthedocs twine wheel \ No newline at end of file diff --git a/src/stalker/db/setup.py b/src/stalker/db/setup.py index 70748634..608447c5 100644 --- a/src/stalker/db/setup.py +++ b/src/stalker/db/setup.py @@ -493,7 +493,7 @@ def register(class_): :class:`.User` s and :class:`.Group` s to be able to interact with the given class. Whatever class you have created needs to be registered. - Example, lets say that you have a data class which is specific to your + Example, let's say that you have a data class which is specific to your studio and it is not present in Stalker Object Model (SOM), so you need to extend SOM with a new data type. Here is a simple Data class inherited from the :class:`.SimpleEntity` class (which is the simplest class you should diff --git a/src/stalker/models/version.py b/src/stalker/models/version.py index 0591d17e..d404f375 100644 --- a/src/stalker/models/version.py +++ b/src/stalker/models/version.py @@ -57,7 +57,7 @@ class Version(Link, DAGMixin): {{project.repository.path}}/{{project.code}}/{%- for parent_task in parent_tasks -%}{{parent_task.nice_name}}/{%- endfor -%} - Or, lets have a setup with environment variables: + Or, let's have a setup with environment variables: $REPO{{project.repository.id}}/{{project.code}}/{%- for parent_task in parent_tasks -%}{{parent_task.nice_name}}/{%- endfor -%} From e54707cafd1106ee8402a89e99bbd2acfc64363d Mon Sep 17 00:00:00 2001 From: Erkan Ozgur Yilmaz Date: Tue, 5 Nov 2024 09:51:31 +0000 Subject: [PATCH 2/2] [#90] Fix a typo in `design.rst` and add the mention of PostgreSQL versions that Stalker is tested with. --- docs/source/design.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/source/design.rst b/docs/source/design.rst index 50e01400..9b3c42a3 100644 --- a/docs/source/design.rst +++ b/docs/source/design.rst @@ -105,8 +105,9 @@ pipeline: 2. **SQLAlchemy Integration:** Leverages SQLAlchemy for its database backend and Object-Relational Mapping (ORM) capabilities, ensuring efficient data - management. - + management. Designed PostgreSQL (versions 14 to 17) in mind but not limited + to it. + 3. **Jinja2 Templates:** Employs Jinja2 for flexible file and folder naming conventions. For a structured naming scheme it is possible to define templates like: @@ -132,9 +133,9 @@ pipeline: 9. **TaskJuggler Integration:** Integrate with TaskJuggler for enhanced task management capabilities, supporting basic task attributes. - 9. **Predefined Task Statuses:** Manage task progress efficiently with a - pre-defined Task Status Workflow, providing a structured approach to - tracking task completion stages. + 10. **Predefined Task Statuses:** Manage task progress efficiently with a + pre-defined Task Status Workflow, providing a structured approach to + tracking task completion stages. For usage examples see :ref:`tutorial_toplevel`\ .