Add sfbuild|pyF4PGA

.github/scripts/prepare_environment.sh

@@ -62,7 +62,7 @@ echo '::endgroup::'
 cd ..
 
 
-echo '::group::🗑️ Remove the wrappers (pre-packaged from arch-defs) and add f4pga-env'
+echo '::group::Add f4pga-env'
 
 case "$F4PGA_FAM" in
   xc7) F4PGA_DIR_ROOT='install';;
@@ -71,6 +71,11 @@ esac
 
 F4PGA_DIR_BIN="$F4PGA_INSTALL_DIR_FAM/$F4PGA_DIR_ROOT"/bin/
 cp $(dirname "$0")/../../f4pga-env "$F4PGA_DIR_BIN"
+
+echo '::endgroup::'
+
+echo '::group::🗑️ Remove the wrappers (pre-packaged from arch-defs)'
+
 cd "$F4PGA_DIR_BIN"
 
 case "$F4PGA_FAM" in

.github/sftest.json

@@ -0,0 +1,43 @@
+{
+    "default_platform": "xc7a50t",
+    "values": {
+        "top": "top"
+    },
+    "dependencies": {
+        "sources": [
+            "xc7/counter_test/counter.v"
+        ],
+        "synth_log": "synth.log",
+        "pack_log": "pack.log"
+    },
+    "xc7a200t": {
+        "default_target": "bitstream",
+        "dependencies": {
+            "xdc": [
+                "arty200.xdc"
+            ],
+            "build_dir": "build/arty_200"
+        }
+    },
+    "xc7a100t": {
+        "default_target": "bitstream",
+        "dependencies": {
+            "xdc": [
+                "arty.xdc"
+            ],
+            "build_dir": "build/arty100"
+        }
+    },
+    "xc7a50t": {
+        "default_target": "bitstream",
+        "dependencies": {
+            "build_dir": "build/arty_35",
+            "xdc": [
+                "arty.xdc"
+            ]
+        },
+        "values": {
+            "part": "xc7a35tcpg236-1"
+        }
+    }
+}

.github/workflows/Pipeline.yml

@@ -21,12 +21,13 @@ on:
 
 jobs:
 
+
   Docs:
     runs-on: ubuntu-latest
     name: '📓 Docs'
     steps:
 
-      - name: '🧰 Checkout'
+      - name: 🧰 Checkout
         uses: actions/checkout@v3
         with:
           submodules: recursive
@@ -62,43 +63,43 @@ jobs:
           git push -u origin +HEAD:gh-pages
 
 
-  Example:
+  Deprecated:
     runs-on: ubuntu-latest
-    name: '🐍 Example'
     strategy:
       fail-fast: false
       matrix:
         include:
           - { fam: xc7, example: counter_test }
           - { fam: eos-s3, example: btn_counter }
+    name: '🚦 Example (deprecated sh) | ${{ matrix.fam }}'
     env:
       F4PGA_INSTALL_DIR: /opt/f4pga
       F4PGA_FAM: ${{ matrix.fam }}
 
     steps:
 
-      - name: '🧰 Checkout'
+      - name: 🧰 Checkout
         uses: actions/checkout@v3
 
-      - name: '🔧 Prepare environment'
+      - name: 🔧 Prepare environment
         run: ./.github/scripts/prepare_environment.sh
 
-      - name: '🐍 Install f4pga (pip)'
+      - name: 🐍 Install f4pga (pip)
         run: |
           . ./.github/scripts/activate.sh
 
           cd f4pga
           pip install --use-feature=in-tree-build .
           cd ..
 
-      - name: '🚧 Test f4pga-env'
+      - name: 🚧 Test f4pga-env
         run: |
           . ./.github/scripts/activate.sh
 
           echo "F4PGA_ENV_BIN=$(f4pga-env bin)" >> "$GITHUB_ENV"
           echo "F4PGA_ENV_SHARE=$(f4pga-env share)" >> "$GITHUB_ENV"
 
-      - name: '🚧 Test make example'
+      - name: 🚧 Test make example
         run: |
           . ./.github/scripts/activate.sh
 
@@ -127,3 +128,91 @@ jobs:
           name: eos-s3-Bitstream
           path: f4pga-examples/eos-s3/btn_counter/build/top.bit
           if-no-files-found: error
+
+
+  pyF4PGA:
+    runs-on: ubuntu-latest
+    name: '🐍 Example | xc7'
+    env:
+      F4PGA_INSTALL_DIR: /opt/f4pga
+      F4PGA_FAM: xc7
+
+    steps:
+
+      - name: 🧰 Checkout
+        uses: actions/checkout@v3
+#        with:
+#          submodules: recursive
+
+      - name: 🔧 Prepare environment
+        run: ./.github/scripts/prepare_environment.sh
+
+      - name: 🐍 Install f4pga (pip)
+        run: |
+          . ./.github/scripts/activate.sh
+
+          cd f4pga
+          pip install --use-feature=in-tree-build .
+          cd ..
+
+      - name: 🚧 Test f4pga build
+        run: |
+          . ./.github/scripts/activate.sh
+
+          cd f4pga-examples
+          f4pga build --flow ../.github/sftest.json -t bitstream
+
+      - name: '📤 Upload artifact: Arty 35 bitstream'
+        uses: actions/upload-artifact@v3
+        with:
+          name: arty_35-Bitstream-pyF4PGA
+          path: f4pga-examples/build/arty_35/top.bit
+
+
+  PYTHONPATH:
+    runs-on: ubuntu-latest
+    name: '🐍 PYTHONPATH'
+    env:
+      F4PGA_INSTALL_DIR: /opt/f4pga
+      F4PGA_FAM: xc7
+
+    steps:
+
+      - name: 🧰 Checkout
+        uses: actions/checkout@v3
+
+      - name: 🚧 Test pyF4PGA (PYTHONPATH)
+        run: |
+          PYTHONPATH=$(pwd) python3 f4pga/__init__.py
+          PYTHONPATH=$(pwd) python3 f4pga/__init__.py -h
+
+
+  pyWrappers:
+    runs-on: ubuntu-latest
+    name: '🐍 Python wrappers'
+    env:
+      F4PGA_INSTALL_DIR: /opt/f4pga
+      F4PGA_FAM: xc7
+
+    steps:
+
+      - name: 🧰 Checkout
+        uses: actions/checkout@v3
+
+      - name: 🔧 Prepare environment
+        run: ./.github/scripts/prepare_environment.sh
+
+      - name: 🐍 Install f4pga (pip)
+        run: |
+          . ./.github/scripts/activate.sh
+
+          cd f4pga
+          pip install --use-feature=in-tree-build .
+          cd ..
+
+      - name: 🚦 Test Python wrappers
+        run: |
+          . ./.github/scripts/activate.sh
+
+          pip3 install -r ./test/requirements.txt
+          pytest -vsrA --color=yes test/wrappers.py

.gitignore

@@ -1,2 +1,3 @@
 *.pyc
 *.sw*
+/f4pga/build/

docs/conf.py

@@ -47,16 +47,24 @@
     'sphinx.ext.intersphinx',
     'sphinx_verilog_domain',
     'sphinxcontrib.bibtex',
+    'myst_parser'
 ]
 
 bibtex_default_style = 'plain'
 bibtex_bibfiles = ['refs.bib']
 
+myst_enable_extensions = [
+    "colon_fence",
+]
+
 numfig = True
 
 templates_path = ['_templates']
 
-source_suffix = ['.rst', '.md']
+source_suffix = {
+    '.rst': 'restructuredtext',
+    '.md': 'markdown'
+}
 
 master_doc = 'index'
 

docs/f4pga/DevNotes.md

@@ -0,0 +1,103 @@
+# Developer's notes
+##### Last update: 2022-05-06
+
+:::{warning}
+These notes are provided as-is and they shouldn't be treated as a full-blown accurate
+documentation, but rather as a helpful resource for those who want to get involved with
+development of _f4pga_. These are not updated regularly.
+
+For more detailed, up-to-date information about the code, refer to the pydoc documentation.
+:::
+
+## Project's structure
+
+* `__init__.py` contains the logic and entry point of the build system
+* `argparser.py` contains boring code for CLI interface
+* `cache.py` contains code needed for tracking modifications in the project.
+* `common.py` contains code shared by the main utility and the modules
+* `flow_config.py` contains code for reading and accessing flow definitions and configurations
+* `module_inspector.py` contains utilities for inspecting I/O of modules
+* `module_runner.py` contains code required to load modules at run-time
+* `module.py` contains definitions required for writing and using f4pga modules
+* `part_db.json` contains mappings from part names to platform names
+* `setup.py` contains a package installation script
+* `stage.py` contains classes relevant  to stage representation
+* `modules` contains loadable modules
+* `platforms` contains platform flow definitions
+
+:::{important}
+Through the codebase _f4pga_ (tool) might be often referenced as _sfbuild_.
+Similarly, _F4PGA_ (toolchain) might get called _Symbiflow_.
+This is due to the project being written back when _F4PGA_ was called _Symbiflow_.
+:::
+
+## Different subsystems and where to find them?
+
+### Building and dependency resolution
+
+All the code regarding dependency resolution is located in `__init__.py` file.
+Take a look at the `Flow` class.
+
+Most of the work is done in `Flow._resolve_dependencies` method. Basically it
+performs a _DFS_ with _stages_ (instances of _f4pga modules_) as its nodes
+which are linked using symbolic names of dependencies on inputs and outputs.
+It queries the modules for information regarding i/o (most importantly the paths
+on which they are going to produce outputs), checks whether
+their inputs are going to be satisfied, checks if dependencies were modified, etc.
+
+The actual building is done using `Flow._build_dep` procedure. It uses a similar
+_DFS_ approach to invoke modules and check their inputs and outputs.
+
+### Modification tracking
+
+Modification tracking is done by taking, comparing and keeping track of `adler32`
+hashes of all dependencies. Each dependency has a set of hashes associated with it.
+The reason for having multiple hashes is that a dependency may have multiple
+"_consumers_", ie. _stages_ which take it as input. Each hash is associated with
+particular consumer. This is necessary, because the system tries to avoid rebuilds
+when possible and status of each file (modified/unmodified) may differ in regards
+to individual stages.
+
+Keeping track of status of each file is done using `SymbiCache` class, which is
+defined in `sf_cache.py` file. `SymbiCache` is used mostly inside `Flow`'s methods.
+
+### Internal environmental variable system
+
+_f4pga_ exposes some data to the user as well as reads some using internal
+environmental variables. These can be referenced by users in
+_platform flow definitions_ and _project flow configurations_ using the
+`${variable_name}` syntax when defining values. They can also be read inside
+_f4pga modules_ by accessing the `ctx.values` namespace.
+
+The core of its system is the `ResolutionEnvironemt` class which can be found
+inside the `common` module.
+
+### Installation
+
+Check `CMakeLists.txt`.
+
+## TODO:
+
+* Define a clear specification for entries in _platform flow definitions_ and
+  _platform flow configurations_. Which environmental variables can be accessed
+  where, and when?
+
+* Force "_on-demand_" outputs if they are required by another stage.
+  This may require redesigning the "on-demand" feature, which currently works
+  by producing a dependency if and only if the user explicitly provides the
+  path. Otherwise the path is unknown.
+
+* Make commenting style consistent
+
+* Document writing flow definitions
+
+* Extend the metadata system for modules, perhaps make it easier to use.
+
+* Add missing metadata for module targets.
+
+* (_suggestion_) Generate platform definitions using CMake.
+
+### Out of the current scope
+
+* Change interfaces of some internal python scripts. This could lead to possibly
+  merging some modules for XC7 and Quicklogic into one common module.