Merge branch 'staging' into miguel/sasrec_unirec

.devcontainer/devcontainer.json

@@ -1,44 +1,54 @@
 {
-	"name": "Recommenders",
-	"build": {
-		"dockerfile": "Dockerfile",
-		"context": "..",
-		"args": { 
-			// Python version: 3, 3.6, 3.7
-			"PYTHON_VERSION": "3.7",
-			"REMOTE_USER": "vscode"
-		}
-	},
+    "name": "Recommenders",
+    "build": {
+        "dockerfile": "../tools/docker/Dockerfile",
+        "context": "..",
+        "target": "deps",
+        "args": {
+            "COMPUTE": "cpu",
+            "PYTHON_VERSION": "3.11"
+        }
+    },
+    "customizations": {
+        "vscode": {
+            // Set default container specific settings.json values on container
+            // create
+            "settings": {
+                "[python]": {
+                    "editor.defaultFormatter": "ms-python.black-formatter",
+                    "editor.formatOnSave": true,
+                    "editor.codeActionsOnSave": {
+                        "source.organizeImports": "explicit"
+                    }
+                },
+                "isort.args": ["--profile", "black"],
+                "python.analysis.autoImportCompletions": true,
+                // Conda env name *must* align with the one in Dockerfle
+                "python.defaultInterpreterPath": "/root/conda/envs/Recommenders/bin/python",
+                "python.testing.pytestEnabled": true,
+                // Test directory
+                "python.testing.pytestArgs": ["tests"]
+            },
+            // VS Code extensions to install on container create
+            "extensions": [
+                // https://marketplace.visualstudio.com/items?itemName=ms-python.black-formatter
+                "ms-python.black-formatter",
+                // https://marketplace.visualstudio.com/items?itemName=ms-python.isort
+                "ms-python.isort",
+                // https://marketplace.visualstudio.com/items?itemName=ms-python.mypy-type-checker
+                "ms-python.mypy-type-checker",
+                // https://marketplace.visualstudio.com/items?itemName=ms-python.pylint
+                "ms-python.pylint",
+                // https://marketplace.visualstudio.com/items?itemName=ms-python.python
+                "ms-python.python",
+                // https://marketplace.visualstudio.com/items?itemName=ms-toolsai.datawrangler
+                "ms-toolsai.datawrangler",
+                // https://marketplace.visualstudio.com/items?itemName=ms-toolsai.jupyter
+                "ms-toolsai.jupyter"
+            ]
+        }
+    },
 
-	// Set *default* container specific settings.json values on container create.
-	"settings": { 
-		"python.pythonPath": "/usr/local/bin/python",
-		"python.languageServer": "Pylance",
-		"python.linting.enabled": true,
-		"python.linting.pylintEnabled": true,
-		"python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
-		"python.formatting.blackPath": "/usr/local/py-utils/bin/black",
-		"python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
-		"python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
-		"python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
-		"python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
-		"python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
-		"python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
-		"python.linting.pylintPath": "/usr/local/py-utils/bin/pylint"
-	},
-
-	// Add the IDs of extensions you want installed when the container is created.
-	"extensions": [
-		"ms-python.python",
-		"ms-python.vscode-pylance"
-	],
-
-	// Use 'forwardPorts' to make a list of ports inside the container available locally.
-	"forwardPorts": [8888],
-
-	// Use 'postCreateCommand' to run commands after the container is created.
-	"postCreateCommand": "pip install -U pip && pip install --user -e .[dev,examples,spark,xlearn]",
-
-	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
-	"remoteUser": "vscode"
+    // Install Recommenders in development mode after container create
+    "postCreateCommand": "bash -i -c 'conda activate Recommenders && conda install -c conda-forge -y openjdk=21 && pip install -e .[dev,spark]'"
 }

.github/CODEOWNERS

@@ -1,5 +1,5 @@
 # See https://help.github.com/articles/about-codeowners/
 # for more info about CODEOWNERS file
 
-*           @miguelgfierro @gramhagen @anargyri @loomlike @wutaomsft @simonyansenzhao
+*           @miguelgfierro @gramhagen @anargyri @loomlike @wav8k @simonyansenzhao
 

.github/ISSUE_TEMPLATE.md

@@ -23,4 +23,11 @@
 <!--- * The tests for SAR PySpark should pass successfully. -->
 
 
+### Willingness to contribute
+<!--- Go over all the following points, and put an `x` in the box that apply. -->
+- [ ] Yes, I can contribute for this issue independently.
+- [ ] Yes, I can contribute for this issue with guidance from Recommenders community.
+- [ ] No, I cannot contribute at this time.
+
+
 ### Other Comments

.github/ISSUE_TEMPLATE/bug_report.md

@@ -28,4 +28,10 @@ assignees: ''
 <!--- For example:  -->
 <!--- * The tests for SAR PySpark should pass successfully. -->
 
+### Willingness to contribute
+<!--- Go over all the following points, and put an `x` in the box that apply. -->
+- [ ] Yes, I can contribute for this issue independently.
+- [ ] Yes, I can contribute for this issue with guidance from Recommenders community.
+- [ ] No, I cannot contribute at this time.
+
 ### Other Comments

.github/ISSUE_TEMPLATE/feature_request.md

@@ -14,4 +14,10 @@ assignees: ''
 <!--- For example:  -->
 <!--- *Adding algorithm xxx will help people understand more about xxx use case scenarios. -->
 
+### Willingness to contribute
+<!--- Go over all the following points, and put an `x` in the box that apply. -->
+- [ ] Yes, I can contribute for this issue independently.
+- [ ] Yes, I can contribute for this issue with guidance from Recommenders community.
+- [ ] No, I cannot contribute at this time.
+
 ### Other Comments

.github/ISSUE_TEMPLATE/general-ask.md

@@ -10,4 +10,10 @@ assignees: ''
 ### Description
 <!--- Describe your general ask in detail -->
 
+### Willingness to contribute
+<!--- Go over all the following points, and put an `x` in the box that apply. -->
+- [ ] Yes, I can contribute for this issue independently.
+- [ ] Yes, I can contribute for this issue with guidance from Recommenders community.
+- [ ] No, I cannot contribute at this time.
+
 ### Other Comments

.github/actions/azureml-test/action.yml

@@ -6,108 +6,93 @@
 name: azureml-tests
 description: "Submit experiment to AzureML cluster"
 inputs:
-  # azureml experiment name
   EXP_NAME:
     required: true
-    type: string
-  # type of test - unit or nightly
+    description: AzureML experiment Name
+  ENV_NAME:
+    required: true
+    description: AzureML environment Name
   TEST_KIND:
     required: true
-    type: string
-  # test environment - cpu, gpu or spark
-  TEST_ENV:
-    required: false
-    type: string
-  # azureml compute credentials
-  AZUREML_TEST_CREDENTIALS:
+    description: Type of test - unit or nightly
+  AZUREML_TEST_UMI_CLIENT_ID:
+    required: true
+    description: AzureML User-managed identity client ID
+  AZUREML_TEST_UMI_TENANT_ID:
+    required: true
+    description: AzureML User-managed identity tenant ID
+  AZUREML_TEST_UMI_SUB_ID:
     required: true
-    type: string
-  # azureml compute subid
+    description: AzureML User-managed identity subscription ID
   AZUREML_TEST_SUBID:
     required: true
-    type: string
-  # python version
+    description: AzureML subscription ID
   PYTHON_VERSION:
     required: true
-    type: string
-  # test group name
+    description: Python version used for the tests
   TEST_GROUP:
     required: true
-    type: string
-  # cpu cluster name
-  CPU_CLUSTER_NAME:
-    required: false
-    type: string
-    default: "cpu-cluster"
-  # gpu cluster name
-  GPU_CLUSTER_NAME:
-    required: false
-    type: string
-    default: "gpu-cluster"
-  # AzureML resource group name
+    description: Test group defined in test_group.py
   RG:
     required: false
-    type: string
+    description: AzureML resource group name
     default: "recommenders_project_resources"
-  # AzureML workspace name
   WS:
     required: false
-    type: string
+    description: AzureML workspace name
     default: "azureml-test-workspace"
-  # test logs path
-  TEST_LOGS_PATH:
+  LOG_DIR:
     required: false
-    type: string
-    default: '"test_logs.log"'
-  # pytest exit code
-  PYTEST_EXIT_CODE:
-    required: false
-    type: string
-    default: "pytest_exit_code.log"
+    description: Directory storing the test logs
+    default: "test_logs"
 
 runs:
   using: "composite"
   steps:
     - name: Setup python
       uses: actions/setup-python@v5
       with:
-        python-version: "3.8"
-    - name: Install azureml-core and azure-cli on a GitHub hosted server
+        python-version: "3.10"
+    - name: Install AzureML Python SDK
       shell: bash
-      run: pip install --quiet "azureml-core>1,<2" "azure-cli>2,<3"
+      run: pip install --quiet "azure-ai-ml>1,<2" mlflow "azureml-mlflow>1,<2" 
     - name: Log in to Azure
       uses: azure/login@v2
       with:
-        creds: ${{inputs.AZUREML_TEST_CREDENTIALS}}
-    - name: Install wheel package
-      shell: bash
-      run: pip install --quiet wheel
+        client-id: ${{ inputs.AZUREML_TEST_UMI_CLIENT_ID }}
+        tenant-id: ${{ inputs.AZUREML_TEST_UMI_TENANT_ID }}
+        subscription-id: ${{ inputs.AZUREML_TEST_UMI_SUB_ID }}
     - name: Submit tests to AzureML
       shell: bash
-      run: >-
+      run: |
+        echo "::group::Running tests ..."
         python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py \
-          --subid ${{inputs.AZUREML_TEST_SUBID}} \
-          --reponame "recommenders" \
-          --branch ${{ github.ref }} \
-          --rg ${{inputs.RG}} \
-          --wsname ${{inputs.WS}} \
-          --expname ${{inputs.EXP_NAME}}_${{inputs.TEST_GROUP}} \
-          --testlogs ${{inputs.TEST_LOGS_PATH}} \
-          --testkind ${{inputs.TEST_KIND}} \
-          --conda_pkg_python ${{inputs.PYTHON_VERSION}} \
-          --testgroup ${{inputs.TEST_GROUP}} \
-          --disable-warnings \
-          --sha "${GITHUB_SHA}" \
-          --clustername $(if [[ ${{inputs.TEST_GROUP}} =~ "gpu" ]]; then echo "${{inputs.GPU_CLUSTER_NAME}}"; else echo "${{inputs.CPU_CLUSTER_NAME}}"; fi) \
-          $(if [[ ${{inputs.TEST_GROUP}} =~ "gpu" ]]; then echo "--add_gpu_dependencies"; fi) \
-          $(if [[ ${{inputs.TEST_GROUP}} =~ "spark" ]]; then echo "--add_spark_dependencies"; fi)
-    - name: Get exit status
+          --subid ${{ inputs.AZUREML_TEST_SUBID }} \
+          --rg ${{ inputs.RG }} \
+          --ws ${{ inputs.WS }} \
+          --cluster ${{ contains(inputs.TEST_GROUP, 'gpu') && 'gpu-cluster' || 'cpu-cluster' }} \
+          --expname ${{ inputs.EXP_NAME }} \
+          --envname ${{ inputs.ENV_NAME }} \
+          --testkind ${{ inputs.TEST_KIND}} \
+          --python-version ${{ inputs.PYTHON_VERSION }} \
+          --testgroup ${{ inputs.TEST_GROUP }} \
+          --sha ${GITHUB_SHA}
+        echo "::endgroup::"
+    - name: Post tests
+      if: ${{ ! cancelled() }}
       shell: bash
-      id: exit_status
-      run: echo "code=$(cat ${{inputs.PYTEST_EXIT_CODE}})" >> $GITHUB_OUTPUT
-    - name: Check Success/Failure
-      if: ${{ steps.exit_status.outputs.code != 0 }}
-      uses: actions/github-script@v7
+      run: |
+        echo "::group::Pytest logs"
+        python tests/ci/azureml_tests/post_pytest.py \
+          --subid ${{ inputs.AZUREML_TEST_SUBID }} \
+          --rg ${{ inputs.RG }} \
+          --ws ${{ inputs.WS }} \
+          --expname ${{ inputs.EXP_NAME }} \
+          --log-dir ${{ inputs.LOG_DIR }}
+        echo "::endgroup::"
+    - name: Save logs
+      if: ${{ ! cancelled() }}
+      uses: actions/upload-artifact@v4
       with:
-        script: |
-            core.setFailed('All tests did not pass!')
+        name: logs-${{ inputs.TEST_GROUP }}-python${{ inputs.PYTHON_VERSION }}
+        path: ${{ inputs.LOG_DIR }}

.github/actions/get-test-groups/action.yml

@@ -6,18 +6,17 @@
 name: get-test-groups
 description: "Get test group names from tests_groups.py"
 inputs:
-  # type of test - unit or nightly
   TEST_KIND:
     required: true
-    type: string
-  # test environment - cpu, gpu or spark
+    description: Type of test - pr gate or nightly
   TEST_ENV:
     required: false
-    type: string
+    description: Test environment - cpu, gpu or spark
     default: 'cpu'
 outputs:
   test_groups:
-    value: ${{steps.get_test_groups.outputs.test_groups}}
+    description: A list of test groups
+    value: ${{ steps.get_test_groups.outputs.test_groups }}
 
 runs:
   using: "composite"