From 382a2b73111ce9254fbc656ee6633963c29aef11 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 11 Feb 2025 10:10:23 +0530 Subject: [PATCH 1/9] Update test-mlc-core-actions.yaml --- .github/workflows/test-mlc-core-actions.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test-mlc-core-actions.yaml b/.github/workflows/test-mlc-core-actions.yaml index 543cbe22e..85aee2c7c 100644 --- a/.github/workflows/test-mlc-core-actions.yaml +++ b/.github/workflows/test-mlc-core-actions.yaml @@ -186,3 +186,8 @@ jobs: mlc rm script --tags=app,image,corner-detection -f mlc rm script 63080407db4d4ac4 -f + - name: Test 21 - Test autoremoval of corrupt entries from repos.json while running a script + run: | + rm -r $HOME/MLC/repos/mlcommons@mlperf-automations + mlc run script --tags=detect,os + mlc pull repo mlcommons@mlperf-automations From 442f4b198171c981f5b9ad3427af72fb4c9d18b1 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 11 Feb 2025 10:13:13 +0530 Subject: [PATCH 2/9] handle corrupt repo entries on runtime --- mlc/main.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/mlc/main.py b/mlc/main.py index b36c441eb..79d4fcc76 100644 --- a/mlc/main.py +++ b/mlc/main.py @@ -148,7 +148,17 @@ def is_curdir_inside_path(base_path): # Iterate through the list of repository paths for repo_path in repo_paths: if not os.path.exists(repo_path): - logger.warning(f"""Warning: {repo_path} not found. Consider doing `mlc rm repo {repo_path}`. Skipping...""") + logger.warning(f"""Warning: {repo_path} not found. Considering it as a corrupt entry and deleting automatically...""") + logger.warning(f"Deleting the {meta_yaml_path} entry from repos.json") + res = self.access( + { + "automation": "repo", + "action": "rm", + "repo": f"{os.path.basename(repo_path)}" + } + ) + if res["return"] > 0: + return res continue if is_curdir_inside_path(repo_path): @@ -163,17 +173,17 @@ def is_curdir_inside_path(base_path): # Check if meta.yaml exists if not os.path.isfile(meta_yaml_path): - logger.warning(f"{meta_yaml_path} not found. Skipping...") - logger.warning(f"Deleting the {meta_yaml_path} entry from repos.json") - res = self.access( - { - "automation": "repo", - "action": "rm", - "repo": f"{os.path.basename(repo_path)}" - } - ) - if res["return"] > 0: - return res + logger.warning(f"{meta_yaml_path} not found. Is it a MLC repo? Skipping...") + # logger.warning(f"Deleting the {meta_yaml_path} entry from repos.json") + # res = self.access( + # { + # "automation": "repo", + # "action": "rm", + # "repo": f"{os.path.basename(repo_path)}" + # } + # ) + # if res["return"] > 0: + # return res continue # Load the YAML file From fe20556c13e1ab291849e6848fb15af383cabc00 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Tue, 11 Feb 2025 10:15:59 +0530 Subject: [PATCH 3/9] Update main.py --- mlc/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlc/main.py b/mlc/main.py index 79d4fcc76..c83a7fa80 100644 --- a/mlc/main.py +++ b/mlc/main.py @@ -173,7 +173,7 @@ def is_curdir_inside_path(base_path): # Check if meta.yaml exists if not os.path.isfile(meta_yaml_path): - logger.warning(f"{meta_yaml_path} not found. Is it a MLC repo? Skipping...") + logger.warning(f"{meta_yaml_path} not found. Could be due to accidental deletion of meta.yaml. Try to stash the changes or reclone by doing `rm repo` and `pull repo`. Skipping...") # logger.warning(f"Deleting the {meta_yaml_path} entry from repos.json") # res = self.access( # { From 0eee8848c0ce64596a7201e2f29a939728591bc5 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 12 Feb 2025 11:16:11 +0530 Subject: [PATCH 4/9] Update test 21 --- .github/workflows/test-mlc-core-actions.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-mlc-core-actions.yaml b/.github/workflows/test-mlc-core-actions.yaml index 89190f7b8..09da76009 100644 --- a/.github/workflows/test-mlc-core-actions.yaml +++ b/.github/workflows/test-mlc-core-actions.yaml @@ -197,6 +197,7 @@ jobs: - name: Test 21 - Test autoremoval of corrupt entries from repos.json while running a script run: | - rm -r $HOME/MLC/repos/mlcommons@mlperf-automations + apt-get install -y jq + TMP_VAR=$HOME/MLC/repos/anandhu@mlperf-automations + jq ". += [\"$TMP_VAR\"]" ${HOME}/MLC/repos/repos.json > tmp.json && mv tmp.json ${HOME}/MLC/repos/repos.json mlc run script --tags=detect,os - mlc pull repo mlcommons@mlperf-automations From daf6f96db373ea4720f1603777256c310ad18d8c Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 12 Feb 2025 11:18:54 +0530 Subject: [PATCH 5/9] Update test-mlc-core-actions.yaml --- .github/workflows/test-mlc-core-actions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlc-core-actions.yaml b/.github/workflows/test-mlc-core-actions.yaml index 09da76009..a374ea138 100644 --- a/.github/workflows/test-mlc-core-actions.yaml +++ b/.github/workflows/test-mlc-core-actions.yaml @@ -197,7 +197,7 @@ jobs: - name: Test 21 - Test autoremoval of corrupt entries from repos.json while running a script run: | - apt-get install -y jq + sudo apt-get install -y jq TMP_VAR=$HOME/MLC/repos/anandhu@mlperf-automations jq ". += [\"$TMP_VAR\"]" ${HOME}/MLC/repos/repos.json > tmp.json && mv tmp.json ${HOME}/MLC/repos/repos.json mlc run script --tags=detect,os From 7b8b56e7970881b4bb4850053700ca4692bf8b94 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 12 Feb 2025 11:27:41 +0530 Subject: [PATCH 6/9] updated test 10 --- .github/workflows/test-mlc-core-actions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlc-core-actions.yaml b/.github/workflows/test-mlc-core-actions.yaml index a374ea138..2a0d9afb6 100644 --- a/.github/workflows/test-mlc-core-actions.yaml +++ b/.github/workflows/test-mlc-core-actions.yaml @@ -136,7 +136,7 @@ jobs: - name: Test 10 - rm cache - More than one cache present run: | - mlc rm cache --tags=get,imagenet-aux 2>&1 --all | tee test7.log + mlc rm cache --tags=get,imagenet-aux -f 2>&1 --all | tee test7.log - name: Test 11 - cp script - Copy mlc script run: | From fa0af0b82f156bc8552f04e3bb30c4e39778d2e3 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Wed, 12 Feb 2025 11:29:40 +0530 Subject: [PATCH 7/9] Update test-mlc-core-actions.yaml --- .github/workflows/test-mlc-core-actions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mlc-core-actions.yaml b/.github/workflows/test-mlc-core-actions.yaml index 2a0d9afb6..3ca011050 100644 --- a/.github/workflows/test-mlc-core-actions.yaml +++ b/.github/workflows/test-mlc-core-actions.yaml @@ -136,7 +136,7 @@ jobs: - name: Test 10 - rm cache - More than one cache present run: | - mlc rm cache --tags=get,imagenet-aux -f 2>&1 --all | tee test7.log + mlc rm cache --tags=get,imagenet-aux -f - name: Test 11 - cp script - Copy mlc script run: | From 5452be3ccdcaae2c62f89c40217367639c8775b2 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 13 Feb 2025 20:16:35 +0530 Subject: [PATCH 8/9] Update test-mlc-core-actions.yaml --- .github/workflows/test-mlc-core-actions.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/test-mlc-core-actions.yaml b/.github/workflows/test-mlc-core-actions.yaml index 3ca011050..d190409ed 100644 --- a/.github/workflows/test-mlc-core-actions.yaml +++ b/.github/workflows/test-mlc-core-actions.yaml @@ -193,11 +193,3 @@ jobs: - name: Test 20 - list cache run: | mlc list cache - - - - name: Test 21 - Test autoremoval of corrupt entries from repos.json while running a script - run: | - sudo apt-get install -y jq - TMP_VAR=$HOME/MLC/repos/anandhu@mlperf-automations - jq ". += [\"$TMP_VAR\"]" ${HOME}/MLC/repos/repos.json > tmp.json && mv tmp.json ${HOME}/MLC/repos/repos.json - mlc run script --tags=detect,os From fde809e721ea72395e56886ccae8151da5ae59b9 Mon Sep 17 00:00:00 2001 From: ANANDHU S <71482562+anandhu-eng@users.noreply.github.com> Date: Thu, 13 Feb 2025 20:17:00 +0530 Subject: [PATCH 9/9] Update main.py --- mlc/main.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/mlc/main.py b/mlc/main.py index 3b32b7d06..c6b41de34 100644 --- a/mlc/main.py +++ b/mlc/main.py @@ -168,16 +168,6 @@ def is_curdir_inside_path(base_path): # Check if meta.yaml exists if not os.path.isfile(meta_yaml_path): logger.warning(f"{meta_yaml_path} not found. Could be due to accidental deletion of meta.yaml. Try to stash the changes or reclone by doing `rm repo` and `pull repo`. Skipping...") - # logger.warning(f"Deleting the {meta_yaml_path} entry from repos.json") - # res = self.access( - # { - # "automation": "repo", - # "action": "rm", - # "repo": f"{os.path.basename(repo_path)}" - # } - # ) - # if res["return"] > 0: - # return res continue # Load the YAML file