AJ's work on for data library

.coveragerc

@@ -27,4 +27,4 @@ exclude_also =
     if __name__ == .__main__.:
 
     ; Don't complain about abstract methods, they aren't run:
-    @(abc\.)?abstractmethod
+    @(abc\.)?abstractmethod

.github/workflows/ci.yml

@@ -13,16 +13,16 @@ jobs:
     name: Security Pipeline
     uses: uc-cdis/.github/.github/workflows/securitypipeline.yaml@master
     with:
-       python-poetry: 'true'
+      python-poetry: 'true'
     secrets: inherit
-
-  UnitTest:
-    name: Python Unit Test with Postgres
-    uses: uc-cdis/.github/.github/workflows/python_unit_test.yaml@master
-    with:
-       test-script: 'test.sh'
-       python-version: '3.9'
-       use-cache: true
+# TODO: Uncomment after repo is public
+#  UnitTest:
+#    name: Python Unit Test with Postgres
+#    uses: uc-cdis/.github/.github/workflows/python_unit_test.yaml@master
+#    with:
+#      test-script: 'test.sh'
+#      python-version: '3.9'
+#      use-cache: true
 
   # this creates linter settings and uploads to an artifact so the configs can be pulled and used across jobs
   LintConfig:
@@ -31,50 +31,53 @@ jobs:
     with:
       python-module-name: "gen3userdatalibrary"
 
-#  # (optional) modify the linter configurations from above. You could omit this if you didn't need to do this
-#  CustomizeLintConfig:
-#    runs-on: ubuntu-latest
-#    name: Customize Lint Config
-#    needs: [LintConfig]
-#    steps:
-#      - uses: actions/download-artifact@v3
-#        with:
-#          # this is uploaded by the lint-create-config.yaml workflow
-#          name: linters-config
-#          path: .github/linters
-#
-#      # modify default isort to specify the module name for proper formatting
-#      - run: echo "known_first_party=gen3userdatalibrary" >> .github/linters/.isort.cfg
-#
-#      # now we need to re-upload the artifacts with the changes
-#      - uses: actions/upload-artifact@v3
-#        with:
-#          name: linters-config
-#          path: |
-#            .github/linters/
-#          if-no-files-found: error
+  #  # (optional) modify the linter configurations from above. You could omit this if you didn't need to do this
+  #  CustomizeLintConfig:
+  #    runs-on: ubuntu-latest
+  #    name: Customize Lint Config
+  #    needs: [LintConfig]
+  #    steps:
+  #      - uses: actions/download-artifact@v3
+  #        with:
+  #          # this is uploaded by the lint-create-config.yaml workflow
+  #          name: linters-config
+  #          path: .github/linters
+  #
+  #      # modify default isort to specify the module name for proper formatting
+  #      - run: echo "known_first_party=gen3userdatalibrary" >> .github/linters/.isort.cfg
+  #
+  #      # now we need to re-upload the artifacts with the changes
+  #      - uses: actions/upload-artifact@v3
+  #        with:
+  #          name: linters-config
+  #          path: |
+  #            .github/linters/
+  #          if-no-files-found: error
+# TODO: Uncomment after repo is public
 
-  RequiredLint:
-    name: Run Required Linters
-    needs: [LintConfig]
-    uses: uc-cdis/.github/.github/workflows/required_lint_check.yaml@master
-    with:
-      python-version: '3.9'
-      use-cache: true
-
-  InformationalLint:
-    name: Run Informational Linters
-    needs: [LintConfig, UnitTest]
-    if: github.ref != 'refs/heads/main'
-    uses: uc-cdis/.github/.github/workflows/optional_lint_check.yaml@master
-    with:
-      python-version: '3.9'
-      use-cache: true
+# TODO: Uncomment after repo is public, this was having some import issues, check out the "fix/lint-path" for some ideas
+#       https://github.com/uc-cdis/.github/compare/master...fix/lint-path
+  # RequiredLint:
+  #   name: Run Required Linters
+  #   needs: [ LintConfig ]
+  #   uses: uc-cdis/.github/.github/workflows/required_lint_check.yaml@master
+  #   with:
+  #     python-version: '3.9'
+  #     use-cache: true
+# TODO: Uncomment after repo is public
+#  InformationalLint:
+#    name: Run Informational Linters
+#    needs: [ LintConfig ] #TODO Add UnitTest
+#    if: github.ref != 'refs/heads/main'
+#    uses: uc-cdis/.github/.github/workflows/optional_lint_check.yaml@master
+#    with:
+#      python-version: '3.9'
+#      use-cache: true
 
   ImageBuildAndPush:
     name: Build Image and Push
     uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
-    needs: [RequiredLint, Security, UnitTest]
+    needs: [ Security ] #TODO Add UnitTest RequiredLint
     with:
       BUILD_PLATFORMS: "linux/amd64"
     secrets:

.gitignore

@@ -134,7 +134,6 @@ celerybeat.pid
 # Environments
 .env
 .venv
-env/
 venv/
 ENV/
 env.bak/

.pre-commit-config.yaml

@@ -1,18 +1,18 @@
 repos:
--   repo: [email protected]:Yelp/detect-secrets
-    rev: v1.4.0
+  - repo: [email protected]:Yelp/detect-secrets
+    rev: v1.5.0
     hooks:
-    -   id: detect-secrets
-        args: ['--baseline', '.secrets.baseline']
+      - id: detect-secrets
+        args: [ '--baseline', '.secrets.baseline' ]
         exclude: poetry.lock
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.5.0
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
     hooks:
-    -   id: trailing-whitespace
-    -   id: end-of-file-fixer
-    -   id: no-commit-to-branch
-        args: [--branch, develop, --branch, master, --pattern, release/.*]
--   repo: https://github.com/psf/black
-    rev: 22.3.0
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: no-commit-to-branch
+        args: [ --branch, develop, --branch, master, --pattern, release/.* ]
+  - repo: https://github.com/psf/black
+    rev: 24.10.0
     hooks:
-    -   id: black
+      - id: black

.secrets.baseline

@@ -0,0 +1,175 @@
+{
+  "version": "1.5.0",
+  "plugins_used": [
+    {
+      "name": "ArtifactoryDetector"
+    },
+    {
+      "name": "AWSKeyDetector"
+    },
+    {
+      "name": "AzureStorageKeyDetector"
+    },
+    {
+      "name": "Base64HighEntropyString",
+      "limit": 4.5
+    },
+    {
+      "name": "BasicAuthDetector"
+    },
+    {
+      "name": "CloudantDetector"
+    },
+    {
+      "name": "DiscordBotTokenDetector"
+    },
+    {
+      "name": "GitHubTokenDetector"
+    },
+    {
+      "name": "GitLabTokenDetector"
+    },
+    {
+      "name": "HexHighEntropyString",
+      "limit": 3.0
+    },
+    {
+      "name": "IbmCloudIamDetector"
+    },
+    {
+      "name": "IbmCosHmacDetector"
+    },
+    {
+      "name": "IPPublicDetector"
+    },
+    {
+      "name": "JwtTokenDetector"
+    },
+    {
+      "name": "KeywordDetector",
+      "keyword_exclude": ""
+    },
+    {
+      "name": "MailchimpDetector"
+    },
+    {
+      "name": "NpmDetector"
+    },
+    {
+      "name": "OpenAIDetector"
+    },
+    {
+      "name": "PrivateKeyDetector"
+    },
+    {
+      "name": "PypiTokenDetector"
+    },
+    {
+      "name": "SendGridDetector"
+    },
+    {
+      "name": "SlackDetector"
+    },
+    {
+      "name": "SoftlayerDetector"
+    },
+    {
+      "name": "SquareOAuthDetector"
+    },
+    {
+      "name": "StripeDetector"
+    },
+    {
+      "name": "TelegramBotTokenDetector"
+    },
+    {
+      "name": "TwilioKeyDetector"
+    }
+  ],
+  "filters_used": [
+    {
+      "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_baseline_file",
+      "filename": ".secrets.baseline"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
+      "min_level": 2
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_indirect_reference"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_likely_id_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_lock_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_potential_uuid"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_sequential_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_swagger_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_templated_secret"
+    }
+  ],
+  "results": {
+    ".github/workflows/ci.yml": [
+      {
+        "type": "Secret Keyword",
+        "filename": ".github/workflows/ci.yml",
+        "hashed_secret": "3e26d6750975d678acb8fa35a0f69237881576b0",
+        "is_verified": false,
+        "line_number": 17
+      }
+    ],
+    "README.md": [
+      {
+        "type": "Basic Auth Credentials",
+        "filename": "README.md",
+        "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3",
+        "is_verified": false,
+        "line_number": 77
+      }
+    ],
+    "gen3userdatalibrary/config.py": [
+      {
+        "type": "Basic Auth Credentials",
+        "filename": "gen3userdatalibrary/config.py",
+        "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
+        "is_verified": false,
+        "line_number": 34
+      },
+      {
+        "type": "Basic Auth Credentials",
+        "filename": "gen3userdatalibrary/config.py",
+        "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3",
+        "is_verified": false,
+        "line_number": 38
+      }
+    ],
+    "tests/.env": [
+      {
+        "type": "Basic Auth Credentials",
+        "filename": "tests/.env",
+        "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3",
+        "is_verified": false,
+        "line_number": 4
+      }
+    ]
+  },
+  "generated_at": "2024-12-03T20:49:50Z"
+}