Merge pull request #36 from huineng/gh

chore: test python

.github/workflows/diem-default.yml

@@ -7,14 +7,14 @@ on:
   # Triggers the workflow on push or pull request events but only for the main branch
   push:
     branches: [main]
-    paths-ignore:
-      - "applications/**"
-      - "packages/**"
+    paths:
+      - "*"
+      - ".github/**"
   pull_request:
     branches: [main]
-    paths-ignore:
-      - "applications/**"
-      - "packages/**"
+    paths:
+      - "*"
+      - ".github/**"
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:

.github/workflows/diem-diemlib.yml

@@ -0,0 +1,62 @@
+# This is a basic workflow to help you get started with Actions
+
+name: CI diem-lib
+
+# Controls when the action will run.
+on:
+  # Triggers the workflow on push or pull request events but only for the main branch
+  push:
+    branches: [main]
+    paths:
+      - "packages/diem-lib/**"
+  pull_request:
+    branches: [main]
+    paths:
+      - "packages/diem-lib/**"
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+defaults:
+  run:
+    working-directory: packages/diem-lib
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest wheel
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+      - name: Build diemlib
+        run: python3 setup.py clean --all && python3 setup.py bdist_wheel
+
+      - name: publish diemlib
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: twine upload dist/*
+        if: github.event_name == 'push' && ( github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' )

.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+  "python.pythonPath": "/usr/bin/python3",
+  "python.linting.enabled": true,
+  "python.linting.pylintEnabled": false,
+  "python.linting.flake8Enabled": true,
+  "python.linting.flake8Args": ["--max-line-length=120"]
+}

packages/diem-lib/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2018 The Python Packaging Authority
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

packages/diem-lib/diemlib.egg-info/PKG-INFO

@@ -0,0 +1,32 @@
+Metadata-Version: 2.1
+Name: diemlib
+Version: 0.0.8
+Summary: Python Utilities for DIEM
+Home-page: https://github.com/IBM/diem/tree/main/packages/diemlib
+Author: Guy Huinen
+Author-email: [email protected]
+License: MIT
+Description: # Python utilities for DIEM
+
+        Diem is an IBM ETL Package. It contains functionality to work with DIEM Jobs
+
+        - main
+          - General logging
+          - Error handling
+          - Communication with Diem Core
+        - cos
+          - Uploading to cloud object storage
+          - Downloading from cloud object storage
+        - mail: sending customized mail
+        - box
+          - downloading from box
+          - uploading to box
+        - util: several small utilities
+        - config: config variables used by jobs
+
+Platform: UNKNOWN
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown

packages/diem-lib/diemlib.egg-info/SOURCES.txt

@@ -0,0 +1,13 @@
+setup.cfg
+setup.py
+diemlib/__init__.py
+diemlib/box.py
+diemlib/config.py
+diemlib/filehandler.py
+diemlib/mailhandler.py
+diemlib/main.py
+diemlib/util.py
+diemlib.egg-info/PKG-INFO
+diemlib.egg-info/SOURCES.txt
+diemlib.egg-info/dependency_links.txt
+diemlib.egg-info/top_level.txt

packages/diem-lib/diemlib.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

packages/diem-lib/diemlib.egg-info/top_level.txt

@@ -0,0 +1 @@
+diemlib

packages/diem-lib/diemlib/__init__.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""Module documentation goes here
+   Diem Lib is a set of utilities that are used by the DIEM Application.
+   It includes
+      - Main functionality for interaction with the main application
+      - Mail Functionality
+      - General Logging
+      - File downloads Cloud Object Storage
+"""

packages/diem-lib/diemlib/box.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# diemlib/box.py
+
+"""
+   These are specific functions to be used for handling files
+   The most important once are getFile and saveFile
+"""
+
+from boxsdk import JWTAuth, Client
+from io import StringIO
+from pathlib import Path
+import pandas as pd
+from diemlib.main import error
+
+
+class Box(object):
+    def __init__(self, auth):
+        config = JWTAuth.from_settings_dictionary(auth)
+        self.client = Client(config)
+
+    def readFile(self, file, **kwargs):
+        # gets a file from box
+
+        try:
+
+            file_type = Path(file).suffix
+
+            file_content = self.client.file(file).content()
+            body = StringIO(str(file_content, "utf-8"))
+
+            if file_type == ".pickle":
+                df = pd.read_pickle(body)
+                return df
+            elif file_type == ".parquet":
+                df = pd.read_parquet(body, **kwargs)
+                return df
+            elif file_type == ".csv":
+                df = pd.read_csv(body, encoding="utf8", **kwargs)
+                return df
+            else:
+                df = pd.read_csv(body, encoding="utf8", **kwargs)
+            return df
+
+        except Exception as e:
+            error(e)
+            raise
+
+    def saveFile(self, df, file, folder=0, **kwargs):
+        # saves a file to box
+
+        try:
+
+            file_type = Path(file).suffix
+
+            body = StringIO()
+
+            if file_type == ".pickle":
+                df.to_pickle(body)
+            elif file_type == ".parquet":
+                df.to_parquet(body, **kwargs)
+            elif file_type == ".csv":
+                df.to_csv(body, encoding="utf8", **kwargs)
+            else:
+                pd.to_csv(body, encoding="utf8", **kwargs)
+
+            self.client.folder(folder).upload_stream(body, file)
+            return file
+
+        except Exception as e:
+            error(e)
+            raise

packages/diem-lib/diemlib/config.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# diemlib/config.py
+
+"""
+   This module contains general variables that can be used
+   througout the complete application
+"""
+
+__count = 0
+__email = None
+__filepath = None
+__id = None
+__jobid = None
+__jobstart: None
+__name = None
+__org: None
+__starttime: None
+__transid = None
+__url = None
+__isservice = False