azure infracost solution more

Trolley-MGMT · Jul 19, 2023 · 24ccecf · 24ccecf
1 parent ba31924
commit 24ccecf
Show file tree

Hide file tree

Showing 9 changed files with 161 additions and 90 deletions.
diff --git a/.github/workflows/az_caching_action.yml b/.github/workflows/az_caching_action.yml
@@ -5,7 +5,7 @@ on:
 
 jobs:
   az_caching:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     permissions:
       contents: read
       pages: write
@@ -20,42 +20,55 @@ jobs:
         with:
           python-version: 3.9
 
+      - name: Checkout infracost pricing repo
+        uses: actions/checkout@v3
+        with:
+          repository: infracost/cloud-pricing-api
+          path: cloud-pricing-api
+
       - name: Echo the values
         run: |
           echo "client_payload is: ${{ github.event.client_payload.client_payload }}"
           echo "azure_credentials is: ${{ github.event.client_payload.azure_credentials }}"            
-          echo "project_name is: ${{ github.event.client_payload.project_name }}"
           echo "client_id is: ${{ github.event.client_payload.client_id }}"
           echo "client_secret is: ${{ github.event.client_payload.client_secret }}"
           echo "tenant_id is: ${{ github.event.client_payload.tenant_id }}"
           echo "infracost_token is: ${{ github.event.client_payload.infracost_token }}"
+          echo "project_name is: ${{ github.event.client_payload.project_name }}"
 
       - name: Az CLI login
         uses: Azure/login@v1
         env:
-          CREDENTIALS:  ${{ github.event.client_payload.azure_credentials }}
+          CREDENTIALS: ${{ github.event.client_payload.azure_credentials }}
         with:
           creds: ${{ env.CREDENTIALS }}
 
+      - name: Installing Infracost
+        if: always()
+        run: |
+          cd cloud-pricing-api
+          export SELF_HOSTED_INFRACOST_API_KEY="${{ github.event.client_payload.infracost_token }}"
+          export INFRACOST_API_KEY="${{ github.event.client_payload.infracost_token }}"
+          docker-compose run init_job
 
       - name: Caching Tasks
-        env:
-          MONGO_URL: ${{ github.event.client_payload.mongo_url }}
-          MONGO_USER: ${{ github.event.client_payload.mongo_user }}
-          MONGO_PASSWORD: ${{ github.event.client_payload.mongo_password }}
-          PROJECT_NAME: ${{ github.event.client_payload.project_name }}
-          AZURE_CLIENT_ID: ${{ github.event.client_payload.client_id }}
-          AZURE_CLIENT_SECRET: ${{ github.event.client_payload.client_secret }}
-          AZURE_TENANT_ID: ${{ github.event.client_payload.tenant_id }}
-          INFRACOST_TOKEN: ${{ github.event.client_payload.infracost_token }}"
-          INFRACOST_URL: http://localhost:4000
-          GITHUB_ACTIONS_ENV: Yes
-
+        if: always()
         run: |
-          /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
-          (echo; echo 'eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"') >> /home/runner/.bash_profile
-          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
-          brew install az
+          cd cloud-pricing-api
+          export SELF_HOSTED_INFRACOST_API_KEY="${{ github.event.client_payload.infracost_token }}"
+          export INFRACOST_API_KEY="${{ github.event.client_payload.infracost_token }}"
+          docker-compose up -d api
+          curl --retry 20 --retry-all-errors http://localhost:4000/health
+          export MONGO_URL=${{ github.event.client_payload.mongo_url }}
+          export MONGO_USER=${{ github.event.client_payload.mongo_user }}
+          export MONGO_PASSWORD=${{ github.event.client_payload.mongo_password }}
+          export PROJECT_NAME=${{ github.event.client_payload.project_name }}
+          export INFRACOST_TOKEN=${{ github.event.client_payload.infracost_token }}
+          export INFRACOST_URL=http://localhost:4000
+          export GITHUB_ACTIONS_ENV=Yes
           export PYTHONPATH=$PYTHONPATH:.
+          cd ..
           pip3 install -r web/requirements.txt
-          python3 web/scripts/az_caching_script.py
+          python3 web/scripts/az_caching_script.py
+          
+
diff --git a/.github/workflows/az_caching_action_.yml b/.github/workflows/az_caching_action_.yml
@@ -0,0 +1,61 @@
+name: AZ Caching
+on:
+  repository_dispatch:
+    types: az-caching-action-trigger
+
+jobs:
+  az_caching:
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      pages: write
+      id-token: write
+    steps:
+      - uses: 'actions/checkout@v3'
+
+      - name: Checkout Code
+        uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Echo the values
+        run: |
+          echo "client_payload is: ${{ github.event.client_payload.client_payload }}"
+          echo "azure_credentials is: ${{ github.event.client_payload.azure_credentials }}"            
+          echo "project_name is: ${{ github.event.client_payload.project_name }}"
+          echo "client_id is: ${{ github.event.client_payload.client_id }}"
+          echo "client_secret is: ${{ github.event.client_payload.client_secret }}"
+          echo "tenant_id is: ${{ github.event.client_payload.tenant_id }}"
+          echo "infracost_token is: ${{ github.event.client_payload.infracost_token }}"
+
+      - name: Az CLI login
+        uses: Azure/login@v1
+        env:
+          CREDENTIALS:  ${{ github.event.client_payload.azure_credentials }}
+        with:
+          creds: ${{ env.CREDENTIALS }}
+
+
+      - name: Caching Tasks
+        env:
+          MONGO_URL: ${{ github.event.client_payload.mongo_url }}
+          MONGO_USER: ${{ github.event.client_payload.mongo_user }}
+          MONGO_PASSWORD: ${{ github.event.client_payload.mongo_password }}
+          PROJECT_NAME: ${{ github.event.client_payload.project_name }}
+          AZURE_CLIENT_ID: ${{ github.event.client_payload.client_id }}
+          AZURE_CLIENT_SECRET: ${{ github.event.client_payload.client_secret }}
+          AZURE_TENANT_ID: ${{ github.event.client_payload.tenant_id }}
+          INFRACOST_TOKEN: ${{ github.event.client_payload.infracost_token }}"
+          INFRACOST_URL: http://localhost:4000
+          GITHUB_ACTIONS_ENV: Yes
+
+        run: |
+          /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
+          (echo; echo 'eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"') >> /home/runner/.bash_profile
+          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
+          brew install az
+          export PYTHONPATH=$PYTHONPATH:.
+          pip3 install -r web/requirements.txt
+          python3 web/scripts/az_caching_script.py
diff --git a/web/main.py b/web/main.py
@@ -339,6 +339,17 @@ def az_caching(user_email: str, project_name: str, azure_credentials: str, githu
         except Exception as e:
             logger.error(f'problem decrypting github_actions_token_decrypted with error {e}')
             return False
+    if not infracost_token:
+        infracost_token_data = mongo_handler.mongo_utils.retrieve_infracost_data_object(user_email)
+        if not infracost_token_data:
+            logger.warning(f'No infracost data for {user_email} user was found. GCP caching will not start')
+            return True
+        try:
+            infracost_token_decrypted = crypter.decrypt(infracost_token_data['infracost_token']).decode("utf-8")
+            content['infracost_token'] = infracost_token_decrypted
+        except Exception as e:
+            logger.error(f'problem decrypting infracost_token_decrypted with error {e}')
+            return False
     cluster_operation = ClusterOperation(**content)
     if cluster_operation.trigger_az_caching():
         return True
@@ -877,6 +888,7 @@ def fetch_regions():
     logger.info(f'A request to fetch regions for {cluster_type} has arrived')
     if cluster_type == AKS:
         regions = mongo_handler.mongo_utils.retrieve_cache(cache_type=LOCATIONS_DICT, provider=AKS)
+        return jsonify(regions), 200
     elif cluster_type == GKE:
         regions = mongo_handler.mongo_utils.retrieve_cache(cache_type=REGIONS_LIST, provider=GKE)
     elif cluster_type == EKS:
@@ -890,7 +902,7 @@ def fetch_regions():
 
 
 @app.route('/fetch_kubernetes_versions', methods=[GET])
-# @login_required
+@login_required
 def fetch_kubernetes_versions():
     cluster_type = request.args.get(CLUSTER_TYPE.lower())
     location_name = request.args.get(LOCATION_NAME.lower())
@@ -920,7 +932,6 @@ def fetch_machine_types():
     cluster_type = request.args.get(CLUSTER_TYPE)
     machine_series = request.args.get(MACHINE_SERIES.lower())
     region_name = request.args.get(REGION_NAME.lower())
-
     logger.info(f'A request to fetch machine types for {cluster_type} has arrived')
     machine_types = mongo_handler.mongo_utils.retrieve_machine_types(machine_series=machine_series,
                                                                      cluster_type=cluster_type, region_name=region_name)

diff --git a/web/mongo_handler/mongo_utils.py b/web/mongo_handler/mongo_utils.py
@@ -105,6 +105,7 @@
 teams: Collection = db.teams
 users: Collection = db.users
 deployment_yamls: Collection = db.deployment_yamls
+
 az_locations_cache: Collection = db.az_locations_cache
 az_resource_groups_cache: Collection = db.az_resource_groups_cache
 az_machines_cache: Collection = db.az_machines_cache
@@ -114,7 +115,6 @@
 
 gke_cache: Collection = db.gke_cache
 gke_machines_cache: Collection = db.gke_machines_cache
-gke_machines_types_cache: Collection = db.gke_machines_types_cache
 gke_machines_series_cache: Collection = db.gke_machines_series_cache
 gke_zones_and_series_cache: Collection = db.gke_zones_and_series_cache
 gke_kubernetes_versions_cache: Collection = db.gke_kubernetes_versions_cache
@@ -123,8 +123,8 @@
 aws_machines_cache: Collection = db.aws_machines_cache
 aws_regions_and_series_cache: Collection = db.aws_regions_and_series_cache
 aws_series_and_machine_types_cache: Collection = db.aws_series_and_machine_types_cache
-fs = gridfs.GridFS(db)
 
+fs = gridfs.GridFS(db)
 k8s_agent_data: Collection = db.k8s_agent_data
 
 providers_data: Collection = db.providers_data
@@ -688,7 +688,7 @@ def retrieve_machine_series(region_name: str = '', cluster_type: str = '') -> li
         machine_series_object = aws_machines_cache.find_one(mongo_query)
     elif cluster_type == AKS:
         mongo_query = {'location_name': region_name}
-        machine_series_object = az_locations_and_series_cache.find_one(mongo_query)
+        machine_series_object = az_machines_cache.find_one(mongo_query)
     else:
         mongo_query = {'zone': region_name}
         machine_series_object = gke_zones_and_series_cache.find_one(mongo_query)
@@ -714,17 +714,19 @@ def retrieve_machine_types(machine_series: str = '', region_name: str = '', clus
         mongo_query = {'region': region_name}
         machine_types_object = aws_machines_cache.find_one(mongo_query)
     elif cluster_type == AKS:
-        mongo_query = {'machine_series': machine_series}
-        machine_types_object = az_series_and_machine_types_cache.find_one(mongo_query)
+        mongo_query = {'location_name': region_name}
+        machine_types_object = az_machines_cache.find_one(mongo_query)
     else:
         mongo_query = {'region': region_name}
         machine_types_object = gke_machines_cache.find_one(mongo_query)
     try:
-        for machine_type in machine_types_object['machines_list']:
-            if machine_type['machine_series'] == machine_series:
-                machine_types_response.append(machine_type)
+        for machine in machine_types_object['machines_list']:
+            if machine['machine_series'] == machine_series:
+                machine['machine_type'] = machine['machine_type'].replace('Standard_', '')
+                machine_types_response.append(machine)
         return machine_types_response
-    except:
+    except Exception as e:
+        logger.error(f'There was some problem here with: {e}')
         return []
 
 
@@ -1559,6 +1561,7 @@ def retrieve_infracost_data_object(user_email: str = '') -> dict:
     del infracost_data_object['_id']
     return infracost_data_object
 
+
 def retrieve_credentials_data_object(provider: str, user_email: str) -> dict:
     """
     """

diff --git a/web/scripts/az_caching_script.py b/web/scripts/az_caching_script.py
@@ -197,9 +197,9 @@ def main():
     # Inserting a AZ Zones and Machine Series Object
     series_list = []
     for location in machine_types_all_locations:
-        for machine_type in machine_types_all_locations[location]:
-            if not machine_type.machine_series in series_list:
-                series_list.append(machine_type.machine_series)
+        for machine in machine_types_all_locations[location]:
+            if not machine['machine_series'] in series_list:
+                series_list.append(machine['machine_series'])
 
         az_zones_and_machine_series_object = AZZonesAndMachineSeriesObject(
             location_name=location,
@@ -211,21 +211,21 @@ def main():
     # Inserting a AZ Machine Series and Machine Types Object
     machines_series_list = []
     for location_name in machine_types_all_locations:
-        for machine_type in machine_types_all_locations[location_name]:
-            if machine_type.machine_series not in machines_series_list:
-                machines_series_list.append(machine_type.machine_series)
+        for machine in machine_types_all_locations[location_name]:
+            if machine['machine_series'] not in machines_series_list:
+                machines_series_list.append(machine['machine_series'])
 
     series_and_machine_types_dict = {}
     for location_name in machine_types_all_locations:
         for machine_series in machines_series_list:
-            for machine_type in machine_types_all_locations[location_name]:
+            for machine in machine_types_all_locations[location_name]:
                 if machine_series not in series_and_machine_types_dict.keys():
-                    if machine_type.machine_series == machine_series:
-                        series_and_machine_types_dict[machine_series] = [machine_type.machine_type]
+                    if machine['machine_series'] == machine_series:
+                        series_and_machine_types_dict[machine_series] = [machine['machine_series']]
                 else:
-                    if machine_type.machine_series == machine_series:
-                        if machine_type.machine_type not in series_and_machine_types_dict[machine_series]:
-                            series_and_machine_types_dict[machine_series].append(machine_type.machine_type)
+                    if machine['machine_series'] == machine_series:
+                        if machine['machine_series'] not in series_and_machine_types_dict[machine_series]:
+                            series_and_machine_types_dict[machine_series].append(machine['machine_type'])
     for machine_series in series_and_machine_types_dict:
         az_series_and_machine_types_object = AZSeriesAndMachineTypesObject(
             machine_series=machine_series,