Skip to content

Commit

Permalink
Added github sync feature (#120)
Browse files Browse the repository at this point in the history
  • Loading branch information
onuratakan authored Apr 29, 2024
1 parent d4dbed1 commit 474c864
Show file tree
Hide file tree
Showing 9 changed files with 250 additions and 10 deletions.
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ RUN apt-get update
RUN apt-get install python3 python3-pip -y
RUN apt-get install nginx redis-server -y
RUN apt-get install openssl -y
RUN apt-get install git-lfs -y
RUN apt-get install git-lfs git -y
RUN apt-get install curl -y

WORKDIR /app/
Expand Down
2 changes: 1 addition & 1 deletion Dockerfile.without_models
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ RUN apt-get update
RUN apt-get install python3 python3-pip -y
RUN apt-get install nginx redis-server -y
RUN apt-get install openssl -y
RUN apt-get install git-lfs -y
RUN apt-get install git-lfs git -y
RUN apt-get install curl -y

WORKDIR /app/
Expand Down
18 changes: 16 additions & 2 deletions upsonic_on_prem/api/operations/user.py
Original file line number Diff line number Diff line change
Expand Up @@ -359,7 +359,7 @@ def get_all_scopes_user():
def delete_scope():
scope = request.form.get("scope")
object = Scope(scope)
return jsonify({"status": True, "result": object.delete()})
return jsonify({"status": True, "result": object.delete(AccessKey(request.authorization.password))})

@app.route(delete_version_url, methods=["POST"])
def delete_version():
Expand Down Expand Up @@ -713,4 +713,18 @@ def get_last_runs():
op = the_scope.get_last_runs(int(last_runs))
else:
op = the_scope.get_last_runs()
return jsonify({"status": True, "result": op})
return jsonify({"status": True, "result": op})



@app.route(get_github_sync_of_scope_url, methods=["POST"])
def get_github_sync_of_scope():
scope = request.form.get("scope")
version = request.form.get("version")
if version != None:
the_scope = Scope.get_version(scope+":"+version)
else:
the_scope = Scope(scope)


return jsonify({"status": True, "result": the_scope.is_it_github_synced()})
5 changes: 3 additions & 2 deletions upsonic_on_prem/api/urls.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
get_last_x_event_url = "/get_last_x_event"

get_document_of_scope_url = "/get_document_of_scope"
get_github_sync_of_scope_url = "/get_github_sync_of_scope"
get_requirements_of_scope_url = "/get_requirements_of_scope"
get_settings_of_scope_url = "/get_settings_of_scope"

Expand Down Expand Up @@ -138,12 +139,12 @@
get_last_runs_url = "/get_last_runs"


user_urs = [dump_run_url, get_last_runs_url, create_readme_url, get_openai_api_key_user, get_default_ai_model, get_readme_url, load_url, dump_url, get_read_scopes_of_me_url, get_write_scopes_of_me_url, get_document_of_scope_url, get_settings_of_scope_url, get_requirements_of_scope_url, get_security_analysis_of_scope_url, get_required_test_types_of_scope_url, get_tags_of_scope_url, get_mistakes_of_scope_url, get_time_complexity_of_scope_url,
user_urs = [dump_run_url, get_last_runs_url, create_readme_url, get_openai_api_key_user, get_default_ai_model, get_readme_url, load_url, dump_url, get_read_scopes_of_me_url, get_write_scopes_of_me_url, get_github_sync_of_scope_url, get_document_of_scope_url, get_settings_of_scope_url, get_requirements_of_scope_url, get_security_analysis_of_scope_url, get_required_test_types_of_scope_url, get_tags_of_scope_url, get_mistakes_of_scope_url, get_time_complexity_of_scope_url,
create_version_url, create_version_prefix_url, search_by_documentation_url, ai_completion_url,
get_dump_history_url, load_specific_dump_url,load_specific_version_url, get_all_scopes_name_prefix_url,
get_type_of_scope_url, get_python_version_of_scope_url, create_document_of_scope_url, create_security_analysis_of_scope_url, create_required_test_types_of_scope_url, create_tags_of_scope_url, create_mistakes_of_scope_url, create_time_complexity_of_scope_url, create_document_of_scope_url_old,
get_all_scopes_user_url, delete_scope_url,delete_version_url, dump_code_url, dump_requirements_url, dump_settings_url, dump_python_version_url, dump_type_url, get_version_history_url, get_module_version_history_url, get_version_code_of_scope_url, get_code_of_scope_url]
user_write_urls = [dump_run_url, dump_url, create_document_of_scope_url, create_security_analysis_of_scope_url, create_required_test_types_of_scope_url, create_tags_of_scope_url, create_mistakes_of_scope_url, create_time_complexity_of_scope_url, create_document_of_scope_url_old, delete_scope_url,delete_version_url,
create_version_url, dump_code_url, dump_requirements_url, dump_settings_url, dump_python_version_url, dump_type_url]
user_read_urls = [get_last_runs_url, load_url, get_document_of_scope_url, get_requirements_of_scope_url, get_settings_of_scope_url, get_security_analysis_of_scope_url, get_required_test_types_of_scope_url, get_tags_of_scope_url, get_mistakes_of_scope_url, get_time_complexity_of_scope_url, get_dump_history_url, get_type_of_scope_url, get_python_version_of_scope_url,
user_read_urls = [get_last_runs_url, load_url, get_document_of_scope_url, get_github_sync_of_scope_url, get_requirements_of_scope_url, get_settings_of_scope_url, get_security_analysis_of_scope_url, get_required_test_types_of_scope_url, get_tags_of_scope_url, get_mistakes_of_scope_url, get_time_complexity_of_scope_url, get_dump_history_url, get_type_of_scope_url, get_python_version_of_scope_url,
load_specific_dump_url,load_specific_version_url, get_version_history_url, get_version_code_of_scope_url, get_code_of_scope_url]
7 changes: 7 additions & 0 deletions upsonic_on_prem/dash/app/api_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,13 @@ def get_documentation(self, scope, version=None):
data["version"] = version
return transform_to_html_bold(self._send_request("POST", "/get_document_of_scope", data=data))

def get_github_sync(self, scope, version=None):
data = {"scope": scope}
if version != None:
data["version"] = version
return self._send_request("POST", "/get_github_sync_of_scope", data=data) == True



def get_requirements(self, scope, version=None):
data = {"scope": scope}
Expand Down
13 changes: 12 additions & 1 deletion upsonic_on_prem/dash/app/templates/libraries/element.html
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,19 @@
</div>
</div>

{% if version == "" and github_active %}
<div class="mb-1">
<a href="{{github_url}}" target="_blank" style="text-decoration: none;">
<img src="/static/images/github.png" style="height: 15px;"/>
{% if github_synced %}
<span>Synced</span>
{% else %}
<span >Not Synced</span>
{% endif %}
</a>
</div>


{% endif %}

<div class="d-flex flex-column scrollabe_righ_side">

Expand Down
15 changes: 15 additions & 0 deletions upsonic_on_prem/dash/app/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,14 @@

load_dotenv(dotenv_path=".env")

github_active = os.environ.get("github_active", "false").lower() == "true"
github_repo_owner = os.environ.get("github_repo_owner")
github_repo_name = os.environ.get("github_repo_name")


def the_connection_code(request):
custom_connection_url = os.getenv("custom_connection_url")

if custom_connection_url == None:
the_connection_code = f"""
from upsonic import Upsonic_On_Prem
Expand Down Expand Up @@ -686,6 +691,7 @@ def delete_openai_api_key_user(request, id):




@login_required
def control_element_version(request, id):

Expand Down Expand Up @@ -1106,6 +1112,12 @@ def control_element(request, id):



github_synced = False

github_url = f"https://github.com/{github_repo_owner}/{github_repo_name}/"
if github_active:
github_synced = API_Integration(request.user.access_key).get_github_sync(id, version=version)



data = {
Expand All @@ -1117,6 +1129,9 @@ def control_element(request, id):
"have_upper": have_upper,
"the_upper": the_upper,
"code": API_Integration(request.user.access_key).get_code(id, version=version),
"github_active":github_active,
"github_synced":github_synced,
"github_url":github_url,
"using_code": using_code,
"documentation": documentation,
"time_complexity": time_complexity,
Expand Down
135 changes: 135 additions & 0 deletions upsonic_on_prem/utils/github_sync.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
import requests

import os

from dotenv import load_dotenv
import base64
load_dotenv(dotenv_path=".env")


class github_:
def __init__(self, token, repo_owner, repo_name):
self.token = token
self.repo_owner = repo_owner
self.repo_name = repo_name
self.headers = {
'Authorization': f'token {self.token}',
'Accept': 'application/vnd.github.v3+json'
}
self.base_url = f'https://api.github.com/repos/{self.repo_owner}/{self.repo_name}'

self.author_name = 'getupsonic'
self.author_email = '[email protected]'

self.github_active = os.environ.get("github_active", "false").lower() == "true"
if self.token is False or self.repo_owner is False or self.repo_name is False:
self.github_active = False

def get_file(self, path):
url = f'{self.base_url}/contents/{path}'
response = requests.get(url, headers=self.headers)
response.raise_for_status()
return response.json()

def create_or_update_file(self, scope, message):
if not self.github_active:
return False


if ":" in scope.key:
path = scope.key.split(":")[0].replace(".", "/")
else:
path = scope.key.replace(".", "/")
path = f'{path}.py'
content = scope.code



# Inside your create_or_update_file function, before the PUT request
encoded_content = base64.b64encode(content.encode('utf-8')).decode('utf-8')
content = encoded_content



# Attempt to get the file to see if it already exists
get_url = f'{self.base_url}/contents/{path}'
get_response = requests.get(get_url, headers=self.headers)

data = {
'message': message,
'content': content,
'author': {
'name': self.author_name,
'email': self.author_email
},
'committer': {
'name': self.author_name,
'email': self.author_email
}
}


# If the file exists, get its SHA to update it
if get_response.status_code == 200:
data['sha'] = get_response.json()['sha']



# Whether the file exists or not, try to create or update it
put_response = requests.put(get_url, headers=self.headers, json=data)
put_response.raise_for_status()

return put_response.json()["content"]['sha']


def get_sha(self, scope):
if not self.github_active:
return False

if ":" in scope.key:
path = scope.key.split(":")[0].replace(".", "/")
else:
path = scope.key.replace(".", "/")
path = f'{path}.py'

# Get the file to retrieve its SHA
file_info = self.get_file(path)
return file_info['sha']


def delete_file(self, scope, message):
if not self.github_active:
return False

if ":" in scope.key:
path = scope.key.split(":")[0].replace(".", "/")
else:
path = scope.key.replace(".", "/")
path = f'{path}.py'

# First, get the file to retrieve its SHA
file_info = self.get_file(path)
sha = file_info['sha']

# Prepare the URL and data for the DELETE request
url = f'{self.base_url}/contents/{path}'
data = {
'message': message,
'sha': sha,
'author': {
'name': self.author_name,
'email': self.author_email
},
'committer': {
'name': self.author_name,
'email': self.author_email
}
}

# Send the DELETE request
response = requests.delete(url, headers=self.headers, json=data)
response.raise_for_status()
return response.json()


github = github_(os.environ.get("github_token", False), os.environ.get("github_repo_owner", False), os.environ.get("github_repo_name", False))
Loading

0 comments on commit 474c864

Please sign in to comment.