Skip to content

Commit

Permalink
github implementation for #165 #24
Browse files Browse the repository at this point in the history
  • Loading branch information
@albertmeronyo
albertmeronyo committed Aug 28, 2019
1 parent 4d9485b commit 513f508
Show file tree
Hide file tree
Showing 5 changed files with 52 additions and 29 deletions.
18 changes: 12 additions & 6 deletions src/fileLoaders.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@


class BaseLoader:
def getTextForName(self, query_name):
def getTextForName(self, query_name, subdir):
# The URIs of all candidates
rq_name = query_name + '.rq'
sparql_name = query_name + '.sparql'
Expand All @@ -24,7 +24,7 @@ def getTextForName(self, query_name):
]

for queryFullName, queryType in candidates:
queryText = self._getText(queryFullName)
queryText = self._getText(queryFullName, subdir)
if queryText:
if (queryType == qType['JSON']):
queryText = json.loads(queryText)
Expand All @@ -50,9 +50,10 @@ def getLicenceURL(self):


class GithubLoader(BaseLoader):
def __init__(self, user, repo, sha, prov):
def __init__(self, user, repo, subdir, sha, prov):
self.user = user
self.repo = repo
self.subdir = subdir
self.sha = sha
self.prov = prov
gh = Github(static.ACCESS_TOKEN)
Expand All @@ -65,6 +66,8 @@ def __init__(self, user, repo, sha, prov):

def fetchFiles(self):
api_repo_content_uri = static.GITHUB_API_BASE_URL + self.user + '/' + self.repo + '/contents'
if self.subdir:
api_repo_content_uri += '/' + str(self.subdir)
params = {
'ref': 'master' if self.sha is None else self.sha
}
Expand All @@ -84,17 +87,20 @@ def getRawRepoUri(self):
raw_repo_uri += '/{}/'.format(self.sha)
return raw_repo_uri

def getTextFor(self, fileItem):
def getTextFor(self, fileItem, subdir=None):
raw_query_uri = fileItem['download_url']
resp = self._getText(raw_query_uri)
resp = self._getText(raw_query_uri, subdir)

# Add query URI as used entity by the logged activity
if self.prov is not None:
self.prov.add_used_entity(raw_query_uri)
return resp

def _getText(self, query_name):
def _getText(self, query_name, subdir=None):
query_uri = self.getRawRepoUri() + query_name
if subdir:
query_uri = self.getRawRepoUri() + subdir + '/' + query_name
print("Requesting query at " + str(query_uri))
req = requests.get(query_uri, headers={'Authorization': 'token {}'.format(static.ACCESS_TOKEN)})
if req.status_code == 200:
return req.text
Expand Down
35 changes: 25 additions & 10 deletions src/server.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,19 +32,23 @@ def query_local(query_name):


@app.route('/api/<user>/<repo>/<query_name>', methods=['GET', 'POST'])
@app.route('/api/<user>/<repo>/<subdir>/<query_name>', methods=['GET', 'POST'])
@app.route('/api/<user>/<repo>/<query_name>.<content>', methods=['GET', 'POST'])
@app.route('/api/<user>/<repo>/<subdir>/<query_name>.<content>', methods=['GET', 'POST'])
@app.route('/api/<user>/<repo>/commit/<sha>/<query_name>', methods=['GET', 'POST'])
@app.route('/api/<user>/<repo>/<subdir>/commit/<sha>/<query_name>', methods=['GET', 'POST'])
@app.route('/api/<user>/<repo>/commit/<sha>/<query_name>.<content>', methods=['GET', 'POST'])
def query(user, repo, query_name, sha=None, content=None):
glogger.info("-----> Executing call name at /{}/{}/{} on commit {}".format(user, repo, query_name, sha))
@app.route('/api/<user>/<repo>/<subdir>/commit/<sha>/<query_name>.<content>', methods=['GET', 'POST'])
def query(user, repo, query_name, subdir=None, sha=None, content=None):
glogger.info("-----> Executing call name at /{}/{}/{}/{} on commit {}".format(user, repo, subdir, query_name, sha))
glogger.debug("Request accept header: " + request.headers["Accept"])

requestArgs = request.args
acceptHeader = request.headers['Accept']
requestUrl = request.url
formData = request.form

query_response, status, headers = utils.dispatch_query(user, repo, query_name,
query_response, status, headers = utils.dispatch_query(user, repo, query_name, subdir,
sha=sha, content=content, requestArgs=requestArgs,
acceptHeader=acceptHeader,
requestUrl=requestUrl, formData=formData)
Expand All @@ -66,19 +70,24 @@ def swagger_spec_local():


@app.route('/api/<user>/<repo>', strict_slashes=False)
@app.route('/api/<user>/<repo>/<subdir>', strict_slashes=False)
@app.route('/api/<user>/<repo>/api-docs')
@app.route('/api/<user>/<repo>/commit/<sha>')
@app.route('/api/<user>/<repo>/commit/<sha>/api-docs')
def api_docs(user, repo, sha=None):
return render_template('api-docs.html', user=user, repo=repo, sha=sha)
@app.route('/api/<user>/<repo>/<subdir>/commit/<sha>')
@app.route('/api/<user>/<repo>/<subdir>/commit/<sha>/api-docs')
def api_docs(user, repo, subdir=None, sha=None):
return render_template('api-docs.html', user=user, repo=repo, subdir=subdir, sha=sha)


@app.route('/api/<user>/<repo>/spec', methods=['GET'])
@app.route('/api/<user>/<repo>/<subdir>/spec', methods=['GET'])
@app.route('/api/<user>/<repo>/commit/<sha>/spec')
def swagger_spec(user, repo, sha=None, content=None):
glogger.info("-----> Generating swagger spec for /{}/{} on commit {}".format(user, repo, sha))
@app.route('/api/<user>/<repo>/<subdir>/commit/<sha>/spec')
def swagger_spec(user, repo, subdir=None, sha=None, content=None):
glogger.info("-----> Generating swagger spec for /{}/{} on commit {}".format(user, repo, subdir, sha))

swag = utils.build_swagger_spec(user, repo, sha, static.SERVER_NAME)
swag = utils.build_swagger_spec(user, repo, subdir, sha, static.SERVER_NAME)

if 'text/turtle' in request.headers['Accept']:
resp_spec = make_response(utils.turtleize(swag))
Expand All @@ -89,9 +98,15 @@ def swagger_spec(user, repo, sha=None, content=None):

resp_spec.headers['Cache-Control'] = static.CACHE_CONTROL_POLICY # Caching JSON specs for 15 minutes

glogger.info("-----> API spec generation for /{}/{} on commit {} complete".format(user, repo, sha))
glogger.info("-----> API spec generation for /{}/{} on commit {} complete".format(user, repo, subdir, sha))
return resp_spec


if __name__ == '__main__':
app.run(host=static.DEFAULT_HOST, port=static.DEFAULT_PORT, debug=static.LOG_DEBUG_MODE)
print("foo")
print("bar")
print("LOG DEBUG MODE is: " + str(static.LOG_DEBUG_MODE))
print("API KEY: " + str(static.ACCESS_TOKEN))
glogger.debug("yo")
app.run(host=static.DEFAULT_HOST, port=static.DEFAULT_PORT, debug=True)
glogger.debug("YO")
4 changes: 2 additions & 2 deletions src/swagger.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,9 +103,9 @@ def get_path_for_item(item):
return item_path


def build_spec(user, repo, sha=None, prov=None, extraMetadata=[]):
def build_spec(user, repo, subdir, sha=None, prov=None, extraMetadata=[]):
"""Build grlc specification for the given github user / repo."""
loader = grlc.utils.getLoader(user, repo, sha=sha, prov=prov)
loader = grlc.utils.getLoader(user, repo, subdir, sha=sha, prov=prov)

files = loader.fetchFiles()
raw_repo_uri = loader.getRawRepoUri()
Expand Down
2 changes: 1 addition & 1 deletion src/templates/api-docs.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@
<script src="{{url_for('.static', filename='js/grlc-api.js')}}"></script>

<script>
url = "{{url_for('swagger_spec', user=user, repo=repo, sha=sha)}}";
url = "{{url_for('swagger_spec', user=user, repo=repo, subdir=subdir, sha=sha)}}";
window.onload = grlcOnLoad(url);
$('#prev-commit').hide();
$('#next-commit').hide();
Expand Down
22 changes: 12 additions & 10 deletions src/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,22 +20,22 @@

glogger = glogging.getGrlcLogger(__name__)

def getLoader(user, repo, sha=None, prov=None):
def getLoader(user, repo, subdir=None, sha=None, prov=None):
"""Build a fileLoader (LocalLoader or GithubLoader) for the given repository."""
if user is None and repo is None:
loader = LocalLoader()
else:
loader = GithubLoader(user, repo, sha, prov)
loader = GithubLoader(user, repo, subdir, sha, prov)
return loader


def build_spec(user, repo, sha=None, prov=None, extraMetadata=[]):
def build_spec(user, repo, subdir, sha=None, prov=None, extraMetadata=[]):
glogger.warning("grlc.utils.build_spec is deprecated and will " \
"be removed in the future. Use grlc.swagger.build_spec instead.")
return swagger.build_spec(user, repo, sha, prov, extraMetadata)
return swagger.build_spec(user, repo, subdir, sha, prov, extraMetadata)


def build_swagger_spec(user, repo, sha, serverName):
def build_swagger_spec(user, repo, subdir, sha, serverName):
"""Build grlc specification for the given github user / repo in swagger format """
if user and repo:
# Init provenance recording
Expand All @@ -47,7 +47,7 @@ def build_swagger_spec(user, repo, sha, serverName):
swag['host'] = serverName

try:
loader = getLoader(user, repo, sha, prov_g)
loader = getLoader(user, repo, subdir, sha, prov_g)
except Exception as e:
# If repo does not exits
swag['info'] = {
Expand All @@ -63,9 +63,11 @@ def build_swagger_spec(user, repo, sha, serverName):
swag['next_commit'] = next_commit
swag['info'] = info
swag['basePath'] = basePath
if subdir:
swag['basePath'] = basePath + subdir

# TODO: can we pass loader to build_spec ?
spec = swagger.build_spec(user, repo, sha, prov_g)
spec = swagger.build_spec(user, repo, subdir, sha, prov_g)
for item in spec:
swag['paths'][item['call_name']] = swagger.get_path_for_item(item)

Expand All @@ -75,10 +77,10 @@ def build_swagger_spec(user, repo, sha, serverName):
return swag


def dispatch_query(user, repo, query_name, sha=None, content=None, requestArgs={}, acceptHeader='application/json',
def dispatch_query(user, repo, query_name, subdir=None, sha=None, content=None, requestArgs={}, acceptHeader='application/json',
requestUrl='http://', formData={}):
loader = getLoader(user, repo, sha=sha, prov=None)
query, q_type = loader.getTextForName(query_name)
loader = getLoader(user, repo, subdir, sha=sha, prov=None)
query, q_type = loader.getTextForName(query_name, subdir)

# Call name implemented with SPARQL query
if q_type == qType['SPARQL'] or q_type == qType['JSON']:
Expand Down

0 comments on commit 513f508

Please sign in to comment.