Skip to content
This repository has been archived by the owner on Nov 16, 2022. It is now read-only.

Owasm executor on google cloud function #1872

Merged
merged 45 commits into from
Jun 12, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
c590ce7
Hello flask
Jun 4, 2020
76c28f1
Add git ignore
Jun 4, 2020
b11e5d5
Missing field handler
Jun 4, 2020
5f5d860
400 Bad request handler
Jun 4, 2020
5c08f51
Add test
Jun 4, 2020
b2766e4
Get constants from env
Jun 4, 2020
095ceb9
Check empty first
Jun 4, 2020
a19295b
Add exception handler
Jun 4, 2020
42aac50
Move get env to outside function
Jun 4, 2020
646cf6a
Add timeout when run process
Jun 4, 2020
0558cca
Convert from millisec to sec
Jun 4, 2020
9414cec
Add test
Jun 4, 2020
61a5a0a
Separate error
Jun 4, 2020
64388b8
Add empty field handler
Jun 4, 2020
d7c1b73
Add execution test cases
Jun 4, 2020
a4a7ba0
Decoded executable
Jun 4, 2020
7c2cb08
Add timeout success case
Jun 4, 2020
95940c4
Call data can be empty
Jun 4, 2020
717f062
Empty executable must return 126
Jun 4, 2020
720d088
Strout and Strerr limit handle
Jun 5, 2020
1846703
Use app from create app
Jun 5, 2020
9872ccc
Remove print
Jun 5, 2020
7fb1273
Separate function to google cloud function file
Jun 5, 2020
ca9b13f
Format by Black
perimeko Jun 5, 2020
574de36
Update changelog
perimeko Jun 5, 2020
f96d275
Use Popen instead of run
perimeko Jun 5, 2020
f9747b1
Fixed all test cases to pass
perimeko Jun 5, 2020
294ae0d
Encode executable to base64
perimeko Jun 5, 2020
81c0ce6
Refactor
perimeko Jun 5, 2020
3022b94
Format code
perimeko Jun 5, 2020
e4c8fbf
Use specific exception type
perimeko Jun 8, 2020
6830a47
Freeze pytest requirement
perimeko Jun 8, 2020
c972f40
Convete timeout string to int
perimeko Jun 8, 2020
2535cb5
Fixed all test cases
perimeko Jun 8, 2020
4a08229
Removed unused
perimeko Jun 8, 2020
09544c8
Use request loaded from schema
perimeko Jun 8, 2020
d3285ca
Use success and bad request function
perimeko Jun 8, 2020
9fb4996
Black format
perimeko Jun 8, 2020
29c9195
Update change log
perimeko Jun 9, 2020
b6a45b4
Remove lines
perimeko Jun 9, 2020
789b725
Fixed test
perimeko Jun 9, 2020
e78927a
Merge branch 'master' into google-cloud
perimeko Jun 12, 2020
c8f0d82
Merge branch 'master' into google-cloud
perimeko Jun 12, 2020
c230c9a
Get request parameter from dict
perimeko Jun 12, 2020
dda192c
Merge branch 'master' into google-cloud
sorawit Jun 12, 2020
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,5 @@
*.swp
*.pyc
*.log

cloud-function/venv/
1 change: 1 addition & 0 deletions CHANGELOG_UNRELEASED.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

### Chain

- (feat) [\#1872](https://github.com/bandprotocol/bandchain/pull/1872) Add Owasm executor on Google Cloud function
- (feat) [\#1951](https://github.com/bandprotocol/bandchain/pull/1951) Add REST endpoint to get genesis file.
- (feat) [\#1908](https://github.com/bandprotocol/bandchain/pull/1908) Support google cloud function REST executor
- (feat) [\#1929](https://github.com/bandprotocol/bandchain/pull/1929) Add report info of validator on CLI and REST endpoint.
Expand Down
107 changes: 107 additions & 0 deletions cloud-function/google_cloud_function.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
from flask import Flask, jsonify, json
import os
import shlex
import subprocess
import base64
import werkzeug
from marshmallow import Schema, fields, ValidationError, validate

# Copy and paste this file on Google Cloud function
# Set environment flag of MAX_EXECUTABLE, MAX_CALLDATA, MAX_TIMEOUT, MAX_STDOUT, MAX_STDERR


def get_env(env, flag):
if flag not in env:
raise Exception(flag + " is missing")
return int(env[flag])


def success(returncode, stdout, stderr, err):
return (
jsonify(
{"returncode": returncode, "stdout": stdout, "stderr": stderr, "err": err}
),
200,
)


def bad_request(err):
return jsonify({"error": err}), 400


def execute(request):
"""Responds to any HTTP request.
Args:
request (flask.Request): HTTP request object.
Returns:
The response text or any set of values that can be turned into a
Response object using
`make_response <http://flask.pocoo.org/docs/1.0/api/#flask.Flask.make_response>`.
"""
env = os.environ.copy()

MAX_EXECUTABLE = get_env(env, "MAX_EXECUTABLE")
MAX_CALLDATA = get_env(env, "MAX_CALLDATA")
MAX_TIMEOUT = get_env(env, "MAX_TIMEOUT")
MAX_STDOUT = get_env(env, "MAX_STDOUT")
MAX_STDERR = get_env(env, "MAX_STDERR")

class Executable(fields.Field):
def _deserialize(self, value, attr, data, **kwargs):
try:
return base64.b64decode(value).decode()
except:
raise ValidationError("Can't decoded executable")

class RequestSchema(Schema):
executable = Executable(
required=True,
validate=validate.Length(max=MAX_EXECUTABLE),
error_messages={"required": "field is missing from JSON request"},
)
calldata = fields.Str(
required=True,
validate=validate.Length(max=MAX_CALLDATA),
error_messages={"required": "field is missing from JSON request"},
)
timeout = fields.Int(
required=True,
validate=validate.Range(min=0, max=MAX_TIMEOUT),
error_messages={"required": "field is missing from JSON request"},
)

try:
request_json = request.get_json(force=True)
except werkzeug.exceptions.BadRequest:
return bad_request("invalid JSON request format")

try:
loaded_request = RequestSchema().load(request_json)
except ValidationError as err:
return bad_request(err.messages)

path = "/tmp/execute.sh"
with open(path, "w") as f:
f.write(loaded_request["executable"])

os.chmod(path, 0o775)

try:
timeout_millisec = loaded_request["timeout"]
timeout_sec = timeout_millisec / 1000

proc = subprocess.Popen(
[path] + shlex.split(loaded_request["calldata"]),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)

proc.wait(timeout=timeout_sec)
returncode = proc.returncode
stdout = proc.stdout.read(MAX_STDOUT).decode()
stderr = proc.stderr.read(MAX_STDERR).decode()
return success(returncode, stdout, stderr, "")
except OSError:
return success(126, "", "", "Execution fail")
except subprocess.TimeoutExpired:
return success(111, "", "", "Execution time limit exceeded")
12 changes: 12 additions & 0 deletions cloud-function/main.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
from flask import Flask, request
from google_cloud_function import execute


def create_app():
app = Flask(__name__)

@app.route("/execute", methods=["POST"])
def run():
return execute(request)

return app
Loading