From f010bca61e5a58970477d0f1ec02df0f1bfadc4a Mon Sep 17 00:00:00 2001 From: Jeff Lowrey Date: Mon, 16 Mar 2020 09:11:51 -0500 Subject: [PATCH 1/5] Create debian templates for the python 3 flask templates. The current python3-flask templates use the Alpine base python image. Precompiled wheels for many important python libraries are not available for Alpine, but are easily available for Debian. Using Debian makes it much simpler and easier to use python packages like Numpy or Pillow. Signed-off-by: Jeff Lowrey --- template/python3-flask-debian/Dockerfile | 45 ++++++++++++ .../python3-flask-debian/function/__init__.py | 0 .../python3-flask-debian/function/handler.py | 7 ++ .../function/requirements.txt | 0 template/python3-flask-debian/index.py | 33 +++++++++ .../python3-flask-debian/requirements.txt | 3 + template/python3-flask-debian/template.yml | 2 + template/python3-http-debian/Dockerfile | 48 +++++++++++++ .../python3-http-debian/function/handler.py | 5 ++ .../function/requirements.txt | 0 template/python3-http-debian/index.py | 69 +++++++++++++++++++ template/python3-http-debian/requirements.txt | 2 + template/python3-http-debian/template.yml | 14 ++++ 13 files changed, 228 insertions(+) create mode 100644 template/python3-flask-debian/Dockerfile create mode 100644 template/python3-flask-debian/function/__init__.py create mode 100644 template/python3-flask-debian/function/handler.py create mode 100644 template/python3-flask-debian/function/requirements.txt create mode 100644 template/python3-flask-debian/index.py create mode 100644 template/python3-flask-debian/requirements.txt create mode 100644 template/python3-flask-debian/template.yml create mode 100644 template/python3-http-debian/Dockerfile create mode 100644 template/python3-http-debian/function/handler.py create mode 100644 template/python3-http-debian/function/requirements.txt create mode 100644 template/python3-http-debian/index.py create mode 100644 template/python3-http-debian/requirements.txt create mode 100644 template/python3-http-debian/template.yml diff --git a/template/python3-flask-debian/Dockerfile b/template/python3-flask-debian/Dockerfile new file mode 100644 index 0000000..c7ecb57 --- /dev/null +++ b/template/python3-flask-debian/Dockerfile @@ -0,0 +1,45 @@ +FROM openfaas/of-watchdog:0.7.2 as watchdog +FROM python:3-slim-buster + +COPY --from=watchdog /fwatchdog /usr/bin/fwatchdog +RUN chmod +x /usr/bin/fwatchdog + +ARG ADDITIONAL_PACKAGE +# Alternatively use ADD https:// (which will not be cached by Docker builder) +RUN apt-get install ${ADDITIONAL_PACKAGE} + + +# Add non root user +RUN addgroup --system app && adduser app --system --ingroup app +#RUN chown app /home/app + +WORKDIR /home/app/ + +COPY index.py . +COPY requirements.txt . +USER root +RUN pip install -r requirements.txt +USER app + +RUN mkdir -p function +RUN touch ./function/__init__.py +WORKDIR /home/app/function/ +COPY function/requirements.txt . +RUN pip install --user -r requirements.txt + +WORKDIR /home/app/ + +USER root +COPY function function +RUN chown -R app:app ./ +USER app + +ENV fprocess="python index.py" + +ENV cgi_headers="true" +ENV mode="http" +ENV upstream_url="http://127.0.0.1:5000" + +HEALTHCHECK --interval=5s CMD [ -e /tmp/.lock ] || exit 1 + +CMD ["fwatchdog"] diff --git a/template/python3-flask-debian/function/__init__.py b/template/python3-flask-debian/function/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/template/python3-flask-debian/function/handler.py b/template/python3-flask-debian/function/handler.py new file mode 100644 index 0000000..a7098fa --- /dev/null +++ b/template/python3-flask-debian/function/handler.py @@ -0,0 +1,7 @@ +def handle(req): + """handle a request to the function + Args: + req (str): request body + """ + + return req diff --git a/template/python3-flask-debian/function/requirements.txt b/template/python3-flask-debian/function/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/template/python3-flask-debian/index.py b/template/python3-flask-debian/index.py new file mode 100644 index 0000000..c7c785f --- /dev/null +++ b/template/python3-flask-debian/index.py @@ -0,0 +1,33 @@ +# Copyright (c) Alex Ellis 2017. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. + +from flask import Flask, request +from function import handler +#from gevent.wsgi import WSGIServer +from gevent.pywsgi import WSGIServer + +app = Flask(__name__) + +@app.before_request +def fix_transfer_encoding(): + """ + Sets the "wsgi.input_terminated" environment flag, thus enabling + Werkzeug to pass chunked requests as streams. The gunicorn server + should set this, but it's not yet been implemented. + """ + + transfer_encoding = request.headers.get("Transfer-Encoding", None) + if transfer_encoding == u"chunked": + request.environ["wsgi.input_terminated"] = True + +@app.route("/", defaults={"path": ""}, methods=["POST", "GET"]) +@app.route("/", methods=["POST", "GET"]) +def main_route(path): + ret = handler.handle(request.get_data()) + return ret + +if __name__ == '__main__': + #app.run(host='0.0.0.0', port=5000, debug=False) + + http_server = WSGIServer(('', 5000), app) + http_server.serve_forever() diff --git a/template/python3-flask-debian/requirements.txt b/template/python3-flask-debian/requirements.txt new file mode 100644 index 0000000..4c0baf0 --- /dev/null +++ b/template/python3-flask-debian/requirements.txt @@ -0,0 +1,3 @@ +flask +gevent + diff --git a/template/python3-flask-debian/template.yml b/template/python3-flask-debian/template.yml new file mode 100644 index 0000000..21a04ea --- /dev/null +++ b/template/python3-flask-debian/template.yml @@ -0,0 +1,2 @@ +language: python3-flask +fprocess: python index.py diff --git a/template/python3-http-debian/Dockerfile b/template/python3-http-debian/Dockerfile new file mode 100644 index 0000000..6fb4a00 --- /dev/null +++ b/template/python3-http-debian/Dockerfile @@ -0,0 +1,48 @@ +FROM openfaas/of-watchdog:0.7.2 as watchdog +FROM python:3-slim-buster + +COPY --from=watchdog /fwatchdog /usr/bin/fwatchdog +RUN chmod +x /usr/bin/fwatchdog + +ARG ADDITIONAL_PACKAGE +# Alternatively use ADD https:// (which will not be cached by Docker builder) +RUN apt-get install ${ADDITIONAL_PACKAGE} + +# Add non root user +RUN addgroup --system app && adduser app --system --ingroup app +RUN chown app /home/app + +USER app + +ENV PATH=$PATH:/home/app/.local/bin + +WORKDIR /home/app/ + +COPY index.py . +COPY requirements.txt . +USER root +RUN pip install -r requirements.txt +USER app + +RUN mkdir -p function +RUN touch ./function/__init__.py +WORKDIR /home/app/function/ +COPY function/requirements.txt . +RUN pip install --user -r requirements.txt + +WORKDIR /home/app/ + +USER root +COPY function function +RUN chown -R app:app ./ +USER app + +# Set up of-watchdog for HTTP mode +ENV fprocess="python index.py" +ENV cgi_headers="true" +ENV mode="http" +ENV upstream_url="http://127.0.0.1:5000" + +HEALTHCHECK --interval=5s CMD [ -e /tmp/.lock ] || exit 1 + +CMD ["fwatchdog"] \ No newline at end of file diff --git a/template/python3-http-debian/function/handler.py b/template/python3-http-debian/function/handler.py new file mode 100644 index 0000000..a1a099c --- /dev/null +++ b/template/python3-http-debian/function/handler.py @@ -0,0 +1,5 @@ +def handle(event, context): + return { + "statusCode": 200, + "body": "Hello from OpenFaaS!" + } diff --git a/template/python3-http-debian/function/requirements.txt b/template/python3-http-debian/function/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/template/python3-http-debian/index.py b/template/python3-http-debian/index.py new file mode 100644 index 0000000..71b8624 --- /dev/null +++ b/template/python3-http-debian/index.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +from flask import Flask, request, jsonify +from waitress import serve +import os + +from function import handler + +app = Flask(__name__) + +class Event: + def __init__(self): + self.body = request.get_data() + self.headers = request.headers + self.method = request.method + self.query = request.args + self.path = request.path + +class Context: + def __init__(self): + self.hostname = os.environ['HOSTNAME'] + +def format_status_code(resp): + if 'statusCode' in resp: + return resp['statusCode'] + + return 200 + +def format_body(resp): + if 'body' not in resp: + return "" + elif type(resp['body']) == dict: + return jsonify(resp['body']) + else: + return str(resp['body']) + +def format_headers(resp): + if 'headers' not in resp: + return [] + elif type(resp['headers']) == dict: + headers = [] + for key in resp['headers'].keys(): + header_tuple = (key, resp['headers'][key]) + headers.append(header_tuple) + return headers + + return resp['headers'] + +def format_response(resp): + if resp == None: + return ('', 200) + + statusCode = format_status_code(resp) + body = format_body(resp) + headers = format_headers(resp) + + return (body, statusCode, headers) + +@app.route('/', defaults={'path': ''}, methods=['GET', 'PUT', 'POST', 'PATCH', 'DELETE']) +@app.route('/', methods=['GET', 'PUT', 'POST', 'PATCH', 'DELETE']) +def call_handler(path): + event = Event() + context = Context() + response_data = handler.handle(event, context) + + resp = format_response(response_data) + return resp + +if __name__ == '__main__': + serve(app, host='0.0.0.0', port=5000) \ No newline at end of file diff --git a/template/python3-http-debian/requirements.txt b/template/python3-http-debian/requirements.txt new file mode 100644 index 0000000..8642e2b --- /dev/null +++ b/template/python3-http-debian/requirements.txt @@ -0,0 +1,2 @@ +flask +waitress \ No newline at end of file diff --git a/template/python3-http-debian/template.yml b/template/python3-http-debian/template.yml new file mode 100644 index 0000000..01b9314 --- /dev/null +++ b/template/python3-http-debian/template.yml @@ -0,0 +1,14 @@ +language: python3-http +fprocess: python index.py +build_options: + - name: dev + packages: + - make + - automake + - gcc + - g++ + - subversion + - python3-dev + - musl-dev + - libffi-dev + - git \ No newline at end of file From d45361d727dcc681560501c16a9feb5b4688725f Mon Sep 17 00:00:00 2001 From: Jeff Lowrey Date: Wed, 25 Mar 2020 08:56:23 -0500 Subject: [PATCH 2/5] Updated existing Alpine Docker files to include comments. Signed-off-by: Jeff Lowrey --- template/python3-flask/Dockerfile | 9 +++++++++ template/python3-http/Dockerfile | 7 ++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/template/python3-flask/Dockerfile b/template/python3-flask/Dockerfile index 52b661e..7af655e 100644 --- a/template/python3-flask/Dockerfile +++ b/template/python3-flask/Dockerfile @@ -5,6 +5,8 @@ COPY --from=watchdog /fwatchdog /usr/bin/fwatchdog RUN chmod +x /usr/bin/fwatchdog ARG ADDITIONAL_PACKAGE +# Alternatively use ADD https:// (which will not be cached by Docker builder) + RUN apk --no-cache add musl-dev gcc make ${ADDITIONAL_PACKAGE} # Add non root user @@ -19,8 +21,11 @@ WORKDIR /home/app/ COPY index.py . COPY requirements.txt . + USER root RUN pip install -r requirements.txt + +#build function directory and install user specified componenets. USER app RUN mkdir -p function @@ -31,9 +36,13 @@ RUN pip install --user -r requirements.txt WORKDIR /home/app/ +#install function code USER root + COPY function function RUN chown -R app:app ./ + +#configure WSGI server and healthcheck USER app ENV fprocess="python index.py" diff --git a/template/python3-http/Dockerfile b/template/python3-http/Dockerfile index 2a8bf73..a0349c3 100644 --- a/template/python3-http/Dockerfile +++ b/template/python3-http/Dockerfile @@ -6,6 +6,7 @@ RUN chmod +x /usr/bin/fwatchdog ARG ADDITIONAL_PACKAGE # Alternatively use ADD https:// (which will not be cached by Docker builder) + RUN apk --no-cache add ${ADDITIONAL_PACKAGE} # Add non root user @@ -22,6 +23,8 @@ COPY index.py . COPY requirements.txt . USER root RUN pip install -r requirements.txt + +#build function directory and install user specified componenets. USER app RUN mkdir -p function @@ -32,12 +35,14 @@ RUN pip install --user -r requirements.txt WORKDIR /home/app/ +#install function code USER root COPY function function RUN chown -R app:app ./ + +#configure WSGI server and healthcheck USER app -# Set up of-watchdog for HTTP mode ENV fprocess="python index.py" ENV cgi_headers="true" ENV mode="http" From f783f8398e59f69125fa9437fc8422887e4f15e2 Mon Sep 17 00:00:00 2001 From: Jeff Lowrey Date: Thu, 26 Mar 2020 15:13:21 -0500 Subject: [PATCH 3/5] Verified that both templates build and added comments to dockerfile Signed-off-by: Jeff Lowrey --- template/python3-flask-debian/Dockerfile | 15 +++++++++++++-- template/python3-http-debian/Dockerfile | 3 ++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/template/python3-flask-debian/Dockerfile b/template/python3-flask-debian/Dockerfile index c7ecb57..a47bb3d 100644 --- a/template/python3-flask-debian/Dockerfile +++ b/template/python3-flask-debian/Dockerfile @@ -6,19 +6,26 @@ RUN chmod +x /usr/bin/fwatchdog ARG ADDITIONAL_PACKAGE # Alternatively use ADD https:// (which will not be cached by Docker builder) -RUN apt-get install ${ADDITIONAL_PACKAGE} +RUN apt-get update && apt-get --assume-yes install musl-dev gcc make ${ADDITIONAL_PACKAGE} # Add non root user RUN addgroup --system app && adduser app --system --ingroup app -#RUN chown app /home/app +RUN chown app /home/app + +USER app + +ENV PATH=$PATH:/home/app/.local/bin WORKDIR /home/app/ COPY index.py . COPY requirements.txt . + USER root RUN pip install -r requirements.txt + +#build function directory and install user specified componenets. USER app RUN mkdir -p function @@ -29,9 +36,13 @@ RUN pip install --user -r requirements.txt WORKDIR /home/app/ +#install function code USER root + COPY function function RUN chown -R app:app ./ + +#configure WSGI server and healthcheck USER app ENV fprocess="python index.py" diff --git a/template/python3-http-debian/Dockerfile b/template/python3-http-debian/Dockerfile index 6fb4a00..3410cb1 100644 --- a/template/python3-http-debian/Dockerfile +++ b/template/python3-http-debian/Dockerfile @@ -6,7 +6,8 @@ RUN chmod +x /usr/bin/fwatchdog ARG ADDITIONAL_PACKAGE # Alternatively use ADD https:// (which will not be cached by Docker builder) -RUN apt-get install ${ADDITIONAL_PACKAGE} + +RUN apt-get update && apt-get --assume-yes install musl-dev gcc make ${ADDITIONAL_PACKAGE} # Add non root user RUN addgroup --system app && adduser app --system --ingroup app From d8193899b90cccb30ef6c7ebd029ddef261af6fa Mon Sep 17 00:00:00 2001 From: Jeff Lowrey Date: Fri, 3 Apr 2020 09:28:11 -0500 Subject: [PATCH 4/5] Template yaml files cleaned, python at 3.7, --assume-yes turned to -qy The template yaml file for the python-http-debian was modified to remove unneeded libraries. Both docker files were changed to force python 3.7. Gevent 1.4 is incompatible with Python 3.8, and the rest of the docker files use python 3.7. The apt-get commands were changed from using --assume-yes to -qy. It's shorter and adds the --quiet (-q) option to reduce information in stdout during docker build. This has been tested with a bare bones basic function and builds, runs, and executes properly. Signed-off-by: Jeff Lowrey --- template/python3-flask-debian/Dockerfile | 4 ++-- template/python3-flask-debian/template.yml | 2 +- template/python3-http-debian/Dockerfile | 4 ++-- template/python3-http-debian/template.yml | 14 +------------- 4 files changed, 6 insertions(+), 18 deletions(-) diff --git a/template/python3-flask-debian/Dockerfile b/template/python3-flask-debian/Dockerfile index a47bb3d..048b637 100644 --- a/template/python3-flask-debian/Dockerfile +++ b/template/python3-flask-debian/Dockerfile @@ -1,5 +1,5 @@ FROM openfaas/of-watchdog:0.7.2 as watchdog -FROM python:3-slim-buster +FROM python:3.7-slim-buster COPY --from=watchdog /fwatchdog /usr/bin/fwatchdog RUN chmod +x /usr/bin/fwatchdog @@ -7,7 +7,7 @@ RUN chmod +x /usr/bin/fwatchdog ARG ADDITIONAL_PACKAGE # Alternatively use ADD https:// (which will not be cached by Docker builder) -RUN apt-get update && apt-get --assume-yes install musl-dev gcc make ${ADDITIONAL_PACKAGE} +RUN apt-get -qy update && apt-get -qy install gcc make ${ADDITIONAL_PACKAGE} # Add non root user RUN addgroup --system app && adduser app --system --ingroup app diff --git a/template/python3-flask-debian/template.yml b/template/python3-flask-debian/template.yml index 21a04ea..4bbcb1b 100644 --- a/template/python3-flask-debian/template.yml +++ b/template/python3-flask-debian/template.yml @@ -1,2 +1,2 @@ -language: python3-flask +language: python3-flask-debian fprocess: python index.py diff --git a/template/python3-http-debian/Dockerfile b/template/python3-http-debian/Dockerfile index 3410cb1..9bdd87e 100644 --- a/template/python3-http-debian/Dockerfile +++ b/template/python3-http-debian/Dockerfile @@ -1,5 +1,5 @@ FROM openfaas/of-watchdog:0.7.2 as watchdog -FROM python:3-slim-buster +FROM python:3.7-slim-buster COPY --from=watchdog /fwatchdog /usr/bin/fwatchdog RUN chmod +x /usr/bin/fwatchdog @@ -7,7 +7,7 @@ RUN chmod +x /usr/bin/fwatchdog ARG ADDITIONAL_PACKAGE # Alternatively use ADD https:// (which will not be cached by Docker builder) -RUN apt-get update && apt-get --assume-yes install musl-dev gcc make ${ADDITIONAL_PACKAGE} +RUN apt-get -qy update && apt-get -qy install ${ADDITIONAL_PACKAGE} # Add non root user RUN addgroup --system app && adduser app --system --ingroup app diff --git a/template/python3-http-debian/template.yml b/template/python3-http-debian/template.yml index 01b9314..36ef6ef 100644 --- a/template/python3-http-debian/template.yml +++ b/template/python3-http-debian/template.yml @@ -1,14 +1,2 @@ -language: python3-http +language: python3-http-debian fprocess: python index.py -build_options: - - name: dev - packages: - - make - - automake - - gcc - - g++ - - subversion - - python3-dev - - musl-dev - - libffi-dev - - git \ No newline at end of file From 2235606b2142dc878aaa8974a3ba6d840ca410a7 Mon Sep 17 00:00:00 2001 From: Jeff Lowrey Date: Sat, 4 Apr 2020 10:49:37 -0500 Subject: [PATCH 5/5] Change listening address for WSGI server. Update the listening address for the WSGI server to use 0.0.0.0 instead of ''. This should resolve errors about Address family not supported by protocol (Python OSError Errno 97. Signed-off-by: Jeff Lowrey --- template/python3-flask-debian/index.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/template/python3-flask-debian/index.py b/template/python3-flask-debian/index.py index c7c785f..62d1379 100644 --- a/template/python3-flask-debian/index.py +++ b/template/python3-flask-debian/index.py @@ -27,7 +27,6 @@ def main_route(path): return ret if __name__ == '__main__': - #app.run(host='0.0.0.0', port=5000, debug=False) - http_server = WSGIServer(('', 5000), app) + http_server = WSGIServer(('0.0.0.0', 5000), app) http_server.serve_forever()