add regular vulnerability scan

.github/workflows/vulnerability-scan.yml

@@ -0,0 +1,43 @@
+name: Vulnerability Scan
+
+on:
+  schedule:
+    # Run once every day
+    - cron: '0 0 * * *'
+
+jobs:
+  scan:
+    name: Vulnerability scan
+    runs-on: ubuntu-latest
+
+    env:
+      DOCKERFILE: Dockerfile_with_poetry_lock
+
+    steps:
+    - name: Checkout git repository 🕝
+      uses: actions/checkout@v2
+
+    - name: Add poetry.lock 🔒
+      # Trivy depends on the presence of `poetry.lock` to scan Python dependencies
+      run: |
+        BASE_IMAGE=rasa/rasa:latest-full
+        docker pull $BASE_IMAGE
+
+        # Create Dockerfile which includes poetry.lock
+        tee -a $DOCKERFILE << END
+        FROM $BASE_IMAGE
+        COPY poetry.lock .
+        END
+
+        IMAGE_NAME=rasa/rasa:latest-scanned
+        docker build -f $DOCKERFILE -t $IMAGE_NAME .
+
+        echo "::set-env name=IMAGE_WITH_POETRY_LOCK::$IMAGE_NAME"
+
+    - name: Scan image 🕵️‍♀️🕵️‍♂️
+      uses: homoluctus/[email protected]
+      with:
+        # Needs the token so it can create an issue once a vulnerability was found
+        token: ${{ secrets.GITHUB_TOKEN }}
+        image: ${{ env.IMAGE_WITH_POETRY_LOCK }}
+        ignore_unfixed: true

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.6-slim as base
+FROM python:3.7-slim as base
 
 RUN apt-get update -qq \
  && apt-get install -y --no-install-recommends \

changelog/5627.improvement.rst

@@ -0,0 +1 @@
+All Rasa Open Source images are now using Python 3.7 instead of Python 3.6.

changelog/5672.bugfix.rst

@@ -0,0 +1,4 @@
+Updated Python dependency ``ruamel.yaml`` to ``>=0.16``. We recommend to use at least
+``0.16.10`` due to the security issue
+`CVE-2019-20478 <https://nvd.nist.gov/vuln/detail/CVE-2019-20478>`_ which is present in
+in prior versions.

docker/Dockerfile_full

@@ -1,4 +1,4 @@
-FROM python:3.6-slim as base
+FROM python:3.7-slim as base
 
 RUN apt-get update -qq \
  && apt-get install -y --no-install-recommends \

docker/Dockerfile_pretrained_embeddings_mitie_en

@@ -1,4 +1,4 @@
-FROM python:3.6-slim as base
+FROM python:3.7-slim as base
 
 RUN apt-get update -qq \
  && apt-get install -y --no-install-recommends \

docker/Dockerfile_pretrained_embeddings_spacy_de

@@ -1,4 +1,4 @@
-FROM python:3.6-slim as base
+FROM python:3.7-slim as base
 
 RUN apt-get update -qq \
  && apt-get install -y --no-install-recommends \

docker/Dockerfile_pretrained_embeddings_spacy_en

@@ -1,4 +1,4 @@
-FROM python:3.6-slim as base
+FROM python:3.7-slim as base
 
 RUN apt-get update -qq \
  && apt-get install -y --no-install-recommends \

pyproject.toml

@@ -80,7 +80,7 @@ networkx = "~2.4.0"
 fbmessenger = "~6.0.0"
 pykwalify = "~1.7.0"
 coloredlogs = "^10.0"
-"ruamel.yaml" = "~0.15"
+"ruamel.yaml" = "^0.16"
 scikit-learn = "^0.22"
 slackclient = "^2.0.0"
 python-telegram-bot = "^11.1"

rasa/constants.py

@@ -25,6 +25,7 @@
 
 CONFIG_SCHEMA_FILE = "nlu/schemas/config.yml"
 DOMAIN_SCHEMA_FILE = "core/schemas/domain.yml"
+YAML_VERSION = (1, 2)
 
 DEFAULT_RASA_X_PORT = 5002
 DEFAULT_RASA_PORT = 5005

rasa/core/utils.py

@@ -31,6 +31,7 @@
     DEFAULT_SANIC_WORKERS,
     ENV_SANIC_WORKERS,
     DEFAULT_ENDPOINTS_PATH,
+    YAML_VERSION,
 )
 
 # backwards compatibility 1.0.x
@@ -195,7 +196,7 @@ def _dump_yaml(obj: Dict, output: Union[Text, Path, StringIO]) -> None:
     yaml_writer = ruamel.yaml.YAML(pure=True, typ="safe")
     yaml_writer.unicode_supplementary = True
     yaml_writer.default_flow_style = False
-    yaml_writer.version = "1.1"
+    yaml_writer.version = YAML_VERSION
 
     yaml_writer.dump(obj, output)
 

rasa/utils/io.py

@@ -17,7 +17,7 @@
 
 import ruamel.yaml as yaml
 
-from rasa.constants import ENV_LOG_LEVEL, DEFAULT_LOG_LEVEL
+from rasa.constants import ENV_LOG_LEVEL, DEFAULT_LOG_LEVEL, YAML_VERSION
 
 if typing.TYPE_CHECKING:
     from prompt_toolkit.validation import Validator
@@ -110,24 +110,22 @@ def read_yaml(content: Text) -> Union[List[Any], Dict[Text, Any]]:
     replace_environment_variables()
 
     yaml_parser = yaml.YAML(typ="safe")
-    yaml_parser.version = "1.2"
-    yaml_parser.unicode_supplementary = True
+    yaml_parser.version = YAML_VERSION
 
-    # noinspection PyUnresolvedReferences
-    try:
-        return yaml_parser.load(content) or {}
-    except yaml.scanner.ScannerError:
-        # A `ruamel.yaml.scanner.ScannerError` might happen due to escaped
-        # unicode sequences that form surrogate pairs. Try converting the input
-        # to a parsable format based on
-        # https://stackoverflow.com/a/52187065/3429596.
+    if _is_ascii(content):
+        # Required to make sure emojis are correctly parsed
         content = (
             content.encode("utf-8")
             .decode("raw_unicode_escape")
             .encode("utf-16", "surrogatepass")
             .decode("utf-16")
         )
-        return yaml_parser.load(content) or {}
+
+    return yaml_parser.load(content) or {}
+
+
+def _is_ascii(text: Text) -> bool:
+    return all(ord(character) < 128 for character in text)
 
 
 def read_file(filename: Text, encoding: Text = DEFAULT_ENCODING) -> Any: