feat(crafter): add Any2Buffer convert

jina/drivers/craft.py

@@ -71,8 +71,9 @@ class DocCraftDriver(BaseCraftDriver):
     def __call__(self, *args, **kwargs):
         for d in self.req.docs:
             ret = self.exec_fn(**pb_obj2dict(d, self.exec.required_keys))
-            for k, v in ret.items():
-                setattr(d, k, v)
+            if ret:
+                for k, v in ret.items():
+                    setattr(d, k, v)
 
 
 class DocMIMEDriver(DocCraftDriver):

jina/executors/crafters/convert.py

@@ -14,8 +14,9 @@ class FilePath2Buffer(BaseDocCrafter):
 
     def craft(self, file_path: str, *args, **kwargs):
         if urllib.parse.urlparse(file_path).scheme in {'http', 'https', 'data'}:
-            tmp = urllib.request.urlopen(file_path)
-            buffer = tmp.file.read()
+            page = urllib.request.Request(file_path, headers={'User-Agent': 'Mozilla/5.0'})
+            tmp = urllib.request.urlopen(page)
+            buffer = tmp.read()
         elif os.path.exists(file_path):
             with open(file_path, 'rb') as fp:
                 buffer = fp.read()
@@ -28,10 +29,22 @@ class DataURI2Buffer(FilePath2Buffer):
     """ Convert a data URI doc to a buffer doc.
     """
 
-    def craft(self, data_uri, *args, **kwargs):
+    def craft(self, data_uri: str, *args, **kwargs):
         return super().craft(data_uri)
 
 
+class Any2Buffer(DataURI2Buffer):
+    def craft(self, file_path: str, data_uri: str, buffer: bytes, *args, **kwargs):
+        if buffer:
+            pass
+        elif file_path:
+            return super(FilePath2Buffer, self).craft(file_path)
+        elif data_uri:
+            return super(DataURI2Buffer, self).craft(data_uri)
+        else:
+            raise ValueError('this document has no "file_path", no "data_uri" and no "buffer" set')
+
+
 class FilePath2DataURI(FilePath2Buffer):
     def __init__(self, charset: str = 'utf-8', base64: bool = False, *args, **kwargs):
         """ Convert file path doc to data uri doc.

jina/logging/sse.py

@@ -94,7 +94,7 @@ def get_podargs():
     @app.route(_config['endpoints']['shutdown'])
     def shutdown():
         from flask import request
-        if not 'werkzeug.server.shutdown' in request.environ:
+        if 'werkzeug.server.shutdown' not in request.environ:
             raise RuntimeError('Not running the development server')
         request.environ['werkzeug.server.shutdown']()
         return 'Server shutting down...'

jina/peapods/gateway.py

@@ -236,11 +236,12 @@ def api(mode):
             if mode_fn is None:
                 return http_error(f'mode: {mode} is not supported yet', 405)
             content = request.json
-            content['mode'] = ClientMode.from_string(mode)
-            content['input_type'] = ClientInputType.DATA_URI
-            if not 'data' in content:
+            if 'data' not in content:
                 return http_error('"data" field is empty', 406)
 
+            content['mode'] = ClientMode.from_string(mode)
+            content['input_type'] = ClientInputType.from_string(content.get('input_type', 'data_uri'))
+
             results = get_result_in_json(getattr(python.request, mode)(**content))
             return Response(asyncio.run(results),
                             status=200,

tests/executors/crafters/test_mime.py

@@ -20,6 +20,14 @@ def input_fn2(pattern='../../*.*'):
         yield g
 
 
+def input_fn3():
+    for g in ['test_mime.py',  # local file
+              'https://github.com/jina-ai/jina/raw/master/.github/1500%D1%85667.gif?raw=true',
+              'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAA2ElEQVR4nADIADf/AxWcWRUeCEeBO68T3u1qLWarHqMaxDnxhAEaLh0Ssu6ZGfnKcjP4CeDLoJok3o4aOPYAJocsjktZfo4Z7Q/WR1UTgppAAdguAhR+AUm9AnqRH2jgdBZ0R+kKxAFoAME32BL7fwQbcLzhw+dXMmY9BS9K8EarXyWLH8VYK1MACkxlLTY4Eh69XfjpROqjE7P0AeBx6DGmA8/lRRlTCmPkL196pC0aWBkVs2wyjqb/LABVYL8Xgeomjl3VtEMxAeaUrGvnIawVh/oBAAD///GwU6v3yCoVAAAAAElFTkSuQmCC',
+              'https://cdn.bulbagarden.net/upload/thumb/2/21/001Bulbasaur.png/240px-001Bulbasaur.png']:
+        yield g
+
+
 class MyTestCase(JinaTestCase):
     def test_dummy_seg(self):
         f = Flow().add(yaml_path='!Buffer2DataURI\nwith: {mimetype: png}')
@@ -43,6 +51,13 @@ def test_aba(self):
         with f:
             f.index(input_fn=input_fn, output_fn=print)
 
+    def test_any2buffer(self):
+        f = (Flow().add(yaml_path='Any2Buffer')
+             .add(yaml_path='Buffer2DataURI'))
+
+        with f:
+            f.index(input_fn=input_fn3, output_fn=print, input_type=ClientInputType.DATA_URI)
+
     # def test_dummy_seg_random(self):
     #     f = Flow().add(yaml_path='../../yaml/dummy-seg-random.yml')
     #     with f: