Merge pull request #642 from TristanCacqueray/legacy-td-removal

Consolidate the task data API with the entity API

CHANGELOG.md

@@ -10,6 +10,8 @@ All notable changes to this project will be documented in this file.
 
 ### Removed
 
+- [api] The legacy task data last_updated/add/commit are replaced by the new crawler commit_info/add/commit api.
+
 ### Fixed
 
 ## [1.1.0] - 2021-09-21

Makefile

@@ -2,8 +2,9 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 .PHONY: up-stage
 
-MESSAGES = monocle/user_group.proto monocle/task_data.proto monocle/search.proto monocle/config.proto
-BACKEND_ONLY = monocle/change.proto monocle/project.proto monocle/crawler.proto
+MESSAGES = monocle/user_group.proto monocle/search.proto monocle/config.proto
+CRAWLER = monocle/change.proto monocle/crawler.proto
+BACKEND_ONLY = monocle/project.proto
 PINCLUDE = -I /usr/include $(PROTOC_FLAGS) -I ./protos/
 
 codegen: codegen-python codegen-javascript codegen-stubs codegen-openapi codegen-haskell codegen-doc
@@ -20,12 +21,12 @@ codegen-stubs:
 	rm -Rf srcgen/
 
 codegen-haskell:
-	sh -c 'for pb in $(MESSAGES) $(BACKEND_ONLY); do compile-proto-file --includeDir /usr/include --includeDir protos/ --includeDir ${PROTOBUF_SRC} --proto $${pb} --out haskell/codegen/; done'
+	sh -c 'for pb in $(MESSAGES) $(CRAWLER) $(BACKEND_ONLY); do compile-proto-file --includeDir /usr/include --includeDir protos/ --includeDir ${PROTOBUF_SRC} --proto $${pb} --out haskell/codegen/; done'
 	find haskell/codegen/ -type f -name "*.hs" -exec sed -i {} -e '1i{-# LANGUAGE NoGeneralisedNewtypeDeriving #-}' \;
 	find haskell/codegen/ -type f -name "*.hs" -exec ormolu -i {} \;
 
 codegen-python:
-	protoc $(PINCLUDE) --python_out=./ --mypy_out=./ $(MESSAGES)
+	protoc $(PINCLUDE) --python_out=./ --mypy_out=./ $(MESSAGES) $(CRAWLER)
 	black monocle/*.py*
 
 codegen-javascript:

codegen/MonocleCodegen.hs

@@ -77,7 +77,6 @@ protoToHaskell = fromProto headers mkService
         "import Monocle.Client (MonocleClient, monocleReq)",
         "import Monocle.Config",
         "import Monocle.Crawler",
-        "import Monocle.TaskData",
         "import Monocle.Search",
         "import Monocle.UserGroup"
       ]

contrib/client.py

@@ -6,7 +6,8 @@
 import argparse
 import json
 import monocle.webapi as M
-import monocle.task_data_pb2 as T
+import monocle.crawler_pb2 as Crawler
+import monocle.change_pb2 as Change
 
 
 def usage():
@@ -48,9 +49,10 @@ def usage():
         print("usage: add-td workspace crawler apikey json")
         exit(1)
     tdjson = json.loads(td)
-    td = T.TaskData(**tdjson)
-    query = M.TaskDataAddRequest(
-        index=workspace, crawler=crawler, apikey=apikey, items=[td]
+    td = Change.TaskData(**tdjson)
+    entity = Crawler.Entity(td_name=crawler)
+    query = M.AddDocRequest(
+        index=workspace, crawler=crawler, apikey=apikey, entity=entity, task_datas=[td]
     )
-    resp = M.task_data_task_data_add(args.url, query)
+    resp = M.crawler_add_doc(args.url, query)
     print(resp)