Skip to content

Commit

Permalink
comment out printlns - temp commit to test int tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@jmaeng72
jmaeng72 committed Jan 2, 2025
1 parent 09a6114 commit 77b6665
Show file tree
Hide file tree
Showing 15 changed files with 135 additions and 77 deletions.
3 changes: 1 addition & 2 deletions common-app-lib/src/cmr/common_app/services/search.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,7 @@
;; use that if so. If the scroll-id is not set and scroll is set to 'defer' then store the
;; search results in the cache and return an empty result (with appropriate headers). Otherwise
;; do normal query/scrolling without using the cache.
(let [_ (println "INSIDE find-concepts...")
scroll-id (:scroll-id query)
(let [scroll-id (:scroll-id query)
[results result-str] (or (pop-scroll-results-from-cache context scroll-id)
(time-concept-search context query))]
(if (and (not scroll-id) (= (:scroll query) "defer"))
Expand Down
3 changes: 1 addition & 2 deletions elastic-utils-lib/src/cmr/elastic_utils/search/query_execution.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,7 @@

(defmethod execute-query :elasticsearch
[context query]
(let [_ (println "INSIDE execute-query :elasticsearch")
[context processed-query] (concept-type-specific-query-processing
(let [[context processed-query] (concept-type-specific-query-processing
context query)
processed-query (pre-process-query-result-features context processed-query)
elastic-results (->> processed-query
Expand Down
13 changes: 6 additions & 7 deletions indexer-app/src/cmr/indexer/data/elasticsearch.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -315,13 +315,14 @@
"Save the document in Elasticsearch, raise error if failed."
[context es-indexes es-type es-doc concept-id revision-id elastic-version options]
(doseq [es-index es-indexes]
(let [_ (println "save-document-in-elastic")
(let [
;_ (println "save-document-in-elastic with options = " options)
conn (context->conn context)
{:keys [ignore-conflict? all-revisions-index?]} options
elastic-id (get-elastic-id concept-id revision-id all-revisions-index?)
_ (println (format "es-helper/put with params: es-index %s es-type %s elastic-id %s elastic-version %s" es-index es-type elastic-id elastic-version))
;_ (println (format "save-document-in-elastic: es-helper/put with params: es-index %s es-type %s elastic-id %s elastic-version %s" es-index es-type elastic-id elastic-version))
result (try-elastic-operation
es-helper/put conn es-index es-type es-doc elastic-id elastic-version)] ;; For all revisions, it will just pick only the concept-id as the elastic-id. Ummm how is this tombstoning the document?
es-helper/put conn es-index es-type es-doc elastic-id elastic-version)]
(when (:error result)
(if (= 409 (:status result))
(if ignore-conflict?
Expand All @@ -342,19 +343,17 @@
([context es-indexes es-type concept-id revision-id elastic-version]
(delete-document context es-indexes es-type concept-id revision-id elastic-version nil))
([context es-indexes _es-type concept-id revision-id elastic-version options]
(println (format "delete-document params: es-indexes %s ^^^^ _es-type %s ^^^^ concept-id %s ^^^^ revision-id %s ^^^^ elastic-version %s ^^^^ options %s ^^^^ " es-indexes _es-type concept-id revision-id elastic-version options))
;(println (format "delete-document params: es-indexes %s ^^^^ _es-type %s ^^^^ concept-id %s ^^^^ revision-id %s ^^^^ elastic-version %s ^^^^ options %s ^^^^ " es-indexes _es-type concept-id revision-id elastic-version options))
(doseq [es-index es-indexes]
;; Cannot use elastisch for deletion as we require special headers on delete
(let [_ (println "delete document in elastic !!!!")
{:keys [admin-token]} (context->es-config context)
(let [{:keys [admin-token]} (context->es-config context)
{:keys [uri http-opts]} (context->conn context)
{:keys [ignore-conflict? all-revisions-index?]} options
elastic-id (get-elastic-id concept-id revision-id all-revisions-index?)
delete-url (if elastic-version
(format "%s/%s/_doc/%s?version=%s&version_type=external_gte"
uri es-index elastic-id elastic-version)
(format "%s/%s/_doc/%s" uri es-index elastic-id))
_ (println "delete-document: delete-url is " delete-url)
response (client/delete delete-url
(merge http-opts
{:headers {"Authorization" admin-token
Expand Down
3 changes: 2 additions & 1 deletion indexer-app/src/cmr/indexer/services/event_handler.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,12 +80,13 @@

(defmethod handle-ingest-event :concept-delete
[context all-revisions-index? {:keys [concept-id revision-id]}]
(println "INSIDE handle-ingest-event JYNA with all-revisions-index = " all-revisions-index?)
;(println "INSIDE handle-ingest-event JYNA with all-revisions-index = " all-revisions-index?)
(when-not (= :humanizer (cc/concept-id->type concept-id))
(let [[tm result] (util/time-execution
(indexer/delete-concept
context concept-id revision-id {:ignore-conflict? true
:all-revisions-index? all-revisions-index?}))]
;(println "Result of handle-ingest-event :concept-delete = " result)
(debug (format (str "Timed function %s handle-ingest-event concept-delete for concept-id %s "
"revision-id %s all-revisions-index? %s took %d ms.")
(str *ns*) concept-id revision-id all-revisions-index? tm))
Expand Down
137 changes: 105 additions & 32 deletions indexer-app/src/cmr/indexer/services/index_service.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -692,73 +692,146 @@
;; ***********************************************
;; Multi method delete-concept

;; TODO maybe create a new delete-concept for just concept-type collection-draft which will delete the CD from both indices -- create a new pathway for it basically
;; and then have publish-draft-concept only write to one queue... this way we aren't worried about the async issue and just do all the deletes at once
(defmulti delete-concept
"Delete the concept with the given id"
(fn [_context concept-id _revision-id _options]
(cs/concept-id->type concept-id)))

;; JYNA this is where the concept delete event goes to in indexer. Indexer picks it up and executes this func.
(defmethod delete-concept :default
[context concept-id revision-id options]
(println (format "****** INSIDE delete-concept :default with params: concept-id %s, revision-id %s, options %s" concept-id revision-id options))
;; Assuming ingest will pass enough info for deletion
;; We should avoid making calls to metadata db to get the necessary info if possible
;(defmethod delete-concept :default
; [context concept-id revision-id options]
; (println (format "****** INSIDE delete-concept :default with params: concept-id %s, revision-id %s, options %s" concept-id revision-id options))
; ;; Assuming ingest will pass enough info for deletion
; ;; We should avoid making calls to metadata db to get the necessary info if possible
; (let [{:keys [all-revisions-index?]} options
; concept-type (cs/concept-id->type concept-id)
; _ (println "******* INSIDE delete-concept: concept-type is : " concept-type)
; ;; CDs are tombstoned when endpoint /publish is used so why would this throw an error? Now I need to look at real-world examples of what is going on
; ;; Now we are seeing this throw errors Jyna, when we keep creating drafts and publishing to the same collection -- because of the two queues executing at diff times problem
; ;; TODO do we really want this to fail if we can't find the concept in the db?
; concept (meta-db/get-concept context concept-id revision-id)
; _ (println "****** delete-concept: THE CONCEPT RETURNED IN DELETE-CONCEPT. SHOULD BE TOMBSTONED CONCEPT.") ;; this gets back the :delete = true metadata from DB
; _ (println (format "****** concept-id= %s, revision-id = %s, deleted = %s, format =%s, native-id = %s" (:concept-id concept) (:revisions-id concept) (:deleted concept) (:format concept) (:native-id concept)))
; elastic-version (get-elastic-version context concept)]
; (when (indexing-applicable? concept-type all-revisions-index?)
; (println (format "****** delete-concept: Indexing is applicable for concept-type: %s and all-revisions-index %s" concept-type all-revisions-index?))
; (info (get-concept-delete-log-string concept-type context concept-id revision-id all-revisions-index?))
; (let [index-names (idx-set/get-concept-index-names context concept-id revision-id options)
; concept-mapping-types (idx-set/get-concept-mapping-types context)
; elastic-options (select-keys options [:all-revisions-index? :ignore-conflict?])
; _ (println "elastic-options = " elastic-options)]
; (if all-revisions-index?
; ;; save tombstone in all revisions collection index
; (let [_ (println "****** delete-concept: all-revisions-index is true") ;; TODO jyna we go into this section
; es-doc (if (cs/generic-concept? concept-type)
; (es/parsed-concept->elastic-doc context concept (json/parse-string (:metadata concept) true))
; (es/parsed-concept->elastic-doc context concept (:extra-fields concept)))
; [tm result] (util/time-execution
; (es/save-document-in-elastic
; context index-names (concept-mapping-types concept-type)
; es-doc concept-id revision-id elastic-version elastic-options))]
; (debug (format "Timed function %s/delete-concept saving tombstone in all-revisions-index took %d ms." (str *ns*) tm))
; (println "****** Saving 'tombstoned' concept resulted in = " result)
; ;; TODO: JYNA Not sure what is going on here... but not convinced that this 'save-document-in-elastic' func is actually deleting the document in elastic like we need it to here...
; result)
; ;; else delete concept from primary concept index
; (do
; (println "****** delete-concept: all-revisions-index is false")
; (es/delete-document
; context index-names (concept-mapping-types concept-type)
; concept-id revision-id elastic-version elastic-options)
; ;; Index a deleted-granule document when granule is deleted
; ;; JYNA all these delete requests to ES are API calls -- are they synchronous?
; (when (= :granule concept-type)
; (let [[tm result] (util/time-execution
; (dg/index-deleted-granule context concept concept-id revision-id elastic-version elastic-options))]
; (debug (format "Timed function %s index-deleted-granule took %d ms." (str *ns*) tm))
; result))
; ;; propagate collection deletion to granules
; (when (= :collection concept-type)
; (let [[tm result] (util/time-execution
; (cascade-collection-delete context concept-mapping-types concept-id revision-id))]
; (debug (format "Timed function %s/cascade-collection-delete took %d ms." (str *ns*) tm))
; result))
; )))
; ;; For draft concept, after the index is deleted, remove it from database.
; (when (cs/is-draft-concept? concept-type)
; (println "****** JYNA this is a draft concept will force delete the draft")
; (meta-db/delete-draft context concept))
; )))

(defn- delete-concept-helper
[context concept concept-id revision-id options]
(let [{:keys [all-revisions-index?]} options
concept-type (cs/concept-id->type concept-id)
;; CDs are tombstoned when endpoint /publish is used so why would this throw an error? Now I need to look at real-world examples of what is going on
;; Now we are seeing this throw errors Jyna, when we keep creating drafts and publishing to the same collection -- because of the two queues executing at diff times problem
;; TODO do we really want this to fail if we can't find the concept in the db?

concept (meta-db/get-concept context concept-id revision-id)
_ (println "****** delete-concept: THE CONCEPT RETURNED IN DELETE-CONCEPT. SHOULD BE TOMBSTONED CONCEPT.") ;; this gets back the :delete = true metadata from DB
_ (println (format "****** concept-id= %s, revision-id = %s, deleted = %s, format =%s, native-id = %s" (:concept-id concept) (:revisions-id concept) (:deleted concept) (:format concept) (:native-id concept)))
elastic-version (get-elastic-version context concept)]
(when (indexing-applicable? concept-type all-revisions-index?)
(println "****** delete-concept: Indexing is applicable")
(println (format "****** delete-concept: Indexing is applicable for concept-type: %s and all-revisions-index %s" concept-type all-revisions-index?))
(info (get-concept-delete-log-string concept-type context concept-id revision-id all-revisions-index?))
(let [index-names (idx-set/get-concept-index-names context concept-id revision-id options)
concept-mapping-types (idx-set/get-concept-mapping-types context)
elastic-options (select-keys options [:all-revisions-index? :ignore-conflict?])]
(if all-revisions-index?
;; save tombstone in all revisions collection index
(let [_ (println "****** delete-concept: all-revisions-index is true") ;; TODO jyna we go into this section
(let [
;_ (println "****** delete-concept: all-revisions-index is true")
es-doc (if (cs/generic-concept? concept-type)
(es/parsed-concept->elastic-doc context concept (json/parse-string (:metadata concept) true))
(es/parsed-concept->elastic-doc context concept (:extra-fields concept)))
[tm result] (util/time-execution
(es/save-document-in-elastic
context index-names (concept-mapping-types concept-type)
es-doc concept-id revision-id elastic-version elastic-options))]
(es/save-document-in-elastic
context index-names (concept-mapping-types concept-type)
es-doc concept-id revision-id elastic-version elastic-options))]
(debug (format "Timed function %s/delete-concept saving tombstone in all-revisions-index took %d ms." (str *ns*) tm))
(println "****** Saving 'tombstoned' concept resulted in = " result)
;(println "****** Saving 'tombstoned' concept resulted in = " result)
;; TODO: JYNA Not sure what is going on here... but not convinced that this 'save-document-in-elastic' func is actually deleting the document in elastic like we need it to here...
result)
;; else delete concept from primary concept index
(do
(println "****** delete-concept: all-revisions-index is false")
;(println "****** delete-concept: all-revisions-index is false")
(es/delete-document
context index-names (concept-mapping-types concept-type)
concept-id revision-id elastic-version elastic-options)
context index-names (concept-mapping-types concept-type)
concept-id revision-id elastic-version elastic-options)
;; Index a deleted-granule document when granule is deleted
;; JYNA all these delete requests to ES are API calls -- are they synchronous?
(when (= :granule concept-type)
(let [[tm result] (util/time-execution
(dg/index-deleted-granule context concept concept-id revision-id elastic-version elastic-options))]
(dg/index-deleted-granule context concept concept-id revision-id elastic-version elastic-options))]
(debug (format "Timed function %s index-deleted-granule took %d ms." (str *ns*) tm))
result))
;; propagate collection deletion to granules
(when (= :collection concept-type)
(let [[tm result] (util/time-execution
(cascade-collection-delete context concept-mapping-types concept-id revision-id))]
(cascade-collection-delete context concept-mapping-types concept-id revision-id))]
(debug (format "Timed function %s/cascade-collection-delete took %d ms." (str *ns*) tm))
result)))))
;; For draft concept, after the index is deleted, remove it from database.
(when (cs/is-draft-concept? concept-type)
(println "****** JYNA this is a draft concept will force delete the draft")
(meta-db/delete-draft context concept))
)))
result))
))))))

(defmethod delete-concept :default
[context concept-id revision-id options]
(println (format "****** INSIDE delete-concept :default with params: concept-id %s, revision-id %s, options %s" concept-id revision-id options))
;; Assuming ingest will pass enough info for deletion
;; We should avoid making calls to metadata db to get the necessary info if possible
(let [concept-type (cs/concept-id->type concept-id)
concept (meta-db/get-concept context concept-id revision-id)]
(delete-concept-helper context concept concept-id revision-id options)
(when (cs/is-draft-concept? concept-type)
(meta-db/delete-draft context concept))))

(defmethod delete-concept :collection-draft
[context concept-id revision-id options]
(println (format "****** INSIDE delete-concept :collection-draft with params: concept-id %s, revision-id %s, options %s" concept-id revision-id options))
(let [concept (meta-db2/get-concept context concept-id revision-id)]
;(if (nil? concept)
; (println "****** INSIDE delete-concept :collection-draft: CONCEPT WAS NIL. DOING NOTHING."))
(when (not (nil? concept))
;(println "****** INSIDE delete-concept :collection-draft: concept was not nil")
(delete-concept-helper context concept concept-id revision-id (assoc options :all-revisions-index? true))
(delete-concept-helper context concept concept-id revision-id (assoc options :all-revisions-index? false))
;; if both all revisions and revision is being processed, then this might have a race condition and throw an error because one deleted the concept before this process could
(try
(meta-db/delete-draft context concept)
(catch Exception e
(println "This either failed to delete or it was because it was already deleted. If already deleted then ignore this error."))))))

(defn- index-association-concept
"Index the association concept identified by the given concept-id and revision-id."
Expand Down
Loading

0 comments on commit 77b6665

Please sign in to comment.