Add search type & log by type & new grafana dashboard

monitoring/grafana/main-dashboard.json

@@ -18,9 +18,116 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 1,
   "links": [],
+  "liveNow": false,
   "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unitScale": true
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 6,
+        "y": 0
+      },
+      "id": 6,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.3.5",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "search_confirm_counter{job=\"search-analysis\"}",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "search_draft_counter{job=\"search-analysis\"}",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "B",
+          "useBackend": false
+        }
+      ],
+      "title": "Total Search",
+      "transformations": [
+        {
+          "id": "calculateField",
+          "options": {
+            "binary": {
+              "left": "{__name__=\"search_confirm_counter\", instance=\"search-service:8081\", job=\"search-analysis\"}",
+              "right": "{__name__=\"search_draft_counter\", instance=\"search-service:8081\", job=\"search-analysis\"}"
+            },
+            "mode": "binary",
+            "reduce": {
+              "reducer": "sum"
+            },
+            "replaceFields": true
+          }
+        }
+      ],
+      "type": "stat"
+    },
     {
       "datasource": {
         "type": "prometheus",
@@ -76,15 +183,16 @@
                 "value": 80
               }
             ]
-          }
+          },
+          "unitScale": true
         },
         "overrides": []
       },
       "gridPos": {
         "h": 11,
         "w": 24,
         "x": 0,
-        "y": 0
+        "y": 8
       },
       "id": 5,
       "options": {
@@ -99,6 +207,7 @@
           "sort": "none"
         }
       },
+      "pluginVersion": "10.3.5",
       "targets": [
         {
           "datasource": {
@@ -107,13 +216,30 @@
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "search_counter{job=\"search-analysis\"}",
+          "expr": "search_confirm_counter{job=\"search-analysis\"}",
           "fullMetaSearch": false,
           "includeNullMetadata": true,
           "key": "Q-299fc1dd-904b-48d5-8c4e-37be4a9fe269-0",
           "legendFormat": "__auto",
           "range": true,
-          "refId": "A",
+          "refId": "Confirm search",
+          "useBackend": false
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "search_draft_counter{job=\"search-analysis\"}",
+          "fullMetaSearch": false,
+          "hide": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "Draft search",
           "useBackend": false
         }
       ],
@@ -129,7 +255,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 11
+        "y": 19
       },
       "id": 2,
       "options": {
@@ -166,7 +292,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 11
+        "y": 19
       },
       "id": 3,
       "options": {
@@ -203,7 +329,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 19
+        "y": 27
       },
       "id": 1,
       "options": {
@@ -240,7 +366,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 19
+        "y": 27
       },
       "id": 4,
       "options": {
@@ -269,6 +395,7 @@
       "type": "logs"
     }
   ],
+  "refresh": "",
   "schemaVersion": 39,
   "tags": [],
   "templating": {
@@ -278,10 +405,11 @@
     "from": "now-1h",
     "to": "now"
   },
+  "timeRangeUpdatedDuringEditOrView": false,
   "timepicker": {},
   "timezone": "browser",
   "title": "Main Dashboard",
   "uid": "fdgi2aql57ocga",
-  "version": 5,
+  "version": 1,
   "weekStart": ""
 }

search-esdb-service/app.env

@@ -8,7 +8,8 @@ RECORD_DATA_PATH=/record
 LDA_DATA_PATH=/lda
 STOPWORD_PATH=/stopword
 LOGS_PATH=/logs
-SEARCH_LOG_PATH=/searchLogs.txt
+SEARCH_LOG_DRAFT_PATH=/searchLogsDraft.txt
+SEARCH_LOG_CONFIRM_PATH=/searchLogsConfirm.txt
 GRPC_PORT=50051
 RABBITMQ_USERNAME=guest
 RABBITMQ_PASSWORD=guest

search-esdb-service/config/config.go

@@ -30,12 +30,13 @@ type (
 		Password string
 	}
 	Static struct {
-		DataPath     string
-		RecordPath   string
-		LDAPath      string
-		StopwordPath string
-		LogsPath     string
-		SearchLogsPath string
+		DataPath              string
+		RecordPath            string
+		LDAPath               string
+		StopwordPath          string
+		LogsPath              string
+		SearchLogsDraftPath   string
+		SearchLogsConfirmPath string
 	}
 )
 
@@ -69,12 +70,13 @@ func ReadConfig() {
 			Password: viper.GetString("RABBITMQ_PASSWORD"),
 		},
 		Static: Static{
-			DataPath:      viper.GetString("STATIC_DATA"),
-			RecordPath:    viper.GetString("RECORD_DATA_PATH"),
-			LDAPath:       viper.GetString("LDA_DATA_PATH"),
-			StopwordPath:  viper.GetString("STOPWORD_PATH"),
-			LogsPath:      viper.GetString("LOGS_PATH"),
-			SearchLogsPath: viper.GetString("SEARCH_LOG_PATH"),
+			DataPath:              viper.GetString("STATIC_DATA"),
+			RecordPath:            viper.GetString("RECORD_DATA_PATH"),
+			LDAPath:               viper.GetString("LDA_DATA_PATH"),
+			StopwordPath:          viper.GetString("STOPWORD_PATH"),
+			LogsPath:              viper.GetString("LOGS_PATH"),
+			SearchLogsDraftPath:   viper.GetString("SEARCH_LOG_DRAFT_PATH"),
+			SearchLogsConfirmPath: viper.GetString("SEARCH_LOG_CONFIRM_PATH"),
 		},
 	}
 }

search-esdb-service/constant/searchType.go

@@ -1,7 +1,9 @@
 package constant
 
 const (
-	SEARCH_BY_DEFAULT = "default"
-	SEARCH_BY_TF_IDF  = "tf-idf"
-	SEARCH_BY_LDA     = "lda"
+	SEARCH_BY_DEFAULT      = "default"
+	SEARCH_BY_TF_IDF       = "tf-idf"
+	SEARCH_BY_LDA          = "lda"
+	SEARCH_STATUS_DRAFTING = "draft"
+	SEARCH_STATUS_CONFIRM  = "confirm"
 )

search-esdb-service/monitoring/prometheus.go

@@ -8,17 +8,36 @@ import (
 //* passed to the usecase but right now we only count the search requests
 //* so we make it simple and just use a global variable
 
-var searchCounter prometheus.Counter
+var searchCounter struct {
+	draftCounter   prometheus.Counter
+	confirmCounter prometheus.Counter
+}
 
 func NewMonitoring() {
-	searchCounter = prometheus.NewCounter(prometheus.CounterOpts{
-		Name: "search_counter",
-		Help: "The total number of searche requests",
+
+	searchDraftCounter := prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "search_draft_counter",
+		Help: "The total number of search requests with status draft",
+	})
+
+	searchConfirmCounter := prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "search_confirm_counter",
+		Help: "The total number of search requests with status confirm",
 	})
 
-	prometheus.MustRegister(searchCounter)
+	prometheus.MustRegister(searchDraftCounter)
+
+	prometheus.MustRegister(searchConfirmCounter)
+
+	searchCounter.draftCounter = searchDraftCounter
+	searchCounter.confirmCounter = searchConfirmCounter
 }
 
-func GetSearchCounter() prometheus.Counter {
-	return searchCounter
+func MonitoringSearch(searchStatus string) {
+	switch searchStatus {
+	case "draft":
+		searchCounter.draftCounter.Inc()
+	case "confirm":
+		searchCounter.confirmCounter.Inc()
+	}
 }

search-esdb-service/record/handlers/recordHttpSearch.go

@@ -14,9 +14,6 @@ import (
 )
 
 func (r *recordHttpHandler) Search(c *gin.Context) {
-	searchCounter := monitoring.GetSearchCounter()
-	searchCounter.Inc()
-
 	handlerOpts := NewHandlerOpts(c)
 	handlerOpts.Params = c.Request.URL.Query()
 
@@ -29,9 +26,6 @@ func (r *recordHttpHandler) Search(c *gin.Context) {
 		return
 	}
 
-	cfg := config.GetConfig()
-	logging.WriteLogsToFile(cfg.Static.LogsPath, cfg.Static.SearchLogsPath, "Search: "+query)
-
 	// retrieve amount
 	sAmount := c.Query("amount")
 	amount := 50 // default to 50 results
@@ -52,6 +46,23 @@ func (r *recordHttpHandler) Search(c *gin.Context) {
 		searchType = constant.SEARCH_BY_DEFAULT
 	}
 
+	// retreive search status
+	searchStatus := c.Query("searchStatus")
+	if searchStatus == "" {
+		searchStatus = constant.SEARCH_STATUS_DRAFTING
+	}
+
+	cfg := config.GetConfig()
+	searchLogsPath := cfg.Static.SearchLogsDraftPath
+	if searchStatus == constant.SEARCH_STATUS_CONFIRM {
+		searchLogsPath = cfg.Static.SearchLogsConfirmPath
+	}
+
+	logging.WriteLogsToFile(cfg.Static.LogsPath, searchLogsPath, "Search: "+query)
+
+	// monitor search
+	monitoring.MonitoringSearch(searchStatus)
+
 	// search for records
 	records, err := r.recordUsecase.Search("record", query, searchType, amount)
 	if err != nil {
@@ -70,6 +81,7 @@ func (r *recordHttpHandler) Search(c *gin.Context) {
 		Response: records,
 		OptionalResponse: &SearchRecordLogResponse{
 			Length: len(records.Results),
+			Status: searchStatus,
 		},
 	}