diff --git a/.github/workflows/datasource.yml b/.github/workflows/datasource.yml index 6ffc33e..3525eac 100644 --- a/.github/workflows/datasource.yml +++ b/.github/workflows/datasource.yml @@ -23,5 +23,5 @@ jobs: - name: Copy Data source directory to Server run: | - scp -i ~/.ssh/id_rsa data ${{ secrets.LINUX_USERNAME }}@${{ secrets.LINUX_HOST }}:~/dhammanava-search + scp -r -i ~/.ssh/id_rsa data ${{ secrets.LINUX_USERNAME }}@${{ secrets.LINUX_HOST }}:~/dhammanava-search \ No newline at end of file diff --git a/.github/workflows/monitoring.yml b/.github/workflows/monitoring.yml index a56cffa..f26d0d2 100644 --- a/.github/workflows/monitoring.yml +++ b/.github/workflows/monitoring.yml @@ -32,4 +32,4 @@ jobs: key: ${{ secrets.LINUX_PRIVATE_KEY }} script: | cd dhammanava-search - docker compose -f docker-compose.prod.yml up -d loki promtail grafana + docker compose -f docker-compose.prod.yml up -d loki promtail grafana prometheus diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index e40d555..65e97f3 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -21,6 +21,7 @@ services: - 8081:8081 depends_on: - elastic-db + - rabbitmq labels: logging: "promtail" loggin_jobname: "containerlogs" @@ -153,6 +154,9 @@ services: loggin_jobname: "containerlogs" depends_on: - data-db + - rabbitmq + - auth-service + - search-service networks: - dhammanava_network @@ -173,6 +177,7 @@ services: rabbitmq: image: "rabbitmq:3.12-management" + container_name: rabbitmq ports: - "5672:5672" - "15672:15672" @@ -223,13 +228,13 @@ services: grafana: image: grafana/grafana:10.3.5 + container_name: grafana ports: - 3000:3000 volumes: - ./monitoring/grafana/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml - ./monitoring/grafana/main-dashboard.json:/var/lib/grafana/dashboards/dashboard.json - ./monitoring/grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/main.yml - - ./monitoring/grafana/grafana.ini:/etc/grafana/grafana.ini environment: - GF_AUTH_ANONYMOUS_ENABLED=true - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index ed6fc69..0345105 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -22,6 +22,7 @@ services: restart: always depends_on: - elastic-db + - rabbitmq networks: - dhammanava_network @@ -57,6 +58,9 @@ services: ports: - 9200:9200 restart: always + labels: + logging: "promtail" + loggin_jobname: "containerlogs" mem_limit: 1g networks: - dhammanava_network @@ -141,6 +145,9 @@ services: restart: always depends_on: - data-db + - rabbitmq + - auth-service + - search-service labels: logging: "promtail" loggin_jobname: "containerlogs" @@ -164,6 +171,7 @@ services: rabbitmq: image: "rabbitmq:3-alpine" + container_name: rabbitmq-container environment: RABBITMQ_DEFAULT_USER: ${RABBITMQ_USERNAME} RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD} @@ -208,6 +216,8 @@ services: ports: - "3100:3100" command: -config.file=/etc/loki/local-config.yaml + volumes: + - ./volumes/monitoring/loki:/loki networks: - dhammanava_network @@ -226,6 +236,7 @@ services: grafana: image: grafana/grafana:10.3.5 + container_name: grafana ports: - 3000:3000 volumes: @@ -234,9 +245,7 @@ services: - ./monitoring/grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/main.yml - ./monitoring/grafana/grafana.ini.prod:/etc/grafana/grafana.ini environment: - - GF_AUTH_ANONYMOUS_ENABLED=true - - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin - - GF_AUTH_DISABLE_LOGIN_FORM=true + - GF_AUTH_DISABLE_LOGIN_FORM=false - GF_SECURITY_ADMIN_USER=${GRAFANA_USERNAME} - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD} networks: @@ -247,6 +256,7 @@ services: container_name: prometheus volumes: - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml + - ./volumes/monitoring/prometheus:/prometheus command: - "--config.file=/etc/prometheus/prometheus.yml" networks: diff --git a/frontend/.env b/frontend/.env new file mode 100644 index 0000000..53a7415 --- /dev/null +++ b/frontend/.env @@ -0,0 +1 @@ +VITE_EVALUATION_FORM_URL=DHAMMANAVA_EVALUATION_FORM_URL \ No newline at end of file diff --git a/frontend/.gitignore b/frontend/.gitignore index 0cd575e..6a22c34 100644 --- a/frontend/.gitignore +++ b/frontend/.gitignore @@ -22,4 +22,4 @@ dist-ssr *.njsproj *.sln *.sw? -.env.production \ No newline at end of file +!.env \ No newline at end of file diff --git a/frontend/src/components/search/SearchField.tsx b/frontend/src/components/search/SearchField.tsx index 07a3595..fee2954 100644 --- a/frontend/src/components/search/SearchField.tsx +++ b/frontend/src/components/search/SearchField.tsx @@ -17,6 +17,7 @@ import { SearchIcon } from "@chakra-ui/icons"; import { useState } from "react"; import { searchService } from "../../service/search"; import { SearchResultInterface } from "../../models/qa"; +import { SEARCH_STATUS, SEARCH_TYPE } from "../../constant"; interface SearchOptions { key: string; @@ -121,7 +122,7 @@ function SearchField({ if (q) { query = q.question; } - const response = await searchService(query); + const response = await searchService(query,SEARCH_TYPE.DEFAULT,SEARCH_STATUS.CONFIRM); const tokens = [query, ...response.tokens]; @@ -151,6 +152,12 @@ function SearchField({ { + if (e.key === "Enter") { + onSelectInputHandle(e); + } + }} variant="search_bar" value={searchParam || ""} placeholder="ค้นหาเลย" diff --git a/frontend/src/constant/index.ts b/frontend/src/constant/index.ts index b3e1755..e19dcd2 100644 --- a/frontend/src/constant/index.ts +++ b/frontend/src/constant/index.ts @@ -58,3 +58,14 @@ export const REQUEST_STATUS = { PENDING: "pending", REVIEWED: "reviewed", }; + +export const SEARCH_STATUS = { + DRAFT : "draft", + CONFIRM : "confirm", + DEFAULT : "draft" +} + +export const SEARCH_TYPE = { + TF_IDF: "tf_idf", + DEFAULT: "tf_idf" +} diff --git a/frontend/src/service/search/search.ts b/frontend/src/service/search/search.ts index 1386f78..e426ae8 100644 --- a/frontend/src/service/search/search.ts +++ b/frontend/src/service/search/search.ts @@ -2,20 +2,23 @@ import axios from "../axiosInstance"; import { CreateCustomError } from "../error"; import { DataItem, SearchResultInterface } from "../../models/qa"; import { searchURL } from "../../constant/serviceURL"; +import { SEARCH_STATUS, SEARCH_TYPE } from "../../constant"; /** * Performs a search query using the specified query string. * * @param {string} query - The search query string. * @param {string} searchType - Optional. The search type to use. Defaults to "tf_idf". + * @param {string} searchStatus - Optional. The search status to use. Defaults to "draft". * @return {Promise} - A promise that resolves with the search results. */ export const searchService = async ( query: string, - searchType: string = "tf_idf" + searchType: string = SEARCH_TYPE.DEFAULT, + searchStatus: string = SEARCH_STATUS.DEFAULT ): Promise => { try { const response = await axios.get( - `${searchURL}/search?query=${query}&searchType=${searchType}` + `${searchURL}/search?query=${query}&searchType=${searchType}&searchStatus=${searchStatus}` ); const records: DataItem[] = response.data.results.map((item: DataItem) => { diff --git a/monitoring/grafana/main-dashboard.json b/monitoring/grafana/main-dashboard.json index 3bd5fa5..0b48d27 100644 --- a/monitoring/grafana/main-dashboard.json +++ b/monitoring/grafana/main-dashboard.json @@ -18,9 +18,116 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 1, "links": [], + "liveNow": false, "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 6, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "search_confirm_counter{job=\"search-analysis\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "search_draft_counter{job=\"search-analysis\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Total Search", + "transformations": [ + { + "id": "calculateField", + "options": { + "binary": { + "left": "{__name__=\"search_confirm_counter\", instance=\"search-service:8081\", job=\"search-analysis\"}", + "right": "{__name__=\"search_draft_counter\", instance=\"search-service:8081\", job=\"search-analysis\"}" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, { "datasource": { "type": "prometheus", @@ -76,7 +183,8 @@ "value": 80 } ] - } + }, + "unitScale": true }, "overrides": [] }, @@ -84,7 +192,7 @@ "h": 11, "w": 24, "x": 0, - "y": 0 + "y": 8 }, "id": 5, "options": { @@ -99,6 +207,7 @@ "sort": "none" } }, + "pluginVersion": "10.3.5", "targets": [ { "datasource": { @@ -107,13 +216,30 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "search_counter{job=\"search-analysis\"}", + "expr": "search_confirm_counter{job=\"search-analysis\"}", "fullMetaSearch": false, "includeNullMetadata": true, "key": "Q-299fc1dd-904b-48d5-8c4e-37be4a9fe269-0", "legendFormat": "__auto", "range": true, - "refId": "A", + "refId": "Confirm search", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "search_draft_counter{job=\"search-analysis\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "Draft search", "useBackend": false } ], @@ -129,7 +255,7 @@ "h": 8, "w": 12, "x": 0, - "y": 11 + "y": 19 }, "id": 2, "options": { @@ -166,7 +292,7 @@ "h": 8, "w": 12, "x": 12, - "y": 11 + "y": 19 }, "id": 3, "options": { @@ -203,7 +329,7 @@ "h": 8, "w": 12, "x": 0, - "y": 19 + "y": 27 }, "id": 1, "options": { @@ -240,7 +366,7 @@ "h": 8, "w": 12, "x": 12, - "y": 19 + "y": 27 }, "id": 4, "options": { @@ -269,6 +395,7 @@ "type": "logs" } ], + "refresh": "", "schemaVersion": 39, "tags": [], "templating": { @@ -278,10 +405,11 @@ "from": "now-1h", "to": "now" }, + "timeRangeUpdatedDuringEditOrView": false, "timepicker": {}, "timezone": "browser", "title": "Main Dashboard", "uid": "fdgi2aql57ocga", - "version": 5, + "version": 1, "weekStart": "" } \ No newline at end of file diff --git a/search-esdb-service/app.env b/search-esdb-service/app.env index 1a13347..274f76d 100644 --- a/search-esdb-service/app.env +++ b/search-esdb-service/app.env @@ -8,7 +8,8 @@ RECORD_DATA_PATH=/record LDA_DATA_PATH=/lda STOPWORD_PATH=/stopword LOGS_PATH=/logs -SEARCH_LOG_PATH=/searchLogs.txt +SEARCH_LOG_DRAFT_PATH=/searchLogsDraft.txt +SEARCH_LOG_CONFIRM_PATH=/searchLogsConfirm.txt GRPC_PORT=50051 RABBITMQ_USERNAME=guest RABBITMQ_PASSWORD=guest diff --git a/search-esdb-service/config/config.go b/search-esdb-service/config/config.go index 1bf5e59..c2ad905 100644 --- a/search-esdb-service/config/config.go +++ b/search-esdb-service/config/config.go @@ -30,12 +30,13 @@ type ( Password string } Static struct { - DataPath string - RecordPath string - LDAPath string - StopwordPath string - LogsPath string - SearchLogsPath string + DataPath string + RecordPath string + LDAPath string + StopwordPath string + LogsPath string + SearchLogsDraftPath string + SearchLogsConfirmPath string } ) @@ -69,12 +70,13 @@ func ReadConfig() { Password: viper.GetString("RABBITMQ_PASSWORD"), }, Static: Static{ - DataPath: viper.GetString("STATIC_DATA"), - RecordPath: viper.GetString("RECORD_DATA_PATH"), - LDAPath: viper.GetString("LDA_DATA_PATH"), - StopwordPath: viper.GetString("STOPWORD_PATH"), - LogsPath: viper.GetString("LOGS_PATH"), - SearchLogsPath: viper.GetString("SEARCH_LOG_PATH"), + DataPath: viper.GetString("STATIC_DATA"), + RecordPath: viper.GetString("RECORD_DATA_PATH"), + LDAPath: viper.GetString("LDA_DATA_PATH"), + StopwordPath: viper.GetString("STOPWORD_PATH"), + LogsPath: viper.GetString("LOGS_PATH"), + SearchLogsDraftPath: viper.GetString("SEARCH_LOG_DRAFT_PATH"), + SearchLogsConfirmPath: viper.GetString("SEARCH_LOG_CONFIRM_PATH"), }, } } diff --git a/search-esdb-service/constant/searchType.go b/search-esdb-service/constant/searchType.go index 0c75adc..40a4d90 100644 --- a/search-esdb-service/constant/searchType.go +++ b/search-esdb-service/constant/searchType.go @@ -1,7 +1,9 @@ package constant const ( - SEARCH_BY_DEFAULT = "default" - SEARCH_BY_TF_IDF = "tf-idf" - SEARCH_BY_LDA = "lda" + SEARCH_BY_DEFAULT = "default" + SEARCH_BY_TF_IDF = "tf-idf" + SEARCH_BY_LDA = "lda" + SEARCH_STATUS_DRAFTING = "draft" + SEARCH_STATUS_CONFIRM = "confirm" ) diff --git a/search-esdb-service/monitoring/prometheus.go b/search-esdb-service/monitoring/prometheus.go index d3348b0..daacf6d 100644 --- a/search-esdb-service/monitoring/prometheus.go +++ b/search-esdb-service/monitoring/prometheus.go @@ -8,17 +8,36 @@ import ( //* passed to the usecase but right now we only count the search requests //* so we make it simple and just use a global variable -var searchCounter prometheus.Counter +var searchCounter struct { + draftCounter prometheus.Counter + confirmCounter prometheus.Counter +} func NewMonitoring() { - searchCounter = prometheus.NewCounter(prometheus.CounterOpts{ - Name: "search_counter", - Help: "The total number of searche requests", + + searchDraftCounter := prometheus.NewCounter(prometheus.CounterOpts{ + Name: "search_draft_counter", + Help: "The total number of search requests with status draft", + }) + + searchConfirmCounter := prometheus.NewCounter(prometheus.CounterOpts{ + Name: "search_confirm_counter", + Help: "The total number of search requests with status confirm", }) - prometheus.MustRegister(searchCounter) + prometheus.MustRegister(searchDraftCounter) + + prometheus.MustRegister(searchConfirmCounter) + + searchCounter.draftCounter = searchDraftCounter + searchCounter.confirmCounter = searchConfirmCounter } -func GetSearchCounter() prometheus.Counter { - return searchCounter +func MonitoringSearch(searchStatus string) { + switch searchStatus { + case "draft": + searchCounter.draftCounter.Inc() + case "confirm": + searchCounter.confirmCounter.Inc() + } } diff --git a/search-esdb-service/record/handlers/recordHttpSearch.go b/search-esdb-service/record/handlers/recordHttpSearch.go index 65bb1d7..53f6f1b 100644 --- a/search-esdb-service/record/handlers/recordHttpSearch.go +++ b/search-esdb-service/record/handlers/recordHttpSearch.go @@ -14,9 +14,6 @@ import ( ) func (r *recordHttpHandler) Search(c *gin.Context) { - searchCounter := monitoring.GetSearchCounter() - searchCounter.Inc() - handlerOpts := NewHandlerOpts(c) handlerOpts.Params = c.Request.URL.Query() @@ -29,9 +26,6 @@ func (r *recordHttpHandler) Search(c *gin.Context) { return } - cfg := config.GetConfig() - logging.WriteLogsToFile(cfg.Static.LogsPath, cfg.Static.SearchLogsPath, "Search: "+query) - // retrieve amount sAmount := c.Query("amount") amount := 50 // default to 50 results @@ -52,6 +46,23 @@ func (r *recordHttpHandler) Search(c *gin.Context) { searchType = constant.SEARCH_BY_DEFAULT } + // retreive search status + searchStatus := c.Query("searchStatus") + if searchStatus == "" { + searchStatus = constant.SEARCH_STATUS_DRAFTING + } + + cfg := config.GetConfig() + searchLogsPath := cfg.Static.SearchLogsDraftPath + if searchStatus == constant.SEARCH_STATUS_CONFIRM { + searchLogsPath = cfg.Static.SearchLogsConfirmPath + } + + logging.WriteLogsToFile(cfg.Static.LogsPath, searchLogsPath, "Search: "+query) + + // monitor search + monitoring.MonitoringSearch(searchStatus) + // search for records records, err := r.recordUsecase.Search("record", query, searchType, amount) if err != nil { @@ -70,6 +81,7 @@ func (r *recordHttpHandler) Search(c *gin.Context) { Response: records, OptionalResponse: &SearchRecordLogResponse{ Length: len(records.Results), + Status: searchStatus, }, } diff --git a/search-esdb-service/record/handlers/responseStruct.go b/search-esdb-service/record/handlers/responseStruct.go index 5058552..b1cbfe9 100644 --- a/search-esdb-service/record/handlers/responseStruct.go +++ b/search-esdb-service/record/handlers/responseStruct.go @@ -21,6 +21,7 @@ func (r Response) LogValue() slog.Value { type SearchRecordLogResponse struct { Length int + Status string } type RecordIndexLogResponse struct { diff --git a/search-esdb-service/record/usecases/recordUsecaseUpdate.go b/search-esdb-service/record/usecases/recordUsecaseUpdate.go index d4d79da..105fa6f 100644 --- a/search-esdb-service/record/usecases/recordUsecaseUpdate.go +++ b/search-esdb-service/record/usecases/recordUsecaseUpdate.go @@ -7,6 +7,7 @@ import ( ) func (r *recordUsecaseImpl) UpdateRecord(record *models.UpdateRecord) error { + //TODO : We have to generate new vector for each record if its update updateRecordEntity := helper.UpdateRecordModelToEntity(record) if err := r.recordRepository.UpdateRecord(updateRecordEntity); err != nil { slog.Error("Failed to update record", diff --git a/search-esdb-service/util/util.go b/search-esdb-service/util/util.go index fcde594..a710014 100644 --- a/search-esdb-service/util/util.go +++ b/search-esdb-service/util/util.go @@ -25,6 +25,11 @@ func RemoveSliceFromArrays(arr []string, slice []string) []string { } result = append(result, a) } + + if len(result) == 0 { + return arr + } + return result }