From 886ec0fea96e044508e81f22f664d7707c353934 Mon Sep 17 00:00:00 2001 From: bhima2001 <56587387+bhima2001@users.noreply.github.com> Date: Sun, 17 Nov 2024 12:08:02 +0530 Subject: [PATCH 1/8] #1260: Runs CI only when source code has changes. (#1265) --- .github/workflows/full-test-suite.yml | 10 +++++++++- .github/workflows/linter.yml | 11 ++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/full-test-suite.yml b/.github/workflows/full-test-suite.yml index fe486d978..1a01a31dd 100644 --- a/.github/workflows/full-test-suite.yml +++ b/.github/workflows/full-test-suite.yml @@ -41,11 +41,19 @@ jobs: key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }} restore-keys: | ${{ runner.os }}-go- + - uses: dorny/paths-filter@v3 + id: file_types + with: + filters: | + watch_file_changes: + - '**/*.go' - name: Install dependencies run: go get -v . # Check if there is any version changes from the cache - name: Build run: make build - name: Run Unit tests + if: steps.file_types.outputs.watch_file_changes == 'true' run: make unittest - name: Run Integration tests - run: make test \ No newline at end of file + if: steps.file_types.outputs.watch_file_changes == 'true' + run: make test diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 36571ac22..fb7c4a708 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -21,9 +21,14 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version: stable + - uses: actions/cache@v4 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-golint-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-golint- - name: golangci-lint uses: golangci/golangci-lint-action@v6 with: From 256228a1546d061a8c1a65d8716953accaca6ca6 Mon Sep 17 00:00:00 2001 From: Harish <79055105+c-harish@users.noreply.github.com> Date: Sun, 17 Nov 2024 12:20:29 +0530 Subject: [PATCH 2/8] feat : add support for bytearray/bitmap to append (#1286) --- docs/src/content/docs/commands/APPEND.md | 28 ++++++++++++++- .../commands/http/append_test.go | 17 ++++++++++ .../commands/resp/append_test.go | 6 ++++ .../commands/websocket/append_test.go | 6 ++++ internal/eval/eval_test.go | 15 +++++--- internal/eval/store_eval.go | 34 +++++++++++-------- 6 files changed, 86 insertions(+), 20 deletions(-) diff --git a/docs/src/content/docs/commands/APPEND.md b/docs/src/content/docs/commands/APPEND.md index 8ada6fd98..2f5cedcf0 100644 --- a/docs/src/content/docs/commands/APPEND.md +++ b/docs/src/content/docs/commands/APPEND.md @@ -3,7 +3,7 @@ title: APPEND description: The `APPEND` command in DiceDB is used to either set the value of a key or append a value to an existing key. This command allows for both creating and updating key-value pairs. --- -The `APPEND` command in DiceDB is used to either set the value of a key or append a value to an existing key. This command allows for both creating and updating key-value pairs. +The `APPEND` command in DiceDB is used to either set the value of a key or append a value to an existing key and returns the length of the value stored at the specified key after appending. This command allows for both creating and updating key-value pairs. ## Syntax @@ -56,8 +56,34 @@ Appending to key `foo` that contains `bar` with `baz` ```bash 127.0.0.1:7379> SET foo bar +OK 127.0.0.1:7379> APPEND foo baz (integer) 6 +127.0.0.1:7379> GET foo +"barbaz" +``` + +Appending "1" to key `bmkey` that contains a bitmap equivalent of `42` + +```bash +127.0.0.1:7379> SETBIT bmkey 2 1 +(integer) 0 +127.0.0.1:7379> SETBIT bmkey 3 1 +(integer) 0 +127.0.0.1:7379> SETBIT bmkey 5 1 +(integer) 0 +127.0.0.1:7379> SETBIT bmkey 10 1 +(integer) 0 +127.0.0.1:7379> SETBIT bmkey 11 1 +(integer) 0 +127.0.0.1:7379> SETBIT bmkey 14 1 +(integer) 0 +127.0.0.1:7379> GET bmkey +"42" +127.0.0.1:7379> APPEND bmkey 1 +(integer) 3 +127.0.0.1:7379> GET bmkey +"421" ``` ### Invalid usage diff --git a/integration_tests/commands/http/append_test.go b/integration_tests/commands/http/append_test.go index 336a37850..3e63f9c1e 100644 --- a/integration_tests/commands/http/append_test.go +++ b/integration_tests/commands/http/append_test.go @@ -101,6 +101,23 @@ func TestAPPEND(t *testing.T) { {Command: "del", Body: map[string]interface{}{"key": "myzset"}}, }, }, + { + name: "APPEND to key created using SETBIT", + commands: []HTTPCommand{ + {Command: "SETBIT", Body: map[string]interface{}{"key": "bitkey", "values": []string{"2", "1"}}}, + {Command: "SETBIT", Body: map[string]interface{}{"key": "bitkey", "values": []string{"3", "1"}}}, + {Command: "SETBIT", Body: map[string]interface{}{"key": "bitkey", "values": []string{"5", "1"}}}, + {Command: "SETBIT", Body: map[string]interface{}{"key": "bitkey", "values": []string{"10", "1"}}}, + {Command: "SETBIT", Body: map[string]interface{}{"key": "bitkey", "values": []string{"11", "1"}}}, + {Command: "SETBIT", Body: map[string]interface{}{"key": "bitkey", "values": []string{"14", "1"}}}, + {Command: "APPEND", Body: map[string]interface{}{"key": "bitkey", "value": "1"}}, + {Command: "GET", Body: map[string]interface{}{"key": "bitkey"}}, + }, + expected: []interface{}{float64(0), float64(0), float64(0), float64(0), float64(0), float64(0), float64(3), "421"}, + cleanup: []HTTPCommand{ + {Command: "del", Body: map[string]interface{}{"key": "bitkey"}}, + }, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { diff --git a/integration_tests/commands/resp/append_test.go b/integration_tests/commands/resp/append_test.go index 6d2481c7d..2d6abb2e4 100644 --- a/integration_tests/commands/resp/append_test.go +++ b/integration_tests/commands/resp/append_test.go @@ -65,6 +65,12 @@ func TestAPPEND(t *testing.T) { expected: []interface{}{int64(1), "WRONGTYPE Operation against a key holding the wrong kind of value"}, cleanup: []string{"del key"}, }, + { + name: "APPEND to key created using SETBIT", + commands: []string{"SETBIT bitkey 2 1", "SETBIT bitkey 3 1", "SETBIT bitkey 5 1", "SETBIT bitkey 10 1", "SETBIT bitkey 11 1", "SETBIT bitkey 14 1", "APPEND bitkey 1", "GET bitkey"}, + expected: []interface{}{int64(0), int64(0), int64(0), int64(0), int64(0), int64(0), int64(3), "421"}, + cleanup: []string{"del bitkey"}, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { diff --git a/integration_tests/commands/websocket/append_test.go b/integration_tests/commands/websocket/append_test.go index 989463d96..63ec1b5f9 100644 --- a/integration_tests/commands/websocket/append_test.go +++ b/integration_tests/commands/websocket/append_test.go @@ -51,6 +51,12 @@ func TestAppend(t *testing.T) { expected: []interface{}{float64(1), "WRONGTYPE Operation against a key holding the wrong kind of value"}, cleanupKey: "key", }, + { + name: "APPEND to key created using SETBIT", + commands: []string{"SETBIT bitkey 2 1", "SETBIT bitkey 3 1", "SETBIT bitkey 5 1", "SETBIT bitkey 10 1", "SETBIT bitkey 11 1", "SETBIT bitkey 14 1", "APPEND bitkey 1", "GET bitkey"}, + expected: []interface{}{float64(0), float64(0), float64(0), float64(0), float64(0), float64(0), float64(3), "421"}, + cleanupKey: "bitkey", + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { diff --git a/internal/eval/eval_test.go b/internal/eval/eval_test.go index e301ce601..c8dc7c7b4 100644 --- a/internal/eval/eval_test.go +++ b/internal/eval/eval_test.go @@ -7031,17 +7031,22 @@ func testEvalAPPEND(t *testing.T, store *dstore.Store) { input: []string{"hashKey", "val"}, migratedOutput: EvalResponse{Result: nil, Error: diceerrors.ErrWrongTypeOperation}, }, - "append to key created using SETBIT": { + "append to key containing byte array": { setup: func() { key := "bitKey" // Create a new byte array object - initialByteArray := NewByteArray(1) // Initialize with 1 byte - initialByteArray.SetBit(0, true) // Set the first bit to 1 + initialByteArray := NewByteArray(2) // Initialize with 2 byte + initialByteArray.SetBit(2, true) // Set the third bit to 1 + initialByteArray.SetBit(3, true) // Set the fourth bit to 1 + initialByteArray.SetBit(5, true) // Set the sixth bit to 1 + initialByteArray.SetBit(10, true) // Set the eleventh bit to 1 + initialByteArray.SetBit(11, true) // Set the twelfth bit to 1 + initialByteArray.SetBit(14, true) // Set the fifteenth bit to 1 obj := store.NewObj(initialByteArray, -1, object.ObjTypeByteArray, object.ObjEncodingByteArray) store.Put(key, obj) }, - input: []string{"bitKey", "val"}, - migratedOutput: EvalResponse{Result: nil, Error: diceerrors.ErrWrongTypeOperation}, + input: []string{"bitKey", "1"}, + migratedOutput: EvalResponse{Result: 3, Error: nil}, }, "append value with leading zeros": { setup: func() { diff --git a/internal/eval/store_eval.go b/internal/eval/store_eval.go index b316af31a..bcbda6248 100644 --- a/internal/eval/store_eval.go +++ b/internal/eval/store_eval.go @@ -1115,25 +1115,33 @@ func evalAPPEND(args []string, store *dstore.Store) *EvalResponse { Error: nil, } } - // Key exists path - if _, ok := obj.Value.(*sortedset.Set); ok { - return &EvalResponse{ - Result: nil, - Error: diceerrors.ErrWrongTypeOperation, - } - } - _, currentEnc := object.ExtractTypeEncoding(obj) var currentValueStr string - switch currentEnc { + switch currentType, currentEnc := object.ExtractTypeEncoding(obj); currentEnc { case object.ObjEncodingInt: // If the encoding is an integer, convert the current value to a string for concatenation currentValueStr = strconv.FormatInt(obj.Value.(int64), 10) case object.ObjEncodingEmbStr, object.ObjEncodingRaw: - // If the encoding is a string, retrieve the string value for concatenation - currentValueStr = obj.Value.(string) + // If the encoding and type is a string, retrieve the string value for concatenation + if currentType == object.ObjTypeString { + currentValueStr = obj.Value.(string) + } else { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongTypeOperation, + } + } + case object.ObjEncodingByteArray: + if val, ok := obj.Value.(*ByteArray); ok { + currentValueStr = string(val.data) + } else { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongTypeOperation, + } + } default: - // If the encoding is neither integer nor string, return a "wrong type" error + // If the encoding is neither integer, string, nor byte array, return a "wrong type" error return &EvalResponse{ Result: nil, Error: diceerrors.ErrWrongTypeOperation, @@ -1401,7 +1409,6 @@ func jsonGETHelper(store *dstore.Store, path, key string) *EvalResponse { // If path is root, return the entire JSON if path == defaultRootPath { resultBytes, err := sonic.Marshal(jsonData) - fmt.Println(string(resultBytes)) if err != nil { return &EvalResponse{ Result: nil, @@ -5212,7 +5219,6 @@ func evalSETBIT(args []string, store *dstore.Store) *EvalResponse { // resize as per the offset byteArray = byteArray.IncreaseSize(int(requiredByteArraySize)) } - resp := byteArray.GetBit(int(offset)) byteArray.SetBit(int(offset), value) From c03ceaa5b0530333c5cf5f6b7b8e97dee5ff3c01 Mon Sep 17 00:00:00 2001 From: Vinit Parekh Date: Sun, 17 Nov 2024 12:35:47 +0530 Subject: [PATCH 3/8] refactor: config management (#1239) --- config/config.go | 521 ++++++------------ config/parser.go | 228 ++++++++ config/validator.go | 97 ++++ dice.toml | 53 -- dicedb.conf | 60 ++ go.mod | 26 +- go.sum | 64 +-- .../commands/async/command_default_test.go | 4 +- integration_tests/commands/http/setup.go | 4 +- .../commands/resp/abort/server_abort_test.go | 6 + integration_tests/commands/resp/dump_test.go | 3 +- .../commands/resp/pfcountwatch_test.go | 98 ++-- .../commands/websocket/json_test.go | 2 + integration_tests/commands/websocket/setup.go | 2 +- integration_tests/config/config_test.go | 151 ++--- integration_tests/config/parser_test.go | 311 +++++++++++ integration_tests/server/server_abort_test.go | 5 + internal/cli/cli.go | 206 +++++++ internal/eval/eval_amd64.go | 2 +- internal/server/httpServer.go | 2 +- internal/server/resp/server.go | 2 +- internal/shard/shard_manager.go | 2 +- main.go | 183 +----- 23 files changed, 1262 insertions(+), 770 deletions(-) create mode 100644 config/parser.go create mode 100644 config/validator.go delete mode 100644 dice.toml create mode 100644 dicedb.conf create mode 100644 integration_tests/config/parser_test.go create mode 100644 internal/cli/cli.go diff --git a/config/config.go b/config/config.go index e23ad1f71..5ece4a6fa 100644 --- a/config/config.go +++ b/config/config.go @@ -1,287 +1,205 @@ package config import ( - "errors" + "fmt" + "log" "log/slog" "os" "path/filepath" - "runtime" - "strings" "time" - - "github.com/dicedb/dice/internal/server/utils" - "github.com/pelletier/go-toml/v2" - "github.com/spf13/viper" ) const ( - DiceDBVersion string = "0.0.5" - - DefaultHost string = "0.0.0.0" - DefaultPort int = 7379 - DefaultConfigName string = "dice.toml" - DefaultConfigFilePath string = "./" + DiceDBVersion string = "0.0.5" + DefaultConfigName string = "dicedb.conf" + DefaultConfigDir string = "." - EvictBatchKeysLRU string = "batch_keys_lru" + EvictSimpleFirst string = "simple-first" + EvictAllKeysRandom string = "allkeys-random" + EvictAllKeysLRU string = "allkeys-lru" + EvictAllKeysLFU string = "allkeys-lfu" DefaultKeysLimit int = 200000000 DefaultEvictionRatio float64 = 0.1 ) -var ( - Host = DefaultHost - Port = DefaultPort - - EnableMultiThreading = false - EnableHTTP = true - HTTPPort = 8082 - - EnableWebsocket = true - WebsocketPort = 8379 - NumShards int = -1 - - // if RequirePass is set to an empty string, no authentication is required - RequirePass = utils.EmptyStr +type Config struct { + Version string `config:"version" default:"0.0.5"` + InstanceID string `config:"instance_id"` + Auth auth `config:"auth"` + AsyncServer asyncServer `config:"async_server"` + HTTP http `config:"http"` + WebSocket websocket `config:"websocket"` + Performance performance `config:"performance"` + Memory memory `config:"memory"` + Persistence persistence `config:"persistence"` + Logging logging `config:"logging"` + Network network `config:"network"` +} - CustomConfigFilePath = utils.EmptyStr - FileLocation = utils.EmptyStr +type auth struct { + UserName string `config:"username" default:"dice"` + Password string `config:"password"` +} - InitConfigCmd = false +type asyncServer struct { + Addr string `config:"addr" default:"0.0.0.0"` + Port int `config:"port" default:"7379" validate:"min=1024,max=65535"` + KeepAlive int32 `config:"keepalive" default:"300"` + Timeout int32 `config:"timeout" default:"300"` + MaxConn int32 `config:"max_conn" default:"0"` +} - KeysLimit = DefaultKeysLimit - EvictionRatio = DefaultEvictionRatio +type http struct { + Enabled bool `config:"enabled" default:"true"` + Port int `config:"port" default:"8082" validate:"min=1024,max=65535"` +} - EnableProfiling = false +type websocket struct { + Enabled bool `config:"enabled" default:"true"` + Port int `config:"port" default:"8379" validate:"min=1024,max=65535"` + MaxWriteResponseRetries int `config:"max_write_response_retries" default:"3" validate:"min=0"` + WriteResponseTimeout time.Duration `config:"write_response_timeout" default:"10s"` +} - EnableWatch = true - LogDir = "" +type performance struct { + WatchChanBufSize int `config:"watch_chan_buf_size" default:"20000"` + ShardCronFrequency time.Duration `config:"shard_cron_frequency" default:"1s"` + MultiplexerPollTimeout time.Duration `config:"multiplexer_poll_timeout" default:"100ms"` + MaxClients int32 `config:"max_clients" default:"20000" validate:"min=0"` + EnableMultiThreading bool `config:"enable_multithreading" default:"false"` + StoreMapInitSize int `config:"store_map_init_size" default:"1024000"` + AdhocReqChanBufSize int `config:"adhoc_req_chan_buf_size" default:"20"` + EnableProfiling bool `config:"profiling" default:"false"` + EnableWatch bool `config:"enable_watch" default:"false"` + NumShards int `config:"num_shards" default:"-1" validate:"oneof=-1|min=1,lte=128"` +} - EnableWAL = true - RestoreFromWAL = false - WALEngine = "sqlite" -) +type memory struct { + MaxMemory int64 `config:"max_memory" default:"0"` + EvictionPolicy string `config:"eviction_policy" default:"allkeys-lfu" validate:"oneof=simple-first allkeys-random allkeys-lru allkeys-lfu"` + EvictionRatio float64 `config:"eviction_ratio" default:"0.9" validate:"min=0,lte=1"` + KeysLimit int `config:"keys_limit" default:"200000000" validate:"min=0"` + LFULogFactor int `config:"lfu_log_factor" default:"10" validate:"min=0"` +} -type Config struct { - Version string `mapstructure:"version"` - InstanceID string `mapstructure:"instance_id"` - AsyncServer struct { - Addr string `mapstructure:"addr"` - Port int `mapstructure:"port"` - KeepAlive int32 `mapstructure:"keepalive"` - Timeout int32 `mapstructure:"timeout"` - MaxConn int32 `mapstructure:"max-conn"` - } `mapstructure:"asyncserver"` - - HTTP struct { - Enabled bool `mapstructure:"enabled"` - Port int `mapstructure:"port"` - } `mapstructure:"http"` - - WebSocket struct { - Enabled bool `mapstructure:"enabled"` - Port int `mapstructure:"port"` - MaxWriteResponseRetries int `mapstructure:"maxwriteresponseretries"` - WriteResponseTimeout time.Duration `mapstructure:"writeresponsetimeout"` - } `mapstructure:"websocket"` - - Performance struct { - WatchChanBufSize int `mapstructure:"watchchanbufsize"` - ShardCronFrequency time.Duration `mapstructure:"shardcronfrequency"` - MultiplexerPollTimeout time.Duration `mapstructure:"servermultiplexerpolltimeout"` - MaxClients int32 `mapstructure:"maxclients"` - EnableMultiThreading bool `mapstructure:"enablemultithreading"` - StoreMapInitSize int `mapstructure:"storemapinitsize"` - AdhocReqChanBufSize int `mapstructure:"adhocreqchanbufsize"` - EnableProfiling bool `mapstructure:"profiling"` - } `mapstructure:"performance"` - - Memory struct { - MaxMemory int64 `mapstructure:"maxmemory"` - EvictionStrategy string `mapstructure:"evictionstrategy"` - EvictionRatio float64 `mapstructure:"evictionratio"` - KeysLimit int `mapstructure:"keyslimit"` - LFULogFactor int `mapstructure:"lfulogfactor"` - } `mapstructure:"memory"` - - Persistence struct { - AOFFile string `mapstructure:"aoffile"` - PersistenceEnabled bool `mapstructure:"persistenceenabled"` - WriteAOFOnCleanup bool `mapstructure:"writeaofoncleanup"` - } `mapstructure:"persistence"` - - Logging struct { - LogLevel string `mapstructure:"loglevel"` - PrettyPrintLogs bool `mapstructure:"prettyprintlogs"` - } `mapstructure:"logging"` - - Auth struct { - UserName string `mapstructure:"username"` - Password string `mapstructure:"password"` - } `mapstructure:"auth"` - - Network struct { - IOBufferLength int `mapstructure:"iobufferlength"` - IOBufferLengthMAX int `mapstructure:"iobufferlengthmax"` - } `mapstructure:"network"` - - NumShards int `mapstructure:"num_shards"` +type persistence struct { + AOFFile string `config:"aof_file" default:"./dice-master.aof" validate:"filepath"` + PersistenceEnabled bool `config:"persistence_enabled" default:"true"` + WriteAOFOnCleanup bool `config:"write_aof_on_cleanup" default:"false"` + EnableWAL bool `config:"enable-wal" default:"true"` + WALDir string `config:"wal-dir" default:"./" validate:"dirpath"` + RestoreFromWAL bool `config:"restore-wal" default:"false"` + WALEngine string `config:"wal-engine" default:"aof" validate:"oneof=sqlite aof"` } -// Default configurations for internal use -var baseConfig = Config{ - Version: DiceDBVersion, - AsyncServer: struct { - Addr string `mapstructure:"addr"` - Port int `mapstructure:"port"` - KeepAlive int32 `mapstructure:"keepalive"` - Timeout int32 `mapstructure:"timeout"` - MaxConn int32 `mapstructure:"max-conn"` - }{ - Addr: DefaultHost, - Port: DefaultPort, - KeepAlive: int32(300), - Timeout: int32(300), - MaxConn: int32(0), - }, - HTTP: struct { - Enabled bool `mapstructure:"enabled"` - Port int `mapstructure:"port"` - }{ - Enabled: EnableHTTP, - Port: HTTPPort, - }, - WebSocket: struct { - Enabled bool `mapstructure:"enabled"` - Port int `mapstructure:"port"` - MaxWriteResponseRetries int `mapstructure:"maxwriteresponseretries"` - WriteResponseTimeout time.Duration `mapstructure:"writeresponsetimeout"` - }{ - Enabled: EnableWebsocket, - Port: WebsocketPort, - MaxWriteResponseRetries: 3, - WriteResponseTimeout: 10 * time.Second, - }, - Performance: struct { - WatchChanBufSize int `mapstructure:"watchchanbufsize"` - ShardCronFrequency time.Duration `mapstructure:"shardcronfrequency"` - MultiplexerPollTimeout time.Duration `mapstructure:"servermultiplexerpolltimeout"` - MaxClients int32 `mapstructure:"maxclients"` - EnableMultiThreading bool `mapstructure:"enablemultithreading"` - StoreMapInitSize int `mapstructure:"storemapinitsize"` - AdhocReqChanBufSize int `mapstructure:"adhocreqchanbufsize"` - EnableProfiling bool `mapstructure:"profiling"` - }{ - WatchChanBufSize: 20000, - ShardCronFrequency: 1 * time.Second, - MultiplexerPollTimeout: 100 * time.Millisecond, - MaxClients: int32(20000), - EnableMultiThreading: false, - StoreMapInitSize: 1024000, - AdhocReqChanBufSize: 20, // assuming we wouldn't have more than 20 adhoc requests being sent at a time. - }, - Memory: struct { - MaxMemory int64 `mapstructure:"maxmemory"` - EvictionStrategy string `mapstructure:"evictionstrategy"` - EvictionRatio float64 `mapstructure:"evictionratio"` - KeysLimit int `mapstructure:"keyslimit"` - LFULogFactor int `mapstructure:"lfulogfactor"` - }{ - MaxMemory: 0, - EvictionStrategy: EvictBatchKeysLRU, - EvictionRatio: DefaultEvictionRatio, - KeysLimit: DefaultKeysLimit, - LFULogFactor: 10, - }, - Persistence: struct { - AOFFile string `mapstructure:"aoffile"` - PersistenceEnabled bool `mapstructure:"persistenceenabled"` - WriteAOFOnCleanup bool `mapstructure:"writeaofoncleanup"` - }{ - PersistenceEnabled: true, - AOFFile: "./dice-master.aof", - WriteAOFOnCleanup: false, - }, - Logging: struct { - LogLevel string `mapstructure:"loglevel"` - PrettyPrintLogs bool `mapstructure:"prettyprintlogs"` - }{ - LogLevel: "info", - PrettyPrintLogs: true, - }, - Auth: struct { - UserName string `mapstructure:"username"` - Password string `mapstructure:"password"` - }{ - UserName: "dice", - Password: RequirePass, - }, - Network: struct { - IOBufferLength int `mapstructure:"iobufferlength"` - IOBufferLengthMAX int `mapstructure:"iobufferlengthmax"` - }{ - IOBufferLength: 512, - IOBufferLengthMAX: 50 * 1024, - }, +type logging struct { + LogLevel string `config:"log_level" default:"info" validate:"oneof=debug info warn error"` } -var defaultConfig Config +type network struct { + IOBufferLengthMAX int `config:"io_buffer_length_max" default:"51200" validate:"min=0,max=1048576"` // max is 1MB' + IOBufferLength int `config:"io_buffer_length" default:"512" validate:"min=0"` +} func init() { - config := baseConfig - config.Logging.PrettyPrintLogs = false - config.Logging.LogLevel = "info" - defaultConfig = config + configFilePath := filepath.Join(DefaultConfigDir, DefaultConfigName) + if err := loadDiceConfig(configFilePath); err != nil { + log.Fatalf("failed to load configuration: %v", err) + } } // DiceConfig is the global configuration object for dice -var DiceConfig = &defaultConfig - -func SetupConfig() { - if InitConfigCmd { - FileLocation = getConfigPath() - createConfigFile(FileLocation) - return - } - - // Check if both -o and -c flags are set - if areBothFlagsSet() { - slog.Error("Both -o and -c flags are set. Please use only one flag.") - return - } - - // Check if -o flag is set - if CustomConfigFilePath != utils.EmptyStr && isValidDirPath() { - createConfigFile(filepath.Join(CustomConfigFilePath, DefaultConfigName)) - return - } +var DiceConfig = &Config{} - // Check if -c flag is set - if FileLocation != utils.EmptyStr || isConfigFilePresent() { - setUpViperConfig(FileLocation) - return - } - - // If no flags are set, use default configurations with prioritizing command line flags - mergeFlagsWithConfig() -} - -func createConfigFile(configFilePath string) { +func CreateConfigFile(configFilePath string) error { + // Check if the config file already exists if _, err := os.Stat(configFilePath); err == nil { - slog.Warn("config file already exists", slog.String("path", configFilePath)) - setUpViperConfig(configFilePath) - return + if err := loadDiceConfig(configFilePath); err != nil { + return fmt.Errorf("failed to load existing configuration: %w", err) + } + return nil } + // Attempt to write a new config file if err := writeConfigFile(configFilePath); err != nil { - slog.Warn("starting DiceDB with default configurations.", slog.Any("error", err)) - return + slog.Warn("Failed to create config file, starting with defaults.", slog.Any("error", err)) + return nil // Continuing with defaults; may reconsider behavior. } - setUpViperConfig(configFilePath) - slog.Info("config file created at", slog.Any("path", configFilePath)) + // Load the new configuration + if err := loadDiceConfig(configFilePath); err != nil { + return fmt.Errorf("failed to load newly created configuration: %w", err) + } + + slog.Info("Config file successfully created.", slog.String("path", configFilePath)) + return nil } +// writeConfigFile writes the default configuration to the specified file path func writeConfigFile(configFilePath string) error { + content := `# Configuration file for Dicedb + +# Version +version = "0.0.5" + +# Async Server Configuration +async_server.addr = "0.0.0.0" +async_server.port = 7379 +async_server.keepalive = 300 +async_server.timeout = 300 +async_server.max_conn = 0 + +# HTTP Configuration +http.enabled = false +http.port = 8082 + +# WebSocket Configuration +websocket.enabled = false +websocket.port = 8379 +websocket.max_write_response_retries = 3 +websocket.write_response_timeout = 10s + +# Performance Configuration +performance.watch_chan_buf_size = 20000 +performance.shard_cron_frequency = 1s +performance.multiplexer_poll_timeout = 100ms +performance.max_clients = 20000 +performance.enable_multithreading = false +performance.store_map_init_size = 1024000 +performance.adhoc_req_chan_buf_size = 20 +performance.enable_profiling = false +performance.enable_watch = false +performance.num_shards = -1 + +# Memory Configuration +memory.max_memory = 0 +memory.eviction_policy = "allkeys-lfu" +memory.eviction_ratio = 0.9 +memory.keys_limit = 200000000 +memory.lfu_log_factor = 10 + +# Persistence Configuration +persistence.aof_file = "./dice-master.aof" +persistence.persistence_enabled = true +persistence.write_aof_on_cleanup = false +persistence.enable-wal = true +persistence.wal-dir = "./" +persistence.restore-wal = false +persistence.wal-engine = "aof" + +# Logging Configuration +logging.log_level = "info" + +# Authentication Configuration +auth.username = "dice" +auth.password = "" + +# Network Configuration +network.io_buffer_length = 512 +network.io_buffer_length_max = 51200` + + // Check if the directory exists or not dir := filepath.Dir(configFilePath) if _, err := os.Stat(dir); err != nil { return err @@ -294,112 +212,35 @@ func writeConfigFile(configFilePath string) error { } defer file.Close() - encoder := toml.NewEncoder(file) - err = encoder.Encode(defaultConfig) - return err -} - -func isValidDirPath() bool { - info, err := os.Stat(CustomConfigFilePath) - if os.IsNotExist(err) || err != nil { - return false - } - - if !info.IsDir() { - return false - } - return true -} - -// This function checks if both -o and -c flags are set or not -func areBothFlagsSet() bool { - return FileLocation != utils.EmptyStr && CustomConfigFilePath != utils.EmptyStr -} - -func setUpViperConfig(configFilePath string) { - if configFilePath != filepath.Join(DefaultConfigFilePath, DefaultConfigName) { - viper.SetConfigName(strings.Split(filepath.Base(configFilePath), ".")[0]) - } else { - viper.SetConfigName("dice") - } - - if configFilePath == utils.EmptyStr { - viper.AddConfigPath(DefaultConfigFilePath) - } else { - viper.AddConfigPath(filepath.Dir(configFilePath)) - } - - viper.SetConfigType("toml") - if err := viper.ReadInConfig(); err != nil { - var configFileNotFoundError viper.ConfigFileNotFoundError - if errors.As(err, &configFileNotFoundError) { - slog.Warn("config file not found. Using default configurations.") - return - } - slog.Error("Error reading config file", slog.Any("error", err)) - } - - if err := viper.Unmarshal(&DiceConfig); err != nil { - slog.Error("Error unmarshalling config file", slog.Any("error", err)) - slog.Warn("starting DiceDB with default configurations.") - return - } - - // override default configurations with command line flags - mergeFlagsWithConfig() -} - -func mergeFlagsWithConfig() { - if RequirePass != utils.EmptyStr { - DiceConfig.Auth.Password = RequirePass - } - - if Host != DefaultHost { - DiceConfig.AsyncServer.Addr = Host - } - - if Port != DefaultPort { - DiceConfig.AsyncServer.Port = Port - } - - if KeysLimit != DefaultKeysLimit { - DiceConfig.Memory.KeysLimit = KeysLimit + if _, err := file.WriteString(content); err != nil { + return err } - if EvictionRatio != DefaultEvictionRatio { - DiceConfig.Memory.EvictionRatio = EvictionRatio - } + return nil } -// This function checks if the config file is present or not at default location or at -c flag location -func isConfigFilePresent() bool { - // If -c flag is not set then look for config file in current directory use it - if _, err := os.Stat(filepath.Join(".", DefaultConfigName)); FileLocation == utils.EmptyStr && err == nil { - FileLocation = filepath.Join(".", DefaultConfigName) - return true +func loadDiceConfig(configFilePath string) error { + parser := NewConfigParser() + if err := parser.ParseFromFile(configFilePath); err != nil { + slog.Warn("Failed to parse config file", slog.String("error", err.Error()), slog.String("message", "Loading default configurations")) + return parser.ParseDefaults(DiceConfig) } - // will be executed if -c flag is used - _, err := os.Stat(FileLocation) - - return err == nil + return parser.Loadconfig(DiceConfig) } // This function returns the config file path based on the OS -func getConfigPath() string { - switch runtime.GOOS { - case "windows": - FileLocation = filepath.Join("C:", "ProgramData", "dice", DefaultConfigName) - case "darwin", "linux": - FileLocation = filepath.Join(string(filepath.Separator), "etc", "dice", DefaultConfigName) - default: - // Default to current directory if OS is unknown - FileLocation = filepath.Join(".", DefaultConfigName) - } - return FileLocation -} +// func getConfigPath() string { +// switch runtime.GOOS { +// case "windows": +// FileLocation = filepath.Join("C:", "ProgramData", "dice", DefaultConfigName) +// case "darwin", "linux": +// FileLocation = filepath.Join(string(filepath.Separator), "etc", "dice", DefaultConfigName) +// default: +// // Default to current directory if OS is unknown +// FileLocation = filepath.Join(".", DefaultConfigName) +// } +// return FileLocation +// } // ResetConfig resets the DiceConfig to default configurations. This function is only used for testing purposes -func ResetConfig() { - DiceConfig = &defaultConfig -} diff --git a/config/parser.go b/config/parser.go new file mode 100644 index 000000000..132c5f64b --- /dev/null +++ b/config/parser.go @@ -0,0 +1,228 @@ +package config + +import ( + "bufio" + "fmt" + "log/slog" + "os" + "reflect" + "strconv" + "strings" + "time" +) + +// ConfigParser handles the parsing of configuration files +type ConfigParser struct { + // store holds the raw key-value pairs from the config file + store map[string]string +} + +// NewConfigParser creates a new instance of ConfigParser +func NewConfigParser() *ConfigParser { + return &ConfigParser{ + store: make(map[string]string), + } +} + +// ParseFromFile reads the configuration data from a file +func (p *ConfigParser) ParseFromFile(filename string) error { + file, err := os.Open(filename) + if err != nil { + return fmt.Errorf("error opening config file: %w", err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + return processConfigData(scanner, p) +} + +// ParseFromStdin reads the configuration data from stdin +func (p *ConfigParser) ParseFromStdin() error { + scanner := bufio.NewScanner(os.Stdin) + return processConfigData(scanner, p) +} + +// ParseDefaults populates a struct with default values based on struct tag `default` +func (p *ConfigParser) ParseDefaults(cfg interface{}) error { + val := reflect.ValueOf(cfg) + if val.Kind() != reflect.Ptr || val.IsNil() { + return fmt.Errorf("config must be a non-nil pointer to a struct") + } + + val = val.Elem() + if val.Kind() != reflect.Struct { + return fmt.Errorf("config must be a pointer to a struct") + } + + return p.unmarshalStruct(val, "") +} + +// Loadconfig populates a struct with configuration values based on struct tags +func (p *ConfigParser) Loadconfig(cfg interface{}) error { + val := reflect.ValueOf(cfg) + if val.Kind() != reflect.Ptr || val.IsNil() { + return fmt.Errorf("config must be a non-nil pointer to a struct") + } + + val = val.Elem() + if val.Kind() != reflect.Struct { + return fmt.Errorf("config must be a pointer to a struct") + } + + if err := p.unmarshalStruct(val, ""); err != nil { + return fmt.Errorf("failed to unmarshal config: %w", err) + } + + if err := validateConfig(DiceConfig); err != nil { + return fmt.Errorf("failed to validate config: %w", err) + } + + return nil +} + +// processConfigData reads the configuration data line by line and stores it in the ConfigParser +func processConfigData(scanner *bufio.Scanner, p *ConfigParser) error { + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + parts := strings.SplitN(line, "=", 2) + if len(parts) != 2 { + slog.Warn("invalid config line", slog.String("line", line)) + continue + } + + key := strings.TrimSpace(parts[0]) + value := strings.Trim(strings.TrimSpace(parts[1]), "\"") + p.store[key] = value + } + + return scanner.Err() +} + +// unmarshalStruct handles the recursive struct parsing. +func (p *ConfigParser) unmarshalStruct(val reflect.Value, prefix string) error { + typ := val.Type() + + for i := 0; i < val.NumField(); i++ { + field := val.Field(i) + fieldType := typ.Field(i) + + // Skip unexported fields just like how encoding/json does + if !field.CanSet() { + continue + } + + // Get config key or field name + key := fieldType.Tag.Get("config") + + // Use field name as key if not specified in tag + if key == "" { + key = strings.ToLower(fieldType.Name) + } + + // Skip fields with "-" tag + if key == "-" { + continue + } + + // Apply nested struct's tag as prefix + fullKey := key + if prefix != "" { + fullKey = fmt.Sprintf("%s.%s", prefix, key) + } + + // Recursively process nested structs with their prefix + if field.Kind() == reflect.Struct { + if err := p.unmarshalStruct(field, fullKey); err != nil { + return err + } + continue + } + + // Fetch and set value for non-struct fields + value, exists := p.store[fullKey] + if !exists { + // Use default value from tag if available + if defaultValue := fieldType.Tag.Get("default"); defaultValue != "" { + value = defaultValue + } else { + continue + } + } + + if err := setField(field, value); err != nil { + return fmt.Errorf("error setting field %s: %w", fullKey, err) + } + } + + return nil +} + +// setField sets the appropriate field value based on its type +func setField(field reflect.Value, value string) error { + switch field.Kind() { + case reflect.String: + field.SetString(value) + + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + if field.Type() == reflect.TypeOf(time.Duration(0)) { + // Handle time.Duration type + duration, err := time.ParseDuration(value) + if err != nil { + return fmt.Errorf("failed to parse duration: %w", err) + } + field.Set(reflect.ValueOf(duration)) + } else { + // Handle other integer types + val, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return fmt.Errorf("failed to parse integer: %w", err) + } + field.SetInt(val) + } + + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + val, err := strconv.ParseUint(value, 10, 64) + if err != nil { + return err + } + field.SetUint(val) + + case reflect.Float32, reflect.Float64: + val, err := strconv.ParseFloat(value, 64) + if err != nil { + return err + } + field.SetFloat(val) + + case reflect.Bool: + val, err := strconv.ParseBool(value) + if err != nil { + return err + } + field.SetBool(val) + + case reflect.Slice: + // Get the type of the elements in the slice + elemType := field.Type().Elem() + values := strings.Split(value, ",") + slice := reflect.MakeSlice(field.Type(), len(values), len(values)) + for i, v := range values { + elem := slice.Index(i) + elemVal := reflect.New(elemType).Elem() + if err := setField(elemVal, strings.TrimSpace(v)); err != nil { + return err + } + elem.Set(elemVal) + } + field.Set(slice) + + default: + return fmt.Errorf("unsupported type: %s", field.Type()) + } + + return nil +} diff --git a/config/validator.go b/config/validator.go new file mode 100644 index 000000000..66669ff43 --- /dev/null +++ b/config/validator.go @@ -0,0 +1,97 @@ +package config + +import ( + "fmt" + "log" + "reflect" + "strings" + + "github.com/go-playground/validator/v10" +) + +func validateConfig(config *Config) error { + validate := validator.New() + validate.RegisterStructValidation(validateShardCount, Config{}) + + if err := validate.Struct(config); err != nil { + validationErrors, ok := err.(validator.ValidationErrors) + if !ok { + return fmt.Errorf("unexpected validation error type: %v", err) + } + + processedFields := make(map[string]bool) + + for _, validationErr := range validationErrors { + fieldName := strings.TrimPrefix(validationErr.Namespace(), "Config.") + + if processedFields[fieldName] { + continue + } + processedFields[fieldName] = true + + log.Printf("Field %s failed validation: %s", fieldName, validationErr.Tag()) + + if err := applyDefaultValuesFromTags(config, fieldName); err != nil { + return fmt.Errorf("error setting default for %s: %v", fieldName, err) + } + } + } + return nil +} + +func validateShardCount(sl validator.StructLevel) { + config := sl.Current().Interface().(Config) + if config.Performance.NumShards <= 0 && config.Performance.NumShards != -1 { + sl.ReportError(config.Performance.NumShards, "NumShards", "NumShards", "invalidValue", "must be -1 or greater than 0") + } +} + +func applyDefaultValuesFromTags(config *Config, fieldName string) error { + configType := reflect.TypeOf(config).Elem() + configValue := reflect.ValueOf(config).Elem() + + // Split the field name if it refers to a nested struct + parts := strings.Split(fieldName, ".") + var field reflect.StructField + var fieldValue reflect.Value + var found bool + + // Traverse the struct to find the nested field + for i, part := range parts { + // If it's the first field, just look in the top-level struct + if i == 0 { + field, found = configType.FieldByName(part) + if !found { + log.Printf("Warning: %s field not found", part) + return fmt.Errorf("field %s not found in config struct", part) + } + fieldValue = configValue.FieldByName(part) + } else { + // Otherwise, the struct is nested, so navigate into it + if fieldValue.Kind() == reflect.Struct { + field, found = fieldValue.Type().FieldByName(part) + if !found { + log.Printf("Warning: %s field not found in %s", part, fieldValue.Type()) + return fmt.Errorf("field %s not found in struct %s", part, fieldValue.Type()) + } + fieldValue = fieldValue.FieldByName(part) + } else { + log.Printf("Warning: %s is not a struct", fieldName) + return fmt.Errorf("%s is not a struct", fieldName) + } + } + } + + defaultValue := field.Tag.Get("default") + if defaultValue == "" { + log.Printf("Warning: %s field has no default value to set, leaving empty string", fieldName) + return nil + } + + if err := setField(fieldValue, defaultValue); err != nil { + return fmt.Errorf("error setting default value for %s: %v", fieldName, err) + } + + log.Printf("Setting default value for %s to: %s", fieldName, defaultValue) + return nil +} diff --git a/dice.toml b/dice.toml deleted file mode 100644 index 02f330fc8..000000000 --- a/dice.toml +++ /dev/null @@ -1,53 +0,0 @@ -# Dice configuration file example. -# -# Note that in order to read the configuration file, Dice must be -# started with the c flag and file path as first argument: -# -# go run main.go -c /path/to/dice.toml - -Version = '0.0.5' -InstanceID = '' - -[AsyncServer] -Addr = '0.0.0.0' -Port = 7379 -KeepAlive = 300 -Timeout = 300 -MaxConn = 0 - -[HTTP] -Enabled = true -Port = 8082 - -[WebSocket] -Enabled = true -Port = 8379 - -[Performance] -WatchChanBufSize = 20000 -ShardCronFrequency = 1000000000 -MultiplexerPollTimeout = 100000000 -MaxClients = 20000 -EnableMultiThreading = false -StoreMapInitSize = 1024000 -AdhocReqChanBufSize = 20 - -[Memory] -MaxMemory = 0 -EvictionPolicy = 'allkeys-lfu' -EvictionRatio = 0.1 -KeysLimit = 200000000 -LFULogFactor = 10 - -[Persistence] -AOFFile = './dice-master.aof' -PersistenceEnabled = true -WriteAOFOnCleanup = false - -[Auth] -UserName = 'dice' -Password = '' - -[Network] -IOBufferLength = 512 -IOBufferLengthMAX = 51200 diff --git a/dicedb.conf b/dicedb.conf new file mode 100644 index 000000000..627d19fc9 --- /dev/null +++ b/dicedb.conf @@ -0,0 +1,60 @@ +# Configuration file for Dicedb + +# Version +version = "0.0.5" + +# Async Server Configuration +async_server.addr = "0.0.0.0" +async_server.port = 7379 +async_server.keepalive = 300 +async_server.timeout = 300 +async_server.max_conn = 0 + +# HTTP Configuration +http.enabled = false +http.port = 8082 + +# WebSocket Configuration +websocket.enabled = false +websocket.port = 8379 +websocket.max_write_response_retries = 3 +websocket.write_response_timeout = 10s + +# Performance Configuration +performance.watch_chan_buf_size = 20000 +performance.shard_cron_frequency = 1s +performance.multiplexer_poll_timeout = 100ms +performance.max_clients = 20000 +performance.enable_multithreading = false +performance.store_map_init_size = 1024000 +performance.adhoc_req_chan_buf_size = 20 +performance.enable_profiling = false +performance.enable_watch = false +performance.num_shards = -1 + +# Memory Configuration +memory.max_memory = 0 +memory.eviction_policy = "allkeys-lfu" +memory.eviction_ratio = 0.9 +memory.keys_limit = 200000000 +memory.lfu_log_factor = 10 + +# Persistence Configuration +persistence.aof_file = "./dice-master.aof" +persistence.persistence_enabled = true +persistence.write_aof_on_cleanup = false +persistence.enable-wal = true +persistence.wal-dir = "./" +persistence.restore-wal = false +persistence.wal-engine = "aof" + +# Logging Configuration +logging.log_level = "info" + +# Authentication Configuration +auth.username = "dice" +auth.password = "" + +# Network Configuration +network.io_buffer_length = 512 +network.io_buffer_length_max = 51200 \ No newline at end of file diff --git a/go.mod b/go.mod index 64bc75043..a193be0cf 100644 --- a/go.mod +++ b/go.mod @@ -11,27 +11,20 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-metro v0.0.0-20211217172704-adc40b04c140 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/fsnotify/fsnotify v1.8.0 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect - github.com/klauspost/cpuid/v2 v2.2.9 // indirect - github.com/magiconair/properties v1.8.7 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/sagikazarmark/locafero v0.6.0 // indirect - github.com/sagikazarmark/slog-shim v0.1.0 // indirect - github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.11.0 // indirect - github.com/spf13/cast v1.7.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/subosito/gotenv v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - go.uber.org/multierr v1.11.0 // indirect - golang.org/x/arch v0.12.0 // indirect + golang.org/x/arch v0.11.0 // indirect + golang.org/x/net v0.21.0 // indirect golang.org/x/sys v0.27.0 // indirect golang.org/x/text v0.20.0 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) @@ -42,6 +35,7 @@ require ( github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964 github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da github.com/dicedb/dicedb-go v0.0.0-20241026093718-570de4575be3 + github.com/go-playground/validator/v10 v10.22.1 github.com/gobwas/glob v0.2.3 github.com/google/btree v1.1.3 github.com/google/go-cmp v0.6.0 @@ -50,10 +44,8 @@ require ( github.com/mattn/go-sqlite3 v1.14.24 github.com/mmcloughlin/geohash v0.10.0 github.com/ohler55/ojg v1.25.0 - github.com/pelletier/go-toml/v2 v2.2.3 github.com/rs/xid v1.6.0 github.com/rs/zerolog v1.33.0 - github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 github.com/twmb/murmur3 v1.1.8 github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 diff --git a/go.sum b/go.sum index 242b25ad3..8a48e609c 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,6 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= -github.com/bytedance/sonic v1.12.3 h1:W2MGa7RCU1QTeYRTPE3+88mVC0yXmsRQRChiyVocVjU= -github.com/bytedance/sonic v1.12.3/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic v1.12.4 h1:9Csb3c9ZJhfUWeMtpCDCq6BUoH5ogfDFLUgQ/jG+R0k= github.com/bytedance/sonic v1.12.4/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= @@ -22,6 +20,7 @@ github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQ github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964 h1:Ew0znI2JatzKy52N1iS5muUsHkf2UJuhocH7uFW7jjs= github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -34,12 +33,16 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dicedb/dicedb-go v0.0.0-20241026093718-570de4575be3 h1:JvnAibMNGA0vQH+T47Y/d5/POURIvfJl3fFk0GIEBkQ= github.com/dicedb/dicedb-go v0.0.0-20241026093718-570de4575be3/go.mod h1:p7x5/3S6wBEmiRMwxavj1I1P1xsSVQS6fcSbeai5ic4= -github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= -github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= -github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA= +github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -51,20 +54,16 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= -github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -73,14 +72,11 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mmcloughlin/geohash v0.10.0 h1:9w1HchfDfdeLc+jFEf/04D27KP7E2QmpDu52wPbJWRE= github.com/mmcloughlin/geohash v0.10.0/go.mod h1:oNZxQo5yWJh0eMQEP/8hwQuVx9Z9tjwFUqcTB1SmG0c= github.com/ohler55/ojg v1.25.0 h1:sDwc4u4zex65Uz5Nm7O1QwDKTT+YRcpeZQTy1pffRkw= github.com/ohler55/ojg v1.25.0/go.mod h1:gQhDVpQLqrmnd2eqGAvJtn+NfKoYJbe/A4Sj3/Vro4o= -github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= -github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= @@ -92,20 +88,6 @@ github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= -github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= -github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= -github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= -github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= -github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= -github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= -github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= -github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= -github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -115,38 +97,26 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= -github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 h1:zzrxE1FKn5ryBNl9eKOeqQ58Y/Qpo3Q9QNxKHX5uzzQ= github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2/go.mod h1:hzfGeIUDq/j97IG+FhNqkowIyEcD88LrW6fyU3K3WqY= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/arch v0.11.0 h1:KXV8WWKCXm6tRpLirl2szsO5j/oOODwZf4hATmGVNs4= golang.org/x/arch v0.11.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= -golang.org/x/arch v0.12.0 h1:UsYJhbzPYGsT0HbEdmYcqtCv8UNGvnaL561NnIUvaKg= -golang.org/x/arch v0.12.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= -golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= -golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= @@ -154,8 +124,6 @@ google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojt gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/integration_tests/commands/async/command_default_test.go b/integration_tests/commands/async/command_default_test.go index f5ca6e542..066fbaa12 100644 --- a/integration_tests/commands/async/command_default_test.go +++ b/integration_tests/commands/async/command_default_test.go @@ -31,9 +31,7 @@ func getCommandDefault(connection net.Conn) []interface{} { return nil } var cmds []interface{} - for _, v := range responseValue.([]interface{}) { - cmds = append(cmds, v) - } + cmds = append(cmds, responseValue.([]interface{})...) return cmds } diff --git a/integration_tests/commands/http/setup.go b/integration_tests/commands/http/setup.go index 26171b2ee..d69f78e3a 100644 --- a/integration_tests/commands/http/setup.go +++ b/integration_tests/commands/http/setup.go @@ -110,11 +110,11 @@ func RunHTTPServer(ctx context.Context, wg *sync.WaitGroup, opt TestServerOption watchChan := make(chan dstore.QueryWatchEvent, config.DiceConfig.Performance.WatchChanBufSize) shardManager := shard.NewShardManager(1, watchChan, nil, globalErrChannel) queryWatcherLocal := querymanager.NewQueryManager() - config.HTTPPort = opt.Port + config.DiceConfig.HTTP.Port = opt.Port // Initialize the HTTPServer testServer := server.NewHTTPServer(shardManager, nil) // Inform the user that the server is starting - fmt.Println("Starting the test server on port", config.HTTPPort) + fmt.Println("Starting the test server on port", config.DiceConfig.HTTP.Port) shardManagerCtx, cancelShardManager := context.WithCancel(ctx) wg.Add(1) go func() { diff --git a/integration_tests/commands/resp/abort/server_abort_test.go b/integration_tests/commands/resp/abort/server_abort_test.go index c3f788e2d..7e13bfd66 100644 --- a/integration_tests/commands/resp/abort/server_abort_test.go +++ b/integration_tests/commands/resp/abort/server_abort_test.go @@ -3,6 +3,7 @@ package abort import ( "context" "fmt" + "log" "net" "sync" "testing" @@ -17,6 +18,11 @@ var testServerOptions = resp.TestServerOptions{ Port: 8740, } +func init() { + config.DiceConfig.AsyncServer.Port = testServerOptions.Port + log.Print("Setting port to ", config.DiceConfig.AsyncServer.Port) +} + func TestAbortCommand(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer ctx.Done() diff --git a/integration_tests/commands/resp/dump_test.go b/integration_tests/commands/resp/dump_test.go index 7e0f04fe8..88f41ace4 100644 --- a/integration_tests/commands/resp/dump_test.go +++ b/integration_tests/commands/resp/dump_test.go @@ -90,8 +90,7 @@ func TestDumpRestore(t *testing.T) { t.Run(tc.name, func(t *testing.T) { FireCommand(conn, "FLUSHALL") for i, cmd := range tc.commands { - var result interface{} - result = FireCommand(conn, cmd) + result := FireCommand(conn, cmd) expected := tc.expected[i] switch exp := expected.(type) { diff --git a/integration_tests/commands/resp/pfcountwatch_test.go b/integration_tests/commands/resp/pfcountwatch_test.go index 267ccc922..60abf9bee 100644 --- a/integration_tests/commands/resp/pfcountwatch_test.go +++ b/integration_tests/commands/resp/pfcountwatch_test.go @@ -19,11 +19,11 @@ type pfcountWatchTestCase struct { } type pfcountWatchWithPFMergeTestCase struct { - destKey1 string - destValue1 []string - destKey2 string - destValue2 []string - result int64 + destKey1 string + destValue1 []string + destKey2 string + destValue2 []string + result int64 } const ( @@ -69,24 +69,24 @@ func TestPFCOUNTWATCH(t *testing.T) { respParsers := setUpRespParsers(t, subscribers) - t.Run("Basic PFCount Operations", func(t *testing.T) { - testPFCountAdd(t, publisher, respParsers) - }, + t.Run("Basic PFCount Operations", func(t *testing.T) { + testPFCountAdd(t, publisher, respParsers) + }, ) - t.Run("PFCount Operations including PFMerge", func(t *testing.T) { - testPFCountMerge(t, publisher, respParsers) - }, + t.Run("PFCount Operations including PFMerge", func(t *testing.T) { + testPFCountMerge(t, publisher, respParsers) + }, ) } func setupSubscribers(count int) []net.Conn { - subscribers := make([]net.Conn, 0, count) - for i := 0; i < count; i++ { - conn := getLocalConnection() - subscribers = append(subscribers, conn) - } - return subscribers + subscribers := make([]net.Conn, 0, count) + for i := 0; i < count; i++ { + conn := getLocalConnection() + subscribers = append(subscribers, conn) + } + return subscribers } func setUpRespParsers(t *testing.T, subscribers []net.Conn) []*clientio.RESPParser { @@ -111,7 +111,7 @@ func testPFCountAdd(t *testing.T, publisher net.Conn, respParsers []*clientio.RE for _, tc := range pfcountWatchTestCases { res := FireCommand(publisher, fmt.Sprintf("PFADD %s %s", tc.key, tc.val)) assert.Equal(t, int64(1), res) - + verifyWatchResults(t, respParsers, tc.result) } } @@ -122,7 +122,7 @@ func testPFCountMerge(t *testing.T, publisher net.Conn, respParsers []*clientio. FireCommand(publisher, fmt.Sprintf("PFADD %s %s", tc.destKey1, tc.destValue1)) FireCommand(publisher, fmt.Sprintf("PFADD %s %s", tc.destKey2, tc.destValue2)) FireCommand(publisher, fmt.Sprintf("PFMERGE %s %s %s", pfcountWatchKey, tc.destKey1, tc.destKey2)) - + verifyWatchResults(t, respParsers, tc.result) } } @@ -157,11 +157,11 @@ type pfcountWatchSDKTestCase struct { } type pfcountWatchSDKWithPFMergeTestCase struct { - destKey1 string - destValue1 []string - destKey2 string - destValue2 []string - result int64 + destKey1 string + destValue1 []string + destKey2 string + destValue2 []string + result int64 } var PFCountWatchSDKTestCases = []pfcountWatchSDKTestCase{ @@ -178,7 +178,7 @@ var pfcountWatchSDKhWithPFMergeTestCases = []pfcountWatchSDKWithPFMergeTestCase{ } func TestPFCountWATCHWithSDK(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) defer cancel() publisher := getLocalSdk() @@ -189,31 +189,31 @@ func TestPFCountWATCHWithSDK(t *testing.T) { channels := setUpWatchChannelsSDK(t, ctx, subscribers) - t.Run("Basic PFCount Operations", func(t *testing.T) { - testPFCountAddSDK(t, ctx, channels, publisher) - }, + t.Run("Basic PFCount Operations", func(t *testing.T) { + testPFCountAddSDK(t, ctx, channels, publisher) + }, ) - t.Run("PFCount Operations including PFMerge", func(t *testing.T) { - testPFCountMergeSDK(t, ctx, channels, publisher) - }, + t.Run("PFCount Operations including PFMerge", func(t *testing.T) { + testPFCountMergeSDK(t, ctx, channels, publisher) + }, ) } -func setupSubscribersSDK(count int) []WatchSubscriber { - subscribers := make([]WatchSubscriber, count) - for i := range subscribers { +func setupSubscribersSDK(count int) []WatchSubscriber { + subscribers := make([]WatchSubscriber, count) + for i := range subscribers { subscribers[i].client = getLocalSdk() - } - return subscribers + } + return subscribers } -func cleanupSubscribersSDK(subscribers []WatchSubscriber) { - for _, sub := range subscribers { - if sub.watch != nil { - sub.watch.Close() - } - } +func cleanupSubscribersSDK(subscribers []WatchSubscriber) { + for _, sub := range subscribers { + if sub.watch != nil { + sub.watch.Close() + } + } } func setUpWatchChannelsSDK(t *testing.T, ctx context.Context, subscribers []WatchSubscriber) []<-chan *dicedb.WatchResult { @@ -253,12 +253,12 @@ func testPFCountMergeSDK(t *testing.T, ctx context.Context, channels []<-chan *d func verifyWatchResultsSDK(t *testing.T, channels []<-chan *dicedb.WatchResult, expected int64) { for _, channel := range channels { select { - case v := <-channel: - assert.Equal(t, pfcountCommandSDK, v.Command) // command - assert.Equal(t, pfcountWatchFingerPrint, v.Fingerprint) // Fingerprint - assert.DeepEqual(t, expected, v.Data) // data - case <-time.After(defaultTimeout): - t.Fatal("timeout waiting for watch result") + case v := <-channel: + assert.Equal(t, pfcountCommandSDK, v.Command) // command + assert.Equal(t, pfcountWatchFingerPrint, v.Fingerprint) // Fingerprint + assert.DeepEqual(t, expected, v.Data) // data + case <-time.After(defaultTimeout): + t.Fatal("timeout waiting for watch result") } } -} \ No newline at end of file +} diff --git a/integration_tests/commands/websocket/json_test.go b/integration_tests/commands/websocket/json_test.go index 75ba0b269..71b85a079 100644 --- a/integration_tests/commands/websocket/json_test.go +++ b/integration_tests/commands/websocket/json_test.go @@ -381,6 +381,7 @@ func TestJsonARRTRIM(t *testing.T) { defer func() { resp1, err := exec.FireCommandAndReadResponse(conn, "DEL a") + assert.Nil(t, err) resp2, err := exec.FireCommandAndReadResponse(conn, "DEL b") assert.Nil(t, err) assert.Equal(t, float64(1), resp1) @@ -472,6 +473,7 @@ func TestJsonARRINSERT(t *testing.T) { defer func() { resp1, err := exec.FireCommandAndReadResponse(conn, "DEL a") + assert.Nil(t, err) resp2, err := exec.FireCommandAndReadResponse(conn, "DEL b") assert.Nil(t, err) assert.Equal(t, float64(1), resp1) diff --git a/integration_tests/commands/websocket/setup.go b/integration_tests/commands/websocket/setup.go index b75b17c4f..b8ea19b8f 100644 --- a/integration_tests/commands/websocket/setup.go +++ b/integration_tests/commands/websocket/setup.go @@ -108,7 +108,7 @@ func RunWebsocketServer(ctx context.Context, wg *sync.WaitGroup, opt TestServerO watchChan := make(chan dstore.QueryWatchEvent, config.DiceConfig.Performance.WatchChanBufSize) shardManager := shard.NewShardManager(1, watchChan, nil, globalErrChannel) queryWatcherLocal := querymanager.NewQueryManager() - config.WebsocketPort = opt.Port + config.DiceConfig.WebSocket.Port = opt.Port testServer := server.NewWebSocketServer(shardManager, testPort1, nil) shardManagerCtx, cancelShardManager := context.WithCancel(ctx) diff --git a/integration_tests/config/config_test.go b/integration_tests/config/config_test.go index acd0de531..c8063c0b0 100644 --- a/integration_tests/config/config_test.go +++ b/integration_tests/config/config_test.go @@ -1,4 +1,4 @@ -package commands +package config_test import ( "os" @@ -8,123 +8,92 @@ import ( "github.com/dicedb/dice/config" ) -// scenario 1: Create a config file if the directory is provided (-o flag) -func TestSetupConfig_CreateAndLoadDefault(t *testing.T) { - config.ResetConfig() - tempDir := t.TempDir() +const configFileName = "dicedb.conf" - // Simulate the flag: -o= - config.CustomConfigFilePath = tempDir - config.SetupConfig() +// TestCreateConfigFile_FileExists tests the scenario when config file already exists +func TestCreateConfigFile_FileExists(t *testing.T) { + tempDir := t.TempDir() + configPath := filepath.Join(tempDir, configFileName) - if config.DiceConfig.AsyncServer.Addr != config.DefaultHost { - t.Fatalf("Expected server addr to be '%s', got '%s'", config.DefaultHost, config.DiceConfig.AsyncServer.Addr) - } - if config.DiceConfig.AsyncServer.Port != config.DefaultPort { - t.Fatalf("Expected server port to be %d, got %d", config.DefaultPort, config.DiceConfig.AsyncServer.Port) + if err := os.WriteFile(configPath, []byte("test config"), 0644); err != nil { + t.Fatalf("Failed to create test config file: %v", err) } -} -// scenario 2: Load default config if no config file or directory is provided -func TestSetupConfig_DefaultConfig(t *testing.T) { - // Simulate no flags being set (default config scenario) - config.ResetConfig() - config.CustomConfigFilePath = "" - config.FileLocation = filepath.Join(config.DefaultConfigFilePath, config.DefaultConfigName) + config.CreateConfigFile(configPath) - // Verify that the configuration was loaded from the default values - if config.DiceConfig.AsyncServer.Addr != config.DefaultHost { - t.Fatalf("Expected server addr to be '%s', got '%s'", config.DefaultHost, config.DiceConfig.AsyncServer.Addr) // 127.0.0.1 + content, err := os.ReadFile(configPath) + if err != nil { + t.Fatalf("Failed to read config file: %v", err) } - if config.DiceConfig.AsyncServer.Port != config.DefaultPort { - t.Fatalf("Expected server port to be %d, got %d", 8739, config.DiceConfig.AsyncServer.Port) + + if string(content) != "test config" { + t.Error("Config file content was modified when it should have been preserved") } } -// scenario 3: Config file is present but not well-structured (Malformed) -func TestSetupConfig_InvalidConfigFile(t *testing.T) { - config.DiceConfig = nil +// TestCreateConfigFile_NewFile tests creating a new config file +func TestCreateConfigFile_NewFile(t *testing.T) { tempDir := t.TempDir() - configFilePath := filepath.Join(tempDir, "dice.toml") - - content := ` - [asyncserver] - addr = 127.0.0.1 // Missing quotes around string value - port = abc // Invalid integer - ` - if err := os.WriteFile(configFilePath, []byte(content), 0666); err != nil { - t.Fatalf("Failed to create invalid test config file: %v", err) - } - - // Simulate the flag: -c= - config.CustomConfigFilePath = "" - config.FileLocation = configFilePath + configPath := filepath.Join(tempDir, configFileName) + config.CreateConfigFile(configPath) - config.SetupConfig() + if _, err := os.Stat(configPath); os.IsNotExist(err) { + t.Error("Config file was not created") + } - if config.DiceConfig.AsyncServer.Addr != config.DefaultHost { - t.Fatalf("Expected server addr to be '%s' after unmarshal error, got '%s'", config.DefaultHost, config.DiceConfig.AsyncServer.Addr) + content, err := os.ReadFile(configPath) + if err != nil { + t.Fatalf("Failed to read created config file: %v", err) } - if config.DiceConfig.AsyncServer.Port != config.DefaultPort { - t.Fatalf("Expected server port to be %d after unmarshal error, got %d", config.DefaultPort, config.DiceConfig.AsyncServer.Port) + + if len(content) == 0 { + t.Error("Created config file is empty") } } -// scenario 4: Config file is present with partial content -func TestSetupConfig_PartialConfigFile(t *testing.T) { - tempDir := t.TempDir() - configFilePath := filepath.Join(tempDir, "dice.toml") - - content := ` - [asyncserver] - addr = "127.0.0.1" - ` - if err := os.WriteFile(configFilePath, []byte(content), 0666); err != nil { - t.Fatalf("Failed to create partial test config file: %v", err) +// TestCreateConfigFile_InvalidPath tests creation with an invalid file path +func TestCreateConfigFile_InvalidPath(t *testing.T) { + configPath := "/nonexistent/directory/dicedb.conf" + config.CreateConfigFile(configPath) + + if _, err := os.Stat(configPath); !os.IsNotExist(err) { + t.Error("Config file should not have been created at invalid path") } +} - // Simulate the flag: -c= - config.CustomConfigFilePath = "" - config.FileLocation = configFilePath +// TestCreateConfigFile_NoPermission tests creation without write permissions +func TestCreateConfigFile_NoPermission(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("Skipping test when running as root") + } - config.SetupConfig() + tempDir := t.TempDir() + err := os.Chmod(tempDir, 0555) // read + execute only + if err != nil { + t.Fatalf("Failed to change directory permissions: %v", err) + } + defer os.Chmod(tempDir, 0755) // restore permissions - t.Log(config.DiceConfig.AsyncServer.Port) + configPath := filepath.Join(tempDir, configFileName) + config.CreateConfigFile(configPath) - if config.DiceConfig.AsyncServer.Addr != "127.0.0.1" { - t.Fatalf("Expected server addr to be '127.0.0.1', got '%s'", config.DiceConfig.AsyncServer.Addr) - } - if config.DiceConfig.AsyncServer.Port != config.DefaultPort { - t.Fatalf("Expected server port to be %d (default), got %d", config.DefaultPort, config.DiceConfig.AsyncServer.Port) + if _, err := os.Stat(configPath); !os.IsNotExist(err) { + t.Error("Config file should not have been created without permissions") } } -// scenario 5: Load config from the provided file path -func TestSetupConfig_LoadFromFile(t *testing.T) { - config.ResetConfig() +// TestCreateConfigFile_ExistingDirectory tests creation in existing directory +func TestCreateConfigFile_ExistingDirectory(t *testing.T) { tempDir := t.TempDir() - configFilePath := filepath.Join(tempDir, "dice.toml") - - content := ` - [asyncserver] - addr = "127.0.0.1" - port = 8739 - ` - if err := os.WriteFile(configFilePath, []byte(content), 0666); err != nil { - t.Fatalf("Failed to write test config file: %v", err) + configDir := filepath.Join(tempDir, "config") + if err := os.MkdirAll(configDir, 0755); err != nil { + t.Fatalf("Failed to create config directory: %v", err) } - // Simulate the flag: -c= - config.CustomConfigFilePath = "" - config.FileLocation = configFilePath - - config.SetupConfig() + configPath := filepath.Join(configDir, configFileName) + config.CreateConfigFile(configPath) - if config.DiceConfig.AsyncServer.Addr != "127.0.0.1" { - t.Fatalf("Expected server addr to be '127.0.0.1', got '%s'", config.DiceConfig.AsyncServer.Addr) + if _, err := os.Stat(configPath); os.IsNotExist(err) { + t.Error("Config file was not created in existing directory") } - if config.DiceConfig.AsyncServer.Port != 8739 { - t.Fatalf("Expected server port to be 8374, got %d", config.DiceConfig.AsyncServer.Port) - } - } diff --git a/integration_tests/config/parser_test.go b/integration_tests/config/parser_test.go new file mode 100644 index 000000000..be594c34a --- /dev/null +++ b/integration_tests/config/parser_test.go @@ -0,0 +1,311 @@ +package config_test + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/dicedb/dice/config" +) + +// TestConfig is a test struct that mimics your actual config structure +type TestConfig struct { + Version string `config:"version" default:"0.0.5"` + InstanceID string `config:"instance_id"` + Auth auth `config:"auth"` + AsyncServer asyncServer `config:"async_server"` + HTTP http `config:"http"` + WebSocket websocket `config:"websocket"` + Performance performance `config:"performance"` + Memory memory `config:"memory"` + Persistence persistence `config:"persistence"` + Logging logging `config:"logging"` + Network network `config:"network"` +} + +type auth struct { + UserName string `config:"username" default:"dice"` + Password string `config:"password"` +} + +type asyncServer struct { + Addr string `config:"addr" default:"0.0.0.0"` + Port int `config:"port" default:"7379" validate:"min=1024,max=65535"` + KeepAlive int32 `config:"keepalive" default:"300"` + Timeout int32 `config:"timeout" default:"300"` + MaxConn int32 `config:"max_conn" default:"0"` +} + +type http struct { + Enabled bool `config:"enabled" default:"true"` + Port int `config:"port" default:"8082" validate:"min=1024,max=65535"` +} + +type websocket struct { + Enabled bool `config:"enabled" default:"true"` + Port int `config:"port" default:"8379" validate:"min=1024,max=65535"` + MaxWriteResponseRetries int `config:"max_write_response_retries" default:"3" validate:"min=0"` + WriteResponseTimeout time.Duration `config:"write_response_timeout" default:"10s"` +} + +type performance struct { + WatchChanBufSize int `config:"watch_chan_buf_size" default:"20000"` + ShardCronFrequency time.Duration `config:"shard_cron_frequency" default:"1s"` + MultiplexerPollTimeout time.Duration `config:"multiplexer_poll_timeout" default:"100ms"` + MaxClients int32 `config:"max_clients" default:"20000" validate:"min=0"` + EnableMultiThreading bool `config:"enable_multithreading" default:"false"` + StoreMapInitSize int `config:"store_map_init_size" default:"1024000"` + AdhocReqChanBufSize int `config:"adhoc_req_chan_buf_size" default:"20"` + EnableProfiling bool `config:"profiling" default:"false"` + EnableWatch bool `config:"enable_watch" default:"false"` + NumShards int `config:"num_shards" default:"-1" validate:"oneof=-1|min=1,lte=128"` +} + +type memory struct { + MaxMemory int64 `config:"max_memory" default:"0"` + EvictionPolicy string `config:"eviction_policy" default:"allkeys-lfu" validate:"oneof=simple-first allkeys-random allkeys-lru allkeys-lfu"` + EvictionRatio float64 `config:"eviction_ratio" default:"0.9" validate:"min=0,lte=1"` + KeysLimit int `config:"keys_limit" default:"200000000" validate:"min=0"` + LFULogFactor int `config:"lfu_log_factor" default:"10" validate:"min=0"` +} + +type persistence struct { + AOFFile string `config:"aof_file" default:"./dice-master.aof" validate:"filepath"` + PersistenceEnabled bool `config:"persistence_enabled" default:"true"` + WriteAOFOnCleanup bool `config:"write_aof_on_cleanup" default:"false"` + EnableWAL bool `config:"enable-wal" default:"false"` + WALDir string `config:"wal-dir" default:"./" validate:"dirpath"` + RestoreFromWAL bool `config:"restore-wal" default:"false"` + WALEngine string `config:"wal-engine" default:"aof" validate:"oneof=sqlite aof"` +} + +type logging struct { + LogLevel string `config:"log_level" default:"info" validate:"oneof=debug info warn error"` +} + +type network struct { + IOBufferLengthMAX int `config:"io_buffer_length_max" default:"51200" validate:"min=0,max=1048576"` // max is 1MB' + IOBufferLength int `config:"io_buffer_length" default:"512" validate:"min=0"` +} + +func TestNewConfigParser(t *testing.T) { + parser := config.NewConfigParser() + if parser == nil { + t.Fatal("NewConfigParser returned nil") + } +} + +func TestParseFromFile(t *testing.T) { + tests := []struct { + name string + content string + wantErr bool + setupErr bool + }{ + { + name: "valid config", + content: `host=testhost +port=9090 +log_level=debug`, + wantErr: false, + }, + { + name: "empty file", + content: "", + wantErr: false, + }, + { + name: "malformed config", + content: `host=testhost +invalid-line +port=9090`, + wantErr: false, + }, + { + name: "non-existent file", + setupErr: true, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + parser := config.NewConfigParser() + + // Create temporary config file + tempDir := t.TempDir() + filename := filepath.Join(tempDir, "dicedb.conf") + + if !tt.setupErr { + err := os.WriteFile(filename, []byte(tt.content), 0644) + if err != nil { + t.Fatalf("Failed to create test file: %v", err) + } + } + + err := parser.ParseFromFile(filename) + if (err != nil) != tt.wantErr { + t.Errorf("ParseFromFile() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestParseFromStdin(t *testing.T) { + tests := []struct { + name string + input string + wantErr bool + }{ + { + name: "valid input", + input: `host=testhost +port=9090 +log_level=debug`, + wantErr: false, + }, + { + name: "empty input", + input: "", + wantErr: false, + }, + { + name: "malformed input", + input: `host=testhost +invalid-line +port=9090`, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + parser := config.NewConfigParser() + + // Store original stdin + oldStdin := os.Stdin + defer func() { os.Stdin = oldStdin }() + + // Create a pipe and pass the test input + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("Failed to create pipe: %v", err) + } + os.Stdin = r + + go func() { + defer w.Close() + w.Write([]byte(tt.input)) + }() + + err = parser.ParseFromStdin() + if (err != nil) != tt.wantErr { + t.Errorf("ParseFromStdin() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestParseDefaults(t *testing.T) { + tests := []struct { + name string + cfg interface{} + wantErr bool + }{ + { + name: "valid struct", + cfg: &TestConfig{}, + wantErr: false, + }, + { + name: "nil pointer", + cfg: nil, + wantErr: true, + }, + { + name: "non-pointer", + cfg: TestConfig{}, + wantErr: true, + }, + { + name: "pointer to non-struct", + cfg: new(string), + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + parser := config.NewConfigParser() + err := parser.ParseDefaults(tt.cfg) + + if (err != nil) != tt.wantErr { + t.Errorf("ParseDefaults() error = %v, wantErr %v", err, tt.wantErr) + } + + if !tt.wantErr && tt.cfg != nil { + cfg := tt.cfg.(*TestConfig) + if cfg.AsyncServer.Addr != "0.0.0.0" || cfg.AsyncServer.Port != 7379 || cfg.Logging.LogLevel != "info" { + t.Error("Default values were not properly set") + } + } + }) + } +} + +// TestLoadconfig tests the Loadconfig method +func TestLoadconfig(t *testing.T) { + tests := []struct { + name string + cfg interface{} + content string + wantErr bool + }{ + { + name: "nil pointer", + cfg: nil, + content: "", + wantErr: true, + }, + { + name: "non-pointer", + cfg: TestConfig{}, + content: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + parser := config.NewConfigParser() + + // Create and populate config file if content is provided + if tt.content != "" { + tempDir := t.TempDir() + filename := filepath.Join(tempDir, "dicedb.conf") + err := os.WriteFile(filename, []byte(tt.content), 0644) + if err != nil { + t.Fatalf("Failed to create test file: %v", err) + } + + err = parser.ParseFromFile(filename) + if err != nil { + t.Fatalf("Failed to parse test file: %v", err) + } + } + + err := parser.Loadconfig(tt.cfg) + if (err != nil) != tt.wantErr { + t.Errorf("Loadconfig() error = %v, wantErr %v", err, tt.wantErr) + } + + if !tt.wantErr && tt.cfg != nil { + cfg := tt.cfg.(*TestConfig) + if tt.content != "" && (cfg.AsyncServer.Addr != "customhost" || cfg.AsyncServer.Port != 9090 || cfg.Logging.LogLevel != "debug") { + t.Error("Config values were not properly loaded") + } + } + }) + } +} diff --git a/integration_tests/server/server_abort_test.go b/integration_tests/server/server_abort_test.go index f4c98ac4f..fb11e69ae 100644 --- a/integration_tests/server/server_abort_test.go +++ b/integration_tests/server/server_abort_test.go @@ -18,6 +18,11 @@ var testServerOptions = commands.TestServerOptions{ Port: 8740, } +func init() { + parser := config.NewConfigParser() + parser.ParseDefaults(config.DiceConfig) +} + func TestAbortCommand(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/internal/cli/cli.go b/internal/cli/cli.go new file mode 100644 index 000000000..34c956846 --- /dev/null +++ b/internal/cli/cli.go @@ -0,0 +1,206 @@ +package cli + +import ( + "fmt" + "log" + "log/slog" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/dicedb/dice/config" +) + +type configEntry struct { + Key string + Value interface{} +} + +var configTable = []configEntry{} + +// configuration function used to add configuration values to the print table at the startup. +// add entry to this function to add a new row in the startup configuration table. +func configuration() { + // Add the version of the DiceDB to the configuration table + addEntry("Version", config.DiceDBVersion) + + // Add the port number on which DiceDB is running to the configuration table + addEntry("Port", config.DiceConfig.AsyncServer.Port) + + // Add whether multi-threading is enabled to the configuration table + addEntry("Multi Threading Enabled", config.DiceConfig.Performance.EnableMultiThreading) + + // Add the number of CPU cores available on the machine to the configuration table + addEntry("Cores", runtime.NumCPU()) + + // Conditionally add the number of shards to be used for DiceDB to the configuration table + if config.DiceConfig.Performance.EnableMultiThreading { + if config.DiceConfig.Performance.NumShards > 0 { + configTable = append(configTable, configEntry{"Shards", config.DiceConfig.Performance.NumShards}) + } else { + configTable = append(configTable, configEntry{"Shards", runtime.NumCPU()}) + } + } else { + configTable = append(configTable, configEntry{"Shards", 1}) + } + + // Add whether the watch feature is enabled to the configuration table + addEntry("Watch Enabled", config.DiceConfig.Performance.EnableWatch) + + // Add whether the watch feature is enabled to the configuration table + addEntry("HTTP Enabled", config.DiceConfig.HTTP.Enabled) + + // Add whether the watch feature is enabled to the configuration table + addEntry("Websocket Enabled", config.DiceConfig.WebSocket.Enabled) +} + +func addEntry(k string, v interface{}) { + configTable = append(configTable, configEntry{k, v}) +} + +// printConfigTable prints key-value pairs in a vertical table format. +func render() { + fmt.Print(` + ██████╗ ██╗ ██████╗███████╗██████╗ ██████╗ + ██╔══██╗██║██╔════╝██╔════╝██╔══██╗██╔══██╗ + ██║ ██║██║██║ █████╗ ██║ ██║██████╔╝ + ██║ ██║██║██║ ██╔══╝ ██║ ██║██╔══██╗ + ██████╔╝██║╚██████╗███████╗██████╔╝██████╔╝ + ╚═════╝ ╚═╝ ╚═════╝╚══════╝╚═════╝ ╚═════╝ + + `) + configuration() + + // Find the longest key to align the values properly + maxKeyLength := 0 + maxValueLength := 20 // Default value length for alignment + for _, entry := range configTable { + if len(entry.Key) > maxKeyLength { + maxKeyLength = len(entry.Key) + } + if len(fmt.Sprintf("%v", entry.Value)) > maxValueLength { + maxValueLength = len(fmt.Sprintf("%v", entry.Value)) + } + } + + // Create the table header and separator line + fmt.Println() + totalWidth := maxKeyLength + maxValueLength + 7 // 7 is for spacing and pipes + fmt.Println(strings.Repeat("-", totalWidth)) + fmt.Printf("| %-*s | %-*s |\n", maxKeyLength, "Configuration", maxValueLength, "Value") + fmt.Println(strings.Repeat("-", totalWidth)) + + // Print each configuration key-value pair without row lines + for _, entry := range configTable { + fmt.Printf("| %-*s | %-20v |\n", maxKeyLength, entry.Key, entry.Value) + } + + // Final bottom line + fmt.Println(strings.Repeat("-", totalWidth)) + fmt.Println() +} + +func Execute() { + if len(os.Args) < 2 { + if err := config.CreateConfigFile(filepath.Join(config.DefaultConfigDir, "dicedb.conf")); err != nil { + log.Fatal(err) + } + render() + return + } + + switch os.Args[1] { + case "-v", "--version": + fmt.Println("dicedb version", config.DiceDBVersion) + os.Exit(0) + + case "-h", "--help": + printUsage() + os.Exit(0) + + case "-": + parser := config.NewConfigParser() + if err := parser.ParseFromStdin(); err != nil { + log.Fatal(err) + } + if err := parser.Loadconfig(config.DiceConfig); err != nil { + log.Fatal(err) + } + fmt.Println(config.DiceConfig.Version) + case "-o", "--output": + if len(os.Args) < 3 { + log.Fatal("Output file path not provided") + } else { + dirPath := os.Args[2] + if dirPath == "" { + log.Fatal("Output file path not provided") + } + + info, err := os.Stat(dirPath) + switch { + case os.IsNotExist(err): + log.Fatal("Output file path does not exist") + case err != nil: + log.Fatalf("Error checking output file path: %v", err) + case !info.IsDir(): + log.Fatal("Output file path is not a directory") + } + + filePath := filepath.Join(dirPath, "dicedb.conf") + if _, err := os.Stat(filePath); err == nil { + slog.Warn("Config file already exists at the specified path", slog.String("path", filePath), slog.String("action", "skipping file creation")) + return + } + if err := config.CreateConfigFile(filePath); err != nil { + log.Fatal(err) + } + render() + } + case "-c", "--config": + if len(os.Args) >= 3 { + filePath := os.Args[2] + if filePath == "" { + log.Fatal("Error: Config file path not provided") + } + + info, err := os.Stat(filePath) + switch { + case os.IsNotExist(err): + log.Fatalf("Config file does not exist: %s", filePath) + case err != nil: + log.Fatalf("Unable to check config file: %v", err) + } + + if info.IsDir() { + log.Fatalf("Config file path points to a directory: %s", filePath) + } + + if !strings.HasSuffix(filePath, ".conf") { + log.Fatalf("Config file must have a .conf extension: %s", filePath) + } + + parser := config.NewConfigParser() + if err := parser.ParseFromFile(filePath); err != nil { + log.Fatal(err) + } + if err := parser.Loadconfig(config.DiceConfig); err != nil { + log.Fatal(err) + } + render() + } else { + log.Fatal("Config file path not provided") + } + + default: + fmt.Printf("Unknown option: %s\n", os.Args[1]) + printUsage() + } +} + +func printUsage() { + fmt.Println(`Usage: ./dicedb [/path/to/dice.conf] [options] [-] + ./dicedb - (read config from stdin) e.g. echo "version=1.0" | ./dicedb - + ./dicedb -v or --version + ./dicedb -h or --help`) +} diff --git a/internal/eval/eval_amd64.go b/internal/eval/eval_amd64.go index d370a64d8..440161b82 100644 --- a/internal/eval/eval_amd64.go +++ b/internal/eval/eval_amd64.go @@ -26,7 +26,7 @@ func EvalBGREWRITEAOF(args []string, store *dstore.Store) []byte { // TODO: Problem at hand: In multi-threaded environment, each shard instance would fork a child process. // TODO: Each child process would now have a copy of the network file descriptor thus resulting in resource leaks. // TODO: We need to find an alternative approach for the multi-threaded environment. - if config.EnableMultiThreading { + if config.DiceConfig.Performance.EnableMultiThreading { return nil } newChild, _, _ := syscall.Syscall(syscall.SYS_FORK, 0, 0, 0) diff --git a/internal/server/httpServer.go b/internal/server/httpServer.go index 2a993c9a0..9cad4e8a6 100644 --- a/internal/server/httpServer.go +++ b/internal/server/httpServer.go @@ -65,7 +65,7 @@ func NewHTTPServer(shardManager *shard.ShardManager, wl wal.AbstractWAL) *HTTPSe mux := http.NewServeMux() caseInsensitiveMux := &CaseInsensitiveMux{mux: mux} srv := &http.Server{ - Addr: fmt.Sprintf(":%d", config.HTTPPort), + Addr: fmt.Sprintf(":%d", config.DiceConfig.HTTP.Port), Handler: caseInsensitiveMux, ReadHeaderTimeout: 5 * time.Second, } diff --git a/internal/server/resp/server.go b/internal/server/resp/server.go index 17244d076..6f857f55d 100644 --- a/internal/server/resp/server.go +++ b/internal/server/resp/server.go @@ -96,7 +96,7 @@ func (s *Server) Run(ctx context.Context) (err error) { } }(wg) - slog.Info("ready to accept and serve requests on", slog.Int("port", config.Port)) + slog.Info("ready to accept and serve requests on", slog.Int("port", config.DiceConfig.AsyncServer.Port)) select { case <-ctx.Done(): diff --git a/internal/shard/shard_manager.go b/internal/shard/shard_manager.go index 86bf41fbf..365a6a687 100644 --- a/internal/shard/shard_manager.go +++ b/internal/shard/shard_manager.go @@ -32,7 +32,7 @@ func NewShardManager(shardCount uint8, queryWatchChan chan dstore.QueryWatchEven shardReqMap := make(map[ShardID]chan *ops.StoreOp) shardErrorChan := make(chan *ShardError) - maxKeysPerShard := config.KeysLimit / int(shardCount) + maxKeysPerShard := config.DiceConfig.Memory.KeysLimit / int(shardCount) for i := uint8(0); i < shardCount; i++ { evictionStrategy := dstore.NewBatchEvictionLRU(maxKeysPerShard, config.DiceConfig.Memory.EvictionRatio) // Shards are numbered from 0 to shardCount-1 diff --git a/main.go b/main.go index a52003d7a..6a5404847 100644 --- a/main.go +++ b/main.go @@ -3,7 +3,6 @@ package main import ( "context" "errors" - "flag" "fmt" "log/slog" "net/http" @@ -12,11 +11,11 @@ import ( "runtime" "runtime/pprof" "runtime/trace" - "strings" "sync" "syscall" "time" + "github.com/dicedb/dice/internal/cli" "github.com/dicedb/dice/internal/logger" "github.com/dicedb/dice/internal/server/abstractserver" "github.com/dicedb/dice/internal/wal" @@ -32,147 +31,11 @@ import ( "github.com/dicedb/dice/internal/worker" ) -type configEntry struct { - Key string - Value interface{} -} - -var configTable = []configEntry{} - -func init() { - flag.StringVar(&config.Host, "host", "0.0.0.0", "host for the DiceDB server") - - flag.IntVar(&config.Port, "port", 7379, "port for the DiceDB server") - - flag.IntVar(&config.HTTPPort, "http-port", 7380, "port for accepting requets over HTTP") - flag.BoolVar(&config.EnableHTTP, "enable-http", false, "enable DiceDB to listen, accept, and process HTTP") - - flag.IntVar(&config.WebsocketPort, "websocket-port", 7381, "port for accepting requets over WebSocket") - flag.BoolVar(&config.EnableWebsocket, "enable-websocket", false, "enable DiceDB to listen, accept, and process WebSocket") - - flag.BoolVar(&config.EnableMultiThreading, "enable-multithreading", false, "enable multithreading execution and leverage multiple CPU cores") - flag.IntVar(&config.NumShards, "num-shards", -1, "number shards to create. defaults to number of cores") - - flag.BoolVar(&config.EnableWatch, "enable-watch", false, "enable support for .WATCH commands and real-time reactivity") - flag.BoolVar(&config.EnableProfiling, "enable-profiling", false, "enable profiling and capture critical metrics and traces in .prof files") - - flag.StringVar(&config.DiceConfig.Logging.LogLevel, "log-level", "info", "log level, values: info, debug") - flag.StringVar(&config.LogDir, "log-dir", "/tmp/dicedb", "log directory path") - - flag.BoolVar(&config.EnableWAL, "enable-wal", false, "enable write-ahead logging") - flag.BoolVar(&config.RestoreFromWAL, "restore-wal", false, "restore the database from the WAL files") - flag.StringVar(&config.WALEngine, "wal-engine", "null", "wal engine to use, values: sqlite, aof") - - flag.StringVar(&config.RequirePass, "requirepass", config.RequirePass, "enable authentication for the default user") - flag.StringVar(&config.CustomConfigFilePath, "o", config.CustomConfigFilePath, "dir path to create the config file") - flag.StringVar(&config.FileLocation, "c", config.FileLocation, "file path of the config file") - flag.BoolVar(&config.InitConfigCmd, "init-config", false, "initialize a new config file") - - flag.IntVar(&config.KeysLimit, "keys-limit", config.KeysLimit, "keys limit for the DiceDB server. "+ - "This flag controls the number of keys each shard holds at startup. You can multiply this number with the "+ - "total number of shard threads to estimate how much memory will be required at system start up.") - flag.Float64Var(&config.EvictionRatio, "eviction-ratio", 0.1, "ratio of keys to evict when the "+ - "keys limit is reached") - - flag.Parse() - - config.SetupConfig() - +func main() { iid := observability.GetOrCreateInstanceID() config.DiceConfig.InstanceID = iid - slog.SetDefault(logger.New()) -} - -func printSplash() { - fmt.Print(` - ██████╗ ██╗ ██████╗███████╗██████╗ ██████╗ - ██╔══██╗██║██╔════╝██╔════╝██╔══██╗██╔══██╗ - ██║ ██║██║██║ █████╗ ██║ ██║██████╔╝ - ██║ ██║██║██║ ██╔══╝ ██║ ██║██╔══██╗ - ██████╔╝██║╚██████╗███████╗██████╔╝██████╔╝ - ╚═════╝ ╚═╝ ╚═════╝╚══════╝╚═════╝ ╚═════╝ - - `) -} - -// configuration function used to add configuration values to the print table at the startup. -// add entry to this function to add a new row in the startup configuration table. -func configuration() { - // Add the version of the DiceDB to the configuration table - addEntry("Version", config.DiceDBVersion) - - // Add the port number on which DiceDB is running to the configuration table - addEntry("Port", config.Port) - - // Add whether multi-threading is enabled to the configuration table - addEntry("Multi Threading Enabled", config.EnableMultiThreading) - - // Add the number of CPU cores available on the machine to the configuration table - addEntry("Cores", runtime.NumCPU()) - - // Conditionally add the number of shards to be used for DiceDB to the configuration table - if config.EnableMultiThreading { - if config.NumShards > 0 { - configTable = append(configTable, configEntry{"Shards", config.NumShards}) - } else { - configTable = append(configTable, configEntry{"Shards", runtime.NumCPU()}) - } - } else { - configTable = append(configTable, configEntry{"Shards", 1}) - } - - // Add whether the watch feature is enabled to the configuration table - addEntry("Watch Enabled", config.EnableWatch) - - // Add whether the watch feature is enabled to the configuration table - addEntry("HTTP Enabled", config.EnableHTTP) - - // Add whether the watch feature is enabled to the configuration table - addEntry("Websocket Enabled", config.EnableWebsocket) -} - -func addEntry(k string, v interface{}) { - configTable = append(configTable, configEntry{k, v}) -} - -// printConfigTable prints key-value pairs in a vertical table format. -func printConfigTable() { - configuration() - - // Find the longest key to align the values properly - maxKeyLength := 0 - maxValueLength := 20 // Default value length for alignment - for _, entry := range configTable { - if len(entry.Key) > maxKeyLength { - maxKeyLength = len(entry.Key) - } - if len(fmt.Sprintf("%v", entry.Value)) > maxValueLength { - maxValueLength = len(fmt.Sprintf("%v", entry.Value)) - } - } - - // Create the table header and separator line - fmt.Println() - totalWidth := maxKeyLength + maxValueLength + 7 // 7 is for spacing and pipes - fmt.Println(strings.Repeat("-", totalWidth)) - fmt.Printf("| %-*s | %-*s |\n", maxKeyLength, "Configuration", maxValueLength, "Value") - fmt.Println(strings.Repeat("-", totalWidth)) - - // Print each configuration key-value pair without row lines - for _, entry := range configTable { - fmt.Printf("| %-*s | %-20v |\n", maxKeyLength, entry.Key, entry.Value) - } - - // Final bottom line - fmt.Println(strings.Repeat("-", totalWidth)) - fmt.Println() -} - -func main() { - printSplash() - printConfigTable() - + cli.Execute() go observability.Ping() ctx, cancel := context.WithCancel(context.Background()) @@ -190,26 +53,26 @@ func main() { ) wl, _ = wal.NewNullWAL() - slog.Info("running with", slog.Bool("enable-wal", config.EnableWAL)) - if config.EnableWAL { - if config.WALEngine == "sqlite" { - _wl, err := wal.NewSQLiteWAL(config.LogDir) + slog.Info("running with", slog.Bool("enable-wal", config.DiceConfig.Persistence.EnableWAL)) + if config.DiceConfig.Persistence.EnableWAL { + if config.DiceConfig.Persistence.WALEngine == "sqlite" { + _wl, err := wal.NewSQLiteWAL(config.DiceConfig.Persistence.WALDir) if err != nil { - slog.Warn("could not create WAL with", slog.String("wal-engine", config.WALEngine), slog.Any("error", err)) + slog.Warn("could not create WAL with", slog.String("wal-engine", config.DiceConfig.Persistence.WALEngine), slog.Any("error", err)) sigs <- syscall.SIGKILL return } wl = _wl - } else if config.WALEngine == "aof" { - _wl, err := wal.NewAOFWAL(config.LogDir) + } else if config.DiceConfig.Persistence.WALEngine == "aof" { + _wl, err := wal.NewAOFWAL(config.DiceConfig.Persistence.WALDir) if err != nil { - slog.Warn("could not create WAL with", slog.String("wal-engine", config.WALEngine), slog.Any("error", err)) + slog.Warn("could not create WAL with", slog.String("wal-engine", config.DiceConfig.Persistence.WALEngine), slog.Any("error", err)) sigs <- syscall.SIGKILL return } wl = _wl } else { - slog.Error("unsupported WAL engine", slog.String("engine", config.WALEngine)) + slog.Error("unsupported WAL engine", slog.String("engine", config.DiceConfig.Persistence.WALEngine)) sigs <- syscall.SIGKILL return } @@ -222,14 +85,14 @@ func main() { slog.Debug("WAL initialization complete") - if config.RestoreFromWAL { + if config.DiceConfig.Persistence.RestoreFromWAL { slog.Info("restoring database from WAL") wal.ReplayWAL(wl) slog.Info("database restored from WAL") } } - if config.EnableWatch { + if config.DiceConfig.Performance.EnableWatch { bufSize := config.DiceConfig.Performance.WatchChanBufSize queryWatchChan = make(chan dstore.QueryWatchEvent, bufSize) cmdWatchChan = make(chan dstore.CmdWatchEvent, bufSize) @@ -241,10 +104,10 @@ func main() { // core count ensures the application can make full use of all available hardware. // If multithreading is not enabled, server will run on a single core. var numShards int - if config.EnableMultiThreading { + if config.DiceConfig.Performance.EnableMultiThreading { numShards = runtime.NumCPU() - if config.NumShards > 0 { - numShards = config.NumShards + if config.DiceConfig.Performance.NumShards > 0 { + numShards = config.DiceConfig.Performance.NumShards } } else { numShards = 1 @@ -269,8 +132,8 @@ func main() { var serverWg sync.WaitGroup - if config.EnableMultiThreading { - if config.EnableProfiling { + if config.DiceConfig.Performance.EnableMultiThreading { + if config.DiceConfig.Performance.EnableProfiling { stopProfiling, err := startProfiling() if err != nil { slog.Error("Profiling could not be started", slog.Any("error", err)) @@ -293,15 +156,15 @@ func main() { serverWg.Add(1) go runServer(ctx, &serverWg, asyncServer, serverErrCh) - if config.EnableHTTP { + if config.DiceConfig.HTTP.Enabled { httpServer := server.NewHTTPServer(shardManager, wl) serverWg.Add(1) go runServer(ctx, &serverWg, httpServer, serverErrCh) } } - if config.EnableWebsocket { - websocketServer := server.NewWebSocketServer(shardManager, config.WebsocketPort, wl) + if config.DiceConfig.WebSocket.Enabled { + websocketServer := server.NewWebSocketServer(shardManager, config.DiceConfig.WebSocket.Port, wl) serverWg.Add(1) go runServer(ctx, &serverWg, websocketServer, serverErrCh) } @@ -328,7 +191,7 @@ func main() { close(sigs) - if config.EnableWAL { + if config.DiceConfig.Persistence.EnableWAL { wal.ShutdownBG() } From 35be5c93f3b5e2fcf1d463ec4aca9e2236a09cf6 Mon Sep 17 00:00:00 2001 From: Jyotinder Singh Date: Sun, 17 Nov 2024 12:42:55 +0530 Subject: [PATCH 4/8] Revert "refactor: config management (#1239)" This reverts commit c03ceaa5b0530333c5cf5f6b7b8e97dee5ff3c01. --- config/config.go | 521 ++++++++++++------ config/parser.go | 228 -------- config/validator.go | 97 ---- dice.toml | 53 ++ dicedb.conf | 60 -- go.mod | 26 +- go.sum | 64 ++- .../commands/async/command_default_test.go | 4 +- integration_tests/commands/http/setup.go | 4 +- .../commands/resp/abort/server_abort_test.go | 6 - integration_tests/commands/resp/dump_test.go | 3 +- .../commands/resp/pfcountwatch_test.go | 98 ++-- .../commands/websocket/json_test.go | 2 - integration_tests/commands/websocket/setup.go | 2 +- integration_tests/config/config_test.go | 151 +++-- integration_tests/config/parser_test.go | 311 ----------- integration_tests/server/server_abort_test.go | 5 - internal/cli/cli.go | 206 ------- internal/eval/eval_amd64.go | 2 +- internal/server/httpServer.go | 2 +- internal/server/resp/server.go | 2 +- internal/shard/shard_manager.go | 2 +- main.go | 183 +++++- 23 files changed, 770 insertions(+), 1262 deletions(-) delete mode 100644 config/parser.go delete mode 100644 config/validator.go create mode 100644 dice.toml delete mode 100644 dicedb.conf delete mode 100644 integration_tests/config/parser_test.go delete mode 100644 internal/cli/cli.go diff --git a/config/config.go b/config/config.go index 5ece4a6fa..e23ad1f71 100644 --- a/config/config.go +++ b/config/config.go @@ -1,205 +1,287 @@ package config import ( - "fmt" - "log" + "errors" "log/slog" "os" "path/filepath" + "runtime" + "strings" "time" + + "github.com/dicedb/dice/internal/server/utils" + "github.com/pelletier/go-toml/v2" + "github.com/spf13/viper" ) const ( - DiceDBVersion string = "0.0.5" - DefaultConfigName string = "dicedb.conf" - DefaultConfigDir string = "." + DiceDBVersion string = "0.0.5" + + DefaultHost string = "0.0.0.0" + DefaultPort int = 7379 + DefaultConfigName string = "dice.toml" + DefaultConfigFilePath string = "./" - EvictSimpleFirst string = "simple-first" - EvictAllKeysRandom string = "allkeys-random" - EvictAllKeysLRU string = "allkeys-lru" - EvictAllKeysLFU string = "allkeys-lfu" + EvictBatchKeysLRU string = "batch_keys_lru" DefaultKeysLimit int = 200000000 DefaultEvictionRatio float64 = 0.1 ) -type Config struct { - Version string `config:"version" default:"0.0.5"` - InstanceID string `config:"instance_id"` - Auth auth `config:"auth"` - AsyncServer asyncServer `config:"async_server"` - HTTP http `config:"http"` - WebSocket websocket `config:"websocket"` - Performance performance `config:"performance"` - Memory memory `config:"memory"` - Persistence persistence `config:"persistence"` - Logging logging `config:"logging"` - Network network `config:"network"` -} +var ( + Host = DefaultHost + Port = DefaultPort -type auth struct { - UserName string `config:"username" default:"dice"` - Password string `config:"password"` -} + EnableMultiThreading = false + EnableHTTP = true + HTTPPort = 8082 -type asyncServer struct { - Addr string `config:"addr" default:"0.0.0.0"` - Port int `config:"port" default:"7379" validate:"min=1024,max=65535"` - KeepAlive int32 `config:"keepalive" default:"300"` - Timeout int32 `config:"timeout" default:"300"` - MaxConn int32 `config:"max_conn" default:"0"` -} + EnableWebsocket = true + WebsocketPort = 8379 + NumShards int = -1 -type http struct { - Enabled bool `config:"enabled" default:"true"` - Port int `config:"port" default:"8082" validate:"min=1024,max=65535"` -} + // if RequirePass is set to an empty string, no authentication is required + RequirePass = utils.EmptyStr -type websocket struct { - Enabled bool `config:"enabled" default:"true"` - Port int `config:"port" default:"8379" validate:"min=1024,max=65535"` - MaxWriteResponseRetries int `config:"max_write_response_retries" default:"3" validate:"min=0"` - WriteResponseTimeout time.Duration `config:"write_response_timeout" default:"10s"` -} + CustomConfigFilePath = utils.EmptyStr + FileLocation = utils.EmptyStr -type performance struct { - WatchChanBufSize int `config:"watch_chan_buf_size" default:"20000"` - ShardCronFrequency time.Duration `config:"shard_cron_frequency" default:"1s"` - MultiplexerPollTimeout time.Duration `config:"multiplexer_poll_timeout" default:"100ms"` - MaxClients int32 `config:"max_clients" default:"20000" validate:"min=0"` - EnableMultiThreading bool `config:"enable_multithreading" default:"false"` - StoreMapInitSize int `config:"store_map_init_size" default:"1024000"` - AdhocReqChanBufSize int `config:"adhoc_req_chan_buf_size" default:"20"` - EnableProfiling bool `config:"profiling" default:"false"` - EnableWatch bool `config:"enable_watch" default:"false"` - NumShards int `config:"num_shards" default:"-1" validate:"oneof=-1|min=1,lte=128"` -} + InitConfigCmd = false -type memory struct { - MaxMemory int64 `config:"max_memory" default:"0"` - EvictionPolicy string `config:"eviction_policy" default:"allkeys-lfu" validate:"oneof=simple-first allkeys-random allkeys-lru allkeys-lfu"` - EvictionRatio float64 `config:"eviction_ratio" default:"0.9" validate:"min=0,lte=1"` - KeysLimit int `config:"keys_limit" default:"200000000" validate:"min=0"` - LFULogFactor int `config:"lfu_log_factor" default:"10" validate:"min=0"` -} + KeysLimit = DefaultKeysLimit + EvictionRatio = DefaultEvictionRatio -type persistence struct { - AOFFile string `config:"aof_file" default:"./dice-master.aof" validate:"filepath"` - PersistenceEnabled bool `config:"persistence_enabled" default:"true"` - WriteAOFOnCleanup bool `config:"write_aof_on_cleanup" default:"false"` - EnableWAL bool `config:"enable-wal" default:"true"` - WALDir string `config:"wal-dir" default:"./" validate:"dirpath"` - RestoreFromWAL bool `config:"restore-wal" default:"false"` - WALEngine string `config:"wal-engine" default:"aof" validate:"oneof=sqlite aof"` -} + EnableProfiling = false + + EnableWatch = true + LogDir = "" -type logging struct { - LogLevel string `config:"log_level" default:"info" validate:"oneof=debug info warn error"` + EnableWAL = true + RestoreFromWAL = false + WALEngine = "sqlite" +) + +type Config struct { + Version string `mapstructure:"version"` + InstanceID string `mapstructure:"instance_id"` + AsyncServer struct { + Addr string `mapstructure:"addr"` + Port int `mapstructure:"port"` + KeepAlive int32 `mapstructure:"keepalive"` + Timeout int32 `mapstructure:"timeout"` + MaxConn int32 `mapstructure:"max-conn"` + } `mapstructure:"asyncserver"` + + HTTP struct { + Enabled bool `mapstructure:"enabled"` + Port int `mapstructure:"port"` + } `mapstructure:"http"` + + WebSocket struct { + Enabled bool `mapstructure:"enabled"` + Port int `mapstructure:"port"` + MaxWriteResponseRetries int `mapstructure:"maxwriteresponseretries"` + WriteResponseTimeout time.Duration `mapstructure:"writeresponsetimeout"` + } `mapstructure:"websocket"` + + Performance struct { + WatchChanBufSize int `mapstructure:"watchchanbufsize"` + ShardCronFrequency time.Duration `mapstructure:"shardcronfrequency"` + MultiplexerPollTimeout time.Duration `mapstructure:"servermultiplexerpolltimeout"` + MaxClients int32 `mapstructure:"maxclients"` + EnableMultiThreading bool `mapstructure:"enablemultithreading"` + StoreMapInitSize int `mapstructure:"storemapinitsize"` + AdhocReqChanBufSize int `mapstructure:"adhocreqchanbufsize"` + EnableProfiling bool `mapstructure:"profiling"` + } `mapstructure:"performance"` + + Memory struct { + MaxMemory int64 `mapstructure:"maxmemory"` + EvictionStrategy string `mapstructure:"evictionstrategy"` + EvictionRatio float64 `mapstructure:"evictionratio"` + KeysLimit int `mapstructure:"keyslimit"` + LFULogFactor int `mapstructure:"lfulogfactor"` + } `mapstructure:"memory"` + + Persistence struct { + AOFFile string `mapstructure:"aoffile"` + PersistenceEnabled bool `mapstructure:"persistenceenabled"` + WriteAOFOnCleanup bool `mapstructure:"writeaofoncleanup"` + } `mapstructure:"persistence"` + + Logging struct { + LogLevel string `mapstructure:"loglevel"` + PrettyPrintLogs bool `mapstructure:"prettyprintlogs"` + } `mapstructure:"logging"` + + Auth struct { + UserName string `mapstructure:"username"` + Password string `mapstructure:"password"` + } `mapstructure:"auth"` + + Network struct { + IOBufferLength int `mapstructure:"iobufferlength"` + IOBufferLengthMAX int `mapstructure:"iobufferlengthmax"` + } `mapstructure:"network"` + + NumShards int `mapstructure:"num_shards"` } -type network struct { - IOBufferLengthMAX int `config:"io_buffer_length_max" default:"51200" validate:"min=0,max=1048576"` // max is 1MB' - IOBufferLength int `config:"io_buffer_length" default:"512" validate:"min=0"` +// Default configurations for internal use +var baseConfig = Config{ + Version: DiceDBVersion, + AsyncServer: struct { + Addr string `mapstructure:"addr"` + Port int `mapstructure:"port"` + KeepAlive int32 `mapstructure:"keepalive"` + Timeout int32 `mapstructure:"timeout"` + MaxConn int32 `mapstructure:"max-conn"` + }{ + Addr: DefaultHost, + Port: DefaultPort, + KeepAlive: int32(300), + Timeout: int32(300), + MaxConn: int32(0), + }, + HTTP: struct { + Enabled bool `mapstructure:"enabled"` + Port int `mapstructure:"port"` + }{ + Enabled: EnableHTTP, + Port: HTTPPort, + }, + WebSocket: struct { + Enabled bool `mapstructure:"enabled"` + Port int `mapstructure:"port"` + MaxWriteResponseRetries int `mapstructure:"maxwriteresponseretries"` + WriteResponseTimeout time.Duration `mapstructure:"writeresponsetimeout"` + }{ + Enabled: EnableWebsocket, + Port: WebsocketPort, + MaxWriteResponseRetries: 3, + WriteResponseTimeout: 10 * time.Second, + }, + Performance: struct { + WatchChanBufSize int `mapstructure:"watchchanbufsize"` + ShardCronFrequency time.Duration `mapstructure:"shardcronfrequency"` + MultiplexerPollTimeout time.Duration `mapstructure:"servermultiplexerpolltimeout"` + MaxClients int32 `mapstructure:"maxclients"` + EnableMultiThreading bool `mapstructure:"enablemultithreading"` + StoreMapInitSize int `mapstructure:"storemapinitsize"` + AdhocReqChanBufSize int `mapstructure:"adhocreqchanbufsize"` + EnableProfiling bool `mapstructure:"profiling"` + }{ + WatchChanBufSize: 20000, + ShardCronFrequency: 1 * time.Second, + MultiplexerPollTimeout: 100 * time.Millisecond, + MaxClients: int32(20000), + EnableMultiThreading: false, + StoreMapInitSize: 1024000, + AdhocReqChanBufSize: 20, // assuming we wouldn't have more than 20 adhoc requests being sent at a time. + }, + Memory: struct { + MaxMemory int64 `mapstructure:"maxmemory"` + EvictionStrategy string `mapstructure:"evictionstrategy"` + EvictionRatio float64 `mapstructure:"evictionratio"` + KeysLimit int `mapstructure:"keyslimit"` + LFULogFactor int `mapstructure:"lfulogfactor"` + }{ + MaxMemory: 0, + EvictionStrategy: EvictBatchKeysLRU, + EvictionRatio: DefaultEvictionRatio, + KeysLimit: DefaultKeysLimit, + LFULogFactor: 10, + }, + Persistence: struct { + AOFFile string `mapstructure:"aoffile"` + PersistenceEnabled bool `mapstructure:"persistenceenabled"` + WriteAOFOnCleanup bool `mapstructure:"writeaofoncleanup"` + }{ + PersistenceEnabled: true, + AOFFile: "./dice-master.aof", + WriteAOFOnCleanup: false, + }, + Logging: struct { + LogLevel string `mapstructure:"loglevel"` + PrettyPrintLogs bool `mapstructure:"prettyprintlogs"` + }{ + LogLevel: "info", + PrettyPrintLogs: true, + }, + Auth: struct { + UserName string `mapstructure:"username"` + Password string `mapstructure:"password"` + }{ + UserName: "dice", + Password: RequirePass, + }, + Network: struct { + IOBufferLength int `mapstructure:"iobufferlength"` + IOBufferLengthMAX int `mapstructure:"iobufferlengthmax"` + }{ + IOBufferLength: 512, + IOBufferLengthMAX: 50 * 1024, + }, } +var defaultConfig Config + func init() { - configFilePath := filepath.Join(DefaultConfigDir, DefaultConfigName) - if err := loadDiceConfig(configFilePath); err != nil { - log.Fatalf("failed to load configuration: %v", err) - } + config := baseConfig + config.Logging.PrettyPrintLogs = false + config.Logging.LogLevel = "info" + defaultConfig = config } // DiceConfig is the global configuration object for dice -var DiceConfig = &Config{} +var DiceConfig = &defaultConfig -func CreateConfigFile(configFilePath string) error { - // Check if the config file already exists - if _, err := os.Stat(configFilePath); err == nil { - if err := loadDiceConfig(configFilePath); err != nil { - return fmt.Errorf("failed to load existing configuration: %w", err) - } - return nil +func SetupConfig() { + if InitConfigCmd { + FileLocation = getConfigPath() + createConfigFile(FileLocation) + return } - // Attempt to write a new config file - if err := writeConfigFile(configFilePath); err != nil { - slog.Warn("Failed to create config file, starting with defaults.", slog.Any("error", err)) - return nil // Continuing with defaults; may reconsider behavior. + // Check if both -o and -c flags are set + if areBothFlagsSet() { + slog.Error("Both -o and -c flags are set. Please use only one flag.") + return } - // Load the new configuration - if err := loadDiceConfig(configFilePath); err != nil { - return fmt.Errorf("failed to load newly created configuration: %w", err) + // Check if -o flag is set + if CustomConfigFilePath != utils.EmptyStr && isValidDirPath() { + createConfigFile(filepath.Join(CustomConfigFilePath, DefaultConfigName)) + return } - slog.Info("Config file successfully created.", slog.String("path", configFilePath)) - return nil + // Check if -c flag is set + if FileLocation != utils.EmptyStr || isConfigFilePresent() { + setUpViperConfig(FileLocation) + return + } + + // If no flags are set, use default configurations with prioritizing command line flags + mergeFlagsWithConfig() +} + +func createConfigFile(configFilePath string) { + if _, err := os.Stat(configFilePath); err == nil { + slog.Warn("config file already exists", slog.String("path", configFilePath)) + setUpViperConfig(configFilePath) + return + } + + if err := writeConfigFile(configFilePath); err != nil { + slog.Warn("starting DiceDB with default configurations.", slog.Any("error", err)) + return + } + + setUpViperConfig(configFilePath) + slog.Info("config file created at", slog.Any("path", configFilePath)) } -// writeConfigFile writes the default configuration to the specified file path func writeConfigFile(configFilePath string) error { - content := `# Configuration file for Dicedb - -# Version -version = "0.0.5" - -# Async Server Configuration -async_server.addr = "0.0.0.0" -async_server.port = 7379 -async_server.keepalive = 300 -async_server.timeout = 300 -async_server.max_conn = 0 - -# HTTP Configuration -http.enabled = false -http.port = 8082 - -# WebSocket Configuration -websocket.enabled = false -websocket.port = 8379 -websocket.max_write_response_retries = 3 -websocket.write_response_timeout = 10s - -# Performance Configuration -performance.watch_chan_buf_size = 20000 -performance.shard_cron_frequency = 1s -performance.multiplexer_poll_timeout = 100ms -performance.max_clients = 20000 -performance.enable_multithreading = false -performance.store_map_init_size = 1024000 -performance.adhoc_req_chan_buf_size = 20 -performance.enable_profiling = false -performance.enable_watch = false -performance.num_shards = -1 - -# Memory Configuration -memory.max_memory = 0 -memory.eviction_policy = "allkeys-lfu" -memory.eviction_ratio = 0.9 -memory.keys_limit = 200000000 -memory.lfu_log_factor = 10 - -# Persistence Configuration -persistence.aof_file = "./dice-master.aof" -persistence.persistence_enabled = true -persistence.write_aof_on_cleanup = false -persistence.enable-wal = true -persistence.wal-dir = "./" -persistence.restore-wal = false -persistence.wal-engine = "aof" - -# Logging Configuration -logging.log_level = "info" - -# Authentication Configuration -auth.username = "dice" -auth.password = "" - -# Network Configuration -network.io_buffer_length = 512 -network.io_buffer_length_max = 51200` - - // Check if the directory exists or not dir := filepath.Dir(configFilePath) if _, err := os.Stat(dir); err != nil { return err @@ -212,35 +294,112 @@ network.io_buffer_length_max = 51200` } defer file.Close() - if _, err := file.WriteString(content); err != nil { - return err + encoder := toml.NewEncoder(file) + err = encoder.Encode(defaultConfig) + return err +} + +func isValidDirPath() bool { + info, err := os.Stat(CustomConfigFilePath) + if os.IsNotExist(err) || err != nil { + return false + } + + if !info.IsDir() { + return false } + return true +} - return nil +// This function checks if both -o and -c flags are set or not +func areBothFlagsSet() bool { + return FileLocation != utils.EmptyStr && CustomConfigFilePath != utils.EmptyStr } -func loadDiceConfig(configFilePath string) error { - parser := NewConfigParser() - if err := parser.ParseFromFile(configFilePath); err != nil { - slog.Warn("Failed to parse config file", slog.String("error", err.Error()), slog.String("message", "Loading default configurations")) - return parser.ParseDefaults(DiceConfig) +func setUpViperConfig(configFilePath string) { + if configFilePath != filepath.Join(DefaultConfigFilePath, DefaultConfigName) { + viper.SetConfigName(strings.Split(filepath.Base(configFilePath), ".")[0]) + } else { + viper.SetConfigName("dice") + } + + if configFilePath == utils.EmptyStr { + viper.AddConfigPath(DefaultConfigFilePath) + } else { + viper.AddConfigPath(filepath.Dir(configFilePath)) + } + + viper.SetConfigType("toml") + if err := viper.ReadInConfig(); err != nil { + var configFileNotFoundError viper.ConfigFileNotFoundError + if errors.As(err, &configFileNotFoundError) { + slog.Warn("config file not found. Using default configurations.") + return + } + slog.Error("Error reading config file", slog.Any("error", err)) + } + + if err := viper.Unmarshal(&DiceConfig); err != nil { + slog.Error("Error unmarshalling config file", slog.Any("error", err)) + slog.Warn("starting DiceDB with default configurations.") + return } - return parser.Loadconfig(DiceConfig) + // override default configurations with command line flags + mergeFlagsWithConfig() +} + +func mergeFlagsWithConfig() { + if RequirePass != utils.EmptyStr { + DiceConfig.Auth.Password = RequirePass + } + + if Host != DefaultHost { + DiceConfig.AsyncServer.Addr = Host + } + + if Port != DefaultPort { + DiceConfig.AsyncServer.Port = Port + } + + if KeysLimit != DefaultKeysLimit { + DiceConfig.Memory.KeysLimit = KeysLimit + } + + if EvictionRatio != DefaultEvictionRatio { + DiceConfig.Memory.EvictionRatio = EvictionRatio + } +} + +// This function checks if the config file is present or not at default location or at -c flag location +func isConfigFilePresent() bool { + // If -c flag is not set then look for config file in current directory use it + if _, err := os.Stat(filepath.Join(".", DefaultConfigName)); FileLocation == utils.EmptyStr && err == nil { + FileLocation = filepath.Join(".", DefaultConfigName) + return true + } + + // will be executed if -c flag is used + _, err := os.Stat(FileLocation) + + return err == nil } // This function returns the config file path based on the OS -// func getConfigPath() string { -// switch runtime.GOOS { -// case "windows": -// FileLocation = filepath.Join("C:", "ProgramData", "dice", DefaultConfigName) -// case "darwin", "linux": -// FileLocation = filepath.Join(string(filepath.Separator), "etc", "dice", DefaultConfigName) -// default: -// // Default to current directory if OS is unknown -// FileLocation = filepath.Join(".", DefaultConfigName) -// } -// return FileLocation -// } +func getConfigPath() string { + switch runtime.GOOS { + case "windows": + FileLocation = filepath.Join("C:", "ProgramData", "dice", DefaultConfigName) + case "darwin", "linux": + FileLocation = filepath.Join(string(filepath.Separator), "etc", "dice", DefaultConfigName) + default: + // Default to current directory if OS is unknown + FileLocation = filepath.Join(".", DefaultConfigName) + } + return FileLocation +} // ResetConfig resets the DiceConfig to default configurations. This function is only used for testing purposes +func ResetConfig() { + DiceConfig = &defaultConfig +} diff --git a/config/parser.go b/config/parser.go deleted file mode 100644 index 132c5f64b..000000000 --- a/config/parser.go +++ /dev/null @@ -1,228 +0,0 @@ -package config - -import ( - "bufio" - "fmt" - "log/slog" - "os" - "reflect" - "strconv" - "strings" - "time" -) - -// ConfigParser handles the parsing of configuration files -type ConfigParser struct { - // store holds the raw key-value pairs from the config file - store map[string]string -} - -// NewConfigParser creates a new instance of ConfigParser -func NewConfigParser() *ConfigParser { - return &ConfigParser{ - store: make(map[string]string), - } -} - -// ParseFromFile reads the configuration data from a file -func (p *ConfigParser) ParseFromFile(filename string) error { - file, err := os.Open(filename) - if err != nil { - return fmt.Errorf("error opening config file: %w", err) - } - defer file.Close() - - scanner := bufio.NewScanner(file) - return processConfigData(scanner, p) -} - -// ParseFromStdin reads the configuration data from stdin -func (p *ConfigParser) ParseFromStdin() error { - scanner := bufio.NewScanner(os.Stdin) - return processConfigData(scanner, p) -} - -// ParseDefaults populates a struct with default values based on struct tag `default` -func (p *ConfigParser) ParseDefaults(cfg interface{}) error { - val := reflect.ValueOf(cfg) - if val.Kind() != reflect.Ptr || val.IsNil() { - return fmt.Errorf("config must be a non-nil pointer to a struct") - } - - val = val.Elem() - if val.Kind() != reflect.Struct { - return fmt.Errorf("config must be a pointer to a struct") - } - - return p.unmarshalStruct(val, "") -} - -// Loadconfig populates a struct with configuration values based on struct tags -func (p *ConfigParser) Loadconfig(cfg interface{}) error { - val := reflect.ValueOf(cfg) - if val.Kind() != reflect.Ptr || val.IsNil() { - return fmt.Errorf("config must be a non-nil pointer to a struct") - } - - val = val.Elem() - if val.Kind() != reflect.Struct { - return fmt.Errorf("config must be a pointer to a struct") - } - - if err := p.unmarshalStruct(val, ""); err != nil { - return fmt.Errorf("failed to unmarshal config: %w", err) - } - - if err := validateConfig(DiceConfig); err != nil { - return fmt.Errorf("failed to validate config: %w", err) - } - - return nil -} - -// processConfigData reads the configuration data line by line and stores it in the ConfigParser -func processConfigData(scanner *bufio.Scanner, p *ConfigParser) error { - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - - parts := strings.SplitN(line, "=", 2) - if len(parts) != 2 { - slog.Warn("invalid config line", slog.String("line", line)) - continue - } - - key := strings.TrimSpace(parts[0]) - value := strings.Trim(strings.TrimSpace(parts[1]), "\"") - p.store[key] = value - } - - return scanner.Err() -} - -// unmarshalStruct handles the recursive struct parsing. -func (p *ConfigParser) unmarshalStruct(val reflect.Value, prefix string) error { - typ := val.Type() - - for i := 0; i < val.NumField(); i++ { - field := val.Field(i) - fieldType := typ.Field(i) - - // Skip unexported fields just like how encoding/json does - if !field.CanSet() { - continue - } - - // Get config key or field name - key := fieldType.Tag.Get("config") - - // Use field name as key if not specified in tag - if key == "" { - key = strings.ToLower(fieldType.Name) - } - - // Skip fields with "-" tag - if key == "-" { - continue - } - - // Apply nested struct's tag as prefix - fullKey := key - if prefix != "" { - fullKey = fmt.Sprintf("%s.%s", prefix, key) - } - - // Recursively process nested structs with their prefix - if field.Kind() == reflect.Struct { - if err := p.unmarshalStruct(field, fullKey); err != nil { - return err - } - continue - } - - // Fetch and set value for non-struct fields - value, exists := p.store[fullKey] - if !exists { - // Use default value from tag if available - if defaultValue := fieldType.Tag.Get("default"); defaultValue != "" { - value = defaultValue - } else { - continue - } - } - - if err := setField(field, value); err != nil { - return fmt.Errorf("error setting field %s: %w", fullKey, err) - } - } - - return nil -} - -// setField sets the appropriate field value based on its type -func setField(field reflect.Value, value string) error { - switch field.Kind() { - case reflect.String: - field.SetString(value) - - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - if field.Type() == reflect.TypeOf(time.Duration(0)) { - // Handle time.Duration type - duration, err := time.ParseDuration(value) - if err != nil { - return fmt.Errorf("failed to parse duration: %w", err) - } - field.Set(reflect.ValueOf(duration)) - } else { - // Handle other integer types - val, err := strconv.ParseInt(value, 10, 64) - if err != nil { - return fmt.Errorf("failed to parse integer: %w", err) - } - field.SetInt(val) - } - - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - val, err := strconv.ParseUint(value, 10, 64) - if err != nil { - return err - } - field.SetUint(val) - - case reflect.Float32, reflect.Float64: - val, err := strconv.ParseFloat(value, 64) - if err != nil { - return err - } - field.SetFloat(val) - - case reflect.Bool: - val, err := strconv.ParseBool(value) - if err != nil { - return err - } - field.SetBool(val) - - case reflect.Slice: - // Get the type of the elements in the slice - elemType := field.Type().Elem() - values := strings.Split(value, ",") - slice := reflect.MakeSlice(field.Type(), len(values), len(values)) - for i, v := range values { - elem := slice.Index(i) - elemVal := reflect.New(elemType).Elem() - if err := setField(elemVal, strings.TrimSpace(v)); err != nil { - return err - } - elem.Set(elemVal) - } - field.Set(slice) - - default: - return fmt.Errorf("unsupported type: %s", field.Type()) - } - - return nil -} diff --git a/config/validator.go b/config/validator.go deleted file mode 100644 index 66669ff43..000000000 --- a/config/validator.go +++ /dev/null @@ -1,97 +0,0 @@ -package config - -import ( - "fmt" - "log" - "reflect" - "strings" - - "github.com/go-playground/validator/v10" -) - -func validateConfig(config *Config) error { - validate := validator.New() - validate.RegisterStructValidation(validateShardCount, Config{}) - - if err := validate.Struct(config); err != nil { - validationErrors, ok := err.(validator.ValidationErrors) - if !ok { - return fmt.Errorf("unexpected validation error type: %v", err) - } - - processedFields := make(map[string]bool) - - for _, validationErr := range validationErrors { - fieldName := strings.TrimPrefix(validationErr.Namespace(), "Config.") - - if processedFields[fieldName] { - continue - } - processedFields[fieldName] = true - - log.Printf("Field %s failed validation: %s", fieldName, validationErr.Tag()) - - if err := applyDefaultValuesFromTags(config, fieldName); err != nil { - return fmt.Errorf("error setting default for %s: %v", fieldName, err) - } - } - } - return nil -} - -func validateShardCount(sl validator.StructLevel) { - config := sl.Current().Interface().(Config) - if config.Performance.NumShards <= 0 && config.Performance.NumShards != -1 { - sl.ReportError(config.Performance.NumShards, "NumShards", "NumShards", "invalidValue", "must be -1 or greater than 0") - } -} - -func applyDefaultValuesFromTags(config *Config, fieldName string) error { - configType := reflect.TypeOf(config).Elem() - configValue := reflect.ValueOf(config).Elem() - - // Split the field name if it refers to a nested struct - parts := strings.Split(fieldName, ".") - var field reflect.StructField - var fieldValue reflect.Value - var found bool - - // Traverse the struct to find the nested field - for i, part := range parts { - // If it's the first field, just look in the top-level struct - if i == 0 { - field, found = configType.FieldByName(part) - if !found { - log.Printf("Warning: %s field not found", part) - return fmt.Errorf("field %s not found in config struct", part) - } - fieldValue = configValue.FieldByName(part) - } else { - // Otherwise, the struct is nested, so navigate into it - if fieldValue.Kind() == reflect.Struct { - field, found = fieldValue.Type().FieldByName(part) - if !found { - log.Printf("Warning: %s field not found in %s", part, fieldValue.Type()) - return fmt.Errorf("field %s not found in struct %s", part, fieldValue.Type()) - } - fieldValue = fieldValue.FieldByName(part) - } else { - log.Printf("Warning: %s is not a struct", fieldName) - return fmt.Errorf("%s is not a struct", fieldName) - } - } - } - - defaultValue := field.Tag.Get("default") - if defaultValue == "" { - log.Printf("Warning: %s field has no default value to set, leaving empty string", fieldName) - return nil - } - - if err := setField(fieldValue, defaultValue); err != nil { - return fmt.Errorf("error setting default value for %s: %v", fieldName, err) - } - - log.Printf("Setting default value for %s to: %s", fieldName, defaultValue) - return nil -} diff --git a/dice.toml b/dice.toml new file mode 100644 index 000000000..02f330fc8 --- /dev/null +++ b/dice.toml @@ -0,0 +1,53 @@ +# Dice configuration file example. +# +# Note that in order to read the configuration file, Dice must be +# started with the c flag and file path as first argument: +# +# go run main.go -c /path/to/dice.toml + +Version = '0.0.5' +InstanceID = '' + +[AsyncServer] +Addr = '0.0.0.0' +Port = 7379 +KeepAlive = 300 +Timeout = 300 +MaxConn = 0 + +[HTTP] +Enabled = true +Port = 8082 + +[WebSocket] +Enabled = true +Port = 8379 + +[Performance] +WatchChanBufSize = 20000 +ShardCronFrequency = 1000000000 +MultiplexerPollTimeout = 100000000 +MaxClients = 20000 +EnableMultiThreading = false +StoreMapInitSize = 1024000 +AdhocReqChanBufSize = 20 + +[Memory] +MaxMemory = 0 +EvictionPolicy = 'allkeys-lfu' +EvictionRatio = 0.1 +KeysLimit = 200000000 +LFULogFactor = 10 + +[Persistence] +AOFFile = './dice-master.aof' +PersistenceEnabled = true +WriteAOFOnCleanup = false + +[Auth] +UserName = 'dice' +Password = '' + +[Network] +IOBufferLength = 512 +IOBufferLengthMAX = 51200 diff --git a/dicedb.conf b/dicedb.conf deleted file mode 100644 index 627d19fc9..000000000 --- a/dicedb.conf +++ /dev/null @@ -1,60 +0,0 @@ -# Configuration file for Dicedb - -# Version -version = "0.0.5" - -# Async Server Configuration -async_server.addr = "0.0.0.0" -async_server.port = 7379 -async_server.keepalive = 300 -async_server.timeout = 300 -async_server.max_conn = 0 - -# HTTP Configuration -http.enabled = false -http.port = 8082 - -# WebSocket Configuration -websocket.enabled = false -websocket.port = 8379 -websocket.max_write_response_retries = 3 -websocket.write_response_timeout = 10s - -# Performance Configuration -performance.watch_chan_buf_size = 20000 -performance.shard_cron_frequency = 1s -performance.multiplexer_poll_timeout = 100ms -performance.max_clients = 20000 -performance.enable_multithreading = false -performance.store_map_init_size = 1024000 -performance.adhoc_req_chan_buf_size = 20 -performance.enable_profiling = false -performance.enable_watch = false -performance.num_shards = -1 - -# Memory Configuration -memory.max_memory = 0 -memory.eviction_policy = "allkeys-lfu" -memory.eviction_ratio = 0.9 -memory.keys_limit = 200000000 -memory.lfu_log_factor = 10 - -# Persistence Configuration -persistence.aof_file = "./dice-master.aof" -persistence.persistence_enabled = true -persistence.write_aof_on_cleanup = false -persistence.enable-wal = true -persistence.wal-dir = "./" -persistence.restore-wal = false -persistence.wal-engine = "aof" - -# Logging Configuration -logging.log_level = "info" - -# Authentication Configuration -auth.username = "dice" -auth.password = "" - -# Network Configuration -network.io_buffer_length = 512 -network.io_buffer_length_max = 51200 \ No newline at end of file diff --git a/go.mod b/go.mod index a193be0cf..64bc75043 100644 --- a/go.mod +++ b/go.mod @@ -11,20 +11,27 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-metro v0.0.0-20211217172704-adc40b04c140 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/gabriel-vasile/mimetype v1.4.3 // indirect - github.com/go-playground/locales v0.14.1 // indirect - github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect - github.com/kr/pretty v0.3.1 // indirect - github.com/leodido/go-urn v1.4.0 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/sagikazarmark/locafero v0.6.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.7.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - golang.org/x/arch v0.11.0 // indirect - golang.org/x/net v0.21.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/arch v0.12.0 // indirect golang.org/x/sys v0.27.0 // indirect golang.org/x/text v0.20.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) @@ -35,7 +42,6 @@ require ( github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964 github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da github.com/dicedb/dicedb-go v0.0.0-20241026093718-570de4575be3 - github.com/go-playground/validator/v10 v10.22.1 github.com/gobwas/glob v0.2.3 github.com/google/btree v1.1.3 github.com/google/go-cmp v0.6.0 @@ -44,8 +50,10 @@ require ( github.com/mattn/go-sqlite3 v1.14.24 github.com/mmcloughlin/geohash v0.10.0 github.com/ohler55/ojg v1.25.0 + github.com/pelletier/go-toml/v2 v2.2.3 github.com/rs/xid v1.6.0 github.com/rs/zerolog v1.33.0 + github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 github.com/twmb/murmur3 v1.1.8 github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 diff --git a/go.sum b/go.sum index 8a48e609c..242b25ad3 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/bytedance/sonic v1.12.3 h1:W2MGa7RCU1QTeYRTPE3+88mVC0yXmsRQRChiyVocVjU= +github.com/bytedance/sonic v1.12.3/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic v1.12.4 h1:9Csb3c9ZJhfUWeMtpCDCq6BUoH5ogfDFLUgQ/jG+R0k= github.com/bytedance/sonic v1.12.4/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= @@ -20,7 +22,6 @@ github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQ github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964 h1:Ew0znI2JatzKy52N1iS5muUsHkf2UJuhocH7uFW7jjs= github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -33,16 +34,12 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dicedb/dicedb-go v0.0.0-20241026093718-570de4575be3 h1:JvnAibMNGA0vQH+T47Y/d5/POURIvfJl3fFk0GIEBkQ= github.com/dicedb/dicedb-go v0.0.0-20241026093718-570de4575be3/go.mod h1:p7x5/3S6wBEmiRMwxavj1I1P1xsSVQS6fcSbeai5ic4= -github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= -github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= -github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= -github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= -github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= -github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= -github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA= -github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -54,16 +51,20 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= -github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -72,11 +73,14 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mmcloughlin/geohash v0.10.0 h1:9w1HchfDfdeLc+jFEf/04D27KP7E2QmpDu52wPbJWRE= github.com/mmcloughlin/geohash v0.10.0/go.mod h1:oNZxQo5yWJh0eMQEP/8hwQuVx9Z9tjwFUqcTB1SmG0c= github.com/ohler55/ojg v1.25.0 h1:sDwc4u4zex65Uz5Nm7O1QwDKTT+YRcpeZQTy1pffRkw= github.com/ohler55/ojg v1.25.0/go.mod h1:gQhDVpQLqrmnd2eqGAvJtn+NfKoYJbe/A4Sj3/Vro4o= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= @@ -88,6 +92,20 @@ github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= +github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= +github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -97,26 +115,38 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 h1:zzrxE1FKn5ryBNl9eKOeqQ58Y/Qpo3Q9QNxKHX5uzzQ= github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2/go.mod h1:hzfGeIUDq/j97IG+FhNqkowIyEcD88LrW6fyU3K3WqY= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/arch v0.11.0 h1:KXV8WWKCXm6tRpLirl2szsO5j/oOODwZf4hATmGVNs4= golang.org/x/arch v0.11.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/arch v0.12.0 h1:UsYJhbzPYGsT0HbEdmYcqtCv8UNGvnaL561NnIUvaKg= +golang.org/x/arch v0.12.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= +golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= +golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= @@ -124,6 +154,8 @@ google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojt gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/integration_tests/commands/async/command_default_test.go b/integration_tests/commands/async/command_default_test.go index 066fbaa12..f5ca6e542 100644 --- a/integration_tests/commands/async/command_default_test.go +++ b/integration_tests/commands/async/command_default_test.go @@ -31,7 +31,9 @@ func getCommandDefault(connection net.Conn) []interface{} { return nil } var cmds []interface{} - cmds = append(cmds, responseValue.([]interface{})...) + for _, v := range responseValue.([]interface{}) { + cmds = append(cmds, v) + } return cmds } diff --git a/integration_tests/commands/http/setup.go b/integration_tests/commands/http/setup.go index d69f78e3a..26171b2ee 100644 --- a/integration_tests/commands/http/setup.go +++ b/integration_tests/commands/http/setup.go @@ -110,11 +110,11 @@ func RunHTTPServer(ctx context.Context, wg *sync.WaitGroup, opt TestServerOption watchChan := make(chan dstore.QueryWatchEvent, config.DiceConfig.Performance.WatchChanBufSize) shardManager := shard.NewShardManager(1, watchChan, nil, globalErrChannel) queryWatcherLocal := querymanager.NewQueryManager() - config.DiceConfig.HTTP.Port = opt.Port + config.HTTPPort = opt.Port // Initialize the HTTPServer testServer := server.NewHTTPServer(shardManager, nil) // Inform the user that the server is starting - fmt.Println("Starting the test server on port", config.DiceConfig.HTTP.Port) + fmt.Println("Starting the test server on port", config.HTTPPort) shardManagerCtx, cancelShardManager := context.WithCancel(ctx) wg.Add(1) go func() { diff --git a/integration_tests/commands/resp/abort/server_abort_test.go b/integration_tests/commands/resp/abort/server_abort_test.go index 7e13bfd66..c3f788e2d 100644 --- a/integration_tests/commands/resp/abort/server_abort_test.go +++ b/integration_tests/commands/resp/abort/server_abort_test.go @@ -3,7 +3,6 @@ package abort import ( "context" "fmt" - "log" "net" "sync" "testing" @@ -18,11 +17,6 @@ var testServerOptions = resp.TestServerOptions{ Port: 8740, } -func init() { - config.DiceConfig.AsyncServer.Port = testServerOptions.Port - log.Print("Setting port to ", config.DiceConfig.AsyncServer.Port) -} - func TestAbortCommand(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer ctx.Done() diff --git a/integration_tests/commands/resp/dump_test.go b/integration_tests/commands/resp/dump_test.go index 88f41ace4..7e0f04fe8 100644 --- a/integration_tests/commands/resp/dump_test.go +++ b/integration_tests/commands/resp/dump_test.go @@ -90,7 +90,8 @@ func TestDumpRestore(t *testing.T) { t.Run(tc.name, func(t *testing.T) { FireCommand(conn, "FLUSHALL") for i, cmd := range tc.commands { - result := FireCommand(conn, cmd) + var result interface{} + result = FireCommand(conn, cmd) expected := tc.expected[i] switch exp := expected.(type) { diff --git a/integration_tests/commands/resp/pfcountwatch_test.go b/integration_tests/commands/resp/pfcountwatch_test.go index 60abf9bee..267ccc922 100644 --- a/integration_tests/commands/resp/pfcountwatch_test.go +++ b/integration_tests/commands/resp/pfcountwatch_test.go @@ -19,11 +19,11 @@ type pfcountWatchTestCase struct { } type pfcountWatchWithPFMergeTestCase struct { - destKey1 string - destValue1 []string - destKey2 string - destValue2 []string - result int64 + destKey1 string + destValue1 []string + destKey2 string + destValue2 []string + result int64 } const ( @@ -69,24 +69,24 @@ func TestPFCOUNTWATCH(t *testing.T) { respParsers := setUpRespParsers(t, subscribers) - t.Run("Basic PFCount Operations", func(t *testing.T) { - testPFCountAdd(t, publisher, respParsers) - }, + t.Run("Basic PFCount Operations", func(t *testing.T) { + testPFCountAdd(t, publisher, respParsers) + }, ) - t.Run("PFCount Operations including PFMerge", func(t *testing.T) { - testPFCountMerge(t, publisher, respParsers) - }, + t.Run("PFCount Operations including PFMerge", func(t *testing.T) { + testPFCountMerge(t, publisher, respParsers) + }, ) } func setupSubscribers(count int) []net.Conn { - subscribers := make([]net.Conn, 0, count) - for i := 0; i < count; i++ { - conn := getLocalConnection() - subscribers = append(subscribers, conn) - } - return subscribers + subscribers := make([]net.Conn, 0, count) + for i := 0; i < count; i++ { + conn := getLocalConnection() + subscribers = append(subscribers, conn) + } + return subscribers } func setUpRespParsers(t *testing.T, subscribers []net.Conn) []*clientio.RESPParser { @@ -111,7 +111,7 @@ func testPFCountAdd(t *testing.T, publisher net.Conn, respParsers []*clientio.RE for _, tc := range pfcountWatchTestCases { res := FireCommand(publisher, fmt.Sprintf("PFADD %s %s", tc.key, tc.val)) assert.Equal(t, int64(1), res) - + verifyWatchResults(t, respParsers, tc.result) } } @@ -122,7 +122,7 @@ func testPFCountMerge(t *testing.T, publisher net.Conn, respParsers []*clientio. FireCommand(publisher, fmt.Sprintf("PFADD %s %s", tc.destKey1, tc.destValue1)) FireCommand(publisher, fmt.Sprintf("PFADD %s %s", tc.destKey2, tc.destValue2)) FireCommand(publisher, fmt.Sprintf("PFMERGE %s %s %s", pfcountWatchKey, tc.destKey1, tc.destKey2)) - + verifyWatchResults(t, respParsers, tc.result) } } @@ -157,11 +157,11 @@ type pfcountWatchSDKTestCase struct { } type pfcountWatchSDKWithPFMergeTestCase struct { - destKey1 string - destValue1 []string - destKey2 string - destValue2 []string - result int64 + destKey1 string + destValue1 []string + destKey2 string + destValue2 []string + result int64 } var PFCountWatchSDKTestCases = []pfcountWatchSDKTestCase{ @@ -178,7 +178,7 @@ var pfcountWatchSDKhWithPFMergeTestCases = []pfcountWatchSDKWithPFMergeTestCase{ } func TestPFCountWATCHWithSDK(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) defer cancel() publisher := getLocalSdk() @@ -189,31 +189,31 @@ func TestPFCountWATCHWithSDK(t *testing.T) { channels := setUpWatchChannelsSDK(t, ctx, subscribers) - t.Run("Basic PFCount Operations", func(t *testing.T) { - testPFCountAddSDK(t, ctx, channels, publisher) - }, + t.Run("Basic PFCount Operations", func(t *testing.T) { + testPFCountAddSDK(t, ctx, channels, publisher) + }, ) - t.Run("PFCount Operations including PFMerge", func(t *testing.T) { - testPFCountMergeSDK(t, ctx, channels, publisher) - }, + t.Run("PFCount Operations including PFMerge", func(t *testing.T) { + testPFCountMergeSDK(t, ctx, channels, publisher) + }, ) } -func setupSubscribersSDK(count int) []WatchSubscriber { - subscribers := make([]WatchSubscriber, count) - for i := range subscribers { +func setupSubscribersSDK(count int) []WatchSubscriber { + subscribers := make([]WatchSubscriber, count) + for i := range subscribers { subscribers[i].client = getLocalSdk() - } - return subscribers + } + return subscribers } -func cleanupSubscribersSDK(subscribers []WatchSubscriber) { - for _, sub := range subscribers { - if sub.watch != nil { - sub.watch.Close() - } - } +func cleanupSubscribersSDK(subscribers []WatchSubscriber) { + for _, sub := range subscribers { + if sub.watch != nil { + sub.watch.Close() + } + } } func setUpWatchChannelsSDK(t *testing.T, ctx context.Context, subscribers []WatchSubscriber) []<-chan *dicedb.WatchResult { @@ -253,12 +253,12 @@ func testPFCountMergeSDK(t *testing.T, ctx context.Context, channels []<-chan *d func verifyWatchResultsSDK(t *testing.T, channels []<-chan *dicedb.WatchResult, expected int64) { for _, channel := range channels { select { - case v := <-channel: - assert.Equal(t, pfcountCommandSDK, v.Command) // command - assert.Equal(t, pfcountWatchFingerPrint, v.Fingerprint) // Fingerprint - assert.DeepEqual(t, expected, v.Data) // data - case <-time.After(defaultTimeout): - t.Fatal("timeout waiting for watch result") + case v := <-channel: + assert.Equal(t, pfcountCommandSDK, v.Command) // command + assert.Equal(t, pfcountWatchFingerPrint, v.Fingerprint) // Fingerprint + assert.DeepEqual(t, expected, v.Data) // data + case <-time.After(defaultTimeout): + t.Fatal("timeout waiting for watch result") } } -} +} \ No newline at end of file diff --git a/integration_tests/commands/websocket/json_test.go b/integration_tests/commands/websocket/json_test.go index 71b85a079..75ba0b269 100644 --- a/integration_tests/commands/websocket/json_test.go +++ b/integration_tests/commands/websocket/json_test.go @@ -381,7 +381,6 @@ func TestJsonARRTRIM(t *testing.T) { defer func() { resp1, err := exec.FireCommandAndReadResponse(conn, "DEL a") - assert.Nil(t, err) resp2, err := exec.FireCommandAndReadResponse(conn, "DEL b") assert.Nil(t, err) assert.Equal(t, float64(1), resp1) @@ -473,7 +472,6 @@ func TestJsonARRINSERT(t *testing.T) { defer func() { resp1, err := exec.FireCommandAndReadResponse(conn, "DEL a") - assert.Nil(t, err) resp2, err := exec.FireCommandAndReadResponse(conn, "DEL b") assert.Nil(t, err) assert.Equal(t, float64(1), resp1) diff --git a/integration_tests/commands/websocket/setup.go b/integration_tests/commands/websocket/setup.go index b8ea19b8f..b75b17c4f 100644 --- a/integration_tests/commands/websocket/setup.go +++ b/integration_tests/commands/websocket/setup.go @@ -108,7 +108,7 @@ func RunWebsocketServer(ctx context.Context, wg *sync.WaitGroup, opt TestServerO watchChan := make(chan dstore.QueryWatchEvent, config.DiceConfig.Performance.WatchChanBufSize) shardManager := shard.NewShardManager(1, watchChan, nil, globalErrChannel) queryWatcherLocal := querymanager.NewQueryManager() - config.DiceConfig.WebSocket.Port = opt.Port + config.WebsocketPort = opt.Port testServer := server.NewWebSocketServer(shardManager, testPort1, nil) shardManagerCtx, cancelShardManager := context.WithCancel(ctx) diff --git a/integration_tests/config/config_test.go b/integration_tests/config/config_test.go index c8063c0b0..acd0de531 100644 --- a/integration_tests/config/config_test.go +++ b/integration_tests/config/config_test.go @@ -1,4 +1,4 @@ -package config_test +package commands import ( "os" @@ -8,92 +8,123 @@ import ( "github.com/dicedb/dice/config" ) -const configFileName = "dicedb.conf" - -// TestCreateConfigFile_FileExists tests the scenario when config file already exists -func TestCreateConfigFile_FileExists(t *testing.T) { +// scenario 1: Create a config file if the directory is provided (-o flag) +func TestSetupConfig_CreateAndLoadDefault(t *testing.T) { + config.ResetConfig() tempDir := t.TempDir() - configPath := filepath.Join(tempDir, configFileName) - - if err := os.WriteFile(configPath, []byte("test config"), 0644); err != nil { - t.Fatalf("Failed to create test config file: %v", err) - } - config.CreateConfigFile(configPath) + // Simulate the flag: -o= + config.CustomConfigFilePath = tempDir + config.SetupConfig() - content, err := os.ReadFile(configPath) - if err != nil { - t.Fatalf("Failed to read config file: %v", err) + if config.DiceConfig.AsyncServer.Addr != config.DefaultHost { + t.Fatalf("Expected server addr to be '%s', got '%s'", config.DefaultHost, config.DiceConfig.AsyncServer.Addr) } - - if string(content) != "test config" { - t.Error("Config file content was modified when it should have been preserved") + if config.DiceConfig.AsyncServer.Port != config.DefaultPort { + t.Fatalf("Expected server port to be %d, got %d", config.DefaultPort, config.DiceConfig.AsyncServer.Port) } } -// TestCreateConfigFile_NewFile tests creating a new config file -func TestCreateConfigFile_NewFile(t *testing.T) { - tempDir := t.TempDir() - configPath := filepath.Join(tempDir, configFileName) - config.CreateConfigFile(configPath) +// scenario 2: Load default config if no config file or directory is provided +func TestSetupConfig_DefaultConfig(t *testing.T) { + // Simulate no flags being set (default config scenario) + config.ResetConfig() + config.CustomConfigFilePath = "" + config.FileLocation = filepath.Join(config.DefaultConfigFilePath, config.DefaultConfigName) - if _, err := os.Stat(configPath); os.IsNotExist(err) { - t.Error("Config file was not created") + // Verify that the configuration was loaded from the default values + if config.DiceConfig.AsyncServer.Addr != config.DefaultHost { + t.Fatalf("Expected server addr to be '%s', got '%s'", config.DefaultHost, config.DiceConfig.AsyncServer.Addr) // 127.0.0.1 } - - content, err := os.ReadFile(configPath) - if err != nil { - t.Fatalf("Failed to read created config file: %v", err) + if config.DiceConfig.AsyncServer.Port != config.DefaultPort { + t.Fatalf("Expected server port to be %d, got %d", 8739, config.DiceConfig.AsyncServer.Port) } +} - if len(content) == 0 { - t.Error("Created config file is empty") +// scenario 3: Config file is present but not well-structured (Malformed) +func TestSetupConfig_InvalidConfigFile(t *testing.T) { + config.DiceConfig = nil + tempDir := t.TempDir() + configFilePath := filepath.Join(tempDir, "dice.toml") + + content := ` + [asyncserver] + addr = 127.0.0.1 // Missing quotes around string value + port = abc // Invalid integer + ` + if err := os.WriteFile(configFilePath, []byte(content), 0666); err != nil { + t.Fatalf("Failed to create invalid test config file: %v", err) } -} -// TestCreateConfigFile_InvalidPath tests creation with an invalid file path -func TestCreateConfigFile_InvalidPath(t *testing.T) { - configPath := "/nonexistent/directory/dicedb.conf" - config.CreateConfigFile(configPath) + // Simulate the flag: -c= + config.CustomConfigFilePath = "" + config.FileLocation = configFilePath - if _, err := os.Stat(configPath); !os.IsNotExist(err) { - t.Error("Config file should not have been created at invalid path") - } -} + config.SetupConfig() -// TestCreateConfigFile_NoPermission tests creation without write permissions -func TestCreateConfigFile_NoPermission(t *testing.T) { - if os.Getuid() == 0 { - t.Skip("Skipping test when running as root") + if config.DiceConfig.AsyncServer.Addr != config.DefaultHost { + t.Fatalf("Expected server addr to be '%s' after unmarshal error, got '%s'", config.DefaultHost, config.DiceConfig.AsyncServer.Addr) + } + if config.DiceConfig.AsyncServer.Port != config.DefaultPort { + t.Fatalf("Expected server port to be %d after unmarshal error, got %d", config.DefaultPort, config.DiceConfig.AsyncServer.Port) } +} +// scenario 4: Config file is present with partial content +func TestSetupConfig_PartialConfigFile(t *testing.T) { tempDir := t.TempDir() - err := os.Chmod(tempDir, 0555) // read + execute only - if err != nil { - t.Fatalf("Failed to change directory permissions: %v", err) + configFilePath := filepath.Join(tempDir, "dice.toml") + + content := ` + [asyncserver] + addr = "127.0.0.1" + ` + if err := os.WriteFile(configFilePath, []byte(content), 0666); err != nil { + t.Fatalf("Failed to create partial test config file: %v", err) } - defer os.Chmod(tempDir, 0755) // restore permissions - configPath := filepath.Join(tempDir, configFileName) - config.CreateConfigFile(configPath) + // Simulate the flag: -c= + config.CustomConfigFilePath = "" + config.FileLocation = configFilePath + + config.SetupConfig() + + t.Log(config.DiceConfig.AsyncServer.Port) - if _, err := os.Stat(configPath); !os.IsNotExist(err) { - t.Error("Config file should not have been created without permissions") + if config.DiceConfig.AsyncServer.Addr != "127.0.0.1" { + t.Fatalf("Expected server addr to be '127.0.0.1', got '%s'", config.DiceConfig.AsyncServer.Addr) + } + if config.DiceConfig.AsyncServer.Port != config.DefaultPort { + t.Fatalf("Expected server port to be %d (default), got %d", config.DefaultPort, config.DiceConfig.AsyncServer.Port) } } -// TestCreateConfigFile_ExistingDirectory tests creation in existing directory -func TestCreateConfigFile_ExistingDirectory(t *testing.T) { +// scenario 5: Load config from the provided file path +func TestSetupConfig_LoadFromFile(t *testing.T) { + config.ResetConfig() tempDir := t.TempDir() - configDir := filepath.Join(tempDir, "config") - if err := os.MkdirAll(configDir, 0755); err != nil { - t.Fatalf("Failed to create config directory: %v", err) + configFilePath := filepath.Join(tempDir, "dice.toml") + + content := ` + [asyncserver] + addr = "127.0.0.1" + port = 8739 + ` + if err := os.WriteFile(configFilePath, []byte(content), 0666); err != nil { + t.Fatalf("Failed to write test config file: %v", err) } - configPath := filepath.Join(configDir, configFileName) - config.CreateConfigFile(configPath) + // Simulate the flag: -c= + config.CustomConfigFilePath = "" + config.FileLocation = configFilePath + + config.SetupConfig() - if _, err := os.Stat(configPath); os.IsNotExist(err) { - t.Error("Config file was not created in existing directory") + if config.DiceConfig.AsyncServer.Addr != "127.0.0.1" { + t.Fatalf("Expected server addr to be '127.0.0.1', got '%s'", config.DiceConfig.AsyncServer.Addr) } + if config.DiceConfig.AsyncServer.Port != 8739 { + t.Fatalf("Expected server port to be 8374, got %d", config.DiceConfig.AsyncServer.Port) + } + } diff --git a/integration_tests/config/parser_test.go b/integration_tests/config/parser_test.go deleted file mode 100644 index be594c34a..000000000 --- a/integration_tests/config/parser_test.go +++ /dev/null @@ -1,311 +0,0 @@ -package config_test - -import ( - "os" - "path/filepath" - "testing" - "time" - - "github.com/dicedb/dice/config" -) - -// TestConfig is a test struct that mimics your actual config structure -type TestConfig struct { - Version string `config:"version" default:"0.0.5"` - InstanceID string `config:"instance_id"` - Auth auth `config:"auth"` - AsyncServer asyncServer `config:"async_server"` - HTTP http `config:"http"` - WebSocket websocket `config:"websocket"` - Performance performance `config:"performance"` - Memory memory `config:"memory"` - Persistence persistence `config:"persistence"` - Logging logging `config:"logging"` - Network network `config:"network"` -} - -type auth struct { - UserName string `config:"username" default:"dice"` - Password string `config:"password"` -} - -type asyncServer struct { - Addr string `config:"addr" default:"0.0.0.0"` - Port int `config:"port" default:"7379" validate:"min=1024,max=65535"` - KeepAlive int32 `config:"keepalive" default:"300"` - Timeout int32 `config:"timeout" default:"300"` - MaxConn int32 `config:"max_conn" default:"0"` -} - -type http struct { - Enabled bool `config:"enabled" default:"true"` - Port int `config:"port" default:"8082" validate:"min=1024,max=65535"` -} - -type websocket struct { - Enabled bool `config:"enabled" default:"true"` - Port int `config:"port" default:"8379" validate:"min=1024,max=65535"` - MaxWriteResponseRetries int `config:"max_write_response_retries" default:"3" validate:"min=0"` - WriteResponseTimeout time.Duration `config:"write_response_timeout" default:"10s"` -} - -type performance struct { - WatchChanBufSize int `config:"watch_chan_buf_size" default:"20000"` - ShardCronFrequency time.Duration `config:"shard_cron_frequency" default:"1s"` - MultiplexerPollTimeout time.Duration `config:"multiplexer_poll_timeout" default:"100ms"` - MaxClients int32 `config:"max_clients" default:"20000" validate:"min=0"` - EnableMultiThreading bool `config:"enable_multithreading" default:"false"` - StoreMapInitSize int `config:"store_map_init_size" default:"1024000"` - AdhocReqChanBufSize int `config:"adhoc_req_chan_buf_size" default:"20"` - EnableProfiling bool `config:"profiling" default:"false"` - EnableWatch bool `config:"enable_watch" default:"false"` - NumShards int `config:"num_shards" default:"-1" validate:"oneof=-1|min=1,lte=128"` -} - -type memory struct { - MaxMemory int64 `config:"max_memory" default:"0"` - EvictionPolicy string `config:"eviction_policy" default:"allkeys-lfu" validate:"oneof=simple-first allkeys-random allkeys-lru allkeys-lfu"` - EvictionRatio float64 `config:"eviction_ratio" default:"0.9" validate:"min=0,lte=1"` - KeysLimit int `config:"keys_limit" default:"200000000" validate:"min=0"` - LFULogFactor int `config:"lfu_log_factor" default:"10" validate:"min=0"` -} - -type persistence struct { - AOFFile string `config:"aof_file" default:"./dice-master.aof" validate:"filepath"` - PersistenceEnabled bool `config:"persistence_enabled" default:"true"` - WriteAOFOnCleanup bool `config:"write_aof_on_cleanup" default:"false"` - EnableWAL bool `config:"enable-wal" default:"false"` - WALDir string `config:"wal-dir" default:"./" validate:"dirpath"` - RestoreFromWAL bool `config:"restore-wal" default:"false"` - WALEngine string `config:"wal-engine" default:"aof" validate:"oneof=sqlite aof"` -} - -type logging struct { - LogLevel string `config:"log_level" default:"info" validate:"oneof=debug info warn error"` -} - -type network struct { - IOBufferLengthMAX int `config:"io_buffer_length_max" default:"51200" validate:"min=0,max=1048576"` // max is 1MB' - IOBufferLength int `config:"io_buffer_length" default:"512" validate:"min=0"` -} - -func TestNewConfigParser(t *testing.T) { - parser := config.NewConfigParser() - if parser == nil { - t.Fatal("NewConfigParser returned nil") - } -} - -func TestParseFromFile(t *testing.T) { - tests := []struct { - name string - content string - wantErr bool - setupErr bool - }{ - { - name: "valid config", - content: `host=testhost -port=9090 -log_level=debug`, - wantErr: false, - }, - { - name: "empty file", - content: "", - wantErr: false, - }, - { - name: "malformed config", - content: `host=testhost -invalid-line -port=9090`, - wantErr: false, - }, - { - name: "non-existent file", - setupErr: true, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - parser := config.NewConfigParser() - - // Create temporary config file - tempDir := t.TempDir() - filename := filepath.Join(tempDir, "dicedb.conf") - - if !tt.setupErr { - err := os.WriteFile(filename, []byte(tt.content), 0644) - if err != nil { - t.Fatalf("Failed to create test file: %v", err) - } - } - - err := parser.ParseFromFile(filename) - if (err != nil) != tt.wantErr { - t.Errorf("ParseFromFile() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func TestParseFromStdin(t *testing.T) { - tests := []struct { - name string - input string - wantErr bool - }{ - { - name: "valid input", - input: `host=testhost -port=9090 -log_level=debug`, - wantErr: false, - }, - { - name: "empty input", - input: "", - wantErr: false, - }, - { - name: "malformed input", - input: `host=testhost -invalid-line -port=9090`, - wantErr: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - parser := config.NewConfigParser() - - // Store original stdin - oldStdin := os.Stdin - defer func() { os.Stdin = oldStdin }() - - // Create a pipe and pass the test input - r, w, err := os.Pipe() - if err != nil { - t.Fatalf("Failed to create pipe: %v", err) - } - os.Stdin = r - - go func() { - defer w.Close() - w.Write([]byte(tt.input)) - }() - - err = parser.ParseFromStdin() - if (err != nil) != tt.wantErr { - t.Errorf("ParseFromStdin() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func TestParseDefaults(t *testing.T) { - tests := []struct { - name string - cfg interface{} - wantErr bool - }{ - { - name: "valid struct", - cfg: &TestConfig{}, - wantErr: false, - }, - { - name: "nil pointer", - cfg: nil, - wantErr: true, - }, - { - name: "non-pointer", - cfg: TestConfig{}, - wantErr: true, - }, - { - name: "pointer to non-struct", - cfg: new(string), - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - parser := config.NewConfigParser() - err := parser.ParseDefaults(tt.cfg) - - if (err != nil) != tt.wantErr { - t.Errorf("ParseDefaults() error = %v, wantErr %v", err, tt.wantErr) - } - - if !tt.wantErr && tt.cfg != nil { - cfg := tt.cfg.(*TestConfig) - if cfg.AsyncServer.Addr != "0.0.0.0" || cfg.AsyncServer.Port != 7379 || cfg.Logging.LogLevel != "info" { - t.Error("Default values were not properly set") - } - } - }) - } -} - -// TestLoadconfig tests the Loadconfig method -func TestLoadconfig(t *testing.T) { - tests := []struct { - name string - cfg interface{} - content string - wantErr bool - }{ - { - name: "nil pointer", - cfg: nil, - content: "", - wantErr: true, - }, - { - name: "non-pointer", - cfg: TestConfig{}, - content: "", - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - parser := config.NewConfigParser() - - // Create and populate config file if content is provided - if tt.content != "" { - tempDir := t.TempDir() - filename := filepath.Join(tempDir, "dicedb.conf") - err := os.WriteFile(filename, []byte(tt.content), 0644) - if err != nil { - t.Fatalf("Failed to create test file: %v", err) - } - - err = parser.ParseFromFile(filename) - if err != nil { - t.Fatalf("Failed to parse test file: %v", err) - } - } - - err := parser.Loadconfig(tt.cfg) - if (err != nil) != tt.wantErr { - t.Errorf("Loadconfig() error = %v, wantErr %v", err, tt.wantErr) - } - - if !tt.wantErr && tt.cfg != nil { - cfg := tt.cfg.(*TestConfig) - if tt.content != "" && (cfg.AsyncServer.Addr != "customhost" || cfg.AsyncServer.Port != 9090 || cfg.Logging.LogLevel != "debug") { - t.Error("Config values were not properly loaded") - } - } - }) - } -} diff --git a/integration_tests/server/server_abort_test.go b/integration_tests/server/server_abort_test.go index fb11e69ae..f4c98ac4f 100644 --- a/integration_tests/server/server_abort_test.go +++ b/integration_tests/server/server_abort_test.go @@ -18,11 +18,6 @@ var testServerOptions = commands.TestServerOptions{ Port: 8740, } -func init() { - parser := config.NewConfigParser() - parser.ParseDefaults(config.DiceConfig) -} - func TestAbortCommand(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/internal/cli/cli.go b/internal/cli/cli.go deleted file mode 100644 index 34c956846..000000000 --- a/internal/cli/cli.go +++ /dev/null @@ -1,206 +0,0 @@ -package cli - -import ( - "fmt" - "log" - "log/slog" - "os" - "path/filepath" - "runtime" - "strings" - - "github.com/dicedb/dice/config" -) - -type configEntry struct { - Key string - Value interface{} -} - -var configTable = []configEntry{} - -// configuration function used to add configuration values to the print table at the startup. -// add entry to this function to add a new row in the startup configuration table. -func configuration() { - // Add the version of the DiceDB to the configuration table - addEntry("Version", config.DiceDBVersion) - - // Add the port number on which DiceDB is running to the configuration table - addEntry("Port", config.DiceConfig.AsyncServer.Port) - - // Add whether multi-threading is enabled to the configuration table - addEntry("Multi Threading Enabled", config.DiceConfig.Performance.EnableMultiThreading) - - // Add the number of CPU cores available on the machine to the configuration table - addEntry("Cores", runtime.NumCPU()) - - // Conditionally add the number of shards to be used for DiceDB to the configuration table - if config.DiceConfig.Performance.EnableMultiThreading { - if config.DiceConfig.Performance.NumShards > 0 { - configTable = append(configTable, configEntry{"Shards", config.DiceConfig.Performance.NumShards}) - } else { - configTable = append(configTable, configEntry{"Shards", runtime.NumCPU()}) - } - } else { - configTable = append(configTable, configEntry{"Shards", 1}) - } - - // Add whether the watch feature is enabled to the configuration table - addEntry("Watch Enabled", config.DiceConfig.Performance.EnableWatch) - - // Add whether the watch feature is enabled to the configuration table - addEntry("HTTP Enabled", config.DiceConfig.HTTP.Enabled) - - // Add whether the watch feature is enabled to the configuration table - addEntry("Websocket Enabled", config.DiceConfig.WebSocket.Enabled) -} - -func addEntry(k string, v interface{}) { - configTable = append(configTable, configEntry{k, v}) -} - -// printConfigTable prints key-value pairs in a vertical table format. -func render() { - fmt.Print(` - ██████╗ ██╗ ██████╗███████╗██████╗ ██████╗ - ██╔══██╗██║██╔════╝██╔════╝██╔══██╗██╔══██╗ - ██║ ██║██║██║ █████╗ ██║ ██║██████╔╝ - ██║ ██║██║██║ ██╔══╝ ██║ ██║██╔══██╗ - ██████╔╝██║╚██████╗███████╗██████╔╝██████╔╝ - ╚═════╝ ╚═╝ ╚═════╝╚══════╝╚═════╝ ╚═════╝ - - `) - configuration() - - // Find the longest key to align the values properly - maxKeyLength := 0 - maxValueLength := 20 // Default value length for alignment - for _, entry := range configTable { - if len(entry.Key) > maxKeyLength { - maxKeyLength = len(entry.Key) - } - if len(fmt.Sprintf("%v", entry.Value)) > maxValueLength { - maxValueLength = len(fmt.Sprintf("%v", entry.Value)) - } - } - - // Create the table header and separator line - fmt.Println() - totalWidth := maxKeyLength + maxValueLength + 7 // 7 is for spacing and pipes - fmt.Println(strings.Repeat("-", totalWidth)) - fmt.Printf("| %-*s | %-*s |\n", maxKeyLength, "Configuration", maxValueLength, "Value") - fmt.Println(strings.Repeat("-", totalWidth)) - - // Print each configuration key-value pair without row lines - for _, entry := range configTable { - fmt.Printf("| %-*s | %-20v |\n", maxKeyLength, entry.Key, entry.Value) - } - - // Final bottom line - fmt.Println(strings.Repeat("-", totalWidth)) - fmt.Println() -} - -func Execute() { - if len(os.Args) < 2 { - if err := config.CreateConfigFile(filepath.Join(config.DefaultConfigDir, "dicedb.conf")); err != nil { - log.Fatal(err) - } - render() - return - } - - switch os.Args[1] { - case "-v", "--version": - fmt.Println("dicedb version", config.DiceDBVersion) - os.Exit(0) - - case "-h", "--help": - printUsage() - os.Exit(0) - - case "-": - parser := config.NewConfigParser() - if err := parser.ParseFromStdin(); err != nil { - log.Fatal(err) - } - if err := parser.Loadconfig(config.DiceConfig); err != nil { - log.Fatal(err) - } - fmt.Println(config.DiceConfig.Version) - case "-o", "--output": - if len(os.Args) < 3 { - log.Fatal("Output file path not provided") - } else { - dirPath := os.Args[2] - if dirPath == "" { - log.Fatal("Output file path not provided") - } - - info, err := os.Stat(dirPath) - switch { - case os.IsNotExist(err): - log.Fatal("Output file path does not exist") - case err != nil: - log.Fatalf("Error checking output file path: %v", err) - case !info.IsDir(): - log.Fatal("Output file path is not a directory") - } - - filePath := filepath.Join(dirPath, "dicedb.conf") - if _, err := os.Stat(filePath); err == nil { - slog.Warn("Config file already exists at the specified path", slog.String("path", filePath), slog.String("action", "skipping file creation")) - return - } - if err := config.CreateConfigFile(filePath); err != nil { - log.Fatal(err) - } - render() - } - case "-c", "--config": - if len(os.Args) >= 3 { - filePath := os.Args[2] - if filePath == "" { - log.Fatal("Error: Config file path not provided") - } - - info, err := os.Stat(filePath) - switch { - case os.IsNotExist(err): - log.Fatalf("Config file does not exist: %s", filePath) - case err != nil: - log.Fatalf("Unable to check config file: %v", err) - } - - if info.IsDir() { - log.Fatalf("Config file path points to a directory: %s", filePath) - } - - if !strings.HasSuffix(filePath, ".conf") { - log.Fatalf("Config file must have a .conf extension: %s", filePath) - } - - parser := config.NewConfigParser() - if err := parser.ParseFromFile(filePath); err != nil { - log.Fatal(err) - } - if err := parser.Loadconfig(config.DiceConfig); err != nil { - log.Fatal(err) - } - render() - } else { - log.Fatal("Config file path not provided") - } - - default: - fmt.Printf("Unknown option: %s\n", os.Args[1]) - printUsage() - } -} - -func printUsage() { - fmt.Println(`Usage: ./dicedb [/path/to/dice.conf] [options] [-] - ./dicedb - (read config from stdin) e.g. echo "version=1.0" | ./dicedb - - ./dicedb -v or --version - ./dicedb -h or --help`) -} diff --git a/internal/eval/eval_amd64.go b/internal/eval/eval_amd64.go index 440161b82..d370a64d8 100644 --- a/internal/eval/eval_amd64.go +++ b/internal/eval/eval_amd64.go @@ -26,7 +26,7 @@ func EvalBGREWRITEAOF(args []string, store *dstore.Store) []byte { // TODO: Problem at hand: In multi-threaded environment, each shard instance would fork a child process. // TODO: Each child process would now have a copy of the network file descriptor thus resulting in resource leaks. // TODO: We need to find an alternative approach for the multi-threaded environment. - if config.DiceConfig.Performance.EnableMultiThreading { + if config.EnableMultiThreading { return nil } newChild, _, _ := syscall.Syscall(syscall.SYS_FORK, 0, 0, 0) diff --git a/internal/server/httpServer.go b/internal/server/httpServer.go index 9cad4e8a6..2a993c9a0 100644 --- a/internal/server/httpServer.go +++ b/internal/server/httpServer.go @@ -65,7 +65,7 @@ func NewHTTPServer(shardManager *shard.ShardManager, wl wal.AbstractWAL) *HTTPSe mux := http.NewServeMux() caseInsensitiveMux := &CaseInsensitiveMux{mux: mux} srv := &http.Server{ - Addr: fmt.Sprintf(":%d", config.DiceConfig.HTTP.Port), + Addr: fmt.Sprintf(":%d", config.HTTPPort), Handler: caseInsensitiveMux, ReadHeaderTimeout: 5 * time.Second, } diff --git a/internal/server/resp/server.go b/internal/server/resp/server.go index 6f857f55d..17244d076 100644 --- a/internal/server/resp/server.go +++ b/internal/server/resp/server.go @@ -96,7 +96,7 @@ func (s *Server) Run(ctx context.Context) (err error) { } }(wg) - slog.Info("ready to accept and serve requests on", slog.Int("port", config.DiceConfig.AsyncServer.Port)) + slog.Info("ready to accept and serve requests on", slog.Int("port", config.Port)) select { case <-ctx.Done(): diff --git a/internal/shard/shard_manager.go b/internal/shard/shard_manager.go index 365a6a687..86bf41fbf 100644 --- a/internal/shard/shard_manager.go +++ b/internal/shard/shard_manager.go @@ -32,7 +32,7 @@ func NewShardManager(shardCount uint8, queryWatchChan chan dstore.QueryWatchEven shardReqMap := make(map[ShardID]chan *ops.StoreOp) shardErrorChan := make(chan *ShardError) - maxKeysPerShard := config.DiceConfig.Memory.KeysLimit / int(shardCount) + maxKeysPerShard := config.KeysLimit / int(shardCount) for i := uint8(0); i < shardCount; i++ { evictionStrategy := dstore.NewBatchEvictionLRU(maxKeysPerShard, config.DiceConfig.Memory.EvictionRatio) // Shards are numbered from 0 to shardCount-1 diff --git a/main.go b/main.go index 6a5404847..a52003d7a 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "context" "errors" + "flag" "fmt" "log/slog" "net/http" @@ -11,11 +12,11 @@ import ( "runtime" "runtime/pprof" "runtime/trace" + "strings" "sync" "syscall" "time" - "github.com/dicedb/dice/internal/cli" "github.com/dicedb/dice/internal/logger" "github.com/dicedb/dice/internal/server/abstractserver" "github.com/dicedb/dice/internal/wal" @@ -31,11 +32,147 @@ import ( "github.com/dicedb/dice/internal/worker" ) -func main() { +type configEntry struct { + Key string + Value interface{} +} + +var configTable = []configEntry{} + +func init() { + flag.StringVar(&config.Host, "host", "0.0.0.0", "host for the DiceDB server") + + flag.IntVar(&config.Port, "port", 7379, "port for the DiceDB server") + + flag.IntVar(&config.HTTPPort, "http-port", 7380, "port for accepting requets over HTTP") + flag.BoolVar(&config.EnableHTTP, "enable-http", false, "enable DiceDB to listen, accept, and process HTTP") + + flag.IntVar(&config.WebsocketPort, "websocket-port", 7381, "port for accepting requets over WebSocket") + flag.BoolVar(&config.EnableWebsocket, "enable-websocket", false, "enable DiceDB to listen, accept, and process WebSocket") + + flag.BoolVar(&config.EnableMultiThreading, "enable-multithreading", false, "enable multithreading execution and leverage multiple CPU cores") + flag.IntVar(&config.NumShards, "num-shards", -1, "number shards to create. defaults to number of cores") + + flag.BoolVar(&config.EnableWatch, "enable-watch", false, "enable support for .WATCH commands and real-time reactivity") + flag.BoolVar(&config.EnableProfiling, "enable-profiling", false, "enable profiling and capture critical metrics and traces in .prof files") + + flag.StringVar(&config.DiceConfig.Logging.LogLevel, "log-level", "info", "log level, values: info, debug") + flag.StringVar(&config.LogDir, "log-dir", "/tmp/dicedb", "log directory path") + + flag.BoolVar(&config.EnableWAL, "enable-wal", false, "enable write-ahead logging") + flag.BoolVar(&config.RestoreFromWAL, "restore-wal", false, "restore the database from the WAL files") + flag.StringVar(&config.WALEngine, "wal-engine", "null", "wal engine to use, values: sqlite, aof") + + flag.StringVar(&config.RequirePass, "requirepass", config.RequirePass, "enable authentication for the default user") + flag.StringVar(&config.CustomConfigFilePath, "o", config.CustomConfigFilePath, "dir path to create the config file") + flag.StringVar(&config.FileLocation, "c", config.FileLocation, "file path of the config file") + flag.BoolVar(&config.InitConfigCmd, "init-config", false, "initialize a new config file") + + flag.IntVar(&config.KeysLimit, "keys-limit", config.KeysLimit, "keys limit for the DiceDB server. "+ + "This flag controls the number of keys each shard holds at startup. You can multiply this number with the "+ + "total number of shard threads to estimate how much memory will be required at system start up.") + flag.Float64Var(&config.EvictionRatio, "eviction-ratio", 0.1, "ratio of keys to evict when the "+ + "keys limit is reached") + + flag.Parse() + + config.SetupConfig() + iid := observability.GetOrCreateInstanceID() config.DiceConfig.InstanceID = iid + slog.SetDefault(logger.New()) - cli.Execute() +} + +func printSplash() { + fmt.Print(` + ██████╗ ██╗ ██████╗███████╗██████╗ ██████╗ + ██╔══██╗██║██╔════╝██╔════╝██╔══██╗██╔══██╗ + ██║ ██║██║██║ █████╗ ██║ ██║██████╔╝ + ██║ ██║██║██║ ██╔══╝ ██║ ██║██╔══██╗ + ██████╔╝██║╚██████╗███████╗██████╔╝██████╔╝ + ╚═════╝ ╚═╝ ╚═════╝╚══════╝╚═════╝ ╚═════╝ + + `) +} + +// configuration function used to add configuration values to the print table at the startup. +// add entry to this function to add a new row in the startup configuration table. +func configuration() { + // Add the version of the DiceDB to the configuration table + addEntry("Version", config.DiceDBVersion) + + // Add the port number on which DiceDB is running to the configuration table + addEntry("Port", config.Port) + + // Add whether multi-threading is enabled to the configuration table + addEntry("Multi Threading Enabled", config.EnableMultiThreading) + + // Add the number of CPU cores available on the machine to the configuration table + addEntry("Cores", runtime.NumCPU()) + + // Conditionally add the number of shards to be used for DiceDB to the configuration table + if config.EnableMultiThreading { + if config.NumShards > 0 { + configTable = append(configTable, configEntry{"Shards", config.NumShards}) + } else { + configTable = append(configTable, configEntry{"Shards", runtime.NumCPU()}) + } + } else { + configTable = append(configTable, configEntry{"Shards", 1}) + } + + // Add whether the watch feature is enabled to the configuration table + addEntry("Watch Enabled", config.EnableWatch) + + // Add whether the watch feature is enabled to the configuration table + addEntry("HTTP Enabled", config.EnableHTTP) + + // Add whether the watch feature is enabled to the configuration table + addEntry("Websocket Enabled", config.EnableWebsocket) +} + +func addEntry(k string, v interface{}) { + configTable = append(configTable, configEntry{k, v}) +} + +// printConfigTable prints key-value pairs in a vertical table format. +func printConfigTable() { + configuration() + + // Find the longest key to align the values properly + maxKeyLength := 0 + maxValueLength := 20 // Default value length for alignment + for _, entry := range configTable { + if len(entry.Key) > maxKeyLength { + maxKeyLength = len(entry.Key) + } + if len(fmt.Sprintf("%v", entry.Value)) > maxValueLength { + maxValueLength = len(fmt.Sprintf("%v", entry.Value)) + } + } + + // Create the table header and separator line + fmt.Println() + totalWidth := maxKeyLength + maxValueLength + 7 // 7 is for spacing and pipes + fmt.Println(strings.Repeat("-", totalWidth)) + fmt.Printf("| %-*s | %-*s |\n", maxKeyLength, "Configuration", maxValueLength, "Value") + fmt.Println(strings.Repeat("-", totalWidth)) + + // Print each configuration key-value pair without row lines + for _, entry := range configTable { + fmt.Printf("| %-*s | %-20v |\n", maxKeyLength, entry.Key, entry.Value) + } + + // Final bottom line + fmt.Println(strings.Repeat("-", totalWidth)) + fmt.Println() +} + +func main() { + printSplash() + printConfigTable() + go observability.Ping() ctx, cancel := context.WithCancel(context.Background()) @@ -53,26 +190,26 @@ func main() { ) wl, _ = wal.NewNullWAL() - slog.Info("running with", slog.Bool("enable-wal", config.DiceConfig.Persistence.EnableWAL)) - if config.DiceConfig.Persistence.EnableWAL { - if config.DiceConfig.Persistence.WALEngine == "sqlite" { - _wl, err := wal.NewSQLiteWAL(config.DiceConfig.Persistence.WALDir) + slog.Info("running with", slog.Bool("enable-wal", config.EnableWAL)) + if config.EnableWAL { + if config.WALEngine == "sqlite" { + _wl, err := wal.NewSQLiteWAL(config.LogDir) if err != nil { - slog.Warn("could not create WAL with", slog.String("wal-engine", config.DiceConfig.Persistence.WALEngine), slog.Any("error", err)) + slog.Warn("could not create WAL with", slog.String("wal-engine", config.WALEngine), slog.Any("error", err)) sigs <- syscall.SIGKILL return } wl = _wl - } else if config.DiceConfig.Persistence.WALEngine == "aof" { - _wl, err := wal.NewAOFWAL(config.DiceConfig.Persistence.WALDir) + } else if config.WALEngine == "aof" { + _wl, err := wal.NewAOFWAL(config.LogDir) if err != nil { - slog.Warn("could not create WAL with", slog.String("wal-engine", config.DiceConfig.Persistence.WALEngine), slog.Any("error", err)) + slog.Warn("could not create WAL with", slog.String("wal-engine", config.WALEngine), slog.Any("error", err)) sigs <- syscall.SIGKILL return } wl = _wl } else { - slog.Error("unsupported WAL engine", slog.String("engine", config.DiceConfig.Persistence.WALEngine)) + slog.Error("unsupported WAL engine", slog.String("engine", config.WALEngine)) sigs <- syscall.SIGKILL return } @@ -85,14 +222,14 @@ func main() { slog.Debug("WAL initialization complete") - if config.DiceConfig.Persistence.RestoreFromWAL { + if config.RestoreFromWAL { slog.Info("restoring database from WAL") wal.ReplayWAL(wl) slog.Info("database restored from WAL") } } - if config.DiceConfig.Performance.EnableWatch { + if config.EnableWatch { bufSize := config.DiceConfig.Performance.WatchChanBufSize queryWatchChan = make(chan dstore.QueryWatchEvent, bufSize) cmdWatchChan = make(chan dstore.CmdWatchEvent, bufSize) @@ -104,10 +241,10 @@ func main() { // core count ensures the application can make full use of all available hardware. // If multithreading is not enabled, server will run on a single core. var numShards int - if config.DiceConfig.Performance.EnableMultiThreading { + if config.EnableMultiThreading { numShards = runtime.NumCPU() - if config.DiceConfig.Performance.NumShards > 0 { - numShards = config.DiceConfig.Performance.NumShards + if config.NumShards > 0 { + numShards = config.NumShards } } else { numShards = 1 @@ -132,8 +269,8 @@ func main() { var serverWg sync.WaitGroup - if config.DiceConfig.Performance.EnableMultiThreading { - if config.DiceConfig.Performance.EnableProfiling { + if config.EnableMultiThreading { + if config.EnableProfiling { stopProfiling, err := startProfiling() if err != nil { slog.Error("Profiling could not be started", slog.Any("error", err)) @@ -156,15 +293,15 @@ func main() { serverWg.Add(1) go runServer(ctx, &serverWg, asyncServer, serverErrCh) - if config.DiceConfig.HTTP.Enabled { + if config.EnableHTTP { httpServer := server.NewHTTPServer(shardManager, wl) serverWg.Add(1) go runServer(ctx, &serverWg, httpServer, serverErrCh) } } - if config.DiceConfig.WebSocket.Enabled { - websocketServer := server.NewWebSocketServer(shardManager, config.DiceConfig.WebSocket.Port, wl) + if config.EnableWebsocket { + websocketServer := server.NewWebSocketServer(shardManager, config.WebsocketPort, wl) serverWg.Add(1) go runServer(ctx, &serverWg, websocketServer, serverErrCh) } @@ -191,7 +328,7 @@ func main() { close(sigs) - if config.DiceConfig.Persistence.EnableWAL { + if config.EnableWAL { wal.ShutdownBG() } From c608b5cdb00b6ec3ff62e32183694792d1b37ad8 Mon Sep 17 00:00:00 2001 From: Vansh Chopra <76000026+vanshavenger@users.noreply.github.com> Date: Sun, 17 Nov 2024 15:09:20 +0530 Subject: [PATCH 5/8] #1283: Feat: ZADD Docs and Error Messages (#1290) Co-authored-by: Jyotinder Singh --- docs/src/content/docs/commands/ZADD.md | 123 ++++++++++++++++++ integration_tests/commands/http/zset_test.go | 70 +++++----- integration_tests/commands/resp/zset_test.go | 70 +++++----- .../commands/websocket/zset_test.go | 70 +++++----- internal/eval/store_eval.go | 6 +- 5 files changed, 231 insertions(+), 108 deletions(-) create mode 100644 docs/src/content/docs/commands/ZADD.md diff --git a/docs/src/content/docs/commands/ZADD.md b/docs/src/content/docs/commands/ZADD.md new file mode 100644 index 000000000..2b684452b --- /dev/null +++ b/docs/src/content/docs/commands/ZADD.md @@ -0,0 +1,123 @@ +--- +title: ZADD +description: The ZADD command adds one or more members with scores to a sorted set in DiceDB. If the key doesn't exist, it creates a new sorted set. If a member already exists, its score is updated. This command is essential for managing sorted data efficiently. +--- + +The ZADD command in DiceDB is used to add one or more members with their associated scores to a sorted set. If the specified key doesn't exist, it creates a new sorted set. For existing members, their scores are updated. This command is crucial for maintaining ordered data structures efficiently. + +## Syntax + +```bash +ZADD key [NX|XX] [GT|LT] [CH] [INCR] score member [score member ...] +``` + +## Parameters + +| Parameter | Description | Type | Required | +| --------- | ----------------------------------------------------------------------------------------------------------------- | ------ | -------- | +| `key` | The key of the sorted set | String | Yes | +| `score` | The score associated with the member | Float | Yes | +| `member` | The member to be added to the sorted set | String | Yes | +| `NX` | Only add new elements. Don't update existing elements. | Flag | No | +| `XX` | Only update existing elements. Don't add new elements. | Flag | No | +| `GT` | Only update existing elements if the new score is greater than the current score | Flag | No | +| `LT` | Only update existing elements if the new score is less than the current score | Flag | No | +| `CH` | Modify the return value from the number of new elements added, to the total number of elements changed | Flag | No | +| `INCR` | When this option is specified, ZADD acts like ZINCRBY. Only one score-element pair can be specified in this mode. | Flag | No | + +## Return values + +| Condition | Return Value | +| ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------ | +| Command executed successfully | The number of elements added to the sorted set (not including elements already existing for which the score was updated) | +| Key holds a value that is not a sorted set | Error message | + +## Behaviour + +- If the key does not exist, a new sorted set is created and the specified members are added with their respective scores. +- If a specified member already exists in the sorted set, its score is updated to the new score provided. +- Members are always added in sorted order according to their score, from the lowest to the highest. +- If multiple score-member pairs are specified, they are processed left to right. +- The `NX` and `XX` options are mutually exclusive and cannot be used together. +- When `CH` is specified, the command returns the total number of elements changed (added and updated). +- The `INCR` option allows the command to behave like ZINCRBY, incrementing the existing score of a member (or setting it if it doesn't exist). + +## Errors + +1. `Wrong type of value or key`: + + - Error Message: `(error) WRONGTYPE Operation against a key holding the wrong kind of value` + - Occurs when attempting to use ZADD on a key that contains a non-sorted set value. + +2. `Invalid score`: + - Error Message: `(error) ERR value is not an integer or a float` + - Occurs when the provided score is not a valid floating-point number. + +## Example Usage + +### Basic Usage + +```bash +127.0.0.1:7379> ZADD myzset 1 "one" 2 "two" 3 "three" +(integer) 3 +127.0.0.1:7379> ZADD myzset 4 "four" +(integer) 1 +``` + +### Updating Existing Members + +```bash +127.0.0.1:7379> ZADD myzset 5 "two" +(integer) 0 +``` + +### Using NX Option + +```bash +127.0.0.1:7379> ZADD myzset NX 6 "six" 7 "two" +(integer) 1 +``` + +### Using XX Option + +```bash +127.0.0.1:7379> ZADD myzset XX 8 "eight" 9 "two" +(integer) 0 +``` + +### Using CH Option + +```bash +127.0.0.1:7379> ZADD myzset CH 10 "ten" 11 "two" +(integer) 2 +``` + +### using INCR Option + +```bash +127.0.0.1:7379> ZADD myzset INCR 1 "two" +(integer) 12 +``` + +## Invalid Usage + +```bash +127.0.0.1:7379> ZADD myzset NX XX 12 "twelve" +(error) ERR XX and NX options at the same time are not compatible +``` + +```bash +127.0.0.1:7379> ZADD myzset LT GT 15 "twelve" +(error) ERR GT, LT, and/or NX options at the same time are not compatible +``` + +## Best Practices + +- Use appropriate score values to maintain the desired order of elements in the sorted set. +- Consider using the `NX` or `XX` options when you want to specifically add new elements or update existing ones, respectively. +- Use the `CH` option when you need to know the total number of elements changed, including both additions and updates. + +## Notes + +- The time complexity of ZADD is O(log(N)) for each item added, where N is the number of elements in the sorted set. +- Scores can be any double-precision floating-point number. diff --git a/integration_tests/commands/http/zset_test.go b/integration_tests/commands/http/zset_test.go index 67e23ce01..1b54678e9 100644 --- a/integration_tests/commands/http/zset_test.go +++ b/integration_tests/commands/http/zset_test.go @@ -284,7 +284,7 @@ func TestZADD(t *testing.T) { commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "XX", "20", "member1"}}}, }, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD XX and CH compatible", @@ -305,21 +305,21 @@ func TestZADD(t *testing.T) { commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"XX", "INCR", "20", "member1", "25", "member2"}}}, }, - expected: []interface{}{"ERR incr option supports a single increment-element pair"}, + expected: []interface{}{"ERR INCR option supports a single increment-element pair"}, }, { name: "ZADD XX, LT and GT are not compatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"XX", "LT", "GT", "20", "member1"}}}, }, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD XX, LT, GT, CH, INCR are not compatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"XX", "LT", "GT", "INCR", "CH", "20", "member1"}}}, }, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD XX, GT and CH compatible", @@ -411,105 +411,105 @@ func TestZADD(t *testing.T) { commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "XX", "20", "member1"}}}, }, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX CH not compatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "XX", "CH", "20", "member1"}}}, }, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX CH INCR not compatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "XX", "CH", "INCR", "20", "member1"}}}, }, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX LT not compatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "XX", "LT", "20", "member1"}}}, }, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX GT not compatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "XX", "GT", "20", "member1"}}}, }, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX LT CH not compatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "XX", "LT", "CH", "20", "member1"}}}, }, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX LT CH INCR not compatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "XX", "LT", "CH", "INCR", "20", "member1"}}}, }, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX GT CH not compatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "XX", "GT", "CH", "20", "member1"}}}, }, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX GT CH INCR not compatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "XX", "GT", "CH", "INCR", "20", "member1"}}}, }, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX INCR not compatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "XX", "INCR", "20", "member1"}}}, }, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX INCR LT not compatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "XX", "INCR", "LT", "20", "member1"}}}, }, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX INCR GT not compatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "XX", "INCR", "GT", "20", "member1"}}}, }, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX LT GT not compatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "XX", "LT", "GT", "20", "member1"}}}, }, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX LT GT CH not compatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "XX", "LT", "GT", "CH", "20", "member1"}}}, }, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX LT GT CH INCR not compatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "XX", "LT", "GT", "CH", "INCR", "20", "member1"}}}, }, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, // NX without XX and all LT GT CH and INCR - all errors @@ -518,84 +518,84 @@ func TestZADD(t *testing.T) { commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "GT", "20", "member1"}}}, }, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX and LT incompatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "LT", "20", "member1"}}}, }, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, LT and GT incompatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "LT", "GT", "20", "member1"}}}, }, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, LT, GT and INCR incompatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "LT", "GT", "INCR", "20", "member1"}}}, }, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, LT, GT and CH incompatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "LT", "GT", "CH", "20", "member1"}}}, }, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, LT, GT, CH and INCR incompatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "LT", "GT", "CH", "INCR", "20", "member1"}}}, }, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, LT, CH not compatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "LT", "CH", "20", "member1"}}}, }, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, LT, INCR not compatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "LT", "INCR", "20", "member1"}}}, }, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, LT, CH, INCR not compatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "LT", "CH", "INCR", "20", "member1"}}}, }, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, GT, CH not compatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "GT", "CH", "20", "member1"}}}, }, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, GT, INCR not compatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "GT", "INCR", "20", "member1"}}}, }, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, GT, CH, INCR not compatible", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"NX", "GT", "CH", "INCR", "20", "member1"}}}, }, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, CH with new member returns CH based - if added or not", @@ -631,28 +631,28 @@ func TestZADD(t *testing.T) { commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"GT", "LT", "15", "member15"}}}, }, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD GT LT CH", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"GT", "LT", "CH", "15", "member15"}}}, }, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD GT LT CH INCR", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"GT", "LT", "CH", "INCR", "15", "member15"}}}, }, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD GT LT INCR", commands: []HTTPCommand{ {Command: "ZADD", Body: map[string]interface{}{"key": "myzset2", "values": [...]string{"GT", "LT", "INCR", "15", "member15"}}}, }, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD GT CH with existing member score less no change hence 0", diff --git a/integration_tests/commands/resp/zset_test.go b/integration_tests/commands/resp/zset_test.go index 9f246d9ed..83217f593 100644 --- a/integration_tests/commands/resp/zset_test.go +++ b/integration_tests/commands/resp/zset_test.go @@ -188,7 +188,7 @@ func TestZADD(t *testing.T) { { name: "ZADD NX and XX not compatible", commands: []string{"ZADD key NX XX 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD XX and CH compatible", @@ -203,18 +203,18 @@ func TestZADD(t *testing.T) { { name: "ZADD INCR and XX not compatible because of more than one member", commands: []string{"ZADD key XX INCR 20 member1 25 member2"}, - expected: []interface{}{"ERR incr option supports a single increment-element pair"}, + expected: []interface{}{"ERR INCR option supports a single increment-element pair"}, }, { name: "ZADD XX, LT and GT are not compatible", commands: []string{"ZADD key XX LT GT 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD XX, LT, GT, CH, INCR are not compatible", commands: []string{"ZADD key XX LT GT INCR CH 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { @@ -290,139 +290,139 @@ func TestZADD(t *testing.T) { { name: "ZADD NX and XX not compatible", commands: []string{"ZADD key NX XX 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX CH not compatible", commands: []string{"ZADD key NX XX CH 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX CH INCR not compatible", commands: []string{"ZADD key NX XX CH INCR 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX LT not compatible", commands: []string{"ZADD key NX XX LT 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX GT not compatible", commands: []string{"ZADD key NX XX GT 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX LT CH not compatible", commands: []string{"ZADD key NX XX LT CH 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX LT CH INCR compatible", commands: []string{"ZADD key NX XX LT CH INCR 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX GT CH not compatible", commands: []string{"ZADD key NX XX GT CH 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX GT CH INCR not compatible", commands: []string{"ZADD key NX XX GT CH INCR 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX INCR not compatible", commands: []string{"ZADD key NX XX INCR 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX INCR LT not compatible", commands: []string{"ZADD key NX XX INCR LT 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX INCR GT not compatible", commands: []string{"ZADD key NX XX INCR GT 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX LT GT not compatible", commands: []string{"ZADD key NX XX LT GT 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX LT GT CH not compatible", commands: []string{"ZADD key NX XX LT GT CH 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX LT GT CH INCR not compatible", commands: []string{"ZADD key NX XX LT GT CH INCR 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, // NX without XX and all LT GT CH and INCR // all are error { name: "ZADD NX and GT incompatible", commands: []string{"ZADD key NX GT 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX and LT incompatible", commands: []string{"ZADD key NX LT 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, LT and GT incompatible", commands: []string{"ZADD key NX LT GT 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, LT, GT and INCR incompatible", commands: []string{"ZADD key NX LT GT INCR 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, LT, GT and CH incompatible", commands: []string{"ZADD key NX LT GT CH 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, LT, GT, CH and INCR incompatible", commands: []string{"ZADD key NX LT GT CH INCR 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, LT, CH not compatible", commands: []string{"ZADD key NX LT CH 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, LT, INCR not compatible", commands: []string{"ZADD key NX LT INCR 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, LT, CH, INCR not compatible", commands: []string{"ZADD key NX LT CH INCR 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, GT, CH not compatible", commands: []string{"ZADD key NX GT CH 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, GT, INCR not compatible", commands: []string{"ZADD key NX GT INCR 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, GT, CH, INCR not compatible", commands: []string{"ZADD key NX GT CH INCR 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { @@ -451,22 +451,22 @@ func TestZADD(t *testing.T) { { name: "ZADD GT and LT", commands: []string{"ZADD key GT LT 15 member15"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD GT LT CH", commands: []string{"ZADD key GT LT CH 15 member15"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD GT LT CH INCR", commands: []string{"ZADD key GT LT CH INCR 15 member15"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD GT LT INCR", commands: []string{"ZADD key GT LT INCR 15 member15"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD GT CH with existing member score less no change hence 0", diff --git a/integration_tests/commands/websocket/zset_test.go b/integration_tests/commands/websocket/zset_test.go index e7720eb0d..30c4bfd56 100644 --- a/integration_tests/commands/websocket/zset_test.go +++ b/integration_tests/commands/websocket/zset_test.go @@ -206,7 +206,7 @@ func TestZADD(t *testing.T) { { name: "ZADD NX and XX not compatible", commands: []string{"ZADD myzset NX XX 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD XX and CH compatible", @@ -221,18 +221,18 @@ func TestZADD(t *testing.T) { { name: "ZADD INCR and XX not compatible because of more than one member", commands: []string{"ZADD myzset XX INCR 20 member1 25 member2"}, - expected: []interface{}{"ERR incr option supports a single increment-element pair"}, + expected: []interface{}{"ERR INCR option supports a single increment-element pair"}, }, { name: "ZADD XX, LT and GT are not compatible", commands: []string{"ZADD key XX LT GT 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD XX, LT, GT, CH, INCR are not compatible", commands: []string{"ZADD key XX LT GT INCR CH 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { @@ -308,139 +308,139 @@ func TestZADD(t *testing.T) { { name: "ZADD NX and XX not compatible", commands: []string{"ZADD key NX XX 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX CH not compatible", commands: []string{"ZADD key NX XX CH 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX CH INCR not compatible", commands: []string{"ZADD key NX XX CH INCR 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX LT not compatible", commands: []string{"ZADD key NX XX LT 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX GT not compatible", commands: []string{"ZADD key NX XX GT 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX LT CH not compatible", commands: []string{"ZADD key NX XX LT CH 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX LT CH INCR compatible", commands: []string{"ZADD key NX XX LT CH INCR 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX GT CH not compatible", commands: []string{"ZADD key NX XX GT CH 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX GT CH INCR not compatible", commands: []string{"ZADD key NX XX GT CH INCR 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX INCR not compatible", commands: []string{"ZADD key NX XX INCR 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX INCR LT not compatible", commands: []string{"ZADD key NX XX INCR LT 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX INCR GT not compatible", commands: []string{"ZADD key NX XX INCR GT 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX LT GT not compatible", commands: []string{"ZADD key NX XX LT GT 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX LT GT CH not compatible", commands: []string{"ZADD key NX XX LT GT CH 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, { name: "ZADD NX XX LT GT CH INCR not compatible", commands: []string{"ZADD key NX XX LT GT CH INCR 20 member1"}, - expected: []interface{}{"ERR xx and nx options at the same time are not compatible"}, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, }, // NX without XX and all LT GT CH and INCR // all are error { name: "ZADD NX and GT incompatible", commands: []string{"ZADD key NX GT 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX and LT incompatible", commands: []string{"ZADD key NX LT 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, LT and GT incompatible", commands: []string{"ZADD key NX LT GT 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, LT, GT and INCR incompatible", commands: []string{"ZADD key NX LT GT INCR 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, LT, GT and CH incompatible", commands: []string{"ZADD key NX LT GT CH 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, LT, GT, CH and INCR incompatible", commands: []string{"ZADD key NX LT GT CH INCR 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, LT, CH not compatible", commands: []string{"ZADD key NX LT CH 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, LT, INCR not compatible", commands: []string{"ZADD key NX LT INCR 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, LT, CH, INCR not compatible", commands: []string{"ZADD key NX LT CH INCR 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, GT, CH not compatible", commands: []string{"ZADD key NX GT CH 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, GT, INCR not compatible", commands: []string{"ZADD key NX GT INCR 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD NX, GT, CH, INCR not compatible", commands: []string{"ZADD key NX GT CH INCR 20 member1"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { @@ -469,22 +469,22 @@ func TestZADD(t *testing.T) { { name: "ZADD GT and LT", commands: []string{"ZADD key GT LT 15 member15"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD GT LT CH", commands: []string{"ZADD key GT LT CH 15 member15"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD GT LT CH INCR", commands: []string{"ZADD key GT LT CH INCR 15 member15"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD GT LT INCR", commands: []string{"ZADD key GT LT INCR 15 member15"}, - expected: []interface{}{"ERR gt and LT and NX options at the same time are not compatible"}, + expected: []interface{}{"ERR GT, LT, and/or NX options at the same time are not compatible"}, }, { name: "ZADD GT CH with existing member score less no change hence 0", diff --git a/internal/eval/store_eval.go b/internal/eval/store_eval.go index bcbda6248..3c4e76c4a 100644 --- a/internal/eval/store_eval.go +++ b/internal/eval/store_eval.go @@ -782,13 +782,13 @@ func validateFlagsAndArgs(args []string, flags map[string]bool) error { return diceerrors.ErrGeneral("syntax error") } if flags[NX] && flags[XX] { - return diceerrors.ErrGeneral("xx and nx options at the same time are not compatible") + return diceerrors.ErrGeneral("XX and NX options at the same time are not compatible") } if (flags[GT] && flags[NX]) || (flags[LT] && flags[NX]) || (flags[GT] && flags[LT]) { - return diceerrors.ErrGeneral("gt and LT and NX options at the same time are not compatible") + return diceerrors.ErrGeneral("GT, LT, and/or NX options at the same time are not compatible") } if flags[INCR] && len(args)/2 > 1 { - return diceerrors.ErrGeneral("incr option supports a single increment-element pair") + return diceerrors.ErrGeneral("INCR option supports a single increment-element pair") } return nil } From 80e16f65e3785a9319651d5b85b417861e4a00c9 Mon Sep 17 00:00:00 2001 From: YuvrajGosain Date: Sun, 17 Nov 2024 15:25:17 +0530 Subject: [PATCH 6/8] #1033Migrating GEOADD and GEODIST command (#1185) Co-authored-by: Yuvraj Gosain Co-authored-by: Jyotinder Singh --- docs/src/content/docs/commands/GEOADD.md | 90 +++++++++ docs/src/content/docs/commands/GEODIST.md | 71 +++++++ integration_tests/commands/http/geo_test.go | 86 +++++++++ integration_tests/commands/resp/geo_test.go | 86 +++++++++ .../commands/websocket/geo_test.go | 87 +++++++++ internal/eval/commands.go | 22 ++- internal/eval/eval.go | 133 ------------- internal/eval/eval_test.go | 129 +++++++++---- internal/eval/store_eval.go | 174 ++++++++++++++++++ internal/server/cmd_meta.go | 10 + internal/worker/cmd_meta.go | 9 + 11 files changed, 717 insertions(+), 180 deletions(-) create mode 100644 docs/src/content/docs/commands/GEOADD.md create mode 100644 docs/src/content/docs/commands/GEODIST.md create mode 100644 integration_tests/commands/http/geo_test.go create mode 100644 integration_tests/commands/resp/geo_test.go create mode 100644 integration_tests/commands/websocket/geo_test.go diff --git a/docs/src/content/docs/commands/GEOADD.md b/docs/src/content/docs/commands/GEOADD.md new file mode 100644 index 000000000..19ba0cafe --- /dev/null +++ b/docs/src/content/docs/commands/GEOADD.md @@ -0,0 +1,90 @@ +--- +title: GEOADD +description: The `GEOADD` command in DiceDB is used to add geospatial items (longitude,latitude) to a specified key, storing them as a sorted set. This would allow for efficient querying of geographical data using commands like GEOSEARCH. +--- + +The `GEOADD` command in DiceDB is used to add geospatial items (longitude,latitude) to a specified key, storing them as a sorted set. This would allow for efficient querying of geographical data using commands like GEOSEARCH. + +## Syntax + +```bash +GEOADD key [NX | XX] [CH] longitude latitude member [longitude latitude member ...] +``` + +## Parameters + +| Parameter | Description | Type | Required | +| --------- | --------------------------------------------------------------------------------- | ------ | -------- | +| key | The name of the sorted set where the geospatial data will be stored. | string | Yes | +| NX | Only add new elements; do not update existing ones. | NONE | No | +| XX | Only update existing elements; do not add new ones. | NONE | No | +| longitude | longitude of the location (must be between -180 and 180 degrees). | float | Yes | +| latitude | latitude of the location (must be between -85.05112878 and 85.05112878 degrees). | float | Yes | +| member | A unique identifier for the location. | string | Yes | + + +## Return Values + +| Condition | Return Value | +| ------------------------------------------------------------ | ----------------------------------------------------------- | +| For each new member added. | 1 | +| No new member is added. | 0 | +| Incorrect Argument Count |`ERR wrong number of arguments for 'geoadd' command` | +| If the longitude is not a valid number or is out of range. |`ERR invalid longitude` | +| If the latitude is not a valid number or is out of range. |`ERR invalid latitude` | + +## Behaviour + +When the GEOADD command is issued, DiceDB performs the following steps: + +1. It checks whether longitude and latitude are valid or not. If not an error is thrown. +2. It checks whether the set exists or not. +3. If set doesn't exist new set is created or else the same set is used. +4. It adds or updates the member in the set. +5. It returns number of members added. + +## Errors + +1.`Wrong number of arguments for 'GEOADD' command` + - Error Message: (error) ERR wrong number of arguments for 'geoadd' command. + - Occurs when the command is executed with an incorrect number of arguments. + +2. `Longitude not a valid number or is out of range ` + - Error Message: (error) ERR invalid longitude. + - Occurs when longitude is out of range(-180 to 180) or not a valid number. + +3. `Latitude not a valid number or is out of range ` + - Error Message: (error) ERR invalid latitude. + - Occurs when latitude is out of range(-85.05112878 to 85.05112878) or not a valid number. + +## Example Usage + +Here are a few examples demonstrating the usage of the GEOADD command: + +### Example : Adding new member to a set + +```bash +127.0.0.1:7379> GEOADD locations 13.361389 38.115556 "Palermo" +1 +``` + +### Example : Updating an already existing member to a set + +```bash +127.0.0.1:7379> GEOADD locations 13.361389 39.115556 "Palermo" +0 +``` + +### Example : Error Adding a member with invalid longitude + +```bash +127.0.0.1:7379> GEOADD locations 181.120332 39.115556 "Jamaica" +(error) ERROR invalid longitude +``` + +### Example : Error Adding a member with invalid latitde + +```bash +127.0.0.1:7379> GEOADD locations 13.361389 91.115556 "Venice" +(error) ERROR invalid latitude +``` diff --git a/docs/src/content/docs/commands/GEODIST.md b/docs/src/content/docs/commands/GEODIST.md new file mode 100644 index 000000000..c40c529d2 --- /dev/null +++ b/docs/src/content/docs/commands/GEODIST.md @@ -0,0 +1,71 @@ +--- +title: GEODIST +description: The `GEODIST` command in Redis is used to calculate the distance between two members (geospatial points) stored in a geospatial index(set). +--- + +The `GEODIST` command in DiceDB is used to calculate the distance between two members (geospatial points) stored in a geospatial index(set). + +## Syntax + +```bash +GEODIST key member1 member2 [m | km | ft | mi] +``` + +## Parameters + +| Parameter | Description | Type | Required | +| --------- | --------------------------------------------------------------------------------- | ------ | -------- | +| key | The name of the sorted set where the geospatial data is stored. | string | Yes | +| member1 | The name of the member1 from where you want to measure the distance. | string | Yes | +| member2 | The name of the member2 to where you want to measure the distance. | string | Yes | +| m | The distance to be measured in meters. | NONE | NO | +| km | The distance to be measured in kilometers. | NONE | NO | +| ft | The distance to be measured in feet. | NONE | NO | +| mi | The distance to be measured in miles. | NONE | NO | + + +## Return Values + +| Condition | Return Value | +| ------------------------------------------------------------ | ----------------------------------------------------------- | +| If both members exist in the set with no option | distance b/w them in meters | +| If both members exist in the set with option km | distance b/w them in kilometers | +| If both members exist in the set with option ft | distance b/w them in feet | +| If both members exist in the set with option mi | distance b/w them in miles | +| If any member doesn't exist in Set | nil | +| Incorrect Argument Count |`ERR wrong number of arguments for 'geodist' command` | + +## Behaviour + +When the GEODIST command is issued, DiceDB performs the following steps: + +1. It gets the sorted set(key). +2. It gets the scores(geohashes) from the sorted sets for both the members. +3. It calculates the distance bw them and returns it. + +## Errors + +1.`Wrong number of arguments for 'GEODIST' command` + - Error Message: (error) ERR wrong number of arguments for 'geodist' command. + - Occurs when the command is executed with an incorrect number of arguments. + + +## Example Usage + +Here are a few examples demonstrating the usage of the GEODIST command: + +### Example : Adding new member to a set + +```bash +127.0.0.1:7379> GEOADD cities -74.0060 40.7128 "New York" +1 +127.0.0.1:7379> GEOADD cities -79.3470 43.6510 "Toronto" +1 +127.0.0.1:7379> GEODIST cities "New York" "Toronto" +"548064.1868" +127.0.0.1:7379> GEODIST cities "New York" "Toronto km" +"548.0642" +127.0.0.1:7379> GEODIST cities "New York" "Toronto mi" +"340.5521" +``` + diff --git a/integration_tests/commands/http/geo_test.go b/integration_tests/commands/http/geo_test.go new file mode 100644 index 000000000..a8e5c6e93 --- /dev/null +++ b/integration_tests/commands/http/geo_test.go @@ -0,0 +1,86 @@ +package http + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGeoAdd(t *testing.T) { + exec := NewHTTPCommandExecutor() + + testCases := []struct { + name string + commands []HTTPCommand + expected []interface{} + }{ + { + name: "GEOADD with wrong number of arguments", + commands: []HTTPCommand{ + {Command: "GEOADD", Body: map[string]interface{}{"key": "mygeo", "values": []interface{}{"1.2", "2.4"}}}, + }, + expected: []interface{}{"ERR wrong number of arguments for 'geoadd' command"}, + }, + { + name: "GEOADD Commands with new member and updating it", + commands: []HTTPCommand{ + {Command: "GEOADD", Body: map[string]interface{}{"key": "mygeo", "values": []interface{}{"1.2", "2.4", "NJ"}}}, + {Command: "GEOADD", Body: map[string]interface{}{"key": "mygeo", "values": []interface{}{"1.24", "2.48", "NJ"}}}, + }, + expected: []interface{}{float64(1), float64(0)}, + }, + { + name: "GEOADD Adding both XX and NX options together", + commands: []HTTPCommand{ + {Command: "GEOADD", Body: map[string]interface{}{"key": "mygeo", "values": []interface{}{"XX", "NX", "1.2", "2.4", "NJ"}}}, + }, + expected: []interface{}{"ERR XX and NX options at the same time are not compatible"}, + }, + { + name: "GEOADD Invalid Longitude", + commands: []HTTPCommand{ + {Command: "GEOADD", Body: map[string]interface{}{"key": "mygeo", "values": []interface{}{"181", "2.4", "MT"}}}, + }, + expected: []interface{}{"ERR invalid longitude"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + for i, cmd := range tc.commands { + result, _ := exec.FireCommand(cmd) + assert.Equal(t, tc.expected[i], result, "Value mismatch for cmd %s", cmd) + } + }) + } +} + +func TestGeoDist(t *testing.T) { + exec := NewHTTPCommandExecutor() + + testCases := []struct { + name string + commands []HTTPCommand + expected []interface{} + }{ + { + name: "GEODIST b/w existing points", + commands: []HTTPCommand{ + {Command: "GEOADD", Body: map[string]interface{}{"key": "points", "values": []interface{}{"13.361389", "38.115556", "Palermo"}}}, + {Command: "GEOADD", Body: map[string]interface{}{"key": "points", "values": []interface{}{"15.087269", "37.502669", "Catania"}}}, + {Command: "GEODIST", Body: map[string]interface{}{"key": "points", "values": []interface{}{"Palermo", "Catania"}}}, + {Command: "GEODIST", Body: map[string]interface{}{"key": "points", "values": []interface{}{"Palermo", "Catania", "km"}}}, + }, + expected: []interface{}{float64(1), float64(1), float64(166274.144), float64(166.2741)}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + for i, cmd := range tc.commands { + result, _ := exec.FireCommand(cmd) + assert.Equal(t, tc.expected[i], result, "Value mismatch for cmd %s", cmd) + } + }) + } +} diff --git a/integration_tests/commands/resp/geo_test.go b/integration_tests/commands/resp/geo_test.go new file mode 100644 index 000000000..57bd2656a --- /dev/null +++ b/integration_tests/commands/resp/geo_test.go @@ -0,0 +1,86 @@ +package resp + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestGeoAdd(t *testing.T) { + conn := getLocalConnection() + defer conn.Close() + + testCases := []struct { + name string + cmds []string + expect []interface{} + }{ + { + name: "GeoAdd With Wrong Number of Arguments", + cmds: []string{"GEOADD mygeo 1 2"}, + expect: []interface{}{"ERR wrong number of arguments for 'geoadd' command"}, + }, + { + name: "GeoAdd With Adding New Member And Updating it", + cmds: []string{"GEOADD mygeo 1.21 1.44 NJ", "GEOADD mygeo 1.22 1.54 NJ"}, + expect: []interface{}{int64(1), int64(0)}, + }, + { + name: "GeoAdd With Adding New Member And Updating it with NX", + cmds: []string{"GEOADD mygeo 1.21 1.44 MD", "GEOADD mygeo 1.22 1.54 MD"}, + expect: []interface{}{int64(1), int64(0)}, + }, + { + name: "GEOADD with both NX and XX options", + cmds: []string{"GEOADD mygeo NX XX 1.21 1.44 MD"}, + expect: []interface{}{"ERR XX and NX options at the same time are not compatible"}, + }, + { + name: "GEOADD invalid longitude", + cmds: []string{"GEOADD mygeo 181.0 1.44 MD"}, + expect: []interface{}{"ERR invalid longitude"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + for i, cmd := range tc.cmds { + result := FireCommand(conn, cmd) + assert.Equal(t, tc.expect[i], result, "Value mismatch for cmd %s", cmd) + } + }) + } +} + +func TestGeoDist(t *testing.T) { + conn := getLocalConnection() + defer conn.Close() + + testCases := []struct { + name string + cmds []string + expect []interface{} + delays []time.Duration + }{ + { + name: "GEODIST b/w existing points", + cmds: []string{ + "GEOADD points 13.361389 38.115556 Palermo", + "GEOADD points 15.087269 37.502669 Catania", + "GEODIST points Palermo Catania", + "GEODIST points Palermo Catania km", + }, + expect: []interface{}{int64(1), int64(1), "166274.144", "166.2741"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + for i, cmd := range tc.cmds { + result := FireCommand(conn, cmd) + assert.Equal(t, tc.expect[i], result, "Value mismatch for cmd %s", cmd) + } + }) + } +} diff --git a/integration_tests/commands/websocket/geo_test.go b/integration_tests/commands/websocket/geo_test.go new file mode 100644 index 000000000..e2c5d240b --- /dev/null +++ b/integration_tests/commands/websocket/geo_test.go @@ -0,0 +1,87 @@ +package websocket + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGeoAdd(t *testing.T) { + exec := NewWebsocketCommandExecutor() + conn := exec.ConnectToServer() + + testCases := []struct { + name string + cmds []string + expect []interface{} + }{ + { + name: "GeoAdd With Wrong Number of Arguments", + cmds: []string{"GEOADD mygeo 1 2"}, + expect: []interface{}{"ERR wrong number of arguments for 'geoadd' command"}, + }, + { + name: "GeoAdd With Adding New Member And Updating it", + cmds: []string{"GEOADD mygeo 1.21 1.44 NJ", "GEOADD mygeo 1.22 1.54 NJ"}, + expect: []interface{}{float64(1), float64(0)}, + }, + { + name: "GeoAdd With Adding New Member And Updating it with NX", + cmds: []string{"GEOADD mygeo NX 1.21 1.44 MD", "GEOADD mygeo 1.22 1.54 MD"}, + expect: []interface{}{float64(1), float64(0)}, + }, + { + name: "GEOADD with both NX and XX options", + cmds: []string{"GEOADD mygeo NX XX 1.21 1.44 DEL"}, + expect: []interface{}{"ERR XX and NX options at the same time are not compatible"}, + }, + { + name: "GEOADD invalid longitude", + cmds: []string{"GEOADD mygeo 181.0 1.44 MD"}, + expect: []interface{}{"ERR invalid longitude"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + for i, cmd := range tc.cmds { + result, err := exec.FireCommandAndReadResponse(conn, cmd) + assert.Nil(t, err) + assert.Equal(t, tc.expect[i], result, "Value mismatch for cmd %s", cmd) + } + }) + } +} + +func TestGeoDist(t *testing.T) { + exec := NewWebsocketCommandExecutor() + conn := exec.ConnectToServer() + defer conn.Close() + + testCases := []struct { + name string + cmds []string + expect []interface{} + }{ + { + name: "GEODIST b/w existing points", + cmds: []string{ + "GEOADD points 13.361389 38.115556 Palermo", + "GEOADD points 15.087269 37.502669 Catania", + "GEODIST points Palermo Catania", + "GEODIST points Palermo Catania km", + }, + expect: []interface{}{float64(1), float64(1), float64(166274.144), float64(166.2741)}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + for i, cmd := range tc.cmds { + result, err := exec.FireCommandAndReadResponse(conn, cmd) + assert.Nil(t, err) + assert.Equal(t, tc.expect[i], result, "Value mismatch for cmd %s", cmd) + } + }) + } +} diff --git a/internal/eval/commands.go b/internal/eval/commands.go index 5c8b1ea51..ae901640f 100644 --- a/internal/eval/commands.go +++ b/internal/eval/commands.go @@ -1275,18 +1275,20 @@ var ( NewEval: evalHINCRBYFLOAT, } geoAddCmdMeta = DiceCmdMeta{ - Name: "GEOADD", - Info: `Adds one or more members to a geospatial index. The key is created if it doesn't exist.`, - Arity: -5, - Eval: evalGEOADD, - KeySpecs: KeySpecs{BeginIndex: 1}, + Name: "GEOADD", + Info: `Adds one or more members to a geospatial index. The key is created if it doesn't exist.`, + Arity: -5, + IsMigrated: true, + NewEval: evalGEOADD, + KeySpecs: KeySpecs{BeginIndex: 1}, } geoDistCmdMeta = DiceCmdMeta{ - Name: "GEODIST", - Info: `Returns the distance between two members in the geospatial index.`, - Arity: -4, - Eval: evalGEODIST, - KeySpecs: KeySpecs{BeginIndex: 1}, + Name: "GEODIST", + Info: `Returns the distance between two members in the geospatial index.`, + Arity: -4, + IsMigrated: true, + NewEval: evalGEODIST, + KeySpecs: KeySpecs{BeginIndex: 1}, } jsonstrappendCmdMeta = DiceCmdMeta{ Name: "JSON.STRAPPEND", diff --git a/internal/eval/eval.go b/internal/eval/eval.go index 11705958b..dea3856a7 100644 --- a/internal/eval/eval.go +++ b/internal/eval/eval.go @@ -5,14 +5,11 @@ import ( "errors" "fmt" "log/slog" - "math" "sort" "strconv" "strings" "time" - "github.com/dicedb/dice/internal/eval/geo" - "github.com/dicedb/dice/internal/eval/sortedset" "github.com/dicedb/dice/internal/object" "github.com/dicedb/dice/internal/sql" @@ -1363,133 +1360,3 @@ func executeBitfieldOps(value *ByteArray, ops []utils.BitFieldOp) []interface{} } return result } -func evalGEOADD(args []string, store *dstore.Store) []byte { - if len(args) < 4 { - return diceerrors.NewErrArity("GEOADD") - } - - key := args[0] - var nx, xx bool - startIdx := 1 - - // Parse options - for startIdx < len(args) { - option := strings.ToUpper(args[startIdx]) - if option == NX { - nx = true - startIdx++ - } else if option == XX { - xx = true - startIdx++ - } else { - break - } - } - - // Check if we have the correct number of arguments after parsing options - if (len(args)-startIdx)%3 != 0 { - return diceerrors.NewErrArity("GEOADD") - } - - if xx && nx { - return diceerrors.NewErrWithMessage("ERR XX and NX options at the same time are not compatible") - } - - // Get or create sorted set - obj := store.Get(key) - var ss *sortedset.Set - if obj != nil { - var err []byte - ss, err = sortedset.FromObject(obj) - if err != nil { - return err - } - } else { - ss = sortedset.New() - } - - added := 0 - for i := startIdx; i < len(args); i += 3 { - longitude, err := strconv.ParseFloat(args[i], 64) - if err != nil || math.IsNaN(longitude) || longitude < -180 || longitude > 180 { - return diceerrors.NewErrWithMessage("ERR invalid longitude") - } - - latitude, err := strconv.ParseFloat(args[i+1], 64) - if err != nil || math.IsNaN(latitude) || latitude < -85.05112878 || latitude > 85.05112878 { - return diceerrors.NewErrWithMessage("ERR invalid latitude") - } - - member := args[i+2] - _, exists := ss.Get(member) - - // Handle XX option: Only update existing elements - if xx && !exists { - continue - } - - // Handle NX option: Only add new elements - if nx && exists { - continue - } - - hash := geo.EncodeHash(latitude, longitude) - - wasInserted := ss.Upsert(hash, member) - if wasInserted { - added++ - } - } - - obj = store.NewObj(ss, -1, object.ObjTypeSortedSet, object.ObjEncodingBTree) - store.Put(key, obj) - - return clientio.Encode(added, false) -} - -func evalGEODIST(args []string, store *dstore.Store) []byte { - if len(args) < 3 || len(args) > 4 { - return diceerrors.NewErrArity("GEODIST") - } - - key := args[0] - member1 := args[1] - member2 := args[2] - unit := "m" - if len(args) == 4 { - unit = strings.ToLower(args[3]) - } - - // Get the sorted set - obj := store.Get(key) - if obj == nil { - return clientio.RespNIL - } - - ss, err := sortedset.FromObject(obj) - if err != nil { - return err - } - - // Get the scores (geohashes) for both members - score1, ok := ss.Get(member1) - if !ok { - return clientio.RespNIL - } - score2, ok := ss.Get(member2) - if !ok { - return clientio.RespNIL - } - - lat1, lon1 := geo.DecodeHash(score1) - lat2, lon2 := geo.DecodeHash(score2) - - distance := geo.GetDistance(lon1, lat1, lon2, lat2) - - result, err := geo.ConvertDistance(distance, unit) - if err != nil { - return err - } - - return clientio.Encode(utils.RoundToDecimals(result, 4), false) -} diff --git a/internal/eval/eval_test.go b/internal/eval/eval_test.go index c8dc7c7b4..bf5dfa76c 100644 --- a/internal/eval/eval_test.go +++ b/internal/eval/eval_test.go @@ -8377,84 +8377,130 @@ func testEvalBitFieldRO(t *testing.T, store *dstore.Store) { func testEvalGEOADD(t *testing.T, store *dstore.Store) { tests := map[string]evalTestCase{ "GEOADD with wrong number of arguments": { - input: []string{"mygeo", "1", "2"}, - output: diceerrors.NewErrArity("GEOADD"), + input: []string{"mygeo", "1", "2"}, + migratedOutput: EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongArgumentCount("GEOADD"), + }, }, "GEOADD with non-numeric longitude": { - input: []string{"mygeo", "long", "40.7128", "NewYork"}, - output: diceerrors.NewErrWithMessage("ERR invalid longitude"), + input: []string{"mygeo", "long", "40.7128", "NewYork"}, + migratedOutput: EvalResponse{ + Result: nil, + Error: diceerrors.ErrGeneral("invalid longitude"), + }, }, "GEOADD with non-numeric latitude": { - input: []string{"mygeo", "-74.0060", "lat", "NewYork"}, - output: diceerrors.NewErrWithMessage("ERR invalid latitude"), + input: []string{"mygeo", "-74.0060", "lat", "NewYork"}, + migratedOutput: EvalResponse{ + Result: nil, + Error: diceerrors.ErrGeneral("invalid latitude"), + }, }, "GEOADD new member to non-existing key": { - setup: func() {}, - input: []string{"mygeo", "-74.0060", "40.7128", "NewYork"}, - output: clientio.Encode(int64(1), false), + setup: func() {}, + input: []string{"mygeo", "-74.0060", "40.7128", "NewYork"}, + migratedOutput: EvalResponse{ + Result: 1, + Error: nil, + }, }, "GEOADD existing member with updated coordinates": { setup: func() { evalGEOADD([]string{"mygeo", "-74.0060", "40.7128", "NewYork"}, store) }, - input: []string{"mygeo", "-73.9352", "40.7304", "NewYork"}, - output: clientio.Encode(int64(0), false), + input: []string{"mygeo", "-73.9352", "40.7304", "NewYork"}, + migratedOutput: EvalResponse{ + Result: 0, + Error: nil, + }, }, "GEOADD multiple members": { setup: func() { evalGEOADD([]string{"mygeo", "-74.0060", "40.7128", "NewYork"}, store) }, - input: []string{"mygeo", "-118.2437", "34.0522", "LosAngeles", "-87.6298", "41.8781", "Chicago"}, - output: clientio.Encode(int64(2), false), + input: []string{"mygeo", "-118.2437", "34.0522", "LosAngeles", "-87.6298", "41.8781", "Chicago"}, + migratedOutput: EvalResponse{ + Result: 2, + Error: nil, + }, }, "GEOADD with NX option (new member)": { - input: []string{"mygeo", "NX", "-122.4194", "37.7749", "SanFrancisco"}, - output: clientio.Encode(int64(1), false), + input: []string{"mygeo", "NX", "-122.4194", "37.7749", "SanFrancisco"}, + migratedOutput: EvalResponse{ + Result: 1, + Error: nil, + }, }, "GEOADD with NX option (existing member)": { setup: func() { evalGEOADD([]string{"mygeo", "-74.0060", "40.7128", "NewYork"}, store) }, - input: []string{"mygeo", "NX", "-73.9352", "40.7304", "NewYork"}, - output: clientio.Encode(int64(0), false), + input: []string{"mygeo", "NX", "-73.9352", "40.7304", "NewYork"}, + migratedOutput: EvalResponse{ + Result: 0, + Error: nil, + }, }, "GEOADD with XX option (new member)": { - input: []string{"mygeo", "XX", "-71.0589", "42.3601", "Boston"}, - output: clientio.Encode(int64(0), false), + input: []string{"mygeo", "XX", "-71.0589", "42.3601", "Boston"}, + migratedOutput: EvalResponse{ + Result: 0, + Error: nil, + }, }, "GEOADD with XX option (existing member)": { setup: func() { evalGEOADD([]string{"mygeo", "-74.0060", "40.7128", "NewYork"}, store) }, - input: []string{"mygeo", "XX", "-73.9352", "40.7304", "NewYork"}, - output: clientio.Encode(int64(0), false), + input: []string{"mygeo", "XX", "-73.9352", "40.7304", "NewYork"}, + migratedOutput: EvalResponse{ + Result: 0, + Error: nil, + }, }, "GEOADD with both NX and XX options": { input: []string{"mygeo", "NX", "XX", "-74.0060", "40.7128", "NewYork"}, output: diceerrors.NewErrWithMessage("ERR XX and NX options at the same time are not compatible"), + migratedOutput: EvalResponse{ + Result: nil, + Error: diceerrors.ErrGeneral("XX and NX options at the same time are not compatible"), + }, }, "GEOADD with invalid option": { - input: []string{"mygeo", "INVALID", "-74.0060", "40.7128", "NewYork"}, - output: diceerrors.NewErrArity("GEOADD"), + input: []string{"mygeo", "INVALID", "-74.0060", "40.7128", "NewYork"}, + migratedOutput: EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongArgumentCount("GEOADD"), + }, }, "GEOADD to a key of wrong type": { setup: func() { store.Put("mygeo", store.NewObj("string_value", -1, object.ObjTypeString, object.ObjEncodingRaw)) }, - input: []string{"mygeo", "-74.0060", "40.7128", "NewYork"}, - output: []byte("-ERR Existing key has wrong Dice type\r\n"), + input: []string{"mygeo", "-74.0060", "40.7128", "NewYork"}, + migratedOutput: EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongTypeOperation, + }, }, "GEOADD with longitude out of range": { - input: []string{"mygeo", "181.0", "40.7128", "Invalid"}, - output: diceerrors.NewErrWithMessage("ERR invalid longitude"), + input: []string{"mygeo", "181.0", "40.7128", "Invalid"}, + migratedOutput: EvalResponse{ + Result: nil, + Error: diceerrors.ErrGeneral("invalid longitude"), + }, }, "GEOADD with latitude out of range": { - input: []string{"mygeo", "-74.0060", "91.0", "Invalid"}, - output: diceerrors.NewErrWithMessage("ERR invalid latitude"), + input: []string{"mygeo", "-74.0060", "91.0", "Invalid"}, + migratedOutput: EvalResponse{ + Result: nil, + Error: diceerrors.ErrGeneral("invalid latitude"), + }, }, } - runEvalTests(t, tests, evalGEOADD, store) + runMigratedEvalTests(t, tests, evalGEOADD, store) } func testEvalGEODIST(t *testing.T, store *dstore.Store) { @@ -8464,28 +8510,37 @@ func testEvalGEODIST(t *testing.T, store *dstore.Store) { evalGEOADD([]string{"points", "13.361389", "38.115556", "Palermo"}, store) evalGEOADD([]string{"points", "15.087269", "37.502669", "Catania"}, store) }, - input: []string{"points", "Palermo", "Catania"}, - output: clientio.Encode(float64(166274.1440), false), // Example value + input: []string{"points", "Palermo", "Catania"}, + migratedOutput: EvalResponse{ + Result: float64(166274.1440), + Error: nil, + }, }, "GEODIST with units (km)": { setup: func() { evalGEOADD([]string{"points", "13.361389", "38.115556", "Palermo"}, store) evalGEOADD([]string{"points", "15.087269", "37.502669", "Catania"}, store) }, - input: []string{"points", "Palermo", "Catania", "km"}, - output: clientio.Encode(float64(166.2741), false), // Example value + input: []string{"points", "Palermo", "Catania", "km"}, + migratedOutput: EvalResponse{ + Result: float64(166.2741), + Error: nil, + }, }, "GEODIST to same point": { setup: func() { evalGEOADD([]string{"points", "13.361389", "38.115556", "Palermo"}, store) }, - input: []string{"points", "Palermo", "Palermo"}, - output: clientio.Encode(float64(0.0000), false), // Expecting distance 0 formatted to 4 decimals + input: []string{"points", "Palermo", "Palermo"}, + migratedOutput: EvalResponse{ + Result: float64(0.0000), + Error: nil, + }, }, // Add other test cases here... } - runEvalTests(t, tests, evalGEODIST, store) + runMigratedEvalTests(t, tests, evalGEODIST, store) } func testEvalSINTER(t *testing.T, store *dstore.Store) { diff --git a/internal/eval/store_eval.go b/internal/eval/store_eval.go index 3c4e76c4a..c01dcec12 100644 --- a/internal/eval/store_eval.go +++ b/internal/eval/store_eval.go @@ -18,6 +18,7 @@ import ( "github.com/dicedb/dice/internal/clientio" "github.com/dicedb/dice/internal/cmd" diceerrors "github.com/dicedb/dice/internal/errors" + "github.com/dicedb/dice/internal/eval/geo" "github.com/dicedb/dice/internal/eval/sortedset" "github.com/dicedb/dice/internal/object" "github.com/dicedb/dice/internal/server/utils" @@ -6171,3 +6172,176 @@ func evalJSONNUMINCRBY(args []string, store *dstore.Store) *EvalResponse { return makeEvalResult(resultString) } + +func evalGEOADD(args []string, store *dstore.Store) *EvalResponse { + if len(args) < 4 { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongArgumentCount("GEOADD"), + } + } + + key := args[0] + var nx, xx bool + startIdx := 1 + + // Parse options + for startIdx < len(args) { + option := strings.ToUpper(args[startIdx]) + if option == "NX" { + nx = true + startIdx++ + } else if option == "XX" { + xx = true + startIdx++ + } else { + break + } + } + + // Check if we have the correct number of arguments after parsing options + if (len(args)-startIdx)%3 != 0 { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongArgumentCount("GEOADD"), + } + } + + if xx && nx { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrGeneral("XX and NX options at the same time are not compatible"), + } + } + + // Get or create sorted set + obj := store.Get(key) + var ss *sortedset.Set + if obj != nil { + var err []byte + ss, err = sortedset.FromObject(obj) + if err != nil { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongTypeOperation, + } + } + } else { + ss = sortedset.New() + } + + added := 0 + for i := startIdx; i < len(args); i += 3 { + longitude, err := strconv.ParseFloat(args[i], 64) + if err != nil || math.IsNaN(longitude) || longitude < -180 || longitude > 180 { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrGeneral("invalid longitude"), + } + } + + latitude, err := strconv.ParseFloat(args[i+1], 64) + if err != nil || math.IsNaN(latitude) || latitude < -85.05112878 || latitude > 85.05112878 { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrGeneral("invalid latitude"), + } + } + + member := args[i+2] + _, exists := ss.Get(member) + + // Handle XX option: Only update existing elements + if xx && !exists { + continue + } + + // Handle NX option: Only add new elements + if nx && exists { + continue + } + + hash := geo.EncodeHash(latitude, longitude) + + wasInserted := ss.Upsert(hash, member) + if wasInserted { + added++ + } + } + + obj = store.NewObj(ss, -1, object.ObjTypeSortedSet, object.ObjEncodingBTree) + store.Put(key, obj) + + return &EvalResponse{ + Result: added, + Error: nil, + } +} + +func evalGEODIST(args []string, store *dstore.Store) *EvalResponse { + if len(args) < 3 || len(args) > 4 { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongArgumentCount("GEODIST"), + } + } + + key := args[0] + member1 := args[1] + member2 := args[2] + unit := "m" + if len(args) == 4 { + unit = strings.ToLower(args[3]) + } + + // Get the sorted set + obj := store.Get(key) + if obj == nil { + return &EvalResponse{ + Result: clientio.NIL, + Error: nil, + } + } + ss, err := sortedset.FromObject(obj) + if err != nil { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongTypeOperation, + } + } + + // Get the scores (geohashes) for both members + score1, ok := ss.Get(member1) + if !ok { + return &EvalResponse{ + Result: nil, + Error: nil, + } + } + score2, ok := ss.Get(member2) + if !ok { + return &EvalResponse{ + Result: nil, + Error: nil, + } + } + + lat1, lon1 := geo.DecodeHash(score1) + lat2, lon2 := geo.DecodeHash(score2) + + distance := geo.GetDistance(lon1, lat1, lon2, lat2) + + result, err := geo.ConvertDistance(distance, unit) + + if err != nil { + return &EvalResponse{ + Result: nil, + Error: diceerrors.ErrWrongTypeOperation, + } + } + + return &EvalResponse{ + Result: utils.RoundToDecimals(result, 4), + Error: nil, + } +} diff --git a/internal/server/cmd_meta.go b/internal/server/cmd_meta.go index 5f0677d7e..1b26c8c4f 100644 --- a/internal/server/cmd_meta.go +++ b/internal/server/cmd_meta.go @@ -435,6 +435,14 @@ var ( Cmd: "RESTORE", CmdType: SingleShard, } + geoaddCmdMeta = CmdsMeta{ + Cmd: "GEOADD", + CmdType: SingleShard, + } + geodistCmdMeta = CmdsMeta{ + Cmd: "GEODIST", + CmdType: SingleShard, + } // Metadata for multishard commands would go here. // These commands require both breakup and gather logic. @@ -559,5 +567,7 @@ func init() { WorkerCmdsMeta["HGETALL"] = hGetAllCmdMeta WorkerCmdsMeta["DUMP"] = dumpCmdMeta WorkerCmdsMeta["RESTORE"] = restoreCmdMeta + WorkerCmdsMeta["GEOADD"] = geoaddCmdMeta + WorkerCmdsMeta["GEODIST"] = geodistCmdMeta // Additional commands (multishard, custom) can be added here as needed. } diff --git a/internal/worker/cmd_meta.go b/internal/worker/cmd_meta.go index 88cb7a0a8..aeaea41c7 100644 --- a/internal/worker/cmd_meta.go +++ b/internal/worker/cmd_meta.go @@ -156,6 +156,8 @@ const ( CmdSmembers = "SMEMBERS" CmdDump = "DUMP" CmdRestore = "RESTORE" + CmdGeoAdd = "GEOADD" + CmdGeoDist = "GEODIST" ) // Watch commands @@ -464,6 +466,13 @@ var CommandsMeta = map[string]CmdMeta{ CmdRestore: { CmdType: SingleShard, }, + // geoCommands + CmdGeoAdd: { + CmdType: SingleShard, + }, + CmdGeoDist: { + CmdType: SingleShard, + }, // Multi-shard commands. CmdRename: { From 76b0ea639836375d8726a1cea0c137a5dca2fafe Mon Sep 17 00:00:00 2001 From: Jyotinder Singh Date: Sun, 17 Nov 2024 16:16:57 +0530 Subject: [PATCH 7/8] Fixes error assertion --- internal/worker/worker.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/worker/worker.go b/internal/worker/worker.go index 49b265f00..f02d1e639 100644 --- a/internal/worker/worker.go +++ b/internal/worker/worker.go @@ -219,10 +219,9 @@ func (w *BaseWorker) executeCommand(ctx context.Context, diceDBCmd *cmd.DiceDBCm if err != nil { var workerErr error // Check if it's a CustomError - if customErr, ok := err.(*diceerrors.PreProcessError); ok { + var customErr *diceerrors.PreProcessError + if errors.As(err, &customErr) { workerErr = w.ioHandler.Write(ctx, customErr.Result) - } else { - workerErr = w.ioHandler.Write(ctx, err) } if workerErr != nil { slog.Debug("Error executing for worker", slog.String("workerID", w.id), slog.Any("error", workerErr)) From f2267bd8a15532db13b9e952adbc757a36112b12 Mon Sep 17 00:00:00 2001 From: Jyotinder Singh Date: Sun, 17 Nov 2024 16:18:51 +0530 Subject: [PATCH 8/8] fixes error assertion if case --- internal/worker/worker.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/worker/worker.go b/internal/worker/worker.go index f02d1e639..b9a4dd20d 100644 --- a/internal/worker/worker.go +++ b/internal/worker/worker.go @@ -222,6 +222,8 @@ func (w *BaseWorker) executeCommand(ctx context.Context, diceDBCmd *cmd.DiceDBCm var customErr *diceerrors.PreProcessError if errors.As(err, &customErr) { workerErr = w.ioHandler.Write(ctx, customErr.Result) + } else { + workerErr = w.ioHandler.Write(ctx, err) } if workerErr != nil { slog.Debug("Error executing for worker", slog.String("workerID", w.id), slog.Any("error", workerErr))