From f3e3366c5de10123ba03a1764c7bbf2e1f5716e2 Mon Sep 17 00:00:00 2001 From: kingecg Date: Wed, 11 Jun 2025 23:40:55 +0800 Subject: [PATCH] =?UTF-8?q?feat(storage):=20=E6=B7=BB=E5=8A=A0=20BoltDB=20?= =?UTF-8?q?=E6=8C=81=E4=B9=85=E5=8C=96=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 在持久化选项中增加了 BoltDB 支持 - 新增 BoltDB 特有的配置项和存储引擎实现 - 更新了配置文件格式和命令行参数以支持 BoltDB - 重构了存储引擎的创建逻辑，支持多种持久化方式 --- README.md | 18 ++++++++++- cmd/server/config.go | 8 +++++ cmd/server/main.go | 45 ++++++++++++++++---------- pkg/storage/engine.go | 2 ++ pkg/storage/factory.go | 72 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 128 insertions(+), 17 deletions(-) create mode 100644 pkg/storage/factory.go diff --git a/README.md b/README.md index a29faf8..f63414f 100644 --- a/README.md +++ b/README.md @@ -41,10 +41,26 @@ go build -o gotidb cmd/server/main.go - `-rest-addr`: REST API 服务地址（默认：":8080"） - `-ws-addr`: WebSocket 服务地址（默认：":8081"） - `-metrics-addr`: 指标服务地址（默认：":8082"） +- `-quic-addr`: QUIC 服务地址（默认：":8083"） - `-nats-url`: NATS 服务器地址（默认："nats://localhost:4222"） -- `-persistence`: 持久化类型（none, wal）（默认："none"） +- `-persistence`: 持久化类型（none, wal, boltdb）（默认："none"） - `-persistence-dir`: 持久化目录（默认："./data"） - `-sync-every`: 每写入多少条数据同步一次（默认：100） +- `-config`: 配置文件路径（默认："config.yaml"） + +### 持久化选项 + +GoTiDB 支持多种持久化方式： + +1. **内存存储（无持久化）**：数据仅保存在内存中，服务重启后数据丢失。 + - 配置：`-persistence=none` + +2. **WAL 日志持久化**：使用预写日志（Write-Ahead Log）进行持久化，支持数据恢复。 + - 配置：`-persistence=wal -persistence-dir=./data -sync-every=100` + +3. **BoltDB 持久化**：使用 BoltDB 进行持久化，提供更高的可靠性和查询性能。 + - 配置：`-persistence=boltdb -persistence-dir=./data` + - 配置文件中可设置：`boltdb_filename`（数据库文件名）和 `boltdb_bucket_size`（数据分桶大小） ## API 使用 diff --git a/cmd/server/config.go b/cmd/server/config.go index e6b535e..5ba1971 100644 --- a/cmd/server/config.go +++ b/cmd/server/config.go @@ -19,6 +19,10 @@ type Config struct { PersistenceDir string `yaml:"persistence_dir"` SyncEvery int `yaml:"sync_every"` QuicConfig *quic.Config `yaml:"quic_config"` + + // BoltDB特有配置 + BoltDBFilename string `yaml:"boltdb_filename"` // BoltDB数据库文件名 + BoltDBBucketSize int `yaml:"boltdb_bucket_size"` // BoltDB数据分桶大小 } func LoadConfig(path string) (*Config, error) { @@ -48,6 +52,10 @@ func GenerateSampleConfig(path string) error { WsAddr: ":8081", QuicAddr: ":8083", QuicConfig: DefaultQuicConfig(), + + // BoltDB默认配置 + BoltDBFilename: "gotidb.db", + BoltDBBucketSize: 30, } // 序列化yaml到path file, err := os.Create(path) diff --git a/cmd/server/main.go b/cmd/server/main.go index b50e9e5..0ac09f6 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -88,24 +88,37 @@ func main() { } } - // 创建存储引擎 - engine := storage.NewMemoryEngine() - - // 如果启用了持久化，配置持久化 - if *persistenceType != "none" { - persistenceConfig := storage.PersistenceConfig{ - Type: storage.PersistenceType(*persistenceType), - Directory: *persistenceDir, - SyncEvery: *syncEvery, - } - - if err := engine.EnablePersistence(persistenceConfig); err != nil { - log.Fatalf("Failed to enable persistence: %v", err) - } - - log.Printf("Persistence enabled: type=%s, directory=%s", *persistenceType, *persistenceDir) + // 创建存储引擎配置 + engineConfig := storage.EngineConfig{ + PersistenceType: storage.PersistenceType(*persistenceType), + PersistenceDir: *persistenceDir, + SyncEvery: *syncEvery, } + // 如果是BoltDB，添加BoltDB特有配置 + if *persistenceType == "boltdb" { + if config != nil { + engineConfig.BoltDBFilename = config.BoltDBFilename + engineConfig.BoltDBOptions = &storage.BoltDBConfig{ + BucketSize: config.BoltDBBucketSize, + } + } else { + // 使用默认值 + engineConfig.BoltDBFilename = "gotidb.db" + engineConfig.BoltDBOptions = &storage.BoltDBConfig{ + BucketSize: 30, + } + } + } + + // 使用工厂方法创建存储引擎 + engine, err := storage.NewStorageEngine(engineConfig) + if err != nil { + log.Fatalf("Failed to create storage engine: %v", err) + } + + log.Printf("Storage engine created: type=%s, directory=%s", *persistenceType, *persistenceDir) + // 创建数据管理器 dataManager := manager.NewDataManager(engine) diff --git a/pkg/storage/engine.go b/pkg/storage/engine.go index 361738c..5a6db63 100644 --- a/pkg/storage/engine.go +++ b/pkg/storage/engine.go @@ -16,6 +16,8 @@ const ( PersistenceTypeNone PersistenceType = "none" // PersistenceTypeWAL 使用WAL日志持久化 PersistenceTypeWAL PersistenceType = "wal" + // PersistenceTypeBoltDB 使用BoltDB持久化 + PersistenceTypeBoltDB PersistenceType = "boltdb" ) // PersistenceConfig 持久化配置 diff --git a/pkg/storage/factory.go b/pkg/storage/factory.go new file mode 100644 index 0000000..114c5a4 --- /dev/null +++ b/pkg/storage/factory.go @@ -0,0 +1,72 @@ +package storage + +import ( + "fmt" + "path/filepath" +) + +// EngineConfig 存储引擎配置 +type EngineConfig struct { + // 持久化配置 + PersistenceType PersistenceType // 持久化类型 + PersistenceDir string // 持久化目录 + SyncEvery int // 每写入多少条数据同步一次 + + // BoltDB特有配置 + BoltDBFilename string // BoltDB文件名 + BoltDBOptions *BoltDBConfig // BoltDB选项 +} + +// NewStorageEngine 创建一个新的存储引擎 +func NewStorageEngine(config EngineConfig) (StorageEngine, error) { + var engine StorageEngine + var err error + + switch config.PersistenceType { + case PersistenceTypeNone, PersistenceTypeWAL: + // 创建内存存储引擎 + memEngine := NewMemoryEngine() + + // 如果需要持久化，启用WAL + if config.PersistenceType == PersistenceTypeWAL { + err = memEngine.EnablePersistence(PersistenceConfig{ + Type: PersistenceTypeWAL, + Directory: config.PersistenceDir, + SyncEvery: config.SyncEvery, + }) + if err != nil { + return nil, fmt.Errorf("failed to enable WAL persistence: %v", err) + } + } + + engine = memEngine + + case PersistenceTypeBoltDB: + // 创建BoltDB存储引擎 + boltDBPath := filepath.Join(config.PersistenceDir, config.BoltDBFilename) + if config.BoltDBFilename == "" { + boltDBPath = filepath.Join(config.PersistenceDir, "gotidb.db") + } + + boltDBConfig := config.BoltDBOptions + if boltDBConfig == nil { + boltDBConfig = &BoltDBConfig{ + FilePath: boltDBPath, + } + } else { + boltDBConfig.FilePath = boltDBPath + } + + boltEngine, err := NewBoltDBEngine(*boltDBConfig) + if err != nil { + return nil, fmt.Errorf("failed to create BoltDB engine: %v", err) + } + + engine = boltEngine + + default: + return nil, fmt.Errorf("unsupported persistence type: %s", config.PersistenceType) + } + + return engine, nil +}