Browse Source

update 添加读写锁 处理

dev
lucas 1 month ago
parent
commit
5a28b87a72
  1. 30
      et_sink/sinkHandler.go

30
et_sink/sinkHandler.go

@ -7,6 +7,8 @@ import (
"gitea.anxinyun.cn/container/common_utils/storage/storageDBs" "gitea.anxinyun.cn/container/common_utils/storage/storageDBs"
"log" "log"
"node/stages" "node/stages"
"slices"
"sync"
"time" "time"
) )
@ -16,6 +18,7 @@ type SinkHandler struct {
dataQueueRaw []common_models.EsRaw dataQueueRaw []common_models.EsRaw
dataQueueVib []common_models.EsVbRaw dataQueueVib []common_models.EsVbRaw
dataQueueTheme []common_models.EsTheme dataQueueTheme []common_models.EsTheme
lock *sync.RWMutex
dataQueueGroup []common_models.EsGroupTheme // 分组主题数据队列 dataQueueGroup []common_models.EsGroupTheme // 分组主题数据队列
signBatch chan bool signBatch chan bool
batchCount int batchCount int
@ -27,7 +30,8 @@ func NewSinkThemeHandler() *SinkHandler {
the := &SinkHandler{ the := &SinkHandler{
stage: stages.NewStage("Theme 数据存储"), stage: stages.NewStage("Theme 数据存储"),
storageConsumers: storageDBs.LoadIStorageConsumer(), storageConsumers: storageDBs.LoadIStorageConsumer(),
dataQueueTheme: make([]common_models.EsTheme, 0), dataQueueTheme: make([]common_models.EsTheme, 0, defaultBatchCount),
lock: &sync.RWMutex{},
signBatch: make(chan bool, 1), signBatch: make(chan bool, 1),
batchCount: defaultBatchCount, batchCount: defaultBatchCount,
} }
@ -44,7 +48,7 @@ func NewSinkGroupHandler() *SinkHandler {
the := &SinkHandler{ the := &SinkHandler{
stage: stages.NewStage("EsGroupTheme 数据存储"), stage: stages.NewStage("EsGroupTheme 数据存储"),
storageConsumers: storageDBs.LoadIStorageConsumer(), storageConsumers: storageDBs.LoadIStorageConsumer(),
dataQueueGroup: make([]common_models.EsGroupTheme, 0), dataQueueGroup: make([]common_models.EsGroupTheme, 0, defaultBatchCount),
batchCount: defaultBatchCount, batchCount: defaultBatchCount,
} }
@ -60,7 +64,9 @@ func NewSinkRawHandler() *SinkHandler {
the := &SinkHandler{ the := &SinkHandler{
stage: stages.NewStage("raws 数据存储"), stage: stages.NewStage("raws 数据存储"),
storageConsumers: storageDBs.LoadIStorageConsumer(), storageConsumers: storageDBs.LoadIStorageConsumer(),
dataQueueRaw: make([]common_models.EsRaw, 0), dataQueueRaw: make([]common_models.EsRaw, 0, defaultBatchCount),
dataQueueVib: make([]common_models.EsVbRaw, 0, defaultBatchCount),
lock: &sync.RWMutex{},
batchCount: defaultBatchCount, batchCount: defaultBatchCount,
} }
go the.dumpRawBatchMonitor() go the.dumpRawBatchMonitor()
@ -90,7 +96,9 @@ func (the *SinkHandler) sinkRawDataToES(deviceData common_models.DeviceData) {
IotaDevice: deviceData.DeviceId, IotaDevice: deviceData.DeviceId,
CreateTime: time.Now().Truncate(time.Millisecond), CreateTime: time.Now().Truncate(time.Millisecond),
} }
the.lock.Lock()
the.dataQueueVib = append(the.dataQueueVib, vbRaws) the.dataQueueVib = append(the.dataQueueVib, vbRaws)
the.lock.Unlock()
case common_models.RawTypeDiag: case common_models.RawTypeDiag:
default: default:
@ -107,7 +115,9 @@ func (the *SinkHandler) sinkRawDataToES(deviceData common_models.DeviceData) {
IotaDevice: deviceData.DeviceId, IotaDevice: deviceData.DeviceId,
CreateTime: time.Now().Truncate(time.Millisecond), CreateTime: time.Now().Truncate(time.Millisecond),
} }
the.lock.Lock()
the.dataQueueRaw = append(the.dataQueueRaw, esRaws) the.dataQueueRaw = append(the.dataQueueRaw, esRaws)
the.lock.Unlock()
} }
} }
@ -124,20 +134,24 @@ func (the *SinkHandler) dumpRawBatchMonitor() {
case <-time.After(200 * time.Millisecond): case <-time.After(200 * time.Millisecond):
} }
if len(the.dataQueueRaw) > 0 { if len(the.dataQueueRaw) > 0 {
the.lock.RLock()
count := len(the.dataQueueRaw) count := len(the.dataQueueRaw)
log.Printf("es写入dataQueueRaw数据 count====> %d", count) log.Printf("es写入dataQueueRaw数据 count====> %d", count)
needDump := make([]common_models.EsRaw, count) needDump := make([]common_models.EsRaw, count)
//避免引用问题 //避免引用问题
copy(needDump, the.dataQueueRaw[:count]) copy(needDump, the.dataQueueRaw[:count])
the.dataQueueRaw = the.dataQueueRaw[count:] the.dataQueueRaw = the.dataQueueRaw[count:]
the.lock.RUnlock()
go the.dumpRaws(needDump) go the.dumpRaws(needDump)
} }
if len(the.dataQueueVib) > 0 { if len(the.dataQueueVib) > 0 {
the.lock.RLock()
count := len(the.dataQueueVib) count := len(the.dataQueueVib)
log.Printf("es写入dataQueueVib数据 count====> %d", count) log.Printf("es写入dataQueueVib数据 count====> %d", count)
needDump := the.dataQueueVib[:count] needDump := the.dataQueueVib[:count]
the.dataQueueVib = the.dataQueueVib[count:] the.dataQueueVib = the.dataQueueVib[count:]
the.lock.RUnlock()
go the.dumpVibRaws(needDump) go the.dumpVibRaws(needDump)
} }
} }
@ -164,12 +178,15 @@ func (the *SinkHandler) dumpThemeBatchMonitor() {
log.Printf("批存储信号Theme,监控器收到") log.Printf("批存储信号Theme,监控器收到")
case <-time.After(200 * time.Millisecond): case <-time.After(200 * time.Millisecond):
} }
if len(the.dataQueueTheme) > 0 { if len(the.dataQueueTheme) > 0 {
count := len(the.dataQueueTheme) the.lock.RLock()
count := len(the.dataQueueTheme) //todo 避免临界操作
needDump := make([]common_models.EsTheme, count) needDump := make([]common_models.EsTheme, count)
//避免引用问题 //避免引用问题
copy(needDump, the.dataQueueTheme[:count]) copy(needDump, the.dataQueueTheme[:count])
the.dataQueueTheme = the.dataQueueTheme[count:] the.dataQueueTheme = the.dataQueueTheme[count:]
the.lock.RUnlock()
go the.dumpThemes(needDump) go the.dumpThemes(needDump)
} }
} }
@ -198,7 +215,10 @@ func (the *SinkHandler) sinkThemeData(stations []common_models.Station) {
} }
EsThemes = append(EsThemes, esTheme) EsThemes = append(EsThemes, esTheme)
} }
the.dataQueueTheme = append(the.dataQueueTheme, EsThemes...) the.lock.Lock()
the.dataQueueTheme = slices.Concat(the.dataQueueTheme, EsThemes)
the.lock.Unlock()
//the.dataQueueTheme = append(the.dataQueueTheme, EsThemes...)
if len(the.dataQueueTheme) >= the.batchCount { if len(the.dataQueueTheme) >= the.batchCount {
the.signBatch <- true the.signBatch <- true
} }

Loading…
Cancel
Save