Golang | 向倒排索引上添加删除文档

syntax = "proto3";package types;message Keyword {string Field = 1; // 属性/类型/名称string Word = 2;  // 关键词
}message Document {string Id = 1; //业务使用的唯一Id，索引上此Id不会重复uint64 IntId = 2; //倒排索引上使用的文档id(业务侧不用管这个字段)uint64 BitsFeature = 3;        //每个bit都表示某种特征的取值repeated Keyword Keywords = 4; //倒排索引的keybytes Bytes = 5;               //业务实体序列化之后的结果
}
// go install github.com/gogo/protobuf/protoc-gen-gogofaster@latest
// protoc --gogofaster_out=./types --proto_path=./types/proto doc.proto

type Document struct {Id          string     `protobuf:"bytes,1,opt,name=Id,proto3" json:"Id,omitempty"`IntId       uint64     `protobuf:"varint,2,opt,name=IntId,proto3" json:"IntId,omitempty"`BitsFeature uint64     `protobuf:"varint,3,opt,name=BitsFeature,proto3" json:"BitsFeature,omitempty"`Keywords    []*Keyword `protobuf:"bytes,4,rep,name=Keywords,proto3" json:"Keywords,omitempty"`Bytes       []byte     `protobuf:"bytes,5,opt,name=Bytes,proto3" json:"Bytes,omitempty"`
}

import "github.com/atopos31/go-velisearch/types"// 倒排索引统一接口
type IRverseIndexer interface {Add(doc types.Document) // keyword定位到某个value IntId在value的链式结构中确定位置Delete(IntId uint64, keyword *types.Keyword) // onFlag：所有 bits 必须完全匹配 offFlag所有 bits 必须完全不匹配 orFlags：bits 必须匹配 `orFlags` 列表中的所有标志中的至少一个。Search(query *types.TermQuery, onFlag uint64, offFlag uint64, orFlags []uint64) []string
}

import ("runtime""sync""github.com/atopos31/go-velisearch/types""github.com/atopos31/go-velisearch/util""github.com/huandu/skiplist"farmhash "github.com/leemcloughlin/gofarmhash"
)var _ IRverseIndexer = (*SkipListInvertedIndexer)(nil)type SkipListInvertedIndexer struct {table *util.ConcurrentHashMap // 使用分段锁保护的并发安全 map，用于存储倒排索引的数据locks []sync.RWMutex          // 针对相同的 key 进行竞争的锁，避免多个协程Add时数据丢失
}type SkipListValue struct {Id          string // 业务侧的IDBitsFeature uint64 // 文件属性位图
}func NewSkipListInvertedIndexer(docNumEstimate int) *SkipListInvertedIndexer {return &SkipListInvertedIndexer{// 小map数根据机器性能 依据核心数设置table: util.NewConcurrentHashMap(runtime.NumCPU(), docNumEstimate),locks: make([]sync.RWMutex, 1000),}
}func (indexer *SkipListInvertedIndexer) Add(doc types.Document) {for _, keyword := range doc.Keywords {Key := keyword.ToString()lock := indexer.getLock(Key)skipListValue := SkipListValue{Id:          doc.Id,BitsFeature: doc.BitsFeature,}lock.Lock()if value, exist := indexer.table.Get(Key); exist {list := value.(*skiplist.SkipList)list.Set(doc.IntId, skipListValue)} else {list := skiplist.New(skiplist.Uint64)list.Set(doc.IntId, skipListValue)indexer.table.Set(Key, list)}lock.Unlock()}
}func (indexer *SkipListInvertedIndexer) getLock(key string) *sync.RWMutex {n := int(farmhash.Hash32WithSeed([]byte(key), 0))return &indexer.locks[n%len(indexer.locks)]
}func (indexer *SkipListInvertedIndexer) Delete(IntId uint64, keyword *types.Keyword) {Key := keyword.ToString()lock := indexer.getLock(Key)lock.Lock()defer lock.Unlock()if value, exist := indexer.table.Get(Key); exist {list := value.(*skiplist.SkipList)list.Remove(IntId)}
}

func (kw Keyword) ToString() string {if len(kw.Word) > 0 {return kw.Field + "\001" + kw.Word}return ""
}