From b55e7584dfce6e4cfef3050fd918827b0a189346 Mon Sep 17 00:00:00 2001 From: SanaeFox <36219542+Hoshinonyaruko@users.noreply.github.com> Date: Fri, 5 Apr 2024 13:12:26 +0800 Subject: [PATCH 1/6] Beta47 (#48) * beta1 * beta2 * beta3 * beta4 * beta5 * beta6 * beta7 * beta8 * beta9 * beta10 * beta11 * beta12 * beta13 * beta14 * beta15 * beta16 * beta16 * beta19 * beta20 * beta21 * beta22 * beta23 * beta24 * beta25 * beta27 * beta28 * beta29 * beta30 * beta31 * beta33 * beta34 * beta35 * beta36 * beta37 * beta38 * beta39 * beta40 * beta41 * beta42 * beta43 * beta44 * beta45 * beta45 * beta46 --- .gitignore | 2 +- applogic/gensokyo.go | 61 ++++--------------- applogic/singlecontext.go | 50 +++++++++++++++ config/config.go | 17 ++++++ go.mod | 5 ++ go.sum | 4 ++ main.go | 56 +++++++++++++---- template/config_template.go | 3 +- utils/blacklist.go | 117 ++++++++++++++++++++++++++++++++++++ utils/utils.go | 61 ++++++++++++++++++- 10 files changed, 310 insertions(+), 66 deletions(-) create mode 100644 applogic/singlecontext.go create mode 100644 utils/blacklist.go diff --git a/.gitignore b/.gitignore index d4db4b7..8ef422f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ # specific -config.yml +*.yml *.sqlite *.txt diff --git a/applogic/gensokyo.go b/applogic/gensokyo.go index 64ee051..0e52fa4 100644 --- a/applogic/gensokyo.go +++ b/applogic/gensokyo.go @@ -100,7 +100,7 @@ func (app *App) GensokyoHandler(w http.ResponseWriter, r *http.Request) { switch msg := message.Message.(type) { case string: // message.Message是一个string - fmtf.Printf("Received string message: %s\n", msg) + fmtf.Printf("userid:[%v]Received string message: %s\n", message.UserID, msg) //是否过滤群信息 if !config.GetGroupmessage() { @@ -125,6 +125,11 @@ func (app *App) GensokyoHandler(w http.ResponseWriter, r *http.Request) { } } + if utils.BlacklistIntercept(message) { + fmtf.Printf("userid:[%v]这位用户在黑名单中,被拦截", message.UserID) + return + } + //处理重置指令 if isResetCommand { fmtf.Println("处理重置操作") @@ -134,55 +139,7 @@ func (app *App) GensokyoHandler(w http.ResponseWriter, r *http.Request) { if !config.GetUsePrivateSSE() { utils.SendPrivateMessage(message.UserID, RestoreResponse) } else { - - // 将字符串转换为rune切片，以正确处理多字节字符 - runes := []rune(RestoreResponse) - - // 计算每部分应该包含的rune数量 - partLength := len(runes) / 3 - - // 初始化用于存储分割结果的切片 - parts := make([]string, 3) - - // 按字符分割字符串 - for i := 0; i < 3; i++ { - if i < 2 { // 前两部分 - start := i * partLength - end := start + partLength - parts[i] = string(runes[start:end]) - } else { // 最后一部分，包含所有剩余的字符 - start := i * partLength - parts[i] = string(runes[start:]) - } - } - - // 开头 - messageSSE := structs.InterfaceBody{ - Content: parts[0], - State: 1, - } - - utils.SendPrivateMessageSSE(message.UserID, messageSSE) - - //中间 - messageSSE = structs.InterfaceBody{ - Content: parts[1], - State: 11, - } - utils.SendPrivateMessageSSE(message.UserID, messageSSE) - - // 从配置中获取promptkeyboard - promptkeyboard := config.GetPromptkeyboard() - - // 创建InterfaceBody结构体实例 - messageSSE = structs.InterfaceBody{ - Content: parts[2], // 假设空格字符串是期望的内容 - State: 20, // 假设的状态码 - PromptKeyboard: promptkeyboard, // 使用更新后的promptkeyboard - } - - // 发送SSE私人消息 - utils.SendPrivateMessageSSE(message.UserID, messageSSE) + utils.SendSSEPrivateRestoreMessage(message.UserID, RestoreResponse) } } else { utils.SendGroupMessage(message.GroupID, RestoreResponse) @@ -282,6 +239,10 @@ func (app *App) GensokyoHandler(w http.ResponseWriter, r *http.Request) { responseText, err := app.GetRandomAnswer(similarTexts[0]) if err == nil { fmtf.Printf("缓存命中,Q:%v,A:%v\n", newmsg, responseText) + //加入上下文 + if app.AddSingleContext(message, responseText) { + fmtf.Printf("缓存加入上下文成功") + } // 发送响应消息 if message.RealMessageType == "group_private" || message.MessageType == "private" { if !config.GetUsePrivateSSE() { diff --git a/applogic/singlecontext.go b/applogic/singlecontext.go new file mode 100644 index 0000000..7064aed --- /dev/null +++ b/applogic/singlecontext.go @@ -0,0 +1,50 @@ +package applogic + +import ( + "time" + + "github.com/hoshinonyaruko/gensokyo-llm/fmtf" + "github.com/hoshinonyaruko/gensokyo-llm/structs" +) + +// 直接根据缓存来储存上下文 +// 其实向量缓存是一个单轮的QA缓存,因为这个项目很初步,很显然无法应对上下文场景的缓存 +// 通过这种方式,将每次缓存的内容也加入上下文,可能会有一个初步的效果提升. +func (app *App) AddSingleContext(message structs.OnebotGroupMessage, responseText string) bool { + // 请求conversation api 增加当前用户上下文 + conversationID, parentMessageID, err := app.handleUserContext(message.UserID) + if err != nil { + fmtf.Printf("error in AddSingleContext app.handleUserContex :%v", err) + return false + } + + // 构造用户消息并添加到上下文 + userMessage := structs.Message{ + ConversationID: conversationID, + ParentMessageID: parentMessageID, + Text: message.Message.(string), + Role: "user", + CreatedAt: time.Now().Format(time.RFC3339), + } + userMessageID, err := app.addMessage(userMessage) + if err != nil { + fmtf.Printf("error in AddSingleContext app.addMessage(userMessage) :%v", err) + return false + } + + // 构造助理消息并添加到上下文 + assistantMessage := structs.Message{ + ConversationID: conversationID, + ParentMessageID: userMessageID, + Text: responseText, + Role: "assistant", + CreatedAt: time.Now().Format(time.RFC3339), + } + _, err = app.addMessage(assistantMessage) + if err != nil { + fmtf.Printf("error in AddSingleContext app.addMessage(assistantMessage) :%v", err) + return false + } + + return true +} diff --git a/config/config.go b/config/config.go index b0a16c2..134439b 100644 --- a/config/config.go +++ b/config/config.go @@ -85,6 +85,7 @@ type Settings struct { LanguagesResponseMessages []string `yaml:"langResponseMessages"` QuestionMaxLenth int `yaml:"questionMaxLenth"` QmlResponseMessages []string `yaml:"qmlResponseMessages"` + BlacklistResponseMessages []string `yaml:"blacklistResponseMessages"` } // LoadConfig 从文件中加载配置并初始化单例配置 @@ -850,3 +851,19 @@ func GetQmlResponseMessages() string { } return "" // 如果列表为空，返回空字符串 } + +// BlacklistResponseMessages 返回语言拦截响应消息列表 +func GetBlacklistResponseMessages() string { + mu.Lock() + defer mu.Unlock() + if instance != nil && len(instance.Settings.BlacklistResponseMessages) > 0 { + // 如果列表中只有一个消息，直接返回这个消息 + if len(instance.Settings.BlacklistResponseMessages) == 1 { + return instance.Settings.BlacklistResponseMessages[0] + } + // 如果有多个消息，随机选择一个返回 + index := rand.Intn(len(instance.Settings.BlacklistResponseMessages)) + return instance.Settings.BlacklistResponseMessages[index] + } + return "" // 如果列表为空，返回空字符串 +} diff --git a/go.mod b/go.mod index 32c4542..97f89f9 100644 --- a/go.mod +++ b/go.mod @@ -10,3 +10,8 @@ require ( ) require github.com/abadojack/whatlanggo v1.0.1 + +require ( + github.com/fsnotify/fsnotify v1.7.0 // indirect + golang.org/x/sys v0.4.0 // indirect +) diff --git a/go.sum b/go.sum index 646cdb2..49a2f5e 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,15 @@ github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4= github.com/abadojack/whatlanggo v1.0.1/go.mod h1:66WiQbSbJBIlOZMsvbKe5m6pzQovxCH9B/K8tQB2uoc= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.839 h1:VGVFNQDaUpDsPkJrh8I9qOxHZ1yj5sJmg9ngsUvTAHM= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.839/go.mod h1:r5r4xbfxSaeR04b166HGsBa/R4U3SueirEUpXGuw+Q0= +golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/main.go b/main.go index 50fc0a9..21077dc 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( "log" "net/http" "os" + "path/filepath" _ "github.com/mattn/go-sqlite3" // 只导入，作为驱动 @@ -19,25 +20,41 @@ import ( ) func main() { - testFlag := flag.Bool("test", false, "Run the test script,test.txt中的是虚拟信息,一行一条") + testFlag := flag.Bool("test", false, "Run the test script, test.txt中的是虚拟信息,一行一条") + ymlPath := flag.String("yml", "", "指定config.yml的路径") flag.Parse() - if _, err := os.Stat("config.yml"); os.IsNotExist(err) { + // 如果用户指定了-yml参数 + configFilePath := "config.yml" // 默认配置文件路径 + if *ymlPath != "" { + configFilePath = *ymlPath + } - // 将修改后的配置写入 config.yml - err = os.WriteFile("config.yml", []byte(template.ConfigTemplate), 0644) - if err != nil { - fmtf.Println("Error writing config.yml:", err) + // 检查配置文件是否存在 + if _, err := os.Stat(configFilePath); os.IsNotExist(err) { + if *ymlPath == "" { + // 用户没有指定-yml参数，按照默认行为处理 + err = os.WriteFile(configFilePath, []byte(template.ConfigTemplate), 0644) + if err != nil { + fmtf.Println("Error writing config.yml:", err) + return + } + fmtf.Println("请配置config.yml然后再次运行.") + fmtf.Print("按下 Enter 继续...") + bufio.NewReader(os.Stdin).ReadBytes('\n') + os.Exit(0) + } else { + // 用户指定了-yml参数，但指定的文件不存在 + fmtf.Println("指定的配置文件不存在:", *ymlPath) return } - - fmtf.Println("请配置config.yml然后再次运行.") - fmtf.Print("按下 Enter 继续...") - bufio.NewReader(os.Stdin).ReadBytes('\n') - os.Exit(0) + } else { + if *ymlPath != "" { + fmtf.Println("载入成功:", *ymlPath) + } } // 加载配置 - conf, err := config.LoadConfig("config.yml") + conf, err := config.LoadConfig(configFilePath) if err != nil { log.Fatalf("error: %v", err) } @@ -125,6 +142,21 @@ func main() { log.Printf("Unknown API type: %d", apiType) } + exePath, err := os.Executable() + if err != nil { + log.Fatal(err) + } + exeDir := filepath.Dir(exePath) + blacklistPath := filepath.Join(exeDir, "blacklist.txt") + + // 载入黑名单 + if err := utils.LoadBlacklist(blacklistPath); err != nil { + log.Fatalf("Failed to load blacklist: %v", err) + } + + // 启动黑名单文件变动监听 + go utils.WatchBlacklist(blacklistPath) + http.HandleFunc("/gensokyo", app.GensokyoHandler) port := config.GetPort() portStr := fmtf.Sprintf(":%d", port) diff --git a/template/config_template.go b/template/config_template.go index 01b4be0..c1d27c7 100644 --- a/template/config_template.go +++ b/template/config_template.go @@ -34,10 +34,11 @@ settings: promptkeyboard : [""] #临时的promptkeyboard超过3个则随机,后期会增加一个ai生成的方式,也会是ai-agent savelogs : false #本地落地日志. #语言过滤 - allowedLanguages : ["Cmn"] #根据自身安全实力,酌情过滤,cmn代表中文,小写字母,[]空数组代表不限制. + allowedLanguages : ["cmn"] #根据自身安全实力,酌情过滤,cmn代表中文,小写字母,[]空数组代表不限制. langResponseMessages : ["抱歉，我不会**这个语言呢","我不会**这门语言,请使用中文和我对话吧"] #定型文,**会自动替换为检测到的语言 questionMaxLenth : 100 #最大问题字数. 0代表不限制 qmlResponseMessages : ["问题太长了,缩短问题试试吧"] #最大问题长度回复. + blacklistResponseMessages : ["目前正在维护中...请稍候再试吧"] #黑名单回复,将userid丢入blacklist.txt 一行一个 #向量缓存(省钱-酌情调整参数)(进阶!!)需要有一定的调试能力,数据库调优能力,计算和数据测试能力. #不同种类的向量,维度和模型不同,所以请一开始决定好使用的向量,或者自行将数据库备份\对应,不同种类向量没有互相检索的能力。 diff --git a/utils/blacklist.go b/utils/blacklist.go new file mode 100644 index 0000000..b1f7423 --- /dev/null +++ b/utils/blacklist.go @@ -0,0 +1,117 @@ +package utils + +import ( + "bufio" + "fmt" + "log" + "os" + "strconv" + "sync" + + "github.com/fsnotify/fsnotify" + "github.com/hoshinonyaruko/gensokyo-llm/config" + "github.com/hoshinonyaruko/gensokyo-llm/structs" +) + +var blacklist = make(map[string]bool) +var mu sync.RWMutex + +// LoadBlacklist 从给定的文件路径载入黑名单ID。 +// 如果文件不存在，则创建该文件。 +func LoadBlacklist(filePath string) error { + file, err := os.Open(filePath) + if err != nil { + if os.IsNotExist(err) { + // 如果文件不存在，则创建一个新文件 + file, err = os.Create(filePath) + if err != nil { + return err // 创建文件失败，返回错误 + } + } else { + return err // 打开文件失败，且原因不是文件不存在 + } + } + defer file.Close() + + scanner := bufio.NewScanner(file) + mu.Lock() + defer mu.Unlock() + blacklist = make(map[string]bool) // 重置黑名单 + + for scanner.Scan() { + blacklist[scanner.Text()] = true + } + + return scanner.Err() +} + +// isInBlacklist 检查给定的ID是否在黑名单中。 +func IsInBlacklist(id string) bool { + mu.RLock() + defer mu.RUnlock() + _, exists := blacklist[id] + return exists +} + +// watchBlacklist 监控黑名单文件的变动并动态更新。 +func WatchBlacklist(filePath string) { + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatal("Error creating watcher:", err) + } + defer watcher.Close() + + done := make(chan bool) + go func() { + for { + select { + case event, ok := <-watcher.Events: + if !ok { + return + } + if event.Op&fsnotify.Write == fsnotify.Write { + fmt.Println("Detected update to blacklist, reloading...") + err := LoadBlacklist(filePath) + if err != nil { + log.Printf("Error reloading blacklist: %v", err) + } + } + case err, ok := <-watcher.Errors: + if !ok { + return + } + log.Println("Watcher error:", err) + } + } + }() + + err = watcher.Add(filePath) + if err != nil { + log.Fatal("Error adding watcher to file:", err) + } + <-done // Keep the watcher alive +} + +// BlacklistIntercept 检查用户ID是否在黑名单中，如果在，则发送预设消息 +func BlacklistIntercept(message structs.OnebotGroupMessage) bool { + // 检查用户ID是否在黑名单中 + if IsInBlacklist(strconv.FormatInt(message.UserID, 10)) { + // 获取黑名单响应消息 + responseMessage := config.GetBlacklistResponseMessages() + + // 根据消息类型发送响应 + if message.RealMessageType == "group_private" || message.MessageType == "private" { + if !config.GetUsePrivateSSE() { + SendPrivateMessage(message.UserID, responseMessage) + } else { + SendSSEPrivateMessage(message.UserID, responseMessage) + } + } else { + SendGroupMessage(message.GroupID, responseMessage) + } + + fmt.Printf("userid:[%v]这位用户在黑名单中,被拦截\n", message.UserID) + return true // 拦截 + } + return false // 用户ID不在黑名单中，不拦截 +} diff --git a/utils/utils.go b/utils/utils.go index f5c66c3..ce14969 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -11,6 +11,7 @@ import ( "os" "regexp" "strings" + "time" "github.com/abadojack/whatlanggo" "github.com/google/uuid" @@ -303,8 +304,12 @@ func PostSensitiveMessages() error { results = append(results, string(responseBody)) } - // 将HTTP响应结果保存到test_result.txt文件中 - return os.WriteFile("test_result.txt", []byte(strings.Join(results, "\n")), 0644) + // 使用当前时间戳生成文件名 + currentTime := time.Now() + fileName := "test_result_" + currentTime.Format("20060102_150405") + ".txt" + + // 将HTTP响应结果保存到指定的文件中 + return os.WriteFile(fileName, []byte(strings.Join(results, "\n")), 0644) } // SendSSEPrivateMessage 分割并发送消息的核心逻辑，直接遍历字符串 @@ -430,6 +435,58 @@ func SendSSEPrivateSafeMessage(userID int64, saveresponse string) { SendPrivateMessageSSE(userID, messageSSE) } +// SendSSEPrivateRestoreMessage 分割并发送重置消息的核心逻辑，直接遍历字符串 +func SendSSEPrivateRestoreMessage(userID int64, RestoreResponse string) { + // 将字符串转换为rune切片，以正确处理多字节字符 + runes := []rune(RestoreResponse) + + // 计算每部分应该包含的rune数量 + partLength := len(runes) / 3 + + // 初始化用于存储分割结果的切片 + parts := make([]string, 3) + + // 按字符分割字符串 + for i := 0; i < 3; i++ { + if i < 2 { // 前两部分 + start := i * partLength + end := start + partLength + parts[i] = string(runes[start:end]) + } else { // 最后一部分，包含所有剩余的字符 + start := i * partLength + parts[i] = string(runes[start:]) + } + } + + // 开头 + messageSSE := structs.InterfaceBody{ + Content: parts[0], + State: 1, + } + + SendPrivateMessageSSE(userID, messageSSE) + + //中间 + messageSSE = structs.InterfaceBody{ + Content: parts[1], + State: 11, + } + SendPrivateMessageSSE(userID, messageSSE) + + // 从配置中获取promptkeyboard + promptkeyboard := config.GetPromptkeyboard() + + // 创建InterfaceBody结构体实例 + messageSSE = structs.InterfaceBody{ + Content: parts[2], // 假设空格字符串是期望的内容 + State: 20, // 假设的状态码 + PromptKeyboard: promptkeyboard, // 使用更新后的promptkeyboard + } + + // 发送SSE私人消息 + SendPrivateMessageSSE(userID, messageSSE) +} + // LanguageIntercept 检查文本语言，如果不在允许列表中，则返回 true 并发送消息 func LanguageIntercept(text string, message structs.OnebotGroupMessage) bool { info := whatlanggo.Detect(text) From 0015a6694b7e2b37b3b57e5d0aca427f12153856 Mon Sep 17 00:00:00 2001 From: SanaeFox <36219542+Hoshinonyaruko@users.noreply.github.com> Date: Sun, 7 Apr 2024 23:29:59 +0800 Subject: [PATCH 2/6] Beta48 (#49) * beta1 * beta2 * beta3 * beta4 * beta5 * beta6 * beta7 * beta8 * beta9 * beta10 * beta11 * beta12 * beta13 * beta14 * beta15 * beta16 * beta16 * beta19 * beta20 * beta21 * beta22 * beta23 * beta24 * beta25 * beta27 * beta28 * beta29 * beta30 * beta31 * beta33 * beta34 * beta35 * beta36 * beta37 * beta38 * beta39 * beta40 * beta41 * beta42 * beta43 * beta44 * beta45 * beta45 * beta46 * beat48 --- applogic/app.go | 5 +++ applogic/embeddings.go | 19 +++++--- applogic/gensokyo.go | 5 ++- applogic/vectorsensitive.go | 86 ++++++++++++++++++++++++++++++++++++- config/config.go | 15 ++++++- go.mod | 2 +- main.go | 10 +++++ template/config_template.go | 1 + 8 files changed, 132 insertions(+), 11 deletions(-) diff --git a/applogic/app.go b/applogic/app.go index 8363ba1..4593e70 100644 --- a/applogic/app.go +++ b/applogic/app.go @@ -4,6 +4,7 @@ import ( "database/sql" "fmt" + "github.com/hoshinonyaruko/gensokyo-llm/config" "github.com/hoshinonyaruko/gensokyo-llm/fmtf" "github.com/hoshinonyaruko/gensokyo-llm/hunyuan" "github.com/hoshinonyaruko/gensokyo-llm/structs" @@ -262,6 +263,10 @@ func (app *App) updateUserContext(userID int64, parentMessageID string) error { } func (app *App) getHistory(conversationID, parentMessageID string) ([]structs.Message, error) { + // 如果不开启上下文 + if config.GetNoContext() { + return nil, nil + } var history []structs.Message // SQL 查询获取历史信息 diff --git a/applogic/embeddings.go b/applogic/embeddings.go index a7aa033..7b3fd60 100644 --- a/applogic/embeddings.go +++ b/applogic/embeddings.go @@ -112,7 +112,7 @@ func (app *App) CalculateTextEmbeddingHunyuan(text string) ([]float64, error) { } if config.GetPrintVector() { - fmt.Printf("混元返回的向量:%v\n", embedding) + fmtf.Printf("混元返回的向量:%v\n", embedding) } return embedding, nil @@ -229,9 +229,16 @@ func (app *App) insertVectorData(text string, vector []float64) (int64, error) { norm := math.Sqrt(sum) n := config.GetCacheN() k := config.GetCacheK() - l := int(norm * k) + // 先进行四舍五入，然后转换为int64 + l := int64(math.Round(norm * k)) groupID := l % n + if config.GetPrintHanming() { + fmtf.Printf("(norm*k): %v\n", norm*k) + fmtf.Printf("(norm*k) mod n ==== (%v) mod %v\n", l, n) + fmtf.Printf("groupid : %v\n", groupID) + } + result, err := app.DB.Exec("INSERT INTO vector_data (text, vector, norm, group_id) VALUES (?, ?, ?, ?)", text, binaryVector, norm, groupID) if err != nil { return 0, err @@ -246,7 +253,7 @@ func (app *App) insertVectorData(text string, vector []float64) (int64, error) { } // searchSimilarText函数根据汉明距离搜索数据库中与给定向量相似的文本 -func (app *App) searchSimilarText(vector []float64, threshold int, targetGroupID int) ([]TextDistance, []int, error) { +func (app *App) searchSimilarText(vector []float64, threshold int, targetGroupID int64) ([]TextDistance, []int, error) { binaryVector := vectorToBinaryConcurrent(vector) // 二值化查询向量 var results []TextDistance var ids []int @@ -289,7 +296,7 @@ func (app *App) searchSimilarText(vector []float64, threshold int, targetGroupID return results, sortedIds, nil } -func calculateGroupID(vector []float64) int { +func calculateGroupID(vector []float64) int64 { var sum float64 for _, v := range vector { sum += v * v @@ -297,9 +304,11 @@ func calculateGroupID(vector []float64) int { norm := math.Sqrt(sum) k := config.GetCacheK() n := config.GetCacheN() - l := int(norm * k) + // 先进行四舍五入，然后转换为int64 + l := int64(math.Round(norm * k)) groupid := l % n // 通过范数计算出一个整数，并将其模n来分配到一个组 if config.GetPrintHanming() { + fmtf.Printf("(norm*k): %v\n", norm*k) fmtf.Printf("(norm*k) mod n ==== (%v) mod %v\n", l, n) fmtf.Printf("groupid : %v\n", groupid) } diff --git a/applogic/gensokyo.go b/applogic/gensokyo.go index 0e52fa4..ed9539b 100644 --- a/applogic/gensokyo.go +++ b/applogic/gensokyo.go @@ -183,6 +183,9 @@ func (app *App) GensokyoHandler(w http.ResponseWriter, r *http.Request) { // 如果使用向量缓存 或者使用 向量安全词 if config.GetUseCache() || config.GetVectorSensitiveFilter() { + if config.GetPrintHanming() { + fmtf.Printf("计算向量的文本: %v", newmsg) + } // 计算文本向量 vector, err = app.CalculateTextEmbedding(newmsg) if err != nil { @@ -194,7 +197,7 @@ func (app *App) GensokyoHandler(w http.ResponseWriter, r *http.Request) { } } - // 向量安全词部分,机器人大安全向量安全屏障 + // 向量安全词部分,机器人向量安全屏障 if config.GetVectorSensitiveFilter() { ret, retstr, err := app.InterceptSensitiveContent(vector, message) if err != nil { diff --git a/applogic/vectorsensitive.go b/applogic/vectorsensitive.go index b483f8a..62e8aa0 100644 --- a/applogic/vectorsensitive.go +++ b/applogic/vectorsensitive.go @@ -25,8 +25,14 @@ func (app *App) insertVectorDataSensitive(text string, vector []float64) (int64, norm := math.Sqrt(sum) n := config.GetCacheN() k := config.GetCacheK() - l := int(norm * k) + // 先进行四舍五入，然后转换为int64 + l := int64(math.Round(norm * k)) groupID := l % n + if config.GetPrintHanming() { + fmtf.Printf("(norm*k): %v\n", norm*k) + fmtf.Printf("(norm*k) mod n ==== (%v) mod %v\n", l, n) + fmtf.Printf("groupid : %v\n", groupID) + } result, err := app.DB.Exec("INSERT INTO sensitive_words (text, vector, norm, group_id) VALUES (?, ?, ?, ?)", text, binaryVector, norm, groupID) if err != nil { @@ -42,7 +48,7 @@ func (app *App) insertVectorDataSensitive(text string, vector []float64) (int64, } // searchSimilarText函数根据汉明距离搜索数据库中与给定向量相似的文本 -func (app *App) searchSimilarTextSensitive(vector []float64, threshold int, targetGroupID int) ([]TextDistance, []int, error) { +func (app *App) searchSimilarTextSensitive(vector []float64, threshold int, targetGroupID int64) ([]TextDistance, []int, error) { binaryVector := vectorToBinaryConcurrent(vector) // 二值化查询向量 var results []TextDistance var ids []int @@ -145,6 +151,7 @@ func (app *App) ProcessSensitiveWords() error { } // 文本在数据库中不存在，计算其向量 + fmtf.Printf("计算向量的敏感词：%s\n", text) vector, err := app.CalculateTextEmbedding(text) if err != nil { return fmt.Errorf("计算文本向量时出错 '%s': %w", text, err) @@ -165,6 +172,81 @@ func (app *App) ProcessSensitiveWords() error { return nil } +func (app *App) ProcessSensitiveWordsV2() error { + // Step 1: 判断是否需要处理敏感词向量 + if !config.GetVectorSensitiveFilter() { + fmt.Println("向量敏感词过滤未启用") + return nil // 不需要处理敏感词向量 + } + + // Step 2: 读取敏感词列表 + file, err := os.Open("vector_sensitive.txt") + if err != nil { + return fmt.Errorf("打开 vector_sensitive.txt 文件时出错: %w", err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + text := scanner.Text() + + // 对每个敏感词重复计算向量10次 + for i := 0; i < 10; i++ { + fmt.Printf("计算向量的敏感词：%s，尝试 #%d\n", text, i+1) + vector, err := app.CalculateTextEmbedding(text) + if err != nil { + return fmt.Errorf("计算文本向量时出错 '%s': %w", text, err) + } + + // 计算groupID + var sum float64 + for _, v := range vector { + sum += v * v + } + norm := math.Sqrt(sum) + k := config.GetCacheK() + l := int64(math.Round(norm * k)) + n := config.GetCacheN() + groupID := l % n + + // 检查数据库中是否存在相同text和groupID的记录 + exists, err := app.textAndGroupIDExistsInDatabase(text, groupID) + if err != nil { + return fmt.Errorf("检查敏感词存在性时出错: %w", err) + } + + if exists { + fmt.Printf("数据库中已存在敏感词及分组ID：%s, %d\n", text, groupID) + continue + } + + // 将新的向量数据插入数据库 + id, err := app.insertVectorDataSensitive(text, vector) + if err != nil { + return fmt.Errorf("将敏感词向量数据插入数据库时出错: %w", err) + } + fmt.Printf("成功插入敏感词及分组ID，ID为：%d\n", id) + } + } + + if err := scanner.Err(); err != nil { + return fmt.Errorf("扫描 vector_sensitive.txt 文件时出错: %w", err) + } + + return nil +} + +// 检查数据库中是否存在相同text和groupID的记录 +func (app *App) textAndGroupIDExistsInDatabase(text string, groupID int64) (bool, error) { + var exists bool + query := "SELECT EXISTS(SELECT 1 FROM sensitive_words WHERE text = ? AND group_id = ? LIMIT 1)" + err := app.DB.QueryRow(query, text, groupID).Scan(&exists) + if err != nil { + return false, fmt.Errorf("查询敏感词和分组ID时出错: %w", err) + } + return exists, nil +} + // textExistsInDatabase 检查给定的文本是否已存在于数据库中 func (app *App) textExistsInDatabase(text string) (bool, error) { var exists bool diff --git a/config/config.go b/config/config.go index 134439b..0edb9f4 100644 --- a/config/config.go +++ b/config/config.go @@ -72,7 +72,7 @@ type Settings struct { GptEmbeddingUrl string `yaml:"gptEmbeddingUrl"` PrintHanming bool `yaml:"printHanming"` CacheK float64 `yaml:"cacheK"` - CacheN int `yaml:"cacheN"` + CacheN int64 `yaml:"cacheN"` PrintVector bool `yaml:"printVector"` VToBThreshold float64 `yaml:"vToBThreshold"` GptModeration bool `yaml:"gptModeration"` @@ -86,6 +86,7 @@ type Settings struct { QuestionMaxLenth int `yaml:"questionMaxLenth"` QmlResponseMessages []string `yaml:"qmlResponseMessages"` BlacklistResponseMessages []string `yaml:"blacklistResponseMessages"` + NoContext bool `yaml:"noContext"` } // LoadConfig 从文件中加载配置并初始化单例配置 @@ -711,7 +712,7 @@ func GetCacheK() float64 { } // 获取CacheN -func GetCacheN() int { +func GetCacheN() int64 { mu.Lock() defer mu.Unlock() if instance != nil { @@ -867,3 +868,13 @@ func GetBlacklistResponseMessages() string { } return "" // 如果列表为空，返回空字符串 } + +// 获取NoContext +func GetNoContext() bool { + mu.Lock() + defer mu.Unlock() + if instance != nil { + return instance.Settings.NoContext + } + return false +} diff --git a/go.mod b/go.mod index 97f89f9..ce85a5a 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,6 @@ require ( require github.com/abadojack/whatlanggo v1.0.1 require ( - github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 golang.org/x/sys v0.4.0 // indirect ) diff --git a/main.go b/main.go index 21077dc..9e32a4d 100644 --- a/main.go +++ b/main.go @@ -22,6 +22,7 @@ import ( func main() { testFlag := flag.Bool("test", false, "Run the test script, test.txt中的是虚拟信息,一行一条") ymlPath := flag.String("yml", "", "指定config.yml的路径") + vFlag := flag.Bool("v", false, "Run ProcessSensitiveWordsV2") flag.Parse() // 如果用户指定了-yml参数 @@ -157,6 +158,15 @@ func main() { // 启动黑名单文件变动监听 go utils.WatchBlacklist(blacklistPath) + // 根据-v参数决定是否运行ProcessSensitiveWordsV2 + if *vFlag { + err := app.ProcessSensitiveWordsV2() + if err != nil { + fmtf.Println("Error running ProcessSensitiveWordsV2:", err) + return + } + } + http.HandleFunc("/gensokyo", app.GensokyoHandler) port := config.GetPort() portStr := fmtf.Sprintf(":%d", port) diff --git a/template/config_template.go b/template/config_template.go index c1d27c7..cb648de 100644 --- a/template/config_template.go +++ b/template/config_template.go @@ -33,6 +33,7 @@ settings: usePrivateSSE : false #不知道是啥的话就不用开 promptkeyboard : [""] #临时的promptkeyboard超过3个则随机,后期会增加一个ai生成的方式,也会是ai-agent savelogs : false #本地落地日志. + noContext : false #不开启上下文 #语言过滤 allowedLanguages : ["cmn"] #根据自身安全实力,酌情过滤,cmn代表中文,小写字母,[]空数组代表不限制. langResponseMessages : ["抱歉，我不会**这个语言呢","我不会**这门语言,请使用中文和我对话吧"] #定型文,**会自动替换为检测到的语言 From 785b1da9ffe275e67bec8efe12ba4d38d2a83a98 Mon Sep 17 00:00:00 2001 From: SanaeFox <36219542+Hoshinonyaruko@users.noreply.github.com> Date: Mon, 8 Apr 2024 17:21:30 +0800 Subject: [PATCH 3/6] Beta49 (#50) * beta1 * beta2 * beta3 * beta4 * beta5 * beta6 * beta7 * beta8 * beta9 * beta10 * beta11 * beta12 * beta13 * beta14 * beta15 * beta16 * beta16 * beta19 * beta20 * beta21 * beta22 * beta23 * beta24 * beta25 * beta27 * beta28 * beta29 * beta30 * beta31 * beta33 * beta34 * beta35 * beta36 * beta37 * beta38 * beta39 * beta40 * beta41 * beta42 * beta43 * beta44 * beta45 * beta45 * beta46 * beat48 * beta49 --- applogic/gensokyo.go | 56 +++++++++-- applogic/vectorsensitive.go | 2 +- config/config.go | 11 ++ template/config_template.go | 1 + utils/blacklist.go | 2 +- utils/utils.go | 194 ++++++++++++++++++++++++++++++++++-- 6 files changed, 251 insertions(+), 15 deletions(-) diff --git a/applogic/gensokyo.go b/applogic/gensokyo.go index ed9539b..67a0e1b 100644 --- a/applogic/gensokyo.go +++ b/applogic/gensokyo.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "encoding/json" + "fmt" "io" "math/rand" "net/http" @@ -142,11 +143,28 @@ func (app *App) GensokyoHandler(w http.ResponseWriter, r *http.Request) { utils.SendSSEPrivateRestoreMessage(message.UserID, RestoreResponse) } } else { - utils.SendGroupMessage(message.GroupID, RestoreResponse) + utils.SendGroupMessage(message.GroupID, message.UserID, RestoreResponse) } return } + withdrawCommand := config.GetWithdrawCommand() + + // 检查checkResetCommand是否在WithdrawCommand列表中 + iswithdrawCommand := false + for _, command := range withdrawCommand { + if checkResetCommand == command { + iswithdrawCommand = true + break + } + } + + // 处理撤回信息 + if iswithdrawCommand { + handleWithdrawMessage(message) + return + } + // newmsg 是一个用于缓存和安全判断的临时量 newmsg := message.Message.(string) // 去除注入的提示词 @@ -254,7 +272,7 @@ func (app *App) GensokyoHandler(w http.ResponseWriter, r *http.Request) { utils.SendSSEPrivateMessage(message.UserID, responseText) } } else { - utils.SendGroupMessage(message.GroupID, responseText) + utils.SendGroupMessage(message.GroupID, message.UserID, responseText) } // 发送响应 w.WriteHeader(http.StatusOK) @@ -299,7 +317,7 @@ func (app *App) GensokyoHandler(w http.ResponseWriter, r *http.Request) { utils.SendSSEPrivateSafeMessage(message.UserID, saveresponse) } } else { - utils.SendGroupMessage(message.GroupID, saveresponse) + utils.SendGroupMessage(message.GroupID, message.UserID, saveresponse) } } // 发送响应 @@ -409,7 +427,7 @@ func (app *App) GensokyoHandler(w http.ResponseWriter, r *http.Request) { utils.SendPrivateMessageSSE(message.UserID, messageSSE) } } else { - utils.SendGroupMessage(message.GroupID, newPart) + utils.SendGroupMessage(message.GroupID, message.UserID, newPart) } } @@ -430,7 +448,7 @@ func (app *App) GensokyoHandler(w http.ResponseWriter, r *http.Request) { utils.SendPrivateMessageSSE(message.UserID, messageSSE) } } else { - utils.SendGroupMessage(message.GroupID, response) + utils.SendGroupMessage(message.GroupID, message.UserID, response) } } } @@ -498,7 +516,7 @@ func (app *App) GensokyoHandler(w http.ResponseWriter, r *http.Request) { if message.RealMessageType == "group_private" || message.MessageType == "private" { utils.SendPrivateMessage(message.UserID, response) } else { - utils.SendGroupMessage(message.GroupID, response) + utils.SendGroupMessage(message.GroupID, message.UserID, response) } } @@ -593,7 +611,7 @@ func processMessage(response string, msg structs.OnebotGroupMessage, newmesssage } } } else { - utils.SendGroupMessage(msg.GroupID, accumulatedMessage) + utils.SendGroupMessage(msg.GroupID, msg.UserID, accumulatedMessage) } messageBuilder.Reset() // 重置消息构建器 @@ -601,3 +619,27 @@ func processMessage(response string, msg structs.OnebotGroupMessage, newmesssage } } } + +// 处理撤回信息的函数 +func handleWithdrawMessage(message structs.OnebotGroupMessage) { + fmt.Println("处理撤回操作") + var id int64 + + // 根据消息类型决定使用哪个ID + switch message.RealMessageType { + case "group_private", "guild_private": + id = message.UserID + case "group", "guild": + id = message.GroupID + default: + fmt.Println("Unsupported message type for withdrawal:", message.RealMessageType) + return + } + + // 调用DeleteLatestMessage函数 + err := utils.DeleteLatestMessage(message.RealMessageType, id, message.UserID) + if err != nil { + fmt.Println("Error deleting latest message:", err) + return + } +} diff --git a/applogic/vectorsensitive.go b/applogic/vectorsensitive.go index 62e8aa0..486e805 100644 --- a/applogic/vectorsensitive.go +++ b/applogic/vectorsensitive.go @@ -288,7 +288,7 @@ func (app *App) InterceptSensitiveContent(vector []float64, message structs.Oneb utils.SendSSEPrivateSafeMessage(message.UserID, saveresponse) } } else { - utils.SendGroupMessage(message.GroupID, saveresponse) + utils.SendGroupMessage(message.GroupID, message.UserID, saveresponse) } return 1, saveresponse, nil } diff --git a/config/config.go b/config/config.go index 0edb9f4..1cf47ed 100644 --- a/config/config.go +++ b/config/config.go @@ -87,6 +87,7 @@ type Settings struct { QmlResponseMessages []string `yaml:"qmlResponseMessages"` BlacklistResponseMessages []string `yaml:"blacklistResponseMessages"` NoContext bool `yaml:"noContext"` + WithdrawCommand []string `yaml:"withdrawCommand"` } // LoadConfig 从文件中加载配置并初始化单例配置 @@ -878,3 +879,13 @@ func GetNoContext() bool { } return false } + +// 获取WithdrawCommand +func GetWithdrawCommand() []string { + mu.Lock() + defer mu.Unlock() + if instance != nil { + return instance.Settings.WithdrawCommand + } + return nil +} diff --git a/template/config_template.go b/template/config_template.go index cb648de..4f0e842 100644 --- a/template/config_template.go +++ b/template/config_template.go @@ -34,6 +34,7 @@ settings: promptkeyboard : [""] #临时的promptkeyboard超过3个则随机,后期会增加一个ai生成的方式,也会是ai-agent savelogs : false #本地落地日志. noContext : false #不开启上下文 + withdrawCommand : ["撤回"] #撤回指令 #语言过滤 allowedLanguages : ["cmn"] #根据自身安全实力,酌情过滤,cmn代表中文,小写字母,[]空数组代表不限制. langResponseMessages : ["抱歉，我不会**这个语言呢","我不会**这门语言,请使用中文和我对话吧"] #定型文,**会自动替换为检测到的语言 diff --git a/utils/blacklist.go b/utils/blacklist.go index b1f7423..789109d 100644 --- a/utils/blacklist.go +++ b/utils/blacklist.go @@ -107,7 +107,7 @@ func BlacklistIntercept(message structs.OnebotGroupMessage) bool { SendSSEPrivateMessage(message.UserID, responseMessage) } } else { - SendGroupMessage(message.GroupID, responseMessage) + SendGroupMessage(message.GroupID, message.UserID, responseMessage) } fmt.Printf("userid:[%v]这位用户在黑名单中,被拦截\n", message.UserID) diff --git a/utils/utils.go b/utils/utils.go index ce14969..a9b98b7 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -10,7 +10,9 @@ import ( "net/http" "os" "regexp" + "strconv" "strings" + "sync" "time" "github.com/abadojack/whatlanggo" @@ -22,6 +24,23 @@ import ( "github.com/hoshinonyaruko/gensokyo-llm/structs" ) +// ResponseData 是用于解析HTTP响应的结构体 +type ResponseData struct { + Data struct { + MessageID int64 `json:"message_id"` + } `json:"data"` +} + +// MessageIDInfo 代表消息ID及其到期时间 +type MessageIDInfo struct { + MessageID int64 // 消息ID + Expires time.Time // 到期时间 +} + +// UserIDMessageIDs 存储每个用户ID对应的消息ID数组及其有效期 +var UserIDMessageIDs = make(map[int64][]MessageIDInfo) +var muUserIDMessageIDs sync.RWMutex // 用于UserIDMessageIDs的读写锁 + func GenerateUUID() string { return uuid.New().String() } @@ -116,7 +135,7 @@ func ExtractEventDetails(eventData map[string]interface{}) (string, structs.Usag return responseTextBuilder.String(), totalUsage } -func SendGroupMessage(groupID int64, message string) error { +func SendGroupMessage(groupID int64, userID int64, message string) error { // 获取基础URL baseURL := config.GetHttpPath() // 假设config.getHttpPath()返回基础URL @@ -130,6 +149,7 @@ func SendGroupMessage(groupID int64, message string) error { // 构造请求体 requestBody, err := json.Marshal(map[string]interface{}{ "group_id": groupID, + "user_id": userID, "message": message, }) if err != nil { @@ -148,7 +168,24 @@ func SendGroupMessage(groupID int64, message string) error { return fmtf.Errorf("received non-OK response status: %s", resp.Status) } - // TODO: 处理响应体（如果需要） + // 读取响应体 + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read response body: %w", err) + } + + // 解析响应体以获取message_id + var responseData ResponseData + if err := json.Unmarshal(bodyBytes, &responseData); err != nil { + return fmt.Errorf("failed to unmarshal response data: %w", err) + } + messageID := responseData.Data.MessageID + + // 添加messageID到全局变量 + AddMessageID(userID, messageID) + + // 输出响应体，这一步是可选的 + fmt.Println("Response Body:", string(bodyBytes)) return nil } @@ -186,7 +223,24 @@ func SendPrivateMessage(UserID int64, message string) error { return fmtf.Errorf("received non-OK response status: %s", resp.Status) } - // TODO: 处理响应体（如果需要） + // 读取响应体 + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read response body: %w", err) + } + + // 解析响应体以获取message_id + var responseData ResponseData + if err := json.Unmarshal(bodyBytes, &responseData); err != nil { + return fmt.Errorf("failed to unmarshal response data: %w", err) + } + messageID := responseData.Data.MessageID + + // 添加messageID到全局变量 + AddMessageID(UserID, messageID) + + // 输出响应体，这一步是可选的 + fmt.Println("Response Body:", string(bodyBytes)) return nil } @@ -224,7 +278,24 @@ func SendPrivateMessageSSE(UserID int64, message structs.InterfaceBody) error { return fmtf.Errorf("received non-OK response status: %s", resp.Status) } - // TODO: 处理响应体（如果需要） + // 读取响应体 + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read response body: %w", err) + } + + // 解析响应体以获取message_id + var responseData ResponseData + if err := json.Unmarshal(bodyBytes, &responseData); err != nil { + return fmt.Errorf("failed to unmarshal response data: %w", err) + } + messageID := responseData.Data.MessageID + + // 添加messageID到全局变量 + AddMessageID(UserID, messageID) + + // 输出响应体，这一步是可选的 + fmt.Println("Response Body:", string(bodyBytes)) return nil } @@ -513,7 +584,7 @@ func LanguageIntercept(text string, message structs.OnebotGroupMessage) bool { SendSSEPrivateMessage(message.UserID, responseMessage) } } else { - SendGroupMessage(message.GroupID, responseMessage) + SendGroupMessage(message.GroupID, message.UserID, responseMessage) } return true // 拦截 @@ -570,10 +641,121 @@ func LengthIntercept(text string, message structs.OnebotGroupMessage) bool { SendSSEPrivateMessage(message.UserID, responseMessage) } } else { - SendGroupMessage(message.GroupID, responseMessage) + SendGroupMessage(message.GroupID, message.UserID, responseMessage) } return true // 拦截 } return false // 长度符合要求，不拦截 } + +// AddMessageID 为指定user_id添加新的消息ID +func AddMessageID(userID int64, messageID int64) { + muUserIDMessageIDs.Lock() + defer muUserIDMessageIDs.Unlock() + + // 消息ID的有效期是120秒 + expiration := time.Now().Add(120 * time.Second) + messageInfo := MessageIDInfo{MessageID: messageID, Expires: expiration} + + // 清理已过期的消息ID + cleanExpiredMessageIDs(userID) + + // 添加新的消息ID + UserIDMessageIDs[userID] = append(UserIDMessageIDs[userID], messageInfo) +} + +// cleanExpiredMessageIDs 清理指定user_id的已过期消息ID +func cleanExpiredMessageIDs(userID int64) { + validMessageIDs := []MessageIDInfo{} + for _, messageInfo := range UserIDMessageIDs[userID] { + if messageInfo.Expires.After(time.Now()) { + validMessageIDs = append(validMessageIDs, messageInfo) + } + } + UserIDMessageIDs[userID] = validMessageIDs +} + +// GetLatestValidMessageID 获取指定user_id当前有效的最新消息ID +func GetLatestValidMessageID(userID int64) (int64, bool) { + muUserIDMessageIDs.RLock() + defer muUserIDMessageIDs.RUnlock() + + // 确保已过期的消息ID被清理 + cleanExpiredMessageIDs(userID) + + // 获取最新的消息ID + if len(UserIDMessageIDs[userID]) > 0 { + latestMessageInfo := UserIDMessageIDs[userID][len(UserIDMessageIDs[userID])-1] + return latestMessageInfo.MessageID, true + } + return 0, false +} + +// sendDeleteRequest 发送删除消息的请求，并输出响应内容 +func sendDeleteRequest(url string, requestBody []byte) error { + // 发送POST请求 + resp, err := http.Post(url, "application/json", bytes.NewBuffer(requestBody)) + if err != nil { + return fmt.Errorf("failed to send POST request: %w", err) + } + defer resp.Body.Close() + + // 检查响应状态 + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("received non-OK response status: %s", resp.Status) + } + + // 读取响应体 + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read response body: %w", err) + } + + // 将响应体转换为字符串，并输出 + bodyString := string(bodyBytes) + fmt.Println("Response Body:", bodyString) + + return nil +} + +func DeleteLatestMessage(messageType string, id int64, userid int64) error { + // 获取基础URL + baseURL := config.GetHttpPath() // 假设config.GetHttpPath()返回基础URL + + // 构建完整的URL + url := baseURL + "/delete_msg" + + // 获取最新的有效消息ID + messageID, valid := GetLatestValidMessageID(userid) + if !valid { + return fmt.Errorf("no valid message ID found for user/group/guild ID: %d", id) + } + + // 构造请求体 + requestBody := make(map[string]interface{}) + requestBody["message_id"] = strconv.FormatInt(messageID, 10) + + // 根据type填充相应的ID字段 + switch messageType { + case "group_private": + requestBody["user_id"] = id + case "group": + requestBody["group_id"] = id + case "guild": + requestBody["channel_id"] = id + case "guild_private": + requestBody["guild_id"] = id + default: + return fmt.Errorf("unsupported message type: %s", messageType) + } + + requestBodyBytes, err := json.Marshal(requestBody) + if err != nil { + return fmt.Errorf("failed to marshal request body: %w", err) + } + fmtf.Printf("发送撤回请求:%v", string(requestBodyBytes)) + + // 发送删除消息请求 + return sendDeleteRequest(url, requestBodyBytes) +} From bfe5eedc27726b8c40077e8109fd8c607f585ddc Mon Sep 17 00:00:00 2001 From: SanaeFox <36219542+Hoshinonyaruko@users.noreply.github.com> Date: Mon, 8 Apr 2024 18:16:45 +0800 Subject: [PATCH 4/6] Beta50 (#51) * beta1 * beta2 * beta3 * beta4 * beta5 * beta6 * beta7 * beta8 * beta9 * beta10 * beta11 * beta12 * beta13 * beta14 * beta15 * beta16 * beta16 * beta19 * beta20 * beta21 * beta22 * beta23 * beta24 * beta25 * beta27 * beta28 * beta29 * beta30 * beta31 * beta33 * beta34 * beta35 * beta36 * beta37 * beta38 * beta39 * beta40 * beta41 * beta42 * beta43 * beta44 * beta45 * beta45 * beta46 * beat48 * beta49 * beta50 --- applogic/chatgpt.go | 20 +++++++++----------- applogic/ernie.go | 20 +++++++++----------- applogic/hunyuan.go | 20 +++++++++----------- 3 files changed, 27 insertions(+), 33 deletions(-) diff --git a/applogic/chatgpt.go b/applogic/chatgpt.go index 0968b59..ebb12b7 100644 --- a/applogic/chatgpt.go +++ b/applogic/chatgpt.go @@ -410,19 +410,17 @@ func truncateHistoryGpt(history []structs.Message, prompt string) []structs.Mess tokenCount += len(msg.Text) } - if tokenCount <= MAX_TOKENS { - return history - } - - // 第一步：从开始逐个移除消息，直到满足令牌数量限制 - for tokenCount > MAX_TOKENS && len(history) > 0 { - tokenCount -= len(history[0].Text) - history = history[1:] - - // 确保移除后，历史记录仍然以user消息结尾 - if len(history) > 0 && history[0].Role == "assistant" { + if tokenCount >= MAX_TOKENS { + // 第一步：从开始逐个移除消息，直到满足令牌数量限制 + for tokenCount > MAX_TOKENS && len(history) > 0 { tokenCount -= len(history[0].Text) history = history[1:] + + // 确保移除后，历史记录仍然以user消息结尾 + if len(history) > 0 && history[0].Role == "assistant" { + tokenCount -= len(history[0].Text) + history = history[1:] + } } } diff --git a/applogic/ernie.go b/applogic/ernie.go index 4573500..f850a3f 100644 --- a/applogic/ernie.go +++ b/applogic/ernie.go @@ -364,19 +364,17 @@ func truncateHistoryErnie(history []structs.Message, prompt string) []structs.Me tokenCount += len(msg.Text) } - if tokenCount <= MAX_TOKENS { - return history - } - - // 第一步：逐个移除消息直到满足令牌数量限制 - for tokenCount > MAX_TOKENS && len(history) > 0 { - tokenCount -= len(history[0].Text) - history = history[1:] - - // 确保移除后，历史记录仍然以user消息结尾 - if len(history) > 0 && history[0].Role == "assistant" { + if tokenCount >= MAX_TOKENS { + // 第一步：逐个移除消息直到满足令牌数量限制 + for tokenCount > MAX_TOKENS && len(history) > 0 { tokenCount -= len(history[0].Text) history = history[1:] + + // 确保移除后，历史记录仍然以user消息结尾 + if len(history) > 0 && history[0].Role == "assistant" { + tokenCount -= len(history[0].Text) + history = history[1:] + } } } diff --git a/applogic/hunyuan.go b/applogic/hunyuan.go index 58eec63..5253b54 100644 --- a/applogic/hunyuan.go +++ b/applogic/hunyuan.go @@ -433,19 +433,17 @@ func truncateHistoryHunYuan(history []structs.Message, prompt string) []structs. tokenCount += len(msg.Text) } - if tokenCount <= MAX_TOKENS { - return history - } - - // 第一步：逐个移除消息直到满足令牌数量限制，同时保证成对的消息交替出现 - for tokenCount > MAX_TOKENS && len(history) > 0 { - tokenCount -= len(history[0].Text) - history = history[1:] - - // 确保移除后，历史记录仍然以user消息结尾 - if len(history) > 0 && history[0].Role == "assistant" { + if tokenCount >= MAX_TOKENS { + // 第一步：逐个移除消息直到满足令牌数量限制，同时保证成对的消息交替出现 + for tokenCount > MAX_TOKENS && len(history) > 0 { tokenCount -= len(history[0].Text) history = history[1:] + + // 确保移除后，历史记录仍然以user消息结尾 + if len(history) > 0 && history[0].Role == "assistant" { + tokenCount -= len(history[0].Text) + history = history[1:] + } } } From 50724d6f4beee235f048b2f3c13b4ab5e702dbd1 Mon Sep 17 00:00:00 2001 From: SanaeFox <36219542+Hoshinonyaruko@users.noreply.github.com> Date: Mon, 8 Apr 2024 20:16:32 +0800 Subject: [PATCH 5/6] Beta51 (#52) * beta1 * beta2 * beta3 * beta4 * beta5 * beta6 * beta7 * beta8 * beta9 * beta10 * beta11 * beta12 * beta13 * beta14 * beta15 * beta16 * beta16 * beta19 * beta20 * beta21 * beta22 * beta23 * beta24 * beta25 * beta27 * beta28 * beta29 * beta30 * beta31 * beta33 * beta34 * beta35 * beta36 * beta37 * beta38 * beta39 * beta40 * beta41 * beta42 * beta43 * beta44 * beta45 * beta45 * beta46 * beat48 * beta49 * beta50 * beta51 --- applogic/ernie.go | 6 +++--- applogic/gensokyo.go | 4 ++-- applogic/hunyuan.go | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/applogic/ernie.go b/applogic/ernie.go index f850a3f..865101b 100644 --- a/applogic/ernie.go +++ b/applogic/ernie.go @@ -389,9 +389,9 @@ func truncateHistoryErnie(history []structs.Message, prompt string) []structs.Me } } - // 第三步：确保以user结尾 - if len(history) > 0 && history[len(history)-1].Role == "assistant" { - for len(history) > 0 && history[len(history)-1].Role != "user" { + // 第三步：确保以assistant结尾 + if len(history) > 0 && history[len(history)-1].Role == "user" { + for len(history) > 0 && history[len(history)-1].Role != "assistant" { history = history[:len(history)-1] } } diff --git a/applogic/gensokyo.go b/applogic/gensokyo.go index 67a0e1b..1fc3839 100644 --- a/applogic/gensokyo.go +++ b/applogic/gensokyo.go @@ -509,9 +509,9 @@ func (app *App) GensokyoHandler(w http.ResponseWriter, r *http.Request) { fmtf.Printf("Error unmarshalling response data: %v\n", err) return } - + var ok bool // 使用提取的response内容发送消息 - if response, ok := responseData["response"].(string); ok && response != "" { + if response, ok = responseData["response"].(string); ok && response != "" { // 判断消息类型，如果是私人消息或私有群消息，发送私人消息；否则，根据配置决定是否发送群消息 if message.RealMessageType == "group_private" || message.MessageType == "private" { utils.SendPrivateMessage(message.UserID, response) diff --git a/applogic/hunyuan.go b/applogic/hunyuan.go index 5253b54..05d80a3 100644 --- a/applogic/hunyuan.go +++ b/applogic/hunyuan.go @@ -460,11 +460,11 @@ func truncateHistoryHunYuan(history []structs.Message, prompt string) []structs. i++ } - // 第三步：确保以user结尾，如果不是则尝试移除直到满足条件 - if len(history) > 0 && history[len(history)-1].Role == "assistant" { - // 尝试找到最近的"user"消息并截断至该点 + // 第三步：确保以assistant结尾，如果不是则尝试移除直到满足条件 + if len(history) > 0 && history[len(history)-1].Role == "user" { + // 尝试找到最近的"assistant"消息并截断至该点 for i := len(history) - 2; i >= 0; i-- { - if history[i].Role == "user" { + if history[i].Role == "assistant" { history = history[:i+1] break } From 971fd4104f1104b667b7c197c8d72f9e20c8b341 Mon Sep 17 00:00:00 2001 From: SanaeFox <36219542+Hoshinonyaruko@users.noreply.github.com> Date: Tue, 9 Apr 2024 02:24:38 +0800 Subject: [PATCH 6/6] Beta52 (#53) * beta1 * beta2 * beta3 * beta4 * beta5 * beta6 * beta7 * beta8 * beta9 * beta10 * beta11 * beta12 * beta13 * beta14 * beta15 * beta16 * beta16 * beta19 * beta20 * beta21 * beta22 * beta23 * beta24 * beta25 * beta27 * beta28 * beta29 * beta30 * beta31 * beta33 * beta34 * beta35 * beta36 * beta37 * beta38 * beta39 * beta40 * beta41 * beta42 * beta43 * beta44 * beta45 * beta45 * beta46 * beat48 * beta49 * beta50 * beta51 * beta52 --- .gitignore | 1 + applogic/gensokyo.go | 17 +++++++---- utils/utils.go | 68 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 78 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 8ef422f..b705a09 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.yml *.sqlite *.txt +*.7z # Go specific *.exe diff --git a/applogic/gensokyo.go b/applogic/gensokyo.go index 1fc3839..a901c3d 100644 --- a/applogic/gensokyo.go +++ b/applogic/gensokyo.go @@ -339,6 +339,7 @@ func (app *App) GensokyoHandler(w http.ResponseWriter, r *http.Request) { w.Write([]byte("Error handling user context")) return } + // 构建并发送请求到conversation接口 port := config.GetPort() portStr := fmtf.Sprintf(":%d", port) @@ -346,6 +347,11 @@ func (app *App) GensokyoHandler(w http.ResponseWriter, r *http.Request) { // 请求模型还是使用原文请求 requestmsg := message.Message.(string) + + if config.GetPrintHanming() { + fmtf.Printf("消息进入替换前:%v", requestmsg) + } + // 替换in替换词规则 if config.GetSensitiveMode() { requestmsg = acnode.CheckWordIN(requestmsg) @@ -359,6 +365,7 @@ func (app *App) GensokyoHandler(w http.ResponseWriter, r *http.Request) { "parentMessageId": parentMessageID, "user_id": message.UserID, }) + if err != nil { fmtf.Printf("Error marshalling request: %v\n", err) return @@ -421,7 +428,7 @@ func (app *App) GensokyoHandler(w http.ResponseWriter, r *http.Request) { } else { //最后一条了 messageSSE := structs.InterfaceBody{ - Content: newPart, + Content: newPart + "\n", State: 11, } utils.SendPrivateMessageSSE(message.UserID, messageSSE) @@ -442,7 +449,7 @@ func (app *App) GensokyoHandler(w http.ResponseWriter, r *http.Request) { } else { //最后一条了 messageSSE := structs.InterfaceBody{ - Content: response, + Content: response + "\n", State: 11, } utils.SendPrivateMessageSSE(message.UserID, messageSSE) @@ -484,7 +491,7 @@ func (app *App) GensokyoHandler(w http.ResponseWriter, r *http.Request) { promptkeyboard := config.GetPromptkeyboard() //最后一条了 messageSSE := structs.InterfaceBody{ - Content: " ", + Content: " " + "\n", State: 20, PromptKeyboard: promptkeyboard, } @@ -595,7 +602,7 @@ func processMessage(response string, msg structs.OnebotGroupMessage, newmesssage //CallbackData := GetStringById(lastMessageID) uerid := strconv.FormatInt(msg.UserID, 10) messageSSE := structs.InterfaceBody{ - Content: accumulatedMessage, + Content: accumulatedMessage + "\n", State: 1, ActionButton: 10, CallbackData: uerid, @@ -604,7 +611,7 @@ func processMessage(response string, msg structs.OnebotGroupMessage, newmesssage } else { //SSE的前半部分 messageSSE := structs.InterfaceBody{ - Content: accumulatedMessage, + Content: accumulatedMessage + "\n", State: 1, } utils.SendPrivateMessageSSE(msg.UserID, messageSSE) diff --git a/utils/utils.go b/utils/utils.go index a9b98b7..7c53197 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -251,12 +251,21 @@ func SendPrivateMessageSSE(UserID int64, message structs.InterfaceBody) error { // 构建完整的URL url := baseURL + "/send_private_msg_sse" + // 调试用的 + if config.GetPrintHanming() { + fmtf.Printf("流式信息替换前:%v", message.Content) + } // 检查是否需要启用敏感词过滤 if config.GetSensitiveModeType() == 1 && message.Content != "" { message.Content = acnode.CheckWordOUT(message.Content) } + // 调试用的 + if config.GetPrintHanming() { + fmtf.Printf("流式信息替换后:%v", message.Content) + } + // 构造请求体，包括InterfaceBody requestBody, err := json.Marshal(map[string]interface{}{ "user_id": UserID, @@ -433,6 +442,7 @@ func SendSSEPrivateMessage(userID int64, content string) { } messageSSE.PromptKeyboard = promptKeyboard + messageSSE.Content = messageSSE.Content + "\n" } // 发送SSE消息函数 @@ -497,9 +507,61 @@ func SendSSEPrivateSafeMessage(userID int64, saveresponse string) { // 创建InterfaceBody结构体实例 messageSSE = structs.InterfaceBody{ - Content: parts[2], // 假设空格字符串是期望的内容 - State: 20, // 假设的状态码 - PromptKeyboard: promptkeyboard, // 使用更新后的promptkeyboard + Content: parts[2] + "\n", // 假设空格字符串是期望的内容 + State: 20, // 假设的状态码 + PromptKeyboard: promptkeyboard, // 使用更新后的promptkeyboard + } + + // 发送SSE私人消息 + SendPrivateMessageSSE(userID, messageSSE) +} + +// SendSSEPrivateRestoreMessage 分割并发送重置消息的核心逻辑，直接遍历字符串 +func SendSSEPrivateRestoreMessage(userID int64, RestoreResponse string) { + // 将字符串转换为rune切片，以正确处理多字节字符 + runes := []rune(RestoreResponse) + + // 计算每部分应该包含的rune数量 + partLength := len(runes) / 3 + + // 初始化用于存储分割结果的切片 + parts := make([]string, 3) + + // 按字符分割字符串 + for i := 0; i < 3; i++ { + if i < 2 { // 前两部分 + start := i * partLength + end := start + partLength + parts[i] = string(runes[start:end]) + } else { // 最后一部分，包含所有剩余的字符 + start := i * partLength + parts[i] = string(runes[start:]) + } + } + + // 开头 + messageSSE := structs.InterfaceBody{ + Content: parts[0], + State: 1, + } + + SendPrivateMessageSSE(userID, messageSSE) + + //中间 + messageSSE = structs.InterfaceBody{ + Content: parts[1], + State: 11, + } + SendPrivateMessageSSE(userID, messageSSE) + + // 从配置中获取promptkeyboard + promptkeyboard := config.GetPromptkeyboard() + + // 创建InterfaceBody结构体实例 + messageSSE = structs.InterfaceBody{ + Content: parts[2] + "\n", // 假设空格字符串是期望的内容 + State: 20, // 假设的状态码 + PromptKeyboard: promptkeyboard, // 使用更新后的promptkeyboard } // 发送SSE私人消息