diff --git a/client/base.go b/client/base.go index e6b63386..59f29e9a 100644 --- a/client/base.go +++ b/client/base.go @@ -7,6 +7,7 @@ import ( "time" "github.com/LagrangeDev/LagrangeGo/cache" + "github.com/LagrangeDev/LagrangeGo/client/highway" "github.com/LagrangeDev/LagrangeGo/event" @@ -20,8 +21,8 @@ import ( const msfwifiServer = "msfwifi.3g.qq.com:8080" -// NewQQclient 创建一个新的QQClient -func NewQQclient(uin uint32, signUrl string, appInfo *info.AppInfo, deviceInfo *info.DeviceInfo, sig *info.SigInfo) *QQClient { +// NewQQClient 创建一个新的QQClient +func NewQQClient(uin uint32, signUrl string, appInfo *info.AppInfo, deviceInfo *info.DeviceInfo, sig *info.SigInfo) *QQClient { client := &QQClient{ Uin: uin, appInfo: appInfo, @@ -31,9 +32,12 @@ func NewQQclient(uin uint32, signUrl string, appInfo *info.AppInfo, deviceInfo * // 128应该够用了吧 pushStore: make(chan *wtlogin.SSOPacket, 128), stopChan: make(chan struct{}), - tcp: &TCPClient{}, - cache: &cache.Cache{}, + highwaySession: highway.Session{ + AppID: uint32(appInfo.AppID), + SubAppID: uint32(appInfo.SubAppID), + }, } + client.highwaySession.Uin = &client.sig.Uin client.Online.Store(false) return client } @@ -53,13 +57,10 @@ type QQClient struct { t106 []byte t16a []byte - tcp *TCPClient + tcp TCPClient + highwaySession highway.Session - cache *cache.Cache - - highwayUri map[uint32][]string - highwaySequence atomic.Uint32 - sigSession []byte + cache cache.Cache GroupMessageEvent EventHandle[*message.GroupMessage] PrivateMessageEvent EventHandle[*message.PrivateMessage] diff --git a/client/highway.go b/client/highway.go index febf8a30..f7c6b585 100644 --- a/client/highway.go +++ b/client/highway.go @@ -10,6 +10,7 @@ import ( "net/url" "strconv" + hw "github.com/LagrangeDev/LagrangeGo/client/highway" highway2 "github.com/LagrangeDev/LagrangeGo/packets/highway" "github.com/LagrangeDev/LagrangeGo/packets/pb/service/highway" "github.com/LagrangeDev/LagrangeGo/utils/binary" @@ -17,29 +18,8 @@ import ( "github.com/RomiChan/protobuf/proto" ) -const ( - uploadBlockSize = 1024 * 1024 - httpServiceType uint32 = 1 -) - -type UpBlock struct { - CommandId int - Uin uint - Sequence uint - FileSize uint64 - Offset uint64 - Ticket []byte - FileMd5 []byte - Block io.Reader - BlockMd5 []byte - BlockSize uint32 - ExtendInfo []byte - Timestamp uint64 -} - func (c *QQClient) EnsureHighwayServers() error { - if c.highwayUri == nil || c.sigSession == nil { - c.highwayUri = make(map[uint32][]string) + if c.highwaySession.SsoAddr == nil || c.highwaySession.SigSession == nil || c.highwaySession.SessionKey == nil { packet, err := highway2.BuildHighWayUrlReq(c.sig.Tgt) if err != nil { return err @@ -52,20 +32,19 @@ func (c *QQClient) EnsureHighwayServers() error { if err != nil { return fmt.Errorf("parse highway server: %v", err) } + c.highwaySession.SigSession = resp.HttpConn.SigSession + c.highwaySession.SessionKey = resp.HttpConn.SessionKey for _, info := range resp.HttpConn.ServerInfos { - servicetype := info.ServiceType + if info.ServiceType != 1 { + continue + } for _, addr := range info.ServerAddrs { - service := c.highwayUri[servicetype] - service = append(service, fmt.Sprintf( - "http://%s:%d/cgi-bin/httpconn?htcmd=0x6FF0087&uin=%d", - le32toipstr(addr.IP), addr.Port, c.sig.Uin, - )) - c.highwayUri[servicetype] = service + networkLogger.Debugln("add highway server", binary.UInt32ToIPV4Address(addr.IP), "port", addr.Port) + c.highwaySession.AppendAddr(addr.IP, addr.Port) } } - c.sigSession = resp.HttpConn.SigSession } - if c.highwayUri == nil || c.sigSession == nil { + if c.highwaySession.SsoAddr == nil || c.highwaySession.SigSession == nil || c.highwaySession.SessionKey == nil { return errors.New("empty highway servers") } return nil @@ -76,30 +55,36 @@ func (c *QQClient) UploadSrcByStream(commonId int, r io.Reader, fileSize uint64, if err != nil { return err } - servers := c.highwayUri[httpServiceType] - server := servers[rand.Intn(len(servers))] - buffer := make([]byte, uploadBlockSize) - for offset := uint64(0); offset < fileSize; offset += uploadBlockSize { - if uploadBlockSize > fileSize-offset { + trans := &hw.Transaction{ + CommandID: uint32(commonId), + Body: r, + Sum: md5, + Size: fileSize, + Ticket: c.highwaySession.SigSession, + LoginSig: c.sig.Tgt, + Ext: extendInfo, + } + _, err = c.highwaySession.Upload(trans) + if err == nil { + return nil + } + // fallback to http upload + servers := c.highwaySession.SsoAddr + saddr := servers[rand.Intn(len(servers))] + server := fmt.Sprintf( + "http://%s:%d/cgi-bin/httpconn?htcmd=0x6FF0087&uin=%d", + binary.UInt32ToIPV4Address(saddr.IP), saddr.Port, c.sig.Uin, + ) + buffer := make([]byte, hw.BlockSize) + for offset := uint64(0); offset < fileSize; offset += hw.BlockSize { + if hw.BlockSize > fileSize-offset { buffer = buffer[:fileSize-offset] } _, err := io.ReadFull(r, buffer) if err != nil { return err } - err = c.SendUpBlock(&UpBlock{ - CommandId: commonId, - Uin: uint(c.sig.Uin), - Sequence: uint(c.highwaySequence.Add(1)), - FileSize: fileSize, - Offset: offset, - Ticket: c.sigSession, - FileMd5: md5, - Block: bytes.NewReader(buffer), - BlockMd5: crypto.MD5Digest(buffer), - BlockSize: uint32(len(buffer)), - ExtendInfo: extendInfo, - }, server) + err = c.SendUpBlock(trans, server, offset, crypto.MD5Digest(buffer), buffer) if err != nil { return err } @@ -107,43 +92,12 @@ func (c *QQClient) UploadSrcByStream(commonId int, r io.Reader, fileSize uint64, return nil } -func (c *QQClient) SendUpBlock(block *UpBlock, server string) error { - head := &highway.DataHighwayHead{ - Version: 1, - Uin: proto.Some(strconv.Itoa(int(block.Uin))), - Command: proto.Some("PicUp.DataUp"), - Seq: proto.Some(uint32(block.Sequence)), - RetryTimes: proto.Some(uint32(0)), - AppId: uint32(c.appInfo.SubAppID), - DataFlag: 16, - CommandId: uint32(block.CommandId), - } - segHead := &highway.SegHead{ - ServiceId: proto.Some(uint32(0)), - Filesize: block.FileSize, - DataOffset: proto.Some(block.Offset), - DataLength: uint32(block.BlockSize), - RetCode: proto.Some(uint32(0)), - ServiceTicket: block.Ticket, - Md5: block.BlockMd5, - FileMd5: block.FileMd5, - CacheAddr: proto.Some(uint32(0)), - CachePort: proto.Some(uint32(0)), - } - loginHead := &highway.LoginSigHead{ - Uint32LoginSigType: 8, - BytesLoginSig: c.sig.Tgt, - AppId: uint32(c.appInfo.AppID), - } - highwayHead := &highway.ReqDataHighwayHead{ - MsgBaseHead: head, - MsgSegHead: segHead, - BytesReqExtendInfo: block.ExtendInfo, - Timestamp: block.Timestamp, - MsgLoginSigHead: loginHead, - } - isEnd := block.Offset+uint64(block.BlockSize) == block.FileSize - payload, err := sendHighwayPacket(highwayHead, block.Block, block.BlockSize, server, isEnd) +func (c *QQClient) SendUpBlock(trans *hw.Transaction, server string, offset uint64, blkmd5 []byte, blk []byte) error { + blksz := uint64(len(blk)) + isEnd := offset+blksz == trans.Size + payload, err := sendHighwayPacket( + trans.Build(&c.highwaySession, offset, uint32(blksz), blkmd5), blk, server, isEnd, + ) if err != nil { return fmt.Errorf("send highway packet: %v", err) } @@ -179,21 +133,27 @@ func parseHighwayPacket(data io.Reader) (head *highway.RespDataHighwayHead, body return head, reader, nil } -func sendHighwayPacket(packet *highway.ReqDataHighwayHead, buffer io.Reader, bufferSize uint32, serverURL string, end bool) (io.ReadCloser, error) { +func sendHighwayPacket(packet *highway.ReqDataHighwayHead, block []byte, serverURL string, end bool) (io.ReadCloser, error) { marshal, err := proto.Marshal(packet) if err != nil { return nil, err } - - writer := binary.NewBuilder(nil). - WriteBytes([]byte{0x28}). - WriteU32(uint32(len(marshal))). - WriteU32(bufferSize). - WriteBytes(marshal) - _, _ = io.Copy(writer, buffer) - _, _ = writer.Write([]byte{0x29}) - - return postHighwayContent(writer.ToReader(), serverURL, end) + buf := hw.Frame(marshal, block) + data, err := io.ReadAll(&buf) + if err != nil { + return nil, err + } + return postHighwayContent(bytes.NewReader(data), serverURL, end) + /* + return postHighwayContent( + binary.NewBuilder(nil). + WriteBytes([]byte{0x28}). + WriteU32(uint32(len(marshal))). + WriteU32(uint32(len(block))). + WriteBytes(marshal). + WriteBytes(block). + WriteBytes([]byte{0x29}).ToReader(), serverURL, end) + */ } func postHighwayContent(content io.Reader, serverURL string, end bool) (io.ReadCloser, error) { diff --git a/client/highway/bdh.go b/client/highway/bdh.go new file mode 100644 index 00000000..5688c824 --- /dev/null +++ b/client/highway/bdh.go @@ -0,0 +1,223 @@ +package highway + +// from https://github.com/Mrs4s/MiraiGo/tree/master/client/internal/highway/bdh.go + +import ( + "crypto/md5" + "io" + "strconv" + "sync" + "sync/atomic" + + ftea "github.com/fumiama/gofastTEA" + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" + + "github.com/LagrangeDev/LagrangeGo/packets/pb/service/highway" + "github.com/LagrangeDev/LagrangeGo/utils/binary" + "github.com/LagrangeDev/LagrangeGo/utils/proto" +) + +const BlockSize = 256 * 1024 + +type Transaction struct { + CommandID uint32 + Body io.Reader + Sum []byte // md5 sum of body + Size uint64 // body size + Ticket []byte + LoginSig []byte + Ext []byte + Encrypt bool +} + +func (trans *Transaction) encrypt(key []byte) error { + if !trans.Encrypt { + return nil + } + if len(key) == 0 { + return errors.New("session key not found. maybe miss some packet?") + } + trans.Ext = ftea.NewTeaCipher(key).Encrypt(trans.Ext) + return nil +} + +func (trans *Transaction) Build(s *Session, offset uint64, length uint32, md5hash []byte) *highway.ReqDataHighwayHead { + return &highway.ReqDataHighwayHead{ + MsgBaseHead: &highway.DataHighwayHead{ + Version: 1, + Uin: proto.Some(strconv.Itoa(int(*s.Uin))), + Command: proto.Some(_REQ_CMD_DATA), + Seq: proto.Some(s.NextSeq()), + RetryTimes: proto.Some(uint32(0)), + AppId: s.SubAppID, + DataFlag: 16, + CommandId: trans.CommandID, + // LocaleId: 2052, + }, + MsgSegHead: &highway.SegHead{ + ServiceId: proto.Some(uint32(0)), + Filesize: trans.Size, + DataOffset: proto.Some(offset), + DataLength: length, + RetCode: proto.Some(uint32(0)), + ServiceTicket: trans.Ticket, + Md5: md5hash, + FileMd5: trans.Sum, + CacheAddr: proto.Some(uint32(0)), + CachePort: proto.Some(uint32(0)), + }, + BytesReqExtendInfo: trans.Ext, + MsgLoginSigHead: &highway.LoginSigHead{ + Uint32LoginSigType: 8, + BytesLoginSig: trans.LoginSig, + AppId: s.AppID, + }, + } +} + +func (s *Session) uploadSingle(trans *Transaction) ([]byte, error) { + pc, err := s.selectConn() + if err != nil { + return nil, err + } + defer s.putIdleConn(pc) + + reader := binary.NewNetworkReader(pc.conn) + var rspExt []byte + offset := 0 + chunk := make([]byte, BlockSize) + for { + chunk = chunk[:cap(chunk)] + rl, err := io.ReadFull(trans.Body, chunk) + if rl == 0 { + break + } + if errors.Is(err, io.ErrUnexpectedEOF) { + chunk = chunk[:rl] + } + ch := md5.Sum(chunk) + head, _ := proto.Marshal(trans.Build(s, uint64(offset), uint32(rl), ch[:])) + offset += rl + buffers := Frame(head, chunk) + _, err = buffers.WriteTo(pc.conn) + if err != nil { + return nil, errors.Wrap(err, "write conn error") + } + rspHead, err := readResponse(reader) + if err != nil { + return nil, errors.Wrap(err, "highway upload error") + } + if rspHead.ErrorCode != 0 { + return nil, errors.Errorf("upload failed: %d", rspHead.ErrorCode) + } + if rspHead.BytesRspExtendInfo != nil { + rspExt = rspHead.BytesRspExtendInfo + } + if rspHead.MsgSegHead != nil && rspHead.MsgSegHead.ServiceTicket != nil { + trans.Ticket = rspHead.MsgSegHead.ServiceTicket + } + } + return rspExt, nil +} + +func (s *Session) Upload(trans *Transaction) ([]byte, error) { + // encrypt ext data + if err := trans.encrypt(s.SessionKey); err != nil { + return nil, err + } + + const maxThreadCount = 4 + threadCount := int(trans.Size) / (6 * BlockSize) // 1 thread upload 1.5 MB + if threadCount > maxThreadCount { + threadCount = maxThreadCount + } + if threadCount < 2 { + // single thread upload + return s.uploadSingle(trans) + } + + // pick a address + // TODO: pick smarter + pc, err := s.selectConn() + if err != nil { + return nil, err + } + addr := pc.addr + s.putIdleConn(pc) + + var ( + rspExt []byte + completedThread uint32 + cond = sync.NewCond(&sync.Mutex{}) + offset = uint64(0) + count = (trans.Size + BlockSize - 1) / BlockSize + id = 0 + ) + doUpload := func() error { + // send signal complete uploading + defer func() { + atomic.AddUint32(&completedThread, 1) + cond.Signal() + }() + + // todo: get from pool? + pc, err := s.connect(addr) + if err != nil { + return err + } + defer s.putIdleConn(pc) + + reader := binary.NewNetworkReader(pc.conn) + chunk := make([]byte, BlockSize) + for { + cond.L.Lock() // lock protect reading + off := offset + offset += BlockSize + id++ + last := uint64(id) == count + if last { // last + for atomic.LoadUint32(&completedThread) != uint32(threadCount-1) { + cond.Wait() + } + } else if uint64(id) > count { + cond.L.Unlock() + break + } + chunk = chunk[:BlockSize] + n, err := io.ReadFull(trans.Body, chunk) + cond.L.Unlock() + + if n == 0 { + break + } + if errors.Is(err, io.ErrUnexpectedEOF) { + chunk = chunk[:n] + } + ch := md5.Sum(chunk) + head, _ := proto.Marshal(trans.Build(s, off, uint32(n), ch[:])) + buffers := Frame(head, chunk) + _, err = buffers.WriteTo(pc.conn) + if err != nil { + return errors.Wrap(err, "write conn error") + } + rspHead, err := readResponse(reader) + if err != nil { + return errors.Wrap(err, "highway upload error") + } + if rspHead.ErrorCode != 0 { + return errors.Errorf("upload failed: %d", rspHead.ErrorCode) + } + if last && rspHead.BytesRspExtendInfo != nil { + rspExt = rspHead.BytesRspExtendInfo + } + } + return nil + } + + group := errgroup.Group{} + for i := 0; i < threadCount; i++ { + group.Go(doUpload) + } + return rspExt, group.Wait() +} diff --git a/client/highway/frame.go b/client/highway/frame.go new file mode 100644 index 00000000..ea812c9a --- /dev/null +++ b/client/highway/frame.go @@ -0,0 +1,37 @@ +package highway + +// from https://github.com/Mrs4s/MiraiGo/tree/master/client/internal/highway/frame.go + +import ( + "encoding/binary" + "net" +) + +var etx = []byte{0x29} + +// Frame 包格式 +// +// - STX: 0x28(40) +// - head length +// - body length +// - head data +// - body data +// - ETX: 0x29(41) +// +// 节省内存, 可被go runtime优化为writev操作 +func Frame(head []byte, body []byte) net.Buffers { + buffers := make(net.Buffers, 4) + // buffer0 format: + // - STX + // - head length + // - body length + buffer0 := make([]byte, 9) + buffer0[0] = 0x28 + binary.BigEndian.PutUint32(buffer0[1:], uint32(len(head))) + binary.BigEndian.PutUint32(buffer0[5:], uint32(len(body))) + buffers[0] = buffer0 + buffers[1] = head + buffers[2] = body + buffers[3] = etx + return buffers +} diff --git a/client/highway/highway.go b/client/highway/highway.go new file mode 100644 index 00000000..2fe5c6ec --- /dev/null +++ b/client/highway/highway.go @@ -0,0 +1,263 @@ +package highway + +// from https://github.com/Mrs4s/MiraiGo/tree/master/client/internal/highway/highway.go + +import ( + "fmt" + "net" + "runtime" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/pkg/errors" + + "github.com/LagrangeDev/LagrangeGo/packets/pb/service/highway" + "github.com/LagrangeDev/LagrangeGo/utils/binary" + "github.com/LagrangeDev/LagrangeGo/utils/proto" +) + +// see com/tencent/mobileqq/highway/utils/BaseConstants.java#L120-L121 +const ( + _REQ_CMD_DATA = "PicUp.DataUp" + _REQ_CMD_HEART_BREAK = "PicUp.Echo" +) + +type Addr struct { + IP uint32 + Port int +} + +func (a Addr) AsNetIP() net.IP { + return net.IPv4(byte(a.IP>>24), byte(a.IP>>16), byte(a.IP>>8), byte(a.IP)) +} + +func (a Addr) String() string { + return fmt.Sprintf("%v:%v", binary.UInt32ToIPV4Address(a.IP), a.Port) +} + +func (a Addr) empty() bool { + return a.IP == 0 || a.Port == 0 +} + +type Session struct { + Uin *uint32 + AppID uint32 + SubAppID uint32 + SigSession []byte + SessionKey []byte + + seq uint32 + + addrMu sync.Mutex + idx int + SsoAddr []Addr + + idleMu sync.Mutex + idleCount int + idle *idle +} + +const highwayMaxResponseSize int32 = 1024 * 100 // 100k + +func (s *Session) AddrLength() int { + s.addrMu.Lock() + defer s.addrMu.Unlock() + return len(s.SsoAddr) +} + +func (s *Session) AppendAddr(ip, port uint32) { + s.addrMu.Lock() + defer s.addrMu.Unlock() + addr := Addr{ + IP: ip, + Port: int(port), + } + s.SsoAddr = append(s.SsoAddr, addr) +} + +func (s *Session) NextSeq() uint32 { + return atomic.AddUint32(&s.seq, 2) +} + +func (s *Session) sendHeartbreak(conn net.Conn) error { + head, _ := proto.Marshal(&highway.ReqDataHighwayHead{ + MsgBaseHead: &highway.DataHighwayHead{ + Version: 1, + Uin: proto.Some(strconv.Itoa(int(*s.Uin))), + Command: proto.Some(_REQ_CMD_HEART_BREAK), + Seq: proto.Some(s.NextSeq()), + AppId: s.SubAppID, + DataFlag: 16, + CommandId: 0, + // LocaleId: 2052, + }, + }) + buffers := Frame(head, nil) + _, err := buffers.WriteTo(conn) + return err +} + +func (s *Session) ping(pc *persistConn) error { + start := time.Now() + err := s.sendHeartbreak(pc.conn) + if err != nil { + return errors.Wrap(err, "echo error") + } + if _, err = readResponse(binary.NewNetworkReader(pc.conn)); err != nil { + return errors.Wrap(err, "echo error") + } + // update delay + pc.ping = time.Since(start).Milliseconds() + return nil +} + +func readResponse(r *binary.NetworkReader) (*highway.RespDataHighwayHead, error) { + _, err := r.ReadByte() + if err != nil { + return nil, errors.Wrap(err, "failed to read byte") + } + hl, _ := r.ReadInt32() + a2, _ := r.ReadInt32() + if hl > highwayMaxResponseSize || a2 > highwayMaxResponseSize { + return nil, errors.Errorf("highway response invild. head size: %v body size: %v", hl, a2) + } + head, _ := r.ReadBytes(int(hl)) + _, _ = r.ReadBytes(int(a2)) // skip payload + _, _ = r.ReadByte() + rsp := new(highway.RespDataHighwayHead) + if err = proto.Unmarshal(head, rsp); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal protobuf message") + } + return rsp, nil +} + +type persistConn struct { + conn net.Conn + addr Addr + ping int64 // echo ping +} + +const maxIdleConn = 7 + +type idle struct { + pc persistConn + next *idle +} + +// getIdleConn ... +func (s *Session) getIdleConn() persistConn { + s.idleMu.Lock() + defer s.idleMu.Unlock() + + // no idle + if s.idle == nil { + return persistConn{} + } + + // switch the fastest idle conn + conn := s.idle.pc + s.idle = s.idle.next + s.idleCount-- + if s.idleCount < 0 { + panic("idle count underflow") + } + + return conn +} + +func (s *Session) putIdleConn(pc persistConn) { + s.idleMu.Lock() + defer s.idleMu.Unlock() + + // check persistConn + if pc.conn == nil || pc.addr.empty() { + panic("put bad idle conn") + } + + cur := &idle{pc: pc} + s.idleCount++ + if s.idle == nil { // quick path + s.idle = cur + return + } + + // insert between pre and succ + var pre, succ *idle + succ = s.idle + for succ != nil && succ.pc.ping < pc.ping { // keep idle list sorted by delay incremental + pre = succ + succ = succ.next + } + if pre != nil { + pre.next = cur + } + cur.next = succ + + // remove the slowest idle conn if idle count greater than maxIdleConn + if s.idleCount > maxIdleConn { + for cur.next != nil { + pre = cur + cur = cur.next + } + pre.next = nil + s.idleCount-- + } +} + +func (s *Session) connect(addr Addr) (persistConn, error) { + conn, err := net.DialTimeout("tcp", addr.String(), time.Second*3) + if err != nil { + return persistConn{}, err + } + _ = conn.(*net.TCPConn).SetKeepAlive(true) + + // close conn + runtime.SetFinalizer(conn, func(conn net.Conn) { + _ = conn.Close() + }) + + pc := persistConn{conn: conn, addr: addr} + if err = s.ping(&pc); err != nil { + return persistConn{}, err + } + return pc, nil +} + +func (s *Session) nextAddr() Addr { + s.addrMu.Lock() + defer s.addrMu.Unlock() + addr := s.SsoAddr[s.idx] + s.idx = (s.idx + 1) % len(s.SsoAddr) + return addr +} + +func (s *Session) selectConn() (pc persistConn, err error) { + for { // select from idle pc + pc = s.getIdleConn() + if pc.conn == nil { + // no idle connection + break + } + + err = s.ping(&pc) // ping + if err == nil { + return + } + } + + try := 0 + for { + addr := s.nextAddr() + pc, err = s.connect(addr) + if err == nil { + break + } + try++ + if try > 5 { + break + } + } + return +} diff --git a/client/http2/client.go b/client/http2/client.go new file mode 100644 index 00000000..b5ac8075 --- /dev/null +++ b/client/http2/client.go @@ -0,0 +1,44 @@ +package http2 + +import ( + "errors" + "io" + "net" + "net/http" + "net/url" + "time" + + "golang.org/x/net/http2" +) + +var ( + ErrEmptyHostAddress = errors.New("empty host addr") +) + +var defaultDialer = net.Dialer{ + Timeout: time.Minute, +} + +func SetDefaultClientTimeout(t time.Duration) { + defaultDialer.Timeout = t +} + +var DefaultClient = http.Client{ + Transport: &http2.Transport{}, +} + +func Get(url string) (resp *http.Response, err error) { + return DefaultClient.Get(url) +} + +func Head(url string) (resp *http.Response, err error) { + return DefaultClient.Head(url) +} + +func Post(url string, contentType string, body io.Reader) (resp *http.Response, err error) { + return DefaultClient.Post(url, contentType, body) +} + +func PostForm(url string, data url.Values) (resp *http.Response, err error) { + return DefaultClient.PostForm(url, data) +} diff --git a/client/listener.go b/client/listener.go index f87bae4e..7abc6fa8 100644 --- a/client/listener.go +++ b/client/listener.go @@ -97,7 +97,7 @@ func decodeOlPushServicePacket(c *QQClient, pkt *wtlogin.SSOPacket) (any, error) if err != nil { return nil, err } - return eventConverter.ParseFriendRequestNotice(&msg, &pb), nil + return eventConverter.ParseFriendRequestNotice(&pb, &msg), nil case 138: // friend recall pb := message.FriendRecall{} err = proto.Unmarshal(pkg.Body.MsgContent, &pb) @@ -112,7 +112,7 @@ func decodeOlPushServicePacket(c *QQClient, pkt *wtlogin.SSOPacket) (any, error) if err != nil { return nil, err } - return eventConverter.ParseFriendRenameEvent(&pb, c.cache), nil + return eventConverter.ParseFriendRenameEvent(&pb, c.cache.GetUin(pb.Body.Data.Uid)), nil case 29: networkLogger.Debugln("self rename") pb := message.SelfRenameMsg{} diff --git a/client/richmedia.go b/client/richmedia.go index 63044198..b5764087 100644 --- a/client/richmedia.go +++ b/client/richmedia.go @@ -7,6 +7,7 @@ import ( "errors" "net/netip" + highway2 "github.com/LagrangeDev/LagrangeGo/client/highway" "github.com/LagrangeDev/LagrangeGo/message" "github.com/LagrangeDev/LagrangeGo/packets/oidb" message2 "github.com/LagrangeDev/LagrangeGo/packets/pb/message" @@ -52,8 +53,9 @@ func (c *QQClient) ImageUploadPrivate(targetUid string, element message.IMessage if err != nil { return nil, err } - ukey := uploadResp.Upload.UKey - if ukey.Unwrap() != "" { + ukey := uploadResp.Upload.UKey.Unwrap() + networkLogger.Debugln("private image upload ukey:", ukey) + if ukey != "" { index := uploadResp.Upload.MsgInfo.MsgInfoBody[0].Index sha1hash, err := hex.DecodeString(index.Info.FileSha1) if err != nil { @@ -61,12 +63,12 @@ func (c *QQClient) ImageUploadPrivate(targetUid string, element message.IMessage } extend := &highway.NTV2RichMediaHighwayExt{ FileUuid: index.FileUuid, - UKey: ukey.Unwrap(), + UKey: ukey, Network: &highway.NTHighwayNetwork{ IPv4S: ConvertNTHighwayNetWork(uploadResp.Upload.IPv4S), }, MsgInfoBody: uploadResp.Upload.MsgInfo.MsgInfoBody, - BlockSize: 1024 * 1024, + BlockSize: uint32(highway2.BlockSize), Hash: &highway.NTHighwayHash{ FileSha1: [][]byte{sha1hash}, }, @@ -114,8 +116,9 @@ func (c *QQClient) ImageUploadGroup(groupUin uint32, element message.IMessageEle if err != nil { return nil, err } - ukey := uploadResp.Upload.UKey - if ukey.Unwrap() != "" { + ukey := uploadResp.Upload.UKey.Unwrap() + networkLogger.Debugln("private image upload ukey:", ukey) + if ukey != "" { index := uploadResp.Upload.MsgInfo.MsgInfoBody[0].Index sha1hash, err := hex.DecodeString(index.Info.FileSha1) if err != nil { @@ -123,12 +126,12 @@ func (c *QQClient) ImageUploadGroup(groupUin uint32, element message.IMessageEle } extend := &highway.NTV2RichMediaHighwayExt{ FileUuid: index.FileUuid, - UKey: ukey.Unwrap(), + UKey: ukey, Network: &highway.NTHighwayNetwork{ IPv4S: ConvertNTHighwayNetWork(uploadResp.Upload.IPv4S), }, MsgInfoBody: uploadResp.Upload.MsgInfo.MsgInfoBody, - BlockSize: 1024 * 1024, + BlockSize: uint32(highway2.BlockSize), Hash: &highway.NTHighwayHash{ FileSha1: [][]byte{sha1hash}, }, diff --git a/entity/groupMember.go b/entity/groupMember.go index b64cea62..119c02d7 100644 --- a/entity/groupMember.go +++ b/entity/groupMember.go @@ -4,7 +4,7 @@ import ( "fmt" ) -type GroupMemberPermission uint +type GroupMemberPermission uint32 const ( Member GroupMemberPermission = iota @@ -24,17 +24,17 @@ type GroupMember struct { Avatar string } -func NewGroupMember(uin uint32, uid string, permission GroupMemberPermission, GroupLevel uint32, MemberCard, - MemberName string, JoinTime, LastMsgTime uint32) *GroupMember { +func NewGroupMember(uin uint32, uid string, permission GroupMemberPermission, groupLevel uint32, + memberCard, memberName string, joinTime, lastMsgTime uint32) *GroupMember { return &GroupMember{ Uin: uin, Uid: uid, Permission: permission, - GroupLevel: GroupLevel, - MemberCard: MemberCard, - MemberName: MemberName, - JoinTime: JoinTime, - LastMsgTime: LastMsgTime, + GroupLevel: groupLevel, + MemberCard: memberCard, + MemberName: memberName, + JoinTime: joinTime, + LastMsgTime: lastMsgTime, Avatar: fmt.Sprintf("https://q1.qlogo.cn/g?b=qq&nk=%v&s=640", uin), } } diff --git a/event/friend.go b/event/friend.go index c5168cef..35764321 100644 --- a/event/friend.go +++ b/event/friend.go @@ -1,7 +1,6 @@ package event import ( - "github.com/LagrangeDev/LagrangeGo/cache" "github.com/LagrangeDev/LagrangeGo/packets/pb/message" ) @@ -27,7 +26,7 @@ type ( } ) -func ParseFriendRequestNotice(msg *message.PushMsg, event *message.FriendRequest) *FriendRequest { +func ParseFriendRequestNotice(event *message.FriendRequest, msg *message.PushMsg) *FriendRequest { info := event.Info return &FriendRequest{ SourceUin: msg.Message.ResponseHead.FromUin, @@ -47,10 +46,10 @@ func ParseFriendRecallEvent(event *message.FriendRecall) *FriendRecall { } } -func ParseFriendRenameEvent(event *message.FriendRenameMsg, cache *cache.Cache) *Rename { +func ParseFriendRenameEvent(event *message.FriendRenameMsg, uin uint32) *Rename { return &Rename{ SubType: 1, - Uin: cache.GetUin(event.Body.Data.Uid), + Uin: uin, Nickname: event.Body.Data.RenameData.NickName, } } diff --git a/go.mod b/go.mod index 3f937fd2..1102841e 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,10 @@ require ( github.com/fumiama/gofastTEA v0.0.10 github.com/fumiama/imgsz v0.0.4 github.com/mattn/go-colorable v0.1.13 + github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 + golang.org/x/net v0.25.0 + golang.org/x/sync v0.7.0 ) require ( @@ -16,4 +19,5 @@ require ( github.com/stretchr/testify v1.8.0 // indirect golang.org/x/image v0.16.0 // indirect golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect ) diff --git a/go.sum b/go.sum index 1188fcb2..8ac2bd20 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= @@ -26,11 +28,17 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= golang.org/x/image v0.16.0 h1:9kloLAKhUufZhA12l5fwnx2NZW39/we1UhBesW433jw= golang.org/x/image v0.16.0/go.mod h1:ugSZItdV4nOxyqp56HmXwH0Ry0nBCpjnZdpDaIHdoPs= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/main.go b/main.go index 1c4fc54b..24dc135d 100644 --- a/main.go +++ b/main.go @@ -25,7 +25,7 @@ func main() { mainLogger.Errorln("load sig error:", err) return } - qqclient := client.NewQQclient(0, "https://sign.lagrangecore.org/api/sign", appInfo, deviceInfo, &sig) + qqclient := client.NewQQClient(0, "https://sign.lagrangecore.org/api/sign", appInfo, deviceInfo, &sig) qqclient.GroupMessageEvent.Subscribe(func(client *client.QQClient, event *message.GroupMessage) { if event.ToString() == "114514" { diff --git a/packets/wtlogin/sso.go b/packets/wtlogin/sso.go index 87fb5b31..fe8566e0 100644 --- a/packets/wtlogin/sso.go +++ b/packets/wtlogin/sso.go @@ -103,7 +103,7 @@ func ParseSSOFrame(buffer []byte, isoicqbody bool) (*SSOPacket, error) { if compressType == 0 { } else if compressType == 1 { - data, _ = utils.DecompressData(data) + data = binary2.ZlibUncompress(data) } else if compressType == 8 { data = data[4:] } else { diff --git a/utils/binary/doc.go b/utils/binary/doc.go deleted file mode 100644 index 6318b9ad..00000000 --- a/utils/binary/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -package binary - -// 看起来很像 https://github.com/Mrs4s/MiraiGo/blob/master/binary下的reader和writer， -// 但是这个真的是没有借鉴MiraiGo diff --git a/utils/binary/network.go b/utils/binary/network.go new file mode 100644 index 00000000..e054ee98 --- /dev/null +++ b/utils/binary/network.go @@ -0,0 +1,44 @@ +package binary + +// from https://github.com/Mrs4s/MiraiGo/blob/master/binary/reader.go + +import ( + "encoding/binary" + "io" + "net" +) + +type NetworkReader struct { + conn net.Conn +} + +func NewNetworkReader(conn net.Conn) *NetworkReader { + return &NetworkReader{conn: conn} +} + +func (r *NetworkReader) ReadByte() (byte, error) { + buf := make([]byte, 1) + n, err := r.conn.Read(buf) + if err != nil { + return 0, err + } + if n != 1 { + return r.ReadByte() + } + return buf[0], nil +} + +func (r *NetworkReader) ReadBytes(len int) ([]byte, error) { + buf := make([]byte, len) + _, err := io.ReadFull(r.conn, buf) + return buf, err +} + +func (r *NetworkReader) ReadInt32() (int32, error) { + b := make([]byte, 4) + _, err := r.conn.Read(b) + if err != nil { + return 0, err + } + return int32(binary.BigEndian.Uint32(b)), nil +} diff --git a/utils/binary/pool.go b/utils/binary/pool.go new file mode 100644 index 00000000..9aa89b1d --- /dev/null +++ b/utils/binary/pool.go @@ -0,0 +1,98 @@ +package binary + +// from https://github.com/Mrs4s/MiraiGo/blob/master/binary/pool.go + +import ( + "bytes" + "compress/gzip" + "compress/zlib" + "sync" + + ftea "github.com/fumiama/gofastTEA" +) + +var bufferPool = sync.Pool{ + New: func() any { + return new(bytes.Buffer) + }, +} + +// SelectBuilder 从池中取出一个 Builder +func SelectBuilder(key []byte) *Builder { + // 因为 bufferPool 定义有 New 函数 + // 所以 bufferPool.Get() 永不为 nil + // 不用判空 + bd := bufferPool.Get().(*Builder) + bd.key = ftea.NewTeaCipher(key) + bd.usetea = len(key) == 16 + return bd +} + +// PutBuilder 将 Builder 放回池中 +func PutBuilder(w *Builder) { + // See https://golang.org/issue/23199 + const maxSize = 32 * 1024 + if w.buffer.Cap() < maxSize { // 对于大Buffer直接丢弃 + w.buffer.Reset() + bufferPool.Put(w) + } +} + +var gzipPool = sync.Pool{ + New: func() any { + buf := new(bytes.Buffer) + w := gzip.NewWriter(buf) + return &GzipWriter{ + w: w, + buf: buf, + } + }, +} + +func AcquireGzipWriter() *GzipWriter { + ret := gzipPool.Get().(*GzipWriter) + ret.buf.Reset() + ret.w.Reset(ret.buf) + return ret +} + +func ReleaseGzipWriter(w *GzipWriter) { + // See https://golang.org/issue/23199 + const maxSize = 1 << 16 + if w.buf.Cap() < maxSize { + w.buf.Reset() + gzipPool.Put(w) + } +} + +type zlibWriter struct { + w *zlib.Writer + buf *bytes.Buffer +} + +var zlibPool = sync.Pool{ + New: func() any { + buf := new(bytes.Buffer) + w := zlib.NewWriter(buf) + return &zlibWriter{ + w: w, + buf: buf, + } + }, +} + +func acquireZlibWriter() *zlibWriter { + ret := zlibPool.Get().(*zlibWriter) + ret.buf.Reset() + ret.w.Reset(ret.buf) + return ret +} + +func releaseZlibWriter(w *zlibWriter) { + // See https://golang.org/issue/23199 + const maxSize = 1 << 16 + if w.buf.Cap() < maxSize { + w.buf.Reset() + zlibPool.Put(w) + } +} diff --git a/utils/binary/utils.go b/utils/binary/utils.go new file mode 100644 index 00000000..b0691496 --- /dev/null +++ b/utils/binary/utils.go @@ -0,0 +1,72 @@ +package binary + +// from https://github.com/Mrs4s/MiraiGo/blob/master/binary/utils.go + +import ( + "bytes" + "compress/gzip" + "compress/zlib" + "encoding/binary" + "net" +) + +type GzipWriter struct { + w *gzip.Writer + buf *bytes.Buffer +} + +func (w *GzipWriter) Write(p []byte) (int, error) { + return w.w.Write(p) +} + +func (w *GzipWriter) Close() error { + return w.w.Close() +} + +func (w *GzipWriter) Bytes() []byte { + return w.buf.Bytes() +} + +func ZlibUncompress(src []byte) []byte { + b := bytes.NewReader(src) + var out bytes.Buffer + r, _ := zlib.NewReader(b) + defer r.Close() + _, _ = out.ReadFrom(r) + return out.Bytes() +} + +func ZlibCompress(data []byte) []byte { + zw := acquireZlibWriter() + _, _ = zw.w.Write(data) + _ = zw.w.Close() + ret := make([]byte, len(zw.buf.Bytes())) + copy(ret, zw.buf.Bytes()) + releaseZlibWriter(zw) + return ret +} + +func GZipCompress(data []byte) []byte { + gw := AcquireGzipWriter() + _, _ = gw.Write(data) + _ = gw.Close() + ret := make([]byte, len(gw.buf.Bytes())) + copy(ret, gw.buf.Bytes()) + ReleaseGzipWriter(gw) + return ret +} + +func GZipUncompress(src []byte) []byte { + b := bytes.NewReader(src) + var out bytes.Buffer + r, _ := gzip.NewReader(b) + defer r.Close() + _, _ = out.ReadFrom(r) + return out.Bytes() +} + +func UInt32ToIPV4Address(i uint32) string { + ip := net.IP{0, 0, 0, 0} + binary.LittleEndian.PutUint32(ip, i) + return ip.String() +} diff --git a/utils/sign.go b/utils/sign.go index b3d3ec68..2775b842 100644 --- a/utils/sign.go +++ b/utils/sign.go @@ -10,6 +10,8 @@ import ( "net/url" "strconv" "time" + + "github.com/LagrangeDev/LagrangeGo/client/http2" ) var ( @@ -119,7 +121,7 @@ func httpGet(rawUrl string, queryParams map[string]string, timeout time.Duration return fmt.Errorf("failed to create GET request: %w", err) } - resp, err := http.DefaultClient.Do(req) + resp, err := http2.DefaultClient.Do(req) if err != nil { if errors.Is(ctx.Err(), context.DeadlineExceeded) { return fmt.Errorf("request timed out") diff --git a/utils/zlib.go b/utils/zlib.go deleted file mode 100644 index 9cf7c3e2..00000000 --- a/utils/zlib.go +++ /dev/null @@ -1,32 +0,0 @@ -package utils - -import ( - "bytes" - "compress/zlib" - "io" -) - -func CompressData(input []byte) ([]byte, error) { - var b bytes.Buffer - w := zlib.NewWriter(&b) - if _, err := w.Write(input); err != nil { - return nil, err - } - if err := w.Close(); err != nil { - return nil, err - } - return b.Bytes(), nil -} - -func DecompressData(compressed []byte) ([]byte, error) { - r, err := zlib.NewReader(bytes.NewReader(compressed)) - if err != nil { - return nil, err - } - defer r.Close() - decompressed, err := io.ReadAll(r) - if err != nil { - return nil, err - } - return decompressed, nil -}