diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..8b48d57 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,13 @@ +Copyright 2015 SensorsData Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md index 17fe0bf..c4c7abf 100644 --- a/README.md +++ b/README.md @@ -1 +1,47 @@ -# sa-sdk-go \ No newline at end of file +# Sensors Analytics + +This is the official Golang SDK for Sensors Analytics. + +## Easy Installation + +You can get Sensors Analytics SDK using go. + +``` + go get github.com/sensorsdata/sa-sdk-go +``` +Or update sdk with +``` + go get -u github.com/sensorsdata/sa-sdk-go + +``` + +Once the SDK is successfully installed, use the Sensors Analytics SDK likes: + +```golang + import sdk "github.com/sensorsdata/sa-sdk-go" + + // Gets the url of Sensors Analytics in the home page. + SA_SERVER_URL = 'YOUR_SERVER_URL' + + // Initialized the Sensors Analytics SDK with Default Consumer + consumer = sdk.InitDefaultConsumer(SA_SERVER_URL) + sa = sdk.InitSensorsAnalytics(consumer) + + properties := map[string]interface{}{ + "price": 12, + "name": "apple", + "somedata": []string{"a", "b"}, + } + + // Track the event 'ServerStart' + sa.track("ABCDEFG1234567", "ServerStart", properties, false) + + sa.Close() +``` + +## More Examples +([Examples](_examples)) + +## To Learn More +See our [full manual](http://www.sensorsdata.cn/manual/golang_sdk.html) + diff --git a/README.txt b/README.txt new file mode 100644 index 0000000..8151564 --- /dev/null +++ b/README.txt @@ -0,0 +1,14 @@ +===================== +Sensors Analytics SDK +===================== + +Sensors Analytics SDK +===================== + +This is the official Golang SDK for Sensors Analytics. + +To Learn More +------------- + +See our `full manual `_. + diff --git a/_examples/example_batch.go b/_examples/example_batch.go new file mode 100644 index 0000000..0e80467 --- /dev/null +++ b/_examples/example_batch.go @@ -0,0 +1,45 @@ +package main +import ( + "fmt" + sdk "github.com/sensorsdata/sa-sdk-go" +) + +func main() { + c, err := sdk.InitBatchConsumer("http://localhost:8106/sa?project=production", 3, 1000) + if err != nil { + fmt.Println(err) + return + } + + sa := sdk.InitSensorsAnalytics(c, "default", false) + defer sa.Close() + + distinctId := "ABCDEF123456" + event := "ViewProduct" + properties := map[string]interface{}{ + "$ip": "2.2.2.2", + "ProductId": "123456", + "ProductCatalog": "Laptop Computer", + "IsAddedToFav": true, + } + + err = sa.Track(distinctId, event, properties, true) + if err != nil { + fmt.Println("track failed", err) + return + } + + err = sa.Track(distinctId, event, properties, true) + if err != nil { + fmt.Println("track failed", err) + return + } + + err = sa.Track(distinctId, event, properties, true) + if err != nil { + fmt.Println("track failed", err) + return + } + + fmt.Println("track done") +} diff --git a/_examples/example_concurrentlogging.go b/_examples/example_concurrentlogging.go new file mode 100644 index 0000000..7125b38 --- /dev/null +++ b/_examples/example_concurrentlogging.go @@ -0,0 +1,33 @@ +package main +import ( + "fmt" + sdk "github.com/sensorsdata/sa-sdk-go" +) + +func main() { + c, err := sdk.InitConcurrentLoggingConsumer("./log.data", false) + if err != nil { + fmt.Println(err) + return + } + + sa := sdk.InitSensorsAnalytics(c, "default", false) + defer sa.Close() + + distinctId := "ABCDEF123456" + event := "ViewProduct" + properties := map[string]interface{}{ + "$ip": "2.2.2.2", + "ProductId": "123456", + "ProductCatalog": "Laptop Computer", + "IsAddedToFav": true, + } + + err = sa.Track(distinctId, event, properties, true) + if err != nil { + fmt.Println("track failed", err) + return + } + + fmt.Println("track done") +} diff --git a/_examples/example_debug.go b/_examples/example_debug.go new file mode 100644 index 0000000..1823405 --- /dev/null +++ b/_examples/example_debug.go @@ -0,0 +1,32 @@ +package main +import ( + "fmt" + sdk "github.com/sensorsdata/sa-sdk-go" +) + +func main() { + c, err := sdk.InitDebugConsumer("http://localhost:8106/sa?project=production", false, 1000) + if err != nil { + fmt.Println(err) + return + } + + sa := sdk.InitSensorsAnalytics(c, "default", false) + + distinctId := "ABCDEF123456777" + event := "ViewInfo" + properties := map[string]interface{}{ + "$ip": "2.2.2.2", + "ProductId": "123456", + "ProductCatalog": "Laptop Computer", + "IsAddedToFav": true, + } + + err = sa.Track(distinctId, event, properties, false) + if err != nil { + fmt.Println("track failed", err) + return + } + + fmt.Println("track done") +} diff --git a/_examples/example_default.go b/_examples/example_default.go new file mode 100644 index 0000000..19dcf28 --- /dev/null +++ b/_examples/example_default.go @@ -0,0 +1,32 @@ +package main +import ( + "fmt" + sdk "github.com/sensorsdata/sa-sdk-go" +) + +func main() { + c, err := sdk.InitDefaultConsumer("http://localhost:8106/sa", 1000) + if err != nil { + fmt.Println(err) + return + } + + sa := sdk.InitSensorsAnalytics(c, "default", false) + defer sa.Close() + + distinctId := "12345" + event := "ViewProduct" + properties := map[string]interface{}{ + "price": 12, + "name": "apple", + "somedata": []string{"a", "b"}, + } + + err = sa.Track(distinctId, event, properties, true) + if err != nil { + fmt.Println("track failed", err) + return + } + + fmt.Println("track done") +} diff --git a/_examples/example_logging.go b/_examples/example_logging.go new file mode 100644 index 0000000..90859cb --- /dev/null +++ b/_examples/example_logging.go @@ -0,0 +1,34 @@ +package main + +import ( + "fmt" + sdk "github.com/sensorsdata/sa-sdk-go" +) + +func main() { + c, err := sdk.InitLoggingConsumer("./log.data", false) + if err != nil { + fmt.Println(err) + return + } + + sa := sdk.InitSensorsAnalytics(c, "default", false) + defer sa.Close() + + distinctId := "ABCDEF123456" + event := "ViewProduct" + properties := map[string]interface{}{ + "$ip": "2.2.2.2", + "ProductId": "123456", + "ProductCatalog": "Laptop Computer", + "IsAddedToFav": true, + } + + err = sa.Track(distinctId, event, properties, true) + if err != nil { + fmt.Println("track failed", err) + return + } + + fmt.Println("track done") +} diff --git a/consumers/batch.go b/consumers/batch.go new file mode 100644 index 0000000..7bfeeed --- /dev/null +++ b/consumers/batch.go @@ -0,0 +1,56 @@ +package consumers + +import ( + "time" + "encoding/json" + + "github.com/sensorsdata/sa-sdk-go/structs" +) + +const ( + BATCH_DEFAULT_MAX = 50 +) + + +type BatchConsumer struct { + Url string + Max int + buffer []structs.EventData + Timeout time.Duration +} + +func InitBatchConsumer(url string, max, timeout int) (*BatchConsumer, error) { + if max > BATCH_DEFAULT_MAX { + max = BATCH_DEFAULT_MAX + } + + c := &BatchConsumer{Url: url, Max: max, Timeout: time.Duration(timeout) * time.Millisecond} + c.buffer = make([]structs.EventData, 0, max) + + return c, nil +} + +func (c *BatchConsumer) Send(data structs.EventData) error { + c.buffer = append(c.buffer, data) + if len(c.buffer) >= c.Max { + return c.Flush() + } + return nil +} + +func (c *BatchConsumer) Flush() error { + jdata, err := json.Marshal(c.buffer) + if err != nil { + return err + } + + send(c.Url, string(jdata), c.Timeout, true) + + c.buffer = c.buffer[:0] + + return nil +} + +func (c *BatchConsumer) Close() error { + return c.Flush() +} diff --git a/consumers/cclogging.go b/consumers/cclogging.go new file mode 100644 index 0000000..24ff41e --- /dev/null +++ b/consumers/cclogging.go @@ -0,0 +1,162 @@ +package consumers + +import ( + "os" + "fmt" + "time" + "sync" + "syscall" + "encoding/json" + + "github.com/sensorsdata/sa-sdk-go/structs" +) + +type ConcurrentLoggingConsumer struct { + w *ConcurrentLogWriter + Fname string + Hour bool +} + +func InitConcurrentLoggingConsumer(fname string, hour bool) (*ConcurrentLoggingConsumer, error) { + w, err := InitConcurrentLogWriter(fname, hour) + if err != nil { + return nil, err + } + + c := &ConcurrentLoggingConsumer{Fname: fname, Hour: hour, w: w} + return c, nil +} + +func (c *ConcurrentLoggingConsumer) Send(data structs.EventData) error { + return c.w.Write(data) +} + +func (c *ConcurrentLoggingConsumer) Flush() error { + c.w.Flush() + return nil +} + +func (c *ConcurrentLoggingConsumer) Close() error { + c.w.Close() + return nil +} + +type ConcurrentLogWriter struct { + rec chan string + + fname string + file *os.File + + day int + hour int + + hourRotate bool + + wg sync.WaitGroup +} + +func (w *ConcurrentLogWriter) Write(data structs.EventData) error { + bdata, err := json.Marshal(data) + if err != nil { + return err + } + + w.rec <- string(bdata) + return nil +} + +func (w *ConcurrentLogWriter) Flush() { + w.file.Sync() +} + +func (w *ConcurrentLogWriter) Close() { + close(w.rec) + w.wg.Wait() +} + +func (w *ConcurrentLogWriter) intRotate() error { + fname := "" + + if w.file != nil { + w.file.Close() + } + + now := time.Now() + today := now.Format("2006-01-02") + w.day = time.Now().Day() + + if w.hourRotate { + hour := now.Hour() + w.hour = hour + + fname = fmt.Sprintf("%s.%s.%d", w.fname, today, hour) + } else { + fname = fmt.Sprintf("%s.%s", w.fname, today) + } + + fd, err := os.OpenFile(fname, os.O_WRONLY | os.O_APPEND | os.O_CREATE, 0644) + if err != nil { + fmt.Printf("open failed: %s\n", err) + return err + } + w.file = fd + + return nil +} + +func InitConcurrentLogWriter(fname string, hourRotate bool) (*ConcurrentLogWriter, error) { + w := &ConcurrentLogWriter{ + fname : fname, + day : time.Now().Day(), + hour : time.Now().Hour(), + hourRotate : hourRotate, + rec : make(chan string, CHANNEL_SIZE), + } + + if err := w.intRotate(); err != nil { + fmt.Fprintf(os.Stderr, "ConcurrentLogWriter(%q): %s\n", w.fname, err) + return nil, err + } + + w.wg.Add(1) + + go func() { + defer func() { + if w.file != nil { + w.file.Sync() + w.file.Close() + } + w.wg.Done() + }() + + for { + select { + case rec, ok := <-w.rec: + if !ok { + return + } + + now := time.Now() + + if (w.hourRotate && now.Hour() != w.hour) || + (now.Day() != w.day) { + if err := w.intRotate(); err != nil { + fmt.Fprintf(os.Stderr, "ConcurrentLogWriter(%q): %s\n", w.fname, err) + return + } + } + + syscall.Flock(int(w.file.Fd()), syscall.LOCK_EX) + _, err := fmt.Fprintln(w.file, rec) + if err != nil { + fmt.Fprintf(os.Stderr, "ConcurrentLogWriter(%q): %s\n", w.fname, err) + syscall.Flock(int(w.file.Fd()), syscall.LOCK_UN) + return + } + syscall.Flock(int(w.file.Fd()), syscall.LOCK_UN) + } + } + }() + + return w, nil +} diff --git a/consumers/consumers.go b/consumers/consumers.go new file mode 100644 index 0000000..7086da4 --- /dev/null +++ b/consumers/consumers.go @@ -0,0 +1,39 @@ +package consumers + +import ( + "time" + + "github.com/sensorsdata/sa-sdk-go/utils" + "github.com/sensorsdata/sa-sdk-go/structs" +) + +type Consumer interface { + Send(data structs.EventData) error + Flush() error + Close() error +} + +func send(url string, data string, to time.Duration, list bool) error { + pdata := "" + + if list { + rdata, err := utils.GeneratePostDataList(data) + if err != nil { + return err + } + pdata = rdata + } else { + rdata, err := utils.GeneratePostData(data) + if err != nil { + return err + } + pdata = rdata + } + + err := utils.DoRequest(url, pdata, to) + if err != nil { + return err + } + + return nil +} diff --git a/consumers/debug.go b/consumers/debug.go new file mode 100644 index 0000000..25d4935 --- /dev/null +++ b/consumers/debug.go @@ -0,0 +1,124 @@ +package consumers + +import ( + "os" + "fmt" + "time" + "bytes" + "errors" + "net/url" + "net/http" + "io/ioutil" + "encoding/json" + + "github.com/sensorsdata/sa-sdk-go/utils" + "github.com/sensorsdata/sa-sdk-go/structs" +) + +type DebugConsumer struct { + Url string + WriteData bool + Timeout time.Duration +} + +func InitDebugConsumer(surl string, writeData bool, timeout int) (*DebugConsumer, error) { + u, err := url.Parse(surl) + if err != nil { + return nil, err + } + u.Path = "/debug" + + return &DebugConsumer{Url: u.String(), WriteData: writeData, Timeout: time.Duration(timeout) * time.Millisecond}, nil +} + +func (c *DebugConsumer) send(url string, data string, to time.Duration, list bool) error { + pdata := "" + + if list { + rdata, err := utils.GeneratePostDataList(data) + if err != nil { + return err + } + pdata = rdata + } else { + rdata, err := utils.GeneratePostData(data) + if err != nil { + return err + } + pdata = rdata + } + + res, status, err := doRequestDebug(url, pdata, to, c.WriteData) + if err != nil && status == 0 { + fmt.Fprintf(os.Stderr, "Send failed: %s\n", err) + return err + } + + if status == 200 { + fmt.Fprintf(os.Stdout, "Valid message: %s\n", string(data)) + return nil + } else { + fmt.Fprintf(os.Stderr, "Invalid message: %s\n", string(data)) + fmt.Fprintf(os.Stderr, "Ret_code: %d\n", status) + fmt.Fprintf(os.Stderr, "Ret_content: %s\n", res) + } + + if status >= 300 { + return errors.New("Bad http status") + } + + return nil +} + +func doRequestDebug(url, args string, to time.Duration, writeData bool) (string, int, error) { + var resp *http.Response + + data := bytes.NewBufferString(args) + + req, _ := http.NewRequest("POST", url , data) + + if !writeData { + req.Header.Add("Dry-Run", "true") + } + + client := &http.Client{Timeout: to} + resp, err := client.Do(req) + + if err != nil { + fmt.Fprintf(os.Stderr, "Request failed to url: %s\n", url) + return "", 0, err + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + fmt.Fprintf(os.Stderr, "Request failed http status is %d\n", resp.StatusCode) + return "", resp.StatusCode, nil + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + fmt.Fprintf(os.Stderr, "Request debug read response failed: %s\n", err) + return "", resp.StatusCode, err + } + + return string(body), resp.StatusCode, nil +} + +func (c *DebugConsumer) Send(data structs.EventData) error { + jdata, err := json.Marshal(data) + if err != nil { + return err + } + + return c.send(c.Url, string(jdata), c.Timeout, false) +} + +func (c *DebugConsumer) Flush() error { + return nil +} + +func (c *DebugConsumer) Close() error { + return nil +} + diff --git a/consumers/default.go b/consumers/default.go new file mode 100644 index 0000000..4b40419 --- /dev/null +++ b/consumers/default.go @@ -0,0 +1,35 @@ +package consumers + +import ( + "time" + "encoding/json" + + "github.com/sensorsdata/sa-sdk-go/structs" +) + +type DefaultConsumer struct { + Url string + Timeout time.Duration +} + +func InitDefaultConsumer(url string, timeout int) (*DefaultConsumer, error) { + return &DefaultConsumer{Url: url, Timeout: time.Duration(timeout) * time.Millisecond}, nil +} + +func (c *DefaultConsumer) Send(data structs.EventData) error { + jdata, err := json.Marshal(data) + if err != nil { + return err + } + + return send(c.Url, string(jdata), c.Timeout, false) +} + +func (c *DefaultConsumer) Flush() error { + return nil +} + +func (c *DefaultConsumer) Close() error { + return nil +} + diff --git a/consumers/logging.go b/consumers/logging.go new file mode 100644 index 0000000..04625c8 --- /dev/null +++ b/consumers/logging.go @@ -0,0 +1,162 @@ +package consumers + +import ( + "os" + "fmt" + "time" + "sync" + "encoding/json" + + "github.com/sensorsdata/sa-sdk-go/structs" +) + +const ( + CHANNEL_SIZE = 1000 +) + +type LoggingConsumer struct { + w *LogWriter + Fname string + Hour bool +} + +func InitLoggingConsumer(fname string, hour bool) (*LoggingConsumer, error) { + w, err := InitLogWriter(fname, hour) + if err != nil { + return nil, err + } + + c := &LoggingConsumer{Fname: fname, Hour: hour, w: w} + return c, nil +} + +func (c *LoggingConsumer) Send(data structs.EventData) error { + return c.w.Write(data) +} + +func (c *LoggingConsumer) Flush() error { + c.w.Flush() + return nil +} + +func (c *LoggingConsumer) Close() error { + c.w.Close() + return nil +} + +type LogWriter struct { + rec chan string + + fname string + file *os.File + + day int + hour int + + hourRotate bool + + wg sync.WaitGroup +} + +func (w *LogWriter) Write(data structs.EventData) error { + bdata, err := json.Marshal(data) + if err != nil { + return err + } + + w.rec <- string(bdata) + return nil +} + +func (w *LogWriter) Flush() { + w.file.Sync() +} + +func (w *LogWriter) Close() { + close(w.rec) + w.wg.Wait() +} + +func (w *LogWriter) intRotate() error { + fname := "" + + if w.file != nil { + w.file.Close() + } + + now := time.Now() + today := now.Format("2006-01-02") + w.day = time.Now().Day() + + if w.hourRotate { + hour := now.Hour() + w.hour = hour + + fname = fmt.Sprintf("%s.%s.%d", w.fname, today, hour) + } else { + fname = fmt.Sprintf("%s.%s", w.fname, today) + } + + fd, err := os.OpenFile(fname, os.O_WRONLY | os.O_APPEND | os.O_CREATE, 0644) + if err != nil { + fmt.Printf("open failed: %s\n", err) + return err + } + w.file = fd + + return nil +} + +func InitLogWriter(fname string, hourRotate bool) (*LogWriter, error) { + w := &LogWriter{ + fname : fname, + day : time.Now().Day(), + hour : time.Now().Hour(), + hourRotate : hourRotate, + rec : make(chan string, CHANNEL_SIZE), + } + + if err := w.intRotate(); err != nil { + fmt.Fprintf(os.Stderr, "LogWriter(%q): %s\n", w.fname, err) + return nil, err + } + + w.wg.Add(1) + + go func() { + defer func() { + if w.file != nil { + w.file.Sync() + w.file.Close() + } + w.wg.Done() + }() + + for { + select { + case rec, ok := <-w.rec: + if !ok { + return + } + + now := time.Now() + + if (w.hourRotate && now.Hour() != w.hour) || + (now.Day() != w.day) { + if err := w.intRotate(); err != nil { + fmt.Fprintf(os.Stderr, "LogWriter(%q): %s\n", w.fname, err) + return + } + } + + _, err := fmt.Fprintln(w.file, rec) + if err != nil { + fmt.Fprintf(os.Stderr, "LogWriter(%q): %s\n", w.fname, err) + return + } + } + } + }() + + return w, nil +} diff --git a/sensorsanalytics.go b/sensorsanalytics.go new file mode 100644 index 0000000..206fcd1 --- /dev/null +++ b/sensorsanalytics.go @@ -0,0 +1,234 @@ +package sensorsanalytics +import ( + "os" + "fmt" + "errors" + "runtime" + + "github.com/sensorsdata/sa-sdk-go/utils" + "github.com/sensorsdata/sa-sdk-go/structs" + "github.com/sensorsdata/sa-sdk-go/consumers" +) + +const ( + TRACK = "track" + TRACK_SIGNUP = "track_signup" + PROFILE_SET = "profile_set" + PROFILE_SET_ONCE = "profile_set_once" + PROFILE_INCREMENT = "profile_increment" + PROFILE_APPEND = "profile_append" + PROFILE_UNSET = "profile_unset" + PROFILE_DELETE = "profile_delete" + + SDK_VERSION = "1.7.5" + LIB_NAME = "Golang" + + MAX_ID_LEN = 255 +) + +type SensorsAnalytics struct { + C consumers.Consumer + ProjectName string + TimeFree bool +} + +func InitSensorsAnalytics(c consumers.Consumer, projectName string, timeFree bool) SensorsAnalytics { + return SensorsAnalytics{C: c, ProjectName: projectName, TimeFree: timeFree} +} + +func (sa *SensorsAnalytics) track(etype, event, distinctId, originId string, properties map[string]interface{}, isLoginId bool) error { + eventTime := utils.NowMs() + if et := extractUserTime(properties); et > 0 { + eventTime = et + } + + data := structs.EventData{ + Type : etype, + Time : eventTime, + DistinctId : distinctId, + Properties : properties, + LibProperties : getLibProperties(), + } + + if sa.ProjectName != "" { + data.Project = sa.ProjectName + } + + if etype == TRACK || etype == TRACK_SIGNUP { + data.Event = event + } + + if etype == TRACK_SIGNUP { + data.OriginId = originId + } + + if sa.TimeFree { + data.TimeFree = true + } + + if isLoginId { + properties["$is_login_id"] = true + } + + err := data.NormalizeData() + if err != nil { + return err + } + + return sa.C.Send(data) +} + +func (sa *SensorsAnalytics) Flush() { + sa.C.Flush() +} + +func (sa *SensorsAnalytics) Close() { + sa.C.Close() +} + +func (sa *SensorsAnalytics) Track(distinctId, event string, properties map[string]interface{}, isLoginId bool) error { + var nproperties map[string]interface{} + + // merge properties + if properties == nil { + properties = make(map[string]interface{}) + } else { + nproperties = utils.DeepCopy(properties) + } + + nproperties["$lib"] = LIB_NAME + nproperties["$lib_version"] = SDK_VERSION + + return sa.track(TRACK, event, distinctId, "", nproperties, isLoginId) +} + +func (sa *SensorsAnalytics) TrackSignup(distinctId, originId string) error { + // check originId and merge properties + if originId == "" { + return errors.New("property [original_id] must not be empty") + } + if len(originId) > MAX_ID_LEN { + return errors.New("the max length of property [original_id] is 255") + } + + properties := make(map[string]interface{}) + + properties["$lib"] = LIB_NAME + properties["$lib_version"] = SDK_VERSION + + return sa.track(TRACK_SIGNUP, "$SignUp", distinctId, originId, properties, false) +} + +func (sa *SensorsAnalytics) ProfileSet(distinctId string, properties map[string]interface{}, isLoginId bool) error { + var nproperties map[string]interface{} + + if properties == nil { + return errors.New("property should not be nil") + } else { + nproperties = utils.DeepCopy(properties) + } + + return sa.track(PROFILE_SET, "", distinctId, "", nproperties, isLoginId) +} + +func (sa *SensorsAnalytics) ProfileSetOnce(distinctId string, properties map[string]interface{}, isLoginId bool) error { + var nproperties map[string]interface{} + + if properties == nil { + return errors.New("property should not be nil") + } else { + nproperties = utils.DeepCopy(properties) + } + + return sa.track(PROFILE_SET_ONCE, "", distinctId, "", nproperties, isLoginId) +} + +func (sa *SensorsAnalytics) ProfileIncrement(distinctId string, properties map[string]interface{}, isLoginId bool) error { + var nproperties map[string]interface{} + + if properties == nil { + return errors.New("property should not be nil") + } else { + nproperties = utils.DeepCopy(properties) + } + + return sa.track(PROFILE_INCREMENT, "", distinctId, "", nproperties, isLoginId) +} + +func (sa *SensorsAnalytics) ProfileAppend(distinctId string, properties map[string]interface{}, isLoginId bool) error { + var nproperties map[string]interface{} + + if properties == nil { + return errors.New("property should not be nil") + } else { + nproperties = utils.DeepCopy(properties) + } + + return sa.track(PROFILE_APPEND, "", distinctId, "", nproperties, isLoginId) +} + +func (sa *SensorsAnalytics) ProfileUnset(distinctId string, properties map[string]interface{}, isLoginId bool) error { + var nproperties map[string]interface{} + + if properties == nil { + return errors.New("property should not be nil") + } else { + nproperties = utils.DeepCopy(properties) + } + + return sa.track(PROFILE_UNSET, "", distinctId, "", nproperties, isLoginId) +} + +func (sa *SensorsAnalytics) ProfileDelete(distinctId string, isLoginId bool) error { + nproperties := make(map[string]interface{}) + + return sa.track(PROFILE_DELETE, "", distinctId, "", nproperties, isLoginId) +} + +func getLibProperties() structs.LibProperties { + lp := structs.LibProperties{} + lp.Lib = LIB_NAME + lp.LibVersion = SDK_VERSION + lp.LibMethod = "code" + if pc, file, line, ok := runtime.Caller(3); ok { //3 means sdk's caller + f := runtime.FuncForPC(pc) + lp.LibDetail = fmt.Sprintf("##%s##%s##%d", f.Name(), file, line) + } + + return lp +} + +func extractUserTime(p map[string]interface{}) int64 { + if t, ok := p["$time"]; ok { + v, ok := t.(int64) + if !ok { + fmt.Fprintln(os.Stderr, "It's not ok for type string") + return 0 + } + delete(p, "$time") + + return v + } + + return 0 +} + +func InitDefaultConsumer(url string, timeout int) (*consumers.DefaultConsumer, error) { + return consumers.InitDefaultConsumer(url, timeout) +} + +func InitBatchConsumer(url string, max, timeout int) (*consumers.BatchConsumer, error) { + return consumers.InitBatchConsumer(url, max, timeout) +} + +func InitLoggingConsumer(filename string, hourRotate bool) (*consumers.LoggingConsumer, error) { + return consumers.InitLoggingConsumer(filename, hourRotate) +} + +func InitConcurrentLoggingConsumer(filename string, hourRotate bool) (*consumers.ConcurrentLoggingConsumer, error) { + return consumers.InitConcurrentLoggingConsumer(filename, hourRotate) +} + +func InitDebugConsumer(url string, writeData bool, timeout int) (*consumers.DebugConsumer, error) { + return consumers.InitDebugConsumer(url, writeData, timeout) +} diff --git a/structs/event.go b/structs/event.go new file mode 100644 index 0000000..3d1a240 --- /dev/null +++ b/structs/event.go @@ -0,0 +1,98 @@ +package structs + +import ( + "time" + "errors" + "regexp" +) + +const ( + KEY_MAX = 256 + VALUE_MAX = 8192 + + NAME_PATTERN_BAD = "^(^distinct_id$|^original_id$|^time$|^properties$|^id$|^first_id$|^second_id$|^users$|^events$|^event$|^user_id$|^date$|^datetime$)$" + NAME_PATTERN_OK = "^[a-zA-Z_$][a-zA-Z\\d_$]{0,99}$" +) + +var patternBad, patternOk *regexp.Regexp + +type EventData struct { + Type string `json:"type"` + Time int64 `json:"time"` + DistinctId string `json:"distinct_id"` + Properties map[string]interface{} `json:"properties"` + LibProperties LibProperties `json:"lib"` + Project string `json:"project"` + Event string `json:"event"` + OriginId string `json:"original_id,omitempty"` + TimeFree bool `json:"time_free,omitempty"` +} + +func init() { + patternBad, _ = regexp.Compile(NAME_PATTERN_BAD) + patternOk, _ = regexp.Compile(NAME_PATTERN_OK) +} + +func (e *EventData) NormalizeData() error { + //check distinct id + if e.DistinctId == "" || len(e.DistinctId) == 0 { + return errors.New("property [original_id] must not be empty") + } + + if len(e.DistinctId) > 255 { + return errors.New("the max length of property [distinct_id] is 255") + } + + //check event + if e.Event != "" { + isMatch := checkPattern([]byte(e.Event)) + if !isMatch { + return errors.New("event name must be a valid variable name.") + } + } + + //check project + if e.Project != "" { + isMatch := checkPattern([]byte(e.Project)) + if !isMatch { + return errors.New("project name must be a valid variable name.") + } + } + + //check properties + if e.Properties != nil { + for k, v := range e.Properties { + //check key + if len(k) > KEY_MAX { + return errors.New("the max length of property key is 256") + } + isMatch := checkPattern([]byte(k)) + if !isMatch { + return errors.New("property key must be a valid variable name.") + } + + //check value + switch v.(type) { + case int: + case bool: + case float64: + case string: + if len(v.(string)) > VALUE_MAX { + return errors.New("the max length of property key is 8192") + } + case []string: //value in properties list MUST be string + case time.Time: //only support time.Time + e.Properties[k] = v.(time.Time).Format("2006-01-02 15:04:05.999") + + default: + return errors.New("property value must be a string/int/float64/bool/time.Time/[]string") + } + } + } + + return nil +} + +func checkPattern(name []byte) bool { + return !patternBad.Match(name) && patternOk.Match(name) +} diff --git a/structs/event_test.go b/structs/event_test.go new file mode 100644 index 0000000..35b4e15 --- /dev/null +++ b/structs/event_test.go @@ -0,0 +1,175 @@ +package structs + +import ( + //"fmt" + "time" + "testing" +) + +var lib LibProperties = LibProperties { + Lib : "Golang", + LibVersion : "1.1.1", + LibMethod : "code", + AppVersion : "1.0.1", + LibDetail : "somedetail", +} + +func TestEventBadDistinctId(t *testing.T) { + //empty distinctid + ed := EventData { + Type: "track", + Time: time.Now().UnixNano() / 1000000, + DistinctId: "", + Properties: map[string]interface{}{ + "name": "alice", + "age": 1, + }, + LibProperties: lib, + Project: "default", + Event: "test", + } + + if err := ed.NormalizeData(); err != nil { + t.Log("found bad type", err) + } else { + t.Fatal("not found bad type") + } + + //too large distinctid + id := make([]byte, 256) + id[255] = byte('a') + ed.DistinctId = string(id) + + if err := ed.NormalizeData(); err != nil { + t.Log("found bad type", err) + } else { + t.Fatal("not found bad type") + } +} + +func TestEventBadEvent(t *testing.T) { + //reserve event name + ed := EventData { + Type: "track", + Time: time.Now().UnixNano() / 1000000, + DistinctId: "123123", + Properties: map[string]interface{}{ + "name": "alice", + "age": 1, + }, + LibProperties: lib, + Project: "default", + Event: "time", + } + + if err := ed.NormalizeData(); err != nil { + t.Log("found bad type", err) + } else { + t.Fatal("not found bad type") + } + + //bad event name pattern + ed.Event = "@$%^abc" + if err := ed.NormalizeData(); err != nil { + t.Log("found bad type", err) + } else { + t.Fatal("not found bad type") + } +} + +func TestEventBadProject(t *testing.T) { + ed := EventData { + Type: "track", + Time: time.Now().UnixNano() / 1000000, + DistinctId: "123123", + Properties: map[string]interface{}{ + "name": "alice", + "age": 1, + }, + LibProperties: lib, + Project: "time", + Event: "sometime", + } + + if err := ed.NormalizeData(); err != nil { + t.Log("found bad type", err) + } else { + t.Fatal("not found bad type") + } + + ed.Project = "@$%^abc" + if err := ed.NormalizeData(); err != nil { + t.Log("found bad type", err) + } else { + t.Fatal("not found bad type") + } +} + +func TestEventBadPropertiesKey(t *testing.T) { + //bad pattern + ed := EventData { + Type: "track", + Time: time.Now().UnixNano() / 1000000, + DistinctId: "123123", + Properties: map[string]interface{}{ + "name": "alice", + "age": 1, + "time": "12345", + }, + LibProperties: lib, + Project: "test", + Event: "sometime", + } + + if err := ed.NormalizeData(); err != nil { + t.Log("found bad type", err) + } else { + t.Fatal("not found bad type") + } + + delete(ed.Properties, "time") + + //bad value type + ed.Properties["abc"] = int64(1) + + if err := ed.NormalizeData(); err != nil { + t.Log("found bad type", err) + } else { + t.Fatal("not found bad type") + } + + //bad value array type + badv := []int{1, 2} + ed.Properties["abc"] = badv + if err := ed.NormalizeData(); err != nil { + t.Log("found bad type", err) + } else { + t.Fatal("not found bad type") + } +} + +func TestEventOkPropertiesKey(t *testing.T) { + //good pattern + ed := EventData { + Type: "track", + Time: time.Now().UnixNano() / 1000000, + DistinctId: "123123", + Properties: map[string]interface{}{ + "name": "alice", + "age": 1, + "size": 0.2, + "flag": true, + "sometime": time.Now(), + "arr": []string{"a", "b"}, + }, + LibProperties: lib, + Project: "test", + Event: "sometime", + } + + if err := ed.NormalizeData(); err != nil { + t.Fatal("found bad type", err) + } else { + t.Log("all type is valid") + } +} diff --git a/structs/property.go b/structs/property.go new file mode 100644 index 0000000..0105f49 --- /dev/null +++ b/structs/property.go @@ -0,0 +1,9 @@ +package structs + +type LibProperties struct { + Lib string `json:"$lib"` + LibVersion string `json:"$lib_version"` + LibMethod string `json:"$lib_method"` + AppVersion string `json:"$app_version,omitempty"` + LibDetail string `json:"$lib_detail"` +} diff --git a/test/batch_test.go b/test/batch_test.go new file mode 100644 index 0000000..2499453 --- /dev/null +++ b/test/batch_test.go @@ -0,0 +1,44 @@ +package test + +import ( + "fmt" + "testing" + + sdk "github.com/sensorsdata/sa-sdk-go" +) + +func TestBatchConsumer(t *testing.T) { + go MockServerRun() + + c, err := sdk.InitBatchConsumer("http://localhost:8106/sa", 3, 1000) + if err != nil { + fmt.Println(err) + return + } + + sa := sdk.InitSensorsAnalytics(c, "default", false) + defer sa.Close() + + distinctId := DemoDistinctId + event := DemoEventString + properties := DemoProperties + properties["$time"] = DemoTime + + err = sa.Track(distinctId, event, properties, true) + if err != nil { + t.Fatal("batch consumer track failed", err) + return + } + err = sa.Track(distinctId, event, properties, true) + if err != nil { + t.Fatal("batch consumer track failed", err) + return + } + err = sa.Track(distinctId, event, properties, true) + if err != nil { + t.Fatal("batch consumer track failed", err) + return + } + + t.Log("batch consumer ok") +} diff --git a/test/cclogging_test.go b/test/cclogging_test.go new file mode 100644 index 0000000..2832f01 --- /dev/null +++ b/test/cclogging_test.go @@ -0,0 +1,67 @@ +package test + +import ( + "fmt" + "time" + "testing" + + sdk "github.com/sensorsdata/sa-sdk-go" + "github.com/sensorsdata/sa-sdk-go/utils" +) + +const ( + CCFILE_NAME = "./test.log" + NUM_WRITERS = 3 + NUM_RECORDS = 100 +) + +func ccwriter() { + c, err := sdk.InitConcurrentLoggingConsumer(CCFILE_NAME, false) + if err != nil { + fmt.Println(err) + return + } + + sa := sdk.InitSensorsAnalytics(c, "default", false) + defer sa.Close() + + distinctId := DemoDistinctId + event := DemoEventString + properties := utils.DeepCopy(DemoProperties) + properties["$time"] = DemoTime + + for i := 0; i < NUM_RECORDS; i++ { + err = sa.Track(distinctId, event, properties, true) + if err != nil { + fmt.Println("concurrentlogging consumer track failed", err) + return + } + } +} + +func TestConcurrentLoggingConsumer(t *testing.T) { + go MockServerRun() + + for i := 0; i < NUM_WRITERS; i++ { + go ccwriter() + } + + //10ms is enough + time.Sleep(time.Millisecond * 10) + + today := time.Now().Format("2006-01-02") + logfile := fmt.Sprintf("%s.%s", CCFILE_NAME, today) + estr, count := judgeFile(logfile) + if estr != "" { + t.Fatal("concurrentlogging consumer track failed", estr) + return + } + + if count != NUM_WRITERS * NUM_RECORDS { + t.Fatal("concurrentlogging consumer track failed, count not match", count) + return + } + + fmt.Println("concurrent records count match") + t.Log("concurrentlogging consumer ok") +} diff --git a/test/debug_test.go b/test/debug_test.go new file mode 100644 index 0000000..d402519 --- /dev/null +++ b/test/debug_test.go @@ -0,0 +1,34 @@ +package test + +import ( + "fmt" + "testing" + + sdk "github.com/sensorsdata/sa-sdk-go" +) + +func TestDebugConsumer(t *testing.T) { + go MockServerRun() + + c, err := sdk.InitDebugConsumer("http://localhost:8106/sa", false, 1000) + if err != nil { + fmt.Println(err) + return + } + + sa := sdk.InitSensorsAnalytics(c, "default", false) + defer sa.Close() + + distinctId := DemoDistinctId + event := DemoEventString + properties := DemoProperties + properties["$time"] = DemoTime + + err = sa.Track(distinctId, event, properties, true) + if err != nil { + t.Fatal("debug consumer track failed", err) + return + } + + t.Log("debug consumer ok") +} diff --git a/test/default_test.go b/test/default_test.go new file mode 100644 index 0000000..ef0e17a --- /dev/null +++ b/test/default_test.go @@ -0,0 +1,35 @@ +package test + +import ( + "fmt" + "testing" + + sdk "github.com/sensorsdata/sa-sdk-go" +) + +func TestDefaultConsumer(t *testing.T) { + go MockServerRun() + + c, err := sdk.InitDefaultConsumer("http://localhost:8106/sa", 1000) + if err != nil { + fmt.Println(err) + return + } + + sa := sdk.InitSensorsAnalytics(c, "default", false) + defer sa.Close() + + distinctId := DemoDistinctId + event := DemoEventString + properties := DemoProperties + properties["$time"] = DemoTime + + err = sa.Track(distinctId, event, properties, true) + if err != nil { + t.Fatal("default consumer track failed", err) + return + } + + t.Log("Default consumer ok") +} + diff --git a/test/demo.go b/test/demo.go new file mode 100644 index 0000000..45cb916 --- /dev/null +++ b/test/demo.go @@ -0,0 +1,127 @@ +package test + +import ( + "github.com/sensorsdata/sa-sdk-go/structs" +) + +var typemap map[string]int = map[string]int{ + "track" : 0, + "track_signup" : 0, + "profile_set" : 0, + "profile_set_once" : 0, + "profile_increment" : 0, + "profile_append" : 0, + "profile_unset" : 0, + "profile_delete" : 0, +} + +var p map[string]interface{} = map[string]interface{}{ + "$ip":"1.1.1.1", + "$is_login_id":true, + "$lib":"Golang", + "$lib_version":"1.7.5", + "ProductCatalog":"Laptop Computer", + "ProductId":"123456", +} + +var demoLibData map[string]interface{} = map[string]interface{}{ + "$lib":"Golang", + "$lib_version":"1.7.5", + "$lib_method":"code", + "$lib_detail":"##main.main.go##track.go##28", +} +var demoEventData map[string]interface{} = map[string]interface{}{ + "type":"track", + "time": DemoTime, + "distinct_id": DemoDistinctId, + "properties": p, + "lib": demoLibData, + "project":"default", +} + +var DemoDistinctId string = "ABCDEF123456" +var DemoEventString string = "ViewProduct" +var DemoTime int64 = 1523329458000 +var DemoProperties map[string]interface{} = map[string]interface{}{ + "$ip": "1.1.1.1", + "ProductId": "123456", + "ProductCatalog": "Laptop Computer", + "IsAddedToFav": true, + "$is_login_id":true, + "$lib": "Golang", + "$lib_version":"1.7.5", +} + +var DemoLib structs.LibProperties = structs.LibProperties{ + Lib : "Golang", + LibVersion : "1.7.5", + LibMethod : "code", + AppVersion : "", + LibDetail : "##main.main.go##track.go##28", +} + +var DemoEvent structs.EventData = structs.EventData{ + Type : "track", + Time : DemoTime, + DistinctId : DemoDistinctId, + Properties : DemoProperties, + LibProperties : DemoLib, + Project : "default", +} + +func demoCompare(ed structs.EventData) string { + if ed.DistinctId != DemoDistinctId { + return "distinctid error" + } + if ed.Time != DemoEvent.Time { + return "time error" + } + if ed.Project != DemoEvent.Project { + return "project error" + } + if _, ok := typemap[ed.Type]; !ok { + return "type error" + } + lib := ed.LibProperties + if lib.Lib != DemoLib.Lib { + return "lib.lib error" + } + if lib.LibVersion != DemoLib.LibVersion{ + return "lib.lib_version error" + } + if lib.LibMethod != DemoLib.LibMethod { + return "lib.lib_method error" + } + if lib.LibDetail == "" { + return "lib.lib_detail error" + } + properties := ed.Properties + if properties["$ip"] != DemoEvent.Properties["$ip"] { + return "properties.$ip error" + } + if properties["$is_login_id"] != DemoEvent.Properties["$is_login_id"] { + return "properties.$is_login_id error" + } + if properties["$lib"] != DemoEvent.Properties["$lib"] { + return "properties.$lib error" + } + if properties["$lib_version"] != DemoEvent.Properties["$lib_version"] { + return "properties.$lib_version error" + } + if properties["ProductId"] != DemoEvent.Properties["ProductId"] { + return "properties.ProductId error" + } + + + return "" +} + +func demoListCompare(edl EDL) string { + for _, ed := range edl { + estr := demoCompare(ed) + if estr != "" { + return estr + } + } + return "" +} diff --git a/test/judge.go b/test/judge.go new file mode 100644 index 0000000..c8da8f7 --- /dev/null +++ b/test/judge.go @@ -0,0 +1,69 @@ +package test +import ( + "os" + "io" + "bufio" + "encoding/json" + "github.com/sensorsdata/sa-sdk-go/structs" +) + +type EDL []structs.EventData + +func judge(data []byte, dataFlag, dataListFlag bool) string { + if dataFlag { + return judgeData(data) + } + if dataListFlag { + return judgeDataList(data) + } + return "error" +} + +func judgeData(data []byte) string { + ed := structs.EventData{} + err := json.Unmarshal(data, &ed) + if err != nil { + return "unmarshal data failed" + } + + return demoCompare(ed) +} + +func judgeDataList(data []byte) string { + edl := EDL{} + err := json.Unmarshal(data, &edl) + if err != nil { + return "unmarshal data failed" + } + + return demoListCompare(edl) +} + +func judgeFile(name string) (string, int) { + defer cleanFile(name) + + file, err := os.OpenFile(name, os.O_RDONLY, 0) + if err != nil { + return "open logging file failed", -1 + } + defer file.Close() + rb := bufio.NewReader(file) + count := 0 + for { + line, _, err := rb.ReadLine() + if err == io.EOF { + break + } + count++ + + estr := judgeData(line) + if estr != "" { + return estr, count + } + } + return "", count +} + +func cleanFile(name string) { + os.Remove(name) +} diff --git a/test/logging_test.go b/test/logging_test.go new file mode 100644 index 0000000..f29a08d --- /dev/null +++ b/test/logging_test.go @@ -0,0 +1,49 @@ +package test + +import ( + "fmt" + "time" + "testing" + + sdk "github.com/sensorsdata/sa-sdk-go" +) + +const ( + FILE_NAME = "./cctest.log" +) + +func TestLoggingConsumer(t *testing.T) { + go MockServerRun() + + c, err := sdk.InitLoggingConsumer(FILE_NAME, false) + if err != nil { + fmt.Println(err) + return + } + + sa := sdk.InitSensorsAnalytics(c, "default", false) + defer sa.Close() + + distinctId := DemoDistinctId + event := DemoEventString + properties := DemoProperties + properties["$time"] = DemoTime + + err = sa.Track(distinctId, event, properties, true) + if err != nil { + t.Fatal("logging consumer track failed", err) + return + } + + time.Sleep(time.Millisecond) + + today := time.Now().Format("2006-01-02") + logfile := fmt.Sprintf("%s.%s", FILE_NAME, today) + estr, _ := judgeFile(logfile) + if estr != "" { + t.Fatal("logging consumer track failed", estr) + return + } + + t.Log("logging consumer ok") +} diff --git a/test/mockserver.go b/test/mockserver.go new file mode 100644 index 0000000..d0d8028 --- /dev/null +++ b/test/mockserver.go @@ -0,0 +1,92 @@ +package test + +import ( + "fmt" + "bytes" + "net/url" + "net/http" + "io/ioutil" + "compress/gzip" + "encoding/base64" +) + +func init() { + http.Handle("/sa", &mockHandler{}) + http.Handle("/debug", &mockHandler{}) +} + +type mockHandler struct{} + +func (h *mockHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method != "POST" { + fmt.Println("method invalid") + http.Error(w, "method invalid", http.StatusBadRequest) + return + } + + result, err := ioutil.ReadAll(r.Body) + if err != nil { + fmt.Println("body invalid") + http.Error(w, "body invalid", http.StatusBadRequest) + return + } + defer r.Body.Close() + + //log.Println("Mock server: raw data is", string(result)) + m, err := url.ParseQuery(string(result)) + if err != nil { + fmt.Println("url query invalid") + http.Error(w, "url query invalid", http.StatusBadRequest) + return + } + dataFlag := false + dataListFlag := false + + rawdata := m.Get("data") + if rawdata != "" { + dataFlag = true + } else { + rawdata = m.Get("data_list") + if rawdata != "" { + dataListFlag = true + } + } + + if rawdata == "" { + fmt.Println("get raw data failed") + http.Error(w, "no data or data_list error", http.StatusBadRequest) + return + } + + data, err := base64.StdEncoding.DecodeString(rawdata) + if err != nil { + fmt.Println("base64 decode failed", err) + http.Error(w, "decode base64 failed", http.StatusBadRequest) + return + } + + buf := bytes.NewBuffer(data) + + zr, err := gzip.NewReader(buf) + if err != nil { + fmt.Println("gzip new reader faild", err) + http.Error(w, "ungzip failed", http.StatusBadRequest) + return + } + defer zr.Close() + + ungzips, _ := ioutil.ReadAll(zr) + //log.Printf("Mock server: data(%d) is %s\n", len(ungzips), string(ungzips)) + estr := judge(ungzips, dataFlag, dataListFlag) + if estr != "" { + fmt.Println("judge failed", estr) + http.Error(w, estr, http.StatusBadRequest) + return + } + + http.Error(w, "", http.StatusOK) +} + +func MockServerRun() { + http.ListenAndServe(":8106", nil) +} diff --git a/utils/utils.go b/utils/utils.go new file mode 100644 index 0000000..dc58636 --- /dev/null +++ b/utils/utils.go @@ -0,0 +1,123 @@ +package utils +import ( + "os" + "fmt" + "time" + "bytes" + "errors" + "net/url" + "net/http" + "compress/gzip" + "encoding/base64" +) + +func DoRequest(url, args string, to time.Duration) error { + var resp *http.Response + + data := bytes.NewBufferString(args) + + req, _ := http.NewRequest("POST", url , data) + + client := &http.Client{Timeout: to} + resp, err := client.Do(req) + + if err != nil { + fmt.Fprintf(os.Stderr, "Request failed to url: %s\n", url) + return err + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + fmt.Fprintf(os.Stderr, "Http status is %d\n", resp.StatusCode) + return errors.New("Bad http status") + } + return nil +} + +func gzipData(data string) ([]byte, error) { + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + + _, err := zw.Write([]byte(data)) + if err != nil { + zw.Close() + return nil, err + } + zw.Close() + + return buf.Bytes(), nil +} + +func encodeData(data string) (string, error) { + gdata, err := gzipData(data) + if err != nil { + return "", err + } + + encoded := base64.StdEncoding.EncodeToString(gdata) + return encoded, nil +} + +func GeneratePostDataList(data string) (string, error) { + edata, err := encodeData(data) + if err != nil { + return "", err + } + + v := url.Values{} + v.Add("data_list", edata) + v.Add("gzip", "1") + + uedata := v.Encode() + + return uedata, nil +} + +func GeneratePostData(data string) (string, error) { + edata, err := encodeData(data) + if err != nil { + return "", err + } + + v := url.Values{} + v.Add("data", edata) + v.Add("gzip", "1") + + uedata := v.Encode() + + return uedata, nil +} + +func NowMs() int64 { + return time.Now().UnixNano() / 1000000 +} + +func DeepCopy(value map[string]interface{}) map[string]interface{} { + ncopy := deepCopy(value) + if nmap, ok := ncopy.(map[string]interface{}); ok { + return nmap + } + + return nil +} + +func deepCopy(value interface{}) interface{} { + if valueMap, ok := value.(map[string]interface{}); ok { + newMap := make(map[string]interface{}) + for k, v := range valueMap { + newMap[k] = deepCopy(v) + } + + return newMap + } else if valueSlice, ok := value.([]interface{}); ok { + newSlice := make([]interface{}, len(valueSlice)) + for k, v := range valueSlice { + newSlice[k] = deepCopy(v) + } + + return newSlice + } + + return value +}