diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 18307b3..0000000 --- a/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM golang:1.21.1 - -WORKDIR /app - -COPY . . - -RUN go build -o siphon . - -ENTRYPOINT "./siphon" \ No newline at end of file diff --git a/Makefile b/Makefile index 399fd8c..32284fa 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ -.PHONY: build run +.PHONY: build build-agent build-gen build: - docker build -t siphon . + go build cmd/siphon/siphon.go -run: - docker run --rm -it siphon +build-agent: + go build cmd/agent/siphon_agent.go + +build-gen: + go build cmd/generator/siphon_gen.go \ No newline at end of file diff --git a/README.md b/README.md index 3892243..980cd87 100644 --- a/README.md +++ b/README.md @@ -10,20 +10,17 @@ A database of basic sample information is built up as often as an un-indexed sam is found by querying threat intelligence APIs. You can view the most recent samples, sorted by time, and download them from their source. -## Installation +## Support -You can either download Siphon from the [releases](https://github.com/pygrum/siphon/releases/latest) -page, or run the application in a Docker container +Siphon is designed with only Unix host support in mind, however, it is possible to set it up on Windows +using Git Bash, WSL or similar applications. -### Using docker -#### Dependencies -- Docker -- Make +## Installation -1. Clone the repository: `git clone https://github.com/pygrum/siphon` -2. Enter the cloned repository and run `make build run` +You can download Siphon source code from the [releases](https://github.com/pygrum/siphon/releases/latest) page, or clone it from this URL. +Then, after entering the containing folder, run `bash scripts/install.sh`. -## Supported Integrations +### Supported Integrations | Name | Setup instructions | |---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -42,4 +39,15 @@ sources: - name: MalwareBazaar apikey: endpoint: https://mb-api.abuse.ch/api/v1/ -``` \ No newline at end of file +``` + +### Changelog + +#### v2.0.0 + +Siphon has introduced honeypot integration! Agents can now be configured and used on decoy hosts to log +information about and cache samples that are used by attackers in real time. These agents provide the same +interface as other integrations - with the ability to query and download recent samples. + +See the [docs](https://github.com/pygrum/siphon/blob/main/docs/DOCS.md) for how to build and configure +agents. diff --git a/cmd/agent/siphon_agent.go b/cmd/agent/siphon_agent.go new file mode 100644 index 0000000..4877c48 --- /dev/null +++ b/cmd/agent/siphon_agent.go @@ -0,0 +1,40 @@ +package main + +import ( + "github.com/pygrum/siphon/internal/agent" + "github.com/pygrum/siphon/internal/logger" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var ( + AgentID string + Interface string + Port string + ClientCertData string // Base64 Encoded certificate data + cfgFile string + + rootCmd = &cobra.Command{ + Use: "siphon_agent", + Short: "A Honeypot-Resident Sample Curator", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + viper.SetConfigFile(cfgFile) + if err := viper.ReadInConfig(); err != nil { + logger.Fatalf("reading configuration file failed: %v", err) + } + agent.Initialize(AgentID, Interface, Port, ClientCertData) + }, + } +) + +func init() { + rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "agent configuration file") + _ = cobra.MarkFlagRequired(rootCmd.PersistentFlags(), "config") +} + +func main() { + if err := rootCmd.Execute(); err != nil { + logger.Fatal(err) + } +} diff --git a/cmd/generator/generator/generator.go b/cmd/generator/generator/generator.go new file mode 100644 index 0000000..cd950ef --- /dev/null +++ b/cmd/generator/generator/generator.go @@ -0,0 +1,74 @@ +package generator + +import ( + "crypto/rand" + "encoding/base64" + "fmt" + "github.com/pygrum/siphon/internal/db" + "github.com/pygrum/siphon/internal/logger" + "github.com/spf13/viper" + "math/big" + "net" + "os" +) + +const ( + AgentIDLength = 16 + AgentIDPrefix = "AA" +) + +var charSet = []byte("0123456789ABCDEF") + +func RandID() string { + ret := make([]byte, AgentIDLength-len(AgentIDPrefix)) + for i := 0; i < AgentIDLength-len(AgentIDPrefix); i++ { + num, _ := rand.Int(rand.Reader, big.NewInt(int64(len(charSet)))) + ret[i] = charSet[num.Int64()] + } + return AgentIDPrefix + string(ret) +} + +func IsAgentID(s string) bool { + return len(s) == AgentIDLength && s[:2] == "AA" +} + +func Generate() error { + name := viper.GetString("name") + goos := viper.GetString("os") + arch := viper.GetString("arch") + iFace := viper.GetString("host") + port := viper.GetString("port") + outFile := viper.GetString("outfile") + siphonCert := viper.GetString("cert_file") + + certData, err := os.ReadFile(siphonCert) + if err != nil { + logger.Fatalf("could not read siphon certificate from %s: %v", siphonCert, err) + } + agentID := RandID() + if len(name) == 0 { + name = agentID + } + mainPath := viper.GetString("src_path") + + builder := NewBuilder("go", goos, arch) + builder.AddSrcPath(mainPath) + builder.SetFlags( + Flag{"main.AgentID", agentID}, + Flag{"main.Interface", iFace}, + Flag{"main.Port", port}, + // Add certificate data base64 encoded to agent, so it is added to its rootCAs + Flag{"main.ClientCertData", base64.StdEncoding.EncodeToString(certData)}, + ) + builder.SetOutFile(outFile) + // Exits if unsuccessful + builder.Build() + + conn := db.Initialize() + agent := &db.Agent{ + AgentID: agentID, + Name: name, + Endpoint: fmt.Sprintf("https://%s/api", net.JoinHostPort(iFace, port)), + } + return conn.Add(agent) +} diff --git a/cmd/generator/generator/wrapper.go b/cmd/generator/generator/wrapper.go new file mode 100644 index 0000000..e2e6c8e --- /dev/null +++ b/cmd/generator/generator/wrapper.go @@ -0,0 +1,78 @@ +package generator + +import ( + "bytes" + "fmt" + "github.com/pygrum/siphon/internal/logger" + "os" + "os/exec" + "strings" +) + +type Builder struct { + CC string + GOOS string + GOARCH string + SrcPaths []string + outFile string + flags []Flag +} + +type Flag struct { + Name string + Value string +} + +const ( + outFileOption = "-o" + flagsOption = "-ldflags" + flagPrefix = "-X" +) + +func NewBuilder(cc, goos, goarch string) *Builder { + return &Builder{ + CC: cc, + GOOS: goos, + GOARCH: goarch, + } +} + +func (b *Builder) AddSrcPath(path string) { + b.SrcPaths = append(b.SrcPaths, path) +} + +func (b *Builder) SetFlags(flags ...Flag) { + b.flags = flags +} + +func (b *Builder) SetOutFile(name string) { + b.outFile = name +} + +func (b *Builder) Build() { + var buildCmd []string + buildCmd = append(buildCmd, "build") + buildCmd = append(buildCmd, outFileOption) + buildCmd = append(buildCmd, b.outFile) + buildCmd = append(buildCmd, flagsOption) + var flags []string + for _, f := range b.flags { + flags = append(flags, flagPrefix) + formatString := "'%s=%s'" + flags = append(flags, fmt.Sprintf(formatString, f.Name, f.Value)) + } + buildCmd = append(buildCmd, strings.Join(flags, " ")) + for _, s := range b.SrcPaths { + buildCmd = append(buildCmd, s) + } + fmt.Println(b.CC, strings.Join(buildCmd, " ")) + var cerr bytes.Buffer + cmd := exec.Command(b.CC, buildCmd...) + // Set arch and os environment vars + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, fmt.Sprintf("GOOS=%s", b.GOOS), fmt.Sprintf("GOARCH=%s", b.GOARCH)) // go-sqlite3 requires cgo + cmd.Stderr = &cerr + if err := cmd.Run(); err != nil { + logger.Fatalf("%v: %s", err, cerr.String()) + } +} diff --git a/cmd/generator/siphon_gen.go b/cmd/generator/siphon_gen.go new file mode 100644 index 0000000..bf513a9 --- /dev/null +++ b/cmd/generator/siphon_gen.go @@ -0,0 +1,47 @@ +package main + +import ( + "fmt" + "github.com/pygrum/siphon/cmd/generator/generator" + "github.com/pygrum/siphon/internal/logger" + "github.com/pygrum/siphon/internal/version" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +func init() { + fmt.Printf("{_/¬ SIPHON GENERATOR %s ¬\\_}\n\n}", version.VersionString()) +} + +var ( + cfgFile string + + rootCmd = &cobra.Command{ + Use: "generator", + Short: "A utility for Siphon agent generation", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + viper.SetConfigFile(cfgFile) + if err := viper.ReadInConfig(); err != nil { + logger.Fatalf("reading configuration file failed: %v", err) + } + if err := generator.Generate(); err != nil { + logger.Fatal(err) + } + logger.Notifyf("agent has successfully been built. For installation instructions, see the docs: %s", + "https://github.com/pygrum/siphon/blob/main/docs/DOCS.md", + ) + }, + } +) + +func init() { + rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "generator configuration file - see https://github.com/pygrum/siphon/blob/main/docs/DOCS.md for help") + _ = cobra.MarkFlagRequired(rootCmd.PersistentFlags(), "config") +} + +func main() { + if err := rootCmd.Execute(); err != nil { + logger.Fatal(err) + } +} diff --git a/cmd/root.go b/cmd/siphon/siphon.go similarity index 83% rename from cmd/root.go rename to cmd/siphon/siphon.go index 61d532a..ef89624 100644 --- a/cmd/root.go +++ b/cmd/siphon/siphon.go @@ -1,4 +1,4 @@ -package cmd +package main import ( "errors" @@ -6,6 +6,7 @@ import ( "github.com/pygrum/siphon/internal/console" "github.com/pygrum/siphon/internal/db" "github.com/pygrum/siphon/internal/logger" + "github.com/pygrum/siphon/internal/version" "github.com/spf13/cobra" "github.com/spf13/viper" "os" @@ -13,12 +14,6 @@ import ( "strings" ) -const ( - VersionMajor = "1" - VersionMinor = "0" - VersionPatch = "0" -) - var ( cfgFile string @@ -33,7 +28,7 @@ var ( fmt.Println(strings.ReplaceAll( title(), "{VER}", - versionString())) + version.VersionString())) console.Start() }, } @@ -56,7 +51,6 @@ func initCfg() { if !errors.Is(err, os.ErrExist) { cobra.CheckErr(err) } - logger.Infof("creating new configuration file at %s", filepath.Join(cfgDir, ".siphon.yaml")) if _, err := os.Stat(filepath.Join(cfgDir, ".siphon.yaml")); os.IsNotExist(err) { err = os.WriteFile(filepath.Join(cfgDir, ".siphon.yaml"), cfgBoilerplate(), 0666) cobra.CheckErr(err) @@ -68,7 +62,7 @@ func initCfg() { } if err := viper.ReadInConfig(); err != nil { - logger.Errorf("reading configuration file ($HOME/.siphon.yaml) failed: %v", err) + logger.Errorf("reading configuration file failed: %v", err) } db.Initialize() } @@ -88,12 +82,10 @@ func title() string { ` } -func Execute() error { - return rootCmd.Execute() -} - -func versionString() string { - return "v" + strings.Join([]string{VersionMajor, VersionMinor, VersionPatch}, ".") +func main() { + if err := rootCmd.Execute(); err != nil { + logger.Fatal(err) + } } func cfgBoilerplate() []byte { diff --git a/configs/siphon.yaml b/configs/siphon.yaml new file mode 100644 index 0000000..27a39ec --- /dev/null +++ b/configs/siphon.yaml @@ -0,0 +1,9 @@ +# Example configuration file for Siphon + +refreshrate: 5 # Refresh sample list every 5 minutes +cert_file: "/path/to/cert/file.crt" +key_file: "/path/to/key/file.crt" +sources: +- name: "malwarebazaar" + endpoint: + apikey: \ No newline at end of file diff --git a/configs/siphon_agent.yaml b/configs/siphon_agent.yaml new file mode 100644 index 0000000..464b14c --- /dev/null +++ b/configs/siphon_agent.yaml @@ -0,0 +1,10 @@ +# Example configuration file for Siphon agent + +cache: true +cert_file: "/path/to/certificate/file" +key_file: "/path/to/key/file" +monitor_folders: + - path: "/path/to/folder_1" + recursive: true + - path: "/path/to/folder_2" + recursive: false \ No newline at end of file diff --git a/configs/siphon_gen.yaml b/configs/siphon_gen.yaml new file mode 100644 index 0000000..7244ce2 --- /dev/null +++ b/configs/siphon_gen.yaml @@ -0,0 +1,10 @@ +# Example configuration file for Siphon's agent generator + +cert_file: "/path/to/siphon/client/certificate" +src_path: "/path/to/siphon/agent/source/file" # This would be ${SRC}/cmd/agent/siphon_agent.go +name: "agent" +os: "windows" +arch: "amd64" +host: "0.0.0.0" +port: "8080" +outfile: "agent.exe" \ No newline at end of file diff --git a/docs/DOCS.md b/docs/DOCS.md index 320f8a0..0cc9140 100644 --- a/docs/DOCS.md +++ b/docs/DOCS.md @@ -1,20 +1,66 @@ # Configuration +## Siphon Here is a full example configuration for Siphon: ```yaml RefreshRate: 5 # Refresh sample list every 5 minutes +cert_file: "/path/to/cert/file.crt" +key_file: "/path/to/key/file.crt" Sources: -- name: VirusTotal +- name: "VirusTotal" endpoint: APIKey: -- name: MalwareBazaar +- name: "MalwareBazaar" endpoint: APIKey: -- name: HybridAnalysis +- name: "HybridAnalysis" endpoint: APIKey: ``` Each entry in `source` correlates to a supported threat intelligence feed integration. -All that is needed is an endpoint and API key. \ No newline at end of file +All that is needed is an endpoint and API key. + +## Siphon agent +The siphon agent monitors folders located in a honeypot. +Here is an example configuration for the Siphon agent: + +```yaml +cache: true +cert_file: "/path/to/certificate/file" +key_file: "/path/to/key/file" +monitor_folders: + - path: "/path/to/folder_1" + recursive: true + - path: "/path/to/folder_2" + recursive: false +``` + +If `cache` is set to true, then files written to disk in the monitored folders +will be saved to a protected folder, in case they get deleted or moved later on. + +The `recursive` monitoring option means that that specific folder and all subfolders will +be monitored for file changes, otherwise, only the top level folder will be monitored. + +## Agent generator +Here is an example agent generator file. These fields are used to build a new agent. +```yaml +cert_file: "/path/to/siphon/client/certificate" +src_path: "/path/to/siphon/source/folder" +name: "agent" +os: "windows" +arch: "amd64" +host: "0.0.0.0" +port: "8080" +outfile: "agent.exe" +``` + +### Agent build steps + +After generating an agent with `siphon_gen`, run the `setup_agent.sh` script (under /scripts) to +create a folder with the agent, tls key pair and default configuration file, which you should tweak as you wish. + +You'll need to update the configuration file with: +1. The path to your key pair when _it's on the target machine_ (this is set to `/root/agent.` by default) +2. Folders to monitor on the host, following the same format as shown under 'Siphon agent' above \ No newline at end of file diff --git a/go.mod b/go.mod index 01b757b..24aa0ac 100644 --- a/go.mod +++ b/go.mod @@ -9,17 +9,35 @@ require ( ) require ( + github.com/alexmullins/zip v0.0.0-20180717182244-4affb64b04d0 // indirect + github.com/bytedance/sonic v1.10.1 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect + github.com/chenzhuoyu/iasm v0.9.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/gin-gonic/gin v1.9.1 // indirect + github.com/go-fsnotify/fsnotify v0.0.0-20180321022601-755488143dae // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.15.4 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jedib0t/go-pretty/v6 v6.4.7 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/leodido/go-urn v1.2.4 // indirect github.com/magiconair/properties v1.8.7 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/mattn/go-sqlite3 v1.14.17 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/reeflective/readline v1.0.10 // indirect github.com/rivo/uniseg v0.4.4 // indirect @@ -29,10 +47,16 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.11 // indirect + golang.org/x/arch v0.5.0 // indirect + golang.org/x/crypto v0.13.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/net v0.15.0 // indirect golang.org/x/sys v0.12.0 // indirect golang.org/x/term v0.12.0 // indirect golang.org/x/text v0.13.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gorm.io/driver/sqlite v1.5.3 // indirect diff --git a/go.sum b/go.sum index 5a2302e..f66107c 100644 --- a/go.sum +++ b/go.sum @@ -38,7 +38,19 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/alexmullins/zip v0.0.0-20180717182244-4affb64b04d0 h1:BVts5dexXf4i+JX8tXlKT0aKoi38JwTXSe+3WUneX0k= +github.com/alexmullins/zip v0.0.0-20180717182244-4affb64b04d0/go.mod h1:FDIQmoMNJJl5/k7upZEnGvgWVZfFeE6qHeN7iCMbCsA= +github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= +github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= +github.com/bytedance/sonic v1.10.1 h1:7a1wuFXL1cMy7a3f7/VFcEtriuXQnUBhtoVfOZiaysc= +github.com/bytedance/sonic v1.10.1/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= +github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0= +github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA= +github.com/chenzhuoyu/iasm v0.9.0 h1:9fhXjVzq5hUy2gkhhgHl95zG2cEAhw9OSGs8toWWAwo= +github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -60,9 +72,25 @@ github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0X github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= +github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= +github.com/go-fsnotify/fsnotify v0.0.0-20180321022601-755488143dae h1:PeVNzgTRtWGm6fVic5i21t+n5ptPGCZuMcSPVMyTWjs= +github.com/go-fsnotify/fsnotify v0.0.0-20180321022601-755488143dae/go.mod h1:BbhqyaehKPCLD83cqfRYdm177Ylm1cdGHu3txjbQSQI= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.15.4 h1:zMXza4EpOdooxPel5xDqXEdXG5r+WggpvnAKMsalBjs= +github.com/go-playground/validator/v10 v10.15.4/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -88,6 +116,7 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -99,8 +128,10 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -133,11 +164,17 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -146,14 +183,23 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -191,16 +237,23 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= +github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -211,6 +264,9 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.5.0 h1:jpGode6huXQxcskEIpOCvrU+tzo81b6+oFLUYXWtH/Y= +golang.org/x/arch v0.5.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -218,6 +274,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -285,6 +343,8 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -341,6 +401,8 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -498,6 +560,9 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -520,6 +585,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/agent/agent.go b/internal/agent/agent.go new file mode 100644 index 0000000..636a171 --- /dev/null +++ b/internal/agent/agent.go @@ -0,0 +1,29 @@ +package agent + +import ( + "github.com/pygrum/siphon/internal/agent/controllers" + "github.com/pygrum/siphon/internal/agent/monitor" + "github.com/pygrum/siphon/internal/logger" +) + +func Initialize(id, iFace, port, clientCertData string) { + agent := controllers.NewAgent(id, iFace, port, clientCertData) + // person who compiled the agent is the only one who can add and remove users - their username 'root' is special + setupControllers(agent) + scout, err := monitor.NewScout(agent) + if err != nil { + logger.Fatalf("failed to create a new scout: %v", err) + } + if err := scout.Start(); err != nil { + logger.Fatalf("failed to start scout: %v", err) + } + logger.Fatal(scout.RunTLS()) +} + +func setupControllers(agent *controllers.Agent) { + apiRouter := agent.Router.Group("/api") + { + apiRouter.GET("samples", agent.GetSamples) + apiRouter.GET("download", agent.GetSampleByHash) + } +} diff --git a/internal/agent/controllers/controllers.go b/internal/agent/controllers/controllers.go new file mode 100644 index 0000000..a757612 --- /dev/null +++ b/internal/agent/controllers/controllers.go @@ -0,0 +1,93 @@ +package controllers + +import ( + "crypto/tls" + "encoding/base64" + "github.com/gin-gonic/gin" + "github.com/pygrum/siphon/internal/agent/utils" + "github.com/pygrum/siphon/internal/db" + "github.com/pygrum/siphon/internal/logger" + "net" + "net/http" + "os" + "time" +) + +// TODO: Use gin to setup API endpoints, that will be used to (DONE) info about recent samples, and (DONE) samples by hash. + +const ( + StatusOk = "ok" + StatusNotFound = "not_found" +) + +func NewAgent(id, iFace, port, clientCertData string) *Agent { + certData, err := base64.StdEncoding.DecodeString(clientCertData) + if err != nil { + logger.Fatalf("could not decode client certificate: %v", err) + } + a := &Agent{ + Router: gin.Default(), + ID: id, + ClientCertData: certData, + BindAddress: net.JoinHostPort(iFace, port), + Conn: db.AgentInitialize(), + } + return a +} + +func (a *Agent) RunTLS(tlsConfig *tls.Config) error { + server := &http.Server{ + Addr: a.BindAddress, + Handler: a.Router, + TLSConfig: tlsConfig, + } + return server.ListenAndServeTLS(a.CertFile, a.KeyFile) +} + +func (a *Agent) GetSamples(c *gin.Context) { + // Return samples discovered within the last hour + samples, err := a.Conn.SamplesByTime(time.Now().Add(-1 * time.Hour)) + if err != nil { + c.JSON(http.StatusInternalServerError, SampleResponse{ + Status: err.Error(), + }) + } + c.JSON(http.StatusOK, SampleResponse{ + Status: StatusOk, + Data: samples, + }) +} + +func (a *Agent) GetSampleByHash(c *gin.Context) { + hash := c.Query("sha256_hash") + // Return samples discovered within the last hour + sample, err := a.Conn.SampleByHash(hash) + if err != nil { + c.JSON(http.StatusInternalServerError, SampleResponse{ + Status: err.Error(), + }) + return + } + if sample == nil { + c.JSON(http.StatusOK, SampleResponse{ + Status: StatusNotFound, + }) + return + } + file, err := utils.ZipFile(sample, "infected") + if err != nil { + c.JSON(http.StatusOK, SampleResponse{ + Status: StatusNotFound, + }) + return + } + bytes, err := os.ReadFile(file) + if err != nil { + c.JSON(http.StatusInternalServerError, SampleResponse{ + Status: err.Error(), + }) + return + } + c.Data(http.StatusOK, "application/zip", bytes) + _ = os.Remove(file) +} diff --git a/internal/agent/controllers/models.go b/internal/agent/controllers/models.go new file mode 100644 index 0000000..3ef4408 --- /dev/null +++ b/internal/agent/controllers/models.go @@ -0,0 +1,21 @@ +package controllers + +import ( + "github.com/gin-gonic/gin" + "github.com/pygrum/siphon/internal/db" +) + +type Agent struct { + Router *gin.Engine + ID string + BindAddress string + CertFile string + KeyFile string + ClientCertData []byte // Get this at compile time + Conn *db.AgentConn +} + +type SampleResponse struct { + Status string `json:"status"` + Data []db.Sample `json:"data"` +} diff --git a/internal/agent/monitor/monitor.go b/internal/agent/monitor/monitor.go new file mode 100644 index 0000000..1aafe6d --- /dev/null +++ b/internal/agent/monitor/monitor.go @@ -0,0 +1,190 @@ +package monitor + +import ( + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "fmt" + "github.com/fsnotify/fsnotify" + "github.com/pygrum/siphon/internal/agent/controllers" + "github.com/pygrum/siphon/internal/db" + "github.com/pygrum/siphon/internal/logger" + "github.com/spf13/viper" + "gorm.io/gorm" + "io" + "log" + "os" + "path/filepath" + "time" +) + +type Scout struct { + Agent *controllers.Agent + Logger *logger.Logger + MonitorPaths []monitorPath +} + +type monitorPath struct { + Path string + Recursive bool +} + +var watcher *fsnotify.Watcher + +func NewScout(agent *controllers.Agent) (*Scout, error) { + var err error + watcher, err = fsnotify.NewWatcher() + if err != nil { + return nil, err + } + var folders []monitorPath + if err := viper.UnmarshalKey("monitor_folders", &folders); err != nil { + return nil, err + } + l, err := logger.NewLogger("./agent_monitor.log") + if err != nil { + return nil, err + } + return &Scout{ + Agent: agent, + MonitorPaths: folders, + Logger: l, + }, nil +} + +func (s *Scout) Start() error { + for _, d := range s.MonitorPaths { + if err := addDirectory(&d); err != nil { + return err + } + } + go s.start() + return nil +} + +func (s *Scout) RunTLS() error { + s.Agent.CertFile = viper.GetString("cert_file") + s.Agent.KeyFile = viper.GetString("key_file") + + cert := s.Agent.ClientCertData + + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(cert) + tlsConfig := &tls.Config{ + ClientCAs: certPool, + InsecureSkipVerify: true, + VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { + certs := make([]*x509.Certificate, len(rawCerts)) + for i, asn1Data := range rawCerts { + cert, err := x509.ParseCertificate(asn1Data) + if err != nil { + return fmt.Errorf("failed to parse certificate: %v", err) + } + certs[i] = cert + } + opts := x509.VerifyOptions{ + Roots: certPool, + CurrentTime: time.Now(), + DNSName: "", // Skip hostname verification + Intermediates: x509.NewCertPool(), + } + + for i, cert := range certs { + if i == 0 { + continue + } + opts.Intermediates.AddCert(cert) + } + _, err := certs[0].Verify(opts) + return err + }, + } + return s.Agent.RunTLS(tlsConfig) +} + +func (s *Scout) start() { + for { + select { + case event := <-watcher.Events: + if event.Has(fsnotify.Create) { + s.Logger.Write(logger.Sinfof("file created: %s", event.Name)) + if err := s.Add(event.Name); err != nil { + s.Logger.Write(logger.Serrorf("failed to add %s to database: %v", event.Name, err)) + } + } else if event.Has(fsnotify.Write) { + s.Logger.Write(logger.Sinfof("file written: %s", event.Name)) + if err := s.Add(event.Name); err != nil { + s.Logger.Write(logger.Serrorf("failed to add %s to database: %v", event.Name, err)) + } + } else if event.Has(fsnotify.Rename) { + s.Logger.Write(logger.Sinfof("file renamed: %s", event.Name)) + } else if event.Has(fsnotify.Remove) { + s.Logger.Write(logger.Sinfof("file removed: %s", event.Name)) + } else { + s.Logger.Write(logger.Sinfof("file mode changed: %s", event.Name)) + } + break + case event := <-watcher.Errors: + s.Logger.Write(logger.Serrorf("error: %s", event.Error())) + break + } + } +} + +func (s *Scout) Add(file string) error { + conn := s.Agent.Conn + fi, err := os.Stat(file) + if err != nil { + return err + } + // Don't save empty files + if fi.Size() == 0 { + return nil + } + // If file caching is on, save copy of file to agent folder (protected) + if viper.GetBool("cache") { + b, _ := os.ReadFile(file) + cwd, _ := os.Getwd() + newFileName := filepath.Join(cwd, filepath.Base(file)) + _ = os.WriteFile(newFileName, b, 0600) + file = newFileName + } + + sample := &db.Sample{ + // Set creation time manually + Model: gorm.Model{ + CreatedAt: fi.ModTime(), + }, + Name: filepath.Base(file), + Path: file, + FileType: filepath.Ext(file)[1:], // Skip the leading '.' + FileSize: uint(fi.Size()), + Source: s.Agent.ID, + Hash: fileHash(file), + } + return conn.Add(sample) +} + +func fileHash(file string) string { + f, _ := os.Open(file) + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + log.Fatal(err) + } + return fmt.Sprintf("%x", h.Sum(nil)) +} + +func addDirectory(mp *monitorPath) error { + if !mp.Recursive { + return watcher.Add(mp.Path) + } + // Add folders to watch recursively + return filepath.Walk(mp.Path, func(path string, fi os.FileInfo, err error) error { + if fi.IsDir() { + return watcher.Add(path) + } + return nil + }) +} diff --git a/internal/agent/utils/utils.go b/internal/agent/utils/utils.go new file mode 100644 index 0000000..d9d6c47 --- /dev/null +++ b/internal/agent/utils/utils.go @@ -0,0 +1,33 @@ +package utils + +import ( + "bytes" + "github.com/alexmullins/zip" + "github.com/pygrum/siphon/internal/db" + "io" + "os" +) + +func ZipFile(sample *db.Sample, pass string) (string, error) { + contents, err := os.ReadFile(sample.Path) + if err != nil { + return "", err + } + zipName := sample.Hash + ".zip" + zipFile, err := os.Create(zipName) + if err != nil { + return "", err + } + defer zipFile.Close() + zipWriter := zip.NewWriter(zipFile) + defer zipWriter.Close() + w, err := zipWriter.Encrypt(sample.Name, pass) + if err != nil { + return "", err + } + _, err = io.Copy(w, bytes.NewReader(contents)) + if err != nil { + return "", err + } + return zipName, nil +} diff --git a/internal/commands/agents/agents.go b/internal/commands/agents/agents.go new file mode 100644 index 0000000..fcd1dd3 --- /dev/null +++ b/internal/commands/agents/agents.go @@ -0,0 +1,47 @@ +package agents + +import ( + "fmt" + "github.com/jedib0t/go-pretty/v6/table" + "github.com/pygrum/siphon/internal/db" + "github.com/pygrum/siphon/internal/logger" +) + +var conn *db.Conn + +func init() { + conn = db.Initialize() +} + +func AgentsCmd() { + // Do nothing if zero sample count + agents := conn.Agents() + if len(agents) == 0 { + logger.Info("no samples loaded - check your internet connection or API configuration") + } else { + RenderTable(agents) + } +} + +func RenderTable(agents []db.Agent) { + t := table.NewWriter() + tmp := table.Table{} + tmp.Render() + + t.SetStyle(table.StyleBold) + + header := table.Row{"ID", "NAME", "ENDPOINT", "CERTIFICATE", "CREATION TIME"} + for _, a := range agents { + row := table.Row{ + a.AgentID, + a.Name, + a.Endpoint, + a.CertPath, + a.CreatedAt, + } + t.AppendRow(row) + } + t.AppendHeader(header) + t.SetAutoIndex(false) + fmt.Println(t.Render()) +} diff --git a/internal/commands/agents/set/set.go b/internal/commands/agents/set/set.go new file mode 100644 index 0000000..e5b462b --- /dev/null +++ b/internal/commands/agents/set/set.go @@ -0,0 +1,30 @@ +package set + +import ( + "github.com/pygrum/siphon/internal/db" + "github.com/pygrum/siphon/internal/logger" +) + +func SetCmd(name, endpoint, certPath string) { + conn := db.Initialize() + a := conn.AgentByName(name) + if a == nil { + a = conn.AgentByID(name) + if a == nil { + logger.Errorf("agent with name/id %s does not exist", name) + return + } + } + if len(endpoint) != 0 { + a.Endpoint = endpoint + } + if len(certPath) != 0 { + a.CertPath = certPath + } + err := conn.Add(a) + if err != nil { + logger.Errorf("failed to update agent fields: %v", err) + return + } + logger.Notify("agent successfully updated") +} diff --git a/internal/commands/commands.go b/internal/commands/commands.go index d829a57..c961c39 100644 --- a/internal/commands/commands.go +++ b/internal/commands/commands.go @@ -1,6 +1,8 @@ package commands import ( + "github.com/pygrum/siphon/internal/commands/agents" + "github.com/pygrum/siphon/internal/commands/agents/set" "github.com/pygrum/siphon/internal/commands/exit" "github.com/pygrum/siphon/internal/commands/get" "github.com/pygrum/siphon/internal/commands/info" @@ -33,7 +35,6 @@ func Commands() *cobra.Command { newCmd.Flags().StringVarP(&newApiKey, "api-key", "k", "", "api key for source") newCmd.Flags().StringVarP(&newEndpoint, "endpoint", "e", "", "source API endpoint") _ = cobra.MarkFlagRequired(newCmd.Flags(), "name") - sourcesCmd.AddCommand(newCmd) var sampleCount string @@ -82,9 +83,32 @@ func Commands() *cobra.Command { }, } + agentsCmd := &cobra.Command{ + Use: "agents", + Short: "View known agents", + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + agents.AgentsCmd() + }, + } + + var endpoint, certFile string + setCmd := &cobra.Command{ + Use: "set [id]", + Short: "Configure agent integration parameters", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + set.SetCmd(args[0], endpoint, certFile) + }, + } + setCmd.Flags().StringVarP(&endpoint, "endpoint", "e", "", "API endpoint for the agent") + setCmd.Flags().StringVarP(&certFile, "cert-file", "c", "", "path to agent certificate file") + agentsCmd.AddCommand(setCmd) + cmd.AddCommand(infoCmd) cmd.AddCommand(getCmd) cmd.AddCommand(sourcesCmd) + cmd.AddCommand(agentsCmd) cmd.AddCommand(samplesCmd) cmd.AddCommand(exitCmd) diff --git a/internal/commands/get/get.go b/internal/commands/get/get.go index c98961d..9200048 100644 --- a/internal/commands/get/get.go +++ b/internal/commands/get/get.go @@ -1,8 +1,9 @@ package get import ( + "github.com/pygrum/siphon/cmd/generator/generator" "github.com/pygrum/siphon/internal/db" - "github.com/pygrum/siphon/internal/db/models" + "github.com/pygrum/siphon/internal/integrations/agent" "github.com/pygrum/siphon/internal/integrations/malwarebazaar" "github.com/pygrum/siphon/internal/logger" "github.com/pygrum/siphon/internal/siphon" @@ -12,19 +13,30 @@ import ( ) func GetCmd(id, out string, persist bool) { + conn := db.Initialize() uid, err := strconv.Atoi(id) - spl := &models.Sample{} + spl := &db.Sample{} if err != nil { - spl = db.SampleByName(id) + spl = conn.SampleByName(id) } else { - spl = db.SampleByID(uint(uid)) + spl = conn.SampleByID(uint(uid)) } if spl == nil { logger.Errorf("sample '%s': not found", id) return } - if strings.ToLower(spl.Source) == malwarebazaar.Source { - fileBytes := getFromMB(spl) + var fileBytes []byte + if strings.ToLower(spl.Source) == malwarebazaar.Source || generator.IsAgentID(spl.Source) { + if strings.ToLower(spl.Source) == malwarebazaar.Source { + fileBytes = getFromMB(spl) + } else if generator.IsAgentID(spl.Source) { + agt := conn.AgentByID(spl.Source) + if agt == nil { + logger.Errorf("agent AgentID '%s' is not present in database", spl.Source) + return + } + fileBytes = getFromAgent(agt, spl) + } if fileBytes != nil { if len(out) == 0 { out = spl.Hash + "." + spl.FileType + ".zip" // MalwareBazaar returns as zip @@ -42,7 +54,7 @@ func GetCmd(id, out string, persist bool) { } } -func getFromMB(spl *models.Sample) []byte { +func getFromMB(spl *db.Sample) []byte { f := malwarebazaar.NewFetcher() if f == nil { logger.Error("cannot fetch from MalwareBazaar: not configured correctly") @@ -59,3 +71,18 @@ func getFromMB(spl *models.Sample) []byte { } return bytes } + +func getFromAgent(agt *db.Agent, spl *db.Sample) []byte { + f := agent.NewFetcher() + readCloser, err := f.Download(agt, spl.Hash) + if err != nil { + logger.Errorf("download failed: %v", err) + return nil + } + defer readCloser.Close() + bytes, err := io.ReadAll(readCloser) + if err != nil { + logger.Errorf("cannot read response body: %v", err) + } + return bytes +} diff --git a/internal/commands/info/info.go b/internal/commands/info/info.go index faea74a..77a8969 100644 --- a/internal/commands/info/info.go +++ b/internal/commands/info/info.go @@ -3,20 +3,25 @@ package info import ( "github.com/pygrum/siphon/internal/commands/samples" "github.com/pygrum/siphon/internal/db" - "github.com/pygrum/siphon/internal/db/models" "github.com/pygrum/siphon/internal/logger" "strconv" ) +var conn *db.Conn + +func init() { + conn = db.Initialize() +} + func InfoCmd(noTrunc bool, ids ...string) { - var sampleList []models.Sample + var sampleList []db.Sample for _, id := range ids { uid, err := strconv.Atoi(id) - spl := &models.Sample{} + spl := &db.Sample{} if err != nil { - spl = db.SampleByName(id) + spl = conn.SampleByName(id) } else { - spl = db.SampleByID(uint(uid)) + spl = conn.SampleByID(uint(uid)) } if spl == nil { logger.Errorf("sample '%s': not found", id) @@ -28,12 +33,12 @@ func InfoCmd(noTrunc bool, ids ...string) { samples.RenderTable(sampleList, noTrunc) } -func clean(spls []models.Sample) []models.Sample { - m := make(map[uint]models.Sample) +func clean(spls []db.Sample) []db.Sample { + m := make(map[uint]db.Sample) for _, x := range spls { m[x.ID] = x } - var cleaned []models.Sample + var cleaned []db.Sample for _, v := range m { cleaned = append(cleaned, v) } diff --git a/internal/commands/samples/samples.go b/internal/commands/samples/samples.go index 44664d1..52402bd 100644 --- a/internal/commands/samples/samples.go +++ b/internal/commands/samples/samples.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/jedib0t/go-pretty/v6/table" "github.com/pygrum/siphon/internal/db" - "github.com/pygrum/siphon/internal/db/models" "github.com/pygrum/siphon/internal/logger" "strconv" "strings" @@ -14,12 +13,18 @@ const ( truncLimit = 40 ) +var conn *db.Conn + +func init() { + conn = db.Initialize() +} + func SamplesCmd(sampleCount string, noTrunc bool) { sc := 0 i, err := strconv.Atoi(sampleCount) if err != nil { if strings.ToLower(sampleCount) == "all" { - sc = db.Count() + sc = conn.Count() } else { logger.Errorf("valid integer argument required") return @@ -33,7 +38,7 @@ func SamplesCmd(sampleCount string, noTrunc bool) { } // Do nothing if zero sample count if sc > 0 { - samples := db.Samples(sc) + samples := conn.Samples(sc) if len(samples) == 0 { logger.Info("no samples loaded - check your internet connection or API configuration") } else { @@ -42,7 +47,7 @@ func SamplesCmd(sampleCount string, noTrunc bool) { } } -func RenderTable(samples []models.Sample, v bool) { +func RenderTable(samples []db.Sample, v bool) { t := table.NewWriter() tmp := table.Table{} tmp.Render() diff --git a/internal/commands/sources/sources.go b/internal/commands/sources/sources.go index a2c703f..9d439f5 100644 --- a/internal/commands/sources/sources.go +++ b/internal/commands/sources/sources.go @@ -35,7 +35,7 @@ func FindSource(sourceName string) *Source { return nil } for _, s := range sources { - if s.Name == sourceName { + if strings.ToLower(s.Name) == strings.ToLower(sourceName) { return &s } } diff --git a/internal/db/db.go b/internal/db/db.go index 1c1d963..4d1caf8 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -1,26 +1,34 @@ package db import ( - "github.com/pygrum/siphon/internal/db/models" + "encoding/json" "github.com/pygrum/siphon/internal/logger" "github.com/spf13/cobra" "gorm.io/driver/sqlite" "gorm.io/gorm" l "gorm.io/gorm/logger" + "runtime" + "strings" "sync" + "time" "os" "path/filepath" ) -var conn struct { +type Conn struct { File string DB *gorm.DB } +type AgentConn struct { + File string +} + var m sync.Mutex -func Initialize() { +func Initialize() *Conn { + conn := Conn{} home, err := os.UserHomeDir() cobra.CheckErr(err) dbFile := filepath.Join(home, ".siphon", "siphon.db") @@ -38,62 +46,176 @@ func Initialize() { if err != nil { logger.Fatalf("failed to open database at %s: %v", dbFile, err) } - _ = db.AutoMigrate(models.Sample{}) + _ = db.AutoMigrate(Sample{}, Agent{}) conn.DB = db + return &conn } -func Samples(count int) []models.Sample { - var samples []models.Sample +func AgentInitialize() *AgentConn { + conn := AgentConn{} + // Safe path to store siphon data based on OS + var RestrictedPath string + switch runtime.GOOS { + case "windows": + RestrictedPath = "C:\\Windows\\System32\\config" + default: + RestrictedPath = "/root" + } + if err := os.Mkdir(filepath.Join(RestrictedPath, ".siphon_agent"), 0700); err != nil && !os.IsExist(err) { + logger.Fatalf("could not create a protected folder in %s: %v", RestrictedPath, err) + } + jsonFile := filepath.Join(RestrictedPath, ".siphon_agent", "agent.json") + if err := os.WriteFile(jsonFile, []byte("[]"), 0600); err != nil { + logger.Fatalf("could not initialise json database: %v", err) + } + conn.File = jsonFile + return &conn +} + +func (conn *Conn) SamplesByTime(dateTime time.Time) []Sample { + var samples []Sample + conn.DB.Where("created_at > ?", dateTime.Format(time.DateTime)).Find(&samples) + return samples +} +func (conn *Conn) Samples(count int) []Sample { + var samples []Sample conn.DB.Order("upload_time DESC, created_at DESC").Limit(count).Find(&samples) return samples } -func Count() int { - var samples []models.Sample +func (conn *Conn) Agents() []Agent { + var agents []Agent + conn.DB.Order("created_at DESC").Find(&agents) + return agents +} + +func (conn *Conn) Count() int { + var samples []Sample var count int64 conn.DB.Find(&samples).Count(&count) return int(count) } -func SampleByHash(md5Hash string) *models.Sample { - sample := &models.Sample{} +func (conn *Conn) SampleByHash(sha256hash string) *Sample { + sample := &Sample{} m.Lock() defer m.Unlock() - conn.DB.Where("hash = ?", md5Hash).First(&sample) + conn.DB.Where("hash = ?", sha256hash).First(&sample) // Return if empty sample received - if (models.Sample{}) == *sample { + if (Sample{}) == *sample { return nil } return sample } -func SampleByName(name string) *models.Sample { - sample := &models.Sample{} +func (conn *Conn) SampleByName(name string) *Sample { + sample := &Sample{} m.Lock() defer m.Unlock() conn.DB.Where("name = ?", name).First(&sample) // Return if empty sample received - if (models.Sample{}) == *sample { + if (Sample{}) == *sample { return nil } return sample } -func SampleByID(id uint) *models.Sample { - sample := &models.Sample{} +func (conn *Conn) AgentByID(id string) *Agent { + agent := &Agent{} m.Lock() defer m.Unlock() - conn.DB.Where("id = ?", id).First(&sample) + conn.DB.Where("agent_id = ?", id).First(&agent) + // Return if empty agent received + if (Agent{}) == *agent { + return nil + } + return agent +} + +func (conn *Conn) AgentByName(name string) *Agent { + agent := &Agent{} + m.Lock() + defer m.Unlock() + conn.DB.Where("name = ?", name).First(&agent) + // Return if empty agent received + if (Agent{}) == *agent { + return nil + } + return agent +} + +func (conn *Conn) SampleByID(id uint) *Sample { + sample := &Sample{} + m.Lock() + defer m.Unlock() + conn.DB.Where("id = ?", id).First(sample) // Return if empty sample received - if (models.Sample{}) == *sample { + if (Sample{}) == *sample { return nil } return sample } -func AddSample(sample *models.Sample) error { +func (conn *Conn) Add(v interface{}) error { m.Lock() defer m.Unlock() - result := conn.DB.Create(sample) + result := conn.DB.Save(v) return result.Error } + +func (aConn *AgentConn) Add(s *Sample) error { + samples, err := aConn.Samples() + if err != nil { + return err + } + samples = append(samples, *s) + return aConn.Save(samples) +} + +func (aConn *AgentConn) Save(samples []Sample) error { + bytes, err := json.Marshal(samples) + if err != nil { + return err + } + return os.WriteFile(aConn.File, bytes, 0600) +} + +func (aConn *AgentConn) Samples() ([]Sample, error) { + bytes, err := os.ReadFile(aConn.File) + if err != nil { + return nil, err + } + var samples []Sample + if err = json.Unmarshal(bytes, &samples); err != nil { + return nil, err + } + return samples, nil +} + +func (aConn *AgentConn) SamplesByTime(dateTime time.Time) ([]Sample, error) { + samples, err := aConn.Samples() + if err != nil { + return nil, err + } + var recentSamples []Sample + for _, sample := range samples { + // If it was created after an hour ago + if sample.CreatedAt.After(dateTime) { + recentSamples = append(recentSamples, sample) + } + } + return recentSamples, nil +} + +func (aConn *AgentConn) SampleByHash(sha256hash string) (*Sample, error) { + samples, err := aConn.Samples() + if err != nil { + return nil, err + } + for _, sample := range samples { + if strings.EqualFold(sample.Hash, sha256hash) { + return &sample, nil + } + } + return nil, nil +} diff --git a/internal/db/models.go b/internal/db/models.go new file mode 100644 index 0000000..597a730 --- /dev/null +++ b/internal/db/models.go @@ -0,0 +1,27 @@ +package db + +import ( + "gorm.io/gorm" + "time" +) + +type Sample struct { + gorm.Model + Name string `json:"name"` + Path string `json:"path"` + FileType string `json:"file_type"` + FileSize uint `json:"file_size"` + Signature string `json:"signature"` + Source string `json:"source"` + + Hash string `json:"hash"` + UploadTime time.Time `json:"upload_time"` +} + +type Agent struct { + gorm.Model + AgentID string `yaml:"agent_id"` + Name string `yaml:"name"` + Endpoint string `yaml:"endpoint"` + CertPath string `yaml:"certificate_path"` +} diff --git a/internal/db/models/models.go b/internal/db/models/models.go deleted file mode 100644 index 559d8d6..0000000 --- a/internal/db/models/models.go +++ /dev/null @@ -1,18 +0,0 @@ -package models - -import ( - "gorm.io/gorm" - "time" -) - -type Sample struct { - gorm.Model - Name string - FileType string - FileSize uint - Signature string - Source string - - Hash string - UploadTime time.Time -} diff --git a/internal/integrations/agent/agent.go b/internal/integrations/agent/agent.go new file mode 100644 index 0000000..e11c3d1 --- /dev/null +++ b/internal/integrations/agent/agent.go @@ -0,0 +1,164 @@ +package agent + +import ( + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "github.com/pygrum/siphon/internal/agent/controllers" + "github.com/pygrum/siphon/internal/db" + "github.com/pygrum/siphon/internal/logger" + "github.com/spf13/viper" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +type Fetcher struct { + Agents []db.Agent + Conn *db.Conn +} + +func NewFetcher() *Fetcher { + f := &Fetcher{ + Conn: db.Initialize(), + } + f.Agents = f.Conn.Agents() + return f +} + +func (f *Fetcher) GetRecent() { + for _, agent := range f.Agents { + a := agent + go func(agent *db.Agent) { + resp, _ := f.BasicRequest(agent, "/samples", "", "") + if resp == nil { + return + } + body, _ := io.ReadAll(resp.Body) + respObject := &controllers.SampleResponse{} + if err := json.Unmarshal(body, respObject); err != nil { + logger.Silentf("%v", err) + return + } + if respObject.Status != controllers.StatusOk { + logger.Silentf("request to (%s:%s) returned error %s", agent.Name, agent.AgentID, respObject.Status) + return + } + if err := f.addSamples(respObject); err != nil { + logger.Silentf("failed to add new samples: %v", err) + } + }(&a) + } +} + +func (f *Fetcher) addSamples(r *controllers.SampleResponse) error { + for _, data := range r.Data { + d := data + d.ID = 0 // Unset ID (primary key) to let gorm create normal record + go func(data db.Sample) { + if f.Conn.SampleByHash(data.Hash) == nil { + if err := f.Conn.Add(&data); err != nil { + logger.Silentf("%v", err) + } + } + }(d) + } + return nil +} + +func (f *Fetcher) BasicRequest(a *db.Agent, endpoint, query, form string) (*http.Response, error) { + r, err := http.NewRequest(http.MethodGet, a.Endpoint+endpoint, strings.NewReader(form)) + if err != nil { + logger.Silentf("unable to create new request for (%s:%s): %v", a.Name, a.AgentID, err) + return nil, err + } + r.URL.RawQuery = query + client, err := f.mTLSClient(a) + if err != nil { + logger.Silentf("failed to create mTLS client: %v", err) + return nil, err + } + resp, err := client.Do(r) + if err != nil { + logger.Silentf("request failed: %v", err) + return nil, err + } + return resp, nil +} + +func (f *Fetcher) mTLSClient(agent *db.Agent) (*http.Client, error) { + serverCertFile := agent.CertPath + certFile, keyFile := viper.GetString("cert_file"), viper.GetString("key_file") + // Read server certificate file and add it to trusted certificate store (certPool). Right now I'm reading my cert instead of the servers + caCert, err := os.ReadFile(serverCertFile) + if err != nil { + return nil, err + } + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(caCert) + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, err + } + return &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certPool, + Certificates: []tls.Certificate{cert}, + InsecureSkipVerify: true, + // function from github.com/digitalbitbox/bitbox-wallet-app/backend/coins/btc/electrum/electrum.go#L76-L111 + VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { + certs := make([]*x509.Certificate, len(rawCerts)) + for i, asn1Data := range rawCerts { + cert, err := x509.ParseCertificate(asn1Data) + if err != nil { + return fmt.Errorf("failed to parse certificate: %v", err) + } + certs[i] = cert + } + opts := x509.VerifyOptions{ + Roots: certPool, + CurrentTime: time.Now(), + DNSName: "", // Skip hostname verification + Intermediates: x509.NewCertPool(), + } + + for i, cert := range certs { + if i == 0 { + continue + } + opts.Intermediates.AddCert(cert) + } + _, err := certs[0].Verify(opts) + return err + }, + }, + }, + }, nil +} + +func (f *Fetcher) Download(agent *db.Agent, sha256Hash string) (io.ReadCloser, error) { + q := url.Values{ + "sha256_hash": {sha256Hash}, + } + resp, err := f.BasicRequest(agent, "/download", q.Encode(), "") + if err != nil { + return nil, err + } + if resp.Header.Get("Content-Type") == "application/json" { + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + respObject := &controllers.SampleResponse{} + if err := json.Unmarshal(body, respObject); err != nil { + return nil, err + } + return nil, fmt.Errorf("request to (%s:%s) returned error %s", agent.Name, agent.AgentID, respObject.Status) + } + return resp.Body, nil +} diff --git a/internal/integrations/integrations.go b/internal/integrations/integrations.go index e49b050..26fb857 100644 --- a/internal/integrations/integrations.go +++ b/internal/integrations/integrations.go @@ -1,6 +1,7 @@ package integrations import ( + "github.com/pygrum/siphon/internal/integrations/agent" "github.com/pygrum/siphon/internal/integrations/malwarebazaar" "github.com/pygrum/siphon/internal/logger" "github.com/spf13/viper" @@ -13,8 +14,12 @@ func Refresh() { logger.Silentf("invalid configuration: refresh rate must be 1 minute or more") } ticker := time.NewTicker(time.Duration(r) * time.Minute) + mbFetcher := malwarebazaar.NewFetcher() + agFetcher := agent.NewFetcher() for range ticker.C { - mbFetcher := malwarebazaar.NewFetcher() - go mbFetcher.GetRecent() + if mbFetcher != nil { + go mbFetcher.GetRecent() + } + go agFetcher.GetRecent() } } diff --git a/internal/integrations/malwarebazaar/malwarebazaar.go b/internal/integrations/malwarebazaar/malwarebazaar.go index cb2aaa9..a4a82e6 100644 --- a/internal/integrations/malwarebazaar/malwarebazaar.go +++ b/internal/integrations/malwarebazaar/malwarebazaar.go @@ -3,9 +3,9 @@ package malwarebazaar import ( "encoding/json" "fmt" + "github.com/gin-gonic/gin/binding" "github.com/pygrum/siphon/internal/commands/sources" "github.com/pygrum/siphon/internal/db" - "github.com/pygrum/siphon/internal/db/models" "github.com/pygrum/siphon/internal/logger" "io" "net/http" @@ -19,8 +19,10 @@ const ( ) func NewFetcher() *Fetcher { - f := &Fetcher{} - mbData := sources.FindSource("MalwareBazaar") + f := &Fetcher{ + Conn: db.Initialize(), + } + mbData := sources.FindSource(Source) if mbData == nil { return nil } @@ -48,21 +50,21 @@ func (f *Fetcher) GetRecent() { logger.Silentf("request to %s returned error %s", Source, respObject.QueryStatus) return } - if err := addSamples(respObject); err != nil { + if err := f.addSamples(respObject); err != nil { logger.Silentf("failed to add new samples: %v", err) } } -func addSamples(r *Response) error { +func (f *Fetcher) addSamples(r *Response) error { for _, data := range r.Data { go func(data Item) { - if db.SampleByHash(data.Sha256Hash) == nil { + if f.Conn.SampleByHash(data.Sha256Hash) == nil { t, err := time.Parse(time.DateTime+" MST", data.FirstSeen+" UTC") if err != nil { - logger.Silentf("could not parse time %s", data.FirstSeen) + logger.Silentf("could not parse time %s: %v", data.FirstSeen, err) t = time.Time{} } - sample := models.Sample{ + sample := db.Sample{ Name: data.FileName, FileType: data.FileType, FileSize: data.FileSize, @@ -72,7 +74,7 @@ func addSamples(r *Response) error { UploadTime: t, // All times for Bazaar are UTC } - if err := db.AddSample(&sample); err != nil { + if err := f.Conn.Add(&sample); err != nil { logger.Silentf("%v", err) } } @@ -88,10 +90,10 @@ func (f *Fetcher) BasicRequest(form url.Values) (*http.Response, error) { return nil, err } r.Header.Add("API-KEY", f.ApiKey) - r.Header.Add("Content-Type", "application/x-www-form-urlencoded") + r.Header.Add("Content-Type", binding.MIMEPOSTForm) client := &http.Client{} - resp, err := client.Do(r) + if err != nil { logger.Silentf("request failed: %v", err) return nil, err diff --git a/internal/integrations/malwarebazaar/models.go b/internal/integrations/malwarebazaar/models.go index b00239a..30be329 100644 --- a/internal/integrations/malwarebazaar/models.go +++ b/internal/integrations/malwarebazaar/models.go @@ -1,5 +1,7 @@ package malwarebazaar +import "github.com/pygrum/siphon/internal/db" + type Response struct { QueryStatus string `json:"query_status"` Data []Item `json:"data"` @@ -17,4 +19,5 @@ type Item struct { type Fetcher struct { Endpoint string ApiKey string + Conn *db.Conn } diff --git a/internal/logger/logger.go b/internal/logger/logger.go index af6d992..06c2b00 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -20,10 +20,31 @@ const ( White = "\033[97m" ) +type Logger struct { + LogFile *os.File +} + +func NewLogger(path string) (*Logger, error) { + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) + if err != nil { + return nil, err + } + defer f.Close() + return &Logger{LogFile: f}, nil +} + +func (l *Logger) Write(s string) { + _, _ = l.LogFile.WriteString(time.Now().Format(time.DateTime) + " " + s + "\n") +} + func Infof(format string, v ...interface{}) { fmt.Printf(Blue+"[-] "+format+"\n"+Reset, v...) } +func Sinfof(format string, v ...interface{}) string { + return fmt.Sprintf(Blue+"[-] "+format+"\n"+Reset, v...) +} + func Info(v interface{}) { fmt.Println(Blue+"[-]", v, Reset) } @@ -62,6 +83,10 @@ func Errorf(format string, v ...interface{}) { fmt.Printf(Red+"[!] "+format+"\n"+Reset, v...) } +func Serrorf(format string, v ...interface{}) string { + return fmt.Sprintf(Red+"[!] "+format+"\n"+Reset, v...) +} + func Silentf(format string, v ...interface{}) { logFile := filepath.Join( filepath.Dir(viper.ConfigFileUsed()), diff --git a/internal/version/version.go b/internal/version/version.go new file mode 100644 index 0000000..a941e51 --- /dev/null +++ b/internal/version/version.go @@ -0,0 +1,13 @@ +package version + +import "strings" + +const ( + versionMajor = "2" + versionMinor = "0" + versionPatch = "0" +) + +func VersionString() string { + return "v" + strings.Join([]string{versionMajor, versionMinor, versionPatch}, ".") +} diff --git a/main.go b/main.go deleted file mode 100644 index d8b7868..0000000 --- a/main.go +++ /dev/null @@ -1,12 +0,0 @@ -package main - -import ( - "github.com/pygrum/siphon/cmd" - "github.com/pygrum/siphon/internal/logger" -) - -func main() { - if err := cmd.Execute(); err != nil { - logger.Fatal(err) - } -} diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100644 index 0000000..19d3a13 --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +set -e + +read -p "both Go & Make must be installed before proceeding. Have you installed both? y/N " yn + +case $yn in +y | Y) + echo "proceeding with installation" + ;; +*) + echo "please install golang and make first, as they are requirements." + exit 1 + ;; +esac + +BASEDIR=$(dirname "$0") +NAME="siphon" +HOMEPATH=${HOME}/.siphon + +if [ ! -d "${HOMEPATH}" ]; then + echo "creating home folder..." + mkdir "${HOMEPATH}" + echo "created home folder at ${HOMEPATH}" +else + echo "home folder ${HOMEPATH} already exists - exiting" + exit 1 +fi + +echo "generating x509 key pair..." +bash "${BASEDIR}/keygen.sh" "$NAME" + +OLDPATH=${PWD} +SRCPATH=$(cd "${BASEDIR}/.." && pwd) + +echo "moving key pair to home folder..." +mv "${NAME}.crt" "${HOMEPATH}/${NAME}.crt" +mv "${NAME}.key" "${HOMEPATH}/${NAME}.key" +echo "done" + +echo "copying source to siphon home folder..." +cp -r "${SRCPATH}" "${HOMEPATH}" +echo "copied" +echo "generating configuration file..." + +CONFIG="${HOMEPATH}/.siphon.yaml" +echo "refreshrate: 1" >> "$CONFIG" +echo "cert_file: ${HOMEPATH}/${NAME}.crt" >> "$CONFIG" +echo "key_file: ${HOMEPATH}/${NAME}.key" >> "$CONFIG" + +echo "building the binary - please wait..." +make build +mv siphon ${HOME}/.local/bin +if ! command -v siphon &> /dev/null; then + mkdir -p "${HOME}/.local/bin" 2>/dev/null + echo 'export PATH=$PATH:$HOME/.local/bin' >> "${HOME}/.profile" +fi +echo "done" +echo "installation complete" +echo "logout and log back in for changes to take effect" +cd "${OLDPATH}" diff --git a/scripts/keygen.sh b/scripts/keygen.sh new file mode 100644 index 0000000..f081adc --- /dev/null +++ b/scripts/keygen.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +NAME="$1" + +# Set cert names to siphon if name not provided +if [ $# -lt 1 ]; then + NAME="siphon" +fi + +openssl req -newkey rsa:4096 \ + -x509 \ + -sha256 \ + -days 3650 \ + -nodes \ + -out "${NAME}.crt" \ + -keyout "${NAME}.key" \ + -subj "/C=US/ST=New York/L=New York City/O=${NAME}/OU=${NAME}/CN=www.${NAME}.com" + +echo "key pair saved at ${NAME}.crt (certificate) and ${NAME}.key (key)" diff --git a/scripts/setup_agent.sh b/scripts/setup_agent.sh new file mode 100644 index 0000000..178ff65 --- /dev/null +++ b/scripts/setup_agent.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -e + +BASEDIR=$(dirname "$0") +BINFILE="$1" +HOMEDIR="${HOME}/.siphon" +NAME="agent" + +if [ $# -ne 1 ]; then + echo "invalid usage" + echo "usage: ${0} " + exit 1 +fi + +OLDDIR=${PWD} +cd "${BASEDIR}/.." + +mkdir "agent_files" +cp "$BINFILE" "agent_files" +cd "agent_files" + +echo "generating agent keys..." +bash ../scripts/keygen.sh "$NAME" + +echo "done" + +AGENT_FOLDER="$(md5sum "$BINFILE" | cut -f1 -d' ')" +mkdir "${HOMEDIR}/${AGENT_FOLDER}" +echo "created agent folder: ${HOMEDIR}/${AGENT_FOLDER} - saving agent certificate to folder..." +cp "${NAME}.crt" "${HOMEDIR}/${AGENT_FOLDER}" +echo "done. configure Siphon to use this certificate for this agent (agents set --cert-file=${HOMEDIR}/${AGENT_FOLDER}/${NAME}.crt)" + +echo "generating default configuration file..." +echo 'cache: true' >> config.yaml +echo "cert_file: /root/${NAME}.crt" >> config.yaml +echo "key_file: /root/${NAME}.key" >> config.yaml +echo "monitor_folders:" >> config.yaml +echo ' - path: "/tmp"' >> config.yaml +echo ' recursive: true' >> config.yaml +echo "saved as config.yaml" + +echo "agent files saved to directory 'agent_files'" +echo "follow the guide at https://github.com/pygrum/siphon/blob/main/docs/DOCS.md to finish configuring the agent." +cd ${OLDDIR}