From c35b535dd2bf9f39ee514c52e3f944625b59bd13 Mon Sep 17 00:00:00 2001 From: Johan Stenstam Date: Wed, 5 Jun 2024 12:43:43 +0200 Subject: [PATCH 1/8] foo --- refreshengine.go | 4 ++++ tem.sample.yaml | 16 ++++------------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/refreshengine.go b/refreshengine.go index adc10c9..e79d983 100644 --- a/refreshengine.go +++ b/refreshengine.go @@ -99,6 +99,10 @@ func (td *TemData) RefreshEngine(conf *Config, stopch chan struct{}) { log.Printf("RefreshEngine: Tapir Observation update evaluated.") case "global-config": + if !strings.HasSuffix(tpkg.Topic, "config") { + log.Printf("RefreshEngine: received global-config message on wrong topic: %s. Ignored", tpkg.Topic) + continue + } td.ProcessTapirGlobalConfig(tpkg.Data) log.Printf("RefreshEngine: Tapir Global Config evaluated.") diff --git a/tem.sample.yaml b/tem.sample.yaml index 1b26754..b3bc218 100644 --- a/tem.sample.yaml +++ b/tem.sample.yaml @@ -17,16 +17,6 @@ bootstrapserver: address: tlsaddress: -# server: -# listen: 127.0.0.1 -# port: 9097 -# upstreams: [ nsa.johani.org:53 ] -# downstreams: [ 127.0.0.1:9999 ] # unbound -# xferzones: [ ] # not useful for TAPIR -# mapzones: [ ] # likely not useful for TAPIR -# slicezones: [ rpz.threat-feed.com ] -# slicezones: [ ] - dnsengine: addresses: 127.0.0.1:5359 @@ -35,13 +25,15 @@ mqtt: server: mqtt.dev.dnstapir.se:8883 uid: johani clientid: this-must-be-unique - topic: events/up/johani/frobozz +# topic: events/up/johani/frobozz cacert: /etc/dnstapir/certs/tapirCA.crt clientcert: /etc/dnstapir/certs/mqttclient.crt clientkey: /etc/dnstapir/certs/mqttclient-key.pem signingkey: /etc/dnstapir/certs/mqttsigner-key.pem - validatorkey: /etc/dnstapir/certs/mqttsigner-pub.pem +# validatorkey: /etc/dnstapir/certs/mqttsigner-pub.pem qos: 2 + configtopic: events/up/johani/config + configvalidatorkey: /etc/dnstapir/certs/mqttsigner-pub.pem output: reaper: From 79acccaec50ff6549adb07ac425068084fb4730f Mon Sep 17 00:00:00 2001 From: Johan Stenstam Date: Wed, 5 Jun 2024 13:03:00 +0200 Subject: [PATCH 2/8] foo --- xfr.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xfr.go b/xfr.go index 1adfd79..d02d1e2 100644 --- a/xfr.go +++ b/xfr.go @@ -161,8 +161,7 @@ func (td *TemData) RpzIxfrOut(w dns.ResponseWriter, r *dns.Msg) (uint32, int, er case *dns.SOA: curserial = rr.Serial default: - td.Logger.Printf("RpzIxfrOut: unexpected RR in IXFR request Authority section:\n%s\n", - rr.String()) + td.Logger.Printf("RpzIxfrOut: unexpected RR in IXFR request Authority section:\n%s\n", rr.String()) } } } From 008d4e26835358a0ed4ffed2e52033e2ab17dcbf Mon Sep 17 00:00:00 2001 From: Johan Stenstam Date: Fri, 7 Jun 2024 13:25:01 +0200 Subject: [PATCH 3/8] foo --- apihandler.go | 134 ++++++++++++++++++++++++++--------------------- bootstrap.go | 45 ++++++++++++++-- config.go | 82 +++++++++++++++++++---------- main.go | 3 +- policy.go | 2 +- refreshengine.go | 2 +- sources.go | 6 +-- tem.sample.yaml | 50 +++++++++--------- xfr.go | 2 +- 9 files changed, 199 insertions(+), 127 deletions(-) diff --git a/apihandler.go b/apihandler.go index 8618623..56f0601 100644 --- a/apihandler.go +++ b/apihandler.go @@ -242,6 +242,12 @@ func APIbootstrap(conf *Config) func(w http.ResponseWriter, r *http.Request) { log.Printf("API: received /bootstrap request (cmd: %s) from %s.\n", bp.Command, r.RemoteAddr) switch bp.Command { + case "greylist-status": + me := conf.TemData.MqttEngine + stats := me.Stats() + resp.MsgCounters = stats.MsgCounters + resp.MsgTimeStamps = stats.MsgTimeStamps + case "export-greylist": td := conf.TemData td.mu.RLock() @@ -501,13 +507,13 @@ func APIdispatcher(conf *Config, done <-chan struct{}) { walkRoutes(router, viper.GetString("apiserver.address")) log.Println("") - address := viper.GetString("apiserver.address") - tlsaddress := viper.GetString("apiserver.tlsaddress") + addresses := viper.GetStringSlice("apiserver.addresses") + tlsaddresses := viper.GetStringSlice("apiserver.tlsaddresses") certfile := viper.GetString("certs.tem.cert") keyfile := viper.GetString("certs.tem.key") - bootstrapaddress := viper.GetString("bootstrapserver.address") - bootstraptlsaddress := viper.GetString("bootstrapserver.tlsaddress") + bootstrapaddresses := viper.GetStringSlice("bootstrapserver.addresses") + bootstraptlsaddresses := viper.GetStringSlice("bootstrapserver.tlsaddresses") bootstraprouter := SetupBootstrapRouter(conf) tlspossible := true @@ -529,79 +535,87 @@ func APIdispatcher(conf *Config, done <-chan struct{}) { TEMExiter("Error creating API server tls config: %v\n", err) } - tlsServer := &http.Server{ - Addr: tlsaddress, - Handler: router, - TLSConfig: tlsConfig, - ReadTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, - } - bootstrapTlsServer := &http.Server{ - Addr: bootstraptlsaddress, - Handler: bootstraprouter, - TLSConfig: tlsConfig, - ReadTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, - } - var wg sync.WaitGroup - log.Println("*** API: Starting API dispatcher #1. Listening on", address) - - if address != "" { - wg.Add(1) - go func(wg *sync.WaitGroup) { - apiServer := &http.Server{ - Addr: address, - Handler: router, - ReadTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, - } - - log.Println("*** API: Starting API dispatcher #1. Listening on", address) - wg.Done() - TEMExiter(apiServer.ListenAndServe()) - }(&wg) - } + // log.Println("*** API: Starting API dispatcher #1. Listening on", address) - if tlsaddress != "" { - if tlspossible { + if len(addresses) > 0 { + for idx, address := range addresses { wg.Add(1) go func(wg *sync.WaitGroup) { - log.Println("*** API: Starting TLS API dispatcher #1. Listening on", tlsaddress) + apiServer := &http.Server{ + Addr: address, + Handler: router, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + } + + log.Printf("*** API: Starting API dispatcher #%d. Listening on %s", idx+1, address) wg.Done() - TEMExiter(tlsServer.ListenAndServeTLS(certfile, keyfile)) + TEMExiter(apiServer.ListenAndServe()) }(&wg) - } else { - log.Printf("*** API: APIdispatcher: Error: Cannot provide TLS service without cert and key files.\n") } } - if bootstrapaddress != "" { - wg.Add(1) - go func(wg *sync.WaitGroup) { - apiServer := &http.Server{ - Addr: bootstrapaddress, - Handler: bootstraprouter, - ReadTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, + if len(tlsaddresses) > 0 { + if tlspossible { + for idx, tlsaddress := range tlsaddresses { + wg.Add(1) + go func(wg *sync.WaitGroup) { + tlsServer := &http.Server{ + Addr: tlsaddress, + Handler: router, + TLSConfig: tlsConfig, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + } + log.Printf("*** API: Starting TLS API dispatcher #%d. Listening on %s", idx+1, tlsaddress) + wg.Done() + TEMExiter(tlsServer.ListenAndServeTLS(certfile, keyfile)) + }(&wg) } - log.Println("*** API: Starting Bootstrap API dispatcher #1. Listening on", bootstrapaddress) - wg.Done() - TEMExiter(apiServer.ListenAndServe()) - }(&wg) - } else { - log.Println("*** API: No bootstrap address specified") + } else { + log.Printf("*** API: APIdispatcher: Error: Cannot provide TLS service without cert and key files.\n") + } } - if bootstraptlsaddress != "" { - if tlspossible { + if len(bootstrapaddresses) > 0 { + for idx, address := range bootstrapaddresses { wg.Add(1) go func(wg *sync.WaitGroup) { - log.Println("*** API: Starting Bootstrap TLS API dispatcher #1. Listening on", bootstraptlsaddress) + apiServer := &http.Server{ + Addr: address, + Handler: bootstraprouter, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + } + log.Printf("*** API: Starting Bootstrap API dispatcher #%d. Listening on %s", idx+1, address) wg.Done() - TEMExiter(bootstrapTlsServer.ListenAndServeTLS(certfile, keyfile)) + TEMExiter(apiServer.ListenAndServe()) }(&wg) + } + } else { + log.Println("*** API: No bootstrap address specified") + } + + if len(bootstraptlsaddresses) > 0 { + if tlspossible { + for idx, address := range bootstraptlsaddresses { + wg.Add(1) + go func(wg *sync.WaitGroup) { + bootstrapTlsServer := &http.Server{ + Addr: address, + Handler: bootstraprouter, + TLSConfig: tlsConfig, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + } + + log.Printf("*** API: Starting Bootstrap TLS API dispatcher #%d. Listening on %s", idx+1, address) + wg.Done() + TEMExiter(bootstrapTlsServer.ListenAndServeTLS(certfile, keyfile)) + }(&wg) + } } else { log.Printf("*** API: APIdispatcher: Error: Cannot provide Bootstrap TLS service without cert and key files.\n") } diff --git a/bootstrap.go b/bootstrap.go index 42a53af..d451edb 100644 --- a/bootstrap.go +++ b/bootstrap.go @@ -42,8 +42,20 @@ func (td *TemData) BootstrapMqttSource(s *tapir.WBGlist, src SourceConf) (*tapir return nil, fmt.Errorf("error setting up TLS for the API client: %v", err) } + bootstrapaddrs := viper.GetStringSlice("bootstrapserver.addresses") + tlsbootstrapaddrs := viper.GetStringSlice("bootstrapserver.tlsaddresses") + bootstrapaddrs = append(bootstrapaddrs, tlsbootstrapaddrs...) + // Iterate over the bootstrap servers for _, server := range src.Bootstrap { + // Is this myself? + for _, bs := range bootstrapaddrs { + if bs == server { + td.Logger.Printf("MQTT bootstrap server %s is myself, skipping", server) + continue + } + } + api.BaseUrl = fmt.Sprintf(src.BootstrapUrl, server) // Send an API ping command @@ -57,6 +69,31 @@ func (td *TemData) BootstrapMqttSource(s *tapir.WBGlist, src SourceConf) (*tapir td.Logger.Printf("MQTT bootstrap server %s uptime: %v. It has processed %d MQTT messages", server, uptime, 17) status, buf, err := api.RequestNG(http.MethodPost, "/bootstrap", tapir.BootstrapPost{ + Command: "greylist-status", + ListName: src.Name, + Encoding: "json", // XXX: This is our default, but we'll test other encodings later + }, true) + if err != nil { + fmt.Printf("Error from RequestNG: %v\n", err) + continue + } + + var br tapir.BootstrapResponse + err = json.Unmarshal(buf, &br) + if err != nil { + td.Logger.Printf("Error decoding greylist-status response from %s: %v. Giving up.\n", server, err) + continue + } + if br.Error { + td.Logger.Printf("Bootstrap server %s responded with error: %s (instead of greylist status)", server, br.ErrorMsg) + } + if len(br.Msg) != 0 { + td.Logger.Printf("Bootstrap server %s responded: %s", server, br.Msg) + } + + td.Logger.Printf("MQTT bootstrap server %s uptime: %v. It has processed %d MQTT messages on the %s topic (last msg arrived at %s), ", server, uptime, br.MsgCounters["greylist"], src.Name, br.MsgTimeStamps["greylist"].Format(time.RFC3339)) + + status, buf, err = api.RequestNG(http.MethodPost, "/bootstrap", tapir.BootstrapPost{ Command: "export-greylist", ListName: src.Name, Encoding: "gob", // XXX: This is our default, but we'll test other encodings later @@ -67,7 +104,7 @@ func (td *TemData) BootstrapMqttSource(s *tapir.WBGlist, src SourceConf) (*tapir } if status != http.StatusOK { - fmt.Printf("HTTP Error: %s\n", buf) + td.Logger.Printf("HTTP Error: %s\n", buf) continue } @@ -94,13 +131,13 @@ func (td *TemData) BootstrapMqttSource(s *tapir.WBGlist, src SourceConf) (*tapir } if td.Debug { - fmt.Printf("%v\n", greylist) - fmt.Printf("Names present in greylist %s:\n", src.Name) + td.Logger.Printf("%v", greylist) + td.Logger.Printf("Names present in greylist %s:", src.Name) out := []string{"Name|Time added|TTL|Tags"} for _, n := range greylist.Names { out = append(out, fmt.Sprintf("%s|%v|%v|%v", n.Name, n.TimeAdded.Format(tapir.TimeLayout), n.TTL, n.TagMask)) } - fmt.Printf("%s\n", columnize.SimpleFormat(out)) + td.Logger.Printf("%s", columnize.SimpleFormat(out)) } // Successfully received and decoded bootstrap data diff --git a/config.go b/config.go index 6a0ddb7..29b628a 100644 --- a/config.go +++ b/config.go @@ -15,13 +15,13 @@ import ( ) type Config struct { - Service ServiceConf - Server ServerConf - Apiserver ApiserverConf - Dnsengine DnsengineConf - Sources map[string]SourceConf - Policy PolicyConf - Log struct { + Services ServicesConf + ApiServer ApiserverConf + DnsEngine DnsengineConf + BootstrapServer BootstrapServerConf + Sources map[string]SourceConf + Policy PolicyConf + Log struct { File string `validate:"required"` Verbose *bool `validate:"required"` Debug *bool `validate:"required"` @@ -36,12 +36,40 @@ type Config struct { BootTime time.Time } -type ServiceConf struct { - Name string `validate:"required"` - // Filter string `validate:"required"` - Reset_Soa_Serial *bool `validate:"required"` - Debug *bool - Verbose *bool +type ServicesConf struct { + Rpz struct { + ZoneName string `validate:"required"` + Primary string `validate:"required"` // XXX: must be an address that DnsEngine listens to + SerialCache string `validate:"required"` + } + + Reaper struct { + Interval int `validate:"required"` + } +} + +type ApiserverConf struct { + Active *bool `validate:"required"` + Name string `validate:"required"` + Key string `validate:"required"` + Addresses []string `validate:"required"` + TlsAddresses []string `validate:"required"` +} + +type DnsengineConf struct { + Active *bool `validate:"required"` + Name string `validate:"required"` + Addresses []string `validate:"required"` + Logfile string `validate:"required"` + // Logger *log.Logger +} + +type BootstrapServerConf struct { + Active *bool `validate:"required"` + Name string `validate:"required"` + Addresses []string `validate:"required"` + TlsAddresses []string `validate:"required"` + Logfile string } type ServerConf struct { @@ -96,16 +124,6 @@ type GreylistConf struct { } } -type ApiserverConf struct { - Address string `validate:"required"` - Key string `validate:"required"` -} -type DnsengineConf struct { - Address string `validate:"required"` - Logfile string `validate:"required"` - // Logger *log.Logger -} - type InternalConf struct { // RefreshZoneCh chan RpzRefresher // RpzCmdCh chan RpzCmdData @@ -128,9 +146,11 @@ func ValidateConfig(v *viper.Viper, cfgfile string) error { var configsections = make(map[string]interface{}, 5) configsections["log"] = config.Log - configsections["service"] = config.Service - configsections["server"] = config.Server - configsections["apiserver"] = config.Apiserver + configsections["services"] = config.Services + // configsections["server"] = config.Server + configsections["apiserver"] = config.ApiServer + configsections["dnsengine"] = config.DnsEngine + configsections["bootstrapserver"] = config.BootstrapServer configsections["policy"] = config.Policy // Cannot validate a map[string]foobar, must validate the individual foobars: @@ -149,11 +169,15 @@ func ValidateBySection(config *Config, configsections map[string]interface{}, cf validate := validator.New() for k, data := range configsections { - log.Printf("%s: Validating config for %s section\n", config.Service.Name, k) + switch data := data.(type) { + case *SourceConf: + log.Printf("%s: Validating config for source %s", data.Name, k) + case *DnsengineConf, *ApiserverConf, *BootstrapServerConf: + // log.Printf("%s: Validating config for service %s", data.Name, k) + } if err := validate.Struct(data); err != nil { log.Printf("ValidateBySection: data that caused validation to fail:\n%v\n", data) - TEMExiter("ValidateBySection: Config %s, section %s: missing required attributes:\n%v\n", - cfgfile, k, err) + TEMExiter("ValidateBySection: Config %s, section %s: missing required attributes:\n%v\n", cfgfile, k, err) } } return nil diff --git a/main.go b/main.go index 9ae5f5e..b30765f 100644 --- a/main.go +++ b/main.go @@ -29,7 +29,7 @@ var TEMExiter = func(args ...interface{}) { func (td *TemData) SaveRpzSerial() error { // Save the current value of td.Downstreams.Serial to a text file - serialFile := viper.GetString("output.rpz.serialcache") + serialFile := viper.GetString("services.rpz.serialcache") if serialFile == "" { log.Fatalf("TEMExiter:No serial cache file specified") } @@ -162,7 +162,6 @@ func main() { } SetupLogging(&conf) - fmt.Printf("Policy Logging to logger: %v\n", conf.Loggers.Policy) err := ValidateConfig(nil, cfgFileUsed) // will terminate on error if err != nil { diff --git a/policy.go b/policy.go index 96a48ad..515edf8 100644 --- a/policy.go +++ b/policy.go @@ -74,7 +74,7 @@ func (td *TemData) ParseOutputs() error { } } // Read the current value of td.Downstreams.Serial from a text file - serialFile := viper.GetString("output.rpz.serialcache") + serialFile := viper.GetString("services.rpz.serialcache") if serialFile != "" { serialFile = filepath.Clean(serialFile) diff --git a/refreshengine.go b/refreshengine.go index e79d983..1e16357 100644 --- a/refreshengine.go +++ b/refreshengine.go @@ -61,7 +61,7 @@ func (td *TemData) RefreshEngine(conf *Config, stopch chan struct{}) { reaperTicker.Reset(td.ReaperInterval) }() - if !viper.GetBool("service.refresh.active") { + if !viper.GetBool("services.refreshengine.active") { log.Printf("Refresh Engine is NOT active. Zones will only be updated on receipt on Notifies.") for range zonerefch { // ensure that we keep reading to keep the channel open diff --git a/sources.go b/sources.go index 5ce9574..1ead879 100644 --- a/sources.go +++ b/sources.go @@ -21,7 +21,7 @@ import ( func NewTemData(conf *Config, lg *log.Logger) (*TemData, error) { rpzdata := RpzData{ CurrentSerial: 1, - ZoneName: viper.GetString("output.rpz.zonename"), + ZoneName: viper.GetString("services.rpz.zonename"), IxfrChain: []RpzIxfr{}, Axfr: RpzAxfr{ Data: map[string]*tapir.RpzName{}, @@ -29,7 +29,7 @@ func NewTemData(conf *Config, lg *log.Logger) (*TemData, error) { // RpzMap: map[string]*tapir.RpzName{}, } - repint := viper.GetInt("output.reaper.interval") + repint := viper.GetInt("services.reaper.interval") if repint == 0 { repint = 60 } @@ -169,7 +169,7 @@ func (td *TemData) ParseSourcesNG() error { td.mu.Unlock() } - td.Logger.Printf("ParseSourcesNG: MQTT Engine: %v", td.MqttEngine) + // td.Logger.Printf("ParseSourcesNG: MQTT Engine: %v", td.MqttEngine) for name, src := range srcs { if !*src.Active { diff --git a/tem.sample.yaml b/tem.sample.yaml index b3bc218..4d38ba9 100644 --- a/tem.sample.yaml +++ b/tem.sample.yaml @@ -10,22 +10,36 @@ cli: apiserver: key: be-nice-to-a-bad-tempered-tapir - address: 127.0.0.1:9099 - tlsaddress: 127.0.0.1:9098 + addresses: [ 127.0.0.1:9099 ] + tlsaddresses: [ 127.0.0.1:9098 ] +# Note: This should only be active for a TEM bootstrapserver bootstrapserver: - address: - tlsaddress: + active: false + addresses: [] + tlsaddresses: [] dnsengine: - addresses: 127.0.0.1:5359 + active: true + addresses: [ 127.0.0.1:5360 ] + logfile: /var/log/dnstapir/tem-dnsengine.log + +services: + reaper: + interval: 60 # seconds, time between runs of deleting expired data + rpz: + zonename: rpz. + primary: 127.0.0.1:5359 # must be an address that the dnsengine listens to + serialcache: /etc/dnstapir/rpz-serial.yaml + refreshengine: + active: true + name: TEM Source Refresher mqtt: logfile: /var/log/dnstapir/tem-mqtt.log - server: mqtt.dev.dnstapir.se:8883 + server: tls://mqtt.dev.dnstapir.se:8883 uid: johani clientid: this-must-be-unique -# topic: events/up/johani/frobozz cacert: /etc/dnstapir/certs/tapirCA.crt clientcert: /etc/dnstapir/certs/mqttclient.crt clientkey: /etc/dnstapir/certs/mqttclient-key.pem @@ -35,25 +49,9 @@ mqtt: configtopic: events/up/johani/config configvalidatorkey: /etc/dnstapir/certs/mqttsigner-pub.pem -output: - reaper: - interval: 60 # seconds, time between runs of deleting expired data - rpz: - zonename: rpz. - primary: 127.0.0.1:5359 # must be an address that the dnsengine listens to - -service: - name: foobar - filter: dnssec+zonemd - reset_soa_serial: true - verbose: true - debug: true - refresh: - active: true - -redirects: - tapir: landing-page.dnstapir.se - police: www.polisen.se +# redirects: +# tapir: landing-page.dnstapir.se +# police: www.polisen.se certs: certdir: /etc/dnstapir/certs diff --git a/xfr.go b/xfr.go index d02d1e2..82b2a4d 100644 --- a/xfr.go +++ b/xfr.go @@ -31,7 +31,7 @@ ${ZONE} IN NS ns2.${ZONE} ns1.${ZONE} IN A 127.0.0.1 ns2.${ZONE} IN AAAA ::1` - rpzzone := viper.GetString("output.rpz.zonename") + rpzzone := viper.GetString("services.rpz.zonename") apex := strings.Replace(apextmpl, "${ZONE}", rpzzone, -1) apex = strings.Replace(apex, "${SERIAL}", fmt.Sprintf("%d", td.Rpz.CurrentSerial), -1) From 7473f7a57e8b7be6e65137d6ff9956bcdaecbe73 Mon Sep 17 00:00:00 2001 From: Johan Stenstam Date: Fri, 7 Jun 2024 18:55:37 +0200 Subject: [PATCH 4/8] foo --- apihandler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/apihandler.go b/apihandler.go index 56f0601..36e75b6 100644 --- a/apihandler.go +++ b/apihandler.go @@ -247,6 +247,7 @@ func APIbootstrap(conf *Config) func(w http.ResponseWriter, r *http.Request) { stats := me.Stats() resp.MsgCounters = stats.MsgCounters resp.MsgTimeStamps = stats.MsgTimeStamps + log.Printf("API: greylist-status: msgs: %d last msg: %v", stats.MsgCounters[bp.ListName], stats.MsgTimeStamps[bp.ListName]) case "export-greylist": td := conf.TemData From b7a0d483aa1b55229bbc3cdc9bd51f382d4a31d0 Mon Sep 17 00:00:00 2001 From: Johan Stenstam Date: Mon, 10 Jun 2024 18:42:04 +0200 Subject: [PATCH 5/8] * more code cleanups --- bootstrap.go | 6 ++++++ logging.go | 2 +- sources.go | 30 ++++++++++++++++++++++++------ tem.globalconfig.sample.yaml | 8 ++++++++ tem.sample.yaml | 33 ++++++++++++++++++++------------- 5 files changed, 59 insertions(+), 20 deletions(-) create mode 100644 tem.globalconfig.sample.yaml diff --git a/bootstrap.go b/bootstrap.go index d451edb..298e7bc 100644 --- a/bootstrap.go +++ b/bootstrap.go @@ -73,11 +73,17 @@ func (td *TemData) BootstrapMqttSource(s *tapir.WBGlist, src SourceConf) (*tapir ListName: src.Name, Encoding: "json", // XXX: This is our default, but we'll test other encodings later }, true) + if err != nil { fmt.Printf("Error from RequestNG: %v\n", err) continue } + if status != http.StatusOK { + td.Logger.Printf("HTTP Error: %s\n", buf) + continue + } + var br tapir.BootstrapResponse err = json.Unmarshal(buf, &br) if err != nil { diff --git a/logging.go b/logging.go index 7cb6464..3fac949 100644 --- a/logging.go +++ b/logging.go @@ -71,7 +71,7 @@ func SetupLogging(conf *Config) { conf.Loggers.Dnsengine = log.Default() } - logfile = viper.GetString("mqtt.logfile") + logfile = viper.GetString("tapir.mqtt.logfile") if logfile != "" { logfile = filepath.Clean(logfile) f, err := os.OpenFile(logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644) // #nosec G302 diff --git a/sources.go b/sources.go index 1ead879..f9d530f 100644 --- a/sources.go +++ b/sources.go @@ -154,7 +154,7 @@ func (td *TemData) ParseSourcesNG() error { td.mu.Unlock() srcs := srcfoo.Sources - td.Logger.Printf("*** ParseSourcesNG: there are %d items in spec.", len(srcs)) + td.Logger.Printf("*** ParseSourcesNG: there are %d sources defined in config", len(srcs)) threads := 0 @@ -162,21 +162,37 @@ func (td *TemData) ParseSourcesNG() error { if td.MqttEngine == nil { td.mu.Lock() - err := td.CreateMqttEngine(viper.GetString("mqtt.clientid"), td.MqttLogger) + err := td.CreateMqttEngine(viper.GetString("tapir.mqtt.clientid"), td.MqttLogger) if err != nil { TEMExiter("Error creating MQTT Engine: %v", err) } td.mu.Unlock() } - // td.Logger.Printf("ParseSourcesNG: MQTT Engine: %v", td.MqttEngine) + // Ensure that the MQTT Engine listens on the DNS TAPIR config topic + cfgtopic := viper.GetString("tapir.config.topic") + if cfgtopic != "" { + if td.Debug { + td.Logger.Printf("ParseSourcesNG: Fetching MQTT validator key for topic %s", cfgtopic) + } + valkey, err := tapir.FetchMqttValidatorKey(cfgtopic, viper.GetString("tapir.config.validatorkey")) + if err != nil { + TEMExiter("Error fetching MQTT validator key for topic %s: %v", cfgtopic, err) + } + err = td.MqttEngine.AddTopic(cfgtopic, nil, valkey) + if err != nil { + TEMExiter("Error adding topic %s to MQTT Engine: %v", cfgtopic, err) + } + } for name, src := range srcs { if !*src.Active { td.Logger.Printf("*** ParseSourcesNG: Source \"%s\" is not active. Ignored.", name) continue } - td.Logger.Printf("=== ParseSourcesNG: Source: %s (%s) will be used (list type %s)", name, src.Name, src.Type) + if td.Debug { + td.Logger.Printf("=== ParseSourcesNG: Source: %s (%s) will be used (list type %s)", name, src.Name, src.Type) + } var err error @@ -204,14 +220,16 @@ func (td *TemData) ParseSourcesNG() error { td.Logger.Printf("ParseSourcesNG: thread %d working on source \"%s\" (%s)", thread, name, src.Source) switch src.Source { case "mqtt": - td.Logger.Printf("ParseSourcesNG: Fetching MQTT validator key for topic %s", src.Topic) + if td.Debug { + td.Logger.Printf("ParseSourcesNG: Fetching MQTT validator key for topic %s", src.Topic) + } valkey, err := tapir.FetchMqttValidatorKey(src.Topic, src.ValidatorKey) if err != nil { td.Logger.Printf("ParseSources: Error fetching MQTT validator key for topic %s: %v", src.Topic, err) } td.Logger.Printf("ParseSourcesNG: Adding topic '%s' to MQTT Engine", src.Topic) - err = td.MqttEngine.AddTopic(src.Topic, valkey) + err = td.MqttEngine.AddTopic(src.Topic, nil, valkey) if err != nil { TEMExiter("Error adding topic %s to MQTT Engine: %v", src.Topic, err) } diff --git a/tem.globalconfig.sample.yaml b/tem.globalconfig.sample.yaml new file mode 100644 index 0000000..e06dfac --- /dev/null +++ b/tem.globalconfig.sample.yaml @@ -0,0 +1,8 @@ +globalconfig: + tapirconfigversion: 17 + rpz: + envelopesize: 400 + bootstrap: + servers: [ 77.72.231.135:5454, 1.2.3.4:1234 ] + baseurl: https://%s/api/v1/ + apikey: be-lean-and-mean-and-in-between diff --git a/tem.sample.yaml b/tem.sample.yaml index 4d38ba9..64b096e 100644 --- a/tem.sample.yaml +++ b/tem.sample.yaml @@ -35,19 +35,26 @@ services: active: true name: TEM Source Refresher -mqtt: - logfile: /var/log/dnstapir/tem-mqtt.log - server: tls://mqtt.dev.dnstapir.se:8883 - uid: johani - clientid: this-must-be-unique - cacert: /etc/dnstapir/certs/tapirCA.crt - clientcert: /etc/dnstapir/certs/mqttclient.crt - clientkey: /etc/dnstapir/certs/mqttclient-key.pem - signingkey: /etc/dnstapir/certs/mqttsigner-key.pem -# validatorkey: /etc/dnstapir/certs/mqttsigner-pub.pem - qos: 2 - configtopic: events/up/johani/config - configvalidatorkey: /etc/dnstapir/certs/mqttsigner-pub.pem +tapir: + mqtt: + logfile: /var/log/dnstapir/tem-mqtt.log + server: tls://mqtt.dev.dnstapir.se:8883 + uid: johani + clientid: this-must-be-unique + cacert: /etc/dnstapir/certs/tapirCA.crt + clientcert: /etc/dnstapir/certs/mqttclient.crt + clientkey: /etc/dnstapir/certs/mqttclient-key.pem + qos: 2 + + observations: + srcname: dns-tapir + topic: events/up/johani/observations + validatorkey: /etc/dnstapir/certs/mqttsigner-pub.pem + + config: + srcname: dns-tapir + topic: events/up/johani/config + validatorkey: /etc/dnstapir/certs/mqttsigner-pub.pem # redirects: # tapir: landing-page.dnstapir.se From 5e7f01d7546c44271f3b68aa74347f7d83900ec0 Mon Sep 17 00:00:00 2001 From: Johan Stenstam Date: Mon, 10 Jun 2024 18:50:28 +0200 Subject: [PATCH 6/8] foo --- main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index b30765f..85f608a 100644 --- a/main.go +++ b/main.go @@ -96,9 +96,9 @@ func mainloop(conf *Config, configfile *string, td *TemData) { case <-hupper: // config file to use has already been set in main() if err := viper.ReadInConfig(); err == nil { - fmt.Fprintln(os.Stderr, "Using config file:", viper.ConfigFileUsed()) + fmt.Fprintln(os.Stderr, "Using config file:", *configfile) } else { - TEMExiter("Could not load config %s: Error: %v", viper.ConfigFileUsed(), err) + TEMExiter("Could not load config %s: Error: %v", *configfile, err) } log.Println("mainloop: SIGHUP received. Forcing refresh of all configured zones.") From ae9e8b74dc75c263cd87e88348eca0cabc002ecd Mon Sep 17 00:00:00 2001 From: Johan Stenstam Date: Wed, 12 Jun 2024 12:30:10 +0200 Subject: [PATCH 7/8] * fixed ensuring that the domain names we receive are FQDNs. - I thought that I fixed this long ago...? --- mqtt.go | 5 +++-- rpz.go | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/mqtt.go b/mqtt.go index 4b3a00e..510b5ea 100644 --- a/mqtt.go +++ b/mqtt.go @@ -10,6 +10,7 @@ import ( "time" "github.com/dnstapir/tapir" + "github.com/miekg/dns" ) func (td *TemData) CreateMqttEngine(clientid string, lg *log.Logger) error { @@ -88,7 +89,7 @@ func (td *TemData) ProcessTapirUpdate(tpkg tapir.MqttPkg) (bool, error) { for _, tname := range tpkg.Data.Added { ttl := time.Duration(tname.TTL) * time.Second tmp := tapir.TapirName{ - Name: tname.Name, + Name: dns.Fqdn(tname.Name), TimeAdded: tname.TimeAdded, TTL: ttl, TagMask: tname.TagMask, @@ -135,7 +136,7 @@ func (td *TemData) ProcessTapirUpdate(tpkg tapir.MqttPkg) (bool, error) { } for _, tname := range tpkg.Data.Removed { - delete(wbgl.Names, tname.Name) + delete(wbgl.Names, dns.Fqdn(tname.Name)) } ixfr, err := td.GenerateRpzIxfr(&tpkg.Data) diff --git a/rpz.go b/rpz.go index 505a692..8c7610e 100644 --- a/rpz.go +++ b/rpz.go @@ -183,6 +183,7 @@ func (td *TemData) GenerateRpzIxfr(data *tapir.TapirMsg) (RpzIxfr, error) { var removeData, addData []*tapir.RpzName td.Policy.Logger.Printf("GenerateRpzIxfr: %d removed names and %d added names", len(data.Removed), len(data.Added)) for _, tn := range data.Removed { + tn.Name = dns.Fqdn(tn.Name) td.Policy.Logger.Printf("GenerateRpzIxfr: evaluating removed name %s", tn.Name) if cur, exist := td.Rpz.Axfr.Data[tn.Name]; exist { newAction := td.ComputeRpzAction(tn.Name) @@ -228,6 +229,7 @@ func (td *TemData) GenerateRpzIxfr(data *tapir.TapirMsg) (RpzIxfr, error) { var addtorpz bool for _, tn := range data.Added { + tn.Name = dns.Fqdn(tn.Name) td.Policy.Logger.Printf("GenerateRpzIxfr: evaluating added name %s", tn.Name) addtorpz = false newAction := td.ComputeRpzAction(tn.Name) From 368da15aa44009f84995fab63ee82d86b1f267b1 Mon Sep 17 00:00:00 2001 From: Johan Stenstam Date: Mon, 17 Jun 2024 11:36:19 +0200 Subject: [PATCH 8/8] foo --- refreshengine.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/refreshengine.go b/refreshengine.go index 1e16357..d2b59f0 100644 --- a/refreshengine.go +++ b/refreshengine.go @@ -107,8 +107,9 @@ func (td *TemData) RefreshEngine(conf *Config, stopch chan struct{}) { log.Printf("RefreshEngine: Tapir Global Config evaluated.") default: - log.Printf("RefreshEngine: Tapir IntelUpdate: unknown msg type: %s", tpkg.Data.MsgType) + log.Printf("RefreshEngine: Tapir Message: unknown msg type: %s", tpkg.Data.MsgType) } + // log.Printf("RefreshEngine: Tapir IntelUpdate: %v", tpkg.Data) case zr = <-zonerefch: zone = zr.Name