Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Vat v2 #1

Open
wants to merge 30 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 101 additions & 0 deletions cmd/vat/analysis/analyzer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
package analysis

import (
"fmt"
"sync"

"github.com/ElrondNetwork/elrond-go-core/core/check"
logger "github.com/ElrondNetwork/elrond-go-logger"
"github.com/elrond-go/cmd/vat/core"
"github.com/elrond-go/cmd/vat/scan"
)

type Analyzer struct {
mut sync.Mutex
discoveredTargets []DiscoveredTarget
discoverer Discoverer
scannerFactory ScannerFactory
analysisType core.AnalysisType
managerCommand int
}

var log = logger.GetOrCreate("vat")

// NewAnalyzer creates a new analyzer used for discovery and parsing activities
func NewAnalyzer(discoverer Discoverer, sf ScannerFactory) (*Analyzer, error) {
if check.IfNil(discoverer) {
return nil, fmt.Errorf("Discoverer needed")
}

if check.IfNil(sf) {
return nil, fmt.Errorf("ScannerFactory needed")
}

a := &Analyzer{}
a.discoverer = discoverer
a.managerCommand = NoCommand
a.discoveredTargets = make([]DiscoveredTarget, 0)
a.scannerFactory = sf

return a, nil
}

// StartJob discovers new targets and start the analysis job
func (a *Analyzer) StartJob(analysisType core.AnalysisType) (scanResults []scan.ScannedTarget) {
a.mut.Lock()
defer a.mut.Unlock()

a.discoverTargets()
// get command from manager
a.analysisType = analysisType
nmapScanResults := a.deployAnalysisWorkers()
p := scan.CreateParser(nmapScanResults, analysisType)
return p.Parse()
}

func (a *Analyzer) discoverTargets() {
a.discoveredTargets = a.discoverer.DiscoverNewTargets(a.discoveredTargets)
}

func (a *Analyzer) deployAnalysisWorkers() (work [][]byte) {
scanResults := make([][]byte, 0)
var wg sync.WaitGroup
for _, h := range a.discoveredTargets {
if (h.ActualStatus() == New) || (h.ActualStatus() == Expired) {
wg.Add(1)
temp := h
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍
or could have been passed as an argument to the anonymous function launched on the go routine

go func() {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

in the future, you might think of a way to create only a predefined number of workers in order to not cripple the host in case a large bunch of targets are discovered at once.

Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

True. Maybe a queue list with no more than 10 workers deployed at once.
Thanks.

defer wg.Done()
scanResults = append(scanResults, a.worker(&temp))
}()
}
}
wg.Wait()
return scanResults
}

// this is concurrent safe because a target is not accessed by two concurrent workers
func (a *Analyzer) worker(h *DiscoveredTarget) (rawScanResults []byte) {
s := a.scannerFactory.CreateScanner(h.Address, core.AnalysisType(a.analysisType))

log.Info("Starting scan for:", "address", h.Address)
// Run the scan
rawResult, err := s.Scan()
if err != nil {
log.Error("Scan failed because %e", err)
}

a.changeTargetStatus(h, core.SCANNED)

log.Info("Scanning done for target:", "address", h.Address)
return rawResult
}

func (a *Analyzer) changeTargetStatus(h *DiscoveredTarget, status core.TargetStatus) {
h.Status = status
}

// IsInterfaceNil returns true if there is no value under the interface
func (a *Analyzer) IsInterfaceNil() bool {
return a == nil
}
158 changes: 158 additions & 0 deletions cmd/vat/analysis/analyzer_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
package analysis

import (
"testing"

"github.com/ElrondNetwork/elrond-go-core/core/check"
"github.com/elrond-go/cmd/vat/core"
"github.com/elrond-go/cmd/vat/scan"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

type FakeDiscoverer struct {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice, test doubles appeared 🥇
As a general rule, we create these doubles in a mock subpackage in order to reuse them & decrease the _test.go file length. We also have a naming convention but that will be another discussion.

}

type FakeParserFactory struct {
}

type FakeScannerFactory struct {
}

func (fd *FakeDiscoverer) DiscoverNewTargets(existingTargets []DiscoveredTarget) (targets []DiscoveredTarget) {
targets = existingTargets

return
}

func (sff *FakeScannerFactory) CreateScanner(target string, analysisType core.AnalysisType) (Scanner scan.Scanner) {
return &scan.NmapScanner{
Name: "TCP-SSH",
Target: target,
Status: core.NOT_STARTED,
Cmd: "Test_Cmd_Should_Fail"}
}

func (fpf *FakeParserFactory) CreateParser(input [][]byte, grammar core.AnalysisType) scan.Parser {
return &scan.ParserData{
Input: input,
AnalyzedTargets: make([]scan.ScannedTarget, 0),
Grammar: grammar,
}
}

func TestNewAnalyzer(t *testing.T) {
fd := &FakeDiscoverer{}
sff := &FakeScannerFactory{}
na, err := NewAnalyzer(fd, sff)
assert.False(t, check.IfNil(na))
assert.Nil(t, err)
}

func TestNewAnalyzer_DiscovererNilCheck(t *testing.T) {
sff := &FakeScannerFactory{}
na, err := NewAnalyzer(nil, sff)
assert.True(t, check.IfNil(na))
expectedErrorString := "Discoverer needed"
assert.EqualErrorf(t, err, expectedErrorString, "wrong message")
}

func TestNewAnalyzer_ScannerFactoryNilCheck(t *testing.T) {
fd := &FakeDiscoverer{}
na, err := NewAnalyzer(fd, nil)
assert.True(t, check.IfNil(na))
expectedErrorString := "ScannerFactory needed"
assert.EqualErrorf(t, err, expectedErrorString, "wrong message")
}

func TestNewAnalyzer_ParserFactoryNilCheck(t *testing.T) {
fd := &FakeDiscoverer{}
sff := &FakeScannerFactory{}
na, err := NewAnalyzer(fd, sff)
assert.True(t, check.IfNil(na))
expectedErrorString := "ParserFactory needed"
assert.EqualErrorf(t, err, expectedErrorString, "wrong message")
}

func TestAnalyzer_DiscoverNewPeers(t *testing.T) {
discovererStub := NewDiscovererStub()
sff := &FakeScannerFactory{}
na, _ := NewAnalyzer(discovererStub, sff)
discovererStub.DiscoverNewTargetsCalled = func(existingTargets []DiscoveredTarget) (targets []DiscoveredTarget) {
return make([]DiscoveredTarget, 2)
}
na.discoverTargets()

require.Equal(t, 2, len(na.discoveredTargets))
}

func TestAnalyzeNewlyDiscoveredTargets(t *testing.T) {
discovererStub := NewDiscovererStub()
sff := &FakeScannerFactory{}
na, _ := NewAnalyzer(discovererStub, sff)
analysisType := core.TCP_WEB
na.StartJob(analysisType)
}

func TestAnalyzeNewlyDiscoveredTargets_ActualStatusIsNew(t *testing.T) {
discovererStub := NewDiscovererStub()
sff := &FakeScannerFactory{}
na, _ := NewAnalyzer(discovererStub, sff)
analysisType := core.TCP_WEB
DiscoveredTarget := DiscoveredTarget{
ID: 0,
Protocol: "Test_Protocol",
Address: "Test_Address",
ConnectionPort: "Test_Port",
Status: core.NEW,
}
na.discoveredTargets = append(na.discoveredTargets, DiscoveredTarget)
na.StartJob(analysisType)
}

func TestAnalyzeNewlyDiscoveredTargets_ActualStatusIsExpired(t *testing.T) {
discovererStub := NewDiscovererStub()
sff := &FakeScannerFactory{}
na, _ := NewAnalyzer(discovererStub, sff)
analysisType := core.TCP_WEB
DiscoveredTarget := DiscoveredTarget{
ID: 0,
Protocol: "Test_Protocol",
Address: "Test_Address",
ConnectionPort: "Test_Port",
Status: core.EXPIRED,
}
na.discoveredTargets = append(na.discoveredTargets, DiscoveredTarget)
na.StartJob(analysisType)
}

func TestAnalyzeNewlyDiscoveredTargets_ActualStatusIsNorNewOrExpired(t *testing.T) {
discovererStub := NewDiscovererStub()
sff := &FakeScannerFactory{}
na, _ := NewAnalyzer(discovererStub, sff)
analysisType := core.TCP_WEB
DiscoveredTarget := DiscoveredTarget{
ID: 0,
Protocol: "Test_Protocol",
Address: "Test_Address",
ConnectionPort: "Test_Port",
Status: core.SCANNED,
}
na.discoveredTargets = append(na.discoveredTargets, DiscoveredTarget)
na.StartJob(analysisType)
}

// IsInterfaceNil returns true if there is no value under the interface
func (fpf *FakeParserFactory) IsInterfaceNil() bool {
return fpf == nil
}

// IsInterfaceNil returns true if there is no value under the interface
func (fsf *FakeScannerFactory) IsInterfaceNil() bool {
return fsf == nil
}

// IsInterfaceNil returns true if there is no value under the interface
func (d *FakeDiscoverer) IsInterfaceNil() bool {
return d == nil
}
37 changes: 37 additions & 0 deletions cmd/vat/analysis/discoverer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package analysis

import (
"strings"

"github.com/ElrondNetwork/elrond-go/p2p"
"github.com/elrond-go/cmd/vat/core"
)

type P2pDiscoverer struct {
messenger p2p.Messenger
}

func NewP2pDiscoverer(messenger p2p.Messenger) *P2pDiscoverer {
return &P2pDiscoverer{
messenger: messenger,
}
}

func (d *P2pDiscoverer) DiscoverNewTargets(targetsDiscoveredLastRound []DiscoveredTarget) (discoveredTargets []DiscoveredTarget) {
discoveredTargets = targetsDiscoveredLastRound
currentlyConnectedTargets := d.messenger.ConnectedAddresses()
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is a coincidence that we have a PR done on elrond-go: multiversx#3727 in which I have added a new way of getting connections info based on what peers try to connect to the network. Might be worth of checking out.

Copy link
Owner Author

@schimih schimih Feb 13, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice.
Should I change it for the vat or should I leave it as it is now?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you can leave this as it is for now. Can be refactored later


for idx, address := range currentlyConnectedTargets {
targetAddress := strings.Split(address, "/")
target := MakeTarget(uint(idx), targetAddress[1], targetAddress[2], targetAddress[4], core.NEW)
if !containsTarget(discoveredTargets, target) {
discoveredTargets = append(discoveredTargets, target)
}
}
return
}

// IsInterfaceNil returns true if there is no value under the interface
func (d *P2pDiscoverer) IsInterfaceNil() bool {
return d == nil
}
22 changes: 22 additions & 0 deletions cmd/vat/analysis/discovererStub.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package analysis

type DiscovererStub struct {
DiscoverNewTargetsCalled func(existingTargets []DiscoveredTarget) (targets []DiscoveredTarget)
}

func NewDiscovererStub() *DiscovererStub {
return &DiscovererStub{}
}

func (stub *DiscovererStub) DiscoverNewTargets(existingTargets []DiscoveredTarget) (targets []DiscoveredTarget) {
if stub.DiscoverNewTargetsCalled != nil {
return stub.DiscoverNewTargetsCalled(existingTargets)
}

return make([]DiscoveredTarget, 0)
}

// IsInterfaceNil returns true if there is no value under the interface
func (stub *DiscovererStub) IsInterfaceNil() bool {
return stub == nil
}
20 changes: 20 additions & 0 deletions cmd/vat/analysis/interface.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package analysis

import (
"github.com/elrond-go/cmd/vat/core"
"github.com/elrond-go/cmd/vat/scan"
)

type Discoverer interface {
DiscoverNewTargets(existingTargets []DiscoveredTarget) (targets []DiscoveredTarget)
IsInterfaceNil() bool
}

type ScannerFactory interface {
CreateScanner(target string, analysisType core.AnalysisType) scan.Scanner
IsInterfaceNil() bool
}
type ParserFactory interface {
CreateParser(input [][]byte, grammar core.AnalysisType) scan.Parser
IsInterfaceNil() bool
}
52 changes: 52 additions & 0 deletions cmd/vat/analysis/target.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package analysis

import (
"github.com/elrond-go/cmd/vat/core"
)

// TargetStatus represents a target's state.
type TargetStatus string

// Enumerates the different possible state values.
const (
New TargetStatus = "NEW"
Scanned TargetStatus = "SCANNED"
Expired TargetStatus = "EXPIRED"
)

const (
NoCommand = iota
CHANGE_STATUS_TO_EXPIRED
)

// ActualStatus returns the status of a target.
func (t DiscoveredTarget) ActualStatus() TargetStatus {
return TargetStatus(t.Status)
}

type DiscoveredTarget struct {
ID uint
Protocol string
Address string
ConnectionPort string
Status core.TargetStatus
}

func MakeTarget(id uint, protocol string, address string, connectionPort string, status core.TargetStatus) DiscoveredTarget {
return DiscoveredTarget{
ID: id,
Protocol: protocol,
Address: address,
ConnectionPort: connectionPort,
Status: status,
}
}

func containsTarget(haystack []DiscoveredTarget, needle DiscoveredTarget) bool {
for _, target := range haystack {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

instead of using a slice, you can use a map, especially that the primary goal is to search through the existing items. Also, this function can be part of the struct that holds the []DiscoveredTarget slice. Advantages for this:

  1. you do not need to provide each time the same slice since it can access it internally
  2. you can better control the slice access with a mutex, making the function concurrent safe
  3. it is somehow related with that slice, so you can keep them together

Copy link
Owner Author

@schimih schimih Feb 14, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Indeed using it as a map would be better. Can I keep it as slice for now, given the number of places where it is referenced and the impact it would have if I will change it?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes

if target.Address == needle.Address {
return true
}
}
return false
}
Loading