diff --git a/go.mod b/go.mod index 45cc7b19f3..de3c094317 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/briandowns/spinner v1.16.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/cheggaaa/pb/v3 v3.0.8 // indirect + github.com/cheggaaa/pb v1.0.27 github.com/d4l3k/messagediff v1.2.2-0.20190829033028-7e0a312ae40b github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 github.com/emersion/go-smtp v0.14.0 @@ -87,6 +87,7 @@ require ( github.com/aquasecurity/go-pep440-version v0.0.0-20210121094942-22b2f8951d46 // indirect github.com/aquasecurity/go-version v0.0.0-20210121072130-637058cfe492 // indirect github.com/caarlos0/env/v6 v6.0.0 // indirect + github.com/cheggaaa/pb/v3 v3.0.8 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect @@ -142,7 +143,7 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.1 // indirect - golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 // indirect + golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 // indirect golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.27.1 // indirect diff --git a/go.sum b/go.sum index f98b181968..c4b7b6e793 100644 --- a/go.sum +++ b/go.sum @@ -2013,8 +2013,8 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 h1:2B5p2L5IfGiD7+b9BOoRMC6DgObAVZV+Fsp050NqXik= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M= diff --git a/models/packages.go b/models/packages.go index 2967094134..8135700f2e 100644 --- a/models/packages.go +++ b/models/packages.go @@ -72,6 +72,31 @@ func (ps Packages) FindByFQPN(nameVerRel string) (*Package, error) { return nil, xerrors.Errorf("Failed to find the package: %s", nameVerRel) } +// ResolveReverseDepsRecursively resolve and set reverse dependencies for each packages recursively +func (ps Packages) ResolveReverseDepsRecursively() { + for name, pkg := range ps { + depsmap := ps.resolveReverseDeps(pkg, map[string]struct{}{}) + delete(depsmap, pkg.Name) + for depname := range depsmap { + pkg.ReverseDependenciesRecursively = append(pkg.ReverseDependenciesRecursively, depname) + } + ps[name] = pkg + } +} + +func (ps Packages) resolveReverseDeps(pkg Package, acc map[string]struct{}) map[string]struct{} { + acc[pkg.Name] = struct{}{} + for _, name := range pkg.ReverseDependencies { + if _, ok := acc[name]; ok { + continue + } + if p, ok := ps[name]; ok { + acc = ps.resolveReverseDeps(p, acc) + } + } + return acc +} + // Package has installed binary packages. type Package struct { Name string `json:"name"` @@ -84,6 +109,18 @@ type Package struct { Changelog *Changelog `json:"changelog,omitempty"` AffectedProcs []AffectedProcess `json:",omitempty"` NeedRestartProcs []NeedRestartProcess `json:",omitempty"` + + // Dependencies are dependent packages + Dependencies []string `json:",omitempty"` + + // ReverseDependencies are packages which depend on this package + ReverseDependencies []string `json:",omitempty"` + + // ReverseDependencies are packages which depend on this package + ReverseDependenciesRecursively []string `json:",omitempty"` + + // DependenciesForUpdate are packages that needs to be updated together + DependenciesForUpdate []string `json:",omitempty"` } // FQPN returns Fully-Qualified-Package-Name diff --git a/models/packages_test.go b/models/packages_test.go index ee07ee89cb..9aa5405875 100644 --- a/models/packages_test.go +++ b/models/packages_test.go @@ -428,3 +428,166 @@ func Test_NewPortStat(t *testing.T) { }) } } + +func TestPackages_resolveReverseDeps(t *testing.T) { + type args struct { + pkg Package + acc map[string]struct{} + } + tests := []struct { + name string + ps Packages + args args + want map[string]struct{} + }{ + { + name: "", + ps: map[string]Package{ + "pkgA": { + Name: "pkgA", + ReverseDependencies: []string{"pkgB"}, + }, + "pkgB": { + Name: "pkgB", + ReverseDependencies: []string{"pkgC"}, + }, + "pkgC": { + Name: "pkgC", + ReverseDependencies: []string{""}, + }, + }, + args: args{ + pkg: Package{ + Name: "pkgA", + ReverseDependencies: []string{"pkgB"}, + }, + acc: map[string]struct{}{}, + }, + want: map[string]struct{}{ + "pkgA": {}, + "pkgB": {}, + "pkgC": {}, + }, + }, + { + name: "", + ps: map[string]Package{ + "pkgA": { + Name: "pkgA", + ReverseDependencies: []string{"pkgB"}, + }, + "pkgB": { + Name: "pkgB", + ReverseDependencies: []string{"pkgA", "pkgC"}, + }, + "pkgC": { + Name: "pkgC", + ReverseDependencies: []string{"pkgB"}, + }, + }, + args: args{ + pkg: Package{ + Name: "pkgA", + ReverseDependencies: []string{"pkgB"}, + }, + acc: map[string]struct{}{}, + }, + want: map[string]struct{}{ + "pkgA": {}, + "pkgB": {}, + "pkgC": {}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.ps.resolveReverseDeps(tt.args.pkg, tt.args.acc); !reflect.DeepEqual(got, tt.want) { + t.Errorf("Packages.resolveReverseDeps() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPackages_resolveReverseDepsRecursively(t *testing.T) { + tests := []struct { + name string + ps Packages + want Packages + }{ + { + name: "", + ps: map[string]Package{ + "pkgA": { + Name: "pkgA", + ReverseDependencies: []string{"pkgB"}, + }, + "pkgB": { + Name: "pkgB", + ReverseDependencies: []string{"pkgC"}, + }, + "pkgC": { + Name: "pkgC", + ReverseDependencies: []string{""}, + }, + }, + want: map[string]Package{ + "pkgA": { + Name: "pkgA", + ReverseDependencies: []string{"pkgB"}, + ReverseDependenciesRecursively: []string{"pkgB", "pkgC"}, + }, + "pkgB": { + Name: "pkgB", + ReverseDependencies: []string{"pkgC"}, + ReverseDependenciesRecursively: []string{"pkgC"}, + }, + "pkgC": { + Name: "pkgC", + ReverseDependencies: []string{""}, + }, + }, + }, + { + name: "", + ps: map[string]Package{ + "pkgA": { + Name: "pkgA", + ReverseDependencies: []string{"pkgB"}, + }, + "pkgB": { + Name: "pkgB", + ReverseDependencies: []string{"pkgA", "pkgC"}, + }, + "pkgC": { + Name: "pkgC", + ReverseDependencies: []string{"pkgB"}, + }, + }, + want: map[string]Package{ + "pkgA": { + Name: "pkgA", + ReverseDependencies: []string{"pkgB"}, + ReverseDependenciesRecursively: []string{"pkgB", "pkgC"}, + }, + "pkgB": { + Name: "pkgB", + ReverseDependencies: []string{"pkgA", "pkgC"}, + ReverseDependenciesRecursively: []string{"pkgA", "pkgC"}, + }, + "pkgC": { + Name: "pkgC", + ReverseDependencies: []string{"pkgB"}, + ReverseDependenciesRecursively: []string{"pkgA", "pkgB"}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.ps.ResolveReverseDepsRecursively() + if !reflect.DeepEqual(tt.ps, tt.want) { + t.Errorf("Packages.resolveReverseDepsRecursively() = \n%+v, want \n%+v", tt.ps, tt.want) + } + }) + } +} diff --git a/scanner/redhatbase.go b/scanner/redhatbase.go index d83bcd2390..b2bf1353aa 100644 --- a/scanner/redhatbase.go +++ b/scanner/redhatbase.go @@ -4,8 +4,10 @@ import ( "bufio" "fmt" "regexp" + "sort" "strings" + "github.com/cheggaaa/pb" "github.com/future-architect/vuls/config" "github.com/future-architect/vuls/constant" "github.com/future-architect/vuls/logging" @@ -218,6 +220,10 @@ func (o *redhatBase) execCheckDeps(packNames []string) error { return nil } +func (o *redhatBase) isDnf() bool { + return o.exec(util.PrependProxyEnv(`repoquery --version | grep dnf`), o.sudo.repoquery()).isSuccess() +} + func (o *redhatBase) preCure() error { if err := o.detectIPAddr(); err != nil { o.log.Warnf("Failed to detect IP addresses: %s", err) @@ -230,8 +236,7 @@ func (o *redhatBase) preCure() error { func (o *redhatBase) postScan() error { if o.isExecYumPS() { if err := o.pkgPs(o.getOwnerPkgs); err != nil { - err = xerrors.Errorf("Failed to execute yum-ps: %w", err) - o.log.Warnf("err: %+v", err) + o.log.Warnf("err: %+v", xerrors.Errorf("Failed to execute yum-ps: %w", err)) o.warns = append(o.warns, err) // Only warning this error } @@ -239,8 +244,7 @@ func (o *redhatBase) postScan() error { if o.isExecNeedsRestarting() { if err := o.needsRestarting(); err != nil { - err = xerrors.Errorf("Failed to execute need-restarting: %w", err) - o.log.Warnf("err: %+v", err) + o.log.Warnf("err: %+v", xerrors.Errorf("Failed to execute need-restarting: %w", err)) o.warns = append(o.warns, err) // Only warning this error } @@ -260,15 +264,10 @@ func (o *redhatBase) scanPackages() (err error) { return xerrors.Errorf("Failed to scan installed packages: %w", err) } - if o.EnabledDnfModules, err = o.detectEnabledDnfModules(); err != nil { - return xerrors.Errorf("Failed to detect installed dnf modules: %w", err) - } - fn := func(pkgName string) execResult { return o.exec(fmt.Sprintf("rpm -q --last %s", pkgName), noSudo) } o.Kernel.RebootRequired, err = o.rebootRequired(fn) if err != nil { - err = xerrors.Errorf("Failed to detect the kernel reboot required: %w", err) - o.log.Warnf("err: %+v", err) + o.log.Warnf("err: %+v", xerrors.Errorf("Failed to detect the kernel reboot required: %w", err)) o.warns = append(o.warns, err) // Only warning this error } @@ -281,15 +280,31 @@ func (o *redhatBase) scanPackages() (err error) { } } + if err := o.yumMakeCache(); err != nil { + return xerrors.Errorf("Failed to `yum makecache`: %w", err) + } + + // `dnf module list` needs yum cache + if o.EnabledDnfModules, err = o.detectEnabledDnfModules(); err != nil { + return xerrors.Errorf("Failed to detect installed dnf modules: %w", err) + } + updatable, err := o.scanUpdatablePackages() if err != nil { - err = xerrors.Errorf("Failed to scan updatable packages: %w", err) - o.log.Warnf("err: %+v", err) + o.log.Warnf("err: %+v", xerrors.Errorf("Failed to scan updatable packages: %w", err)) o.warns = append(o.warns, err) // Only warning this error } else { o.Packages.MergeNewVersion(updatable) } + + if o.getServerInfo().Mode.IsDeep() { + resolver := yumDependentResolver{redhat: o, isDnf: o.isDnf()} + resolver.detectDependenciesForUpdate() + resolver.detectReverseDependencies() + o.Packages.ResolveReverseDepsRecursively() + } + return nil } @@ -420,13 +435,8 @@ func (o *redhatBase) yumMakeCache() error { } func (o *redhatBase) scanUpdatablePackages() (models.Packages, error) { - if err := o.yumMakeCache(); err != nil { - return nil, xerrors.Errorf("Failed to `yum makecache`: %w", err) - } - - isDnf := o.exec(util.PrependProxyEnv(`repoquery --version | grep dnf`), o.sudo.repoquery()).isSuccess() cmd := `repoquery --all --pkgnarrow=updates --qf='%{NAME} %{EPOCH} %{VERSION} %{RELEASE} %{REPO}'` - if isDnf { + if o.isDnf() { cmd = `repoquery --upgrades --qf='%{NAME} %{EPOCH} %{VERSION} %{RELEASE} %{REPONAME}' -q` } for _, repo := range o.getServerInfo().Enablerepo { @@ -700,9 +710,6 @@ func (o *redhatBase) detectEnabledDnfModules() ([]string, error) { cmd := `dnf --nogpgcheck --cacheonly --color=never --quiet module list --enabled` r := o.exec(util.PrependProxyEnv(cmd), noSudo) if !r.isSuccess() { - if strings.Contains(r.Stdout, "Cache-only enabled but no cache") { - return nil, xerrors.Errorf("sudo yum check-update to make local cache before scanning: %s", r) - } return nil, xerrors.Errorf("Failed to dnf module list: %s", r) } return o.parseDnfModuleList(r.Stdout) @@ -723,3 +730,122 @@ func (o *redhatBase) parseDnfModuleList(stdout string) (labels []string, err err } return } + +type yumDependentResolver struct { + redhat *redhatBase + isDnf bool +} + +func (o *yumDependentResolver) detectDependenciesForUpdate() { + o.redhat.log.Infof("Detecting the dependencies for each packages when yum updating") + o.redhat.log.Infof("If it is too slow, `yum clean all` may make it faster") + bar := pb.StartNew(len(o.redhat.Packages)) + for name, pkg := range o.redhat.Packages { + bar.Increment() + if pkg.Version == pkg.NewVersion && pkg.Release == pkg.NewRelease { + // only updatable pkgs + continue + } + names, err := o.scanUpdatablePkgDeps(name) + if err != nil { + o.redhat.log.Warnf("err: %+v", xerrors.Errorf("Failed to scan dependent packages for update: %w", err)) + o.redhat.warns = append(o.redhat.warns, err) + // Only warning this error + } + sort.Strings(names) + pkg.DependenciesForUpdate = names + o.redhat.Packages[name] = pkg + } + bar.Finish() + return +} + +func (o *yumDependentResolver) scanUpdatablePkgDeps(name string) (depsPkgNames []string, err error) { + cmd := fmt.Sprintf("LANGUAGE=en_US.UTF-8 yum install --assumeno --cacheonly %s", name) + r := o.redhat.exec(cmd, true) + if !r.isSuccess(0, 1) { + return nil, xerrors.Errorf("Failed to SSH: %s", r) + } + names := o.parseYumInstall(r.Stdout) + for _, n := range names { + if _, ok := o.redhat.Packages[n]; ok { + depsPkgNames = append(depsPkgNames, n) + } + } + return +} + +func (o *yumDependentResolver) parseYumInstall(stdout string) []string { + names, inDepsLines := []string{}, false + scanner := bufio.NewScanner(strings.NewReader(stdout)) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "Updating for dependencies:") { + inDepsLines = true + continue + } + if inDepsLines { + ss := strings.Fields(line) + if len(ss) == 0 { + break + } + names = append(names, ss[0]) + } + } + return names +} + +func (o *yumDependentResolver) detectReverseDependencies() { + o.redhat.log.Infof("Detecting the reverse dependencies for each packages") + bar := pb.StartNew(len(o.redhat.Packages)) + for name, pkg := range o.redhat.Packages { + bar.Increment() + // if pkg.Version == pkg.NewVersion && pkg.Release == pkg.NewRelease { + // continue + // } + names, err := o.repoqueryWhatRequires(name) + if err != nil { + o.redhat.log.Warnf("err: %+v", xerrors.Errorf("Failed to scan reverse dependent packages: %w", err)) + o.redhat.warns = append(o.redhat.warns, err) + // Only warning this error + } + sort.Strings(names) + pkg.ReverseDependencies = names + o.redhat.Packages[name] = pkg + } + bar.Finish() + return +} + +func (o *yumDependentResolver) repoqueryWhatRequires(pkgName string) (depsPkgNames []string, err error) { + cmd := `LANGUAGE=en_US.UTF-8 repoquery --cache --resolve --pkgnarrow=installed --qf "%{name}" --whatrequires ` + pkgName + if o.isDnf { + cmd = `LANGUAGE=en_US.UTF-8 repoquery --cache --installed --qf "%{name}" --whatrequires ` + pkgName + } + r := o.redhat.exec(cmd, true) + if !r.isSuccess() { + return nil, xerrors.Errorf("Failed to SSH: %s", r) + } + names := o.parseRepoqueryWhatRequires(r.Stdout) + for _, n := range names { + if pkgName == n { + continue + } + if _, ok := o.redhat.Packages[n]; ok { + depsPkgNames = append(depsPkgNames, n) + } + } + return +} + +func (o *yumDependentResolver) parseRepoqueryWhatRequires(stdout string) []string { + names := []string{} + scanner := bufio.NewScanner(strings.NewReader(stdout)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line != "" { + names = append(names, line) + } + } + return names +} diff --git a/scanner/redhatbase_test.go b/scanner/redhatbase_test.go index 9355dd7fb6..316661f7ef 100644 --- a/scanner/redhatbase_test.go +++ b/scanner/redhatbase_test.go @@ -10,11 +10,6 @@ import ( "github.com/k0kubun/pp" ) -// func unixtimeNoerr(s string) time.Time { -// t, _ := unixtime(s) -// return t -// } - func TestParseInstalledPackagesLinesRedhat(t *testing.T) { r := newRHEL(config.ServerInfo{}) r.Distro = config.Distro{Family: constant.RedHat} @@ -641,3 +636,125 @@ kernel-3.10.0-1062.12.1.el7.x86_64 Sat 29 Feb 2020 12:09:00 PM UTC`, }) } } + +func Test_redhatBase_parseYumInstall(t *testing.T) { + type fields struct { + base base + sudo rootPriv + } + type args struct { + stdout string + } + tests := []struct { + name string + fields fields + args args + wantPkgNames []string + }{ + { + name: "ok", + fields: fields{ + base: base{}, + }, + args: args{ + stdout: `=========================================================================================================================================================== + Package Arch Version Repository Size +=========================================================================================================================================================== +Updating: + glibc x86_64 2.17-325.el7_9 updates 3.6 M +Updating for dependencies: + glibc-common x86_64 2.17-325.el7_9 updates 12 M + glibc-devel x86_64 2.17-325.el7_9 updates 1.1 M + glibc-headers x86_64 2.17-325.el7_9 updates 691 k + +Transaction Summary +=========================================================================================================================================================== +Upgrade 1 Package (+3 Dependent packages) + +Exiting on user command +Your transaction was saved, rerun it with: + yum load-transaction /tmp/yum_save_tx.2021-10-25.08-24.1EXcRc.yumtx`, + }, + wantPkgNames: []string{ + "glibc-common", + "glibc-devel", + "glibc-headers", + }, + }, + { + name: "no deps", + fields: fields{ + base: base{}, + }, + args: args{ + stdout: `Package zlib-1.2.7-19.el7_9.x86_64 already installed and latest version`, + }, + wantPkgNames: []string{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + o := &redhatBase{ + base: tt.fields.base, + } + resolver := yumDependentResolver{redhat: o} + if gotPkgNames := resolver.parseYumInstall(tt.args.stdout); !reflect.DeepEqual(gotPkgNames, tt.wantPkgNames) { + t.Errorf("redhatBase.parseYumInstall() = %v, want %v", gotPkgNames, tt.wantPkgNames) + } + }) + } +} + +func Test_yumDependentResolver_parseRepoqueryWhatRequires(t *testing.T) { + type fields struct { + redhat *redhatBase + } + type args struct { + stdout string + } + tests := []struct { + name string + fields fields + args args + want []string + }{ + { + name: "no deps", + fields: fields{redhat: &redhatBase{}}, + args: args{ + stdout: "", + }, + want: []string{}, + }, + { + name: "no deps", + fields: fields{redhat: &redhatBase{}}, + args: args{ + stdout: `which +whois +wpa_supplicant +xcb-util +xfsprogs +xinetd`, + }, + want: []string{ + "which", + "whois", + "wpa_supplicant", + "xcb-util", + "xfsprogs", + "xinetd", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + o := &yumDependentResolver{ + redhat: tt.fields.redhat, + } + if got := o.parseRepoqueryWhatRequires(tt.args.stdout); !reflect.DeepEqual(got, tt.want) { + t.Errorf("yumDependentResolver.parseRepoqueryWhatRequires() = %v, want %v", got, tt.want) + } + }) + } +}