From 3cdcb80161830991ad206583e08cd98b822777f0 Mon Sep 17 00:00:00 2001 From: Marco Mariani Date: Mon, 20 Nov 2023 17:47:51 +0100 Subject: [PATCH] Separate Item and ItemState; fill BelongsToCollectsion with all ancestors and for uninstalled items too --- cmd/crowdsec-cli/config_backup.go | 16 ++--- cmd/crowdsec-cli/itemcommands.go | 4 +- cmd/crowdsec-cli/items.go | 8 +-- cmd/crowdsec-cli/simulation.go | 2 +- cmd/crowdsec-cli/utils_table.go | 2 +- cmd/crowdsec/main.go | 4 +- pkg/cwhub/enable.go | 14 ++--- pkg/cwhub/enable_test.go | 32 +++++----- pkg/cwhub/helpers.go | 38 ++++++------ pkg/cwhub/helpers_test.go | 86 +++++++++++++-------------- pkg/cwhub/hub.go | 2 +- pkg/cwhub/items.go | 83 +++++++++++++++++--------- pkg/cwhub/items_test.go | 16 ++--- pkg/cwhub/sync.go | 98 ++++++++++++++++--------------- pkg/hubtest/hubtest_item.go | 6 +- pkg/leakybucket/manager_load.go | 4 +- pkg/parser/unix_parser.go | 4 +- test/bats/20_hub_items.bats | 13 ++++ 18 files changed, 237 insertions(+), 195 deletions(-) diff --git a/cmd/crowdsec-cli/config_backup.go b/cmd/crowdsec-cli/config_backup.go index 30a70830ec14..93772d611c78 100644 --- a/cmd/crowdsec-cli/config_backup.go +++ b/cmd/crowdsec-cli/config_backup.go @@ -40,13 +40,13 @@ func backupHub(dirPath string) error { clog = clog.WithFields(log.Fields{ "file": v.Name, }) - if !v.Installed { //only backup installed ones + if !v.State.Installed { //only backup installed ones clog.Debugf("[%s] : not installed", k) continue } //for the local/tainted ones, we back up the full file - if v.Tainted || v.IsLocal() || !v.UpToDate { + if v.State.Tainted || v.IsLocal() || !v.State.UpToDate { //we need to backup stages for parsers if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS { fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage) @@ -54,16 +54,16 @@ func backupHub(dirPath string) error { return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err) } } - clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.IsLocal(), v.UpToDate) + clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.State.Tainted, v.IsLocal(), v.State.UpToDate) tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName) - if err = CopyFile(v.LocalPath, tfile); err != nil { - return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err) + if err = CopyFile(v.State.LocalPath, tfile); err != nil { + return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.State.LocalPath, tfile, err) } - clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile) + clog.Infof("local/tainted saved %s to %s", v.State.LocalPath, tfile) continue } - clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate) - clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate) + clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.State.UpToDate) + clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.State.UpToDate) upstreamParsers = append(upstreamParsers, v.Name) } //write the upstream items diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcommands.go index de7aab68ac10..c4c883e5f8d6 100644 --- a/cmd/crowdsec-cli/itemcommands.go +++ b/cmd/crowdsec-cli/itemcommands.go @@ -331,8 +331,8 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e return fmt.Errorf("can't find '%s' in %s", itemName, it.name) } - if !force && len(item.BelongsToCollections) > 0 { - log.Warningf("%s belongs to collections: %s", item.Name, item.BelongsToCollections) + if !force && len(item.State.BelongsToCollections) > 0 { + log.Warningf("%s belongs to collections: %s", item.Name, item.State.BelongsToCollections) log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, it.singular) continue diff --git a/cmd/crowdsec-cli/items.go b/cmd/crowdsec-cli/items.go index aeefd4263036..d445d19364c5 100644 --- a/cmd/crowdsec-cli/items.go +++ b/cmd/crowdsec-cli/items.go @@ -41,7 +41,7 @@ func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly b for _, itemName := range itemNames { item := hub.GetItem(itemType, itemName) - if installedOnly && !item.Installed { + if installedOnly && !item.State.Installed { continue } @@ -78,8 +78,8 @@ func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item status, emo := item.Status() hubStatus[itemType][i] = itemHubStatus{ Name: item.Name, - LocalVersion: item.LocalVersion, - LocalPath: item.LocalPath, + LocalVersion: item.State.LocalVersion, + LocalPath: item.State.LocalPath, Description: item.Description, Status: status, UTF8Status: fmt.Sprintf("%v %s", emo, status), @@ -111,7 +111,7 @@ func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item row := []string{ item.Name, status, - item.LocalVersion, + item.State.LocalVersion, item.Description, } if len(itemTypes) > 1 { diff --git a/cmd/crowdsec-cli/simulation.go b/cmd/crowdsec-cli/simulation.go index c43e22a68da0..27aea5831d09 100644 --- a/cmd/crowdsec-cli/simulation.go +++ b/cmd/crowdsec-cli/simulation.go @@ -154,7 +154,7 @@ func NewSimulationEnableCmd() *cobra.Command { log.Errorf("'%s' doesn't exist or is not a scenario", scenario) continue } - if !item.Installed { + if !item.State.Installed { log.Warningf("'%s' isn't enabled", scenario) } isExcluded := slices.Contains(csConfig.Cscli.SimulationConfig.Exclusions, scenario) diff --git a/cmd/crowdsec-cli/utils_table.go b/cmd/crowdsec-cli/utils_table.go index b4f86718e403..ee3915c73008 100644 --- a/cmd/crowdsec-cli/utils_table.go +++ b/cmd/crowdsec-cli/utils_table.go @@ -19,7 +19,7 @@ func listHubItemTable(out io.Writer, title string, items []*cwhub.Item) { for _, item := range items { status, emo := item.Status() - t.AddRow(item.Name, fmt.Sprintf("%v %s", emo, status), item.LocalVersion, item.LocalPath) + t.AddRow(item.Name, fmt.Sprintf("%v %s", emo, status), item.State.LocalVersion, item.State.LocalPath) } renderTableTitle(out, title) t.Render() diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index 61bbfad86ffb..8c7fb2991ffe 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -81,8 +81,8 @@ func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error { files []string ) for _, hubScenarioItem := range hub.GetItemMap(cwhub.SCENARIOS) { - if hubScenarioItem.Installed { - files = append(files, hubScenarioItem.LocalPath) + if hubScenarioItem.State.Installed { + files = append(files, hubScenarioItem.State.LocalPath) } } buckets = leakybucket.NewBuckets() diff --git a/pkg/cwhub/enable.go b/pkg/cwhub/enable.go index b38f74e97408..2a17fc7243ab 100644 --- a/pkg/cwhub/enable.go +++ b/pkg/cwhub/enable.go @@ -65,8 +65,8 @@ func (i *Item) createInstallLink() error { // enable enables the item by creating a symlink to the downloaded content, and also enables sub-items func (i *Item) enable() error { - if i.Installed { - if i.Tainted { + if i.State.Installed { + if i.State.Tainted { return fmt.Errorf("%s is tainted, won't enable unless --force", i.Name) } @@ -75,7 +75,7 @@ func (i *Item) enable() error { } // if it's a collection, check sub-items even if the collection file itself is up-to-date - if i.UpToDate && !i.HasSubItems() { + if i.State.UpToDate && !i.HasSubItems() { log.Tracef("%s is installed and up-to-date, skip.", i.Name) return nil } @@ -92,14 +92,14 @@ func (i *Item) enable() error { } log.Infof("Enabled %s: %s", i.Type, i.Name) - i.Installed = true + i.State.Installed = true return nil } // purge removes the actual config file that was downloaded func (i *Item) purge() error { - if !i.Downloaded { + if !i.State.Downloaded { log.Infof("removing %s: not downloaded -- no need to remove", i.Name) return nil } @@ -118,7 +118,7 @@ func (i *Item) purge() error { return fmt.Errorf("while removing file: %w", err) } - i.Downloaded = false + i.State.Downloaded = false log.Infof("Removed source file [%s]: %s", i.Name, src) return nil @@ -179,7 +179,7 @@ func (i *Item) disable(purge bool, force bool) error { return err } - i.Installed = false + i.State.Installed = false if purge { if err := i.purge(); err != nil { diff --git a/pkg/cwhub/enable_test.go b/pkg/cwhub/enable_test.go index fc9863d9ea54..35e56915b364 100644 --- a/pkg/cwhub/enable_test.go +++ b/pkg/cwhub/enable_test.go @@ -16,9 +16,9 @@ func testInstall(hub *Hub, t *testing.T, item *Item) { err = hub.localSync() require.NoError(t, err, "failed to run localSync") - assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should be up-to-date", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted", item.Name) err = item.enable() require.NoError(t, err, "failed to enable %s", item.Name) @@ -26,14 +26,14 @@ func testInstall(hub *Hub, t *testing.T, item *Item) { err = hub.localSync() require.NoError(t, err, "failed to run localSync") - assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].State.Installed, "%s should be installed", item.Name) } func testTaint(hub *Hub, t *testing.T, item *Item) { - assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted", item.Name) // truncate the file - f, err := os.Create(item.LocalPath) + f, err := os.Create(item.State.LocalPath) require.NoError(t, err) f.Close() @@ -41,11 +41,11 @@ func testTaint(hub *Hub, t *testing.T, item *Item) { err = hub.localSync() require.NoError(t, err, "failed to run localSync") - assert.True(t, hub.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should be tainted", item.Name) } func testUpdate(hub *Hub, t *testing.T, item *Item) { - assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should not be up-to-date", item.Name) // Update it + check status _, err := item.downloadLatest(true, true) @@ -55,12 +55,12 @@ func testUpdate(hub *Hub, t *testing.T, item *Item) { err = hub.localSync() require.NoError(t, err, "failed to run localSync") - assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should be up-to-date", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted anymore", item.Name) } func testDisable(hub *Hub, t *testing.T, item *Item) { - assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].State.Installed, "%s should be installed", item.Name) // Remove err := item.disable(false, false) @@ -71,9 +71,9 @@ func testDisable(hub *Hub, t *testing.T, item *Item) { require.NoError(t, err, "failed to run localSync") require.Empty(t, hub.Warnings) - assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) - assert.True(t, hub.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted anymore", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed anymore", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].State.Downloaded, "%s should still be downloaded", item.Name) // Purge err = item.disable(true, false) @@ -84,8 +84,8 @@ func testDisable(hub *Hub, t *testing.T, item *Item) { require.NoError(t, err, "failed to run localSync") require.Empty(t, hub.Warnings) - assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) - assert.False(t, hub.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed anymore", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].State.Downloaded, "%s should not be downloaded", item.Name) } func TestInstallParser(t *testing.T) { diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index f014a2a7d1f7..bd31235aa27d 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -21,7 +21,7 @@ import ( // Install installs the item from the hub, downloading it if needed func (i *Item) Install(force bool, downloadOnly bool) error { - if downloadOnly && i.Downloaded && i.UpToDate { + if downloadOnly && i.State.Downloaded && i.State.UpToDate { log.Infof("%s is already downloaded and up-to-date", i.Name) if !force { @@ -100,11 +100,11 @@ func (i *Item) Remove(purge bool, force bool) (bool, error) { return false, fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name) } - if i.Tainted && !force { + if i.State.Tainted && !force { return false, fmt.Errorf("%s is tainted, use '--force' to remove", i.Name) } - if !i.Installed && !purge { + if !i.State.Installed && !purge { log.Infof("removing %s: not installed -- no need to remove", i.Name) return false, nil } @@ -117,7 +117,7 @@ func (i *Item) Remove(purge bool, force bool) (bool, error) { } for _, sub := range i.SubItems() { - if !sub.Installed { + if !sub.State.Installed { continue } @@ -156,15 +156,15 @@ func (i *Item) Remove(purge bool, force bool) (bool, error) { func (i *Item) Upgrade(force bool) (bool, error) { updated := false - if !i.Downloaded { + if !i.State.Downloaded { return false, fmt.Errorf("can't upgrade %s: not installed", i.Name) } - if !i.Installed { + if !i.State.Installed { return false, fmt.Errorf("can't upgrade %s: downloaded but not installed", i.Name) } - if i.UpToDate { + if i.State.UpToDate { log.Infof("%s: up-to-date", i.Name) if err := i.DownloadDataIfNeeded(force); err != nil { @@ -181,8 +181,8 @@ func (i *Item) Upgrade(force bool) (bool, error) { return false, fmt.Errorf("%s: download failed: %w", i.Name, err) } - if !i.UpToDate { - if i.Tainted { + if !i.State.UpToDate { + if i.State.Tainted { log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, i.Name) } else if i.IsLocal() { log.Infof("%v %s is local", emoji.Prohibited, i.Name) @@ -205,12 +205,12 @@ func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) { log.Debugf("Downloading %s %s", i.Type, i.Name) for _, sub := range i.SubItems() { - if !sub.Installed && updateOnly && sub.Downloaded { + if !sub.State.Installed && updateOnly && sub.State.Downloaded { log.Debugf("skipping upgrade of %s: not installed", i.Name) continue } - log.Debugf("Download %s sub-item: %s %s (%t -> %t)", i.Name, sub.Type, sub.Name, i.Installed, updateOnly) + log.Debugf("Download %s sub-item: %s %s (%t -> %t)", i.Name, sub.Type, sub.Name, i.State.Installed, updateOnly) // recurse as it's a collection if sub.HasSubItems() { @@ -221,7 +221,7 @@ func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) { } } - downloaded := sub.Downloaded + downloaded := sub.State.Downloaded if _, err := sub.download(overwrite); err != nil { return "", fmt.Errorf("while downloading %s: %w", sub.Name, err) @@ -229,14 +229,14 @@ func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) { // We need to enable an item when it has been added to a collection since latest release of the collection. // We check if sub.Downloaded is false because maybe the item has been disabled by the user. - if !sub.Installed && !downloaded { + if !sub.State.Installed && !downloaded { if err := sub.enable(); err != nil { return "", fmt.Errorf("enabling '%s': %w", sub.Name, err) } } } - if !i.Installed && updateOnly && i.Downloaded { + if !i.State.Installed && updateOnly && i.State.Downloaded { log.Debugf("skipping upgrade of %s: not installed", i.Name) return "", nil } @@ -291,12 +291,12 @@ func (i *Item) fetch() ([]byte, error) { func (i *Item) download(overwrite bool) (string, error) { // if user didn't --force, don't overwrite local, tainted, up-to-date files if !overwrite { - if i.Tainted { + if i.State.Tainted { log.Debugf("%s: tainted, not updated", i.Name) return "", nil } - if i.UpToDate { + if i.State.UpToDate { // We still have to check if data files are present log.Debugf("%s: up-to-date, not updated", i.Name) } @@ -333,9 +333,9 @@ func (i *Item) download(overwrite bool) (string, error) { return "", fmt.Errorf("while writing %s: %w", finalPath, err) } - i.Downloaded = true - i.Tainted = false - i.UpToDate = true + i.State.Downloaded = true + i.State.Tainted = false + i.State.UpToDate = true if err = downloadDataSet(i.hub.local.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil { return "", fmt.Errorf("while downloading data for %s: %w", i.FileName, err) diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index 0ad647b9917a..58d1a56cc821 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -14,16 +14,16 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) { hub := envSetup(t) // fresh install of collection - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") require.NoError(t, item.Install(false, false)) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) // This is the scenario that gets added in next version of collection require.Nil(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"]) @@ -44,10 +44,10 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) { hub = getHubOrFail(t, hub.local, remote) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") didUpdate, err := item.Upgrade(false) @@ -55,8 +55,8 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) { require.True(t, didUpdate) assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection") - require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) - require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Downloaded) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Installed) } // Install a collection, disable a scenario. @@ -65,18 +65,18 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { hub := envSetup(t) // fresh install of collection - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") require.NoError(t, item.Install(false, false)) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection") item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario") @@ -92,11 +92,11 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { hub = getHubOrFail(t, hub.local, remote) // scenario referenced by collection was deleted hence, collection should be tainted - require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) hub, err = NewHub(hub.local, remote, true) require.NoError(t, err, "failed to download index: %s", err) @@ -107,7 +107,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) { require.False(t, didUpdate) hub = getHubOrFail(t, hub.local, remote) - require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) } // getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test @@ -125,18 +125,18 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te hub := envSetup(t) // fresh install of collection - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") require.NoError(t, item.Install(false, false)) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection") item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario") @@ -152,12 +152,12 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te hub = getHubOrFail(t, hub.local, remote) // scenario referenced by collection was deleted hence, collection should be tainted - require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Downloaded) // this fails + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate) // collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario" // we now attempt to upgrade the collection, however it shouldn't install the foobar_scenario @@ -167,7 +167,7 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te hub, err = NewHub(hub.local, remote, true) require.NoError(t, err, "failed to download index: %s", err) - require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) hub = getHubOrFail(t, hub.local, remote) item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") @@ -176,8 +176,8 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te require.True(t, didUpdate) hub = getHubOrFail(t, hub.local, remote) - require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Installed) } func assertCollectionDepsInstalled(t *testing.T, hub *Hub, collection string) { diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 82be29b01d4e..01f7eef0426e 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -109,7 +109,7 @@ func (h *Hub) ItemStats() []string { local++ } - if item.Tainted { + if item.State.Tainted { tainted++ } } diff --git a/pkg/cwhub/items.go b/pkg/cwhub/items.go index 8492a07f349a..4a1c3cddc391 100644 --- a/pkg/cwhub/items.go +++ b/pkg/cwhub/items.go @@ -39,35 +39,41 @@ type ItemVersion struct { Deprecated bool `json:"deprecated,omitempty"` } +// ItemState is used to keep the local state (i.e. at runtime) of an item +// This data is not stored in the index, but is displayed in the output of "cscli ... inspect" +type ItemState struct { + LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR} + LocalVersion string `json:"local_version,omitempty"` + LocalHash string `json:"local_hash,omitempty"` // the local meow + Installed bool `json:"installed"` + Downloaded bool `json:"downloaded"` + UpToDate bool `json:"up_to_date"` + Tainted bool `json:"tainted"` // has it been locally modified? + BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any +} + // Item represents an object managed in the hub. It can be a parser, scenario, collection.. type Item struct { // back pointer to the hub, to retrieve subitems and call install/remove methods hub *Hub + // local (deployed) info + State ItemState + // descriptive info - Type string `json:"type,omitempty" yaml:"type,omitempty"` // can be any of the ItemTypes - Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-... - Name string `json:"name,omitempty"` // as seen in .index.json, usually "author/name" - FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml - Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .index.json - Author string `json:"author,omitempty"` // as seen in .index.json - References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .index.json - BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any + Type string `json:"type,omitempty" yaml:"type,omitempty"` // can be any of the ItemTypes + Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-... + Name string `json:"name,omitempty"` // as seen in .index.json, usually "author/name" + FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml + Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .index.json + Author string `json:"author,omitempty"` // as seen in .index.json + References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .index.json // remote (hub) info RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // the path relative to (git | hub API) ie. /parsers/stage/author/file.yaml Version string `json:"version,omitempty"` // the last version Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // the list of existing versions - // local (deployed) info - LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR} - LocalVersion string `json:"local_version,omitempty"` - LocalHash string `json:"local_hash,omitempty"` // the local meow - Installed bool `json:"installed"` - Downloaded bool `json:"downloaded"` - UpToDate bool `json:"up_to_date"` - Tainted bool `json:"tainted"` // has it been locally modified? - // if it's a collection, it can have sub items Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"` PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"` @@ -80,7 +86,7 @@ func (i *Item) HasSubItems() bool { } func (i *Item) IsLocal() bool { - return i.Installed && !i.Downloaded + return i.State.Installed && !i.State.Downloaded } // MarshalJSON is used to add the "local" field to the json output @@ -91,10 +97,27 @@ func (i Item) MarshalJSON() ([]byte, error) { return json.Marshal(&struct { Alias - Local bool `json:"local"` + // we have to repeat the fields here, json will have inline support in v2 + LocalPath string `json:"local_path,omitempty"` + LocalVersion string `json:"local_version,omitempty"` + LocalHash string `json:"local_hash,omitempty"` + Installed bool `json:"installed"` + Downloaded bool `json:"downloaded"` + UpToDate bool `json:"up_to_date"` + Tainted bool `json:"tainted"` + Local bool `json:"local"` + BelongsToCollections []string `json:"belongs_to_collections,omitempty"` }{ - Alias: Alias(i), - Local: i.IsLocal(), + Alias: Alias(i), + LocalPath: i.State.LocalPath, + LocalVersion: i.State.LocalVersion, + LocalHash: i.State.LocalHash, + Installed: i.State.Installed, + Downloaded: i.State.Downloaded, + UpToDate: i.State.UpToDate, + Tainted: i.State.Tainted, + BelongsToCollections: i.State.BelongsToCollections, + Local: i.IsLocal(), }) } @@ -106,9 +129,11 @@ func (i Item) MarshalYAML() (interface{}, error) { return &struct { Alias `yaml:",inline"` - Local bool `yaml:"local"` + State ItemState `yaml:",inline"` + Local bool `yaml:"local"` }{ Alias: Alias(i), + State: i.State, Local: i.IsLocal(), }, nil } @@ -190,7 +215,7 @@ func (i *Item) logMissingSubItems() { func (i *Item) parentCollections() []*Item { ret := make([]*Item, 0) - for _, parentName := range i.BelongsToCollections { + for _, parentName := range i.State.BelongsToCollections { parent := i.hub.GetItem(COLLECTIONS, parentName) if parent == nil { continue @@ -208,7 +233,7 @@ func (i *Item) Status() (string, emoji.Emoji) { status := "disabled" ok := false - if i.Installed { + if i.State.Installed { ok = true status = "enabled" } @@ -220,10 +245,10 @@ func (i *Item) Status() (string, emoji.Emoji) { } warning := false - if i.Tainted { + if i.State.Tainted { warning = true status += ",tainted" - } else if !i.UpToDate && !i.IsLocal() { + } else if !i.State.UpToDate && !i.IsLocal() { warning = true status += ",update-available" } @@ -233,7 +258,7 @@ func (i *Item) Status() (string, emoji.Emoji) { switch { case !managed: emo = emoji.House - case !i.Installed: + case !i.State.Installed: emo = emoji.Prohibited case warning: emo = emoji.Warning @@ -246,7 +271,7 @@ func (i *Item) Status() (string, emoji.Emoji) { // versionStatus: semver requires 'v' prefix func (i *Item) versionStatus() int { - local, err := semver.NewVersion(i.LocalVersion) + local, err := semver.NewVersion(i.State.LocalVersion) if err != nil { return VersionUnknown } @@ -328,7 +353,7 @@ func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) { retItems := make([]*Item, 0) for _, item := range items { - if item.Installed { + if item.State.Installed { retItems = append(retItems, item) } } diff --git a/pkg/cwhub/items_test.go b/pkg/cwhub/items_test.go index cb9424330317..caffb2eb7a0d 100644 --- a/pkg/cwhub/items_test.go +++ b/pkg/cwhub/items_test.go @@ -18,18 +18,18 @@ func TestItemStatus(t *testing.T) { item := hub.GetItem(COLLECTIONS, k) require.NotNil(t, item) - item.Installed = true - item.UpToDate = false - item.Tainted = false - item.Downloaded = true + item.State.Installed = true + item.State.UpToDate = false + item.State.Tainted = false + item.State.Downloaded = true txt, _ := item.Status() require.Equal(t, "enabled,update-available", txt) - item.Installed = true - item.UpToDate = false - item.Tainted = false - item.Downloaded = false + item.State.Installed = true + item.State.UpToDate = false + item.State.Tainted = false + item.State.Downloaded = false txt, _ = item.Status() require.Equal(t, "enabled,local", txt) diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index a755e10fec2e..2274b9c0fce1 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -12,7 +12,6 @@ import ( "github.com/Masterminds/semver/v3" log "github.com/sirupsen/logrus" - "slices" ) func isYAMLFileName(path string) bool { @@ -154,14 +153,16 @@ func newLocalItem(h *Hub, path string, info *itemFileInfo) *Item { _, fileName := filepath.Split(path) return &Item{ - hub: h, - Name: info.fname, - Stage: info.stage, - Installed: true, - Type: info.ftype, - LocalPath: path, - UpToDate: true, - FileName: fileName, + hub: h, + Name: info.fname, + Stage: info.stage, + Type: info.ftype, + FileName: fileName, + State: ItemState{ + LocalPath: path, + Installed: true, + UpToDate: true, + }, } } @@ -244,7 +245,7 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { if path == src { log.Tracef("marking %s as downloaded", item.Name) - item.Downloaded = true + item.State.Downloaded = true } } else if !hasPathSuffix(hubpath, item.RemotePath) { // wrong file @@ -279,45 +280,41 @@ func (h *Hub) checkSubItems(v *Item) error { } // ensure all the sub-items are installed, or tag the parent as tainted - log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed) + log.Tracef("checking submembers of %s installed:%t", v.Name, v.State.Installed) for _, sub := range v.SubItems() { - log.Tracef("check %s installed:%t", sub.Name, sub.Installed) + log.Tracef("check %s installed:%t", sub.Name, sub.State.Installed) - if !v.Installed { + if !v.State.Installed { continue } if err := h.checkSubItems(sub); err != nil { - if sub.Tainted { - v.Tainted = true + if sub.State.Tainted { + v.State.Tainted = true } return fmt.Errorf("sub collection %s is broken: %w", sub.Name, err) } - if sub.Tainted { - v.Tainted = true + if sub.State.Tainted { + v.State.Tainted = true // XXX: improve msg return fmt.Errorf("tainted %s %s, tainted", sub.Type, sub.Name) } - if !sub.Installed && v.Installed { - v.Tainted = true + if !sub.State.Installed && v.State.Installed { + v.State.Tainted = true // XXX: improve msg return fmt.Errorf("missing %s %s, tainted", sub.Type, sub.Name) } - if !sub.UpToDate { - v.UpToDate = false + if !sub.State.UpToDate { + v.State.UpToDate = false return fmt.Errorf("outdated %s %s", sub.Type, sub.Name) } - if !slices.Contains(sub.BelongsToCollections, v.Name) { - sub.BelongsToCollections = append(sub.BelongsToCollections, v.Name) - } - - log.Tracef("checking for %s - tainted:%t uptodate:%t", sub.Name, v.Tainted, v.UpToDate) + log.Tracef("checking for %s - tainted:%t uptodate:%t", sub.Name, v.State.Tainted, v.State.UpToDate) } return nil @@ -363,11 +360,18 @@ func (h *Hub) localSync() error { warnings := make([]string, 0) for _, item := range h.Items[COLLECTIONS] { - if _, err := item.allDependencies(); err != nil { + // check for cyclic dependencies + subs, err := item.allDependencies() + if err != nil { return err } - if !item.Installed { + // populate the sub- and sub-sub-items with the collection they belong to + for _, sub := range subs { + sub.State.BelongsToCollections = append(sub.State.BelongsToCollections, item.Name) + } + + if !item.State.Installed { continue } @@ -378,14 +382,14 @@ func (h *Hub) localSync() error { warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err)) } case VersionUpdateAvailable: // not up-to-date - warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version)) + warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version)) case VersionFuture: - warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version)) + warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version)) case VersionUnknown: warnings = append(warnings, fmt.Sprintf("collection %s is tainted (latest:%s)", item.Name, item.Version)) } - log.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.LocalVersion, item.Version, item.Versions) + log.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.State.LocalVersion, item.Version, item.Versions) } h.Warnings = warnings @@ -396,7 +400,7 @@ func (h *Hub) localSync() error { func (i *Item) setVersionState(path string, inhub bool) error { var err error - i.LocalHash, err = getSHA256(path) + i.State.LocalHash, err = getSHA256(path) if err != nil { return fmt.Errorf("failed to get sha256 of %s: %w", path, err) } @@ -412,44 +416,44 @@ func (i *Item) setVersionState(path string, inhub bool) error { return fmt.Errorf("while syncing %s %s: %w", i.Type, i.FileName, err) } - i.LocalVersion = "?" + i.State.LocalVersion = "?" for _, version := range versions { - if i.Versions[version].Digest == i.LocalHash { - i.LocalVersion = version + if i.Versions[version].Digest == i.State.LocalHash { + i.State.LocalVersion = version break } } - if i.LocalVersion == "?" { + if i.State.LocalVersion == "?" { log.Tracef("got tainted match for %s: %s", i.Name, path) if !inhub { - i.LocalPath = path - i.Installed = true + i.State.LocalPath = path + i.State.Installed = true } - i.UpToDate = false - i.Tainted = true + i.State.UpToDate = false + i.State.Tainted = true return nil } // we got an exact match, update struct - i.Downloaded = true + i.State.Downloaded = true if !inhub { - log.Tracef("found exact match for %s, version is %s, latest is %s", i.Name, i.LocalVersion, i.Version) - i.LocalPath = path - i.Tainted = false + log.Tracef("found exact match for %s, version is %s, latest is %s", i.Name, i.State.LocalVersion, i.Version) + i.State.LocalPath = path + i.State.Tainted = false // if we're walking the hub, present file doesn't means installed file - i.Installed = true + i.State.Installed = true } - if i.LocalVersion == i.Version { + if i.State.LocalVersion == i.Version { log.Tracef("%s is up-to-date", i.Name) - i.UpToDate = true + i.State.UpToDate = true } return nil diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index 82107c6fe3a4..717c876eda77 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -409,7 +409,7 @@ func (t *HubTestItem) InstallHub() error { // install data for parsers if needed ret := hub.GetItemMap(cwhub.PARSERS) for parserName, item := range ret { - if item.Installed { + if item.State.Installed { if err := item.DownloadDataIfNeeded(true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", parserName, err) } @@ -421,7 +421,7 @@ func (t *HubTestItem) InstallHub() error { // install data for scenarios if needed ret = hub.GetItemMap(cwhub.SCENARIOS) for scenarioName, item := range ret { - if item.Installed { + if item.State.Installed { if err := item.DownloadDataIfNeeded(true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", scenarioName, err) } @@ -433,7 +433,7 @@ func (t *HubTestItem) InstallHub() error { // install data for postoverflows if needed ret = hub.GetItemMap(cwhub.POSTOVERFLOWS) for postoverflowName, item := range ret { - if item.Installed { + if item.State.Installed { if err := item.DownloadDataIfNeeded(true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", postoverflowName, err) } diff --git a/pkg/leakybucket/manager_load.go b/pkg/leakybucket/manager_load.go index 0e84d9547f16..d4d9d7d0e0f6 100644 --- a/pkg/leakybucket/manager_load.go +++ b/pkg/leakybucket/manager_load.go @@ -243,8 +243,8 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, hub *cwhub.Hub, files []str bucketFactory.Simulated = cscfg.SimulationConfig.IsSimulated(hubItem.Name) } if hubItem != nil { - bucketFactory.ScenarioVersion = hubItem.LocalVersion - bucketFactory.hash = hubItem.LocalHash + bucketFactory.ScenarioVersion = hubItem.State.LocalVersion + bucketFactory.hash = hubItem.State.LocalHash } else { log.Errorf("scenario %s (%s) couldn't be find in hub (ignore if in unit tests)", bucketFactory.Name, bucketFactory.Filename) } diff --git a/pkg/parser/unix_parser.go b/pkg/parser/unix_parser.go index 02dfb435a9ec..617e46189f35 100644 --- a/pkg/parser/unix_parser.go +++ b/pkg/parser/unix_parser.go @@ -67,9 +67,9 @@ func NewParsers(hub *cwhub.Hub) *Parsers { for _, itemType := range []string{cwhub.PARSERS, cwhub.POSTOVERFLOWS} { for _, hubParserItem := range hub.GetItemMap(itemType) { - if hubParserItem.Installed { + if hubParserItem.State.Installed { stagefile := Stagefile{ - Filename: hubParserItem.LocalPath, + Filename: hubParserItem.State.LocalPath, Stage: hubParserItem.Stage, } if itemType == cwhub.PARSERS { diff --git a/test/bats/20_hub_items.bats b/test/bats/20_hub_items.bats index ca40beea0906..1a795e625d3c 100644 --- a/test/bats/20_hub_items.bats +++ b/test/bats/20_hub_items.bats @@ -57,6 +57,19 @@ teardown() { assert_json '["1.10",false,false]' } +@test "do not unmarshal state attributes" { + new_hub=$( \ + jq <"$INDEX_PATH" \ + '. * {parsers:{"crowdsecurity/syslog-logs":{"tainted":true, "installed":true, "local":true}}}' + ) + echo "$new_hub" >"$INDEX_PATH" + + rune -0 cscli parsers inspect crowdsecurity/syslog-logs --no-metrics + assert_output --partial 'tainted: false' + assert_output --partial 'installed: false' + assert_output --partial 'local: false' +} + @test "hub index with invalid (non semver) version numbers" { rune -0 cscli collections remove crowdsecurity/sshd --purge