-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
1,607 changed files
with
778,781 additions
and
21 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1 +1 @@ | ||
| v0.3.10 | ||
| v0.3.11 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,127 @@ | ||
| package bq | ||
|
|
||
| import ( | ||
| "context" | ||
| "strings" | ||
|
|
||
| "cloud.google.com/go/bigquery" | ||
| "github.com/pkg/errors" | ||
| "github.com/rs/zerolog/log" | ||
| "google.golang.org/api/iterator" | ||
| ) | ||
|
|
||
| func configureTarget(ctx context.Context, t *targetConfig) error { | ||
| if t == nil { | ||
| return errors.New("nil target config") | ||
| } | ||
|
|
||
| log.Debug(). | ||
| Str("project", t.ProjectID). | ||
| Str("dataset", t.DatasetID). | ||
| Str("table", t.TableID). | ||
| Msg("configuring target") | ||
|
|
||
| exists, err := datasetExists(ctx, t) | ||
| if err != nil { | ||
| return errors.Wrap(err, "failed to check if dataset exists") | ||
| } | ||
|
|
||
| if !exists { | ||
| if err := createDataset(ctx, t); err != nil { | ||
| return errors.Wrap(err, "failed to create dataset") | ||
| } | ||
| } | ||
|
|
||
| exists, err = tableExists(ctx, t) | ||
| if err != nil { | ||
| return errors.Wrap(err, "failed to check if table exists") | ||
| } | ||
|
|
||
| if !exists { | ||
| if err := createTable(ctx, t, vulnerabilitySchema); err != nil { | ||
| return errors.Wrapf(err, "failed to create table with ID: %s", t.TableID) | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func createTable(ctx context.Context, t *targetConfig, schema bigquery.Schema) error { | ||
| client, err := bigquery.NewClient(ctx, t.ProjectID) | ||
| if err != nil { | ||
| return errors.Wrapf(err, "failed to create bigquery client for project %s", t.ProjectID) | ||
| } | ||
| defer client.Close() | ||
|
|
||
| metaData := &bigquery.TableMetadata{ | ||
| Schema: schema, | ||
| } | ||
|
|
||
| tableRef := client.Dataset(t.DatasetID).Table(t.TableID) | ||
| if err := tableRef.Create(ctx, metaData); err != nil { | ||
| return errors.Wrapf(err, "failed to create table %s", t.TableID) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func createDataset(ctx context.Context, t *targetConfig) error { | ||
| client, err := bigquery.NewClient(ctx, t.ProjectID) | ||
| if err != nil { | ||
| return errors.Wrapf(err, "failed to create bigquery client for project %s", t.ProjectID) | ||
| } | ||
| defer client.Close() | ||
|
|
||
| meta := &bigquery.DatasetMetadata{Location: t.Location} | ||
| if err := client.Dataset(t.DatasetID).Create(ctx, meta); err != nil { | ||
| return errors.Wrapf(err, "failed to create dataset %s", t.DatasetID) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func datasetExists(ctx context.Context, t *targetConfig) (bool, error) { | ||
| client, err := bigquery.NewClient(ctx, t.ProjectID) | ||
| if err != nil { | ||
| return false, errors.Wrapf(err, "failed to create bigquery client for project %s", t.ProjectID) | ||
| } | ||
| defer client.Close() | ||
|
|
||
| it := client.Datasets(ctx) | ||
| for { | ||
| dataset, err := it.Next() | ||
| if errors.Is(err, iterator.Done) { | ||
| break | ||
| } | ||
| if err != nil { | ||
| return false, errors.Wrapf(err, "failed to list datasets for project %s", t.ProjectID) | ||
| } | ||
|
|
||
| if strings.EqualFold(dataset.DatasetID, t.DatasetID) { | ||
| return true, nil | ||
| } | ||
| } | ||
| return false, nil | ||
| } | ||
|
|
||
| func tableExists(ctx context.Context, t *targetConfig) (bool, error) { | ||
| client, err := bigquery.NewClient(ctx, t.ProjectID) | ||
| if err != nil { | ||
| return false, errors.Wrapf(err, "failed to create bigquery client for project %s", t.ProjectID) | ||
| } | ||
| defer client.Close() | ||
|
|
||
| it := client.Dataset(t.DatasetID).Tables(ctx) | ||
| for { | ||
| table, err := it.Next() | ||
| if errors.Is(err, iterator.Done) { | ||
| break | ||
| } | ||
| if err != nil { | ||
| return false, errors.Wrapf(err, "failed to list datasets for project %s", t.ProjectID) | ||
| } | ||
|
|
||
| if strings.EqualFold(table.TableID, t.TableID) { | ||
| return true, nil | ||
| } | ||
| } | ||
| return false, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| package bq | ||
|
|
||
| import ( | ||
| "context" | ||
| "strings" | ||
|
|
||
| "cloud.google.com/go/bigquery" | ||
| "github.com/pkg/errors" | ||
| ) | ||
|
|
||
| const ( | ||
| importDefaultLocation = "US" | ||
| importTargetParts = 3 | ||
| importTargetPartProject = 0 | ||
| importTargetPartDataset = 1 | ||
| importTargetPartTable = 2 | ||
|
|
||
| expectedFormat = "bq://project.dataset.table" | ||
|
|
||
| targetSchema = "bq://" | ||
| ) | ||
|
|
||
| type targetConfig struct { | ||
| ProjectID string | ||
| Location string | ||
| DatasetID string | ||
| TableID string | ||
| } | ||
|
|
||
| // ParseImportRequest parses import request. | ||
| // e.g. bq://cloudy-demos.disco.packages | ||
| func parseTarget(uri string) (*targetConfig, error) { | ||
| if !strings.HasPrefix(uri, "bq://") { | ||
| return nil, errors.Errorf("invalid import target schema, expected %s, got: %s", targetSchema, uri) | ||
| } | ||
|
|
||
| v := strings.Replace(uri, targetSchema, "", 1) | ||
|
|
||
| parts := strings.Split(v, ".") | ||
| if len(parts) != importTargetParts { | ||
| return nil, errors.Errorf("invalid import target: %s, expected %d part format: %s", uri, importTargetParts, expectedFormat) | ||
| } | ||
|
|
||
| t := &targetConfig{ | ||
| Location: importDefaultLocation, | ||
| ProjectID: parts[importTargetPartProject], | ||
| DatasetID: parts[importTargetPartDataset], | ||
| TableID: parts[importTargetPartTable], | ||
| } | ||
|
|
||
| if t.ProjectID == "" || t.DatasetID == "" || t.TableID == "" { | ||
| return nil, errors.Errorf("invalid import target: %+v", t) | ||
| } | ||
|
|
||
| return t, nil | ||
| } | ||
|
|
||
| func insert(ctx context.Context, t *targetConfig, items interface{}) error { | ||
| if t == nil || t.ProjectID == "" || t.DatasetID == "" || t.TableID == "" { | ||
| return errors.New("project, dataset and table must be specified") | ||
| } | ||
|
|
||
| client, err := bigquery.NewClient(ctx, t.ProjectID) | ||
| if err != nil { | ||
| return errors.Wrap(err, "failed to create bigquery client") | ||
| } | ||
| defer client.Close() | ||
|
|
||
| inserter := client.Dataset(t.DatasetID).Table(t.TableID).Inserter() | ||
| inserter.SkipInvalidRows = true | ||
| if err := inserter.Put(ctx, items); err != nil { | ||
| return errors.Wrap(err, "failed to insert rows") | ||
| } | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| package bq | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
| ) | ||
|
|
||
| func TestTargetParsing(t *testing.T) { | ||
| _, err := parseTarget("bq://") | ||
| assert.Error(t, err) | ||
|
|
||
| _, err = parseTarget("bq://project") | ||
| assert.Error(t, err) | ||
|
|
||
| _, err = parseTarget("bq://project.dataset") | ||
| assert.Error(t, err) | ||
|
|
||
| _, err = parseTarget("bq://project.dataset.table.port") | ||
| assert.Error(t, err) | ||
|
|
||
| tr, err := parseTarget("bq://test.vimp.vuls") | ||
| assert.NoError(t, err) | ||
| assert.NotNil(t, t) | ||
| assert.Equal(t, "test", tr.ProjectID) | ||
| assert.Equal(t, "vimp", tr.DatasetID) | ||
| assert.Equal(t, "vuls", tr.TableID) | ||
| assert.Equal(t, importDefaultLocation, tr.Location) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| package bq | ||
|
|
||
| import ( | ||
| "context" | ||
|
|
||
| "cloud.google.com/go/bigquery" | ||
| "github.com/mchmarny/vimp/pkg/data" | ||
| "github.com/pkg/errors" | ||
| "github.com/rs/zerolog/log" | ||
| ) | ||
|
|
||
| var ( | ||
| vulnerabilitySchema = bigquery.Schema{ | ||
| {Name: "image", Type: bigquery.StringFieldType, Required: true}, | ||
| {Name: "digest", Type: bigquery.StringFieldType, Required: true}, | ||
| {Name: "source", Type: bigquery.StringFieldType, Required: true}, | ||
| {Name: "processed", Type: bigquery.TimestampFieldType, Required: true}, | ||
| {Name: "cve", Type: bigquery.StringFieldType, Required: true}, | ||
| {Name: "package", Type: bigquery.StringFieldType, Required: true}, | ||
| {Name: "version", Type: bigquery.StringFieldType, Required: true}, | ||
| {Name: "severity", Type: bigquery.StringFieldType, Required: true}, | ||
| {Name: "score", Type: bigquery.FloatFieldType, Required: true}, | ||
| {Name: "fixed", Type: bigquery.BooleanFieldType, Required: true}, | ||
| } | ||
| ) | ||
|
|
||
| func Import(uri string, vuls []*data.ImageVulnerability) error { | ||
| t, err := parseTarget(uri) | ||
| if err != nil { | ||
| return errors.Wrap(err, "failed to parse target") | ||
| } | ||
|
|
||
| ctx := context.Background() | ||
| if err := configureTarget(ctx, t); err != nil { | ||
| return errors.Wrap(err, "errors checking target configuration") | ||
| } | ||
|
|
||
| rows := makeVulnerabilityRows(vuls) | ||
| if err := insert(ctx, t, rows); err != nil { | ||
| return errors.Wrap(err, "failed to insert rows") | ||
| } | ||
|
|
||
| log.Info().Msgf("inserted %d records into %s.%s.%s", len(rows), t.ProjectID, t.DatasetID, t.TableID) | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func makeVulnerabilityRows(in []*data.ImageVulnerability) []*VulnerabilityRow { | ||
| list := make([]*VulnerabilityRow, 0) | ||
|
|
||
| for _, r := range in { | ||
| list = append(list, &VulnerabilityRow{ | ||
| vul: r, | ||
| }) | ||
| } | ||
|
|
||
| return list | ||
| } | ||
|
|
||
| type VulnerabilityRow struct { | ||
| vul *data.ImageVulnerability | ||
| } | ||
|
|
||
| func (v *VulnerabilityRow) Save() (map[string]bigquery.Value, string, error) { | ||
| return map[string]bigquery.Value{ | ||
| "image": v.vul.Image, | ||
| "digest": v.vul.Digest, | ||
| "source": v.vul.Source, | ||
| "processed": v.vul.ProcessedAt, | ||
| "cve": v.vul.ID, | ||
| "package": v.vul.Package, | ||
| "version": v.vul.Version, | ||
| "severity": v.vul.Severity, | ||
| "score": v.vul.Score, | ||
| "fixed": v.vul.IsFixed, | ||
| }, "", nil | ||
| } |
Oops, something went wrong.