diff --git a/binary/proto/proto.go b/binary/proto/proto.go index 562a82db..ebef5fe5 100644 --- a/binary/proto/proto.go +++ b/binary/proto/proto.go @@ -257,6 +257,7 @@ func setProtoMetadata(meta any, i *spb.Inventory) { DpkgMetadata: &spb.DPKGPackageMetadata{ PackageName: m.PackageName, SourceName: m.SourceName, + Status: m.Status, SourceVersion: m.SourceVersion, PackageVersion: m.PackageVersion, OsId: m.OSID, diff --git a/binary/proto/scan_result.proto b/binary/proto/scan_result.proto index 90b62969..d430aeb4 100644 --- a/binary/proto/scan_result.proto +++ b/binary/proto/scan_result.proto @@ -200,6 +200,7 @@ message APKPackageMetadata { } // The additional data found in DPKG packages. +// Next ID: 11 message DPKGPackageMetadata { string package_name = 1; string source_name = 2; @@ -210,6 +211,7 @@ message DPKGPackageMetadata { string os_version_id = 7; string maintainer = 8; string architecture = 9; + string status = 10; } // The additional data found in RPM packages. diff --git a/extractor/os/dpkg/extractor.go b/extractor/os/dpkg/extractor.go index 65e4435a..2fed9f61 100644 --- a/extractor/os/dpkg/extractor.go +++ b/extractor/os/dpkg/extractor.go @@ -40,6 +40,9 @@ const ( // defaultMaxFileSize is the maximum file size an extractor will unmarshal. // If Extract gets a bigger file, it will return an error. defaultMaxFileSize = 100 * units.MiB + + // defaultIncludeNotInstalled is the default value for the IncludeNotInstalled option. + defaultIncludeNotInstalled = false ) // Config is the configuration for the Extractor. @@ -47,18 +50,23 @@ type Config struct { // MaxFileSize is the maximum file size an extractor will unmarshal. // If Extract gets a bigger file, it will return an error. MaxFileSize int64 + // IncludeNotInstalled includes packages that are not installed + // (e.g. `deinstall`, `purge`, and those missing a status field). + IncludeNotInstalled bool } // DefaultConfig returns the default configuration for the DPKG extractor. func DefaultConfig() Config { return Config{ - MaxFileSize: defaultMaxFileSize, + MaxFileSize: defaultMaxFileSize, + IncludeNotInstalled: defaultIncludeNotInstalled, } } // Extractor extracts packages from DPKG files. type Extractor struct { - maxFileSize int64 + maxFileSize int64 + includeNotInstalled bool } // New returns a DPKG extractor. @@ -69,7 +77,16 @@ type Extractor struct { // ``` func New(cfg Config) *Extractor { return &Extractor{ - maxFileSize: cfg.MaxFileSize, + maxFileSize: cfg.MaxFileSize, + includeNotInstalled: cfg.IncludeNotInstalled, + } +} + +// Config returns the configuration of the extractor. +func (e Extractor) Config() Config { + return Config{ + MaxFileSize: e.maxFileSize, + IncludeNotInstalled: e.includeNotInstalled, } } @@ -125,9 +142,14 @@ func (e Extractor) Extract(ctx context.Context, input *extractor.ScanInput) ([]* return pkgs, err } } + // Distroless distributions have their packages in status.d, which does not contain the Status // value. - if !strings.Contains(input.Path, "status.d") || h.Get("Status") != "" { + if !e.includeNotInstalled && (!strings.Contains(input.Path, "status.d") || h.Get("Status") != "") { + if h.Get("Status") == "" { + log.Warnf("Package %q has no status field", h.Get("Package")) + continue + } installed, err := statusInstalled(h.Get("Status")) if err != nil { return pkgs, fmt.Errorf("statusInstalled(%q): %w", h.Get("Status"), err) @@ -136,6 +158,7 @@ func (e Extractor) Extract(ctx context.Context, input *extractor.ScanInput) ([]* continue } } + pkgName := h.Get("Package") pkgVersion := h.Get("Version") if pkgName == "" || pkgVersion == "" { @@ -153,6 +176,7 @@ func (e Extractor) Extract(ctx context.Context, input *extractor.ScanInput) ([]* Metadata: &Metadata{ PackageName: pkgName, PackageVersion: pkgVersion, + Status: h.Get("Status"), OSID: m["ID"], OSVersionCodename: m["VERSION_CODENAME"], OSVersionID: m["VERSION_ID"], diff --git a/extractor/os/dpkg/extractor_test.go b/extractor/os/dpkg/extractor_test.go index 9f7f180a..800adee5 100644 --- a/extractor/os/dpkg/extractor_test.go +++ b/extractor/os/dpkg/extractor_test.go @@ -19,15 +19,54 @@ import ( "fmt" "os" "path/filepath" + "reflect" "testing" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/internal/units" "github.com/google/osv-scalibr/extractor/os/dpkg" "github.com/google/osv-scalibr/purl" ) +func TestNew(t *testing.T) { + tests := []struct { + name string + cfg dpkg.Config + wantCfg dpkg.Config + }{ + { + name: "default", + cfg: dpkg.DefaultConfig(), + wantCfg: dpkg.Config{ + MaxFileSize: 100 * units.MiB, + IncludeNotInstalled: false, + }, + }, + { + name: "custom", + cfg: dpkg.Config{ + MaxFileSize: 10, + IncludeNotInstalled: true, + }, + wantCfg: dpkg.Config{ + MaxFileSize: 10, + IncludeNotInstalled: true, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := dpkg.New(tt.cfg) + if !reflect.DeepEqual(got.Config(), tt.wantCfg) { + t.Errorf("New(%+v).Config(): got %+v, want %+v", tt.cfg, got.Config(), tt.wantCfg) + } + }) + } +} + func TestFileRequired(t *testing.T) { var e extractor.InventoryExtractor = dpkg.Extractor{} @@ -90,6 +129,7 @@ func TestExtract(t *testing.T) { Metadata: &dpkg.Metadata{ PackageName: "accountsservice", PackageVersion: "22.08.8-6", + Status: "install ok installed", OSID: "debian", OSVersionCodename: "bookworm", OSVersionID: "12", @@ -105,6 +145,7 @@ func TestExtract(t *testing.T) { Metadata: &dpkg.Metadata{ PackageName: "acl", PackageVersion: "2.3.1-3", + Status: "install ok installed", OSID: "debian", OSVersionCodename: "bookworm", OSVersionID: "12", @@ -120,6 +161,7 @@ func TestExtract(t *testing.T) { Metadata: &dpkg.Metadata{ PackageName: "adduser", PackageVersion: "3.131", + Status: "install ok installed", OSID: "debian", OSVersionCodename: "bookworm", OSVersionID: "12", @@ -135,6 +177,7 @@ func TestExtract(t *testing.T) { Metadata: &dpkg.Metadata{ PackageName: "admin-session", PackageVersion: "2023.06.26.c543406313-00", + Status: "install ok installed", OSID: "debian", OSVersionCodename: "bookworm", OSVersionID: "12", @@ -150,6 +193,7 @@ func TestExtract(t *testing.T) { Metadata: &dpkg.Metadata{ PackageName: "attr", PackageVersion: "1:2.5.1-4", + Status: "install ok installed", OSID: "debian", OSVersionCodename: "bookworm", OSVersionID: "12", @@ -166,6 +210,7 @@ func TestExtract(t *testing.T) { Metadata: &dpkg.Metadata{ PackageName: "libacl1", PackageVersion: "2.3.1-3", + Status: "install ok installed", SourceName: "acl", OSID: "debian", OSVersionCodename: "bookworm", @@ -183,6 +228,7 @@ func TestExtract(t *testing.T) { Metadata: &dpkg.Metadata{ PackageName: "util-linux-extra", PackageVersion: "2.38.1-5+b1", + Status: "install ok installed", SourceName: "util-linux", SourceVersion: "2.38.1-5", OSID: "debian", @@ -207,6 +253,7 @@ func TestExtract(t *testing.T) { Metadata: &dpkg.Metadata{ PackageName: "foo", PackageVersion: "1.0", + Status: "install ok installed", OSID: "debian", OSVersionCodename: "bookworm", OSVersionID: "12", @@ -220,6 +267,7 @@ func TestExtract(t *testing.T) { Metadata: &dpkg.Metadata{ PackageName: "bar", PackageVersion: "2.0", + Status: "install ok installed", OSID: "debian", OSVersionCodename: "bookworm", OSVersionID: "12", @@ -240,6 +288,7 @@ func TestExtract(t *testing.T) { Metadata: &dpkg.Metadata{ PackageName: "foo", PackageVersion: "1.0", + Status: "install ok installed", OSID: "debian", OSVersionCodename: "bookworm", OSVersionID: "12", @@ -253,6 +302,7 @@ func TestExtract(t *testing.T) { Metadata: &dpkg.Metadata{ PackageName: "bar", PackageVersion: "2.0", + Status: "install ok installed", OSID: "debian", OSVersionCodename: "bookworm", OSVersionID: "12", @@ -273,6 +323,59 @@ func TestExtract(t *testing.T) { Metadata: &dpkg.Metadata{ PackageName: "wantinstall_installed", PackageVersion: "1.0", + Status: "install ok installed", + OSID: "debian", + OSVersionCodename: "bookworm", + OSVersionID: "12", + }, + Locations: []string{"testdata/statusfield"}, + Extractor: dpkg.Name, + }, + &extractor.Inventory{ + Name: "wantdeinstall_installed", + Version: "1.0", + Metadata: &dpkg.Metadata{ + PackageName: "wantdeinstall_installed", + PackageVersion: "1.0", + Status: "deinstall reinstreq installed", + OSID: "debian", + OSVersionCodename: "bookworm", + OSVersionID: "12", + }, + Locations: []string{"testdata/statusfield"}, + Extractor: dpkg.Name, + }, + &extractor.Inventory{ + Name: "wantpurge_installed", + Version: "1.0", + Metadata: &dpkg.Metadata{ + PackageName: "wantpurge_installed", + PackageVersion: "1.0", + Status: "purge ok installed", + OSID: "debian", + OSVersionCodename: "bookworm", + OSVersionID: "12", + }, + Locations: []string{"testdata/statusfield"}, + Extractor: dpkg.Name, + }, + }, + }, + { + name: "statusfield including not installed", + path: "testdata/statusfield", + osrelease: DebianBookworm, + cfg: dpkg.Config{ + IncludeNotInstalled: true, + }, + wantInventory: []*extractor.Inventory{ + &extractor.Inventory{ + Name: "wantinstall_installed", + Version: "1.0", + Metadata: &dpkg.Metadata{ + PackageName: "wantinstall_installed", + PackageVersion: "1.0", + Status: "install ok installed", OSID: "debian", OSVersionCodename: "bookworm", OSVersionID: "12", @@ -286,6 +389,35 @@ func TestExtract(t *testing.T) { Metadata: &dpkg.Metadata{ PackageName: "wantdeinstall_installed", PackageVersion: "1.0", + Status: "deinstall reinstreq installed", + OSID: "debian", + OSVersionCodename: "bookworm", + OSVersionID: "12", + }, + Locations: []string{"testdata/statusfield"}, + Extractor: dpkg.Name, + }, + &extractor.Inventory{ + Name: "wantdeinstall_configfiles", + Version: "1.0", + Metadata: &dpkg.Metadata{ + PackageName: "wantdeinstall_configfiles", + PackageVersion: "1.0", + Status: "deinstall ok config-files", + OSID: "debian", + OSVersionCodename: "bookworm", + OSVersionID: "12", + }, + Locations: []string{"testdata/statusfield"}, + Extractor: dpkg.Name, + }, + &extractor.Inventory{ + Name: "wantinstall_unpacked", + Version: "1.0", + Metadata: &dpkg.Metadata{ + PackageName: "wantinstall_unpacked", + PackageVersion: "1.0", + Status: "install ok unpacked", OSID: "debian", OSVersionCodename: "bookworm", OSVersionID: "12", @@ -299,6 +431,34 @@ func TestExtract(t *testing.T) { Metadata: &dpkg.Metadata{ PackageName: "wantpurge_installed", PackageVersion: "1.0", + Status: "purge ok installed", + OSID: "debian", + OSVersionCodename: "bookworm", + OSVersionID: "12", + }, + Locations: []string{"testdata/statusfield"}, + Extractor: dpkg.Name, + }, + &extractor.Inventory{ + Name: "wantinstall_halfinstalled", + Version: "1.0", + Metadata: &dpkg.Metadata{ + PackageName: "wantinstall_halfinstalled", + PackageVersion: "1.0", + Status: "install reinstreq half-installed", + OSID: "debian", + OSVersionCodename: "bookworm", + OSVersionID: "12", + }, + Locations: []string{"testdata/statusfield"}, + Extractor: dpkg.Name, + }, + &extractor.Inventory{ + Name: "wantnostatus", + Version: "1.0", + Metadata: &dpkg.Metadata{ + PackageName: "wantnostatus", + PackageVersion: "1.0", OSID: "debian", OSVersionCodename: "bookworm", OSVersionID: "12", @@ -313,7 +473,6 @@ func TestExtract(t *testing.T) { path: "testdata/empty", osrelease: DebianBookworm, wantInventory: []*extractor.Inventory{}, - wantErr: cmpopts.AnyError, }, { name: "invalid", @@ -334,6 +493,7 @@ func TestExtract(t *testing.T) { Metadata: &dpkg.Metadata{ PackageName: "acl", PackageVersion: "2.3.1-3", + Status: "install ok installed", OSID: "debian", OSVersionID: "12", Maintainer: "Guillem Jover ", @@ -355,6 +515,7 @@ func TestExtract(t *testing.T) { Metadata: &dpkg.Metadata{ PackageName: "acl", PackageVersion: "2.3.1-3", + Status: "install ok installed", OSID: "debian", Maintainer: "Guillem Jover ", Architecture: "amd64", @@ -375,6 +536,7 @@ func TestExtract(t *testing.T) { Metadata: &dpkg.Metadata{ PackageName: "acl", PackageVersion: "2.3.1-3", + Status: "install ok installed", OSVersionCodename: "bookworm", Maintainer: "Guillem Jover ", Architecture: "amd64", @@ -398,6 +560,7 @@ func TestExtract(t *testing.T) { Metadata: &dpkg.Metadata{ PackageName: "acl", PackageVersion: "2.3.1-3", + Status: "install ok installed", OSID: "ubuntu", OSVersionCodename: "jammy", OSVersionID: "22.04", @@ -489,6 +652,7 @@ func TestExtractNonexistentOSRelease(t *testing.T) { Metadata: &dpkg.Metadata{ PackageName: "acl", PackageVersion: "2.3.1-3", + Status: "install ok installed", OSID: "", OSVersionID: "", Maintainer: "Guillem Jover ", @@ -638,5 +802,10 @@ func defaultConfigWith(cfg dpkg.Config) dpkg.Config { if cfg.MaxFileSize > 0 { newCfg.MaxFileSize = cfg.MaxFileSize } + + if cfg.IncludeNotInstalled { + newCfg.IncludeNotInstalled = cfg.IncludeNotInstalled + } + return newCfg } diff --git a/extractor/os/dpkg/metadata.go b/extractor/os/dpkg/metadata.go index b891b9b8..da678784 100644 --- a/extractor/os/dpkg/metadata.go +++ b/extractor/os/dpkg/metadata.go @@ -17,6 +17,7 @@ package dpkg // Metadata holds parsing information for an dpkg package. type Metadata struct { PackageName string + Status string SourceName string SourceVersion string PackageVersion string diff --git a/extractor/os/dpkg/testdata/statusfield b/extractor/os/dpkg/testdata/statusfield index 6bd13d60..500a25a0 100644 --- a/extractor/os/dpkg/testdata/statusfield +++ b/extractor/os/dpkg/testdata/statusfield @@ -21,3 +21,6 @@ Version: 1.0 Package: wantinstall_halfinstalled Status: install reinstreq half-installed Version: 1.0 + +Package: wantnostatus +Version: 1.0