From 2bf75464d6027b8079431c34410ab9d070b818f4 Mon Sep 17 00:00:00 2001 From: Arnout Vandecappelle Date: Wed, 7 Feb 2024 16:35:17 +0100 Subject: [PATCH] support/scripts/pkg-stats: remove checking the CPE database We currently check the CPE database to see if the CPE ID we use (including the version) is already in the database. However, the version part of the CPE ID is not actually used for CVE matching. Instead, the CVEs have a range of versions associated with them and we match against those ranges. In addition, NVD is moving to a new API for accessing the CPE database. It will not longer be possible to simply download all the CPE IDs, and due to rate limiting, the download will have to be done in several queries. Since all of this is anyway of limited use, drop the CPE database lookup entirely. Instead, as long as a CPE ID is defined in a package, it is considered OK, without any checks. Signed-off-by: Arnout Vandecappelle Signed-off-by: Thomas Petazzoni --- support/scripts/pkg-stats | 54 ++------------------------------------- 1 file changed, 2 insertions(+), 52 deletions(-) diff --git a/support/scripts/pkg-stats b/support/scripts/pkg-stats index 05f0cd9e4a..7835cb3d62 100755 --- a/support/scripts/pkg-stats +++ b/support/scripts/pkg-stats @@ -28,10 +28,6 @@ import re import subprocess import json import sys -import time -import gzip -import xml.etree.ElementTree -import requests brpath = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", "..")) @@ -40,7 +36,6 @@ from getdeveloperlib import parse_developers # noqa: E402 INFRA_RE = re.compile(r"\$\(eval \$\(([a-z-]*)-package\)\)") URL_RE = re.compile(r"\s*https?://\S*\s*$") -CPEDB_URL = "https://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz" RM_API_STATUS_ERROR = 1 RM_API_STATUS_FOUND_BY_DISTRO = 2 @@ -282,8 +277,7 @@ class Package: if var in self.all_cpeids: self.cpeid = self.all_cpeids[var] - # Set a preliminary status, it might be overridden by check_package_cpes() - self.status['cpe'] = ("warning", "not checked against CPE dictionary") + self.status['cpe'] = ("ok", "(not checked against CPE dictionary)") else: self.status['cpe'] = ("error", "no verified CPE identifier") @@ -674,47 +668,6 @@ def check_package_cves(nvd_path, packages): pkg.status['cve'] = ("ok", "not affected by CVEs") -def check_package_cpes(nvd_path, packages): - class CpeXmlParser: - cpes = [] - - def start(self, tag, attrib): - if tag == "{http://scap.nist.gov/schema/cpe-extension/2.3}cpe23-item": - self.cpes.append(attrib['name']) - - def close(self): - return self.cpes - - print("CPE: Setting up NIST dictionary") - if not os.path.exists(os.path.join(nvd_path, "cpe")): - os.makedirs(os.path.join(nvd_path, "cpe")) - - cpe_dict_local = os.path.join(nvd_path, "cpe", os.path.basename(CPEDB_URL)) - if not os.path.exists(cpe_dict_local) or os.stat(cpe_dict_local).st_mtime < time.time() - 86400: - print("CPE: Fetching xml manifest from [" + CPEDB_URL + "]") - cpe_dict = requests.get(CPEDB_URL) - open(cpe_dict_local, "wb").write(cpe_dict.content) - - print("CPE: Unzipping xml manifest...") - nist_cpe_file = gzip.GzipFile(fileobj=open(cpe_dict_local, 'rb')) - - parser = xml.etree.ElementTree.XMLParser(target=CpeXmlParser()) - while True: - c = nist_cpe_file.read(1024*1024) - if not c: - break - parser.feed(c) - cpes = parser.close() - - for p in packages: - if not p.cpeid: - continue - if p.cpeid in cpes: - p.status['cpe'] = ("ok", "verified CPE identifier") - else: - p.status['cpe'] = ("error", "CPE version unknown in CPE database") - - def calculate_stats(packages): stats = defaultdict(int) stats['packages'] = len(packages) @@ -1309,7 +1262,7 @@ def parse_args(): parser.add_argument('--nvd-path', dest='nvd_path', help='Path to the local NVD database', type=resolvepath) parser.add_argument('--disable', type=list_str, - help='Features to disable, comma-separated (cve, upstream, url, cpe, warning)', + help='Features to disable, comma-separated (cve, upstream, url, warning)', default=[]) args = parser.parse_args() if not args.html and not args.json: @@ -1372,9 +1325,6 @@ def __main__(): if "cve" not in args.disable and args.nvd_path: print("Checking packages CVEs") check_package_cves(args.nvd_path, packages) - if "cpe" not in args.disable and args.nvd_path: - print("Checking packages CPEs") - check_package_cpes(args.nvd_path, packages) print("Calculate stats") stats = calculate_stats(packages) if args.html: