From fbea83fc47046653778e0cf49863b2c4a3bac8e7 Mon Sep 17 00:00:00 2001 From: Juan Carrano Date: Wed, 2 Nov 2022 11:34:27 +0100 Subject: [PATCH] support/pkg-stats: list packages from external trees. Search the external trees for package files and add them to the list. The list of directories walked and excluded are the same as for the main tree, and should work out of the box if the user sticks to the directory structure suggested in the manual. Two additional properties were added to the Package class, the tree name and the path. For consistency and to simplify the code, packages in the main tree are marked as coming from "BR2". The HTML output has a new column listing the external name (or "BR2") and the json output has a new property "tree". Signed-off-by: Juan Carrano [Arnout: - fix flake8 error "'itertools' imported but unused"; - use str.split instead of str.partition; - use BR2_EXTERNAL_BUILDROOT_PATH instead of BR2_EXTERNAL_BR2_PATH; - remove pkgdir variable, instead use self.pkgdir. ] Signed-off-by: Arnout Vandecappelle --- support/scripts/pkg-stats | 92 +++++++++++++++++++++++++-------------- 1 file changed, 60 insertions(+), 32 deletions(-) diff --git a/support/scripts/pkg-stats b/support/scripts/pkg-stats index 9349a0df57..05f0cd9e4a 100755 --- a/support/scripts/pkg-stats +++ b/support/scripts/pkg-stats @@ -23,7 +23,7 @@ import asyncio import datetime import fnmatch import os -from collections import defaultdict +from collections import defaultdict, namedtuple import re import subprocess import json @@ -77,6 +77,19 @@ def get_defconfig_list(): ] +Br2Tree = namedtuple("Br2Tree", ["name", "path"]) + + +def get_trees(): + raw_variables = subprocess.check_output(["make", "--no-print-directory", "-s", + "BR2_HAVE_DOT_CONFIG=y", "printvars", + "VARS=BR2_EXTERNAL_NAMES BR2_EXTERNAL_%_PATH"]) + variables = dict(line.split("=") for line in raw_variables.decode().split("\n") if line) + variables["BR2_EXTERNAL_BUILDROOT_PATH"] = brpath + externals = ["BUILDROOT", *variables["BR2_EXTERNAL_NAMES"].split()] + return [Br2Tree(name, os.path.normpath(variables[f"BR2_EXTERNAL_{name}_PATH"])) for name in externals] + + class Package: all_licenses = dict() all_license_files = list() @@ -89,7 +102,9 @@ class Package: status_checks = ['cve', 'developers', 'hash', 'license', 'license-files', 'patches', 'pkg-check', 'url', 'version'] - def __init__(self, name, path): + def __init__(self, tree, name, path): + self.tree = tree.name + self.tree_path = tree.path self.name = name self.path = path self.pkg_path = os.path.dirname(path) @@ -118,15 +133,26 @@ class Package: def pkgvar(self): return self.name.upper().replace("-", "_") + @property + def pkgdir(self): + return os.path.join(self.tree_path, self.pkg_path) + + @property + def pkgfile(self): + return os.path.join(self.tree_path, self.path) + + @property + def hashpath(self): + return self.pkgfile.replace(".mk", ".hash") + def set_url(self): """ Fills in the .url field """ self.status['url'] = ("warning", "no Config.in") - pkgdir = os.path.dirname(os.path.join(brpath, self.path)) - for filename in os.listdir(pkgdir): + for filename in os.listdir(self.pkgdir): if fnmatch.fnmatch(filename, 'Config.*'): - fp = open(os.path.join(pkgdir, filename), "r") + fp = open(os.path.join(self.pkgdir, filename), "r") for config_line in fp: if URL_RE.match(config_line): self.url = config_line.strip() @@ -172,7 +198,7 @@ class Package: keep_target = True self.infras = list() - with open(os.path.join(brpath, self.path), 'r') as f: + with open(self.pkgfile, 'r') as f: lines = f.readlines() for line in lines: match = INFRA_RE.match(line) @@ -211,8 +237,7 @@ class Package: self.status['hash-license'] = ("na", "no valid package infra") return - hashpath = self.path.replace(".mk", ".hash") - if os.path.exists(os.path.join(brpath, hashpath)): + if os.path.exists(self.hashpath): self.status['hash'] = ("ok", "found") else: self.status['hash'] = ("error", "missing") @@ -225,8 +250,7 @@ class Package: self.status['patches'] = ("na", "no valid package infra") return - pkgdir = os.path.dirname(os.path.join(brpath, self.path)) - for subdir, _, _ in os.walk(pkgdir): + for subdir, _, _ in os.walk(self.pkgdir): self.patch_files = fnmatch.filter(os.listdir(subdir), '*.patch') if self.patch_count == 0: @@ -268,9 +292,8 @@ class Package: Fills in the .warnings and .status['pkg-check'] fields """ cmd = [os.path.join(brpath, "utils/check-package")] - pkgdir = os.path.dirname(os.path.join(brpath, self.path)) self.status['pkg-check'] = ("error", "Missing") - for root, dirs, files in os.walk(pkgdir): + for root, dirs, files in os.walk(self.pkgdir): for f in files: cmd.append(os.path.join(root, f)) o = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[1] @@ -327,7 +350,7 @@ class Package: self.is_status_ok('license-files'), self.status['hash'], self.patch_count) -def get_pkglist(npackages, package_list): +def get_pkglist(trees, npackages, package_list): """ Builds the list of Buildroot packages, returning a list of Package objects. Only the .name and .path fields of the Package object are @@ -362,8 +385,8 @@ def get_pkglist(npackages, package_list): "toolchain/toolchain-wrapper.mk"] packages = list() count = 0 - for root, dirs, files in os.walk(brpath): - root = os.path.relpath(root, brpath) + for br_tree, root, dirs, files in ((tree, *rdf) for tree in trees for rdf in os.walk(tree.path)): + root = os.path.relpath(root, br_tree.path) rootdir = root.split("/") if len(rootdir) < 1: continue @@ -384,7 +407,7 @@ def get_pkglist(npackages, package_list): continue if skip: continue - p = Package(pkgname, pkgpath) + p = Package(br_tree, pkgname, pkgpath) packages.append(p) count += 1 if npackages and count == npackages: @@ -858,7 +881,7 @@ function expandField(fieldId){ #package-grid, #results-grid { display: grid; grid-gap: 2px; - grid-template-columns: 1fr repeat(12, min-content); + grid-template-columns: min-content 1fr repeat(12, min-content); } #results-grid { grid-template-columns: 3fr 1fr; @@ -924,6 +947,8 @@ def boolean_str(b): def dump_html_pkg(f, pkg): pkg_css_class = pkg.path.replace("/", "_")[:-3] + f.write(f'
{pkg.tree}
\n') f.write(f'
{pkg.path}
\n') # Patch count @@ -1128,31 +1153,33 @@ def dump_html_pkg(f, pkg): def dump_html_all_pkgs(f, packages): f.write("""
-
Tree
+
Package
-
Patch count
-
Infrastructure
-
License
-
License files
-
Hash file
-
Current version
-
Latest version
-
Warnings
-
Upstream URL
-
CVEs
-
CVEs Ignored
-
CPE ID
""") for pkg in sorted(packages): @@ -1223,7 +1250,7 @@ def dump_html(packages, stats, date, commit, output): def dump_json(packages, defconfigs, stats, date, commit, output): # Format packages as a dictionnary instead of a list # Exclude local field that does not contains real date - excluded_fields = ['url_worker', 'name'] + excluded_fields = ['url_worker', 'name', 'tree_path'] pkgs = { pkg.name: { k: v @@ -1311,7 +1338,8 @@ def __main__(): 'rev-parse', 'HEAD']).splitlines()[0].decode() print("Build package list ...") - packages = get_pkglist(args.npackages, package_list) + all_trees = get_trees() + packages = get_pkglist(all_trees, args.npackages, package_list) print("Getting developers ...") developers = parse_developers() print("Build defconfig list ...")