Source code for meta_package_manager.managers.pkg

# Copyright Kevin Deldycke <[email protected]> and contributors.
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.

"""FreeBSD package managers.

Two managers share this module because they share the FreeBSD ecosystem and
the same on-disk install database:

- {class}`~meta_package_manager.managers.pkg.PKG` wraps the binary `pkg` frontend, which fetches
  pre-compiled artifacts from the official FreeBSD repository.
- {class}`Ports` wraps the source-build workflow rooted at `/usr/ports`,
  driving {command}`make` recipes directly and delegating registry queries
  back to `pkg`.

References:

- [`pkg(8)` man page](https://man.freebsd.org/cgi/man.cgi?query=pkg&sektion=8)
- [FreeBSD handbook on ports](https://docs.freebsd.org/en/books/handbook/ports/)
- [`ports(7)` man page](https://man.freebsd.org/cgi/man.cgi?query=ports&sektion=7)
"""

from __future__ import annotations

import json
import re
from functools import cached_property
from pathlib import Path
from typing import ClassVar

from extra_platforms import FREEBSD

from ..capabilities import Delegate, version_not_implemented
from ..manager import PackageManager

TYPE_CHECKING = False
if TYPE_CHECKING:
    from collections.abc import Iterator

    from ..package import Package


PORTS_TREE = Path("/usr/ports")
"""Canonical location of the FreeBSD ports tree.

The Handbook documents this path as the convention; `PORTSDIR` can override
it, but every tool and consumer in the wild assumes this default.
"""


[docs] class PKG(PackageManager): """FreeBSD's binary pkg frontend, fetching pre-compiled artifacts from the official FreeBSD repository. Only root may modify the package database, so mutating operations escalate through `sudo` by default, like the {class}`Ports` sibling. ```{note} `outdated` parses `pkg upgrade --dry-run` rather than `pkg version`, because only the dry-run names the target version each package would move to. ``` ```{caution} `sync` forces `IGNORE_OSVERSION=yes`: a package built for a newer FreeBSD than the running kernel would otherwise trigger an interactive confirmation that hangs the subprocess. It is passed as a `-o` command-line option rather than an environment variable, which sudo's environment reset would strip from the escalated call. Support for that setting is also why the version floor is `1.11`. ``` """ name = "FreeBSD pkg" homepage_url = "https://github.com/freebsd/pkg" logo = "freebsd" keywords = ("freebsd",) platforms = FREEBSD default_sudo = True requirement = ">=1.11" """1.11 is the first version to support the `IGNORE_OSVERSION` setting. ```{code-block} shell-session $ pkg --version 1.20.9 ``` """ """`--quiet` is deliberately absent from `pre_args`, and cannot go back. `pkg` accepts it only *after* the subcommand: as a global argument both `--quiet` and `-q` are answered with ``pkg: unrecognized option `--quiet'`` and exit `1`, which broke every operation on `pkg` 2.x. Each subcommand that takes the flag therefore carries it in its own argument list, and `query` never gets it at all, being the one subcommand that rejects it. """ _INSTALLED_REGEXP = re.compile(r"(\S+) (\S+) (.+)") _ORPHANS_REGEXP = re.compile( r"^\s+(?P<package_id>\S+): (?P<installed_version>\S+)$", re.MULTILINE, ) """Extract the indented `<name>: <version>` lines of `autoremove`'s removal manifest, skipping the flush-left narration around them.""" _OUTDATED_REGEXP = re.compile(r"(\S+): (\S+) -> (\S+) .+") @property def installed(self) -> Iterator[Package]: """Fetch installed packages. ```{code-block} shell-session $ pkg query "%n %v %c" 7-zip 21.07_2 Console version of the 7-Zip file archiver ap24-mod_mpm_itk 2.4.7_2 Run each vhost under a separate uid and gid apache24 2.4.57 Version 2.4.x of Apache web server aquantia-atlantic-kmod 0.0.5_1 Aquantia AQtion (Atlantic) Network Driver arcconf 3.07.23971,1 Adaptec SCSI/SAS RAID administration tool areca-cli-amd64 1.14.7.150519,1 Command Line Interface for ARC-xxxx RAID base64 1.5_1 Utility to encode and decode base64 files bash 5.1.12 GNU Project's Bourne Again SHell beadm 1.4_1 Solaris-like utility to manage Boot Environments on ZFS ``` """ output = self.run_cli("query", "%n %v %c") for package in output.splitlines(): match = self._INSTALLED_REGEXP.match(package) if match: package_id, installed_version, description = match.groups() yield self.package( id=package_id, description=description, installed_version=installed_version, ) @property def outdated(self) -> Iterator[Package]: """Fetch outdated packages. ```{code-block} shell-session $ pkg upgrade --quiet --dry-run Updating FreeBSD repository catalogue... FreeBSD repository is up to date. All repositories are up to date. Checking for upgrades (312 candidates): 100% Processing candidates (312 candidates): 100% The following 466 package(s) will be affected (of 0 checked): Installed packages to be REMOVED: freenas-files: 13.0_1700495253 py39-midcli: 20190509171453 py39-middlewared: 13.0_1700495253 New packages to be INSTALLED: abseil: 20230125.3 [FreeBSD] argp-standalone: 1.5.0 [FreeBSD] brotli: 1.1.0,1 [FreeBSD] Installed packages to be UPGRADED: 7-zip: 21.07_2 -> 23.01 [FreeBSD] apache24: 2.4.57 -> 2.4.58_1 [FreeBSD] apr: 1.7.0.1.6.1_1 -> 1.7.3.1.6.3_1 [FreeBSD] aquantia-atlantic-kmod: 0.0.5_1 -> 0.0.5_2 [FreeBSD] bash: 5.1.12 -> 5.2.21 [FreeBSD] ``` :::{note} We rely on `pkg upgrade` instead of `pkg version` because the latter does not provides the new version: ```{code-block} console $ pkg version --like "<" Updating FreeBSD repository catalogue... FreeBSD repository is up to date. All repositories are up to date. 7-zip-21.07_2 < apache24-2.4.57 < apr-1.7.0.1.6.1_1 < aquantia-atlantic-kmod-0.0.5_1 < bash-5.1.12 < ``` ::: """ output = self.run_cli("upgrade", "--quiet", "--dry-run") outdated_list = output.split("Installed packages to be UPGRADED:", 1)[1].strip() for package in outdated_list.splitlines(): match = self._OUTDATED_REGEXP.match(package.strip()) if match: package_id, installed_version, latest_version = match.groups() yield self.package( id=package_id, latest_version=latest_version, installed_version=installed_version, ) @property def orphans(self) -> Iterator[Package]: """Fetch packages installed as dependencies that nothing requires anymore. `--dry-run` turns `autoremove` into a read-only report of the would-be-removed packages. ```{code-block} shell-session $ pkg autoremove --quiet --dry-run Checking integrity... done (0 conflicting) Deinstallation has been requested for the following 2 packages: Installed packages to be REMOVED: libiconv: 1.17 pcre: 8.45_3 Number of packages to be removed: 2 ``` """ output = self.run_cli("autoremove", "--quiet", "--dry-run") yield from self.parse_regex_lines(self._ORPHANS_REGEXP, output)
[docs] def search(self, query: str, extended: bool, exact: bool) -> Iterator[Package]: """Fetch matching packages. ```{caution} The result is a single JSON *array*, not one object per line. `--raw` wraps every match in one, and `json-compact` only strips the whitespace. The stream of bare objects this once parsed was `pkg` 1.x's shape; the array below is what 2.7.5 returns. ``` ```{caution} `--quiet` stays out of this command even though `search` accepts it: it overrides `--raw` and collapses the output to bare `<name>-<version>` tokens, which carry neither the version field nor the comment. ``` A search matching nothing exits `1` with an empty array on `<stdout>` and nothing on `<stderr>`, which is why no `must_succeed` is passed: mpm only counts a non-zero exit as a failure when `<stderr>` is non-empty, so the empty result reads as one. Default search on ID substring, truncated for width: ```{code-block} console $ pkg search --raw --raw-format json-compact --search name nyancat [{"name":"nyancat","origin":"net/nyancat","version":"1.5.2,1","comment":"Animated telnet server that renders a loop of the nyan cat animation",(...)}] ``` Exact search on ID: ```{code-block} console $ pkg search --raw --raw-format json-compact --search name --exact nyancat ``` Extended search over the comment and description fields: ```{code-block} console $ pkg search --raw --raw-format json-compact \ --search name --search comment --search description nyancat ``` """ # The `search` subcommand is part of the command, not implied: without # it `pkg` reads `--raw` as a global argument and refuses to run. search_args = [ "search", "--raw", "--raw-format", "json-compact", "--search", "name", ] if exact: search_args.append("--exact") # Expand search to the comment and description fields. if extended: search_args += ["--search", "comment", "--search", "description"] # No `must_succeed`: `pkg` exits `1` on a search that matches nothing, # writing an empty JSON array and nothing to `<stderr>`, so the default # non-strict rule reads that as the empty result it is. output = self.run_cli(search_args, query) # A single top-level array, not one object per line: `json-compact` # compacts the whitespace and keeps the array `--raw` wraps results in. for package in json.loads(output) if output.strip() else (): yield self.package( id=package["name"], description=package["comment"], latest_version=package["version"], )
[docs] @version_not_implemented def install(self, package_id: str, version: str | None = None) -> str: """Install one package. ```{code-block} shell-session $ sudo pkg install --quiet --yes dmg2img Updating FreeBSD repository catalogue... FreeBSD repository is up to date. All repositories are up to date. Checking integrity... done (0 conflicting) The following 1 package(s) will be affected (of 0 checked): New packages to be INSTALLED: dmg2img: 1.6.7 [FreeBSD] Number of packages to be installed: 1 [1/1] Installing dmg2img-1.6.7... [1/1] Extracting dmg2img-1.6.7: 100% ``` """ return self.run_cli("install", "--quiet", "--yes", package_id, sudo=True)
[docs] def upgrade_all_cli(self) -> tuple[str, ...]: """Generates the CLI to upgrade all outdated packages. ```{code-block} shell-session $ sudo pkg upgrade --quiet --yes ``` """ return self.build_cli("upgrade", "--quiet", "--yes", sudo=True)
[docs] @version_not_implemented def upgrade_one_cli( self, package_id: str, version: str | None = None, ) -> tuple[str, ...]: """Generates the CLI to upgrade the provided package. ```{code-block} shell-session $ sudo pkg upgrade --quiet --yes dmg2img ``` """ return self.build_cli("upgrade", "--quiet", "--yes", package_id, sudo=True)
[docs] def remove(self, package_id: str) -> str: """Remove one package. ```{code-block} shell-session $ sudo pkg delete --quiet --yes dmg2img Checking integrity... done (0 conflicting) Deinstallation has been requested for the following 1 packages: Installed packages to be REMOVED: dmg2img: 1.6.7 Number of packages to be removed: 1 [1/1] Deinstalling dmg2img-1.6.7... [1/1] Deleting files for dmg2img-1.6.7: 100% pkg: Package database is busy while closing! ``` """ return self.run_cli("delete", "--quiet", "--yes", package_id, sudo=True)
[docs] def sync(self) -> None: """Sync package metadata. ```{code-block} shell-session $ sudo pkg -o IGNORE_OSVERSION=yes update --quiet Updating FreeBSD repository catalogue... Fetching meta.conf: 100% 163 B 0.2kB/s 00:01 Fetching packagesite.pkg: 100% 7 MiB 3.6MB/s 00:02 Processing entries: 100% FreeBSD repository update completed. 33804 packages processed. All repositories are up to date. ``` The `IGNORE_OSVERSION=yes` prevents blocking update: ```{code-block} console $ pkg update --quiet Updating FreeBSD repository catalogue... Fetching meta.conf: 100% 163 B 0.2kB/s 00:01 Fetching packagesite.pkg: 100% 7 MiB 3.6MB/s 00:02 Processing entries: 0% Newer FreeBSD version for package zziplib: To ignore this error set IGNORE_OSVERSION=yes - package: 1302001 - running kernel: 1301000 Ignore the mismatch and continue? [y/N]: ``` """ # The -o command-line form survives sudo's environment reset, which would # strip an IGNORE_OSVERSION passed as a plain environment variable. self.run_cli("-o", "IGNORE_OSVERSION=yes", "update", "--quiet", sudo=True)
[docs] def cleanup_orphan(self) -> None: """Remove every package installed as a dependency and no longer required. ```{code-block} shell-session $ sudo pkg autoremove --quiet --yes Checking integrity... done (0 conflicting) Nothing to do. ``` """ self.run_cli("autoremove", "--quiet", "--yes", sudo=True)
[docs] def cleanup_cache(self) -> None: """Delete every cached package from the local cache directory. ```{code-block} shell-session $ sudo pkg clean --quiet --yes --all Nothing to do. ``` """ self.run_cli("clean", "--quiet", "--yes", "--all", sudo=True)
[docs] def doctor_cli(self) -> tuple[str, ...]: """Generates the CLI running the native self-diagnosis. `check --checksums` validates every installed package's files against their recorded checksums, exiting non-zero on mismatches. ```{code-block} shell-session $ pkg check --checksums --all ``` """ return self.build_cli("check", "--checksums", "--all")
[docs] class Ports(PackageManager): """FreeBSD ports tree: the source-build workflow rooted at `/usr/ports`. ```{note} Coexists with {class}`~meta_package_manager.managers.pkg.PKG` on the same system: both share the install database maintained by `pkg`. `Ports` builds and tracks ports compiled from source under `/usr/ports`, while `PKG` handles binary packages from the FreeBSD repository. Listing operations may overlap because `pkg` does not distinguish ports-built from binary-installed packages once they are registered. ``` ```{note} `installed` and `outdated` delegate to the sibling `pkg` binary, since the ports tree keeps no registry of its own. Builds drive FreeBSD's `make` directly with `BATCH=yes` to accept default build options without prompting. Upgrades shell out to the third-party `portmaster`: the ports tree ships no batch upgrader. `sync` refreshes the tree with `git` (`portsnap` was removed after FreeBSD 13). ``` ```{caution} Mutating operations require root privileges and a populated ports tree at `/usr/ports`. The manager flags itself unavailable when the tree is missing. ``` :::{caution} Mutating operations compile from source, so their duration is set by the port rather than by the network. A small library finishes well inside the 500 second ceiling `mpm` puts on a state-changing command (`expat` takes about a minute), where a compiler or a browser runs for hours and reports `Timed out after 500s`. Raise the ceiling for this manager alone, in the configuration file: ```toml [mpm.overrides.ports] timeout = 14400 ``` No single value fits, a build scaling with the port, its dependency tree and the machine. `sync` and `installed` are unaffected, delegating to `git` and `pkg` rather than building. ::: """ # Removal goes through the shared install database, identical to PKG. _pkg = Delegate(PKG) name = "FreeBSD Ports Collection" homepage_url = "https://www.freebsd.org/ports/" logo = "freebsd" keywords = ("freebsd ports",) platforms = FREEBSD default_sudo = True cli_names = ("make",) """The ports tree is driven by FreeBSD's {command}`make`. No dedicated frontend exists; each port is a directory whose `Makefile` targets are invoked directly. """ extra_env: ClassVar = {"BATCH": "yes"} """Force non-interactive builds. Many ports prompt for build option dialogs by default. `BATCH=yes` accepts the saved or default options without user interaction, which is the only sensible behavior for an automated tool. See `ports(7)`. """ version_cli_options = ("-V", "MAKE_VERSION") """FreeBSD `make` exposes its version via internal variable expansion. GNU Make's `--version` flag does not work on BSD make, so the probe reads the variable instead, which also avoids matching a GNU Make installation shadowing the BSD binary. The variable is `MAKE_VERSION`, the one `make(1)` documents as "the version of make (...) typically the date of last import from NetBSD". The dotted `.MAKE.VERSION` this once read is not among the `.MAKE.*` family that page lists, and expands to the empty string: the probe then found no version at all, which left `ports` permanently unavailable rather than merely misreported. """ version_regexes = (r"(?P<version>\d{8,})",) """BSD make reports its version as a date-like integer (e.g. `20240218`).""" _OUTDATED_REGEXP = re.compile( r"^(?P<package_id>\S+)\s+<\s+needs updating\s+\(port has (?P<latest_version>\S+)\)", re.MULTILINE, ) """Match outdated entries from `pkg version -vPL=` output. Format per line: `<pkgname-pkgver> <op> needs updating (port has <latest_version>)` """ _NAME_VERSION_REGEXP = re.compile(r"^(?P<package_id>.+)-(?P<version>[^-]+)$") """Split `<name>-<version>` strings reported by `pkg version`. The version starts at the last hyphen; everything before it is the package name, including embedded hyphens (e.g. `py311-pip-23.2`). """
[docs] @cached_property def available(self) -> bool: """Available only when `make` is found *and* the ports tree exists. The {command}`make` binary alone is not enough: without a populated `/usr/ports` directory, every operation would fail. Treat the tree as part of the manager's runtime requirement. """ if not (self.supported and self.cli_path and self.executable and self.fresh): return False return (PORTS_TREE / "Makefile").is_file()
@property def installed(self) -> Iterator[Package]: """Fetch packages currently registered as installed. Delegates to `pkg query` because the ports tree itself maintains no registry: ports installs are recorded in the same database as binary `pkg` installs. ```{code-block} shell-session $ pkg query "%n %v %o %c" curl 8.7.1 ftp/curl Non-interactive tool to get files from FTP/HTTP servers python311 3.11.9 lang/python311 Interpreted object-oriented programming language ``` """ pkg_path = self.sibling_cli("pkg") output = self.run_cli( "query", # No quotes around the format: mpm builds an argv rather than a # shell line, so `pkg` would take them as literal format text and # wrap every record in them, leaving each id with a leading `"`. "%n %v %o %c", override_cli_path=pkg_path, auto_pre_args=False, auto_extra_env=False, ) for line in output.splitlines(): parts = line.split(" ", 3) if len(parts) < 4: continue package_id, installed_version, _origin, description = parts yield self.package( id=package_id, description=description, installed_version=installed_version, ) @property def outdated(self) -> Iterator[Package]: """Fetch packages whose installed version lags the ports tree. Uses `pkg version` in ports-comparison mode (`-P`): it walks the local tree for each installed package and reports those with a newer `Makefile` version available. `-L =` drops the packages that are already current, so every reported line is an actionable one. `-I`, which reads `/usr/ports/INDEX-<major>` instead of the tree, is not combined with it: `pkg` accepts only one source and exits with a usage error on `-vIPL=`. The tree is also what {attr}`available` already requires, where the index is a separate file the user has to fetch. ```{code-block} shell-session $ pkg version -vPL= expat-2.8.2 < needs updating (port has 2.8.3) git-2.54.0 < needs updating (port has 2.55.0) libffi-3.6.0 < needs updating (port has 3.8.0) FreeBSD-acct-15.1 ? orphaned: base/FreeBSD-acct ``` """ pkg_path = self.sibling_cli("pkg") output = self.run_cli( "version", "-vPL=", override_cli_path=pkg_path, auto_pre_args=False, auto_extra_env=False, ) for match in self._OUTDATED_REGEXP.finditer(output): split = self.split_name_version(match.group("package_id")) if not split: continue package_id, installed_version = split yield self.package( id=package_id, installed_version=installed_version, latest_version=match.group("latest_version"), )
[docs] @version_not_implemented def install(self, package_id: str, version: str | None = None) -> str: """Build and install a port from source. `package_id` may be either a bare port name like `nginx` or its full origin like `www/nginx`. A bare name is resolved to its origin through `_resolve_origin`, which queries the active repository. The block below illustrates rather than captures: the origin is resolved by a query, so the corpus cannot rebuild this command from a stand-in package id. Read the exact argv off `mpm --plan install`. ```{code-block} console $ sudo make -C /usr/ports/www/nginx install clean BATCH=yes ``` """ origin = self._resolve_origin(package_id) port_dir = PORTS_TREE / origin return self.run_cli( "-C", str(port_dir), "install", "clean", sudo=True, )
[docs] def upgrade_all_cli(self) -> tuple[str, ...]: """Generate the CLI to upgrade every outdated port. The ports tree has no first-party batch upgrader; the workflow relies on the third-party `portmaster` tool. We build the command line without checking that `portmaster` is installed, because upgrade commands are typically printed for the user to inspect before running. `-G` is what makes the run unattended, and `--no-confirm` does not cover it: `portmaster` invokes `make config` for any port whose options were never saved, which spawns the `portconfig` dialog even under `BATCH=yes`. With no terminal to answer it, the dialog spins until `mpm` times out the whole command, having built nothing. Both flags stay short because `portmaster` 3.35 offers no long form for either: its parser accepts `--force-config`, the *opposite* of `-G`, and nothing spelling `-a`. ```{code-block} shell-session $ sudo portmaster --no-confirm --no-term-title -G -a ``` """ portmaster = self.which("portmaster") or Path("portmaster") return self.build_cli( "--no-confirm", "--no-term-title", "-G", "-a", override_cli_path=portmaster, auto_pre_args=False, sudo=True, )
[docs] @version_not_implemented def upgrade_one_cli( self, package_id: str, version: str | None = None, ) -> tuple[str, ...]: """Generate the CLI to upgrade one port via `portmaster`. Carries `-G` for the reason {meth}`upgrade_all_cli` gives: without it `portmaster` stops on the `portconfig` options dialog. ```{code-block} console $ sudo portmaster --no-confirm --no-term-title -G www/nginx ``` """ origin = self._resolve_origin(package_id) portmaster = self.which("portmaster") or Path("portmaster") return self.build_cli( "--no-confirm", "--no-term-title", "-G", origin, override_cli_path=portmaster, auto_pre_args=False, sudo=True, )
remove = _pkg.remove """Reuses {meth}`PKG.remove`: the ports tree has no native uninstaller, and removal goes through the shared install database regardless of how the package was originally built. """
[docs] def sync(self) -> None: """Refresh the local ports tree from upstream. Modern FreeBSD distributes the ports tree via Git; `portsnap` was deprecated and removed after FreeBSD 13. We pull from whatever remote the tree was checked out from. ```{code-block} shell-session $ sudo git -C /usr/ports pull --ff-only ``` """ git_path = self.sibling_cli("git") self.run_cli( "-C", PORTS_TREE.as_posix(), "pull", "--ff-only", override_cli_path=git_path, auto_pre_args=False, auto_extra_env=False, sudo=True, )
[docs] def cleanup_cache(self) -> None: """Remove cached build artifacts from the ports tree. Walks the tree once and invokes {command}`make clean` at the root, which recursively cleans every port's work directory. `DISTCLEAN=yes` also removes downloaded distfiles. ```{code-block} shell-session $ sudo make -C /usr/ports clean DISTCLEAN=yes BATCH=yes ``` """ self.run_cli( "-C", PORTS_TREE.as_posix(), "clean", "DISTCLEAN=yes", sudo=True, )
def _resolve_origin(self, package_id: str) -> str: """Resolve a port name to its `category/portname` origin. Accepts either form and returns the origin verbatim when already slashed. Otherwise queries the `pkg` binary to look up the origin from the configured repository. The lookup runs under `force_exec`, like the `go env GOBIN` probe that `go` cannot build a command without. Plan mode captures by the *operation* in flight rather than by the command, so a read issued from inside `install` is recorded instead of run: without the flag this helper reads back an empty string and reports every package as unresolvable under `mpm --plan install`. `--search name` is passed even though [`pkg-search(8)`](https://man.freebsd.org/cgi/man.cgi?query=pkg-search) documents that field as the default for a term holding no `/`: under `--exact`, `pkg` 2.7.5 matches nothing without it, so `pkg search --exact --quiet --origins curl` reports no result on a host where `ftp/curl` is installed. `--origins` does not select the field either, being an output modifier equivalent to `-L origin`. """ if "/" in package_id: return package_id pkg_path = self.sibling_cli("pkg") output = self.run_cli( "search", "--exact", "--quiet", "--search", "name", "--origins", package_id, override_cli_path=pkg_path, auto_pre_args=False, auto_extra_env=False, force_exec=True, ) first_line = output.strip().splitlines() if not first_line: msg = f"Could not resolve port origin for {package_id!r}." raise ValueError(msg) return first_line[0].strip()