Source code for craft_parts.executor.filesets

# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
# Copyright 2015-2021 Canonical Ltd.
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License version 3 as published by the Free Software Foundation.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# Lesser General Public License for more details.
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <>.

"""Definitions and helpers to handle filesets."""

import os
from glob import iglob
from typing import List, Set, Tuple

from craft_parts import errors
from craft_parts.utils import path_utils

[docs]class Fileset: """Helper class to process string lists.""" def __init__(self, entries: List[str], *, name: str = ""): self._name = name self._list = entries def __repr__(self) -> str: return f"Fileset({self._list!r}, name={self._name!r})" @property def name(self) -> str: """Return the fileset name.""" return self._name @property def entries(self) -> List[str]: """Return the list of entries in this fileset.""" return self._list.copy() @property def includes(self) -> List[str]: """Return the list of files to be included.""" return [path_utils.get_partitioned_path(x) for x in self._list if x[0] != "-"] @property def excludes(self) -> List[str]: """Return the list of files to be excluded.""" return [ path_utils.get_partitioned_path(x[1:]) for x in self._list if x[0] == "-" ]
[docs] def remove(self, item: str) -> None: """Remove this entry from the list of files. :param item: The item to remove. """ self._list.remove(item)
[docs] def combine(self, other: "Fileset") -> None: """Combine the entries in this fileset with entries from another fileset. :param other: The fileset to combine with. """ to_combine = False # combine if the fileset has a wildcard if "*" in self.entries: to_combine = True self.remove("*") other_excludes = set(other.excludes) my_includes = set(self.includes) contradicting_set = set.intersection(other_excludes, my_includes) if contradicting_set: raise errors.FilesetConflict(contradicting_set) # combine if the fileset is only excludes if {x[0] for x in self.entries} == set("-"): to_combine = True if to_combine: self._list = list(set(self._list + other.entries))
[docs]def migratable_filesets(fileset: Fileset, srcdir: str) -> Tuple[Set[str], Set[str]]: """Return the files and directories that can be migrated. :param fileset: The fileset to migrate. :return: A tuple containing the set of files and the set of directories that can be migrated. """ includes, excludes = _get_file_list(fileset) include_files = _generate_include_set(srcdir, includes) exclude_files, exclude_dirs = _generate_exclude_set(srcdir, excludes) files = include_files - exclude_files for exclude_dir in exclude_dirs: files = {x for x in files if not x.startswith(exclude_dir + "/")} # Separate dirs from files. dirs = { x for x in files if os.path.isdir(os.path.join(srcdir, x)) and not os.path.islink(os.path.join(srcdir, x)) } # Remove dirs from files. files = files - dirs # Include (resolved) parent directories for each selected file. for filename in files: filename = _get_resolved_relative_path(filename, srcdir) dirname = os.path.dirname(filename) while dirname: dirs.add(dirname) dirname = os.path.dirname(dirname) # Resolve parent paths for dirs and files. resolved_dirs = set() for dirname in dirs: resolved_dirs.add(_get_resolved_relative_path(dirname, srcdir)) resolved_files = set() for filename in files: resolved_files.add(_get_resolved_relative_path(filename, srcdir)) return resolved_files, resolved_dirs
def _get_file_list(fileset: Fileset) -> Tuple[List[str], List[str]]: """Split a fileset to obtain include and exclude file filters. :param fileset: The fileset to split. :return: A tuple containing the include and exclude lists. """ includes: List[str] = [] excludes: List[str] = [] for item in fileset.entries: if item.startswith("-"): excludes.append(item[1:]) elif item.startswith("\\"): includes.append(item[1:]) else: includes.append(item) # paths must be relative for entry in includes + excludes: if os.path.isabs(entry): raise errors.FilesetError(, message=f"path {entry!r} must be relative." ) includes = includes or ["*"] processed_includes: List[str] = [] processed_excludes: List[str] = [] for file in includes: processed_includes.append(path_utils.get_partitioned_path(file)) for file in excludes: processed_excludes.append(path_utils.get_partitioned_path(file)) return processed_includes, processed_excludes def _generate_include_set(directory: str, includes: List[str]) -> Set[str]: """Obtain the list of files to include based on include file filter. :param directory: The path to the tree containing the files to filter. :return: The set of files to include. """ include_files = set() for include in includes: if "*" in include: pattern = os.path.join(directory, include) matches = iglob(pattern, recursive=True) include_files |= set(matches) else: include_files |= set([os.path.join(directory, include)]) include_dirs = [ x for x in include_files if os.path.isdir(x) and not os.path.islink(x) ] include_files = {os.path.relpath(x, directory) for x in include_files} # Expand includeFiles, so that an exclude like '*/*.so' will still match # files from an include like 'lib' for include_dir in include_dirs: for root, dirs, files in os.walk(include_dir): include_files |= { os.path.relpath(os.path.join(root, d), directory) for d in dirs } include_files |= { os.path.relpath(os.path.join(root, f), directory) for f in files } return include_files def _generate_exclude_set( directory: str, excludes: List[str] ) -> Tuple[Set[str], Set[str]]: """Obtain the list of files to exclude based on exclude file filter. :param directory: The path to the tree containing the files to filter. :return: The set of files to exclude. """ exclude_files = set() for exclude in excludes: pattern = os.path.join(directory, exclude) matches = iglob(pattern, recursive=True) exclude_files |= set(matches) exclude_dirs = { os.path.relpath(x, directory) for x in exclude_files if os.path.isdir(x) } exclude_files = {os.path.relpath(x, directory) for x in exclude_files} return exclude_files, exclude_dirs def _get_resolved_relative_path(relative_path: str, base_directory: str) -> str: """Resolve path components against target base_directory. If the resulting target path is a symlink, it will not be followed. Only the path's parents are fully resolved against base_directory, and the relative path is returned. :param relative_path: Path of target, relative to base_directory. :param base_directory: Base path of target. :return: Resolved path, relative to base_directory. """ parent_relpath, filename = os.path.split(relative_path) parent_abspath = os.path.realpath(os.path.join(base_directory, parent_relpath)) filename_abspath = os.path.join(parent_abspath, filename) filename_relpath = os.path.relpath(filename_abspath, base_directory) return filename_relpath