394 lines
15 KiB
Python
394 lines
15 KiB
Python
|
from functools import partial
|
||
|
|
from glob import glob
|
||
|
|
from distutils.util import convert_path
|
||
|
|
import distutils.command.build_py as orig
|
||
|
|
import os
|
||
|
|
import fnmatch
|
||
|
|
import textwrap
|
||
|
|
import distutils.errors
|
||
|
|
import itertools
|
||
|
|
import stat
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import Dict, Iterable, Iterator, List, Optional, Tuple
|
||
|
|
|
||
|
|
from ..extern.more_itertools import unique_everseen
|
||
|
|
from ..warnings import SetuptoolsDeprecationWarning
|
||
|
|
|
||
|
|
|
||
|
|
_IMPLICIT_DATA_FILES = ('*.pyi', 'py.typed')
|
||
|
|
|
||
|
|
|
||
|
|
def make_writable(target):
|
||
|
|
os.chmod(target, os.stat(target).st_mode | stat.S_IWRITE)
|
||
|
|
|
||
|
|
|
||
|
|
class build_py(orig.build_py):
|
||
|
|
"""Enhanced 'build_py' command that includes data files with packages
|
||
|
|
|
||
|
|
The data files are specified via a 'package_data' argument to 'setup()'.
|
||
|
|
See 'setuptools.dist.Distribution' for more details.
|
||
|
|
|
||
|
|
Also, this version of the 'build_py' command allows you to specify both
|
||
|
|
'py_modules' and 'packages' in the same setup operation.
|
||
|
|
"""
|
||
|
|
|
||
|
|
editable_mode: bool = False
|
||
|
|
existing_egg_info_dir: Optional[str] = None #: Private API, internal use only.
|
||
|
|
|
||
|
|
def finalize_options(self):
|
||
|
|
orig.build_py.finalize_options(self)
|
||
|
|
self.package_data = self.distribution.package_data
|
||
|
|
self.exclude_package_data = self.distribution.exclude_package_data or {}
|
||
|
|
if 'data_files' in self.__dict__:
|
||
|
|
del self.__dict__['data_files']
|
||
|
|
self.__updated_files = []
|
||
|
|
|
||
|
|
def copy_file(
|
||
|
|
self, infile, outfile, preserve_mode=1, preserve_times=1, link=None, level=1
|
||
|
|
):
|
||
|
|
# Overwrite base class to allow using links
|
||
|
|
if link:
|
||
|
|
infile = str(Path(infile).resolve())
|
||
|
|
outfile = str(Path(outfile).resolve())
|
||
|
|
return super().copy_file(
|
||
|
|
infile, outfile, preserve_mode, preserve_times, link, level
|
||
|
|
)
|
||
|
|
|
||
|
|
def run(self):
|
||
|
|
"""Build modules, packages, and copy data files to build directory"""
|
||
|
|
if not (self.py_modules or self.packages) or self.editable_mode:
|
||
|
|
return
|
||
|
|
|
||
|
|
if self.py_modules:
|
||
|
|
self.build_modules()
|
||
|
|
|
||
|
|
if self.packages:
|
||
|
|
self.build_packages()
|
||
|
|
self.build_package_data()
|
||
|
|
|
||
|
|
# Only compile actual .py files, using our base class' idea of what our
|
||
|
|
# output files are.
|
||
|
|
self.byte_compile(orig.build_py.get_outputs(self, include_bytecode=0))
|
||
|
|
|
||
|
|
def __getattr__(self, attr):
|
||
|
|
"lazily compute data files"
|
||
|
|
if attr == 'data_files':
|
||
|
|
self.data_files = self._get_data_files()
|
||
|
|
return self.data_files
|
||
|
|
return orig.build_py.__getattr__(self, attr)
|
||
|
|
|
||
|
|
def build_module(self, module, module_file, package):
|
||
|
|
outfile, copied = orig.build_py.build_module(self, module, module_file, package)
|
||
|
|
if copied:
|
||
|
|
self.__updated_files.append(outfile)
|
||
|
|
return outfile, copied
|
||
|
|
|
||
|
|
def _get_data_files(self):
|
||
|
|
"""Generate list of '(package,src_dir,build_dir,filenames)' tuples"""
|
||
|
|
self.analyze_manifest()
|
||
|
|
return list(map(self._get_pkg_data_files, self.packages or ()))
|
||
|
|
|
||
|
|
def get_data_files_without_manifest(self):
|
||
|
|
"""
|
||
|
|
Generate list of ``(package,src_dir,build_dir,filenames)`` tuples,
|
||
|
|
but without triggering any attempt to analyze or build the manifest.
|
||
|
|
"""
|
||
|
|
# Prevent eventual errors from unset `manifest_files`
|
||
|
|
# (that would otherwise be set by `analyze_manifest`)
|
||
|
|
self.__dict__.setdefault('manifest_files', {})
|
||
|
|
return list(map(self._get_pkg_data_files, self.packages or ()))
|
||
|
|
|
||
|
|
def _get_pkg_data_files(self, package):
|
||
|
|
# Locate package source directory
|
||
|
|
src_dir = self.get_package_dir(package)
|
||
|
|
|
||
|
|
# Compute package build directory
|
||
|
|
build_dir = os.path.join(*([self.build_lib] + package.split('.')))
|
||
|
|
|
||
|
|
# Strip directory from globbed filenames
|
||
|
|
filenames = [
|
||
|
|
os.path.relpath(file, src_dir)
|
||
|
|
for file in self.find_data_files(package, src_dir)
|
||
|
|
]
|
||
|
|
return package, src_dir, build_dir, filenames
|
||
|
|
|
||
|
|
def find_data_files(self, package, src_dir):
|
||
|
|
"""Return filenames for package's data files in 'src_dir'"""
|
||
|
|
patterns = self._get_platform_patterns(
|
||
|
|
self.package_data,
|
||
|
|
package,
|
||
|
|
src_dir,
|
||
|
|
extra_patterns=_IMPLICIT_DATA_FILES,
|
||
|
|
)
|
||
|
|
globs_expanded = map(partial(glob, recursive=True), patterns)
|
||
|
|
# flatten the expanded globs into an iterable of matches
|
||
|
|
globs_matches = itertools.chain.from_iterable(globs_expanded)
|
||
|
|
glob_files = filter(os.path.isfile, globs_matches)
|
||
|
|
files = itertools.chain(
|
||
|
|
self.manifest_files.get(package, []),
|
||
|
|
glob_files,
|
||
|
|
)
|
||
|
|
return self.exclude_data_files(package, src_dir, files)
|
||
|
|
|
||
|
|
def get_outputs(self, include_bytecode=1) -> List[str]:
|
||
|
|
"""See :class:`setuptools.commands.build.SubCommand`"""
|
||
|
|
if self.editable_mode:
|
||
|
|
return list(self.get_output_mapping().keys())
|
||
|
|
return super().get_outputs(include_bytecode)
|
||
|
|
|
||
|
|
def get_output_mapping(self) -> Dict[str, str]:
|
||
|
|
"""See :class:`setuptools.commands.build.SubCommand`"""
|
||
|
|
mapping = itertools.chain(
|
||
|
|
self._get_package_data_output_mapping(),
|
||
|
|
self._get_module_mapping(),
|
||
|
|
)
|
||
|
|
return dict(sorted(mapping, key=lambda x: x[0]))
|
||
|
|
|
||
|
|
def _get_module_mapping(self) -> Iterator[Tuple[str, str]]:
|
||
|
|
"""Iterate over all modules producing (dest, src) pairs."""
|
||
|
|
for package, module, module_file in self.find_all_modules():
|
||
|
|
package = package.split('.')
|
||
|
|
filename = self.get_module_outfile(self.build_lib, package, module)
|
||
|
|
yield (filename, module_file)
|
||
|
|
|
||
|
|
def _get_package_data_output_mapping(self) -> Iterator[Tuple[str, str]]:
|
||
|
|
"""Iterate over package data producing (dest, src) pairs."""
|
||
|
|
for package, src_dir, build_dir, filenames in self.data_files:
|
||
|
|
for filename in filenames:
|
||
|
|
target = os.path.join(build_dir, filename)
|
||
|
|
srcfile = os.path.join(src_dir, filename)
|
||
|
|
yield (target, srcfile)
|
||
|
|
|
||
|
|
def build_package_data(self):
|
||
|
|
"""Copy data files into build directory"""
|
||
|
|
for target, srcfile in self._get_package_data_output_mapping():
|
||
|
|
self.mkpath(os.path.dirname(target))
|
||
|
|
_outf, _copied = self.copy_file(srcfile, target)
|
||
|
|
make_writable(target)
|
||
|
|
|
||
|
|
def analyze_manifest(self):
|
||
|
|
self.manifest_files = mf = {}
|
||
|
|
if not self.distribution.include_package_data:
|
||
|
|
return
|
||
|
|
src_dirs = {}
|
||
|
|
for package in self.packages or ():
|
||
|
|
# Locate package source directory
|
||
|
|
src_dirs[assert_relative(self.get_package_dir(package))] = package
|
||
|
|
|
||
|
|
if (
|
||
|
|
getattr(self, 'existing_egg_info_dir', None)
|
||
|
|
and Path(self.existing_egg_info_dir, "SOURCES.txt").exists()
|
||
|
|
):
|
||
|
|
egg_info_dir = self.existing_egg_info_dir
|
||
|
|
manifest = Path(egg_info_dir, "SOURCES.txt")
|
||
|
|
files = manifest.read_text(encoding="utf-8").splitlines()
|
||
|
|
else:
|
||
|
|
self.run_command('egg_info')
|
||
|
|
ei_cmd = self.get_finalized_command('egg_info')
|
||
|
|
egg_info_dir = ei_cmd.egg_info
|
||
|
|
files = ei_cmd.filelist.files
|
||
|
|
|
||
|
|
check = _IncludePackageDataAbuse()
|
||
|
|
for path in self._filter_build_files(files, egg_info_dir):
|
||
|
|
d, f = os.path.split(assert_relative(path))
|
||
|
|
prev = None
|
||
|
|
oldf = f
|
||
|
|
while d and d != prev and d not in src_dirs:
|
||
|
|
prev = d
|
||
|
|
d, df = os.path.split(d)
|
||
|
|
f = os.path.join(df, f)
|
||
|
|
if d in src_dirs:
|
||
|
|
if f == oldf:
|
||
|
|
if check.is_module(f):
|
||
|
|
continue # it's a module, not data
|
||
|
|
else:
|
||
|
|
importable = check.importable_subpackage(src_dirs[d], f)
|
||
|
|
if importable:
|
||
|
|
check.warn(importable)
|
||
|
|
mf.setdefault(src_dirs[d], []).append(path)
|
||
|
|
|
||
|
|
def _filter_build_files(self, files: Iterable[str], egg_info: str) -> Iterator[str]:
|
||
|
|
"""
|
||
|
|
``build_meta`` may try to create egg_info outside of the project directory,
|
||
|
|
and this can be problematic for certain plugins (reported in issue #3500).
|
||
|
|
|
||
|
|
Extensions might also include between their sources files created on the
|
||
|
|
``build_lib`` and ``build_temp`` directories.
|
||
|
|
|
||
|
|
This function should filter this case of invalid files out.
|
||
|
|
"""
|
||
|
|
build = self.get_finalized_command("build")
|
||
|
|
build_dirs = (egg_info, self.build_lib, build.build_temp, build.build_base)
|
||
|
|
norm_dirs = [os.path.normpath(p) for p in build_dirs if p]
|
||
|
|
|
||
|
|
for file in files:
|
||
|
|
norm_path = os.path.normpath(file)
|
||
|
|
if not os.path.isabs(file) or all(d not in norm_path for d in norm_dirs):
|
||
|
|
yield file
|
||
|
|
|
||
|
|
def get_data_files(self):
|
||
|
|
pass # Lazily compute data files in _get_data_files() function.
|
||
|
|
|
||
|
|
def check_package(self, package, package_dir):
|
||
|
|
"""Check namespace packages' __init__ for declare_namespace"""
|
||
|
|
try:
|
||
|
|
return self.packages_checked[package]
|
||
|
|
except KeyError:
|
||
|
|
pass
|
||
|
|
|
||
|
|
init_py = orig.build_py.check_package(self, package, package_dir)
|
||
|
|
self.packages_checked[package] = init_py
|
||
|
|
|
||
|
|
if not init_py or not self.distribution.namespace_packages:
|
||
|
|
return init_py
|
||
|
|
|
||
|
|
for pkg in self.distribution.namespace_packages:
|
||
|
|
if pkg == package or pkg.startswith(package + '.'):
|
||
|
|
break
|
||
|
|
else:
|
||
|
|
return init_py
|
||
|
|
|
||
|
|
with open(init_py, 'rb') as f:
|
||
|
|
contents = f.read()
|
||
|
|
if b'declare_namespace' not in contents:
|
||
|
|
raise distutils.errors.DistutilsError(
|
||
|
|
"Namespace package problem: %s is a namespace package, but "
|
||
|
|
"its\n__init__.py does not call declare_namespace()! Please "
|
||
|
|
'fix it.\n(See the setuptools manual under '
|
||
|
|
'"Namespace Packages" for details.)\n"' % (package,)
|
||
|
|
)
|
||
|
|
return init_py
|
||
|
|
|
||
|
|
def initialize_options(self):
|
||
|
|
self.packages_checked = {}
|
||
|
|
orig.build_py.initialize_options(self)
|
||
|
|
self.editable_mode = False
|
||
|
|
self.existing_egg_info_dir = None
|
||
|
|
|
||
|
|
def get_package_dir(self, package):
|
||
|
|
res = orig.build_py.get_package_dir(self, package)
|
||
|
|
if self.distribution.src_root is not None:
|
||
|
|
return os.path.join(self.distribution.src_root, res)
|
||
|
|
return res
|
||
|
|
|
||
|
|
def exclude_data_files(self, package, src_dir, files):
|
||
|
|
"""Filter filenames for package's data files in 'src_dir'"""
|
||
|
|
files = list(files)
|
||
|
|
patterns = self._get_platform_patterns(
|
||
|
|
self.exclude_package_data,
|
||
|
|
package,
|
||
|
|
src_dir,
|
||
|
|
)
|
||
|
|
match_groups = (fnmatch.filter(files, pattern) for pattern in patterns)
|
||
|
|
# flatten the groups of matches into an iterable of matches
|
||
|
|
matches = itertools.chain.from_iterable(match_groups)
|
||
|
|
bad = set(matches)
|
||
|
|
keepers = (fn for fn in files if fn not in bad)
|
||
|
|
# ditch dupes
|
||
|
|
return list(unique_everseen(keepers))
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _get_platform_patterns(spec, package, src_dir, extra_patterns=[]):
|
||
|
|
"""
|
||
|
|
yield platform-specific path patterns (suitable for glob
|
||
|
|
or fn_match) from a glob-based spec (such as
|
||
|
|
self.package_data or self.exclude_package_data)
|
||
|
|
matching package in src_dir.
|
||
|
|
"""
|
||
|
|
raw_patterns = itertools.chain(
|
||
|
|
extra_patterns,
|
||
|
|
spec.get('', []),
|
||
|
|
spec.get(package, []),
|
||
|
|
)
|
||
|
|
return (
|
||
|
|
# Each pattern has to be converted to a platform-specific path
|
||
|
|
os.path.join(src_dir, convert_path(pattern))
|
||
|
|
for pattern in raw_patterns
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def assert_relative(path):
|
||
|
|
if not os.path.isabs(path):
|
||
|
|
return path
|
||
|
|
from distutils.errors import DistutilsSetupError
|
||
|
|
|
||
|
|
msg = (
|
||
|
|
textwrap.dedent(
|
||
|
|
"""
|
||
|
|
Error: setup script specifies an absolute path:
|
||
|
|
|
||
|
|
%s
|
||
|
|
|
||
|
|
setup() arguments must *always* be /-separated paths relative to the
|
||
|
|
setup.py directory, *never* absolute paths.
|
||
|
|
"""
|
||
|
|
).lstrip()
|
||
|
|
% path
|
||
|
|
)
|
||
|
|
raise DistutilsSetupError(msg)
|
||
|
|
|
||
|
|
|
||
|
|
class _IncludePackageDataAbuse:
|
||
|
|
"""Inform users that package or module is included as 'data file'"""
|
||
|
|
|
||
|
|
class _Warning(SetuptoolsDeprecationWarning):
|
||
|
|
_SUMMARY = """
|
||
|
|
Package {importable!r} is absent from the `packages` configuration.
|
||
|
|
"""
|
||
|
|
|
||
|
|
_DETAILS = """
|
||
|
|
############################
|
||
|
|
# Package would be ignored #
|
||
|
|
############################
|
||
|
|
Python recognizes {importable!r} as an importable package[^1],
|
||
|
|
but it is absent from setuptools' `packages` configuration.
|
||
|
|
|
||
|
|
This leads to an ambiguous overall configuration. If you want to distribute this
|
||
|
|
package, please make sure that {importable!r} is explicitly added
|
||
|
|
to the `packages` configuration field.
|
||
|
|
|
||
|
|
Alternatively, you can also rely on setuptools' discovery methods
|
||
|
|
(for example by using `find_namespace_packages(...)`/`find_namespace:`
|
||
|
|
instead of `find_packages(...)`/`find:`).
|
||
|
|
|
||
|
|
You can read more about "package discovery" on setuptools documentation page:
|
||
|
|
|
||
|
|
- https://setuptools.pypa.io/en/latest/userguide/package_discovery.html
|
||
|
|
|
||
|
|
If you don't want {importable!r} to be distributed and are
|
||
|
|
already explicitly excluding {importable!r} via
|
||
|
|
`find_namespace_packages(...)/find_namespace` or `find_packages(...)/find`,
|
||
|
|
you can try to use `exclude_package_data`, or `include-package-data=False` in
|
||
|
|
combination with a more fine grained `package-data` configuration.
|
||
|
|
|
||
|
|
You can read more about "package data files" on setuptools documentation page:
|
||
|
|
|
||
|
|
- https://setuptools.pypa.io/en/latest/userguide/datafiles.html
|
||
|
|
|
||
|
|
|
||
|
|
[^1]: For Python, any directory (with suitable naming) can be imported,
|
||
|
|
even if it does not contain any `.py` files.
|
||
|
|
On the other hand, currently there is no concept of package data
|
||
|
|
directory, all directories are treated like packages.
|
||
|
|
"""
|
||
|
|
# _DUE_DATE: still not defined as this is particularly controversial.
|
||
|
|
# Warning initially introduced in May 2022. See issue #3340 for discussion.
|
||
|
|
|
||
|
|
def __init__(self):
|
||
|
|
self._already_warned = set()
|
||
|
|
|
||
|
|
def is_module(self, file):
|
||
|
|
return file.endswith(".py") and file[: -len(".py")].isidentifier()
|
||
|
|
|
||
|
|
def importable_subpackage(self, parent, file):
|
||
|
|
pkg = Path(file).parent
|
||
|
|
parts = list(itertools.takewhile(str.isidentifier, pkg.parts))
|
||
|
|
if parts:
|
||
|
|
return ".".join([parent, *parts])
|
||
|
|
return None
|
||
|
|
|
||
|
|
def warn(self, importable):
|
||
|
|
if importable not in self._already_warned:
|
||
|
|
self._Warning.emit(importable=importable)
|
||
|
|
self._already_warned.add(importable)
|