_known_annotated_metadata.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403
  1. from __future__ import annotations
  2. from collections import defaultdict
  3. from collections.abc import Iterable
  4. from copy import copy
  5. from functools import lru_cache, partial
  6. from typing import TYPE_CHECKING, Any
  7. from pydantic_core import CoreSchema, PydanticCustomError, ValidationError, to_jsonable_python
  8. from pydantic_core import core_schema as cs
  9. from ._fields import PydanticMetadata
  10. from ._import_utils import import_cached_field_info
  11. if TYPE_CHECKING:
  12. pass
  13. STRICT = {'strict'}
  14. FAIL_FAST = {'fail_fast'}
  15. LENGTH_CONSTRAINTS = {'min_length', 'max_length'}
  16. INEQUALITY = {'le', 'ge', 'lt', 'gt'}
  17. NUMERIC_CONSTRAINTS = {'multiple_of', *INEQUALITY}
  18. ALLOW_INF_NAN = {'allow_inf_nan'}
  19. STR_CONSTRAINTS = {
  20. *LENGTH_CONSTRAINTS,
  21. *STRICT,
  22. 'strip_whitespace',
  23. 'to_lower',
  24. 'to_upper',
  25. 'pattern',
  26. 'coerce_numbers_to_str',
  27. 'ascii_only',
  28. }
  29. BYTES_CONSTRAINTS = {*LENGTH_CONSTRAINTS, *STRICT}
  30. LIST_CONSTRAINTS = {*LENGTH_CONSTRAINTS, *STRICT, *FAIL_FAST}
  31. TUPLE_CONSTRAINTS = {*LENGTH_CONSTRAINTS, *STRICT, *FAIL_FAST}
  32. SET_CONSTRAINTS = {*LENGTH_CONSTRAINTS, *STRICT, *FAIL_FAST}
  33. DICT_CONSTRAINTS = {*LENGTH_CONSTRAINTS, *STRICT}
  34. GENERATOR_CONSTRAINTS = {*LENGTH_CONSTRAINTS, *STRICT}
  35. SEQUENCE_CONSTRAINTS = {*LENGTH_CONSTRAINTS, *FAIL_FAST}
  36. FLOAT_CONSTRAINTS = {*NUMERIC_CONSTRAINTS, *ALLOW_INF_NAN, *STRICT}
  37. DECIMAL_CONSTRAINTS = {'max_digits', 'decimal_places', *FLOAT_CONSTRAINTS}
  38. INT_CONSTRAINTS = {*NUMERIC_CONSTRAINTS, *ALLOW_INF_NAN, *STRICT}
  39. BOOL_CONSTRAINTS = STRICT
  40. UUID_CONSTRAINTS = STRICT
  41. DATE_TIME_CONSTRAINTS = {*NUMERIC_CONSTRAINTS, *STRICT}
  42. TIMEDELTA_CONSTRAINTS = {*NUMERIC_CONSTRAINTS, *STRICT}
  43. TIME_CONSTRAINTS = {*NUMERIC_CONSTRAINTS, *STRICT}
  44. LAX_OR_STRICT_CONSTRAINTS = STRICT
  45. ENUM_CONSTRAINTS = STRICT
  46. COMPLEX_CONSTRAINTS = STRICT
  47. UNION_CONSTRAINTS = {'union_mode'}
  48. URL_CONSTRAINTS = {
  49. 'max_length',
  50. 'allowed_schemes',
  51. 'host_required',
  52. 'default_host',
  53. 'default_port',
  54. 'default_path',
  55. }
  56. TEXT_SCHEMA_TYPES = ('str', 'bytes', 'url', 'multi-host-url')
  57. SEQUENCE_SCHEMA_TYPES = ('list', 'tuple', 'set', 'frozenset', 'generator', *TEXT_SCHEMA_TYPES)
  58. NUMERIC_SCHEMA_TYPES = ('float', 'int', 'date', 'time', 'timedelta', 'datetime')
  59. CONSTRAINTS_TO_ALLOWED_SCHEMAS: dict[str, set[str]] = defaultdict(set)
  60. constraint_schema_pairings: list[tuple[set[str], tuple[str, ...]]] = [
  61. (STR_CONSTRAINTS, TEXT_SCHEMA_TYPES),
  62. (BYTES_CONSTRAINTS, ('bytes',)),
  63. (LIST_CONSTRAINTS, ('list',)),
  64. (TUPLE_CONSTRAINTS, ('tuple',)),
  65. (SET_CONSTRAINTS, ('set', 'frozenset')),
  66. (DICT_CONSTRAINTS, ('dict',)),
  67. (GENERATOR_CONSTRAINTS, ('generator',)),
  68. (FLOAT_CONSTRAINTS, ('float',)),
  69. (INT_CONSTRAINTS, ('int',)),
  70. (DATE_TIME_CONSTRAINTS, ('date', 'time', 'datetime', 'timedelta')),
  71. # TODO: this is a bit redundant, we could probably avoid some of these
  72. (STRICT, (*TEXT_SCHEMA_TYPES, *SEQUENCE_SCHEMA_TYPES, *NUMERIC_SCHEMA_TYPES, 'typed-dict', 'model')),
  73. (UNION_CONSTRAINTS, ('union',)),
  74. (URL_CONSTRAINTS, ('url', 'multi-host-url')),
  75. (BOOL_CONSTRAINTS, ('bool',)),
  76. (UUID_CONSTRAINTS, ('uuid',)),
  77. (LAX_OR_STRICT_CONSTRAINTS, ('lax-or-strict',)),
  78. (ENUM_CONSTRAINTS, ('enum',)),
  79. (DECIMAL_CONSTRAINTS, ('decimal',)),
  80. (COMPLEX_CONSTRAINTS, ('complex',)),
  81. ]
  82. for constraints, schemas in constraint_schema_pairings:
  83. for c in constraints:
  84. CONSTRAINTS_TO_ALLOWED_SCHEMAS[c].update(schemas)
  85. def as_jsonable_value(v: Any) -> Any:
  86. if type(v) not in (int, str, float, bytes, bool, type(None)):
  87. return to_jsonable_python(v)
  88. return v
  89. def expand_grouped_metadata(annotations: Iterable[Any]) -> Iterable[Any]:
  90. """Expand the annotations.
  91. Args:
  92. annotations: An iterable of annotations.
  93. Returns:
  94. An iterable of expanded annotations.
  95. Example:
  96. ```python
  97. from annotated_types import Ge, Len
  98. from pydantic._internal._known_annotated_metadata import expand_grouped_metadata
  99. print(list(expand_grouped_metadata([Ge(4), Len(5)])))
  100. #> [Ge(ge=4), MinLen(min_length=5)]
  101. ```
  102. """
  103. import annotated_types as at
  104. FieldInfo = import_cached_field_info()
  105. for annotation in annotations:
  106. if isinstance(annotation, at.GroupedMetadata):
  107. yield from annotation
  108. elif isinstance(annotation, FieldInfo):
  109. yield from annotation.metadata
  110. # this is a bit problematic in that it results in duplicate metadata
  111. # all of our "consumers" can handle it, but it is not ideal
  112. # we probably should split up FieldInfo into:
  113. # - annotated types metadata
  114. # - individual metadata known only to Pydantic
  115. annotation = copy(annotation)
  116. annotation.metadata = []
  117. yield annotation
  118. else:
  119. yield annotation
  120. @lru_cache
  121. def _get_at_to_constraint_map() -> dict[type, str]:
  122. """Return a mapping of annotated types to constraints.
  123. Normally, we would define a mapping like this in the module scope, but we can't do that
  124. because we don't permit module level imports of `annotated_types`, in an attempt to speed up
  125. the import time of `pydantic`. We still only want to have this dictionary defined in one place,
  126. so we use this function to cache the result.
  127. """
  128. import annotated_types as at
  129. return {
  130. at.Gt: 'gt',
  131. at.Ge: 'ge',
  132. at.Lt: 'lt',
  133. at.Le: 'le',
  134. at.MultipleOf: 'multiple_of',
  135. at.MinLen: 'min_length',
  136. at.MaxLen: 'max_length',
  137. }
  138. def apply_known_metadata(annotation: Any, schema: CoreSchema) -> CoreSchema | None: # noqa: C901
  139. """Apply `annotation` to `schema` if it is an annotation we know about (Gt, Le, etc.).
  140. Otherwise return `None`.
  141. This does not handle all known annotations. If / when it does, it can always
  142. return a CoreSchema and return the unmodified schema if the annotation should be ignored.
  143. Assumes that GroupedMetadata has already been expanded via `expand_grouped_metadata`.
  144. Args:
  145. annotation: The annotation.
  146. schema: The schema.
  147. Returns:
  148. An updated schema with annotation if it is an annotation we know about, `None` otherwise.
  149. Raises:
  150. RuntimeError: If a constraint can't be applied to a specific schema type.
  151. ValueError: If an unknown constraint is encountered.
  152. """
  153. import annotated_types as at
  154. from ._validators import NUMERIC_VALIDATOR_LOOKUP, forbid_inf_nan_check
  155. schema = schema.copy()
  156. schema_update, other_metadata = collect_known_metadata([annotation])
  157. schema_type = schema['type']
  158. chain_schema_constraints: set[str] = {
  159. 'pattern',
  160. 'strip_whitespace',
  161. 'to_lower',
  162. 'to_upper',
  163. 'coerce_numbers_to_str',
  164. 'ascii_only',
  165. }
  166. chain_schema_steps: list[CoreSchema] = []
  167. for constraint, value in schema_update.items():
  168. if constraint not in CONSTRAINTS_TO_ALLOWED_SCHEMAS:
  169. raise ValueError(f'Unknown constraint {constraint}')
  170. allowed_schemas = CONSTRAINTS_TO_ALLOWED_SCHEMAS[constraint]
  171. # if it becomes necessary to handle more than one constraint
  172. # in this recursive case with function-after or function-wrap, we should refactor
  173. # this is a bit challenging because we sometimes want to apply constraints to the inner schema,
  174. # whereas other times we want to wrap the existing schema with a new one that enforces a new constraint.
  175. if schema_type in {'function-before', 'function-wrap', 'function-after'} and constraint == 'strict':
  176. schema['schema'] = apply_known_metadata(annotation, schema['schema']) # type: ignore # schema is function schema
  177. return schema
  178. # if we're allowed to apply constraint directly to the schema, like le to int, do that
  179. if schema_type in allowed_schemas:
  180. if constraint == 'union_mode' and schema_type == 'union':
  181. schema['mode'] = value # type: ignore # schema is UnionSchema
  182. else:
  183. schema[constraint] = value
  184. continue
  185. # else, apply a function after validator to the schema to enforce the corresponding constraint
  186. if constraint in chain_schema_constraints:
  187. def _apply_constraint_with_incompatibility_info(
  188. value: Any, handler: cs.ValidatorFunctionWrapHandler
  189. ) -> Any:
  190. try:
  191. x = handler(value)
  192. except ValidationError as ve:
  193. # if the error is about the type, it's likely that the constraint is incompatible the type of the field
  194. # for example, the following invalid schema wouldn't be caught during schema build, but rather at this point
  195. # with a cryptic 'string_type' error coming from the string validator,
  196. # that we'd rather express as a constraint incompatibility error (TypeError)
  197. # Annotated[list[int], Field(pattern='abc')]
  198. if 'type' in ve.errors()[0]['type']:
  199. raise TypeError(
  200. f"Unable to apply constraint '{constraint}' to supplied value {value} for schema of type '{schema_type}'" # noqa: B023
  201. )
  202. raise ve
  203. return x
  204. chain_schema_steps.append(
  205. cs.no_info_wrap_validator_function(
  206. _apply_constraint_with_incompatibility_info, cs.str_schema(**{constraint: value})
  207. )
  208. )
  209. elif constraint in NUMERIC_VALIDATOR_LOOKUP:
  210. if constraint in LENGTH_CONSTRAINTS:
  211. inner_schema = schema
  212. while inner_schema['type'] in {'function-before', 'function-wrap', 'function-after'}:
  213. inner_schema = inner_schema['schema'] # type: ignore
  214. inner_schema_type = inner_schema['type']
  215. if inner_schema_type == 'list' or (
  216. inner_schema_type == 'json-or-python' and inner_schema['json_schema']['type'] == 'list' # type: ignore
  217. ):
  218. js_constraint_key = 'minItems' if constraint == 'min_length' else 'maxItems'
  219. else:
  220. js_constraint_key = 'minLength' if constraint == 'min_length' else 'maxLength'
  221. else:
  222. js_constraint_key = constraint
  223. schema = cs.no_info_after_validator_function(
  224. partial(NUMERIC_VALIDATOR_LOOKUP[constraint], **{constraint: value}), schema
  225. )
  226. metadata = schema.get('metadata', {})
  227. if (existing_json_schema_updates := metadata.get('pydantic_js_updates')) is not None:
  228. metadata['pydantic_js_updates'] = {
  229. **existing_json_schema_updates,
  230. **{js_constraint_key: as_jsonable_value(value)},
  231. }
  232. else:
  233. metadata['pydantic_js_updates'] = {js_constraint_key: as_jsonable_value(value)}
  234. schema['metadata'] = metadata
  235. elif constraint == 'allow_inf_nan' and value is False:
  236. schema = cs.no_info_after_validator_function(
  237. forbid_inf_nan_check,
  238. schema,
  239. )
  240. else:
  241. # It's rare that we'd get here, but it's possible if we add a new constraint and forget to handle it
  242. # Most constraint errors are caught at runtime during attempted application
  243. raise RuntimeError(f"Unable to apply constraint '{constraint}' to schema of type '{schema_type}'")
  244. for annotation in other_metadata:
  245. if (annotation_type := type(annotation)) in (at_to_constraint_map := _get_at_to_constraint_map()):
  246. constraint = at_to_constraint_map[annotation_type]
  247. validator = NUMERIC_VALIDATOR_LOOKUP.get(constraint)
  248. if validator is None:
  249. raise ValueError(f'Unknown constraint {constraint}')
  250. schema = cs.no_info_after_validator_function(
  251. partial(validator, {constraint: getattr(annotation, constraint)}), schema
  252. )
  253. continue
  254. elif isinstance(annotation, (at.Predicate, at.Not)):
  255. predicate_name = f'{annotation.func.__qualname__!r} ' if hasattr(annotation.func, '__qualname__') else ''
  256. # Note: B023 is ignored because even though we iterate over `other_metadata`, it is guaranteed
  257. # to be of length 1. `apply_known_metadata()` is called from `GenerateSchema`, where annotations
  258. # were already expanded via `expand_grouped_metadata()`. Confusing, but this falls into the annotations
  259. # refactor.
  260. if isinstance(annotation, at.Predicate):
  261. def val_func(v: Any) -> Any:
  262. predicate_satisfied = annotation.func(v) # noqa: B023
  263. if not predicate_satisfied:
  264. raise PydanticCustomError(
  265. 'predicate_failed',
  266. f'Predicate {predicate_name}failed', # pyright: ignore[reportArgumentType] # noqa: B023
  267. )
  268. return v
  269. else:
  270. def val_func(v: Any) -> Any:
  271. predicate_satisfied = annotation.func(v) # noqa: B023
  272. if predicate_satisfied:
  273. raise PydanticCustomError(
  274. 'not_operation_failed',
  275. f'Not of {predicate_name}failed', # pyright: ignore[reportArgumentType] # noqa: B023
  276. )
  277. return v
  278. schema = cs.no_info_after_validator_function(val_func, schema)
  279. else:
  280. # ignore any other unknown metadata
  281. return None
  282. if chain_schema_steps:
  283. chain_schema_steps = [schema] + chain_schema_steps
  284. return cs.chain_schema(chain_schema_steps)
  285. return schema
  286. def collect_known_metadata(annotations: Iterable[Any]) -> tuple[dict[str, Any], list[Any]]:
  287. """Split `annotations` into known metadata and unknown annotations.
  288. Args:
  289. annotations: An iterable of annotations.
  290. Returns:
  291. A tuple contains a dict of known metadata and a list of unknown annotations.
  292. Example:
  293. ```python
  294. from annotated_types import Gt, Len
  295. from pydantic._internal._known_annotated_metadata import collect_known_metadata
  296. print(collect_known_metadata([Gt(1), Len(42), ...]))
  297. #> ({'gt': 1, 'min_length': 42}, [Ellipsis])
  298. ```
  299. """
  300. annotations = expand_grouped_metadata(annotations)
  301. res: dict[str, Any] = {}
  302. remaining: list[Any] = []
  303. for annotation in annotations:
  304. # isinstance(annotation, PydanticMetadata) also covers ._fields:_PydanticGeneralMetadata
  305. if isinstance(annotation, PydanticMetadata):
  306. res.update(annotation.__dict__)
  307. # we don't use dataclasses.asdict because that recursively calls asdict on the field values
  308. elif (annotation_type := type(annotation)) in (at_to_constraint_map := _get_at_to_constraint_map()):
  309. constraint = at_to_constraint_map[annotation_type]
  310. res[constraint] = getattr(annotation, constraint)
  311. elif isinstance(annotation, type) and issubclass(annotation, PydanticMetadata):
  312. # also support PydanticMetadata classes being used without initialisation,
  313. # e.g. `Annotated[int, Strict]` as well as `Annotated[int, Strict()]`
  314. res.update({k: v for k, v in vars(annotation).items() if not k.startswith('_')})
  315. else:
  316. remaining.append(annotation)
  317. # Nones can sneak in but pydantic-core will reject them
  318. # it'd be nice to clean things up so we don't put in None (we probably don't _need_ to, it was just easier)
  319. # but this is simple enough to kick that can down the road
  320. res = {k: v for k, v in res.items() if v is not None}
  321. return res, remaining
  322. def check_metadata(metadata: dict[str, Any], allowed: Iterable[str], source_type: Any) -> None:
  323. """A small utility function to validate that the given metadata can be applied to the target.
  324. More than saving lines of code, this gives us a consistent error message for all of our internal implementations.
  325. Args:
  326. metadata: A dict of metadata.
  327. allowed: An iterable of allowed metadata.
  328. source_type: The source type.
  329. Raises:
  330. TypeError: If there is metadatas that can't be applied on source type.
  331. """
  332. unknown = metadata.keys() - set(allowed)
  333. if unknown:
  334. raise TypeError(
  335. f'The following constraints cannot be applied to {source_type!r}: {", ".join([f"{k!r}" for k in unknown])}'
  336. )