Mercurial > hgrepos > Python > libs > data-schema

comparison data_schema/__init__.py @ 5:84dfd1a94926

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add the existing implementation. All tests work. The documentation as text file is included also.
author Franz Glasner <fzglas.hg@dom66.de>
date Thu, 06 Jul 2023 23:41:41 +0200
parents
children f4e1b6d6fe63
comparison
equal deleted inserted replaced
4:d715f0c13c60 5:84dfd1a94926
1 # -*- coding: utf-8 -*-
2 r"""
3 Object schema validation support.
4
5 Somewhat modelled after JSON schema.
6
7 .. seealso:: https://json-schema.org/understanding-json-schema/index.html
8
9 :Author: Franz Glasner <fzglas.hg@dom66.de>
10 :Copyright: \(c) 2023 Franz Glasner
11 :License: BSD 3-Clause "New" or "Revised" License.
12 See :ref:`LICENSE.txt <license>` for details.
13 :ID: @(#) $Header$
14
15 """
16
17 __version__ = "0.1.dev1"
18
19 __revision__ = "|VCSRevision|"
20
21 __date__ = "|VCSJustDate|"
22
23 __all__ = ["ERROR", "WARNING", "INFO", "ERRORS", "WARNINGS",
24 "level_name", "problem_message",
25 "ValidationProblem", "SchemaError",
26 "validate",
27 "log_problem_cause"]
28
29
30 import ast
31 import collections
32 import copy
33 import datetime
34 import re
35 import urllib.parse
36
37 import rfc3986
38
39 import configmix.yaml
40
41 from .util import get_data_stream
42
43
44 def NC_(ctx, msg):
45 """Mimimum dummy translation support"""
46 return msg
47
48
49 ERROR = 40
50 WARNING = 30
51 INFO = 20
52
53 _level_to_name = {
54 ERROR: "ERROR",
55 WARNING: "WARNING",
56 INFO: "INFO",
57 }
58 _name_to_level = {name: level for (level, name) in _level_to_name.items()}
59
60 ERRORS = {
61 10000: NC_("schema-msg", "dict expected"),
62 10001: NC_("schema-msg", "list expected"),
63 10002: NC_("schema-msg", "string expected"),
64 10003: NC_("schema-msg", "dict key must be a string"),
65 10004: NC_("schema-msg", "additional key encountered"),
66 10005: NC_("schema-msg", "required key(s) missing"),
67 10006: NC_("schema-msg", "min string length encountered"),
68 10007: NC_("schema-msg", "max string length exceeded"),
69 10008: NC_("schema-msg", "string value does not match the required RE pattern"),
70 10009: NC_("schema-msg", "string value does not validate"),
71 10010: NC_("schema-msg", "validation error"),
72 10011: NC_("schema-msg", "None/Null object expected"),
73 10012: NC_("schema-msg", "min list length encountered"),
74 10013: NC_("schema-msg", "max list length exceeded"),
75 10014: NC_("schema-msg", "tuple expected"),
76 10015: NC_("schema-msg", "min tuple length encountered"),
77 10016: NC_("schema-msg", "max tuple length exceeded"),
78 10017: NC_("schema-msg", "additional items in tuple not allowed"),
79 10018: NC_("schema-msg", "object is not empty"),
80 10019: NC_("schema-msg", "more than one match in `one-of' detected"),
81 10020: NC_("schema-msg", "int expected"),
82 10021: NC_("schema-msg", "int value lower than minValue"),
83 10022: NC_("schema-msg", "int value greater than maxValue"),
84 10023: NC_("schema-msg", "float expected"),
85 10024: NC_("schema-msg", "float value lower than minValue"),
86 10025: NC_("schema-msg", "float value greater than maxValue"),
87 10026: NC_("schema-msg", "boolean value expected"),
88 10027: NC_("schema-msg", "boolean true expected"),
89 10028: NC_("schema-msg", "boolean false expected"),
90 10029: NC_("schema-msg", "`not' expected problems but got none"),
91 10030: NC_("schema-msg", "numeric type (int or float) expected"),
92 10031: NC_("schema-msg", "numeric value lower than minValue"),
93 10032: NC_("schema-msg", "numeric value greater than maxValue"),
94 10033: NC_("schema-msg", "a plain scalar value expected"),
95 10034: NC_("schema-msg", "dict key does not match required schema"),
96 10035: NC_("schema-msg", "binary data expected"),
97 10036: NC_("schema-msg", "length of binary data lower than minValue"),
98 10037: NC_("schema-msg", "length of binary data exceeds maxValue"),
99 10038: NC_("schema-msg", "a set is expected"),
100 10039: NC_("schema-msg", "length of set lower than minLength"),
101 10040: NC_("schema-msg", "length of set greater than maxLength"),
102 10041: NC_("schema-msg", "timestamp expected"),
103 10042: NC_("schema-msg", "value of timestamp does not validate"),
104 10043: NC_("schema-msg", "enumerated string value expected but not found"),
105 10044: NC_("schema-msg", "referenced object doest not exist"),
106 10045: NC_("schema-msg", "key is not contained in referenced object"),
107 10046: NC_("schema-msg", "referenced object is not a container"),
108 10047: NC_("schema-msg", "binary data does not match the required RE pattern"),
109 10048: NC_("schema-msg", "enumerated integer value expected but not found"),
110 10049: NC_("schema-msg", "enumerated number value expected but not found"),
111 10050: NC_("schema-msg", "min dict length encountered"),
112 10051: NC_("schema-msg", "max dict length exceeded"),
113 10052: NC_("schema-msg", "index constraint violated"),
114 10053: NC_("schema-msg", "`one-of' failed"),
115 10054: NC_("schema-msg", "failing `one-of' item"),
116 10055: NC_("schema-msg", "`any-of' failed"),
117 10056: NC_("schema-msg", "failing `any-of' item"),
118 10057: NC_("schema-msg", "`all-of' failed"),
119 10058: NC_("schema-msg", "failing `all-of' item"),
120 }
121
122 WARNINGS = {
123 80000: NC_("schema-msg", "duplicate dict key"),
124 }
125
126 if not set(ERRORS.keys()).isdisjoint(set(WARNINGS.keys())):
127 raise ValueError("ERRORS and WARNINGS must be disjoint")
128
129
130 TYPE_RE = type(re.compile(r"\A.+\Z"))
131
132 _SENTINEL = object()
133
134 SCHEMA_REF_KEY = "$ref"
135 """Key name for schema references (like a symlink within a schema)"""
136
137 SCHEMA_PATH_ROOT = "$root"
138 """URI path to the root schema"""
139
140 SCHEMA_PATH_SELF = "$self"
141 """URI path to the current schema"""
142
143
144 def level_name(level):
145 name = _level_to_name.get(level)
146 if name is None:
147 name = "Level {}".format(level)
148 return name
149
150
151 def problem_message(pr):
152 if isinstance(pr, ValidationProblem):
153 code = getattr(pr, "code", None)
154 else:
155 code = pr
156 msg = ERRORS.get(code, None)
157 if msg is None:
158 msg = WARNINGS[code]
159 return msg
160
161
162 class ValidationProblem(object):
163
164 __slots__ = ("code", "severity", "hint", "context", "cause", "index")
165
166 def __init__(self,
167 code=None,
168 severity=None,
169 hint=None,
170 context=None,
171 cause=None,
172 index=None):
173 if code is not None:
174 # check validity
175 if code not in ERRORS and code not in WARNINGS:
176 raise ValueError(
177 "unknown validation error code: {}".format(code))
178 self.code = code
179 if severity is None:
180 # autodetermine
181 if code in ERRORS:
182 self.severity = ERROR
183 elif code in WARNINGS:
184 self.severity = WARNING
185 else:
186 assert False
187 else:
188 self.severity = severity
189 else:
190 raise TypeError("`code' must be given")
191 self.hint = hint
192 self.context = context
193 if cause:
194 if not isinstance(cause, (list, tuple, set, frozenset)):
195 cause = (cause, )
196 for c in cause:
197 if not isinstance(c, ValidationProblem):
198 raise SchemaError(
199 "can only nest other `ValidationProblem' instances")
200 self.cause = cause
201 self.index = index
202
203 def __repr__(self):
204 try:
205 msg = " (" + problem_message(self) + ")"
206 except LookupError:
207 msg = ""
208 if self.index is None:
209 return "ValidationProblem(code={!r}{}, severity={!r}, hint={}, context=[depth={}]{})".format(
210 self.code, msg, self.severity, self.hint, self.context.depth, self.context)
211 else:
212 return "ValidationProblem(code={!r}{}, severity={!r}, hint={}, context=[depth={}]{}, index={})".format(
213 self.code, msg, self.severity, self.hint, self.context.depth, self.context, self.index)
214
215
216 class SchemaError(Exception):
217 """An error within the schema itself"""
218 pass
219
220
221 ValidationSettings = collections.namedtuple(
222 "ValidationSettings",
223 ["skip_keys", "break_on_keynames_problems"])
224
225
226 class _Schema(dict):
227
228 __slots__ = ("parent", "is_sub_root", "_schema_cache")
229
230 def __init__(self, parent, is_sub_root, *args, **kwds):
231 super().__init__(*args, **kwds)
232 if parent is None or isinstance(parent, _Schema):
233 self.parent = parent
234 else:
235 raise TypeError("`_Schema' or `None' expected")
236 if parent is None:
237 self._schema_cache = {}
238 if not is_sub_root:
239 raise ValueError(
240 "the root schmema must be a sub-root (aka `$self') also")
241 self.is_sub_root = True
242 else:
243 self.is_sub_root = is_sub_root
244
245 @property
246 def ROOT(self):
247 """Get the root schema"""
248 r = self
249 while r.parent is not None:
250 r = r.parent
251 return r
252
253 @property
254 def SELF(self):
255 r = self
256 while not r.is_sub_root:
257 r = r.parent
258 return r
259
260 def copy(self):
261 return _Schema(self.parent, self.is_sub_root, self)
262
263 def get_child(self, name, default=None):
264 return self.ensure_child_schema(self.get(name, default))
265
266 def ensure_child_schema(self, v):
267 if v is None:
268 return None
269 elif isinstance(v, _Schema):
270 return v
271 elif isinstance(v, dict):
272 return _Schema(self, False, v)
273 else:
274 return v
275
276 def ensure_list_of_child_schemata(self, v):
277 if isinstance(v, (list, tuple)):
278 return [_Schema(self, False, i) for i in v]
279 else:
280 return v
281
282 def __eq__(self, other):
283 if not isinstance(other, _Schema):
284 return NotImplemented
285 return (self.parent is other.parent
286 and bool(self.is_sub_root) == bool(other.is_sub_root)
287 and dict(self) == dict(other))
288
289 def __ne__(self, other):
290 #
291 # While the default in Python3 is sensible implementing is recommended
292 # when a built-in __eq__ is overwritten (Raymond Hettinger).
293 #
294 # Do not use not self == other because NotImplemented is not handled
295 # properly in some early Python versions (including Py2).
296 #
297 equal = self.__eq__(other)
298 return NotImplemented if equal is NotImplemented else not equal
299
300 def __copy__(self):
301 return _Schema(self.parent, self.is_sub_root, self)
302
303 def __deepcopy__(self, memo):
304 return _Schema(self.parent,
305 self.is_sub_root,
306 copy.deepcopy(dict(self), memo))
307
308 def __str__(self):
309 return "<_Schema " + super().__str__() + ">"
310
311 def __repr__(self):
312 return "<_Schema " + super().__repr__() + ">"
313
314 def get_cached_schema(self, key, load_if_needed=True):
315 root = self.ROOT
316 s = root._schema_cache.get(key, None)
317 if s is None and load_if_needed:
318 with get_data_stream(key) as schemastream:
319 # load schema a new `$self' (i.e. sub-root is True)
320 s = _Schema(self, True, configmix.yaml.load(schemastream))
321 root._schema_cache[key] = s
322 return s
323
324 def add_cached_schema(self, key, schema):
325 r = self.ROOT
326 assert isinstance(schema, _Schema)
327 r._schema_cache[key] = schema
328
329
330 class Context(object):
331
332 __slots__ = ("_parent", "_key", "_key_index",
333 "_index",
334 "root_object", "root_schema",
335 "_current_object",
336 "_settings")
337
338 def __init__(self, parent, *, key=_SENTINEL, index=_SENTINEL,
339 root_object=_SENTINEL, root_schema=_SENTINEL,
340 current_object=_SENTINEL,
341 settings=_SENTINEL,
342 key_index=_SENTINEL):
343 if parent is None:
344 if key is not _SENTINEL:
345 raise TypeError("the root context may not have a key")
346 if index is not _SENTINEL:
347 raise TypeError("the root context may not have an index")
348 if settings is _SENTINEL:
349 raise TypeError("the root context must have settings")
350 self.root_object = root_object
351 if current_object is _SENTINEL:
352 current_object = root_object
353 self.root_schema = root_schema
354 else:
355 if key is _SENTINEL and index is _SENTINEL:
356 raise TypeError("one of `key` and `index` must be given in a non-root context")
357 if root_object is not _SENTINEL:
358 raise TypeError("non-root context may not have a root object")
359 if root_schema is not _SENTINEL:
360 raise TypeError("non-root context may not have a root schema")
361 if key is not _SENTINEL and index is not _SENTINEL:
362 raise ValueError("only one of `key` and `index` may be given in a context")
363 if key_index is not _SENTINEL and key is _SENTINEL:
364 raise ValueError("when having a `key_index` a `key` also must be given")
365 self._parent = parent
366 self._key = key
367 self._key_index = key_index
368 self._index = index
369 self._current_object = current_object
370 self._settings = settings
371
372 @property
373 def parent(self):
374 return self._parent
375
376 @property
377 def safe_parent(self):
378 if self.is_root:
379 raise TypeError("the root context has no parent")
380 return self.parent
381
382 @property
383 def root(self):
384 """Get the root context"""
385 ctx = self
386 while not ctx.is_root:
387 ctx = ctx.parent
388 return ctx
389
390 @property
391 def is_root(self):
392 return not bool(self.parent)
393
394 @property
395 def key(self):
396 if self._key is _SENTINEL:
397 raise AttributeError("no `key' in Context")
398 return self._key
399
400 @property
401 def index(self):
402 if self._index is _SENTINEL:
403 raise AttributeError("no `index' in Context")
404 return self._index
405
406 @property
407 def key_index(self):
408 if self._key_index is _SENTINEL:
409 raise AttributeError("no `key_index' in Context")
410 return self._key_index
411
412 @property
413 def current_object(self):
414 if self._current_object is _SENTINEL:
415 raise AttributeError("no `current_object' in Context")
416 return self._current_object
417
418 @property
419 def settings(self):
420 s = self._settings
421 return s if s is not _SENTINEL else self.parent.settings
422
423 @property
424 def depth(self):
425 if self._key is _SENTINEL and self._index is _SENTINEL and self.is_root:
426 return 0
427 n = 0
428 ctx = self
429 while not ctx.is_root:
430 n += 1
431 ctx = ctx.parent
432 return n
433
434 def __str__(self):
435 if self._key is _SENTINEL and self._index is _SENTINEL and self.is_root:
436 return "<ROOT>"
437 chain = []
438 ctx = self
439 while not ctx.is_root:
440 if ctx._key is not _SENTINEL:
441 chain.append(str(ctx.key))
442 elif ctx._index is not _SENTINEL:
443 chain.append("INDEX:{}".format(ctx.index))
444 else:
445 chain.append("")
446 ctx = ctx.parent
447 chain.reverse()
448 return " / ".join(chain)
449
450 def __repr__(self):
451 return "<Context path=`{}'>".format(str(self))
452
453
454 def _get_one_of(d, *keys, default=None, strict=True):
455 """Get the first found key and its value of `keys` from dict `d`.
456
457 """
458 for k in keys:
459 v = d.get(k, _SENTINEL)
460 if v is not _SENTINEL:
461 if strict:
462 #
463 # check that all no other key of `keys` besides of `k` is
464 # in `d`
465 #
466 other_keys = set(keys)
467 other_keys.remove(k)
468 for k2 in other_keys:
469 if k2 in d:
470 raise SchemaError("ambiguous key from: {}".format(
471 ", ".join(keys)))
472 return k, v
473 return None, default
474
475
476 def validate(obj, schema, **kwds):
477 """Validate object `obj` against the *specific* schema `schema`.
478
479 Yields errors and warnings
480
481 """
482 settings = {
483 "skip_keys": None,
484 "break_on_keynames_problems": True,
485 }
486 settings.update(kwds)
487 if not isinstance(schema, _Schema):
488 if not isinstance(schema, dict):
489 raise SchemaError("Schema must be a dict-alike."
490 " Got: {!r}".format(schema))
491 schema = _Schema(None, True, schema)
492 context = Context(None, root_object=obj, root_schema=schema,
493 settings=ValidationSettings(**settings))
494 yield from _validate(obj, schema, context, is_root=True)
495
496
497 def _validate(obj, schema, context, is_root=False):
498 """Validate object `obj` against the *specific* schema `schema`.
499
500 Yields errors and warnings
501
502 """
503 if not isinstance(schema, _Schema):
504 raise SchemaError("Schema must be a `_Schema'."
505 " Got: {!r}. Context: {!s}".format(schema, context))
506 # 1. Process "cond" or "match"
507 schema = process_schema_conditionals(schema, context)
508 # 2. Process "$ref" schema references
509 schema = process_schema_references(
510 schema, context, check_single_ref_key=not is_root)
511
512 # 3. Real validation
513
514 # check combinator shortcuts without "type" indirection
515 combinator, combinator_schema = _get_one_of(
516 schema, "not", "all-of", "any-of", "one-of")
517 if combinator is None:
518 try:
519 t = schema["type"]
520 except KeyError:
521 raise SchemaError("Schema has no `type' key: {!r}."
522 " Context: {!s}".format(schema, context))
523 else:
524 #
525 # Construct a temporary schema with the proper indirection for
526 # the check below
527 #
528 t = {"type": {combinator: combinator_schema}}
529 if combinator_schema is None:
530 raise SchemaError("a combinator requires a child")
531 if callable(t):
532 yield from t(obj, schema, context)
533 elif t is None:
534 yield from validate_null(obj, schema, context)
535 elif isinstance(t, dict):
536 if len(t) != 1:
537 raise SchemaError("type dict must be of length 1")
538 # Check whether a shortcut is already seen above
539 if combinator is None:
540 combinator = list(t.keys())[0]
541 combinator_schema = t[combinator]
542 if combinator == "not":
543 yield from validate_not(
544 obj, schema.ensure_child_schema(combinator_schema), context)
545 elif combinator == "all-of":
546 yield from validate_allOf(
547 obj,
548 schema.ensure_list_of_child_schemata(combinator_schema),
549 context)
550 elif combinator == "any-of":
551 yield from validate_anyOf(
552 obj,
553 schema.ensure_list_of_child_schemata(combinator_schema),
554 context)
555 elif combinator == "one-of":
556 yield from validate_oneOf(
557 obj,
558 schema.ensure_list_of_child_schemata(combinator_schema),
559 context)
560 else:
561 raise SchemaError("unknown combinator: {}".format(combinator))
562 elif isinstance(t, (list, tuple)):
563 # a simple list is "any-of"
564 yield from validate_anyOf(
565 obj, schema.ensure_list_of_child_schemata(t), context)
566 elif t in ("dict", "map", "object"):
567 yield from validate_dict(obj, schema, context)
568 elif t in ("list", "array",):
569 yield from validate_list(obj, schema, context)
570 elif t in ("tuple", "record"):
571 yield from validate_tuple(obj, schema, context)
572 elif t in ("set", "frozenset"):
573 yield from validate_set(obj, schema, context)
574 elif t in ("string", "str"):
575 yield from validate_str(obj, schema, context)
576 elif t in ("deny", ):
577 yield from validate_deny(obj, schema, context)
578 elif t in ("accept", ):
579 yield from validate_accept(obj, schema, context)
580 elif t in ("none", "null", "nil"):
581 yield from validate_null(obj, schema, context)
582 elif t in ("empty", ):
583 yield from validate_empty(obj, schema, context)
584 elif t in ("integer", "int"):
585 yield from validate_integer(obj, schema, context)
586 elif t in ("float", "real", "double"):
587 yield from validate_float(obj, schema, context)
588 elif t in ("number", "num"):
589 yield from validate_number(obj, schema, context)
590 elif t in ("bool", "boolean"):
591 yield from validate_bool(obj, schema, context)
592 elif t in ("scalar", ):
593 yield from validate_scalar(obj, schema, context)
594 elif t in ("binary", ):
595 yield from validate_binary(obj, schema, context)
596 elif t in ("timestamp", "datetime"):
597 yield from validate_timestamp(obj, schema, context)
598 else:
599 raise SchemaError("unknown type in schema: {}".format(t))
600
601
602 def _is_in_skip_keys(key, skip_keys):
603 if not skip_keys:
604 return False
605 for sk in skip_keys:
606 if isinstance(sk, str):
607 if key == sk:
608 return True
609 else:
610 if sk.search(key):
611 return True
612 return False
613
614
615 def _is_null_allowed_for_object(obj, schema, context):
616 if obj is None and schema.get("nullable", False):
617 return True
618 return False
619
620
621 def _validate_index_constraint(obj, schema, context):
622 # No evaluation of index constraints for the root context
623 if context.is_root:
624 return
625 try:
626 index_constraints = schema["index-constraint"]
627 except KeyError:
628 return # no constraints
629 else:
630 if not isinstance(index_constraints, (list, tuple, set, frozenset)):
631 index_constraints = [index_constraints]
632 if not index_constraints:
633 return
634 parent = context.safe_parent
635 try:
636 effective_index = context.index
637 except AttributeError:
638 try:
639 effective_index = context.key_index
640 except AttributeError:
641 raise SchemaError("parent container has no usable index")
642 for idx in index_constraints:
643 if idx < 0:
644 idx = len(parent.current_object) + idx
645 if idx == effective_index:
646 break
647 else:
648 yield ValidationProblem(code=10052, context=context)
649
650
651 def validate_dict(obj, schema, context):
652 if _is_null_allowed_for_object(obj, schema, context):
653 return
654 if not isinstance(obj, dict):
655 yield ValidationProblem(code=10000, hint="got: {}".format(type(obj).__name__), context=context)
656 return
657 yield from _validate_index_constraint(obj, schema, context)
658 minlen = schema.get("minLength", None)
659 if minlen:
660 if len(obj) < minlen:
661 yield ValidationProblem(code=10050, hint=obj, context=context)
662 maxlen = schema.get("maxLength", None)
663 if maxlen is not None:
664 if len(obj) > maxlen:
665 yield ValidationProblem(code=10051, hint=obj, context=context)
666 schema_keys = schema.get("keys", {}) if schema else {}
667 seen_keys = set()
668 schema_keynames = schema.get_child("keyNames", None)
669 idx = -1
670 for key, item in obj.items():
671 idx += 1
672 if schema_keynames is None:
673 if not isinstance(key, str):
674 yield ValidationProblem(code=10003, hint=repr(key), context=context)
675 else:
676 # validate the key against given schema
677 new_context = Context(context, key=key, key_index=idx, current_object=key)
678 key_probs = list(_validate(key, schema_keynames, new_context))
679 if key_probs:
680 yield ValidationProblem(
681 code=10034, hint=key, context=context, cause=key_probs)
682 if context.settings.break_on_keynames_problems:
683 return
684 if key in seen_keys:
685 yield ValidationProblem(code=80000, hint=key, context=context)
686 else:
687 seen_keys.add(key)
688 # XXX FIXME: context: new leaf context with new key for recursion
689 if key in schema_keys:
690 new_context = Context(context, key=key, key_index=idx, current_object=item)
691 yield from _validate(item, schema.ensure_child_schema(schema_keys[key]), new_context)
692 else:
693 # check whether additional keys are allowed
694 additional_keys = schema.get_child("additionalKeys", False)
695 if isinstance(additional_keys, bool):
696 if not additional_keys:
697 if not _is_in_skip_keys(key, context.settings.skip_keys):
698 yield ValidationProblem(code=10004, hint=str(key), context=context)
699 else:
700 if not _is_in_skip_keys(key, context.settings.skip_keys):
701 # try this as the common schema for all the additional keys
702 new_context = Context(context, key=key, key_index=idx, current_object=item)
703 yield from _validate(item, additional_keys, new_context)
704 # check whether all required keys are seen
705 try:
706 required_keys = set(schema.get("required", set()))
707 except (TypeError, ValueError):
708 raise SchemaError("`required` must be an iterable")
709 if not required_keys <= seen_keys:
710 hs = [str(i) for i in required_keys - seen_keys]
711 yield ValidationProblem(code=10005, hint=sorted(hs), context=context)
712
713
714 def validate_list(obj, schema, context):
715 if _is_null_allowed_for_object(obj, schema, context):
716 return
717 if not isinstance(obj, (list, tuple)):
718 yield ValidationProblem(code=10001, hint="got: {}".format(type(obj).__name__), context=context)
719 return
720 yield from _validate_index_constraint(obj, schema, context)
721 minlen = schema.get("minLength", None)
722 if minlen:
723 if len(obj) < minlen:
724 yield ValidationProblem(code=10012, hint=obj, context=context)
725 maxlen = schema.get("maxLength", None)
726 if maxlen is not None:
727 if len(obj) > maxlen:
728 yield ValidationProblem(code=10013, hint=obj, context=context)
729 try:
730 schema_items = schema.ensure_child_schema(schema["items"])
731 except KeyError:
732 schema_items = _Schema(schema, False, {"type": validate_deny})
733 for idx, o in enumerate(obj):
734 new_context = Context(parent=context, index=idx, current_object=o)
735 yield from _validate(o, schema_items, new_context)
736
737
738 def validate_set(obj, schema, context):
739 if _is_null_allowed_for_object(obj, schema, context):
740 return
741 if not isinstance(obj, (set, frozenset)):
742 yield ValidationProblem(code=10038, hint="got: {}".format(type(obj).__name__), context=context)
743 return
744 yield from _validate_index_constraint(obj, schema, context)
745 minlen = schema.get("minLength", None)
746 if minlen:
747 if len(obj) < minlen:
748 yield ValidationProblem(code=10039, hint=obj, context=context)
749 maxlen = schema.get("maxLength", None)
750 if maxlen is not None:
751 if len(obj) > maxlen:
752 yield ValidationProblem(code=10040, hint=obj, context=context)
753 try:
754 schema_items = schema.ensure_child_schema(schema["items"])
755 except KeyError:
756 schema_items = _Schema(schema, False, {"type": validate_deny})
757 for o in obj:
758 new_context = Context(parent=context, key=o, current_object=o)
759 yield from _validate(o, schema_items, new_context)
760
761
762 def validate_tuple(obj, schema, context):
763 if _is_null_allowed_for_object(obj, schema, context):
764 return
765 if not isinstance(obj, (list, tuple)):
766 yield ValidationProblem(code=10014, hint="got: {}".format(type(obj).__name__), context=context)
767 return
768 yield from _validate_index_constraint(obj, schema, context)
769 minlen = schema.get("minLength", None)
770 if minlen:
771 if len(obj) < minlen:
772 yield ValidationProblem(code=10015, hint=obj, context=context)
773 maxlen = schema.get("maxLength", None)
774 if maxlen is not None:
775 if len(obj) > maxlen:
776 yield ValidationProblem(code=10016, hint=obj, context=context)
777 schema_items = schema.get("items", [])
778 if not isinstance(schema_items, (list, tuple)):
779 raise SchemaError("tuple items require a list of schemata in items")
780 for idx, o in enumerate(obj):
781 # early exit at maxlen
782 if maxlen is not None and idx >= maxlen:
783 break
784 new_context = Context(parent=context, index=idx, current_object=o)
785 try:
786 schema_index = schema.ensure_child_schema(schema_items[idx])
787 except IndexError:
788 additional_items = schema.get_child("additionalItems", False)
789 if isinstance(additional_items, bool):
790 if not additional_items:
791 yield ValidationProblem(code=10017, context=new_context)
792 else:
793 yield from _validate(o, additional_items, new_context)
794 else:
795 yield from _validate(o, schema_index, new_context)
796
797
798 def validate_str(obj, schema, context):
799 if _is_null_allowed_for_object(obj, schema, context):
800 return
801 if not isinstance(obj, str):
802 yield ValidationProblem(code=10002, hint=obj, context=context)
803 else:
804 yield from _validate_index_constraint(obj, schema, context)
805 enumvalues = schema.get("enum", None)
806 if enumvalues is not None:
807 for ev in enumvalues:
808 if ev == obj:
809 break
810 else:
811 yield ValidationProblem(code=10043, hint=obj, context=context)
812 minlen = schema.get("minLength", None)
813 if minlen:
814 if len(obj) < minlen:
815 yield ValidationProblem(code=10006, hint=obj, context=context)
816 maxlen = schema.get("maxLength", None)
817 if maxlen is not None:
818 if len(obj) > maxlen:
819 yield ValidationProblem(code=10007, hint=obj, context=context)
820 pattern = schema.get("pattern", None)
821 if pattern is not None:
822 if isinstance(pattern, str):
823 mo = re.search(pattern, obj)
824 if not mo:
825 yield ValidationProblem(code=10008, context=context)
826 elif isinstance(pattern, TYPE_RE):
827 mo = pattern.search(obj)
828 if not mo:
829 yield ValidationProblem(code=10008, context=context)
830 elif callable(pattern):
831 yield from pattern(obj, schema, context)
832 else:
833 raise SchemaError("unknown pattern type")
834 is_contained = schema.get("is-contained-in-ref", None)
835 if is_contained is not None:
836 refobj = try_get_reference(is_contained,
837 context,
838 schema,
839 default=_SENTINEL)
840 if refobj is _SENTINEL:
841 yield ValidationProblem(code=10044, context=context)
842 else:
843 try:
844 if obj not in refobj:
845 yield ValidationProblem(code=10045, context=context)
846 except TypeError:
847 yield ValidationProblem(code=10046, context=context)
848
849
850 def validate_binary(obj, schema, context):
851 if not isinstance(obj, (bytes, bytearray)):
852 yield ValidationProblem(code=10035, hint=obj, context=context)
853 else:
854 yield from _validate_index_constraint(obj, schema, context)
855 minlen = schema.get("minLength", None)
856 if minlen:
857 if len(obj) < minlen:
858 yield ValidationProblem(code=10036, hint=obj, context=context)
859 maxlen = schema.get("maxLength", None)
860 if maxlen is not None:
861 if len(obj) > maxlen:
862 yield ValidationProblem(code=10037, hint=obj, context=context)
863 pattern = schema.get("pattern", None)
864 if pattern is not None:
865 if isinstance(pattern, (str, bytes, bytearray)):
866 if isinstance(pattern, str):
867 if "'''" not in pattern:
868 bytes_pattern = ast.literal_eval(
869 "b'''" + pattern + "'''")
870 elif '"""' not in pattern:
871 bytes_pattern = ast.literal_eval(
872 'b"""' + pattern + '"""')
873 else:
874 raise SchemaError("incompatible bytes pattern")
875 else:
876 bytes_pattern = pattern
877 mo = re.search(bytes_pattern, obj)
878 if not mo:
879 yield ValidationProblem(code=10047, context=context)
880 elif isinstance(pattern, TYPE_RE):
881 mo = pattern.search(obj)
882 if not mo:
883 yield ValidationProblem(code=10047, context=context)
884 elif callable(pattern):
885 yield from pattern(obj, schema, context)
886 else:
887 raise SchemaError("unknown pattern type")
888
889
890 def validate_timestamp(obj, schema, context):
891 if not isinstance(obj, datetime.datetime):
892 yield ValidationProblem(code=10041, hint=obj, context=context)
893 else:
894 yield from _validate_index_constraint(obj, schema, context)
895 value = schema.get("value", None)
896 if value is not None:
897 if callable(value):
898 yield from value(obj, schema, context)
899 else:
900 raise SchemaError("unknown value validator (only a callable allowed)")
901
902
903 def validate_integer(obj, schema, context):
904 if _is_null_allowed_for_object(obj, schema, context):
905 return
906 if not isinstance(obj, int):
907 yield ValidationProblem(code=10020, hint=obj, context=context)
908 else:
909 yield from _validate_index_constraint(obj, schema, context)
910 minValue = schema.get("minValue", None)
911 if minValue is not None and obj < minValue:
912 yield ValidationProblem(code=10021, hint=obj, context=context)
913 maxValue = schema.get("maxValue", None)
914 if maxValue is not None and obj > maxValue:
915 yield ValidationProblem(code=10022, hint=obj, context=context)
916 enumvalues = schema.get("enum", None)
917 if enumvalues is not None:
918 for ev in enumvalues:
919 if ev == obj:
920 break
921 else:
922 yield ValidationProblem(code=10048, hint=obj, context=context)
923 value = schema.get("value", None)
924 if value is not None:
925 if callable(value):
926 yield from value(obj, schema, context)
927 else:
928 raise SchemaError("unknown value validator (only a callable allowed)")
929
930
931 def validate_float(obj, schema, context):
932 if _is_null_allowed_for_object(obj, schema, context):
933 return
934 if not isinstance(obj, float):
935 yield ValidationProblem(code=10023, hint=obj, context=context)
936 else:
937 yield from _validate_index_constraint(obj, schema, context)
938 minValue = schema.get("minValue", None)
939 if minValue is not None and obj < minValue:
940 yield ValidationProblem(code=10024, hint=obj, context=context)
941 maxValue = schema.get("maxValue", None)
942 if maxValue is not None and obj > maxValue:
943 yield ValidationProblem(code=10025, hint=obj, context=context)
944 value = schema.get("value", None)
945 if value is not None:
946 if callable(value):
947 yield from value(obj, schema, context)
948 else:
949 raise SchemaError("unknown value validator (only a callable allowed)")
950
951
952 def validate_number(obj, schema, context):
953 if _is_null_allowed_for_object(obj, schema, context):
954 return
955 if not isinstance(obj, (int, float)):
956 yield ValidationProblem(code=10030, hint=obj, context=context)
957 else:
958 yield from _validate_index_constraint(obj, schema, context)
959 minValue = schema.get("minValue", None)
960 if minValue is not None and isinstance(obj, float):
961 minValue *= 1.0
962 if minValue is not None and obj < minValue:
963 yield ValidationProblem(code=10031, hint=obj, context=context)
964 maxValue = schema.get("maxValue", None)
965 if maxValue is not None and isinstance(obj, float):
966 maxValue *= 1.0
967 if maxValue is not None and obj > maxValue:
968 yield ValidationProblem(code=10032, hint=obj, context=context)
969 enumvalues = schema.get("enum", None)
970 if enumvalues is not None:
971 for ev in enumvalues:
972 if ev == obj:
973 break
974 else:
975 yield ValidationProblem(code=10049, hint=obj, context=context)
976 value = schema.get("value", None)
977 if value is not None:
978 if callable(value):
979 yield from value(obj, schema, context)
980 else:
981 raise SchemaError("unknown value validator (only a callable allowed)")
982
983
984 def validate_scalar(obj, schema, context):
985 if _is_null_allowed_for_object(obj, schema, context):
986 return
987 yield from _validate_index_constraint(obj, schema, context)
988 if obj is None:
989 yield ValidationProblem(code=10033, hint=obj, context=context)
990 if isinstance(obj, (dict, list, tuple, set, frozenset)):
991 yield ValidationProblem(code=10033, hint=obj, context=context)
992
993
994 def validate_deny(obj, schema, context):
995 yield from _validate_index_constraint(obj, schema, context)
996 yield ValidationProblem(code=10010, context=context)
997
998
999 def validate_accept(obj, schema, context):
1000 yield from _validate_index_constraint(obj, schema, context)
1001
1002
1003 def validate_null(obj, schema, context):
1004 yield from _validate_index_constraint(obj, schema, context)
1005 if obj is not None:
1006 yield ValidationProblem(code=10011, context=context)
1007
1008
1009 def validate_empty(obj, schema, context):
1010 yield from _validate_index_constraint(obj, schema, context)
1011 if obj is None:
1012 return
1013 if isinstance(obj, (dict, list, tuple, set, frozenset)) and not obj:
1014 return
1015 yield ValidationProblem(10018, context=context)
1016
1017
1018 def validate_bool(obj, schema, context):
1019 if _is_null_allowed_for_object(obj, schema, context):
1020 return
1021 if not isinstance(obj, bool):
1022 yield ValidationProblem(code=10026, hint=obj, context=context)
1023 else:
1024 yield from _validate_index_constraint(obj, schema, context)
1025 value = schema.get("value", None)
1026 if value is not None:
1027 if callable(value):
1028 yield from value(obj, schema, context)
1029 elif value and not obj:
1030 yield ValidationProblem(code=10027, hint=obj, context=context)
1031 elif not value and obj:
1032 yield ValidationProblem(code=10028, hint=obj, context=context)
1033
1034
1035 def validate_allOf(obj, schema, context):
1036 if not isinstance(schema, (list, tuple)):
1037 raise SchemaError("require a list of schematas for `all-of'")
1038 res = []
1039 for idx, s in enumerate(schema):
1040 assert isinstance(s, _Schema)
1041 tr = list(_validate(obj, s, context))
1042 if tr:
1043 res.append((idx, tr, ))
1044 if res:
1045 yield ValidationProblem(
1046 code=10057,
1047 context=context,
1048 cause=[
1049 ValidationProblem(
1050 code=10058,
1051 context=context,
1052 cause=tr,
1053 index=idx) for (idx, tr) in res])
1054
1055
1056 def validate_anyOf(obj, schema, context):
1057 if not isinstance(schema, (list, tuple)):
1058 raise SchemaError("require a list of schematas for `any-of'")
1059 res = []
1060 for s in schema:
1061 assert isinstance(s, _Schema)
1062 tr = list(_validate(obj, s, context))
1063 if tr:
1064 res.append(tr)
1065 else:
1066 # Erfolg: gleich positiv zurueck ohne Meldungen
1067 return
1068 # Ansonsten: alle Fehlschlaege protokollieren
1069 if res:
1070 yield ValidationProblem(
1071 code=10055,
1072 context=context,
1073 cause=[
1074 ValidationProblem(
1075 code=10056,
1076 context=context,
1077 cause=tr) for tr in res])
1078
1079
1080 def validate_oneOf(obj, schema, context):
1081 if not isinstance(schema, (list, tuple)):
1082 raise SchemaError("require a list of schematas for `one-of'")
1083 success_res = []
1084 failed_res = []
1085 for idx, s in enumerate(schema):
1086 assert isinstance(s, _Schema)
1087 tr = list(_validate(obj, s, context))
1088 if tr:
1089 failed_res.append((idx, tr, ))
1090 else:
1091 success_res.append(idx)
1092 if len(success_res) == 1:
1093 return
1094 elif len(success_res) == 0:
1095 # Ansonsten: alle Fehlschlaege protokollieren
1096 if failed_res:
1097 yield ValidationProblem(
1098 code=10053,
1099 context=context,
1100 cause=[
1101 ValidationProblem(
1102 code=10054,
1103 context=context,
1104 cause=tr,
1105 index=idx) for (idx, tr) in failed_res])
1106 else:
1107 # Die Indizes der "zuvielen" in "hint" anzeigen
1108 yield ValidationProblem(code=10019, hint=",".join([str(k) for k in success_res]))
1109
1110
1111 def validate_not(obj, schema, context):
1112 assert isinstance(schema, _Schema)
1113 res = list(_validate(obj, schema, context))
1114 if not res:
1115 yield ValidationProblem(code=10029, hint=obj, context=context,
1116 cause=res)
1117
1118
1119 def process_schema_references(schema, context, check_single_ref_key=True):
1120 try:
1121 ref = schema[SCHEMA_REF_KEY]
1122 except (KeyError, TypeError):
1123 return schema
1124 # if `$ref' is found it MUST be the only key
1125 if check_single_ref_key and len(schema) != 1:
1126 raise SchemaError("`{}' must be the single key if it exists")
1127 schema = try_get_reference(ref, context, schema)
1128 if not isinstance(schema, _Schema):
1129 raise SchemaError(
1130 "dereferenced schema is not a `_Schema': {}".format(ref))
1131 schema = copy.deepcopy(schema)
1132 return process_schema_references(schema, context, check_single_ref_key=True)
1133
1134
1135 def process_schema_conditionals(schema, context):
1136 """Lisp-like `cond` to provide schema modifications
1137
1138 :param schema: the input schema
1139 :param context: the validation context with a valid
1140 `context.root.root_object`
1141 :returns: the processed schema: the schema itself if it is unchanged and
1142 a copy of the schema if has been changed
1143
1144 """
1145 what, conds = _get_one_of(schema, "cond", "match", default=None)
1146 if what is None or conds is None:
1147 return schema
1148 if not isinstance(conds, (list, tuple)):
1149 raise SchemaError("the conditions of a cond must be a sequence")
1150 if what == "cond":
1151 return _process_schema_conditionals_cond(schema, conds, context)
1152 elif what == "match":
1153 return _process_schema_conditionals_match(schema, conds, context)
1154 else:
1155 assert False, "unreachable"
1156
1157
1158 def _process_schema_conditionals_cond(schema, conds, context):
1159 for cond in conds:
1160 if not isinstance(cond, dict):
1161 raise SchemaError("a single condition must be a dict")
1162 if eval_condition(cond, context, schema):
1163 rep_type, rep_schema = _get_one_of(
1164 cond, "then", "then-replace", "then-merge")
1165 rep_schema = schema.ensure_child_schema(rep_schema)
1166 if rep_type in ("then", "then-replace"):
1167 do_merge = False
1168 elif rep_type == "then-merge":
1169 do_merge = True
1170 else:
1171 raise SchemaError("unknown then type: {}".format(rep_type))
1172 break
1173 else:
1174 #
1175 # No condition was true: just remove the "cond" to get the
1176 # effective schema.
1177 #
1178 rep_schema = None
1179 do_merge = False
1180
1181 new_schema = schema.copy()
1182 del new_schema["cond"]
1183 if rep_schema:
1184 rep_schema = process_schema_references(rep_schema, context)
1185 # this could insert a new nested "cond" or "match" again
1186 if do_merge:
1187 rep_schema = copy.deepcopy(rep_schema)
1188 new_schema = _merge(rep_schema, new_schema)
1189 else:
1190 new_schema.update(rep_schema)
1191 # Recursively apply "cond/match" evaluation to the resulting schema
1192 return process_schema_conditionals(new_schema, context)
1193
1194
1195 def _process_schema_conditionals_match(schema, conds, context):
1196 rep_schemata = []
1197 for cond in conds:
1198 if not isinstance(cond, dict):
1199 raise SchemaError("a single condition must be a dict")
1200 if eval_condition(cond, context, schema):
1201 rep_type, rep_schema = _get_one_of(
1202 cond, "then", "then-replace", "then-merge")
1203 rep_schema = schema.ensure_child_schema(rep_schema)
1204 if rep_type in ("then", "then-replace"):
1205 rep_schemata.append((False, rep_schema))
1206 elif rep_type == "then-merge":
1207 rep_schemata.append((True, rep_schema))
1208 else:
1209 raise SchemaError("unknown then type: {}".format(rep_type))
1210
1211 new_schema = schema.copy()
1212 del new_schema["match"]
1213 for do_merge, rep_schema in rep_schemata:
1214 rep_schema = process_schema_references(rep_schema, context)
1215 # this could insert a new nested "cond" or "match" again
1216 if do_merge:
1217 rep_schema = copy.deepcopy(rep_schema)
1218 new_schema = _merge(rep_schema, new_schema)
1219 else:
1220 new_schema.update(rep_schema)
1221 # Recursively apply "cond/match" evaluation to the resulting schema
1222 return process_schema_conditionals(new_schema, context)
1223
1224
1225 def eval_condition(cond, context, schema):
1226 """Eval the condition in `cond` and return a tuple `(hit, predval)`
1227
1228 """
1229 pred, predval = _get_one_of(
1230 cond,
1231 "when-ref-true", "when-ref-exists", "when",
1232 default=_SENTINEL)
1233
1234 if pred == "when":
1235 # rekursive evaluation of `predval` as the real predicate
1236 return eval_pred(predval, context, schema)
1237 elif pred == "when-ref-true":
1238 refobj = try_get_reference(predval, context, schema, default=None)
1239 return bool(refobj)
1240 elif pred == "when-ref-exists":
1241 refobj = try_get_reference(predval, context, schema, default=_SENTINEL)
1242 return refobj is not _SENTINEL
1243 else:
1244 raise SchemaError("unknown condition type: {}".format(pred))
1245
1246
1247 def eval_pred(pred, context, schema):
1248 if isinstance(pred, dict):
1249 combinator, combinator_val = _get_one_of(
1250 pred,
1251 "not", "all-of", "any-of", "one-of",
1252 default=None)
1253 if combinator:
1254 if combinator == "not":
1255 return not eval_pred(combinator_val, context, schema)
1256 elif combinator == "all-of":
1257 if not isinstance(combinator_val, (list, tuple)):
1258 raise SchemaError("`all-of' requires a list of childs")
1259 for cv in combinator_val:
1260 if not eval_pred(cv, context, schema):
1261 return False
1262 return True
1263 elif combinator == "any-of":
1264 if not isinstance(combinator_val, (list, tuple)):
1265 raise SchemaError("`any-of' requires a list of childs")
1266 for cv in combinator_val:
1267 if eval_pred(cv, context, schema):
1268 return True
1269 return False
1270 elif combinator == "one-of":
1271 if not isinstance(combinator_val, (list, tuple)):
1272 raise SchemaError("`one-of' requires a list of childs")
1273 num_true = 0
1274 for cv in combinator_val:
1275 if eval_pred(cv, context, schema):
1276 num_true += 1
1277 # shortcut
1278 if num_true > 1:
1279 return False
1280 if num_true == 1:
1281 return True
1282 else:
1283 return False
1284 else:
1285 raise SchemaError(
1286 "unknown logical operator: {}".format(combinator))
1287 else:
1288 pred_key, pred_val = _get_one_of(
1289 pred,
1290 "ref-true", "ref-exists", "equals",
1291 default=None)
1292 if pred_key == "ref-true":
1293 refobj = try_get_reference(
1294 pred_val, context, schema, default=None)
1295 return bool(refobj)
1296 elif pred_key == "ref-exists":
1297 refobj = try_get_reference(
1298 pred_val, context, schema, default=_SENTINEL)
1299 return refobj is not _SENTINEL
1300 elif pred_key == "equals":
1301 if not isinstance(pred_val, (list, tuple)):
1302 raise SchemaError("`equals' requires a list as childs")
1303 if len(pred_val) != 2:
1304 raise SchemaError("`equals' requires a list of len 2")
1305 op1 = eval_comparison_operator_operand(
1306 pred_val[0], context, schema)
1307 op2 = eval_comparison_operator_operand(
1308 pred_val[1], context, schema)
1309 return op1 == op2
1310 else:
1311 raise SchemaError("unknown predicate: {}".format(pred))
1312 elif isinstance(pred, list):
1313 # implicit all-of (aka AND)
1314 for cv in pred:
1315 if not eval_pred(cv, context, schema):
1316 return False
1317 return True
1318 else:
1319 return pred
1320
1321
1322 def eval_comparison_operator_operand(op, context, schema):
1323 if not isinstance(op, dict):
1324 raise SchemaError("an operand must be a dict")
1325 opkey, opval = _get_one_of(op, "ref", "val", "value")
1326 if opkey is None:
1327 raise SchemaError("no operant given in {!r}".format(op))
1328 if opkey == "ref":
1329 return try_get_reference(opval, context, schema)
1330 elif opkey in ("val", "value"):
1331 return opval
1332 else:
1333 assert False
1334
1335
1336 def try_get_reference(ref, context, schema, default=None):
1337 """Get the object referenced in `ref`
1338
1339 Use `context` as data/object context and `schema` as the current schema
1340 context.
1341
1342 """
1343 uri = rfc3986.URIReference.from_string(ref).normalize()
1344 if not uri.scheme:
1345 uri = uri.copy_with(scheme="object")
1346 if uri.scheme == "object":
1347 if ref.startswith("object#"):
1348 for attr in ("authority", "path", "query"):
1349 if getattr(uri, attr, None) is not None:
1350 raise SchemaError(
1351 "bogus {} in URI reference `{}'".format(attr, ref))
1352 if uri.fragment is None:
1353 raise SchemaError("fragment required in reference")
1354 if not uri.fragment:
1355 return context.root.root_object
1356 elif uri.fragment == '.':
1357 return context.current_object
1358 parts = uri.fragment.split('.') # use '.' separator as in configmix
1359 if parts[0]:
1360 # absolute
1361 d = context.root.root_object
1362 else:
1363 # relative
1364 d = context.current_object
1365 parts = parts[1:]
1366 c = context # needed to determine relative object references
1367 relative_refs_allowed = True
1368 for part in [urllib.parse.unquote(p) for p in parts]:
1369 if part:
1370 relative_refs_allowed = False
1371 try:
1372 d = d[part]
1373 except (KeyError, IndexError, TypeError):
1374 return default
1375 else:
1376 if not relative_refs_allowed:
1377 raise SchemaError(
1378 "empty part in path to object reference not allowed")
1379 c = c.safe_parent
1380 d = c.current_object
1381 return d
1382 elif uri.scheme == "schema":
1383 if not uri.path or (uri.path == SCHEMA_PATH_SELF):
1384 s = schema.SELF
1385 elif uri.path == SCHEMA_PATH_ROOT:
1386 s = schema.ROOT
1387 else:
1388 s = schema.get_cached_schema(uri.path, load_if_needed=True)
1389 if uri.fragment is None:
1390 raise SchemaError("fragment required in reference")
1391
1392 if not uri.fragment.startswith('/'):
1393 raise SchemaError("references to parts of a schema must be absolute (begin with `/')")
1394 if uri.fragment == '/':
1395 return s
1396 parts = uri.fragment.split('/')
1397 parent_for_subschema = s
1398 for part in [urllib.parse.unquote(p) for p in parts[1:]]:
1399 try:
1400 v = s[part]
1401 except (KeyError, IndexError, TypeError):
1402 return default
1403 else:
1404 if isinstance(v, _Schema):
1405 pass
1406 elif isinstance(v, dict):
1407 s = _Schema(parent_for_subschema, False, v)
1408 else:
1409 # need not try further
1410 return default
1411 return s
1412 else:
1413 raise SchemaError("Unknown schema reference scheme: {}".format(uri.scheme))
1414
1415
1416 _DEL_VALUE = '{{::DEL::}}'
1417 """Sigil to mark keys to be deleted in the target when merging"""
1418
1419
1420 def _merge(user, default):
1421 """Logically merge the configuration in `user` into `default`.
1422
1423 :param dict user:
1424 the new configuration that will be logically merged
1425 into `default`
1426 :param dict default:
1427 the base configuration where `user` is logically merged into
1428 :returns: `user` with the necessary amendments from `default`.
1429 If `user` is ``None`` then `default` is returned.
1430
1431 .. note:: Implementation: The configuration in `user` is
1432 augmented/changed **inplace**.
1433
1434 If a value in `user` is equal to :data:`._DEL_VALUE`
1435 (``{{::DEL::}}``) the corresponding key will be deleted from the
1436 merged output.
1437
1438 From http://stackoverflow.com/questions/823196/yaml-merge-in-python
1439
1440 """
1441 if user is None:
1442 _filter_deletions(default)
1443 return default
1444 if isinstance(user, dict) and isinstance(default, dict):
1445 for k, v in default.items():
1446 if k in user:
1447 if user[k] == _DEL_VALUE:
1448 # do not copy and delete the marker
1449 del user[k]
1450 else:
1451 user[k] = _merge_item(user[k], v)
1452 else:
1453 user[k] = v
1454 else:
1455 raise SchemaError("can only merge two dicts on top-level")
1456 _filter_deletions(user)
1457 return user
1458
1459
1460 def _merge_item(user, default):
1461 """Recursion helper for :func:`._merge`
1462
1463 """
1464 if isinstance(user, dict) and isinstance(default, dict):
1465 for k, v in default.items():
1466 if k in user:
1467 if user[k] == _DEL_VALUE:
1468 # do not copy and delete the marker
1469 del user[k]
1470 else:
1471 user[k] = _merge_item(user[k], v)
1472 else:
1473 user[k] = v
1474 elif isinstance(user, (list, tuple)) and isinstance(default, (list, tuple)):
1475 for idx, v in enumerate(default):
1476 user.insert(idx, v)
1477 return user
1478
1479
1480 def _filter_deletions(d):
1481 """Recursively filter deletions in the dict `d`.
1482
1483 Deletions have values that equal :data:`._DEL_VALUE`.
1484
1485 """
1486 if not isinstance(d, dict):
1487 return
1488 # use a copy of the items because we change `d` while iterating
1489 for k, v in list(d.items()):
1490 if v == _DEL_VALUE:
1491 del d[k]
1492 else:
1493 if isinstance(d[k], dict):
1494 _filter_deletions(d[k])
1495
1496
1497 def _log_problem_cause_all(logger, loglevel, level, problems):
1498 if not problems:
1499 return
1500 for pr in problems:
1501 logger.log(loglevel, "%s> %r", "-"*((level*2)+2), pr)
1502 _log_problem_cause_all(logger, loglevel, level+1, pr.cause)
1503
1504
1505 def _build_problems_by_level_and_depth(by_level, by_depth, level, problems):
1506 for pr in problems:
1507 if not pr.cause:
1508 continue
1509 try:
1510 prl = by_level[level]
1511 except LookupError:
1512 prl= []
1513 by_level[level] = prl
1514 prl.append(pr)
1515
1516 depth = pr.context.depth
1517 try:
1518 prd = by_depth[depth]
1519 except LookupError:
1520 prd= []
1521 by_depth[depth] = prd
1522 prd.append(pr)
1523 _build_problems_by_level_and_depth(
1524 by_level, by_depth, level+1, pr.cause)
1525
1526
1527 def _log_problem_cause(logger, loglevel, max_level, max_depth, level, problems):
1528 for pr in problems:
1529 #
1530 # Check whether we will start logging from this level downwards
1531 # all problems
1532 #
1533 if max_level is None or level == max_level:
1534 new_max_level = None # trigger logging
1535 else:
1536 new_max_level = max_level
1537 if max_depth is None or max_depth == pr.context.depth:
1538 new_max_depth = None # trigger logging
1539 else:
1540 new_max_depth = max_depth
1541 if new_max_level is None or new_max_depth is None:
1542 logger.log(loglevel, "%s> %r", "-"*((level*2)+2), pr)
1543 if pr.cause:
1544 _log_problem_cause(
1545 logger, loglevel,
1546 new_max_level, new_max_depth,
1547 level+1, pr.cause)
1548
1549
1550 def log_problem_cause(logger, loglevel, debug, level, problems):
1551 if not problems:
1552 return
1553 if debug:
1554 _log_problem_cause_all(logger, loglevel, level, problems)
1555 else:
1556 by_level = {} # to determine maximum problem nesting level
1557 by_depth = {} # to determine maximum context nexting level
1558 _build_problems_by_level_and_depth(by_level, by_depth, level, problems)
1559
1560 max_level = max(by_level.keys())
1561 max_depth = max(by_depth.keys())
1562
1563 _log_problem_cause(
1564 logger, loglevel, max_level, max_depth, level, problems)