Migrate Twister schemas to JSON Schema

boards/snps/em_starterkit/board.yml

@@ -8,7 +8,7 @@ board:
   - name: emsk_em11d
   revision:
     format: major.minor.patch
-    default: "2.3"
+    default: "2.3.0"
     revisions:
-    - name: "2.2"
-    - name: "2.3"
+    - name: "2.2.0"
+    - name: "2.3.0"

boards/snps/em_starterkit/em_starterkit_emsk_em7d_2_2.yaml → boards/snps/em_starterkit/em_starterkit_emsk_em7d_2_2_0.yaml

@@ -1,4 +1,4 @@
-identifier: em_starterkit@2.2/emsk_em7d
+identifier: em_starterkit@2.2.0/emsk_em7d
 name: EM Starterkit EM7D (rev. 2.2)
 type: mcu
 arch: arc

drivers/i2c/i2c_dw.c

@@ -11,7 +11,6 @@
 #ifdef CONFIG_CPU_CORTEX_M
 #include <cmsis_core.h>
 #endif
-#include <soc.h>
 
 #include <stddef.h>
 #include <zephyr/types.h>

scripts/list_boards.py

@@ -11,21 +11,24 @@
 from dataclasses import dataclass, field
 from pathlib import Path
 
+import jsonschema
 import list_hardware
-import pykwalify.core
 import yaml
+from jsonschema.exceptions import best_match
 from list_hardware import unique_paths
 
 try:
     from yaml import CSafeLoader as SafeLoader
 except ImportError:
     from yaml import SafeLoader
 
-BOARD_SCHEMA_PATH = str(Path(__file__).parent / 'schemas' / 'board-schema.yml')
+BOARD_SCHEMA_PATH = str(Path(__file__).parent / 'schemas' / 'board-schema.yaml')
 with open(BOARD_SCHEMA_PATH) as f:
     board_schema = yaml.load(f.read(), Loader=SafeLoader)
 
-BOARD_VALIDATOR = pykwalify.core.Core(schema_data=board_schema, source_data={})
+validator_class = jsonschema.validators.validator_for(board_schema)
+validator_class.check_schema(board_schema)
+board_validator = validator_class(board_schema)
 
 BOARD_YML = 'board.yml'
 
@@ -231,23 +234,14 @@ def load_v2_boards(board_name, board_yml, systems):
         with board_yml.open('r', encoding='utf-8') as f:
             b = yaml.load(f.read(), Loader=SafeLoader)
 
-        try:
-            BOARD_VALIDATOR.source = b
-            BOARD_VALIDATOR.validate()
-        except pykwalify.errors.SchemaError as e:
-            sys.exit(f'ERROR: Malformed "build" section in file: {board_yml.as_posix()}\n{e}')
-
-        mutual_exclusive = {'board', 'boards'}
-        if len(mutual_exclusive - b.keys()) < 1:
-            sys.exit(f'ERROR: Malformed content in file: {board_yml.as_posix()}\n'
-                     f'{mutual_exclusive} are mutual exclusive at this level.')
+        errors = list(board_validator.iter_errors(b))
+        if errors:
+            sys.exit('ERROR: Malformed board YAML file: '
+                     f'{board_yml.as_posix()}\n'
+                     f'{best_match(errors).message} in {best_match(errors).json_path}')
 
         board_array = b.get('boards', [b.get('board', None)])
         for board in board_array:
-            mutual_exclusive = {'name', 'extend'}
-            if len(mutual_exclusive - board.keys()) < 1:
-                sys.exit(f'ERROR: Malformed "board" section in file: {board_yml.as_posix()}\n'
-                         f'{mutual_exclusive} are mutual exclusive at this level.')
 
             # This is a extending an existing board, place in array to allow later processing.
             if 'extend' in board:
@@ -260,19 +254,6 @@ def load_v2_boards(board_name, board_yml, systems):
                 # Not the board we're looking for, ignore.
                 continue
 
-            board_revision = board.get('revision')
-            if board_revision is not None and board_revision.get('format') != 'custom':
-                if board_revision.get('default') is None:
-                    sys.exit(f'ERROR: Malformed "board" section in file: {board_yml.as_posix()}\n'
-                             "Cannot find required key 'default'. Path: '/board/revision.'")
-                if board_revision.get('revisions') is None:
-                    sys.exit(f'ERROR: Malformed "board" section in file: {board_yml.as_posix()}\n'
-                             "Cannot find required key 'revisions'. Path: '/board/revision.'")
-
-            mutual_exclusive = {'socs', 'variants'}
-            if len(mutual_exclusive - board.keys()) < 1:
-                sys.exit(f'ERROR: Malformed "board" section in file: {board_yml.as_posix()}\n'
-                         f'{mutual_exclusive} are mutual exclusive at this level.')
             socs = [Soc.from_soc(systems.get_soc(s['name']), s.get('variants', []))
                     for s in board.get('socs', {})]
 

scripts/list_hardware.py

@@ -9,26 +9,31 @@
 from dataclasses import dataclass
 from pathlib import Path, PurePath
 
-import pykwalify.core
+import jsonschema
 import yaml
+from jsonschema.exceptions import best_match
 
 try:
     from yaml import CSafeLoader as SafeLoader
 except ImportError:
     from yaml import SafeLoader
 
 
-SOC_SCHEMA_PATH = str(Path(__file__).parent / 'schemas' / 'soc-schema.yml')
+SOC_SCHEMA_PATH = str(Path(__file__).parent / 'schemas' / 'soc-schema.yaml')
 with open(SOC_SCHEMA_PATH) as f:
     soc_schema = yaml.load(f.read(), Loader=SafeLoader)
 
-SOC_VALIDATOR = pykwalify.core.Core(schema_data=soc_schema, source_data={})
-
-ARCH_SCHEMA_PATH = str(Path(__file__).parent / 'schemas' / 'arch-schema.yml')
+ARCH_SCHEMA_PATH = str(Path(__file__).parent / 'schemas' / 'arch-schema.yaml')
 with open(ARCH_SCHEMA_PATH) as f:
     arch_schema = yaml.load(f.read(), Loader=SafeLoader)
 
-ARCH_VALIDATOR = pykwalify.core.Core(schema_data=arch_schema, source_data={})
+validator_class = jsonschema.validators.validator_for(soc_schema)
+validator_class.check_schema(soc_schema)
+soc_validator = validator_class(soc_schema)
+
+validator_class = jsonschema.validators.validator_for(arch_schema)
+validator_class.check_schema(arch_schema)
+arch_validator = validator_class(arch_schema)
 
 SOC_YML = 'soc.yml'
 ARCHS_YML_PATH = PurePath('arch/archs.yml')
@@ -44,12 +49,12 @@ def __init__(self, folder='', soc_yaml=None):
         if soc_yaml is None:
             return
 
-        try:
-            data = yaml.load(soc_yaml, Loader=SafeLoader)
-            SOC_VALIDATOR.source = data
-            SOC_VALIDATOR.validate()
-        except (yaml.YAMLError, pykwalify.errors.SchemaError) as e:
-            sys.exit(f'ERROR: Malformed yaml {soc_yaml.as_posix()}', e)
+        data = yaml.load(soc_yaml, Loader=SafeLoader)
+        errors = list(soc_validator.iter_errors(data))
+        if errors:
+            sys.exit('ERROR: Malformed soc YAML file: \n'
+                        f'{soc_yaml}\n'
+                        f'{best_match(errors).message} in {best_match(errors).json_path}')
 
         for f in data.get('family', []):
             family = Family(f['name'], [folder], [], [])
@@ -82,10 +87,6 @@ def __init__(self, folder='', soc_yaml=None):
             self._socs.extend(socs)
 
         for soc in data.get('socs', []):
-            mutual_exclusive = {'name', 'extend'}
-            if len(mutual_exclusive - soc.keys()) < 1:
-                sys.exit(f'ERROR: Malformed content in SoC file: {soc_yaml}\n'
-                         f'{mutual_exclusive} are mutual exclusive at this level.')
             if soc.get('name') is not None:
                 self._socs.append(Soc(soc['name'], [c['name'] for c in soc.get('cpuclusters', [])],
                                   [folder], '', ''))
@@ -94,8 +95,9 @@ def __init__(self, folder='', soc_yaml=None):
                                            [c['name'] for c in soc.get('cpuclusters', [])],
                                            [folder], '', ''))
             else:
+                # This should not happen if schema validation passed
                 sys.exit(f'ERROR: Malformed "socs" section in SoC file: {soc_yaml}\n'
-                         f'Cannot find one of required keys {mutual_exclusive}.')
+                         f'SoC entry must have either "name" or "extend" property.')
 
         # Ensure that any runner configuration matches socs and cpuclusters declared in the same
         # soc.yml file
@@ -217,11 +219,11 @@ def find_v2_archs(args):
             with Path(archs_yml).open('r', encoding='utf-8') as f:
                 archs = yaml.load(f.read(), Loader=SafeLoader)
 
-            try:
-                ARCH_VALIDATOR.source = archs
-                ARCH_VALIDATOR.validate()
-            except pykwalify.errors.SchemaError as e:
-                sys.exit(f'ERROR: Malformed "build" section in file: {archs_yml.as_posix()}\n{e}')
+            errors = list(arch_validator.iter_errors(archs))
+            if errors:
+                sys.exit('ERROR: Malformed arch YAML file: '
+                         f'{archs_yml.as_posix()}\n'
+                         f'{best_match(errors).message} in {best_match(errors).json_path}')
 
             if args.arch is not None:
                 archs = {'archs': list(filter(

scripts/pylib/twister/scl.py

@@ -8,8 +8,12 @@
 # Set of code that other projects can also import to do things on
 # Zephyr's sanity check testcases.
 
+import jsonschema
 import logging
 import yaml
+
+from jsonschema.exceptions import best_match
+
 try:
     # Use the C LibYAML parser if available, rather than the Python parser.
     # It's much faster.
@@ -20,7 +24,8 @@
     from yaml import Loader, SafeLoader, Dumper
 
 log = logging.getLogger("scl")
-
+# Cache for validators to avoid recreating them for the same JSON schema
+_validator_cache = {}
 
 class EmptyYamlFileException(Exception):
     pass
@@ -51,23 +56,21 @@ def yaml_load(filename):
                   e.note, cmark.name, cmark.line, cmark.column, e.context)
         raise
 
-# If pykwalify is installed, then the validate function will work --
-# otherwise, it is a stub and we'd warn about it.
-try:
-    import pykwalify.core
-    # Don't print error messages yourself, let us do it
-    logging.getLogger("pykwalify.core").setLevel(50)
+def _yaml_validate(data, schema):
+    if not schema:
+        return
 
-    def _yaml_validate(data, schema):
-        if not schema:
-            return
-        c = pykwalify.core.Core(source_data=data, schema_data=schema)
-        c.validate(raise_exception=True)
+    schema_key = id(schema)
+    if schema_key not in _validator_cache:
+        validator_class = jsonschema.validators.validator_for(schema)
+        validator_class.check_schema(schema)
+        _validator_cache[schema_key] = validator_class(schema)
 
-except ImportError as e:
-    log.warning("can't import pykwalify; won't validate YAML (%s)", e)
-    def _yaml_validate(data, schema):
-        pass
+    validator = _validator_cache[schema_key]
+    errors = list(validator.iter_errors(data))
+    if errors:
+        err = best_match(errors)
+        raise jsonschema.ValidationError(f"Error: {err.message} in {err.json_path}")
 
 def yaml_load_verify(filename, schema):
     """
@@ -79,7 +82,7 @@ def yaml_load_verify(filename, schema):
 
     # 'document.yaml' contains a single YAML document.
     :raises yaml.scanner.ScannerError: on YAML parsing error
-    :raises pykwalify.errors.SchemaError: on Schema violation error
+    :raises jsonschema.exceptions.ValidationError: on Schema violation error
     """
     # 'document.yaml' contains a single YAML document.
     y = yaml_load(filename)

scripts/requirements-actions.in

@@ -10,6 +10,7 @@ gitlint>=0.19.1
 gitpython>=3.1.41
 ijson
 intelhex
+jsonschema
 junit2html
 junitparser>=4.0.1
 mypy