diff --git a/misc/scripts/convert-yaml-data-extensions-to-json.py b/misc/scripts/convert-yaml-data-extensions-to-json.py new file mode 100755 index 000000000000..3d6688845853 --- /dev/null +++ b/misc/scripts/convert-yaml-data-extensions-to-json.py @@ -0,0 +1,388 @@ +#!/usr/bin/env python3 + +"""Convert YAML data extension files to JSON while preserving comments. + +USAGE: + +Pass files directly to convert a specific set of data extensions: + + convert-yaml-data-extensions-to-json.py [--delete-yaml] FILE [FILE ...] + +Each file will be converted to a JSON file with the same base name. By default, the original YAML +file is retained; pass --delete-yaml to remove it after successful conversion. + +Alternatively, pass a qlpack.yml manifest: + + convert-yaml-data-extensions-to-json.py [--delete-yaml] --qlpack PATH/qlpack.yml + +In this mode, files will be converted as above, but the qlpack.yml manifest itself will be updated: +the existing YAML patterns will be preserved (so that they will continue to match if any new YAML +extensions are added), and new file patterns matching the JSON files will be added. + +WARNING: + +This script was generated by an LLM and has not been reviewed by a human; always check its output +for correctness. +""" + +import argparse +import json +import sys +from collections import defaultdict +from io import StringIO +from pathlib import Path +from typing import Any + +from ruamel.yaml import YAML + +YAML_PARSER = YAML(typ="rt") +YAML_PARSER.allow_duplicate_keys = False +YAML_PARSER.indent(mapping=2, sequence=4, offset=2) +YAML_PARSER.width = 4096 + + +def load_yaml(path: Path) -> tuple[Any, str]: + """Load a YAML file and return both its parsed value and source text.""" + text = path.read_text(encoding="utf-8") + return YAML_PARSER.load(text), text + + +def encode_scalar(value: Any) -> str: + """Encode a scalar value as compact JSON.""" + return json.dumps(value, ensure_ascii=False, separators=(",", ":")) + + +def yaml_comment_start(line: str) -> int | None: + """Find the start of a YAML comment outside quoted strings.""" + single_quoted = False + double_quoted = False + escaped = False + index = 0 + while index < len(line): + character = line[index] + if double_quoted: + if escaped: + escaped = False + elif character == "\\": + escaped = True + elif character == '"': + double_quoted = False + elif single_quoted: + if character == "'" and index + 1 < len(line) and line[index + 1] == "'": + index += 1 + elif character == "'": + single_quoted = False + elif character == '"': + double_quoted = True + elif character == "'": + single_quoted = True + elif character == "#" and (index == 0 or line[index - 1].isspace()): + return index + index += 1 + return None + + +def node_anchors( + value: Any, path: tuple[Any, ...] = () +) -> list[tuple[int, tuple[Any, ...]]]: + """Map parsed YAML nodes to their source lines and object paths.""" + anchors = [] + if path == () and hasattr(value, "lc"): + anchors.append((value.lc.line, path)) + if isinstance(value, dict): + for key, child in value.items(): + child_path = path + (key,) + anchors.append((value.lc.key(key)[0], child_path)) + anchors.extend(node_anchors(child, child_path)) + elif isinstance(value, list): + for index, child in enumerate(value): + child_path = path + (index,) + anchors.append((value.lc.item(index)[0], child_path)) + if not path or path[-1] != "data": + anchors.extend(node_anchors(child, child_path)) + return anchors + + +def associate_comments( + value: Any, source: str +) -> tuple[ + dict[tuple[Any, ...], list[str]], + dict[tuple[Any, ...], str], + list[str], +]: + """Associate YAML comments with nearby parsed nodes.""" + anchors_by_line: dict[int, set[tuple[Any, ...]]] = defaultdict(set) + for line, path in node_anchors(value): + anchors_by_line[line].add(path) + + pre_comments: dict[tuple[Any, ...], list[str]] = defaultdict(list) + inline_comments: dict[tuple[Any, ...], str] = {} + trailing_comments: list[str] = [] + pending: list[str] = [] + + for line_number, line in enumerate(source.splitlines()): + comment_start = yaml_comment_start(line) + if comment_start is None: + continue + comment = line[comment_start + 1 :].strip() + if line[:comment_start].strip(): + candidates = anchors_by_line.get(line_number) + if candidates: + path = min( + candidates, key=lambda candidate: (len(candidate), repr(candidate)) + ) + inline_comments[path] = comment + else: + pending.append(comment) + later_lines = [ + anchor_line + for anchor_line in anchors_by_line + if anchor_line > line_number + ] + if later_lines: + next_line = min(later_lines) + path = min( + anchors_by_line[next_line], + key=lambda candidate: (len(candidate), repr(candidate)), + ) + pre_comments[path].extend(pending) + pending = [] + + trailing_comments.extend(pending) + return pre_comments, inline_comments, trailing_comments + + +def format_json( + value: Any, + pre_comments: dict[tuple[Any, ...], list[str]], + inline_comments: dict[tuple[Any, ...], str], + level: int = 0, + inline_rows: bool = False, + path: tuple[Any, ...] = (), +) -> str: + """Format a parsed value as JSON while restoring associated comments.""" + indent = " " * level + child_indent = " " * (level + 1) + + if not isinstance(value, (dict, list)): + return encode_scalar(value) + + if isinstance(value, dict): + if not value: + return "{}" + entries = [] + items = list(value.items()) + for index, (key, child) in enumerate(items): + child_path = path + (key,) + encoded_key = json.dumps(key, ensure_ascii=False) + encoded_child = format_json( + child, + pre_comments, + inline_comments, + level + 1, + inline_rows=(key == "data"), + path=child_path, + ) + entry = f"{child_indent}{encoded_key}: {encoded_child}" + if index + 1 < len(items): + entry += "," + if comment := inline_comments.get(child_path): + entry += f" // {comment}" + comment_lines = [ + f"{child_indent}// {comment}".rstrip() + for comment in pre_comments.get(child_path, []) + ] + entries.append("\n".join(comment_lines + [entry])) + return "{\n" + "\n".join(entries) + f"\n{indent}}}" + + if not value: + return "[]" + if inline_rows: + entries = [] + for index, row in enumerate(value): + child_path = path + (index,) + entry = ( + f"{child_indent}" + f"{json.dumps(row, ensure_ascii=False, separators=(', ', ': '))}" + ) + if index + 1 < len(value): + entry += "," + if comment := inline_comments.get(child_path): + entry += f" // {comment}" + comment_lines = [ + f"{child_indent}// {comment}".rstrip() + for comment in pre_comments.get(child_path, []) + ] + entries.append("\n".join(comment_lines + [entry])) + return "[\n" + "\n".join(entries) + f"\n{indent}]" + + entries = [] + for index, child in enumerate(value): + child_path = path + (index,) + entry = ( + f"{child_indent}" + f"{format_json(child, pre_comments, inline_comments, level + 1, path=child_path)}" + ) + if index + 1 < len(value): + entry += "," + if comment := inline_comments.get(child_path): + entry += f" // {comment}" + comment_lines = [ + f"{child_indent}// {comment}".rstrip() + for comment in pre_comments.get(child_path, []) + ] + entries.append("\n".join(comment_lines + [entry])) + return "[\n" + "\n".join(entries) + f"\n{indent}]" + + +def format_json_with_comments(value: Any, source: str) -> str: + """Format a parsed YAML value as JSON with its original comments.""" + pre_comments, inline_comments, trailing_comments = associate_comments(value, source) + lines = [f"// {comment}".rstrip() for comment in pre_comments.pop((), [])] + lines.append(format_json(value, pre_comments, inline_comments)) + lines.extend(f"// {comment}".rstrip() for comment in trailing_comments) + return "\n".join(lines) + + +def write_text_atomically(path: Path, contents: str) -> None: + """Write text through a temporary file and atomically replace the target.""" + temporary = path.with_name(f".{path.name}.tmp") + try: + temporary.write_text(contents, encoding="utf-8") + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + + +def convert_data_extension(source: Path, delete_yaml: bool = False) -> Path: + """Convert a YAML data extension to JSON, optionally removing the source.""" + target = source.with_suffix(".json") + model, source_text = load_yaml(source) + write_text_atomically(target, format_json_with_comments(model, source_text) + "\n") + if delete_yaml: + source.unlink() + return target + + +def data_extensions_from_qlpack(path: Path) -> list[Path]: + """Find YAML data extensions matched by a qlpack manifest.""" + manifest, _ = load_yaml(path) + if not isinstance(manifest, dict): + raise ValueError(f"qlpack manifest is not a mapping: {path}") + + patterns = manifest.get("dataExtensions") + if not isinstance(patterns, list): + raise ValueError(f"qlpack manifest has no dataExtensions list: {path}") + + files = set() + for pattern in patterns: + if not isinstance(pattern, str): + raise ValueError(f"dataExtensions pattern is not a string: {pattern!r}") + files.update( + match.resolve() + for match in path.parent.glob(pattern.lstrip("/")) + if match.is_file() and match.suffix in {".yml", ".yaml"} + ) + return sorted(files) + + +def add_json_data_extension_patterns(path: Path) -> None: + """Add JSON equivalents of YAML data-extension patterns to a manifest.""" + manifest, _ = load_yaml(path) + if not isinstance(manifest, dict): + raise ValueError(f"qlpack manifest is not a mapping: {path}") + + patterns = manifest.get("dataExtensions") + if not isinstance(patterns, list): + raise ValueError(f"qlpack manifest has no dataExtensions list: {path}") + + existing_patterns = set() + for pattern in patterns: + if not isinstance(pattern, str): + raise ValueError(f"dataExtensions pattern is not a string: {pattern!r}") + existing_patterns.add(pattern) + + json_patterns = [] + for pattern in patterns: + if pattern.endswith(".yml"): + json_pattern = pattern.removesuffix(".yml") + ".json" + elif pattern.endswith(".yaml"): + json_pattern = pattern.removesuffix(".yaml") + ".json" + else: + continue + if json_pattern not in existing_patterns: + json_patterns.append(json_pattern) + existing_patterns.add(json_pattern) + + if not json_patterns: + return + + patterns.extend(json_patterns) + output = StringIO() + YAML_PARSER.dump(manifest, output) + write_text_atomically(path, output.getvalue()) + + +def validate_yaml_file(path: Path) -> Path: + """Resolve and validate a YAML input file.""" + path = path.expanduser().resolve() + if not path.is_file(): + raise ValueError(f"YAML file does not exist: {path}") + if path.suffix not in {".yml", ".yaml"}: + raise ValueError(f"not a YAML file: {path}") + return path + + +def main() -> None: + """Parse command-line arguments and convert the selected data extensions.""" + parser = argparse.ArgumentParser(description=__doc__) + inputs = parser.add_mutually_exclusive_group(required=True) + inputs.add_argument( + "files", + metavar="YAML_FILE", + nargs="*", + type=Path, + help="YAML data extension file to convert", + ) + inputs.add_argument( + "--qlpack", + metavar="QLPACK_FILE", + type=Path, + help="convert data extensions matched by a qlpack.yml manifest", + ) + parser.add_argument( + "--delete-yaml", + action="store_true", + help="delete YAML files after successful conversion", + ) + args = parser.parse_args() + + if args.qlpack is not None: + qlpack = args.qlpack.expanduser().resolve() + if not qlpack.is_file(): + raise ValueError(f"qlpack manifest does not exist: {qlpack}") + files = data_extensions_from_qlpack(qlpack) + else: + files = [] + for file in args.files: + if file.name == "qlpack.yml": + print( + f"Warning: skipping qlpack.yml passed as a data extension. Try running with --qlpack {file}", + file=sys.stderr, + flush=True, + ) + else: + files.append(file) + + files = [validate_yaml_file(file) for file in files] + for file in files: + convert_data_extension(file, delete_yaml=args.delete_yaml) + + if args.qlpack is not None: + add_json_data_extension_patterns(qlpack) + + print(f"Converted {len(files)} data extensions.", flush=True) + + +if __name__ == "__main__": + main()