#!/usr/bin/env python3
"""
Generate C file to handle keysym names
"""
import argparse
import itertools
import random
import re
import sys
from collections import defaultdict
from dataclasses import dataclass
from enum import Enum
from pathlib import Path
from typing import DefaultDict, Generator, Iterable, Iterator
import perfect_hash
# Root of the project
SCRIPT = Path(__file__)
ROOT = SCRIPT.parent.parent
# Parse commands
parser = argparse.ArgumentParser(description="Generate C file to handle keysym names")
parser.add_argument(
"c_header", type=Path, help="Path to the libxkbcommon keysym header"
)
parser.add_argument("gperf", type=Path, help="Path to the gperf file")
args = parser.parse_args()
# Set the seed explicitly, so we reduce diff
random.seed(b"libxkbcommon")
KEYSYM_ENTRY_PATTERN = re.compile(
r"""
^\#define\s+
XKB_KEY_(?P<name>\w+)\s+
(?P<value>0x[0-9a-fA-F]+)\s*
(?:/\*\s*
(?:
(?P<deprecated>deprecated)|
\(<U\+(?P<unicode_alt_semantics>[0-9a-fA-F]{4,}>)\)|
\(U\+(?P<deprecated_unicode>[0-9a-fA-F]{4,})\s(?:\s|\w|-)+\)|
.*
)
)?
""",
re.VERBOSE,
)
class Deprecation(Enum):
NONE = "none"
"No deprecation"
EXPLICIT = "explicit"
"Explicit deprecation in comment: /* deprecated */"
IMPLICIT = "implicit"
"""
Implicit deprecation: the keysym has already been defined with a previous
name, and the present name has not been declared explicitly as an alias.
"""
@dataclass
class Keysym:
name: str
value: int
deprecated: Deprecation
alias: bool
def parse_keysyms(path: Path) -> Iterator[Keysym]:
with path.open("rt", encoding="utf-8") as fd:
for line in fd:
if m := KEYSYM_ENTRY_PATTERN.match(line):
yield Keysym(
name=m.group("name"),
value=int(m.group("value"), 16),
deprecated=Deprecation.EXPLICIT
if m.group("deprecated") or m.group("deprecated_unicode")
else Deprecation.NONE,
alias="alias for" in line.casefold()
or m.group("unicode_alt_semantics"),
)
def get_keysyms(path: Path) -> dict[int, list[Keysym]]:
keysyms: DefaultDict[int, list[Keysym]] = defaultdict(list)
for keysym in parse_keysyms(path):
if (
(ks := keysyms.get(keysym.value))
and keysym.deprecated is Deprecation.NONE
and not keysym.alias
# deal with first name being deprecated
and any(k.deprecated is Deprecation.NONE for k in ks)
):
keysym.deprecated = Deprecation.IMPLICIT
keysyms[keysym.value].append(keysym)
return keysyms
keysyms_by_value = get_keysyms(args.c_header)
entries = tuple(itertools.chain.from_iterable(keysyms_by_value.values()))
# Sort based on the keysym name:
# 1. Sort by the casefolded name: e.g. kana_ya < kana_YO.
# 2. If same casefolded name, then sort by cased name, i.e for
# ASCII: upper before lower: e.g kana_YA < kana_ya.
# E.g. kana_YA < kana_ya < kana_YO < kana_yo
# WARNING: this sort must not be changed, as some functions e.g.
# xkb_keysym_from_name rely on upper case variant occuring first.
entries_isorted = sorted(entries, key=lambda e: (e.name.casefold(), e.name))
# Sort based on keysym value. Sort is stable so in case of duplicate, the first
# keysym occurence stays first.
entries_kssorted = sorted(entries, key=lambda e: e.value)
print(
f"""
/**
* This file comes from libxkbcommon and was generated by {SCRIPT.name}
* You can always fetch the latest version from:
* https://raw.github.com/xkbcommon/libxkbcommon/master/src/ks_tables.h
*/
#pragma once
"""
)
entry_offsets: dict[str, int] = {}
UINT16_MAX = (1 << 16) - 1
DEPRECATED_KEYSYM = UINT16_MAX
MAX_EXPLICIT_DEPRECATED_ALIAS_INDEX_LOG2 = 8
MAX_EXPLICIT_DEPRECATED_ALIAS_INDEX = 1 << MAX_EXPLICIT_DEPRECATED_ALIAS_INDEX_LOG2
MAX_EXPLICIT_DEPRECATED_ALIAS_COUNT_LOG2 = 4
MAX_EXPLICIT_DEPRECATED_ALIAS_COUNT = 1 << MAX_EXPLICIT_DEPRECATED_ALIAS_COUNT_LOG2
MAX_OFFSET = DEPRECATED_KEYSYM - 1
print(
"""
#include <stddef.h>
#include <stdint.h>
#include "xkbcommon/xkbcommon.h"
#ifdef __GNUC__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Woverlength-strings"
#endif
static const char *keysym_names =
""".strip()
)
offs = 0
for keysym in entries_isorted:
if offs >= MAX_OFFSET:
raise ValueError(f"Offset must be kept under {MAX_OFFSET}, got: {offs}.")
entry_offsets[keysym.name] = offs
print(f' "{keysym.name}\\0"')
offs += len(keysym.name) + 1
print(
"""
;
#ifdef __GNUC__
#pragma GCC diagnostic pop
#endif
""".strip()
)
template = r"""
static const uint16_t keysym_name_G[] = {
$G
};
static size_t
keysym_name_hash_f(const char *key, const char *T)
{
size_t sum = 0;
for (size_t i = 0; key[i] != '\0'; i++)
sum += (size_t) (T[i % $NS] * key[i]);
return sum % $NG;
}
static inline size_t
keysym_name_perfect_hash(const char *key)
{
return (
keysym_name_G[keysym_name_hash_f(key, "$S1")] +
keysym_name_G[keysym_name_hash_f(key, "$S2")]
) % $NG;
}
"""
print(
perfect_hash.generate_code(
keys=[keysym.name for keysym in entries_isorted],
template=template,
)
)
print(
"""
struct name_keysym {
xkb_keysym_t keysym;
uint16_t offset;
};\n"""
)
def print_entries(entries: Iterable[Keysym]):
for entry in entries:
print(
" {{ 0x{value:08x}, {offs} }}, /* {name} */".format(
offs=entry_offsets[entry.name], value=entry.value, name=entry.name
)
)
print("static const struct name_keysym name_to_keysym[] = {")
print_entries(entries_isorted)
print("};\n")
# *.sort() is stable so we always get the first keysym for duplicate
print("static const struct name_keysym keysym_to_name[] = {")
print_entries(
next(g[1]) for g in itertools.groupby(entries_kssorted, key=lambda e: e.value)
)
print("};\n")
def make_deprecated_entry(
value,
keysyms: list[Keysym],
entry_offsets: dict[str, int],
explicit_deprecated_aliases_index: int,
) -> tuple[str | None, tuple[int, ...]]:
assert keysyms
non_deprecated_ks = tuple(k for k in keysyms if k.deprecated is Deprecation.NONE)
explicit_deprecated_aliases: tuple[int, ...] = ()
if non_deprecated_ks:
# Keysym is not deprecated. Check if none of its aliases are.
if len(keysyms) == 1 or all(
ks.alias and ks.deprecated is Deprecation.NONE for ks in keysyms[1:]
):
return None, ()
ref = non_deprecated_ks[0].name
canonical_name = f"Reference: {ref}. "
assert ref in entry_offsets
canonical_index = str(entry_offsets[ref])
deprecated_ks = tuple(k for k in keysyms if k not in non_deprecated_ks)
if any(ks.alias and ks.deprecated is Deprecation.NONE for ks in keysyms[1:]):
# keysym has both explicit and deprecated aliases
explicit_deprecated_aliases = tuple(
entry_offsets[ks.name]
for ks in keysyms[1:]
if ks.deprecated is not Deprecation.NONE
)
assert (
explicit_deprecated_aliases_index < MAX_EXPLICIT_DEPRECATED_ALIAS_INDEX
)
assert (
len(explicit_deprecated_aliases) < MAX_EXPLICIT_DEPRECATED_ALIAS_COUNT
)
else:
explicit_deprecated_aliases_index = 0
else:
# Keysym is deprecated
canonical_name = ""
canonical_index = "DEPRECATED_KEYSYM"
deprecated_ks = keysyms
explicit_deprecated_aliases_index = 0
if non_deprecated_ks[1:]:
non_deprecated = (
"Non deprecated aliases: "
+ ", ".join(ks.name for ks in non_deprecated_ks[1:])
+ ". "
)
else:
non_deprecated = ""
deprecated = ", ".join(ks.name for ks in deprecated_ks)
comment = f"{canonical_name}{non_deprecated}Deprecated: {deprecated}"
return (
f" {{ 0x{value:0>8x}, {canonical_index: <17}, {explicit_deprecated_aliases_index}, {len(explicit_deprecated_aliases)} }}, /* {comment} */",
explicit_deprecated_aliases,
)
def generate_deprecated_keysyms(
keysyms_by_value: dict[int, list[Keysym]], entry_offsets: dict[str, int]
) -> Generator[tuple[int, ...], None, None]:
explicit_deprecated_aliases_index = 0
for value, keysyms in sorted(keysyms_by_value.items(), key=lambda e: e[0]):
assert keysyms
c_entry, explicit_deprecated_aliases = make_deprecated_entry(
value, keysyms, entry_offsets, explicit_deprecated_aliases_index
)
if c_entry is not None:
print(c_entry)
if explicit_deprecated_aliases:
yield explicit_deprecated_aliases
explicit_deprecated_aliases_index += len(explicit_deprecated_aliases)
def generate_mixed_aliases(aliases: Iterable[Iterable[int]]):
for xs in aliases:
for x in xs:
print(f" {x},")
print(f"#define DEPRECATED_KEYSYM 0x{DEPRECATED_KEYSYM:x}")
# NOTE: Alternative implementation, useful the day the indexes do not fit uint16_t.
# print(f"""
# struct deprecated_keysym {{
# xkb_keysym_t keysym;
# union {{
# uint32_t offset;
# struct {{
# uint32_t offset:{MAX_OFFSET_LOG2};
# /* Explicit deprecated aliases start index & count */
# uint8_t explicit_index:{MAX_EXPLICIT_DEPRECATED_ALIAS_INDEX_LOG2};
# uint8_t explicit_count:{MAX_EXPLICIT_DEPRECATED_ALIAS_COUNT_LOG2};
# }} details;
# }};
# }};
# """)
print("""
struct deprecated_keysym {
xkb_keysym_t keysym;
uint16_t offset;
/* Explicit deprecated aliases start index & count */
uint8_t explicit_index;
uint8_t explicit_count;
};
""")
print("static const struct deprecated_keysym deprecated_keysyms[] = {")
explicit_deprecated_aliases = tuple(
generate_deprecated_keysyms(keysyms_by_value, entry_offsets)
)
print("};\n")
print("static const uint32_t explicit_deprecated_aliases[] = {")
generate_mixed_aliases(explicit_deprecated_aliases)
print("};")
print(f"max name offset: {max(entry_offsets.values())}", file=sys.stderr)
# Check that the keywords of our XKB parser that clash with keysyms are handled properly
def parse_gperf_keywords(path: Path) -> Iterator[str]:
with path.open("rt", encoding="utf-8") as fd:
in_keyword_section = False
for line in fd:
if line.startswith(r"%%"):
# This is a boundary of the keywords section
if in_keyword_section:
break
in_keyword_section = True
elif in_keyword_section:
# Parse the keywords
keyword, *_ = line.split(",")
yield keyword.strip().casefold()
# Skip any line until we reach the keywords
else:
raise ValueError("Parse error: keywords section boundary not found")
SUPPORTED_KEYWORDS_CLASHES = {"section"}
UNSUPPORTED_KEYWORDS_CLASHES = frozenset(parse_gperf_keywords(args.gperf)).difference(
SUPPORTED_KEYWORDS_CLASHES
)
expected_clashes: set[str] = set()
errors = 0
for entry in entries:
if entry.name.casefold() in UNSUPPORTED_KEYWORDS_CLASHES:
print(
f"ERROR: keysym “{entry.name}” (0x{entry.value:0>4x}) clashes with keywords",
"and cannot be parsed properly.",
file=sys.stderr,
)
errors += 1
elif (lower := entry.name.lower()) in SUPPORTED_KEYWORDS_CLASHES:
if not entry.name.islower():
# Keywords’s atoms are registered in *lower* case, so the keysym will be
# replaced by the keysym with the corresponding name, but they may not match.
entry2: Keysym = Keysym("NoSymbol", 0, Deprecation.NONE, False)
if any(
e.name == lower
for e in keysyms_by_value[entry.value]
if e.name != entry.value
):
# There is a keysym in lower case that is an alias
print(
f"WARNING: keysym “{entry.name}”",
f"will be parsed as “{lower}” (expected)",
file=sys.stderr,
)
else:
# Lookup the keysym mismatch
for e in entries:
if e.name == lower:
entry2 = e
break
print(
f"ERROR: keysym “{entry.name}” (0x{entry.value:0>4x})",
r"clashes with keywords and will be replaced by",
f"“{entry2.name}” (0x{entry2.value:0>4x}).",
file=sys.stderr,
)
errors += 1
else:
print(
f"WARNING: keysym “{entry.name}” clashing with keywords (expected)",
file=sys.stderr,
)
expected_clashes.add(entry.name)
if diff := SUPPORTED_KEYWORDS_CLASHES.difference(expected_clashes):
print(f"ERROR: Unexpected missing clashing keysyms: {diff}", file=sys.stderr)
errors += 1
if errors:
print(
f" {errors} ERRORS ".center(80, "-"),
"Please update the parser file `parser.y` to handle keysyms causing clashes.",
"The relevant entries are:",
"- Keysym",
"- Element (for modmap, parsed via: Expr -> Term -> Lhs -> FieldSpec -> Element)",
file=sys.stderr,
sep="\n",
)
exit(1)
