1From 0514a1cb2f45ab6dd814118780d56a713f4925a2 Mon Sep 17 00:00:00 2001
2From: Niels Dossche <7771979+nielsdos@users.noreply.github.com>
3Date: Thu, 24 Aug 2023 22:57:48 +0200
4Subject: [PATCH 3/6] Patch utilities and data structure to be able to generate
5 smaller lookup tables
6
7Changed the generation script to check if everything fits in 32-bits.
8And change the actual field types to 32-bits. This decreases the hash
9tables in size.
10---
11 source/lexbor/core/shs.h             |  4 ++--
12 utils/lexbor/encoding/multi-byte.py  |  2 +-
13 utils/lexbor/encoding/single-byte.py |  2 +-
14 utils/lexbor/lexbor/LXB.py           | 12 +++++++++---
15 4 files changed, 13 insertions(+), 7 deletions(-)
16
17diff --git a/source/lexbor/core/shs.h b/source/lexbor/core/shs.h
18index 7a63a07..c84dfaa 100644
19--- a/source/lexbor/core/shs.h
20+++ b/source/lexbor/core/shs.h
21@@ -27,9 +27,9 @@ lexbor_shs_entry_t;
22
23 typedef struct {
24     uint32_t key;
25-    void     *value;
26+    uint32_t value;
27
28-    size_t   next;
29+    uint32_t next;
30 }
31 lexbor_shs_hash_t;
32
33diff --git a/utils/lexbor/encoding/multi-byte.py b/utils/lexbor/encoding/multi-byte.py
34index ff9ddf1..f8af2d4 100755
35--- a/utils/lexbor/encoding/multi-byte.py
36+++ b/utils/lexbor/encoding/multi-byte.py
37@@ -151,7 +151,7 @@ class MultiByte:
38
39                 key_id = entries[1].decode('utf-8')
40
41-                hash_key.append(key_id, '(void *) {}'.format(idx))
42+                hash_key.append(key_id, idx)
43
44         return hash_key.create(rate = 1)
45
46diff --git a/utils/lexbor/encoding/single-byte.py b/utils/lexbor/encoding/single-byte.py
47index 9a85d54..ec2023c 100755
48--- a/utils/lexbor/encoding/single-byte.py
49+++ b/utils/lexbor/encoding/single-byte.py
50@@ -128,7 +128,7 @@ class SingleByte:
51                 entries = values[idx]
52                 key_id = entries[1].decode('utf-8')
53
54-                hash_key.append(key_id, '(void *) {}'.format(idx + 0x80))
55+                hash_key.append(key_id, idx + 0x80)
56
57         return hash_key.create(rate = 1)
58
59diff --git a/utils/lexbor/lexbor/LXB.py b/utils/lexbor/lexbor/LXB.py
60index 3e75812..2370c66 100755
61--- a/utils/lexbor/lexbor/LXB.py
62+++ b/utils/lexbor/lexbor/LXB.py
63@@ -94,7 +94,7 @@ class HashKey:
64     def append(self, key_id, value):
65         self.buffer.append([self.hash_id(int(key_id, 0)), value])
66
67-    def create(self, terminate_value = '{0, NULL, 0}', rate = 2, is_const = True, data_before = None):
68+    def create(self, terminate_value = '{0, 0, 0}', rate = 2, is_const = True, data_before = None):
69         test = self.test(int(self.max_table_size / 1.2), int(self.max_table_size * 1.2))
70
71         rate_dn = rate - 1
72@@ -142,9 +142,12 @@ class HashKey:
73             entry = table[idx]
74
75             if entry:
76+                assert entry[0] < 2**32
77+                assert entry[1] < 2**32
78+                assert entry[2] < 2**32
79                 result.append("{{{}, {}, {}}},".format(entry[0], entry[1], entry[2]))
80             else:
81-                result.append("{0, NULL, 0},")
82+                result.append("{0, 0, 0},")
83
84             if int(idx) % rate == rate_dn:
85                 result.append("\n    ")
86@@ -154,9 +157,12 @@ class HashKey:
87         if len(table):
88             entry = table[-1]
89             if entry:
90+                assert entry[0] < 2**32
91+                assert entry[1] < 2**32
92+                assert entry[2] < 2**32
93                 result.append("{{{}, {}, {}}}\n".format(entry[0], entry[1], entry[2]))
94             else:
95-                result.append("{0, NULL, 0}\n")
96+                result.append("{0, 0, 0}\n")
97
98         result.append("};")
99
100--
1012.44.0
102
103