1 /*************************************************
2 *      Perl-Compatible Regular Expressions       *
3 *************************************************/
4 
5 /* PCRE is a library of functions to support regular expressions whose syntax
6 and semantics are as close as possible to those of the Perl 5 language.
7 
8                        Written by Philip Hazel
9                     This module by Zoltan Herczeg
10      Original API code Copyright (c) 1997-2012 University of Cambridge
11           New API code Copyright (c) 2016-2019 University of Cambridge
12 
13 -----------------------------------------------------------------------------
14 Redistribution and use in source and binary forms, with or without
15 modification, are permitted provided that the following conditions are met:
16 
17     * Redistributions of source code must retain the above copyright notice,
18       this list of conditions and the following disclaimer.
19 
20     * Redistributions in binary form must reproduce the above copyright
21       notice, this list of conditions and the following disclaimer in the
22       documentation and/or other materials provided with the distribution.
23 
24     * Neither the name of the University of Cambridge nor the names of its
25       contributors may be used to endorse or promote products derived from
26       this software without specific prior written permission.
27 
28 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
29 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
32 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
33 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
34 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
35 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
36 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38 POSSIBILITY OF SUCH DAMAGE.
39 -----------------------------------------------------------------------------
40 */
41 
42 #if !(defined SUPPORT_VALGRIND)
43 
44 #if ((defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) \
45      || (defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X))
46 
47 typedef enum {
48   vector_compare_match1,
49   vector_compare_match1i,
50   vector_compare_match2,
51 } vector_compare_type;
52 
max_fast_forward_char_pair_offset(void)53 static SLJIT_INLINE sljit_u32 max_fast_forward_char_pair_offset(void)
54 {
55 #if PCRE2_CODE_UNIT_WIDTH == 8
56 return 15;
57 #elif PCRE2_CODE_UNIT_WIDTH == 16
58 return 7;
59 #elif PCRE2_CODE_UNIT_WIDTH == 32
60 return 3;
61 #else
62 #error "Unsupported unit width"
63 #endif
64 }
65 
66 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
jump_if_utf_char_start(struct sljit_compiler * compiler,sljit_s32 reg)67 static struct sljit_jump *jump_if_utf_char_start(struct sljit_compiler *compiler, sljit_s32 reg)
68 {
69 #if PCRE2_CODE_UNIT_WIDTH == 8
70 OP2(SLJIT_AND, reg, 0, reg, 0, SLJIT_IMM, 0xc0);
71 return CMP(SLJIT_NOT_EQUAL, reg, 0, SLJIT_IMM, 0x80);
72 #elif PCRE2_CODE_UNIT_WIDTH == 16
73 OP2(SLJIT_AND, reg, 0, reg, 0, SLJIT_IMM, 0xfc00);
74 return CMP(SLJIT_NOT_EQUAL, reg, 0, SLJIT_IMM, 0xdc00);
75 #else
76 #error "Unknown code width"
77 #endif
78 }
79 #endif
80 
81 #endif /* SLJIT_CONFIG_X86 || SLJIT_CONFIG_S390X */
82 
83 #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)
84 
character_to_int32(PCRE2_UCHAR chr)85 static sljit_s32 character_to_int32(PCRE2_UCHAR chr)
86 {
87 sljit_u32 value = chr;
88 #if PCRE2_CODE_UNIT_WIDTH == 8
89 #define SSE2_COMPARE_TYPE_INDEX 0
90 return (sljit_s32)((value << 24) | (value << 16) | (value << 8) | value);
91 #elif PCRE2_CODE_UNIT_WIDTH == 16
92 #define SSE2_COMPARE_TYPE_INDEX 1
93 return (sljit_s32)((value << 16) | value);
94 #elif PCRE2_CODE_UNIT_WIDTH == 32
95 #define SSE2_COMPARE_TYPE_INDEX 2
96 return (sljit_s32)(value);
97 #else
98 #error "Unsupported unit width"
99 #endif
100 }
101 
load_from_mem_sse2(struct sljit_compiler * compiler,sljit_s32 dst_xmm_reg,sljit_s32 src_general_reg,sljit_s8 offset)102 static void load_from_mem_sse2(struct sljit_compiler *compiler, sljit_s32 dst_xmm_reg, sljit_s32 src_general_reg, sljit_s8 offset)
103 {
104 sljit_u8 instruction[5];
105 
106 SLJIT_ASSERT(dst_xmm_reg < 8);
107 SLJIT_ASSERT(src_general_reg < 8);
108 
109 /* MOVDQA xmm1, xmm2/m128 */
110 instruction[0] = ((sljit_u8)offset & 0xf) == 0 ? 0x66 : 0xf3;
111 instruction[1] = 0x0f;
112 instruction[2] = 0x6f;
113 
114 if (offset == 0)
115   {
116   instruction[3] = (dst_xmm_reg << 3) | src_general_reg;
117   sljit_emit_op_custom(compiler, instruction, 4);
118   return;
119   }
120 
121 instruction[3] = 0x40 | (dst_xmm_reg << 3) | src_general_reg;
122 instruction[4] = (sljit_u8)offset;
123 sljit_emit_op_custom(compiler, instruction, 5);
124 }
125 
fast_forward_char_pair_sse2_compare(struct sljit_compiler * compiler,vector_compare_type compare_type,int step,sljit_s32 dst_ind,sljit_s32 cmp1_ind,sljit_s32 cmp2_ind,sljit_s32 tmp_ind)126 static void fast_forward_char_pair_sse2_compare(struct sljit_compiler *compiler, vector_compare_type compare_type,
127   int step, sljit_s32 dst_ind, sljit_s32 cmp1_ind, sljit_s32 cmp2_ind, sljit_s32 tmp_ind)
128 {
129 sljit_u8 instruction[4];
130 instruction[0] = 0x66;
131 instruction[1] = 0x0f;
132 
133 SLJIT_ASSERT(step >= 0 && step <= 3);
134 
135 if (compare_type != vector_compare_match2)
136   {
137   if (step == 0)
138     {
139     if (compare_type == vector_compare_match1i)
140       {
141       /* POR xmm1, xmm2/m128 */
142       /* instruction[0] = 0x66; */
143       /* instruction[1] = 0x0f; */
144       instruction[2] = 0xeb;
145       instruction[3] = 0xc0 | (dst_ind << 3) | cmp2_ind;
146       sljit_emit_op_custom(compiler, instruction, 4);
147       }
148     return;
149     }
150 
151   if (step != 2)
152     return;
153 
154   /* PCMPEQB/W/D xmm1, xmm2/m128 */
155   /* instruction[0] = 0x66; */
156   /* instruction[1] = 0x0f; */
157   instruction[2] = 0x74 + SSE2_COMPARE_TYPE_INDEX;
158   instruction[3] = 0xc0 | (dst_ind << 3) | cmp1_ind;
159   sljit_emit_op_custom(compiler, instruction, 4);
160   return;
161   }
162 
163 switch (step)
164   {
165   case 0:
166   /* MOVDQA xmm1, xmm2/m128 */
167   /* instruction[0] = 0x66; */
168   /* instruction[1] = 0x0f; */
169   instruction[2] = 0x6f;
170   instruction[3] = 0xc0 | (tmp_ind << 3) | dst_ind;
171   sljit_emit_op_custom(compiler, instruction, 4);
172   return;
173 
174   case 1:
175   /* PCMPEQB/W/D xmm1, xmm2/m128 */
176   /* instruction[0] = 0x66; */
177   /* instruction[1] = 0x0f; */
178   instruction[2] = 0x74 + SSE2_COMPARE_TYPE_INDEX;
179   instruction[3] = 0xc0 | (dst_ind << 3) | cmp1_ind;
180   sljit_emit_op_custom(compiler, instruction, 4);
181   return;
182 
183   case 2:
184   /* PCMPEQB/W/D xmm1, xmm2/m128 */
185   /* instruction[0] = 0x66; */
186   /* instruction[1] = 0x0f; */
187   instruction[2] = 0x74 + SSE2_COMPARE_TYPE_INDEX;
188   instruction[3] = 0xc0 | (tmp_ind << 3) | cmp2_ind;
189   sljit_emit_op_custom(compiler, instruction, 4);
190   return;
191 
192   case 3:
193   /* POR xmm1, xmm2/m128 */
194   /* instruction[0] = 0x66; */
195   /* instruction[1] = 0x0f; */
196   instruction[2] = 0xeb;
197   instruction[3] = 0xc0 | (dst_ind << 3) | tmp_ind;
198   sljit_emit_op_custom(compiler, instruction, 4);
199   return;
200   }
201 }
202 
203 #define JIT_HAS_FAST_FORWARD_CHAR_SIMD (sljit_has_cpu_feature(SLJIT_HAS_SSE2))
204 
fast_forward_char_simd(compiler_common * common,PCRE2_UCHAR char1,PCRE2_UCHAR char2,sljit_s32 offset)205 static void fast_forward_char_simd(compiler_common *common, PCRE2_UCHAR char1, PCRE2_UCHAR char2, sljit_s32 offset)
206 {
207 DEFINE_COMPILER;
208 sljit_u8 instruction[8];
209 struct sljit_label *start;
210 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
211 struct sljit_label *restart;
212 #endif
213 struct sljit_jump *quit;
214 struct sljit_jump *partial_quit[2];
215 vector_compare_type compare_type = vector_compare_match1;
216 sljit_s32 tmp1_reg_ind = sljit_get_register_index(TMP1);
217 sljit_s32 str_ptr_reg_ind = sljit_get_register_index(STR_PTR);
218 sljit_s32 data_ind = 0;
219 sljit_s32 tmp_ind = 1;
220 sljit_s32 cmp1_ind = 2;
221 sljit_s32 cmp2_ind = 3;
222 sljit_u32 bit = 0;
223 int i;
224 
225 SLJIT_UNUSED_ARG(offset);
226 
227 if (char1 != char2)
228   {
229   bit = char1 ^ char2;
230   compare_type = vector_compare_match1i;
231 
232   if (!is_powerof2(bit))
233     {
234     bit = 0;
235     compare_type = vector_compare_match2;
236     }
237   }
238 
239 partial_quit[0] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
240 if (common->mode == PCRE2_JIT_COMPLETE)
241   add_jump(compiler, &common->failed_match, partial_quit[0]);
242 
243 /* First part (unaligned start) */
244 
245 OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char1 | bit));
246 
247 SLJIT_ASSERT(tmp1_reg_ind < 8);
248 
249 /* MOVD xmm, r/m32 */
250 instruction[0] = 0x66;
251 instruction[1] = 0x0f;
252 instruction[2] = 0x6e;
253 instruction[3] = 0xc0 | (cmp1_ind << 3) | tmp1_reg_ind;
254 sljit_emit_op_custom(compiler, instruction, 4);
255 
256 if (char1 != char2)
257   {
258   OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(bit != 0 ? bit : char2));
259 
260   /* MOVD xmm, r/m32 */
261   instruction[3] = 0xc0 | (cmp2_ind << 3) | tmp1_reg_ind;
262   sljit_emit_op_custom(compiler, instruction, 4);
263   }
264 
265 OP1(SLJIT_MOV, TMP2, 0, STR_PTR, 0);
266 
267 /* PSHUFD xmm1, xmm2/m128, imm8 */
268 /* instruction[0] = 0x66; */
269 /* instruction[1] = 0x0f; */
270 instruction[2] = 0x70;
271 instruction[3] = 0xc0 | (cmp1_ind << 3) | cmp1_ind;
272 instruction[4] = 0;
273 sljit_emit_op_custom(compiler, instruction, 5);
274 
275 if (char1 != char2)
276   {
277   /* PSHUFD xmm1, xmm2/m128, imm8 */
278   instruction[3] = 0xc0 | (cmp2_ind << 3) | cmp2_ind;
279   sljit_emit_op_custom(compiler, instruction, 5);
280   }
281 
282 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
283 restart = LABEL();
284 #endif
285 OP2(SLJIT_AND, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, ~0xf);
286 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xf);
287 
288 load_from_mem_sse2(compiler, data_ind, str_ptr_reg_ind, 0);
289 for (i = 0; i < 4; i++)
290   fast_forward_char_pair_sse2_compare(compiler, compare_type, i, data_ind, cmp1_ind, cmp2_ind, tmp_ind);
291 
292 /* PMOVMSKB reg, xmm */
293 /* instruction[0] = 0x66; */
294 /* instruction[1] = 0x0f; */
295 instruction[2] = 0xd7;
296 instruction[3] = 0xc0 | (tmp1_reg_ind << 3) | data_ind;
297 sljit_emit_op_custom(compiler, instruction, 4);
298 
299 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
300 OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, TMP2, 0);
301 
302 quit = CMP(SLJIT_NOT_ZERO, TMP1, 0, SLJIT_IMM, 0);
303 
304 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
305 
306 /* Second part (aligned) */
307 start = LABEL();
308 
309 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 16);
310 
311 partial_quit[1] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
312 if (common->mode == PCRE2_JIT_COMPLETE)
313   add_jump(compiler, &common->failed_match, partial_quit[1]);
314 
315 load_from_mem_sse2(compiler, data_ind, str_ptr_reg_ind, 0);
316 for (i = 0; i < 4; i++)
317   fast_forward_char_pair_sse2_compare(compiler, compare_type, i, data_ind, cmp1_ind, cmp2_ind, tmp_ind);
318 
319 /* PMOVMSKB reg, xmm */
320 /* instruction[0] = 0x66; */
321 /* instruction[1] = 0x0f; */
322 instruction[2] = 0xd7;
323 instruction[3] = 0xc0 | (tmp1_reg_ind << 3) | data_ind;
324 sljit_emit_op_custom(compiler, instruction, 4);
325 
326 CMPTO(SLJIT_ZERO, TMP1, 0, SLJIT_IMM, 0, start);
327 
328 JUMPHERE(quit);
329 
330 /* BSF r32, r/m32 */
331 instruction[0] = 0x0f;
332 instruction[1] = 0xbc;
333 instruction[2] = 0xc0 | (tmp1_reg_ind << 3) | tmp1_reg_ind;
334 sljit_emit_op_custom(compiler, instruction, 3);
335 
336 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
337 
338 if (common->mode != PCRE2_JIT_COMPLETE)
339   {
340   JUMPHERE(partial_quit[0]);
341   JUMPHERE(partial_quit[1]);
342   OP2U(SLJIT_SUB | SLJIT_SET_GREATER, STR_PTR, 0, STR_END, 0);
343   CMOV(SLJIT_GREATER, STR_PTR, STR_END, 0);
344   }
345 else
346   add_jump(compiler, &common->failed_match, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
347 
348 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
349 if (common->utf && offset > 0)
350   {
351   SLJIT_ASSERT(common->mode == PCRE2_JIT_COMPLETE);
352 
353   OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-offset));
354 
355   quit = jump_if_utf_char_start(compiler, TMP1);
356 
357   OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
358   add_jump(compiler, &common->failed_match, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
359   OP1(SLJIT_MOV, TMP2, 0, STR_PTR, 0);
360   JUMPTO(SLJIT_JUMP, restart);
361 
362   JUMPHERE(quit);
363   }
364 #endif
365 }
366 
367 #define JIT_HAS_FAST_REQUESTED_CHAR_SIMD (sljit_has_cpu_feature(SLJIT_HAS_SSE2))
368 
fast_requested_char_simd(compiler_common * common,PCRE2_UCHAR char1,PCRE2_UCHAR char2)369 static jump_list *fast_requested_char_simd(compiler_common *common, PCRE2_UCHAR char1, PCRE2_UCHAR char2)
370 {
371 DEFINE_COMPILER;
372 sljit_u8 instruction[8];
373 struct sljit_label *start;
374 struct sljit_jump *quit;
375 jump_list *not_found = NULL;
376 vector_compare_type compare_type = vector_compare_match1;
377 sljit_s32 tmp1_reg_ind = sljit_get_register_index(TMP1);
378 sljit_s32 str_ptr_reg_ind = sljit_get_register_index(STR_PTR);
379 sljit_s32 data_ind = 0;
380 sljit_s32 tmp_ind = 1;
381 sljit_s32 cmp1_ind = 2;
382 sljit_s32 cmp2_ind = 3;
383 sljit_u32 bit = 0;
384 int i;
385 
386 if (char1 != char2)
387   {
388   bit = char1 ^ char2;
389   compare_type = vector_compare_match1i;
390 
391   if (!is_powerof2(bit))
392     {
393     bit = 0;
394     compare_type = vector_compare_match2;
395     }
396   }
397 
398 add_jump(compiler, &not_found, CMP(SLJIT_GREATER_EQUAL, TMP1, 0, STR_END, 0));
399 OP1(SLJIT_MOV, TMP2, 0, TMP1, 0);
400 OP1(SLJIT_MOV, TMP3, 0, STR_PTR, 0);
401 
402 /* First part (unaligned start) */
403 
404 OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char1 | bit));
405 
406 SLJIT_ASSERT(tmp1_reg_ind < 8);
407 
408 /* MOVD xmm, r/m32 */
409 instruction[0] = 0x66;
410 instruction[1] = 0x0f;
411 instruction[2] = 0x6e;
412 instruction[3] = 0xc0 | (cmp1_ind << 3) | tmp1_reg_ind;
413 sljit_emit_op_custom(compiler, instruction, 4);
414 
415 if (char1 != char2)
416   {
417   OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(bit != 0 ? bit : char2));
418 
419   /* MOVD xmm, r/m32 */
420   instruction[3] = 0xc0 | (cmp2_ind << 3) | tmp1_reg_ind;
421   sljit_emit_op_custom(compiler, instruction, 4);
422   }
423 
424 OP1(SLJIT_MOV, STR_PTR, 0, TMP2, 0);
425 
426 /* PSHUFD xmm1, xmm2/m128, imm8 */
427 /* instruction[0] = 0x66; */
428 /* instruction[1] = 0x0f; */
429 instruction[2] = 0x70;
430 instruction[3] = 0xc0 | (cmp1_ind << 3) | cmp1_ind;
431 instruction[4] = 0;
432 sljit_emit_op_custom(compiler, instruction, 5);
433 
434 if (char1 != char2)
435   {
436   /* PSHUFD xmm1, xmm2/m128, imm8 */
437   instruction[3] = 0xc0 | (cmp2_ind << 3) | cmp2_ind;
438   sljit_emit_op_custom(compiler, instruction, 5);
439   }
440 
441 OP2(SLJIT_AND, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, ~0xf);
442 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xf);
443 
444 load_from_mem_sse2(compiler, data_ind, str_ptr_reg_ind, 0);
445 for (i = 0; i < 4; i++)
446   fast_forward_char_pair_sse2_compare(compiler, compare_type, i, data_ind, cmp1_ind, cmp2_ind, tmp_ind);
447 
448 /* PMOVMSKB reg, xmm */
449 /* instruction[0] = 0x66; */
450 /* instruction[1] = 0x0f; */
451 instruction[2] = 0xd7;
452 instruction[3] = 0xc0 | (tmp1_reg_ind << 3) | data_ind;
453 sljit_emit_op_custom(compiler, instruction, 4);
454 
455 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
456 OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, TMP2, 0);
457 
458 quit = CMP(SLJIT_NOT_ZERO, TMP1, 0, SLJIT_IMM, 0);
459 
460 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
461 
462 /* Second part (aligned) */
463 start = LABEL();
464 
465 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 16);
466 
467 add_jump(compiler, &not_found, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
468 
469 load_from_mem_sse2(compiler, data_ind, str_ptr_reg_ind, 0);
470 for (i = 0; i < 4; i++)
471   fast_forward_char_pair_sse2_compare(compiler, compare_type, i, data_ind, cmp1_ind, cmp2_ind, tmp_ind);
472 
473 /* PMOVMSKB reg, xmm */
474 /* instruction[0] = 0x66; */
475 /* instruction[1] = 0x0f; */
476 instruction[2] = 0xd7;
477 instruction[3] = 0xc0 | (tmp1_reg_ind << 3) | data_ind;
478 sljit_emit_op_custom(compiler, instruction, 4);
479 
480 CMPTO(SLJIT_ZERO, TMP1, 0, SLJIT_IMM, 0, start);
481 
482 JUMPHERE(quit);
483 
484 /* BSF r32, r/m32 */
485 instruction[0] = 0x0f;
486 instruction[1] = 0xbc;
487 instruction[2] = 0xc0 | (tmp1_reg_ind << 3) | tmp1_reg_ind;
488 sljit_emit_op_custom(compiler, instruction, 3);
489 
490 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, STR_PTR, 0);
491 add_jump(compiler, &not_found, CMP(SLJIT_GREATER_EQUAL, TMP1, 0, STR_END, 0));
492 
493 OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0);
494 return not_found;
495 }
496 
497 #ifndef _WIN64
498 
499 #define JIT_HAS_FAST_FORWARD_CHAR_PAIR_SIMD (sljit_has_cpu_feature(SLJIT_HAS_SSE2))
500 
fast_forward_char_pair_simd(compiler_common * common,sljit_s32 offs1,PCRE2_UCHAR char1a,PCRE2_UCHAR char1b,sljit_s32 offs2,PCRE2_UCHAR char2a,PCRE2_UCHAR char2b)501 static void fast_forward_char_pair_simd(compiler_common *common, sljit_s32 offs1,
502   PCRE2_UCHAR char1a, PCRE2_UCHAR char1b, sljit_s32 offs2, PCRE2_UCHAR char2a, PCRE2_UCHAR char2b)
503 {
504 DEFINE_COMPILER;
505 sljit_u8 instruction[8];
506 vector_compare_type compare1_type = vector_compare_match1;
507 vector_compare_type compare2_type = vector_compare_match1;
508 sljit_u32 bit1 = 0;
509 sljit_u32 bit2 = 0;
510 sljit_u32 diff = IN_UCHARS(offs1 - offs2);
511 sljit_s32 tmp1_reg_ind = sljit_get_register_index(TMP1);
512 sljit_s32 tmp2_reg_ind = sljit_get_register_index(TMP2);
513 sljit_s32 str_ptr_reg_ind = sljit_get_register_index(STR_PTR);
514 sljit_s32 data1_ind = 0;
515 sljit_s32 data2_ind = 1;
516 sljit_s32 tmp1_ind = 2;
517 sljit_s32 tmp2_ind = 3;
518 sljit_s32 cmp1a_ind = 4;
519 sljit_s32 cmp1b_ind = 5;
520 sljit_s32 cmp2a_ind = 6;
521 sljit_s32 cmp2b_ind = 7;
522 struct sljit_label *start;
523 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
524 struct sljit_label *restart;
525 #endif
526 struct sljit_jump *jump[2];
527 int i;
528 
529 SLJIT_ASSERT(common->mode == PCRE2_JIT_COMPLETE && offs1 > offs2);
530 SLJIT_ASSERT(diff <= IN_UCHARS(max_fast_forward_char_pair_offset()));
531 SLJIT_ASSERT(tmp1_reg_ind < 8 && tmp2_reg_ind == 1);
532 
533 /* Initialize. */
534 if (common->match_end_ptr != 0)
535   {
536   OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr);
537   OP1(SLJIT_MOV, TMP3, 0, STR_END, 0);
538   OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(offs1 + 1));
539 
540   OP2U(SLJIT_SUB | SLJIT_SET_LESS, TMP1, 0, STR_END, 0);
541   CMOV(SLJIT_LESS, STR_END, TMP1, 0);
542   }
543 
544 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offs1));
545 add_jump(compiler, &common->failed_match, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
546 
547 /* MOVD xmm, r/m32 */
548 instruction[0] = 0x66;
549 instruction[1] = 0x0f;
550 instruction[2] = 0x6e;
551 
552 if (char1a == char1b)
553   OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char1a));
554 else
555   {
556   bit1 = char1a ^ char1b;
557   if (is_powerof2(bit1))
558     {
559     compare1_type = vector_compare_match1i;
560     OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char1a | bit1));
561     OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, character_to_int32(bit1));
562     }
563   else
564     {
565     compare1_type = vector_compare_match2;
566     bit1 = 0;
567     OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char1a));
568     OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, character_to_int32(char1b));
569     }
570   }
571 
572 instruction[3] = 0xc0 | (cmp1a_ind << 3) | tmp1_reg_ind;
573 sljit_emit_op_custom(compiler, instruction, 4);
574 
575 if (char1a != char1b)
576   {
577   instruction[3] = 0xc0 | (cmp1b_ind << 3) | tmp2_reg_ind;
578   sljit_emit_op_custom(compiler, instruction, 4);
579   }
580 
581 if (char2a == char2b)
582   OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char2a));
583 else
584   {
585   bit2 = char2a ^ char2b;
586   if (is_powerof2(bit2))
587     {
588     compare2_type = vector_compare_match1i;
589     OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char2a | bit2));
590     OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, character_to_int32(bit2));
591     }
592   else
593     {
594     compare2_type = vector_compare_match2;
595     bit2 = 0;
596     OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, character_to_int32(char2a));
597     OP1(SLJIT_MOV, TMP2, 0, SLJIT_IMM, character_to_int32(char2b));
598     }
599   }
600 
601 instruction[3] = 0xc0 | (cmp2a_ind << 3) | tmp1_reg_ind;
602 sljit_emit_op_custom(compiler, instruction, 4);
603 
604 if (char2a != char2b)
605   {
606   instruction[3] = 0xc0 | (cmp2b_ind << 3) | tmp2_reg_ind;
607   sljit_emit_op_custom(compiler, instruction, 4);
608   }
609 
610 /* PSHUFD xmm1, xmm2/m128, imm8 */
611 /* instruction[0] = 0x66; */
612 /* instruction[1] = 0x0f; */
613 instruction[2] = 0x70;
614 instruction[4] = 0;
615 
616 instruction[3] = 0xc0 | (cmp1a_ind << 3) | cmp1a_ind;
617 sljit_emit_op_custom(compiler, instruction, 5);
618 
619 if (char1a != char1b)
620   {
621   instruction[3] = 0xc0 | (cmp1b_ind << 3) | cmp1b_ind;
622   sljit_emit_op_custom(compiler, instruction, 5);
623   }
624 
625 instruction[3] = 0xc0 | (cmp2a_ind << 3) | cmp2a_ind;
626 sljit_emit_op_custom(compiler, instruction, 5);
627 
628 if (char2a != char2b)
629   {
630   instruction[3] = 0xc0 | (cmp2b_ind << 3) | cmp2b_ind;
631   sljit_emit_op_custom(compiler, instruction, 5);
632   }
633 
634 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
635 restart = LABEL();
636 #endif
637 
638 OP2(SLJIT_SUB, TMP1, 0, STR_PTR, 0, SLJIT_IMM, diff);
639 OP1(SLJIT_MOV, TMP2, 0, STR_PTR, 0);
640 OP2(SLJIT_AND, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, ~0xf);
641 
642 load_from_mem_sse2(compiler, data1_ind, str_ptr_reg_ind, 0);
643 
644 jump[0] = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, STR_PTR, 0);
645 
646 load_from_mem_sse2(compiler, data2_ind, str_ptr_reg_ind, -(sljit_s8)diff);
647 jump[1] = JUMP(SLJIT_JUMP);
648 
649 JUMPHERE(jump[0]);
650 
651 /* MOVDQA xmm1, xmm2/m128 */
652 /* instruction[0] = 0x66; */
653 /* instruction[1] = 0x0f; */
654 instruction[2] = 0x6f;
655 instruction[3] = 0xc0 | (data2_ind << 3) | data1_ind;
656 sljit_emit_op_custom(compiler, instruction, 4);
657 
658 /* PSLLDQ xmm1, imm8 */
659 /* instruction[0] = 0x66; */
660 /* instruction[1] = 0x0f; */
661 instruction[2] = 0x73;
662 instruction[3] = 0xc0 | (7 << 3) | data2_ind;
663 instruction[4] = diff;
664 sljit_emit_op_custom(compiler, instruction, 5);
665 
666 JUMPHERE(jump[1]);
667 
668 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, 0xf);
669 
670 for (i = 0; i < 4; i++)
671   {
672   fast_forward_char_pair_sse2_compare(compiler, compare2_type, i, data2_ind, cmp2a_ind, cmp2b_ind, tmp2_ind);
673   fast_forward_char_pair_sse2_compare(compiler, compare1_type, i, data1_ind, cmp1a_ind, cmp1b_ind, tmp1_ind);
674   }
675 
676 /* PAND xmm1, xmm2/m128 */
677 /* instruction[0] = 0x66; */
678 /* instruction[1] = 0x0f; */
679 instruction[2] = 0xdb;
680 instruction[3] = 0xc0 | (data1_ind << 3) | data2_ind;
681 sljit_emit_op_custom(compiler, instruction, 4);
682 
683 /* PMOVMSKB reg, xmm */
684 /* instruction[0] = 0x66; */
685 /* instruction[1] = 0x0f; */
686 instruction[2] = 0xd7;
687 instruction[3] = 0xc0 | (tmp1_reg_ind << 3) | 0;
688 sljit_emit_op_custom(compiler, instruction, 4);
689 
690 /* Ignore matches before the first STR_PTR. */
691 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
692 OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, TMP2, 0);
693 
694 jump[0] = CMP(SLJIT_NOT_ZERO, TMP1, 0, SLJIT_IMM, 0);
695 
696 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
697 
698 /* Main loop. */
699 start = LABEL();
700 
701 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 16);
702 add_jump(compiler, &common->failed_match, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
703 
704 load_from_mem_sse2(compiler, data1_ind, str_ptr_reg_ind, 0);
705 load_from_mem_sse2(compiler, data2_ind, str_ptr_reg_ind, -(sljit_s8)diff);
706 
707 for (i = 0; i < 4; i++)
708   {
709   fast_forward_char_pair_sse2_compare(compiler, compare1_type, i, data1_ind, cmp1a_ind, cmp1b_ind, tmp2_ind);
710   fast_forward_char_pair_sse2_compare(compiler, compare2_type, i, data2_ind, cmp2a_ind, cmp2b_ind, tmp1_ind);
711   }
712 
713 /* PAND xmm1, xmm2/m128 */
714 /* instruction[0] = 0x66; */
715 /* instruction[1] = 0x0f; */
716 instruction[2] = 0xdb;
717 instruction[3] = 0xc0 | (data1_ind << 3) | data2_ind;
718 sljit_emit_op_custom(compiler, instruction, 4);
719 
720 /* PMOVMSKB reg, xmm */
721 /* instruction[0] = 0x66; */
722 /* instruction[1] = 0x0f; */
723 instruction[2] = 0xd7;
724 instruction[3] = 0xc0 | (tmp1_reg_ind << 3) | 0;
725 sljit_emit_op_custom(compiler, instruction, 4);
726 
727 CMPTO(SLJIT_ZERO, TMP1, 0, SLJIT_IMM, 0, start);
728 
729 JUMPHERE(jump[0]);
730 
731 /* BSF r32, r/m32 */
732 instruction[0] = 0x0f;
733 instruction[1] = 0xbc;
734 instruction[2] = 0xc0 | (tmp1_reg_ind << 3) | tmp1_reg_ind;
735 sljit_emit_op_custom(compiler, instruction, 3);
736 
737 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
738 
739 add_jump(compiler, &common->failed_match, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
740 
741 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
742 if (common->utf)
743   {
744   OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-offs1));
745 
746   jump[0] = jump_if_utf_char_start(compiler, TMP1);
747 
748   OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
749   CMPTO(SLJIT_LESS, STR_PTR, 0, STR_END, 0, restart);
750 
751   add_jump(compiler, &common->failed_match, JUMP(SLJIT_JUMP));
752 
753   JUMPHERE(jump[0]);
754   }
755 #endif
756 
757 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offs1));
758 
759 if (common->match_end_ptr != 0)
760   OP1(SLJIT_MOV, STR_END, 0, TMP3, 0);
761 }
762 
763 #endif /* !_WIN64 */
764 
765 #undef SSE2_COMPARE_TYPE_INDEX
766 
767 #endif /* SLJIT_CONFIG_X86 */
768 
769 #if (defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64 && (defined __ARM_NEON || defined __ARM_NEON__))
770 
771 #include <arm_neon.h>
772 
773 typedef union {
774   unsigned int x;
775   struct { unsigned char c1, c2, c3, c4; } c;
776 } int_char;
777 
778 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
utf_continue(PCRE2_SPTR s)779 static SLJIT_INLINE int utf_continue(PCRE2_SPTR s)
780 {
781 #if PCRE2_CODE_UNIT_WIDTH == 8
782 return (*s & 0xc0) == 0x80;
783 #elif PCRE2_CODE_UNIT_WIDTH == 16
784 return (*s & 0xfc00) == 0xdc00;
785 #else
786 #error "Unknown code width"
787 #endif
788 }
789 #endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 */
790 
791 #if PCRE2_CODE_UNIT_WIDTH == 8
792 # define VECTOR_FACTOR 16
793 # define vect_t uint8x16_t
794 # define VLD1Q(X) vld1q_u8((sljit_u8 *)(X))
795 # define VCEQQ vceqq_u8
796 # define VORRQ vorrq_u8
797 # define VST1Q vst1q_u8
798 # define VDUPQ vdupq_n_u8
799 # define VEXTQ vextq_u8
800 # define VANDQ vandq_u8
801 typedef union {
802        uint8_t mem[16];
803        uint64_t dw[2];
804 } quad_word;
805 #elif PCRE2_CODE_UNIT_WIDTH == 16
806 # define VECTOR_FACTOR 8
807 # define vect_t uint16x8_t
808 # define VLD1Q(X) vld1q_u16((sljit_u16 *)(X))
809 # define VCEQQ vceqq_u16
810 # define VORRQ vorrq_u16
811 # define VST1Q vst1q_u16
812 # define VDUPQ vdupq_n_u16
813 # define VEXTQ vextq_u16
814 # define VANDQ vandq_u16
815 typedef union {
816        uint16_t mem[8];
817        uint64_t dw[2];
818 } quad_word;
819 #else
820 # define VECTOR_FACTOR 4
821 # define vect_t uint32x4_t
822 # define VLD1Q(X) vld1q_u32((sljit_u32 *)(X))
823 # define VCEQQ vceqq_u32
824 # define VORRQ vorrq_u32
825 # define VST1Q vst1q_u32
826 # define VDUPQ vdupq_n_u32
827 # define VEXTQ vextq_u32
828 # define VANDQ vandq_u32
829 typedef union {
830        uint32_t mem[4];
831        uint64_t dw[2];
832 } quad_word;
833 #endif
834 
835 #define FFCS
836 #include "pcre2_jit_neon_inc.h"
837 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
838 # define FF_UTF
839 # include "pcre2_jit_neon_inc.h"
840 # undef FF_UTF
841 #endif
842 #undef FFCS
843 
844 #define FFCS_2
845 #include "pcre2_jit_neon_inc.h"
846 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
847 # define FF_UTF
848 # include "pcre2_jit_neon_inc.h"
849 # undef FF_UTF
850 #endif
851 #undef FFCS_2
852 
853 #define FFCS_MASK
854 #include "pcre2_jit_neon_inc.h"
855 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
856 # define FF_UTF
857 # include "pcre2_jit_neon_inc.h"
858 # undef FF_UTF
859 #endif
860 #undef FFCS_MASK
861 
862 #define JIT_HAS_FAST_FORWARD_CHAR_SIMD 1
863 
fast_forward_char_simd(compiler_common * common,PCRE2_UCHAR char1,PCRE2_UCHAR char2,sljit_s32 offset)864 static void fast_forward_char_simd(compiler_common *common, PCRE2_UCHAR char1, PCRE2_UCHAR char2, sljit_s32 offset)
865 {
866 DEFINE_COMPILER;
867 int_char ic;
868 struct sljit_jump *partial_quit;
869 /* Save temporary registers. */
870 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, STR_PTR, 0);
871 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS1, TMP3, 0);
872 
873 /* Prepare function arguments */
874 OP1(SLJIT_MOV, SLJIT_R0, 0, STR_END, 0);
875 OP1(SLJIT_MOV, SLJIT_R1, 0, STR_PTR, 0);
876 OP1(SLJIT_MOV, SLJIT_R2, 0, SLJIT_IMM, offset);
877 
878 if (char1 == char2)
879   {
880     ic.c.c1 = char1;
881     ic.c.c2 = char2;
882     OP1(SLJIT_MOV, SLJIT_R4, 0, SLJIT_IMM, ic.x);
883 
884 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
885   if (common->utf && offset > 0)
886     sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4(W, W, W, W, W),
887                      SLJIT_IMM, SLJIT_FUNC_ADDR(ffcs_utf));
888   else
889     sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4(W, W, W, W, W),
890                      SLJIT_IMM, SLJIT_FUNC_ADDR(ffcs));
891 #else
892   sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4(W, W, W, W, W),
893                    SLJIT_IMM, SLJIT_FUNC_ADDR(ffcs));
894 #endif
895   }
896 else
897   {
898   PCRE2_UCHAR mask = char1 ^ char2;
899   if (is_powerof2(mask))
900     {
901     ic.c.c1 = char1 | mask;
902     ic.c.c2 = mask;
903     OP1(SLJIT_MOV, SLJIT_R4, 0, SLJIT_IMM, ic.x);
904 
905 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
906     if (common->utf && offset > 0)
907       sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4(W, W, W, W, W),
908                        SLJIT_IMM, SLJIT_FUNC_ADDR(ffcs_mask_utf));
909     else
910       sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4(W, W, W, W, W),
911                        SLJIT_IMM, SLJIT_FUNC_ADDR(ffcs_mask));
912 #else
913     sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4(W, W, W, W, W),
914                      SLJIT_IMM, SLJIT_FUNC_ADDR(ffcs_mask));
915 #endif
916     }
917   else
918     {
919       ic.c.c1 = char1;
920       ic.c.c2 = char2;
921       OP1(SLJIT_MOV, SLJIT_R4, 0, SLJIT_IMM, ic.x);
922 
923 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
924     if (common->utf && offset > 0)
925       sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4(W, W, W, W, W),
926                        SLJIT_IMM, SLJIT_FUNC_ADDR(ffcs_2_utf));
927     else
928       sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4(W, W, W, W, W),
929                        SLJIT_IMM, SLJIT_FUNC_ADDR(ffcs_2));
930 #else
931     sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4(W, W, W, W, W),
932                      SLJIT_IMM, SLJIT_FUNC_ADDR(ffcs_2));
933 #endif
934     }
935   }
936 /* Restore registers. */
937 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
938 OP1(SLJIT_MOV, TMP3, 0, SLJIT_MEM1(SLJIT_SP), LOCALS1);
939 
940 /* Check return value. */
941 partial_quit = CMP(SLJIT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0);
942 if (common->mode == PCRE2_JIT_COMPLETE)
943   add_jump(compiler, &common->failed_match, partial_quit);
944 
945 /* Fast forward STR_PTR to the result of memchr. */
946 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_RETURN_REG, 0);
947 
948 if (common->mode != PCRE2_JIT_COMPLETE)
949   JUMPHERE(partial_quit);
950 }
951 
952 typedef enum {
953   compare_match1,
954   compare_match1i,
955   compare_match2,
956 } compare_type;
957 
fast_forward_char_pair_compare(compare_type ctype,vect_t dst,vect_t cmp1,vect_t cmp2)958 static inline vect_t fast_forward_char_pair_compare(compare_type ctype, vect_t dst, vect_t cmp1, vect_t cmp2)
959 {
960 if (ctype == compare_match2)
961   {
962   vect_t tmp = dst;
963   dst = VCEQQ(dst, cmp1);
964   tmp = VCEQQ(tmp, cmp2);
965   dst = VORRQ(dst, tmp);
966   return dst;
967   }
968 
969 if (ctype == compare_match1i)
970   dst = VORRQ(dst, cmp2);
971 dst = VCEQQ(dst, cmp1);
972 return dst;
973 }
974 
max_fast_forward_char_pair_offset(void)975 static SLJIT_INLINE sljit_u32 max_fast_forward_char_pair_offset(void)
976 {
977 #if PCRE2_CODE_UNIT_WIDTH == 8
978 return 15;
979 #elif PCRE2_CODE_UNIT_WIDTH == 16
980 return 7;
981 #elif PCRE2_CODE_UNIT_WIDTH == 32
982 return 3;
983 #else
984 #error "Unsupported unit width"
985 #endif
986 }
987 
988 /* ARM doesn't have a shift left across lanes. */
shift_left_n_lanes(vect_t a,sljit_u8 n)989 static SLJIT_INLINE vect_t shift_left_n_lanes(vect_t a, sljit_u8 n)
990 {
991 vect_t zero = VDUPQ(0);
992 SLJIT_ASSERT(0 < n && n < VECTOR_FACTOR);
993 /* VEXTQ takes an immediate as last argument. */
994 #define C(X) case X: return VEXTQ(zero, a, VECTOR_FACTOR - X);
995 switch (n)
996   {
997   C(1); C(2); C(3);
998 #if PCRE2_CODE_UNIT_WIDTH != 32
999   C(4); C(5); C(6); C(7);
1000 # if PCRE2_CODE_UNIT_WIDTH != 16
1001   C(8); C(9); C(10); C(11); C(12); C(13); C(14); C(15);
1002 # endif
1003 #endif
1004   default:
1005     /* Based on the ASSERT(0 < n && n < VECTOR_FACTOR) above, this won't
1006        happen. The return is still here for compilers to not warn. */
1007     return a;
1008   }
1009 }
1010 
1011 #define FFCPS
1012 #define FFCPS_DIFF1
1013 #define FFCPS_CHAR1A2A
1014 
1015 #define FFCPS_0
1016 #include "pcre2_jit_neon_inc.h"
1017 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
1018 # define FF_UTF
1019 # include "pcre2_jit_neon_inc.h"
1020 # undef FF_UTF
1021 #endif
1022 #undef FFCPS_0
1023 
1024 #undef FFCPS_CHAR1A2A
1025 
1026 #define FFCPS_1
1027 #include "pcre2_jit_neon_inc.h"
1028 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
1029 # define FF_UTF
1030 # include "pcre2_jit_neon_inc.h"
1031 # undef FF_UTF
1032 #endif
1033 #undef FFCPS_1
1034 
1035 #undef FFCPS_DIFF1
1036 
1037 #define FFCPS_DEFAULT
1038 #include "pcre2_jit_neon_inc.h"
1039 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
1040 # define FF_UTF
1041 # include "pcre2_jit_neon_inc.h"
1042 # undef FF_UTF
1043 #endif
1044 #undef FFCPS
1045 
1046 #define JIT_HAS_FAST_FORWARD_CHAR_PAIR_SIMD 1
1047 
fast_forward_char_pair_simd(compiler_common * common,sljit_s32 offs1,PCRE2_UCHAR char1a,PCRE2_UCHAR char1b,sljit_s32 offs2,PCRE2_UCHAR char2a,PCRE2_UCHAR char2b)1048 static void fast_forward_char_pair_simd(compiler_common *common, sljit_s32 offs1,
1049   PCRE2_UCHAR char1a, PCRE2_UCHAR char1b, sljit_s32 offs2, PCRE2_UCHAR char2a, PCRE2_UCHAR char2b)
1050 {
1051 DEFINE_COMPILER;
1052 sljit_u32 diff = IN_UCHARS(offs1 - offs2);
1053 struct sljit_jump *partial_quit;
1054 int_char ic;
1055 SLJIT_ASSERT(common->mode == PCRE2_JIT_COMPLETE && offs1 > offs2);
1056 SLJIT_ASSERT(diff <= IN_UCHARS(max_fast_forward_char_pair_offset()));
1057 SLJIT_ASSERT(compiler->scratches == 5);
1058 
1059 /* Save temporary register STR_PTR. */
1060 OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCALS0, STR_PTR, 0);
1061 
1062 /* Prepare arguments for the function call. */
1063 if (common->match_end_ptr == 0)
1064    OP1(SLJIT_MOV, SLJIT_R0, 0, STR_END, 0);
1065 else
1066   {
1067   OP1(SLJIT_MOV, SLJIT_R0, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr);
1068   OP2(SLJIT_ADD, SLJIT_R0, 0, SLJIT_R0, 0, SLJIT_IMM, IN_UCHARS(offs1 + 1));
1069 
1070   OP2U(SLJIT_SUB | SLJIT_SET_LESS, STR_END, 0, SLJIT_R0, 0);
1071   CMOV(SLJIT_LESS, SLJIT_R0, STR_END, 0);
1072   }
1073 
1074 OP1(SLJIT_MOV, SLJIT_R1, 0, STR_PTR, 0);
1075 OP1(SLJIT_MOV_S32, SLJIT_R2, 0, SLJIT_IMM, offs1);
1076 OP1(SLJIT_MOV_S32, SLJIT_R3, 0, SLJIT_IMM, offs2);
1077 ic.c.c1 = char1a;
1078 ic.c.c2 = char1b;
1079 ic.c.c3 = char2a;
1080 ic.c.c4 = char2b;
1081 OP1(SLJIT_MOV_U32, SLJIT_R4, 0, SLJIT_IMM, ic.x);
1082 
1083 if (diff == 1) {
1084   if (char1a == char1b && char2a == char2b) {
1085 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
1086     if (common->utf)
1087       sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4(W, W, W, W, W),
1088                        SLJIT_IMM, SLJIT_FUNC_ADDR(ffcps_0_utf));
1089     else
1090 #endif
1091       sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4(W, W, W, W, W),
1092                        SLJIT_IMM, SLJIT_FUNC_ADDR(ffcps_0));
1093   } else {
1094 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
1095     if (common->utf)
1096       sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4(W, W, W, W, W),
1097                        SLJIT_IMM, SLJIT_FUNC_ADDR(ffcps_1_utf));
1098     else
1099 #endif
1100       sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4(W, W, W, W, W),
1101                        SLJIT_IMM, SLJIT_FUNC_ADDR(ffcps_1));
1102   }
1103 } else {
1104 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
1105   if (common->utf)
1106     sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4(W, W, W, W, W),
1107                      SLJIT_IMM, SLJIT_FUNC_ADDR(ffcps_default_utf));
1108   else
1109 #endif
1110     sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4(W, W, W, W, W),
1111                      SLJIT_IMM, SLJIT_FUNC_ADDR(ffcps_default));
1112 }
1113 
1114 /* Restore STR_PTR register. */
1115 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), LOCALS0);
1116 
1117 /* Check return value. */
1118 partial_quit = CMP(SLJIT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0);
1119 add_jump(compiler, &common->failed_match, partial_quit);
1120 
1121 /* Fast forward STR_PTR to the result of memchr. */
1122 OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_RETURN_REG, 0);
1123 
1124 JUMPHERE(partial_quit);
1125 }
1126 
1127 #endif /* SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64 */
1128 
1129 #if (defined SLJIT_CONFIG_S390X && SLJIT_CONFIG_S390X)
1130 
1131 #if PCRE2_CODE_UNIT_WIDTH == 8
1132 #define VECTOR_ELEMENT_SIZE 0
1133 #elif PCRE2_CODE_UNIT_WIDTH == 16
1134 #define VECTOR_ELEMENT_SIZE 1
1135 #elif PCRE2_CODE_UNIT_WIDTH == 32
1136 #define VECTOR_ELEMENT_SIZE 2
1137 #else
1138 #error "Unsupported unit width"
1139 #endif
1140 
load_from_mem_vector(struct sljit_compiler * compiler,BOOL vlbb,sljit_s32 dst_vreg,sljit_s32 base_reg,sljit_s32 index_reg)1141 static void load_from_mem_vector(struct sljit_compiler *compiler, BOOL vlbb, sljit_s32 dst_vreg,
1142   sljit_s32 base_reg, sljit_s32 index_reg)
1143 {
1144 sljit_u16 instruction[3];
1145 
1146 instruction[0] = (sljit_u16)(0xe700 | (dst_vreg << 4) | index_reg);
1147 instruction[1] = (sljit_u16)(base_reg << 12);
1148 instruction[2] = (sljit_u16)((0x8 << 8) | (vlbb ? 0x07 : 0x06));
1149 
1150 sljit_emit_op_custom(compiler, instruction, 6);
1151 }
1152 
1153 #if PCRE2_CODE_UNIT_WIDTH == 32
1154 
replicate_imm_vector(struct sljit_compiler * compiler,int step,sljit_s32 dst_vreg,PCRE2_UCHAR chr,sljit_s32 tmp_general_reg)1155 static void replicate_imm_vector(struct sljit_compiler *compiler, int step, sljit_s32 dst_vreg,
1156   PCRE2_UCHAR chr, sljit_s32 tmp_general_reg)
1157 {
1158 sljit_u16 instruction[3];
1159 
1160 SLJIT_ASSERT(step >= 0 && step <= 1);
1161 
1162 if (chr < 0x7fff)
1163   {
1164   if (step == 1)
1165     return;
1166 
1167   /* VREPI */
1168   instruction[0] = (sljit_u16)(0xe700 | (dst_vreg << 4));
1169   instruction[1] = (sljit_u16)chr;
1170   instruction[2] = (sljit_u16)((VECTOR_ELEMENT_SIZE << 12) | (0x8 << 8) | 0x45);
1171   sljit_emit_op_custom(compiler, instruction, 6);
1172   return;
1173   }
1174 
1175 if (step == 0)
1176   {
1177   OP1(SLJIT_MOV, tmp_general_reg, 0, SLJIT_IMM, chr);
1178 
1179   /* VLVG */
1180   instruction[0] = (sljit_u16)(0xe700 | (dst_vreg << 4) | sljit_get_register_index(tmp_general_reg));
1181   instruction[1] = 0;
1182   instruction[2] = (sljit_u16)((VECTOR_ELEMENT_SIZE << 12) | (0x8 << 8) | 0x22);
1183   sljit_emit_op_custom(compiler, instruction, 6);
1184   return;
1185   }
1186 
1187 /* VREP */
1188 instruction[0] = (sljit_u16)(0xe700 | (dst_vreg << 4) | dst_vreg);
1189 instruction[1] = 0;
1190 instruction[2] = (sljit_u16)((VECTOR_ELEMENT_SIZE << 12) | (0xc << 8) | 0x4d);
1191 sljit_emit_op_custom(compiler, instruction, 6);
1192 }
1193 
1194 #endif
1195 
fast_forward_char_pair_sse2_compare(struct sljit_compiler * compiler,vector_compare_type compare_type,int step,sljit_s32 dst_ind,sljit_s32 cmp1_ind,sljit_s32 cmp2_ind,sljit_s32 tmp_ind)1196 static void fast_forward_char_pair_sse2_compare(struct sljit_compiler *compiler, vector_compare_type compare_type,
1197   int step, sljit_s32 dst_ind, sljit_s32 cmp1_ind, sljit_s32 cmp2_ind, sljit_s32 tmp_ind)
1198 {
1199 sljit_u16 instruction[3];
1200 
1201 SLJIT_ASSERT(step >= 0 && step <= 2);
1202 
1203 if (step == 1)
1204   {
1205   /* VCEQ */
1206   instruction[0] = (sljit_u16)(0xe700 | (dst_ind << 4) | dst_ind);
1207   instruction[1] = (sljit_u16)(cmp1_ind << 12);
1208   instruction[2] = (sljit_u16)((VECTOR_ELEMENT_SIZE << 12) | (0xe << 8) | 0xf8);
1209   sljit_emit_op_custom(compiler, instruction, 6);
1210   return;
1211   }
1212 
1213 if (compare_type != vector_compare_match2)
1214   {
1215   if (step == 0 && compare_type == vector_compare_match1i)
1216     {
1217     /* VO */
1218     instruction[0] = (sljit_u16)(0xe700 | (dst_ind << 4) | dst_ind);
1219     instruction[1] = (sljit_u16)(cmp2_ind << 12);
1220     instruction[2] = (sljit_u16)((0xe << 8) | 0x6a);
1221     sljit_emit_op_custom(compiler, instruction, 6);
1222     }
1223   return;
1224   }
1225 
1226 switch (step)
1227   {
1228   case 0:
1229   /* VCEQ */
1230   instruction[0] = (sljit_u16)(0xe700 | (tmp_ind << 4) | dst_ind);
1231   instruction[1] = (sljit_u16)(cmp2_ind << 12);
1232   instruction[2] = (sljit_u16)((VECTOR_ELEMENT_SIZE << 12) | (0xe << 8) | 0xf8);
1233   sljit_emit_op_custom(compiler, instruction, 6);
1234   return;
1235 
1236   case 2:
1237   /* VO */
1238   instruction[0] = (sljit_u16)(0xe700 | (dst_ind << 4) | dst_ind);
1239   instruction[1] = (sljit_u16)(tmp_ind << 12);
1240   instruction[2] = (sljit_u16)((0xe << 8) | 0x6a);
1241   sljit_emit_op_custom(compiler, instruction, 6);
1242   return;
1243   }
1244 }
1245 
1246 #define JIT_HAS_FAST_FORWARD_CHAR_SIMD 1
1247 
fast_forward_char_simd(compiler_common * common,PCRE2_UCHAR char1,PCRE2_UCHAR char2,sljit_s32 offset)1248 static void fast_forward_char_simd(compiler_common *common, PCRE2_UCHAR char1, PCRE2_UCHAR char2, sljit_s32 offset)
1249 {
1250 DEFINE_COMPILER;
1251 sljit_u16 instruction[3];
1252 struct sljit_label *start;
1253 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
1254 struct sljit_label *restart;
1255 #endif
1256 struct sljit_jump *quit;
1257 struct sljit_jump *partial_quit[2];
1258 vector_compare_type compare_type = vector_compare_match1;
1259 sljit_s32 tmp1_reg_ind = sljit_get_register_index(TMP1);
1260 sljit_s32 str_ptr_reg_ind = sljit_get_register_index(STR_PTR);
1261 sljit_s32 data_ind = 0;
1262 sljit_s32 tmp_ind = 1;
1263 sljit_s32 cmp1_ind = 2;
1264 sljit_s32 cmp2_ind = 3;
1265 sljit_s32 zero_ind = 4;
1266 sljit_u32 bit = 0;
1267 int i;
1268 
1269 SLJIT_UNUSED_ARG(offset);
1270 
1271 if (char1 != char2)
1272   {
1273   bit = char1 ^ char2;
1274   compare_type = vector_compare_match1i;
1275 
1276   if (!is_powerof2(bit))
1277     {
1278     bit = 0;
1279     compare_type = vector_compare_match2;
1280     }
1281   }
1282 
1283 partial_quit[0] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1284 if (common->mode == PCRE2_JIT_COMPLETE)
1285   add_jump(compiler, &common->failed_match, partial_quit[0]);
1286 
1287 /* First part (unaligned start) */
1288 
1289 OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, 16);
1290 
1291 #if PCRE2_CODE_UNIT_WIDTH != 32
1292 
1293 /* VREPI */
1294 instruction[0] = (sljit_u16)(0xe700 | (cmp1_ind << 4));
1295 instruction[1] = (sljit_u16)(char1 | bit);
1296 instruction[2] = (sljit_u16)((VECTOR_ELEMENT_SIZE << 12) | (0x8 << 8) | 0x45);
1297 sljit_emit_op_custom(compiler, instruction, 6);
1298 
1299 if (char1 != char2)
1300   {
1301   /* VREPI */
1302   instruction[0] = (sljit_u16)(0xe700 | (cmp2_ind << 4));
1303   instruction[1] = (sljit_u16)(bit != 0 ? bit : char2);
1304   /* instruction[2] = (sljit_u16)((VECTOR_ELEMENT_SIZE << 12) | (0x8 << 8) | 0x45); */
1305   sljit_emit_op_custom(compiler, instruction, 6);
1306   }
1307 
1308 #else /* PCRE2_CODE_UNIT_WIDTH == 32 */
1309 
1310 for (int i = 0; i < 2; i++)
1311   {
1312   replicate_imm_vector(compiler, i, cmp1_ind, char1 | bit, TMP1);
1313 
1314   if (char1 != char2)
1315     replicate_imm_vector(compiler, i, cmp2_ind, bit != 0 ? bit : char2, TMP1);
1316   }
1317 
1318 #endif /* PCRE2_CODE_UNIT_WIDTH != 32 */
1319 
1320 if (compare_type == vector_compare_match2)
1321   {
1322   /* VREPI */
1323   instruction[0] = (sljit_u16)(0xe700 | (zero_ind << 4));
1324   instruction[1] = 0;
1325   instruction[2] = (sljit_u16)((0x8 << 8) | 0x45);
1326   sljit_emit_op_custom(compiler, instruction, 6);
1327   }
1328 
1329 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
1330 restart = LABEL();
1331 #endif
1332 
1333 load_from_mem_vector(compiler, TRUE, data_ind, str_ptr_reg_ind, 0);
1334 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, ~15);
1335 
1336 if (compare_type != vector_compare_match2)
1337   {
1338   if (compare_type == vector_compare_match1i)
1339     fast_forward_char_pair_sse2_compare(compiler, compare_type, 0, data_ind, cmp1_ind, cmp2_ind, tmp_ind);
1340 
1341   /* VFEE */
1342   instruction[0] = (sljit_u16)(0xe700 | (data_ind << 4) | data_ind);
1343   instruction[1] = (sljit_u16)((cmp1_ind << 12) | (1 << 4));
1344   instruction[2] = (sljit_u16)((VECTOR_ELEMENT_SIZE << 12) | (0xe << 8) | 0x80);
1345   sljit_emit_op_custom(compiler, instruction, 6);
1346   }
1347 else
1348   {
1349   for (i = 0; i < 3; i++)
1350     fast_forward_char_pair_sse2_compare(compiler, compare_type, i, data_ind, cmp1_ind, cmp2_ind, tmp_ind);
1351 
1352   /* VFENE */
1353   instruction[0] = (sljit_u16)(0xe700 | (data_ind << 4) | data_ind);
1354   instruction[1] = (sljit_u16)((zero_ind << 12) | (1 << 4));
1355   instruction[2] = (sljit_u16)((0xe << 8) | 0x81);
1356   sljit_emit_op_custom(compiler, instruction, 6);
1357   }
1358 
1359 /* VLGVB */
1360 instruction[0] = (sljit_u16)(0xe700 | (tmp1_reg_ind << 4) | data_ind);
1361 instruction[1] = 7;
1362 instruction[2] = (sljit_u16)((0x4 << 8) | 0x21);
1363 sljit_emit_op_custom(compiler, instruction, 6);
1364 
1365 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
1366 quit = CMP(SLJIT_LESS, STR_PTR, 0, TMP2, 0);
1367 
1368 OP2(SLJIT_SUB, STR_PTR, 0, TMP2, 0, SLJIT_IMM, 16);
1369 
1370 /* Second part (aligned) */
1371 start = LABEL();
1372 
1373 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 16);
1374 
1375 partial_quit[1] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
1376 if (common->mode == PCRE2_JIT_COMPLETE)
1377   add_jump(compiler, &common->failed_match, partial_quit[1]);
1378 
1379 load_from_mem_vector(compiler, TRUE, data_ind, str_ptr_reg_ind, 0);
1380 
1381 if (compare_type != vector_compare_match2)
1382   {
1383   if (compare_type == vector_compare_match1i)
1384     fast_forward_char_pair_sse2_compare(compiler, compare_type, 0, data_ind, cmp1_ind, cmp2_ind, tmp_ind);
1385 
1386   /* VFEE */
1387   instruction[0] = (sljit_u16)(0xe700 | (data_ind << 4) | data_ind);
1388   instruction[1] = (sljit_u16)((cmp1_ind << 12) | (1 << 4));
1389   instruction[2] = (sljit_u16)((VECTOR_ELEMENT_SIZE << 12) | (0xe << 8) | 0x80);
1390   sljit_emit_op_custom(compiler, instruction, 6);
1391   }
1392 else
1393   {
1394   for (i = 0; i < 3; i++)
1395     fast_forward_char_pair_sse2_compare(compiler, compare_type, i, data_ind, cmp1_ind, cmp2_ind, tmp_ind);
1396 
1397   /* VFENE */
1398   instruction[0] = (sljit_u16)(0xe700 | (data_ind << 4) | data_ind);
1399   instruction[1] = (sljit_u16)((zero_ind << 12) | (1 << 4));
1400   instruction[2] = (sljit_u16)((0xe << 8) | 0x81);
1401   sljit_emit_op_custom(compiler, instruction, 6);
1402   }
1403 
1404 sljit_set_current_flags(compiler, SLJIT_SET_OVERFLOW);
1405 JUMPTO(SLJIT_OVERFLOW, start);
1406 
1407 /* VLGVB */
1408 instruction[0] = (sljit_u16)(0xe700 | (tmp1_reg_ind << 4) | data_ind);
1409 instruction[1] = 7;
1410 instruction[2] = (sljit_u16)((0x4 << 8) | 0x21);
1411 sljit_emit_op_custom(compiler, instruction, 6);
1412 
1413 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
1414 
1415 JUMPHERE(quit);
1416 
1417 if (common->mode != PCRE2_JIT_COMPLETE)
1418   {
1419   JUMPHERE(partial_quit[0]);
1420   JUMPHERE(partial_quit[1]);
1421   OP2U(SLJIT_SUB | SLJIT_SET_GREATER, STR_PTR, 0, STR_END, 0);
1422   CMOV(SLJIT_GREATER, STR_PTR, STR_END, 0);
1423   }
1424 else
1425   add_jump(compiler, &common->failed_match, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
1426 
1427 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
1428 if (common->utf && offset > 0)
1429   {
1430   SLJIT_ASSERT(common->mode == PCRE2_JIT_COMPLETE);
1431 
1432   OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-offset));
1433 
1434   quit = jump_if_utf_char_start(compiler, TMP1);
1435 
1436   OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
1437   add_jump(compiler, &common->failed_match, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
1438 
1439   OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, 16);
1440   JUMPTO(SLJIT_JUMP, restart);
1441 
1442   JUMPHERE(quit);
1443   }
1444 #endif
1445 }
1446 
1447 #define JIT_HAS_FAST_REQUESTED_CHAR_SIMD 1
1448 
fast_requested_char_simd(compiler_common * common,PCRE2_UCHAR char1,PCRE2_UCHAR char2)1449 static jump_list *fast_requested_char_simd(compiler_common *common, PCRE2_UCHAR char1, PCRE2_UCHAR char2)
1450 {
1451 DEFINE_COMPILER;
1452 sljit_u16 instruction[3];
1453 struct sljit_label *start;
1454 struct sljit_jump *quit;
1455 jump_list *not_found = NULL;
1456 vector_compare_type compare_type = vector_compare_match1;
1457 sljit_s32 tmp1_reg_ind = sljit_get_register_index(TMP1);
1458 sljit_s32 tmp3_reg_ind = sljit_get_register_index(TMP3);
1459 sljit_s32 data_ind = 0;
1460 sljit_s32 tmp_ind = 1;
1461 sljit_s32 cmp1_ind = 2;
1462 sljit_s32 cmp2_ind = 3;
1463 sljit_s32 zero_ind = 4;
1464 sljit_u32 bit = 0;
1465 int i;
1466 
1467 if (char1 != char2)
1468   {
1469   bit = char1 ^ char2;
1470   compare_type = vector_compare_match1i;
1471 
1472   if (!is_powerof2(bit))
1473     {
1474     bit = 0;
1475     compare_type = vector_compare_match2;
1476     }
1477   }
1478 
1479 add_jump(compiler, &not_found, CMP(SLJIT_GREATER_EQUAL, TMP1, 0, STR_END, 0));
1480 
1481 /* First part (unaligned start) */
1482 
1483 OP2(SLJIT_ADD, TMP2, 0, TMP1, 0, SLJIT_IMM, 16);
1484 
1485 #if PCRE2_CODE_UNIT_WIDTH != 32
1486 
1487 /* VREPI */
1488 instruction[0] = (sljit_u16)(0xe700 | (cmp1_ind << 4));
1489 instruction[1] = (sljit_u16)(char1 | bit);
1490 instruction[2] = (sljit_u16)((VECTOR_ELEMENT_SIZE << 12) | (0x8 << 8) | 0x45);
1491 sljit_emit_op_custom(compiler, instruction, 6);
1492 
1493 if (char1 != char2)
1494   {
1495   /* VREPI */
1496   instruction[0] = (sljit_u16)(0xe700 | (cmp2_ind << 4));
1497   instruction[1] = (sljit_u16)(bit != 0 ? bit : char2);
1498   /* instruction[2] = (sljit_u16)((VECTOR_ELEMENT_SIZE << 12) | (0x8 << 8) | 0x45); */
1499   sljit_emit_op_custom(compiler, instruction, 6);
1500   }
1501 
1502 #else /* PCRE2_CODE_UNIT_WIDTH == 32 */
1503 
1504 for (int i = 0; i < 2; i++)
1505   {
1506   replicate_imm_vector(compiler, i, cmp1_ind, char1 | bit, TMP3);
1507 
1508   if (char1 != char2)
1509     replicate_imm_vector(compiler, i, cmp2_ind, bit != 0 ? bit : char2, TMP3);
1510   }
1511 
1512 #endif /* PCRE2_CODE_UNIT_WIDTH != 32 */
1513 
1514 if (compare_type == vector_compare_match2)
1515   {
1516   /* VREPI */
1517   instruction[0] = (sljit_u16)(0xe700 | (zero_ind << 4));
1518   instruction[1] = 0;
1519   instruction[2] = (sljit_u16)((0x8 << 8) | 0x45);
1520   sljit_emit_op_custom(compiler, instruction, 6);
1521   }
1522 
1523 load_from_mem_vector(compiler, TRUE, data_ind, tmp1_reg_ind, 0);
1524 OP2(SLJIT_AND, TMP2, 0, TMP2, 0, SLJIT_IMM, ~15);
1525 
1526 if (compare_type != vector_compare_match2)
1527   {
1528   if (compare_type == vector_compare_match1i)
1529     fast_forward_char_pair_sse2_compare(compiler, compare_type, 0, data_ind, cmp1_ind, cmp2_ind, tmp_ind);
1530 
1531   /* VFEE */
1532   instruction[0] = (sljit_u16)(0xe700 | (data_ind << 4) | data_ind);
1533   instruction[1] = (sljit_u16)((cmp1_ind << 12) | (1 << 4));
1534   instruction[2] = (sljit_u16)((VECTOR_ELEMENT_SIZE << 12) | (0xe << 8) | 0x80);
1535   sljit_emit_op_custom(compiler, instruction, 6);
1536   }
1537 else
1538   {
1539   for (i = 0; i < 3; i++)
1540     fast_forward_char_pair_sse2_compare(compiler, compare_type, i, data_ind, cmp1_ind, cmp2_ind, tmp_ind);
1541 
1542   /* VFENE */
1543   instruction[0] = (sljit_u16)(0xe700 | (data_ind << 4) | data_ind);
1544   instruction[1] = (sljit_u16)((zero_ind << 12) | (1 << 4));
1545   instruction[2] = (sljit_u16)((0xe << 8) | 0x81);
1546   sljit_emit_op_custom(compiler, instruction, 6);
1547   }
1548 
1549 /* VLGVB */
1550 instruction[0] = (sljit_u16)(0xe700 | (tmp3_reg_ind << 4) | data_ind);
1551 instruction[1] = 7;
1552 instruction[2] = (sljit_u16)((0x4 << 8) | 0x21);
1553 sljit_emit_op_custom(compiler, instruction, 6);
1554 
1555 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP3, 0);
1556 quit = CMP(SLJIT_LESS, TMP1, 0, TMP2, 0);
1557 
1558 OP2(SLJIT_SUB, TMP1, 0, TMP2, 0, SLJIT_IMM, 16);
1559 
1560 /* Second part (aligned) */
1561 start = LABEL();
1562 
1563 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 16);
1564 
1565 add_jump(compiler, &not_found, CMP(SLJIT_GREATER_EQUAL, TMP1, 0, STR_END, 0));
1566 
1567 load_from_mem_vector(compiler, TRUE, data_ind, tmp1_reg_ind, 0);
1568 
1569 if (compare_type != vector_compare_match2)
1570   {
1571   if (compare_type == vector_compare_match1i)
1572     fast_forward_char_pair_sse2_compare(compiler, compare_type, 0, data_ind, cmp1_ind, cmp2_ind, tmp_ind);
1573 
1574   /* VFEE */
1575   instruction[0] = (sljit_u16)(0xe700 | (data_ind << 4) | data_ind);
1576   instruction[1] = (sljit_u16)((cmp1_ind << 12) | (1 << 4));
1577   instruction[2] = (sljit_u16)((VECTOR_ELEMENT_SIZE << 12) | (0xe << 8) | 0x80);
1578   sljit_emit_op_custom(compiler, instruction, 6);
1579   }
1580 else
1581   {
1582   for (i = 0; i < 3; i++)
1583     fast_forward_char_pair_sse2_compare(compiler, compare_type, i, data_ind, cmp1_ind, cmp2_ind, tmp_ind);
1584 
1585   /* VFENE */
1586   instruction[0] = (sljit_u16)(0xe700 | (data_ind << 4) | data_ind);
1587   instruction[1] = (sljit_u16)((zero_ind << 12) | (1 << 4));
1588   instruction[2] = (sljit_u16)((0xe << 8) | 0x81);
1589   sljit_emit_op_custom(compiler, instruction, 6);
1590   }
1591 
1592 sljit_set_current_flags(compiler, SLJIT_SET_OVERFLOW);
1593 JUMPTO(SLJIT_OVERFLOW, start);
1594 
1595 /* VLGVB */
1596 instruction[0] = (sljit_u16)(0xe700 | (tmp3_reg_ind << 4) | data_ind);
1597 instruction[1] = 7;
1598 instruction[2] = (sljit_u16)((0x4 << 8) | 0x21);
1599 sljit_emit_op_custom(compiler, instruction, 6);
1600 
1601 OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP3, 0);
1602 
1603 JUMPHERE(quit);
1604 add_jump(compiler, &not_found, CMP(SLJIT_GREATER_EQUAL, TMP1, 0, STR_END, 0));
1605 
1606 return not_found;
1607 }
1608 
1609 #define JIT_HAS_FAST_FORWARD_CHAR_PAIR_SIMD 1
1610 
fast_forward_char_pair_simd(compiler_common * common,sljit_s32 offs1,PCRE2_UCHAR char1a,PCRE2_UCHAR char1b,sljit_s32 offs2,PCRE2_UCHAR char2a,PCRE2_UCHAR char2b)1611 static void fast_forward_char_pair_simd(compiler_common *common, sljit_s32 offs1,
1612   PCRE2_UCHAR char1a, PCRE2_UCHAR char1b, sljit_s32 offs2, PCRE2_UCHAR char2a, PCRE2_UCHAR char2b)
1613 {
1614 DEFINE_COMPILER;
1615 sljit_u16 instruction[3];
1616 struct sljit_label *start;
1617 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
1618 struct sljit_label *restart;
1619 #endif
1620 struct sljit_jump *quit;
1621 struct sljit_jump *jump[2];
1622 vector_compare_type compare1_type = vector_compare_match1;
1623 vector_compare_type compare2_type = vector_compare_match1;
1624 sljit_u32 bit1 = 0;
1625 sljit_u32 bit2 = 0;
1626 sljit_s32 diff = IN_UCHARS(offs2 - offs1);
1627 sljit_s32 tmp1_reg_ind = sljit_get_register_index(TMP1);
1628 sljit_s32 tmp2_reg_ind = sljit_get_register_index(TMP2);
1629 sljit_s32 str_ptr_reg_ind = sljit_get_register_index(STR_PTR);
1630 sljit_s32 data1_ind = 0;
1631 sljit_s32 data2_ind = 1;
1632 sljit_s32 tmp1_ind = 2;
1633 sljit_s32 tmp2_ind = 3;
1634 sljit_s32 cmp1a_ind = 4;
1635 sljit_s32 cmp1b_ind = 5;
1636 sljit_s32 cmp2a_ind = 6;
1637 sljit_s32 cmp2b_ind = 7;
1638 sljit_s32 zero_ind = 8;
1639 int i;
1640 
1641 SLJIT_ASSERT(common->mode == PCRE2_JIT_COMPLETE && offs1 > offs2);
1642 SLJIT_ASSERT(-diff <= (sljit_s32)IN_UCHARS(max_fast_forward_char_pair_offset()));
1643 SLJIT_ASSERT(tmp1_reg_ind != 0 && tmp2_reg_ind != 0);
1644 
1645 if (char1a != char1b)
1646   {
1647   bit1 = char1a ^ char1b;
1648   compare1_type = vector_compare_match1i;
1649 
1650   if (!is_powerof2(bit1))
1651     {
1652     bit1 = 0;
1653     compare1_type = vector_compare_match2;
1654     }
1655   }
1656 
1657 if (char2a != char2b)
1658   {
1659   bit2 = char2a ^ char2b;
1660   compare2_type = vector_compare_match1i;
1661 
1662   if (!is_powerof2(bit2))
1663     {
1664     bit2 = 0;
1665     compare2_type = vector_compare_match2;
1666     }
1667   }
1668 
1669 /* Initialize. */
1670 if (common->match_end_ptr != 0)
1671   {
1672   OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr);
1673   OP1(SLJIT_MOV, TMP3, 0, STR_END, 0);
1674   OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(offs1 + 1));
1675 
1676   OP2U(SLJIT_SUB | SLJIT_SET_LESS, TMP1, 0, STR_END, 0);
1677   CMOV(SLJIT_LESS, STR_END, TMP1, 0);
1678   }
1679 
1680 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offs1));
1681 add_jump(compiler, &common->failed_match, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
1682 OP2(SLJIT_AND, TMP2, 0, STR_PTR, 0, SLJIT_IMM, ~15);
1683 
1684 #if PCRE2_CODE_UNIT_WIDTH != 32
1685 
1686 OP2(SLJIT_SUB, TMP1, 0, STR_PTR, 0, SLJIT_IMM, -diff);
1687 
1688 /* VREPI */
1689 instruction[0] = (sljit_u16)(0xe700 | (cmp1a_ind << 4));
1690 instruction[1] = (sljit_u16)(char1a | bit1);
1691 instruction[2] = (sljit_u16)((VECTOR_ELEMENT_SIZE << 12) | (0x8 << 8) | 0x45);
1692 sljit_emit_op_custom(compiler, instruction, 6);
1693 
1694 if (char1a != char1b)
1695   {
1696   /* VREPI */
1697   instruction[0] = (sljit_u16)(0xe700 | (cmp1b_ind << 4));
1698   instruction[1] = (sljit_u16)(bit1 != 0 ? bit1 : char1b);
1699   /* instruction[2] = (sljit_u16)((VECTOR_ELEMENT_SIZE << 12) | (0x8 << 8) | 0x45); */
1700   sljit_emit_op_custom(compiler, instruction, 6);
1701   }
1702 
1703 /* VREPI */
1704 instruction[0] = (sljit_u16)(0xe700 | (cmp2a_ind << 4));
1705 instruction[1] = (sljit_u16)(char2a | bit2);
1706 /* instruction[2] = (sljit_u16)((VECTOR_ELEMENT_SIZE << 12) | (0x8 << 8) | 0x45); */
1707 sljit_emit_op_custom(compiler, instruction, 6);
1708 
1709 if (char2a != char2b)
1710   {
1711   /* VREPI */
1712   instruction[0] = (sljit_u16)(0xe700 | (cmp2b_ind << 4));
1713   instruction[1] = (sljit_u16)(bit2 != 0 ? bit2 : char2b);
1714   /* instruction[2] = (sljit_u16)((VECTOR_ELEMENT_SIZE << 12) | (0x8 << 8) | 0x45); */
1715   sljit_emit_op_custom(compiler, instruction, 6);
1716   }
1717 
1718 #else /* PCRE2_CODE_UNIT_WIDTH == 32 */
1719 
1720 for (int i = 0; i < 2; i++)
1721   {
1722   replicate_imm_vector(compiler, i, cmp1a_ind, char1a | bit1, TMP1);
1723 
1724   if (char1a != char1b)
1725     replicate_imm_vector(compiler, i, cmp1b_ind, bit1 != 0 ? bit1 : char1b, TMP1);
1726 
1727   replicate_imm_vector(compiler, i, cmp2a_ind, char2a | bit2, TMP1);
1728 
1729   if (char2a != char2b)
1730     replicate_imm_vector(compiler, i, cmp2b_ind, bit2 != 0 ? bit2 : char2b, TMP1);
1731   }
1732 
1733 OP2(SLJIT_SUB, TMP1, 0, STR_PTR, 0, SLJIT_IMM, -diff);
1734 
1735 #endif /* PCRE2_CODE_UNIT_WIDTH != 32 */
1736 
1737 /* VREPI */
1738 instruction[0] = (sljit_u16)(0xe700 | (zero_ind << 4));
1739 instruction[1] = 0;
1740 instruction[2] = (sljit_u16)((0x8 << 8) | 0x45);
1741 sljit_emit_op_custom(compiler, instruction, 6);
1742 
1743 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
1744 restart = LABEL();
1745 #endif
1746 
1747 jump[0] = CMP(SLJIT_LESS, TMP1, 0, TMP2, 0);
1748 load_from_mem_vector(compiler, TRUE, data2_ind, tmp1_reg_ind, 0);
1749 jump[1] = JUMP(SLJIT_JUMP);
1750 JUMPHERE(jump[0]);
1751 load_from_mem_vector(compiler, FALSE, data2_ind, tmp1_reg_ind, 0);
1752 JUMPHERE(jump[1]);
1753 
1754 load_from_mem_vector(compiler, TRUE, data1_ind, str_ptr_reg_ind, 0);
1755 OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, 16);
1756 
1757 for (i = 0; i < 3; i++)
1758   {
1759   fast_forward_char_pair_sse2_compare(compiler, compare1_type, i, data1_ind, cmp1a_ind, cmp1b_ind, tmp1_ind);
1760   fast_forward_char_pair_sse2_compare(compiler, compare2_type, i, data2_ind, cmp2a_ind, cmp2b_ind, tmp2_ind);
1761   }
1762 
1763 /* VN */
1764 instruction[0] = (sljit_u16)(0xe700 | (data1_ind << 4) | data1_ind);
1765 instruction[1] = (sljit_u16)(data2_ind << 12);
1766 instruction[2] = (sljit_u16)((0xe << 8) | 0x68);
1767 sljit_emit_op_custom(compiler, instruction, 6);
1768 
1769 /* VFENE */
1770 instruction[0] = (sljit_u16)(0xe700 | (data1_ind << 4) | data1_ind);
1771 instruction[1] = (sljit_u16)((zero_ind << 12) | (1 << 4));
1772 instruction[2] = (sljit_u16)((0xe << 8) | 0x81);
1773 sljit_emit_op_custom(compiler, instruction, 6);
1774 
1775 /* VLGVB */
1776 instruction[0] = (sljit_u16)(0xe700 | (tmp1_reg_ind << 4) | data1_ind);
1777 instruction[1] = 7;
1778 instruction[2] = (sljit_u16)((0x4 << 8) | 0x21);
1779 sljit_emit_op_custom(compiler, instruction, 6);
1780 
1781 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
1782 quit = CMP(SLJIT_LESS, STR_PTR, 0, TMP2, 0);
1783 
1784 OP2(SLJIT_SUB, STR_PTR, 0, TMP2, 0, SLJIT_IMM, 16);
1785 OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, diff);
1786 
1787 /* Main loop. */
1788 start = LABEL();
1789 
1790 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, 16);
1791 add_jump(compiler, &common->failed_match, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
1792 
1793 load_from_mem_vector(compiler, FALSE, data1_ind, str_ptr_reg_ind, 0);
1794 load_from_mem_vector(compiler, FALSE, data2_ind, str_ptr_reg_ind, tmp1_reg_ind);
1795 
1796 for (i = 0; i < 3; i++)
1797   {
1798   fast_forward_char_pair_sse2_compare(compiler, compare1_type, i, data1_ind, cmp1a_ind, cmp1b_ind, tmp1_ind);
1799   fast_forward_char_pair_sse2_compare(compiler, compare2_type, i, data2_ind, cmp2a_ind, cmp2b_ind, tmp2_ind);
1800   }
1801 
1802 /* VN */
1803 instruction[0] = (sljit_u16)(0xe700 | (data1_ind << 4) | data1_ind);
1804 instruction[1] = (sljit_u16)(data2_ind << 12);
1805 instruction[2] = (sljit_u16)((0xe << 8) | 0x68);
1806 sljit_emit_op_custom(compiler, instruction, 6);
1807 
1808 /* VFENE */
1809 instruction[0] = (sljit_u16)(0xe700 | (data1_ind << 4) | data1_ind);
1810 instruction[1] = (sljit_u16)((zero_ind << 12) | (1 << 4));
1811 instruction[2] = (sljit_u16)((0xe << 8) | 0x81);
1812 sljit_emit_op_custom(compiler, instruction, 6);
1813 
1814 sljit_set_current_flags(compiler, SLJIT_SET_OVERFLOW);
1815 JUMPTO(SLJIT_OVERFLOW, start);
1816 
1817 /* VLGVB */
1818 instruction[0] = (sljit_u16)(0xe700 | (tmp2_reg_ind << 4) | data1_ind);
1819 instruction[1] = 7;
1820 instruction[2] = (sljit_u16)((0x4 << 8) | 0x21);
1821 sljit_emit_op_custom(compiler, instruction, 6);
1822 
1823 OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP2, 0);
1824 
1825 JUMPHERE(quit);
1826 
1827 add_jump(compiler, &common->failed_match, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
1828 
1829 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
1830 if (common->utf)
1831   {
1832   SLJIT_ASSERT(common->mode == PCRE2_JIT_COMPLETE);
1833 
1834   OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-offs1));
1835 
1836   quit = jump_if_utf_char_start(compiler, TMP1);
1837 
1838   OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
1839   add_jump(compiler, &common->failed_match, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
1840 
1841   /* TMP1 contains diff. */
1842   OP2(SLJIT_AND, TMP2, 0, STR_PTR, 0, SLJIT_IMM, ~15);
1843   OP2(SLJIT_SUB, TMP1, 0, STR_PTR, 0, SLJIT_IMM, -diff);
1844   JUMPTO(SLJIT_JUMP, restart);
1845 
1846   JUMPHERE(quit);
1847   }
1848 #endif
1849 
1850 OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offs1));
1851 
1852 if (common->match_end_ptr != 0)
1853   OP1(SLJIT_MOV, STR_END, 0, TMP3, 0);
1854 }
1855 
1856 #endif /* SLJIT_CONFIG_S390X */
1857 
1858 #endif /* !SUPPORT_VALGRIND */
1859