Lines Matching refs:tok
2321 fetch_range_quantifier(UChar** src, UChar* end, OnigToken* tok, ScanEnv* env) in fetch_range_quantifier() argument
2397 tok->type = TK_INTERVAL; in fetch_range_quantifier()
2398 tok->u.repeat.lower = low; in fetch_range_quantifier()
2399 tok->u.repeat.upper = up; in fetch_range_quantifier()
2483 static int fetch_token(OnigToken* tok, UChar** src, UChar* end, ScanEnv* env);
2911 fetch_token_in_cc(OnigToken* tok, UChar** src, UChar* end, ScanEnv* env) in fetch_token_in_cc() argument
2922 tok->type = TK_EOT; in fetch_token_in_cc()
2923 return tok->type; in fetch_token_in_cc()
2927 tok->type = TK_CHAR; in fetch_token_in_cc()
2928 tok->base = 0; in fetch_token_in_cc()
2929 tok->u.c = c; in fetch_token_in_cc()
2930 tok->escaped = 0; in fetch_token_in_cc()
2933 tok->type = TK_CC_CLOSE; in fetch_token_in_cc()
2936 tok->type = TK_CC_RANGE; in fetch_token_in_cc()
2945 tok->escaped = 1; in fetch_token_in_cc()
2946 tok->u.c = c; in fetch_token_in_cc()
2949 tok->type = TK_CHAR_TYPE; in fetch_token_in_cc()
2950 tok->u.prop.ctype = ONIGENC_CTYPE_WORD; in fetch_token_in_cc()
2951 tok->u.prop.not = 0; in fetch_token_in_cc()
2954 tok->type = TK_CHAR_TYPE; in fetch_token_in_cc()
2955 tok->u.prop.ctype = ONIGENC_CTYPE_WORD; in fetch_token_in_cc()
2956 tok->u.prop.not = 1; in fetch_token_in_cc()
2959 tok->type = TK_CHAR_TYPE; in fetch_token_in_cc()
2960 tok->u.prop.ctype = ONIGENC_CTYPE_DIGIT; in fetch_token_in_cc()
2961 tok->u.prop.not = 0; in fetch_token_in_cc()
2964 tok->type = TK_CHAR_TYPE; in fetch_token_in_cc()
2965 tok->u.prop.ctype = ONIGENC_CTYPE_DIGIT; in fetch_token_in_cc()
2966 tok->u.prop.not = 1; in fetch_token_in_cc()
2969 tok->type = TK_CHAR_TYPE; in fetch_token_in_cc()
2970 tok->u.prop.ctype = ONIGENC_CTYPE_SPACE; in fetch_token_in_cc()
2971 tok->u.prop.not = 0; in fetch_token_in_cc()
2974 tok->type = TK_CHAR_TYPE; in fetch_token_in_cc()
2975 tok->u.prop.ctype = ONIGENC_CTYPE_SPACE; in fetch_token_in_cc()
2976 tok->u.prop.not = 1; in fetch_token_in_cc()
2980 tok->type = TK_CHAR_TYPE; in fetch_token_in_cc()
2981 tok->u.prop.ctype = ONIGENC_CTYPE_XDIGIT; in fetch_token_in_cc()
2982 tok->u.prop.not = 0; in fetch_token_in_cc()
2986 tok->type = TK_CHAR_TYPE; in fetch_token_in_cc()
2987 tok->u.prop.ctype = ONIGENC_CTYPE_XDIGIT; in fetch_token_in_cc()
2988 tok->u.prop.not = 1; in fetch_token_in_cc()
2997 tok->type = TK_CHAR_PROPERTY; in fetch_token_in_cc()
2998 tok->u.prop.not = (c == 'P' ? 1 : 0); in fetch_token_in_cc()
3003 tok->u.prop.not = (tok->u.prop.not == 0 ? 1 : 0); in fetch_token_in_cc()
3027 tok->type = TK_CODE_POINT; in fetch_token_in_cc()
3028 tok->base = 16; in fetch_token_in_cc()
3029 tok->u.code = (OnigCodePoint )num; in fetch_token_in_cc()
3042 tok->type = TK_RAW_BYTE; in fetch_token_in_cc()
3043 tok->base = 16; in fetch_token_in_cc()
3044 tok->u.c = num; in fetch_token_in_cc()
3058 tok->type = TK_CODE_POINT; in fetch_token_in_cc()
3059 tok->base = 16; in fetch_token_in_cc()
3060 tok->u.code = (OnigCodePoint )num; in fetch_token_in_cc()
3074 tok->type = TK_RAW_BYTE; in fetch_token_in_cc()
3075 tok->base = 8; in fetch_token_in_cc()
3076 tok->u.c = num; in fetch_token_in_cc()
3084 if (tok->u.c != num) { in fetch_token_in_cc()
3085 tok->u.code = (OnigCodePoint )num; in fetch_token_in_cc()
3086 tok->type = TK_CODE_POINT; in fetch_token_in_cc()
3094 tok->backp = p; /* point at '[' is readed */ in fetch_token_in_cc()
3098 tok->type = TK_POSIX_BRACKET_OPEN; in fetch_token_in_cc()
3108 tok->type = TK_CC_CC_OPEN; in fetch_token_in_cc()
3119 tok->type = TK_CC_AND; in fetch_token_in_cc()
3125 return tok->type; in fetch_token_in_cc()
3129 fetch_token(OnigToken* tok, UChar** src, UChar* end, ScanEnv* env) in fetch_token() argument
3141 tok->type = TK_EOT; in fetch_token()
3142 return tok->type; in fetch_token()
3145 tok->type = TK_STRING; in fetch_token()
3146 tok->base = 0; in fetch_token()
3147 tok->backp = p; in fetch_token()
3153 tok->backp = p; in fetch_token()
3156 tok->u.c = c; in fetch_token()
3157 tok->escaped = 1; in fetch_token()
3161 tok->type = TK_OP_REPEAT; in fetch_token()
3162 tok->u.repeat.lower = 0; in fetch_token()
3163 tok->u.repeat.upper = REPEAT_INFINITE; in fetch_token()
3169 tok->type = TK_OP_REPEAT; in fetch_token()
3170 tok->u.repeat.lower = 1; in fetch_token()
3171 tok->u.repeat.upper = REPEAT_INFINITE; in fetch_token()
3177 tok->type = TK_OP_REPEAT; in fetch_token()
3178 tok->u.repeat.lower = 0; in fetch_token()
3179 tok->u.repeat.upper = 1; in fetch_token()
3184 tok->u.repeat.greedy = 0; in fetch_token()
3185 tok->u.repeat.possessive = 0; in fetch_token()
3191 tok->type != TK_INTERVAL) || in fetch_token()
3193 tok->type == TK_INTERVAL))) { in fetch_token()
3195 tok->u.repeat.greedy = 1; in fetch_token()
3196 tok->u.repeat.possessive = 1; in fetch_token()
3199 tok->u.repeat.greedy = 1; in fetch_token()
3200 tok->u.repeat.possessive = 0; in fetch_token()
3207 r = fetch_range_quantifier(&p, end, tok, env); in fetch_token()
3221 tok->type = TK_ALT; in fetch_token()
3226 tok->type = TK_SUBEXP_OPEN; in fetch_token()
3231 tok->type = TK_SUBEXP_CLOSE; in fetch_token()
3236 tok->type = TK_CHAR_TYPE; in fetch_token()
3237 tok->u.prop.ctype = ONIGENC_CTYPE_WORD; in fetch_token()
3238 tok->u.prop.not = 0; in fetch_token()
3243 tok->type = TK_CHAR_TYPE; in fetch_token()
3244 tok->u.prop.ctype = ONIGENC_CTYPE_WORD; in fetch_token()
3245 tok->u.prop.not = 1; in fetch_token()
3250 tok->type = TK_ANCHOR; in fetch_token()
3251 tok->u.anchor = ANCHOR_WORD_BOUND; in fetch_token()
3256 tok->type = TK_ANCHOR; in fetch_token()
3257 tok->u.anchor = ANCHOR_NOT_WORD_BOUND; in fetch_token()
3263 tok->type = TK_ANCHOR; in fetch_token()
3264 tok->u.anchor = ANCHOR_WORD_BEGIN; in fetch_token()
3269 tok->type = TK_ANCHOR; in fetch_token()
3270 tok->u.anchor = ANCHOR_WORD_END; in fetch_token()
3276 tok->type = TK_CHAR_TYPE; in fetch_token()
3277 tok->u.prop.ctype = ONIGENC_CTYPE_SPACE; in fetch_token()
3278 tok->u.prop.not = 0; in fetch_token()
3283 tok->type = TK_CHAR_TYPE; in fetch_token()
3284 tok->u.prop.ctype = ONIGENC_CTYPE_SPACE; in fetch_token()
3285 tok->u.prop.not = 1; in fetch_token()
3290 tok->type = TK_CHAR_TYPE; in fetch_token()
3291 tok->u.prop.ctype = ONIGENC_CTYPE_DIGIT; in fetch_token()
3292 tok->u.prop.not = 0; in fetch_token()
3297 tok->type = TK_CHAR_TYPE; in fetch_token()
3298 tok->u.prop.ctype = ONIGENC_CTYPE_DIGIT; in fetch_token()
3299 tok->u.prop.not = 1; in fetch_token()
3304 tok->type = TK_CHAR_TYPE; in fetch_token()
3305 tok->u.prop.ctype = ONIGENC_CTYPE_XDIGIT; in fetch_token()
3306 tok->u.prop.not = 0; in fetch_token()
3311 tok->type = TK_CHAR_TYPE; in fetch_token()
3312 tok->u.prop.ctype = ONIGENC_CTYPE_XDIGIT; in fetch_token()
3313 tok->u.prop.not = 1; in fetch_token()
3319 tok->type = TK_ANCHOR; in fetch_token()
3320 tok->u.subtype = ANCHOR_BEGIN_BUF; in fetch_token()
3325 tok->type = TK_ANCHOR; in fetch_token()
3326 tok->u.subtype = ANCHOR_SEMI_END_BUF; in fetch_token()
3332 tok->type = TK_ANCHOR; in fetch_token()
3333 tok->u.subtype = ANCHOR_END_BUF; in fetch_token()
3338 tok->type = TK_ANCHOR; in fetch_token()
3339 tok->u.subtype = ANCHOR_BEGIN_POSITION; in fetch_token()
3367 tok->type = TK_CODE_POINT; in fetch_token()
3368 tok->u.code = (OnigCodePoint )num; in fetch_token()
3381 tok->type = TK_RAW_BYTE; in fetch_token()
3382 tok->base = 16; in fetch_token()
3383 tok->u.c = num; in fetch_token()
3397 tok->type = TK_CODE_POINT; in fetch_token()
3398 tok->base = 16; in fetch_token()
3399 tok->u.code = (OnigCodePoint )num; in fetch_token()
3419 tok->type = TK_BACKREF; in fetch_token()
3420 tok->u.backref.num = 1; in fetch_token()
3421 tok->u.backref.ref1 = num; in fetch_token()
3422 tok->u.backref.by_name = 0; in fetch_token()
3424 tok->u.backref.exist_level = 0; in fetch_token()
3446 tok->type = TK_RAW_BYTE; in fetch_token()
3447 tok->base = 8; in fetch_token()
3448 tok->u.c = num; in fetch_token()
3469 env, &back_num, &tok->u.backref.level); in fetch_token()
3470 if (r == 1) tok->u.backref.exist_level = 1; in fetch_token()
3471 else tok->u.backref.exist_level = 0; in fetch_token()
3489 tok->type = TK_BACKREF; in fetch_token()
3490 tok->u.backref.by_name = 0; in fetch_token()
3491 tok->u.backref.num = 1; in fetch_token()
3492 tok->u.backref.ref1 = back_num; in fetch_token()
3510 tok->type = TK_BACKREF; in fetch_token()
3511 tok->u.backref.by_name = 1; in fetch_token()
3513 tok->u.backref.num = 1; in fetch_token()
3514 tok->u.backref.ref1 = backs[0]; in fetch_token()
3517 tok->u.backref.num = num; in fetch_token()
3518 tok->u.backref.refs = backs; in fetch_token()
3540 tok->type = TK_CALL; in fetch_token()
3541 tok->u.call.name = prev; in fetch_token()
3542 tok->u.call.name_end = name_end; in fetch_token()
3543 tok->u.call.gnum = gnum; in fetch_token()
3553 tok->type = TK_QUOTE_OPEN; in fetch_token()
3562 tok->type = TK_CHAR_PROPERTY; in fetch_token()
3563 tok->u.prop.not = (c == 'P' ? 1 : 0); in fetch_token()
3568 tok->u.prop.not = (tok->u.prop.not == 0 ? 1 : 0); in fetch_token()
3581 if (tok->u.c != num) { in fetch_token()
3582 tok->type = TK_CODE_POINT; in fetch_token()
3583 tok->u.code = (OnigCodePoint )num; in fetch_token()
3587 SAFE_ENC_LEN(enc, tok->backp, end, len); in fetch_token()
3588 p = tok->backp + len; in fetch_token()
3594 tok->u.c = c; in fetch_token()
3595 tok->escaped = 0; in fetch_token()
3609 tok->type = TK_ANYCHAR_ANYTIME; in fetch_token()
3621 tok->type = TK_ANYCHAR; in fetch_token()
3629 tok->type = TK_OP_REPEAT; in fetch_token()
3630 tok->u.repeat.lower = 0; in fetch_token()
3631 tok->u.repeat.upper = REPEAT_INFINITE; in fetch_token()
3640 tok->type = TK_OP_REPEAT; in fetch_token()
3641 tok->u.repeat.lower = 1; in fetch_token()
3642 tok->u.repeat.upper = REPEAT_INFINITE; in fetch_token()
3651 tok->type = TK_OP_REPEAT; in fetch_token()
3652 tok->u.repeat.lower = 0; in fetch_token()
3653 tok->u.repeat.upper = 1; in fetch_token()
3659 r = fetch_range_quantifier(&p, end, tok, env); in fetch_token()
3673 tok->type = TK_ALT; in fetch_token()
3698 tok->type = TK_SUBEXP_OPEN; in fetch_token()
3703 tok->type = TK_SUBEXP_CLOSE; in fetch_token()
3708 tok->type = TK_ANCHOR; in fetch_token()
3709 tok->u.subtype = (IS_SINGLELINE(env->option) in fetch_token()
3715 tok->type = TK_ANCHOR; in fetch_token()
3716 tok->u.subtype = (IS_SINGLELINE(env->option) in fetch_token()
3722 tok->type = TK_CC_OPEN; in fetch_token()
3757 return tok->type; in fetch_token()
4025 parse_char_property(Node** np, OnigToken* tok, UChar** src, UChar* end, in parse_char_property() argument
4039 if (tok->u.prop.not != 0) NCCLASS_SET_NOT(cc); in parse_char_property()
4186 parse_char_class(Node** np, OnigToken* tok, UChar** src, UChar* end, in parse_char_class() argument
4202 r = fetch_token_in_cc(tok, src, end, env); in parse_char_class()
4203 if (r == TK_CHAR && tok->u.c == '^' && tok->escaped == 0) { in parse_char_class()
4205 r = fetch_token_in_cc(tok, src, end, env); in parse_char_class()
4218 r = tok->type = TK_CHAR; /* allow []...] */ in parse_char_class()
4232 len = ONIGENC_CODE_TO_MBCLEN(env->enc, tok->u.c); in parse_char_class()
4244 v = (OnigCodePoint )tok->u.c; in parse_char_class()
4251 if (! ONIGENC_IS_SINGLEBYTE(env->enc) && tok->base != 0) { in parse_char_class()
4255 int i, base = tok->base; in parse_char_class()
4257 buf[0] = tok->u.c; in parse_char_class()
4259 r = fetch_token_in_cc(tok, &p, end, env); in parse_char_class()
4261 if (r != TK_RAW_BYTE || tok->base != base) { in parse_char_class()
4265 buf[i] = tok->u.c; in parse_char_class()
4281 r = fetch_token_in_cc(tok, &p, end, env); in parse_char_class()
4296 v = (OnigCodePoint )tok->u.c; in parse_char_class()
4305 v = tok->u.code; in parse_char_class()
4325 p = tok->backp; in parse_char_class()
4326 v = (OnigCodePoint )tok->u.c; in parse_char_class()
4334 r = add_ctype_to_cc(cc, tok->u.prop.ctype, tok->u.prop.not, env); in parse_char_class()
4348 r = add_ctype_to_cc(cc, ctype, tok->u.prop.not, env); in parse_char_class()
4356 r = fetch_token_in_cc(tok, &p, end, env); in parse_char_class()
4373 v = (OnigCodePoint )tok->u.c; in parse_char_class()
4376 r = fetch_token_in_cc(tok, &p, end, env); in parse_char_class()
4390 r = fetch_token_in_cc(tok, &p, end, env); in parse_char_class()
4413 r = parse_char_class(&anode, tok, &p, end, env); in parse_char_class()
4459 r = tok->type; in parse_char_class()
4461 r = fetch_token_in_cc(tok, &p, end, env); in parse_char_class()
4512 static int parse_subexp(Node** top, OnigToken* tok, int term,
4516 parse_enclose(Node** np, OnigToken* tok, int term, UChar** src, UChar* end, in parse_enclose() argument
4545 r = fetch_token(tok, &p, end, env); in parse_enclose()
4547 r = parse_subexp(np, tok, term, &p, end, env); in parse_enclose()
4705 r = fetch_token(tok, &p, end, env); in parse_enclose()
4707 r = parse_subexp(&target, tok, term, &p, end, env); in parse_enclose()
4739 r = fetch_token(tok, &p, end, env); in parse_enclose()
4741 r = parse_subexp(&target, tok, term, &p, end, env); in parse_enclose()
5048 parse_exp(Node** np, OnigToken* tok, int term, in parse_exp() argument
5056 if (tok->type == (enum TokenSyms )term) in parse_exp()
5059 switch (tok->type) { in parse_exp()
5064 return tok->type; in parse_exp()
5068 r = parse_enclose(np, tok, TK_SUBEXP_CLOSE, src, end, env); in parse_exp()
5076 r = fetch_token(tok, src, end, env); in parse_exp()
5078 r = parse_subexp(&target, tok, term, src, end, env); in parse_exp()
5082 return tok->type; in parse_exp()
5090 if (tok->escaped) goto tk_raw_byte; in parse_exp()
5097 *np = node_new_str(tok->backp, *src); in parse_exp()
5101 r = fetch_token(tok, src, end, env); in parse_exp()
5105 r = onig_node_str_cat(*np, tok->backp, *src); in parse_exp()
5118 *np = node_new_str_raw_char((UChar )tok->u.c); in parse_exp()
5124 r = fetch_token(tok, src, end, env); in parse_exp()
5130 r = fetch_token(tok, src, end, env); in parse_exp()
5148 r = node_str_cat_char(*np, (UChar )tok->u.c); in parse_exp()
5159 int num = ONIGENC_CODE_TO_MBC(env->enc, tok->u.code, buf); in parse_exp()
5190 switch (tok->u.prop.ctype) { in parse_exp()
5192 *np = node_new_ctype(tok->u.prop.ctype, tok->u.prop.not); in parse_exp()
5206 r = ONIGENC_GET_CTYPE_CODE_RANGE(env->enc, tok->u.prop.ctype, in parse_exp()
5215 key.not = tok->u.prop.not; in parse_exp()
5216 key.type = tok->u.prop.ctype; in parse_exp()
5236 *np = node_new_cclass_by_codepoint_range(tok->u.prop.not, in parse_exp()
5257 add_ctype_to_cc(cc, tok->u.prop.ctype, 0, env); in parse_exp()
5258 if (tok->u.prop.not != 0) NCCLASS_SET_NOT(cc); in parse_exp()
5273 r = parse_char_property(np, tok, src, end, env); in parse_exp()
5281 r = parse_char_class(np, tok, src, end, env); in parse_exp()
5326 len = tok->u.backref.num; in parse_exp()
5328 (len > 1 ? tok->u.backref.refs : &(tok->u.backref.ref1)), in parse_exp()
5329 tok->u.backref.by_name, in parse_exp()
5331 tok->u.backref.exist_level, in parse_exp()
5332 tok->u.backref.level, in parse_exp()
5341 int gnum = tok->u.call.gnum; in parse_exp()
5348 *np = node_new_call(tok->u.call.name, tok->u.call.name_end, gnum); in parse_exp()
5356 *np = onig_node_new_anchor(tok->u.anchor); in parse_exp()
5381 r = fetch_token(tok, src, end, env); in parse_exp()
5389 qn = node_new_quantifier(tok->u.repeat.lower, tok->u.repeat.upper, in parse_exp()
5392 NQTFR(qn)->greedy = tok->u.repeat.greedy; in parse_exp()
5399 if (tok->u.repeat.possessive != 0) { in parse_exp()
5439 parse_branch(Node** top, OnigToken* tok, int term, in parse_branch() argument
5446 r = parse_exp(&node, tok, term, src, end, env); in parse_branch()
5456 r = parse_exp(&node, tok, term, src, end, env); in parse_branch()
5476 parse_subexp(Node** top, OnigToken* tok, int term, in parse_subexp() argument
5483 r = parse_branch(&node, tok, term, src, end, env); in parse_subexp()
5496 r = fetch_token(tok, src, end, env); in parse_subexp()
5498 r = parse_branch(&node, tok, term, src, end, env); in parse_subexp()
5505 if (tok->type != (enum TokenSyms )term) in parse_subexp()
5523 OnigToken tok; in parse_regexp() local
5525 r = fetch_token(&tok, src, end, env); in parse_regexp()
5527 r = parse_subexp(top, &tok, TK_EOT, src, end, env); in parse_regexp()
