From 21422e8536a6b95e5a8d1ce4a6083073e562db24 Mon Sep 17 00:00:00 2001 From: K Date: Fri, 14 May 2021 11:32:46 +0200 Subject: [PATCH 001/229] Optimize unpack() for named fields (#6958) Create name using either zend_init_string_fast (no repetitions) or by concatenating the name with zend_print_ulong_to_buf. This is much more efficient than using snprintf. We also avoid repeated strlen() calculations. --- ext/standard/pack.c | 103 ++++++++++++-------- ext/standard/tests/strings/pack_arrays.phpt | 42 ++++++++ 2 files changed, 106 insertions(+), 39 deletions(-) create mode 100644 ext/standard/tests/strings/pack_arrays.phpt diff --git a/ext/standard/pack.c b/ext/standard/pack.c index 3fad071aab662..bfad808121680 100644 --- a/ext/standard/pack.c +++ b/ext/standard/pack.c @@ -739,24 +739,24 @@ PHP_FUNCTION(unpack) while (formatlen-- > 0) { char type = *(format++); char c; - int arg = 1, argb; + int repetitions = 1, argb; char *name; int namelen; - int size=0; + int size = 0; /* Handle format arguments if any */ if (formatlen > 0) { c = *format; if (c >= '0' && c <= '9') { - arg = atoi(format); + repetitions = atoi(format); while (formatlen > 0 && *format >= '0' && *format <= '9') { format++; formatlen--; } } else if (c == '*') { - arg = -1; + repetitions = -1; format++; formatlen--; } @@ -764,7 +764,7 @@ PHP_FUNCTION(unpack) /* Get of new value in array */ name = format; - argb = arg; + argb = repetitions; while (formatlen > 0 && *format != '/') { formatlen--; @@ -780,9 +780,9 @@ PHP_FUNCTION(unpack) /* Never use any input */ case 'X': size = -1; - if (arg < 0) { + if (repetitions < 0) { php_error_docref(NULL, E_WARNING, "Type %c: '*' ignored", type); - arg = 1; + repetitions = 1; } break; @@ -793,14 +793,14 @@ PHP_FUNCTION(unpack) case 'a': case 'A': case 'Z': - size = arg; - arg = 1; + size = repetitions; + repetitions = 1; break; case 'h': case 'H': - size = (arg > 0) ? (arg + (arg % 2)) / 2 : arg; - arg = 1; + size = (repetitions > 0) ? (repetitions + (repetitions % 2)) / 2 : repetitions; + repetitions = 1; break; /* Use 1 byte of input */ @@ -870,18 +870,9 @@ PHP_FUNCTION(unpack) RETURN_FALSE; } - /* Do actual unpacking */ - for (i = 0; i != arg; i++ ) { - /* Space for name + number, safe as namelen is ensured <= 200 */ - char n[256]; - if (arg != 1 || namelen == 0) { - /* Need to add element number to name */ - snprintf(n, sizeof(n), "%.*s%d", namelen, name, i + 1); - } else { - /* Truncate name to next format code or end of string */ - snprintf(n, sizeof(n), "%.*s", namelen, name); - } + /* Do actual unpacking */ + for (i = 0; i != repetitions; i++ ) { if (size != 0 && size != -1 && INT_MAX - size + 1 < inputpos) { php_error_docref(NULL, E_WARNING, "Type %c: integer overflow", type); @@ -890,6 +881,22 @@ PHP_FUNCTION(unpack) } if ((inputpos + size) <= inputlen) { + + zend_string* real_name; + zval val; + + if (repetitions == 1 && namelen > 0) { + /* Use a part of the formatarg argument directly as the name. */ + real_name = zend_string_init_fast(name, namelen); + + } else { + /* Need to add the 1-based element number to the name */ + char buf[MAX_LENGTH_OF_LONG + 1]; + char *res = zend_print_ulong_to_buf(buf + sizeof(buf) - 1, i+1); + size_t digits = buf + sizeof(buf) - 1 - res; + real_name = zend_string_concat2(name, namelen, res, digits); + } + switch ((int) type) { case 'a': { /* a will not strip any trailing whitespace or null padding */ @@ -902,7 +909,8 @@ PHP_FUNCTION(unpack) size = len; - add_assoc_stringl(return_value, n, &input[inputpos], len); + ZVAL_STRINGL(&val, &input[inputpos], len); + zend_symtable_update(Z_ARRVAL_P(return_value), real_name, &val); break; } case 'A': { @@ -928,7 +936,8 @@ PHP_FUNCTION(unpack) break; } - add_assoc_stringl(return_value, n, &input[inputpos], len + 1); + ZVAL_STRINGL(&val, &input[inputpos], len + 1); + zend_symtable_update(Z_ARRVAL_P(return_value), real_name, &val); break; } /* New option added for Z to remain in-line with the Perl implementation */ @@ -952,7 +961,8 @@ PHP_FUNCTION(unpack) } len = s; - add_assoc_stringl(return_value, n, &input[inputpos], len); + ZVAL_STRINGL(&val, &input[inputpos], len); + zend_symtable_update(Z_ARRVAL_P(return_value), real_name, &val); break; } @@ -995,7 +1005,9 @@ PHP_FUNCTION(unpack) } ZSTR_VAL(buf)[len] = '\0'; - add_assoc_str(return_value, n, buf); + + ZVAL_STR(&val, buf); + zend_symtable_update(Z_ARRVAL_P(return_value), real_name, &val); break; } @@ -1003,7 +1015,9 @@ PHP_FUNCTION(unpack) case 'C': { /* unsigned */ uint8_t x = input[inputpos]; zend_long v = (type == 'c') ? (int8_t) x : x; - add_assoc_long(return_value, n, v); + + ZVAL_LONG(&val, v); + zend_symtable_update(Z_ARRVAL_P(return_value), real_name, &val); break; } @@ -1022,7 +1036,8 @@ PHP_FUNCTION(unpack) v = x; } - add_assoc_long(return_value, n, v); + ZVAL_LONG(&val, v); + zend_symtable_update(Z_ARRVAL_P(return_value), real_name, &val); break; } @@ -1030,7 +1045,9 @@ PHP_FUNCTION(unpack) case 'I': { /* unsigned integer, machine size, machine endian */ unsigned int x = *((unaligned_uint*) &input[inputpos]); zend_long v = (type == 'i') ? (int) x : x; - add_assoc_long(return_value, n, v); + + ZVAL_LONG(&val, v); + zend_symtable_update(Z_ARRVAL_P(return_value), real_name, &val); break; } @@ -1049,7 +1066,9 @@ PHP_FUNCTION(unpack) v = x; } - add_assoc_long(return_value, n, v); + ZVAL_LONG(&val, v); + zend_symtable_update(Z_ARRVAL_P(return_value), real_name, &val); + break; } @@ -1069,7 +1088,8 @@ PHP_FUNCTION(unpack) v = x; } - add_assoc_long(return_value, n, v); + ZVAL_LONG(&val, v); + zend_symtable_update(Z_ARRVAL_P(return_value), real_name, &val); break; } #endif @@ -1088,7 +1108,8 @@ PHP_FUNCTION(unpack) memcpy(&v, &input[inputpos], sizeof(float)); } - add_assoc_double(return_value, n, (double)v); + ZVAL_DOUBLE(&val, v); + zend_symtable_update(Z_ARRVAL_P(return_value), real_name, &val); break; } @@ -1105,7 +1126,9 @@ PHP_FUNCTION(unpack) } else { memcpy(&v, &input[inputpos], sizeof(double)); } - add_assoc_double(return_value, n, v); + + ZVAL_DOUBLE(&val, v); + zend_symtable_update(Z_ARRVAL_P(return_value), real_name, &val); break; } @@ -1116,25 +1139,27 @@ PHP_FUNCTION(unpack) case 'X': if (inputpos < size) { inputpos = -size; - i = arg - 1; /* Break out of for loop */ + i = repetitions - 1; /* Break out of for loop */ - if (arg >= 0) { + if (repetitions >= 0) { php_error_docref(NULL, E_WARNING, "Type %c: outside of string", type); } } break; case '@': - if (arg <= inputlen) { - inputpos = arg; + if (repetitions <= inputlen) { + inputpos = repetitions; } else { php_error_docref(NULL, E_WARNING, "Type %c: outside of string", type); } - i = arg - 1; /* Done, break out of for loop */ + i = repetitions - 1; /* Done, break out of for loop */ break; } + zend_string_release(real_name); + inputpos += size; if (inputpos < 0) { if (size != -1) { /* only print warning if not working with * */ @@ -1142,7 +1167,7 @@ PHP_FUNCTION(unpack) } inputpos = 0; } - } else if (arg < 0) { + } else if (repetitions < 0) { /* Reached end of input for '*' repeater */ break; } else { diff --git a/ext/standard/tests/strings/pack_arrays.phpt b/ext/standard/tests/strings/pack_arrays.phpt new file mode 100644 index 0000000000000..301897006c1b4 --- /dev/null +++ b/ext/standard/tests/strings/pack_arrays.phpt @@ -0,0 +1,42 @@ +--TEST-- +test unpack() to array with named keys +--FILE-- + +--EXPECT-- +Array +( + [aa] => 66051 + [bb] => 67438087 + [cc] => 134810123 +) +Array +( + [aa1] => 66051 + [aa2] => 67438087 + [cc] => 134810123 +) +Array +( + [aa1] => 66051 + [aa2] => 67438087 + [aa3] => 134810123 +) +Array +( + [aa1] => 66051 + [aa2] => 67438087 + [aa3] => 134810123 +) +Array +( + [1] => 66051 + [2] => 67438087 + [3] => 134810123 +) From d764f1dc12a1778d0c82c474430cb1da16038473 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Tantau?= Date: Fri, 14 May 2021 11:43:55 +0200 Subject: [PATCH 002/229] Fix #77372: Retain full path of files for directory uploads (#6917) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To fix https://bugs.php.net/bug.php?id=77372 and improve support of `` I introduced another item to the `$_FILES` array called `full_path`, containing the full filename, as supplied by the user-agent. Co-authored-by: Björn Tantau --- ext/session/tests/rfc1867.phpt | 8 +- ext/session/tests/rfc1867_cleanup.phpt | 8 +- ext/session/tests/rfc1867_disabled.phpt | 8 +- ext/session/tests/rfc1867_disabled_2.phpt | 8 +- ext/session/tests/rfc1867_inter.phpt | 8 +- ext/session/tests/rfc1867_no_name.phpt | 8 +- ext/session/tests/rfc1867_sid_cookie.phpt | 8 +- ext/session/tests/rfc1867_sid_get.phpt | 8 +- ext/session/tests/rfc1867_sid_get_2.phpt | 8 +- ext/session/tests/rfc1867_sid_invalid.phpt | 8 +- .../tests/rfc1867_sid_only_cookie.phpt | 8 +- .../tests/rfc1867_sid_only_cookie_2.phpt | 8 +- ext/session/tests/rfc1867_sid_post.phpt | 8 +- main/rfc1867.c | 15 +++- sapi/cli/tests/php_cli_server_005.phpt | 4 +- tests/basic/021.phpt | 4 +- tests/basic/029.phpt | 4 +- tests/basic/bug55500.phpt | 7 +- tests/basic/rfc1867_anonymous_upload.phpt | 8 +- tests/basic/rfc1867_array_upload.phpt | 11 ++- tests/basic/rfc1867_empty_upload.phpt | 12 ++- tests/basic/rfc1867_max_file_size.phpt | 12 ++- .../rfc1867_max_file_uploads_empty_files.phpt | 16 +++- tests/basic/rfc1867_missing_boundary_2.phpt | 4 +- .../rfc1867_multiple_webkitdirectory.phpt | 74 +++++++++++++++++++ tests/basic/rfc1867_post_max_filesize.phpt | 12 ++- 26 files changed, 238 insertions(+), 49 deletions(-) create mode 100644 tests/basic/rfc1867_multiple_webkitdirectory.phpt diff --git a/ext/session/tests/rfc1867.phpt b/ext/session/tests/rfc1867.phpt index a5ae8b22181f7..aed179d757333 100644 --- a/ext/session/tests/rfc1867.phpt +++ b/ext/session/tests/rfc1867.phpt @@ -54,9 +54,11 @@ string(%d) "rfc1867" bool(true) array(2) { ["file1"]=> - array(5) { + array(6) { ["name"]=> string(9) "file1.txt" + ["full_path"]=> + string(9) "file1.txt" ["type"]=> string(0) "" ["tmp_name"]=> @@ -67,9 +69,11 @@ array(2) { int(1) } ["file2"]=> - array(5) { + array(6) { ["name"]=> string(9) "file2.txt" + ["full_path"]=> + string(9) "file2.txt" ["type"]=> string(0) "" ["tmp_name"]=> diff --git a/ext/session/tests/rfc1867_cleanup.phpt b/ext/session/tests/rfc1867_cleanup.phpt index 9baa6144ba688..9a0bf626f33ad 100644 --- a/ext/session/tests/rfc1867_cleanup.phpt +++ b/ext/session/tests/rfc1867_cleanup.phpt @@ -54,9 +54,11 @@ string(%d) "rfc1867-cleanup" bool(true) array(2) { ["file1"]=> - array(5) { + array(6) { ["name"]=> string(9) "file1.txt" + ["full_path"]=> + string(9) "file1.txt" ["type"]=> string(0) "" ["tmp_name"]=> @@ -67,9 +69,11 @@ array(2) { int(1) } ["file2"]=> - array(5) { + array(6) { ["name"]=> string(9) "file2.txt" + ["full_path"]=> + string(9) "file2.txt" ["type"]=> string(0) "" ["tmp_name"]=> diff --git a/ext/session/tests/rfc1867_disabled.phpt b/ext/session/tests/rfc1867_disabled.phpt index 8d8effd1f42ca..43c1079064d65 100644 --- a/ext/session/tests/rfc1867_disabled.phpt +++ b/ext/session/tests/rfc1867_disabled.phpt @@ -47,9 +47,11 @@ session_destroy(); string(%d) "rfc1867-disabled" array(2) { ["file1"]=> - array(5) { + array(6) { ["name"]=> string(9) "file1.txt" + ["full_path"]=> + string(9) "file1.txt" ["type"]=> string(0) "" ["tmp_name"]=> @@ -60,9 +62,11 @@ array(2) { int(1) } ["file2"]=> - array(5) { + array(6) { ["name"]=> string(9) "file2.txt" + ["full_path"]=> + string(9) "file2.txt" ["type"]=> string(0) "" ["tmp_name"]=> diff --git a/ext/session/tests/rfc1867_disabled_2.phpt b/ext/session/tests/rfc1867_disabled_2.phpt index c539c6eaeae4e..f2ff6ebb96424 100644 --- a/ext/session/tests/rfc1867_disabled_2.phpt +++ b/ext/session/tests/rfc1867_disabled_2.phpt @@ -47,9 +47,11 @@ session_destroy(); string(%d) "rfc1867-disabled-2" array(2) { ["file1"]=> - array(5) { + array(6) { ["name"]=> string(9) "file1.txt" + ["full_path"]=> + string(9) "file1.txt" ["type"]=> string(0) "" ["tmp_name"]=> @@ -60,9 +62,11 @@ array(2) { int(1) } ["file2"]=> - array(5) { + array(6) { ["name"]=> string(9) "file2.txt" + ["full_path"]=> + string(9) "file2.txt" ["type"]=> string(0) "" ["tmp_name"]=> diff --git a/ext/session/tests/rfc1867_inter.phpt b/ext/session/tests/rfc1867_inter.phpt index fd28dfe07a30c..db0ca00a05dcc 100644 --- a/ext/session/tests/rfc1867_inter.phpt +++ b/ext/session/tests/rfc1867_inter.phpt @@ -57,9 +57,11 @@ session_destroy(); string(%d) "rfc1867-inter" array(2) { ["file1"]=> - array(5) { + array(6) { ["name"]=> string(9) "file1.txt" + ["full_path"]=> + string(9) "file1.txt" ["type"]=> string(0) "" ["tmp_name"]=> @@ -70,9 +72,11 @@ array(2) { int(1) } ["file2"]=> - array(5) { + array(6) { ["name"]=> string(9) "file2.txt" + ["full_path"]=> + string(9) "file2.txt" ["type"]=> string(0) "" ["tmp_name"]=> diff --git a/ext/session/tests/rfc1867_no_name.phpt b/ext/session/tests/rfc1867_no_name.phpt index 15877a664e644..b27120dc5bb33 100644 --- a/ext/session/tests/rfc1867_no_name.phpt +++ b/ext/session/tests/rfc1867_no_name.phpt @@ -47,9 +47,11 @@ session_destroy(); string(%d) "rfc1867-no-name" array(2) { ["file1"]=> - array(5) { + array(6) { ["name"]=> string(9) "file1.txt" + ["full_path"]=> + string(9) "file1.txt" ["type"]=> string(0) "" ["tmp_name"]=> @@ -60,9 +62,11 @@ array(2) { int(1) } ["file2"]=> - array(5) { + array(6) { ["name"]=> string(9) "file2.txt" + ["full_path"]=> + string(9) "file2.txt" ["type"]=> string(0) "" ["tmp_name"]=> diff --git a/ext/session/tests/rfc1867_sid_cookie.phpt b/ext/session/tests/rfc1867_sid_cookie.phpt index 85c28934f440b..9acd8d68f808b 100644 --- a/ext/session/tests/rfc1867_sid_cookie.phpt +++ b/ext/session/tests/rfc1867_sid_cookie.phpt @@ -53,9 +53,11 @@ string(%d) "rfc1867-sid-cookie" bool(true) array(2) { ["file1"]=> - array(5) { + array(6) { ["name"]=> string(9) "file1.txt" + ["full_path"]=> + string(9) "file1.txt" ["type"]=> string(0) "" ["tmp_name"]=> @@ -66,9 +68,11 @@ array(2) { int(1) } ["file2"]=> - array(5) { + array(6) { ["name"]=> string(9) "file2.txt" + ["full_path"]=> + string(9) "file2.txt" ["type"]=> string(0) "" ["tmp_name"]=> diff --git a/ext/session/tests/rfc1867_sid_get.phpt b/ext/session/tests/rfc1867_sid_get.phpt index dfb192cb47cb0..b9dde7bb2ed01 100644 --- a/ext/session/tests/rfc1867_sid_get.phpt +++ b/ext/session/tests/rfc1867_sid_get.phpt @@ -51,9 +51,11 @@ string(%d) "rfc1867-sid-get" bool(true) array(2) { ["file1"]=> - array(5) { + array(6) { ["name"]=> string(9) "file1.txt" + ["full_path"]=> + string(9) "file1.txt" ["type"]=> string(0) "" ["tmp_name"]=> @@ -64,9 +66,11 @@ array(2) { int(1) } ["file2"]=> - array(5) { + array(6) { ["name"]=> string(9) "file2.txt" + ["full_path"]=> + string(9) "file2.txt" ["type"]=> string(0) "" ["tmp_name"]=> diff --git a/ext/session/tests/rfc1867_sid_get_2.phpt b/ext/session/tests/rfc1867_sid_get_2.phpt index 33e4489cc8bd8..4f0f598a8a517 100644 --- a/ext/session/tests/rfc1867_sid_get_2.phpt +++ b/ext/session/tests/rfc1867_sid_get_2.phpt @@ -53,9 +53,11 @@ string(%d) "rfc1867-sid-get-2" bool(true) array(2) { ["file1"]=> - array(5) { + array(6) { ["name"]=> string(9) "file1.txt" + ["full_path"]=> + string(9) "file1.txt" ["type"]=> string(0) "" ["tmp_name"]=> @@ -66,9 +68,11 @@ array(2) { int(1) } ["file2"]=> - array(5) { + array(6) { ["name"]=> string(9) "file2.txt" + ["full_path"]=> + string(9) "file2.txt" ["type"]=> string(0) "" ["tmp_name"]=> diff --git a/ext/session/tests/rfc1867_sid_invalid.phpt b/ext/session/tests/rfc1867_sid_invalid.phpt index 4d8372c538149..23e3bdcd37cf7 100644 --- a/ext/session/tests/rfc1867_sid_invalid.phpt +++ b/ext/session/tests/rfc1867_sid_invalid.phpt @@ -65,9 +65,11 @@ string(%d) "" bool(true) array(2) { ["file1"]=> - array(5) { + array(6) { ["name"]=> string(9) "file1.txt" + ["full_path"]=> + string(9) "file1.txt" ["type"]=> string(0) "" ["tmp_name"]=> @@ -78,9 +80,11 @@ array(2) { int(1) } ["file2"]=> - array(5) { + array(6) { ["name"]=> string(9) "file2.txt" + ["full_path"]=> + string(9) "file2.txt" ["type"]=> string(0) "" ["tmp_name"]=> diff --git a/ext/session/tests/rfc1867_sid_only_cookie.phpt b/ext/session/tests/rfc1867_sid_only_cookie.phpt index 54897b91c8582..d438068c8f661 100644 --- a/ext/session/tests/rfc1867_sid_only_cookie.phpt +++ b/ext/session/tests/rfc1867_sid_only_cookie.phpt @@ -53,9 +53,11 @@ string(%d) "rfc1867-sid-only-cookie" bool(true) array(2) { ["file1"]=> - array(5) { + array(6) { ["name"]=> string(9) "file1.txt" + ["full_path"]=> + string(9) "file1.txt" ["type"]=> string(0) "" ["tmp_name"]=> @@ -66,9 +68,11 @@ array(2) { int(1) } ["file2"]=> - array(5) { + array(6) { ["name"]=> string(9) "file2.txt" + ["full_path"]=> + string(9) "file2.txt" ["type"]=> string(0) "" ["tmp_name"]=> diff --git a/ext/session/tests/rfc1867_sid_only_cookie_2.phpt b/ext/session/tests/rfc1867_sid_only_cookie_2.phpt index 3fd46148d724e..698fc1609d36e 100644 --- a/ext/session/tests/rfc1867_sid_only_cookie_2.phpt +++ b/ext/session/tests/rfc1867_sid_only_cookie_2.phpt @@ -50,9 +50,11 @@ string(%d) "%s" bool(true) array(2) { ["file1"]=> - array(5) { + array(6) { ["name"]=> string(9) "file1.txt" + ["full_path"]=> + string(9) "file1.txt" ["type"]=> string(0) "" ["tmp_name"]=> @@ -63,9 +65,11 @@ array(2) { int(1) } ["file2"]=> - array(5) { + array(6) { ["name"]=> string(9) "file2.txt" + ["full_path"]=> + string(9) "file2.txt" ["type"]=> string(0) "" ["tmp_name"]=> diff --git a/ext/session/tests/rfc1867_sid_post.phpt b/ext/session/tests/rfc1867_sid_post.phpt index f22f1534039dc..4ab3e6c33efb8 100644 --- a/ext/session/tests/rfc1867_sid_post.phpt +++ b/ext/session/tests/rfc1867_sid_post.phpt @@ -49,9 +49,11 @@ string(%d) "rfc1867-sid-post" bool(true) array(2) { ["file1"]=> - array(5) { + array(6) { ["name"]=> string(9) "file1.txt" + ["full_path"]=> + string(9) "file1.txt" ["type"]=> string(0) "" ["tmp_name"]=> @@ -62,9 +64,11 @@ array(2) { int(1) } ["file2"]=> - array(5) { + array(6) { ["name"]=> string(9) "file2.txt" + ["full_path"]=> + string(9) "file2.txt" ["type"]=> string(0) "" ["tmp_name"]=> diff --git a/main/rfc1867.c b/main/rfc1867.c index 583b3166d537f..315e87b041425 100644 --- a/main/rfc1867.c +++ b/main/rfc1867.c @@ -55,7 +55,7 @@ PHPAPI int (*php_rfc1867_callback)(unsigned int event, void *event_data, void ** static void safe_php_register_variable(char *var, char *strval, size_t val_len, zval *track_vars_array, bool override_protection); /* The longest property name we use in an uploaded file array */ -#define MAX_SIZE_OF_INDEX sizeof("[tmp_name]") +#define MAX_SIZE_OF_INDEX sizeof("[full_path]") /* The longest anonymous name */ #define MAX_SIZE_ANONNAME 33 @@ -1142,9 +1142,20 @@ SAPI_API SAPI_POST_HANDLER_FUNC(rfc1867_post_handler) /* {{{ */ snprintf(lbuf, llen, "%s[name]", param); } register_http_post_files_variable(lbuf, s, &PG(http_globals)[TRACK_VARS_FILES], 0); - efree(filename); s = NULL; + /* Add full path of supplied file for folder uploads via + * + */ + /* Add $foo[full_path] */ + if (is_arr_upload) { + snprintf(lbuf, llen, "%s[full_path][%s]", abuf, array_index); + } else { + snprintf(lbuf, llen, "%s[full_path]", param); + } + register_http_post_files_variable(lbuf, filename, &PG(http_globals)[TRACK_VARS_FILES], 0); + efree(filename); + /* Possible Content-Type: */ if (cancel_upload || !(cd = php_mime_get_hdr_value(header, "Content-Type"))) { cd = ""; diff --git a/sapi/cli/tests/php_cli_server_005.phpt b/sapi/cli/tests/php_cli_server_005.phpt index 41e57881a10b9..339f25424499b 100644 --- a/sapi/cli/tests/php_cli_server_005.phpt +++ b/sapi/cli/tests/php_cli_server_005.phpt @@ -52,9 +52,11 @@ Content-type: text/html; charset=UTF-8 array(1) { ["userfile"]=> - array(5) { + array(6) { ["name"]=> string(12) "laruence.txt" + ["full_path"]=> + string(12) "laruence.txt" ["type"]=> string(10) "text/plain" ["tmp_name"]=> diff --git a/tests/basic/021.phpt b/tests/basic/021.phpt index eeaf58869b0b5..37e853e58ac70 100644 --- a/tests/basic/021.phpt +++ b/tests/basic/021.phpt @@ -24,9 +24,11 @@ var_dump($_POST); --EXPECTF-- array(1) { ["pics"]=> - array(5) { + array(6) { ["name"]=> string(12) "bug37276.txt" + ["full_path"]=> + string(12) "bug37276.txt" ["type"]=> string(10) "text/plain" ["tmp_name"]=> diff --git a/tests/basic/029.phpt b/tests/basic/029.phpt index 21d9082cffb8b..d720cbc6ba9a0 100644 --- a/tests/basic/029.phpt +++ b/tests/basic/029.phpt @@ -32,9 +32,11 @@ var_dump($_POST); --EXPECTF-- array(1) { ["pics"]=> - array(5) { + array(6) { ["name"]=> string(9) "file1.txt" + ["full_path"]=> + string(9) "file1.txt" ["type"]=> string(10) "text/plain" ["tmp_name"]=> diff --git a/tests/basic/bug55500.phpt b/tests/basic/bug55500.phpt index 2f9e393348aec..4f2387e0f184e 100644 --- a/tests/basic/bug55500.phpt +++ b/tests/basic/bug55500.phpt @@ -36,12 +36,17 @@ var_dump($_POST); --EXPECTF-- array(1) { ["file"]=> - array(5) { + array(6) { ["name"]=> array(1) { [0]=> string(9) "file1.txt" } + ["full_path"]=> + array(1) { + [0]=> + string(9) "file1.txt" + } ["type"]=> array(1) { [0]=> diff --git a/tests/basic/rfc1867_anonymous_upload.phpt b/tests/basic/rfc1867_anonymous_upload.phpt index 5650b5cd5db09..923ee3e258d1c 100644 --- a/tests/basic/rfc1867_anonymous_upload.phpt +++ b/tests/basic/rfc1867_anonymous_upload.phpt @@ -25,9 +25,11 @@ var_dump($_POST); --EXPECTF-- array(2) { [%d]=> - array(5) { + array(6) { ["name"]=> string(9) "file1.txt" + ["full_path"]=> + string(9) "file1.txt" ["type"]=> string(16) "text/plain-file1" ["tmp_name"]=> @@ -38,9 +40,11 @@ array(2) { int(1) } [%d]=> - array(5) { + array(6) { ["name"]=> string(9) "file2.txt" + ["full_path"]=> + string(9) "file2.txt" ["type"]=> string(16) "text/plain-file2" ["tmp_name"]=> diff --git a/tests/basic/rfc1867_array_upload.phpt b/tests/basic/rfc1867_array_upload.phpt index 90ed0c36e02b5..9f48e59913c50 100644 --- a/tests/basic/rfc1867_array_upload.phpt +++ b/tests/basic/rfc1867_array_upload.phpt @@ -30,7 +30,7 @@ var_dump($_POST); --EXPECTF-- array(1) { ["file"]=> - array(5) { + array(6) { ["name"]=> array(3) { [0]=> @@ -40,6 +40,15 @@ array(1) { [3]=> string(9) "file3.txt" } + ["full_path"]=> + array(3) { + [0]=> + string(9) "file1.txt" + [2]=> + string(9) "file2.txt" + [3]=> + string(9) "file3.txt" + } ["type"]=> array(3) { [0]=> diff --git a/tests/basic/rfc1867_empty_upload.phpt b/tests/basic/rfc1867_empty_upload.phpt index 2b89ca8888b71..c8a96955be905 100644 --- a/tests/basic/rfc1867_empty_upload.phpt +++ b/tests/basic/rfc1867_empty_upload.phpt @@ -40,9 +40,11 @@ if (is_uploaded_file($_FILES["file3"]["tmp_name"])) { --EXPECTF-- array(3) { ["file1"]=> - array(5) { + array(6) { ["name"]=> string(9) "file1.txt" + ["full_path"]=> + string(9) "file1.txt" ["type"]=> string(16) "text/plain-file1" ["tmp_name"]=> @@ -53,9 +55,11 @@ array(3) { int(1) } ["file2"]=> - array(5) { + array(6) { ["name"]=> string(0) "" + ["full_path"]=> + string(0) "" ["type"]=> string(0) "" ["tmp_name"]=> @@ -66,9 +70,11 @@ array(3) { int(0) } ["file3"]=> - array(5) { + array(6) { ["name"]=> string(9) "file3.txt" + ["full_path"]=> + string(9) "file3.txt" ["type"]=> string(16) "text/plain-file3" ["tmp_name"]=> diff --git a/tests/basic/rfc1867_max_file_size.phpt b/tests/basic/rfc1867_max_file_size.phpt index 8d585f750322b..81133e30a2dec 100644 --- a/tests/basic/rfc1867_max_file_size.phpt +++ b/tests/basic/rfc1867_max_file_size.phpt @@ -40,9 +40,11 @@ if (is_uploaded_file($_FILES["file3"]["tmp_name"])) { --EXPECTF-- array(3) { ["file1"]=> - array(5) { + array(6) { ["name"]=> string(9) "file1.txt" + ["full_path"]=> + string(9) "file1.txt" ["type"]=> string(16) "text/plain-file1" ["tmp_name"]=> @@ -53,9 +55,11 @@ array(3) { int(1) } ["file2"]=> - array(5) { + array(6) { ["name"]=> string(9) "file2.txt" + ["full_path"]=> + string(9) "file2.txt" ["type"]=> string(0) "" ["tmp_name"]=> @@ -66,9 +70,11 @@ array(3) { int(0) } ["file3"]=> - array(5) { + array(6) { ["name"]=> string(9) "file3.txt" + ["full_path"]=> + string(20) "C:\foo\bar/file3.txt" ["type"]=> string(16) "text/plain-file3" ["tmp_name"]=> diff --git a/tests/basic/rfc1867_max_file_uploads_empty_files.phpt b/tests/basic/rfc1867_max_file_uploads_empty_files.phpt index b85ed20971989..e95b8454d1337 100644 --- a/tests/basic/rfc1867_max_file_uploads_empty_files.phpt +++ b/tests/basic/rfc1867_max_file_uploads_empty_files.phpt @@ -40,9 +40,11 @@ if (is_uploaded_file($_FILES["file4"]["tmp_name"])) { --EXPECTF-- array(4) { ["file2"]=> - array(5) { + array(6) { ["name"]=> string(0) "" + ["full_path"]=> + string(0) "" ["type"]=> string(0) "" ["tmp_name"]=> @@ -53,9 +55,11 @@ array(4) { int(0) } ["file3"]=> - array(5) { + array(6) { ["name"]=> string(0) "" + ["full_path"]=> + string(0) "" ["type"]=> string(0) "" ["tmp_name"]=> @@ -66,9 +70,11 @@ array(4) { int(0) } ["file4"]=> - array(5) { + array(6) { ["name"]=> string(9) "file4.txt" + ["full_path"]=> + string(9) "file4.txt" ["type"]=> string(15) "text/plain-file" ["tmp_name"]=> @@ -79,9 +85,11 @@ array(4) { int(0) } ["file1"]=> - array(5) { + array(6) { ["name"]=> string(9) "file1.txt" + ["full_path"]=> + string(9) "file1.txt" ["type"]=> string(15) "text/plain-file" ["tmp_name"]=> diff --git a/tests/basic/rfc1867_missing_boundary_2.phpt b/tests/basic/rfc1867_missing_boundary_2.phpt index d3f93f838715c..7011a2d86b8dd 100644 --- a/tests/basic/rfc1867_missing_boundary_2.phpt +++ b/tests/basic/rfc1867_missing_boundary_2.phpt @@ -18,9 +18,11 @@ var_dump($_POST); --EXPECT-- array(1) { ["file1"]=> - array(5) { + array(6) { ["name"]=> string(9) "file1.txt" + ["full_path"]=> + string(9) "file1.txt" ["type"]=> string(0) "" ["tmp_name"]=> diff --git a/tests/basic/rfc1867_multiple_webkitdirectory.phpt b/tests/basic/rfc1867_multiple_webkitdirectory.phpt new file mode 100644 index 0000000000000..00dc12fda6086 --- /dev/null +++ b/tests/basic/rfc1867_multiple_webkitdirectory.phpt @@ -0,0 +1,74 @@ +--TEST-- +Request #77372 (Relative file path is removed from uploaded file) +--INI-- +file_uploads=1 +upload_max_filesize=1024 +max_file_uploads=10 +--POST_RAW-- +Content-Type: multipart/form-data; boundary=---------------------------64369134225794159231042985467 +-----------------------------64369134225794159231042985467 +Content-Disposition: form-data; name="files[]"; filename="directory/subdirectory/file2.txt" +Content-Type: text/plain + +2 +-----------------------------64369134225794159231042985467 +Content-Disposition: form-data; name="files[]"; filename="directory/file1.txt" +Content-Type: text/plain + +1 +-----------------------------64369134225794159231042985467-- +--FILE-- + +--EXPECTF-- +array(1) { + ["files"]=> + array(6) { + ["name"]=> + array(2) { + [0]=> + string(9) "file2.txt" + [1]=> + string(9) "file1.txt" + } + ["full_path"]=> + array(2) { + [0]=> + string(32) "directory/subdirectory/file2.txt" + [1]=> + string(19) "directory/file1.txt" + } + ["type"]=> + array(2) { + [0]=> + string(10) "text/plain" + [1]=> + string(10) "text/plain" + } + ["tmp_name"]=> + array(2) { + [0]=> + string(%d) "%s" + [1]=> + string(%d) "%s" + } + ["error"]=> + array(2) { + [0]=> + int(0) + [1]=> + int(0) + } + ["size"]=> + array(2) { + [0]=> + int(1) + [1]=> + int(1) + } + } +} +array(0) { +} diff --git a/tests/basic/rfc1867_post_max_filesize.phpt b/tests/basic/rfc1867_post_max_filesize.phpt index b03220e915a7b..f8e99e574fd08 100644 --- a/tests/basic/rfc1867_post_max_filesize.phpt +++ b/tests/basic/rfc1867_post_max_filesize.phpt @@ -36,9 +36,11 @@ if (is_uploaded_file($_FILES["file3"]["tmp_name"])) { --EXPECTF-- array(3) { ["file1"]=> - array(5) { + array(6) { ["name"]=> string(9) "file1.txt" + ["full_path"]=> + string(9) "file1.txt" ["type"]=> string(16) "text/plain-file1" ["tmp_name"]=> @@ -49,9 +51,11 @@ array(3) { int(1) } ["file2"]=> - array(5) { + array(6) { ["name"]=> string(9) "file2.txt" + ["full_path"]=> + string(9) "file2.txt" ["type"]=> string(0) "" ["tmp_name"]=> @@ -62,9 +66,11 @@ array(3) { int(0) } ["file3"]=> - array(5) { + array(6) { ["name"]=> string(9) "file3.txt" + ["full_path"]=> + string(9) "file3.txt" ["type"]=> string(16) "text/plain-file3" ["tmp_name"]=> From 6d5c60e945f2adb4e5bc1bd229b0ec59aff3d1aa Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Fri, 14 May 2021 11:47:14 +0200 Subject: [PATCH 003/229] Add UPGRADING/NEWS for full_path for file uploads [ci skip] --- NEWS | 2 ++ UPGRADING | 3 +++ 2 files changed, 5 insertions(+) diff --git a/NEWS b/NEWS index 482088e0e09a8..d94b43464ff64 100644 --- a/NEWS +++ b/NEWS @@ -9,6 +9,8 @@ PHP NEWS function). (Nikita) . Fixed bug #53826 (__callStatic fired in base class through a parent call if the method is private). (Nikita) + . Implemented FR #77372 (Relative file path is removed from uploaded file). + (Björn Tantau) - Date: . Fixed bug #52480 (Incorrect difference using DateInterval) (Derick) diff --git a/UPGRADING b/UPGRADING index 5a6c736e364eb..f5342a7577be6 100644 --- a/UPGRADING +++ b/UPGRADING @@ -176,6 +176,9 @@ PHP 8.1 UPGRADE NOTES RFC: https://wiki.php.net/rfc/noreturn_type . Added support for fibers. RFC: https://wiki.php.net/rfc/fibers + . File uploads now provide an additional full_path key, which contains the + full path (rather than just the basename) of the uploaded file. This is + intended for use in conjunction with "upload webkitdirectory". - Curl: . Added CURLOPT_DOH_URL option. From 840c78b2d84561bd61af63bf334f94011dde52fa Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Fri, 14 May 2021 12:02:28 +0200 Subject: [PATCH 004/229] Check RAND_egd after setting up openssl Noticed this by accident: We should check functions in the library only after setting it up. --- ext/openssl/config0.m4 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ext/openssl/config0.m4 b/ext/openssl/config0.m4 index e08a76897aeeb..ffd4e0751cc6b 100644 --- a/ext/openssl/config0.m4 +++ b/ext/openssl/config0.m4 @@ -28,14 +28,15 @@ if test "$PHP_OPENSSL" != "no"; then PHP_EVAL_LIBLINE($KERBEROS_LIBS, OPENSSL_SHARED_LIBADD) fi - AC_CHECK_FUNCS([RAND_egd]) - PHP_SETUP_OPENSSL(OPENSSL_SHARED_LIBADD, [ AC_DEFINE(HAVE_OPENSSL_EXT,1,[ ]) ], [ AC_MSG_ERROR([OpenSSL check failed. Please check config.log for more information.]) ]) + + AC_CHECK_FUNCS([RAND_egd]) + if test "$PHP_SYSTEM_CIPHERS" != "no"; then AC_DEFINE(USE_OPENSSL_SYSTEM_CIPHERS,1,[ Use system default cipher list instead of hardcoded value ]) fi From 959e5787bdf7c088a57dce5f4f7570abd7fe35f8 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Wed, 12 May 2021 10:02:02 +0200 Subject: [PATCH 005/229] Disable -a mode without readline To avoid confusing, as -a without readline is not actually interactive. Discussion: https://externals.io/message/114426 Closes GH-6976. --- UPGRADING | 6 ++++++ sapi/cli/php_cli.c | 18 ++++++------------ sapi/cli/tests/009.phpt | 2 ++ sapi/cli/tests/012-2.phpt | 2 ++ 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/UPGRADING b/UPGRADING index f5342a7577be6..6d19c4281813d 100644 --- a/UPGRADING +++ b/UPGRADING @@ -277,6 +277,12 @@ PHP 8.1 UPGRADE NOTES 3. Changes in SAPI modules ======================================== +- CLI: + . Using -a without the readline extension will now result in an error. + Previously, -a without readline had the same behavior as calling php without + any arguments, apart from printing an additional "Interactive mode enabled" + message. This mode was not, in fact, interactive. + ======================================== 4. Deprecated Functionality ======================================== diff --git a/sapi/cli/php_cli.c b/sapi/cli/php_cli.c index e0a1191c3de3e..a6c47e185909b 100644 --- a/sapi/cli/php_cli.c +++ b/sapi/cli/php_cli.c @@ -488,11 +488,7 @@ static void php_cli_usage(char *argv0) " %s [options] -- [args...]\n" " %s [options] -a\n" "\n" -#if (defined(HAVE_LIBREADLINE) || defined(HAVE_LIBEDIT)) && !defined(COMPILE_DL_READLINE) - " -a Run as interactive shell\n" -#else - " -a Run interactively\n" -#endif + " -a Run as interactive shell (requires readline extension)\n" " -c | Look for php.ini file in this directory\n" " -n No configuration (ini) files will be used\n" " -d foo[=bar] Define INI entry foo with value 'bar'\n" @@ -694,6 +690,10 @@ static int do_cli(int argc, char **argv) /* {{{ */ switch (c) { case 'a': /* interactive mode */ + if (!cli_shell_callbacks.cli_shell_run) { + param_error = "Interactive shell (-a) requires the readline extension.\n"; + break; + } if (!interactive) { if (behavior != PHP_MODE_STANDARD) { param_error = param_mode_conflict; @@ -874,13 +874,7 @@ static int do_cli(int argc, char **argv) /* {{{ */ #endif if (interactive) { - if (cli_shell_callbacks.cli_shell_run) { - printf("Interactive shell\n\n"); - } else { - printf("Interactive mode enabled\n\n"); - /* Treat as non-interactive apart from the stdin input */ - interactive = false; - } + printf("Interactive shell\n\n"); fflush(stdout); } diff --git a/sapi/cli/tests/009.phpt b/sapi/cli/tests/009.phpt index c110966458326..904f680c7f378 100644 --- a/sapi/cli/tests/009.phpt +++ b/sapi/cli/tests/009.phpt @@ -1,5 +1,7 @@ --TEST-- using invalid combinations of cmdline options +--EXTENSIONS-- +readline --SKIPIF-- --FILE-- diff --git a/sapi/cli/tests/012-2.phpt b/sapi/cli/tests/012-2.phpt index da73fa9b26f22..8ef97595691e3 100644 --- a/sapi/cli/tests/012-2.phpt +++ b/sapi/cli/tests/012-2.phpt @@ -1,5 +1,7 @@ --TEST-- more invalid arguments and error messages +--EXTENSIONS-- +readline --SKIPIF-- --FILE-- From 6ce7b10e14b8d5d8407d36d4fb9be8357ec31551 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Fri, 14 May 2021 12:14:52 +0200 Subject: [PATCH 006/229] Add pkg.m4 upstream Per https://github.com/php/php-src/pull/6907#issuecomment-826362572. Closes GH-6907. [ci skip] --- CONTRIBUTING.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 21c6d21650ad6..8784e71447042 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -147,6 +147,7 @@ locations. ├─ config.sub # https://git.savannah.gnu.org/cgit/config.git ├─ libtool.m4 # https://git.savannah.gnu.org/cgit/libtool.git ├─ ltmain.sh # https://git.savannah.gnu.org/cgit/libtool.git + ├─ pkg.m4 # https://gitlab.freedesktop.org/pkg-config/pkg-config ├─ shtool # https://www.gnu.org/software/shtool/ └─ ... ├─ docs/ # PHP internals and repository documentation From 8d409d955fb3499ef51d78bdc232b00af2a8369a Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Fri, 14 May 2021 12:20:00 +0200 Subject: [PATCH 007/229] Drop test-phpdbg and clean-phpdbg targets test-phpdbg doesn't work entirely, and there doesn't seem to be any cause for having a target that cleans phpdbg in particular. --- sapi/phpdbg/Makefile.frag | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/sapi/phpdbg/Makefile.frag b/sapi/phpdbg/Makefile.frag index 1572bfd9f078e..2e2faf803b00d 100644 --- a/sapi/phpdbg/Makefile.frag +++ b/sapi/phpdbg/Makefile.frag @@ -29,13 +29,3 @@ install-phpdbg: $(BUILD_BINARY) @echo "Installing phpdbg man page: $(INSTALL_ROOT)$(mandir)/man1/" @$(mkinstalldirs) $(INSTALL_ROOT)$(mandir)/man1 @$(INSTALL_DATA) sapi/phpdbg/phpdbg.1 $(INSTALL_ROOT)$(mandir)/man1/$(program_prefix)phpdbg$(program_suffix).1 - -clean-phpdbg: - @echo "Cleaning phpdbg object files ..." - find sapi/phpdbg/ -name *.lo -o -name *.o | xargs rm -f - -test-phpdbg: - @echo "Running phpdbg tests ..." - @$(top_builddir)/sapi/cli/php sapi/phpdbg/tests/run-tests.php --phpdbg sapi/phpdbg/phpdbg - -.PHONY: clean-phpdbg test-phpdbg From 28e7addb9dfd2280c2f9e6990d353070321c69e7 Mon Sep 17 00:00:00 2001 From: Flavio Heleno Date: Wed, 12 May 2021 13:36:43 +0200 Subject: [PATCH 008/229] Fix #81032: GD install is affected by external libgd installation This PR replaces the bundled libgd includes from #include with #include "foo.h" for gd-related headers to avoid including headers that may be available in system directories instead of the expected local headers. Closes GH-6975. --- NEWS | 4 ++++ ext/gd/gd.c | 24 +++++++++++++++++------- ext/gd/libgd/gd_crop.c | 3 ++- ext/gd/libgd/gd_interpolation.c | 2 +- ext/gd/libgd/gd_wbmp.c | 6 +++--- 5 files changed, 27 insertions(+), 12 deletions(-) diff --git a/NEWS b/NEWS index 8753ab57f6ab3..77d14f0538b6f 100644 --- a/NEWS +++ b/NEWS @@ -18,6 +18,10 @@ PHP NEWS . Fixed bug #80901 (Info leak in ftp extension). (cmb) . Fixed bug #79100 (Wrong FTP error messages). (cmb) +- GD: + . Fixed bug #81032 (GD install is affected by external libgd installation). + (Flavio Heleno, cmb) + - MBString: . Fixed bug #81011 (mb_convert_encoding removes references from arrays). (cmb) diff --git a/ext/gd/gd.c b/ext/gd/gd.c index 3480699a70b92..336a739692671 100644 --- a/ext/gd/gd.c +++ b/ext/gd/gd.c @@ -60,13 +60,23 @@ static int le_gd, le_gd_font; -#include -#include -#include /* 1 Tiny font */ -#include /* 2 Small font */ -#include /* 3 Medium bold font */ -#include /* 4 Large font */ -#include /* 5 Giant font */ +#ifdef HAVE_GD_BUNDLED +# include "libgd/gd.h" +# include "libgd/gd_errors.h" +# include "libgd/gdfontt.h" /* 1 Tiny font */ +# include "libgd/gdfonts.h" /* 2 Small font */ +# include "libgd/gdfontmb.h" /* 3 Medium bold font */ +# include "libgd/gdfontl.h" /* 4 Large font */ +# include "libgd/gdfontg.h" /* 5 Giant font */ +#else +# include +# include +# include /* 1 Tiny font */ +# include /* 2 Small font */ +# include /* 3 Medium bold font */ +# include /* 4 Large font */ +# include /* 5 Giant font */ +#endif #if defined(HAVE_GD_FREETYPE) && defined(HAVE_GD_BUNDLED) # include diff --git a/ext/gd/libgd/gd_crop.c b/ext/gd/libgd/gd_crop.c index b4bff27006889..676545c4dbc92 100644 --- a/ext/gd/libgd/gd_crop.c +++ b/ext/gd/libgd/gd_crop.c @@ -19,11 +19,12 @@ * (end code) **/ -#include #include #include #include +#include "gd.h" + static int gdGuessBackgroundColorFromCorners(gdImagePtr im, int *color); static int gdColorMatch(gdImagePtr im, int col1, int col2, float threshold); diff --git a/ext/gd/libgd/gd_interpolation.c b/ext/gd/libgd/gd_interpolation.c index 886b3b023bab7..5247ad40a0ba9 100644 --- a/ext/gd/libgd/gd_interpolation.c +++ b/ext/gd/libgd/gd_interpolation.c @@ -58,7 +58,7 @@ #include #include -#include +#include "gd.h" #include "gdhelpers.h" #ifdef _MSC_VER diff --git a/ext/gd/libgd/gd_wbmp.c b/ext/gd/libgd/gd_wbmp.c index 13dc9e38d6186..22d1c4f4c66d5 100644 --- a/ext/gd/libgd/gd_wbmp.c +++ b/ext/gd/libgd/gd_wbmp.c @@ -51,13 +51,13 @@ ---------------------------------------------------------------------------- */ -#include -#include -#include #include #include #include +#include "gd.h" +#include "gdfonts.h" +#include "gd_errors.h" #include "wbmp.h" From 6afbb7419458f0b2f40284a295be07f03b3b5a65 Mon Sep 17 00:00:00 2001 From: Kamil Tekiela Date: Thu, 13 May 2021 13:50:32 +0100 Subject: [PATCH 009/229] Fixed bug #81037 PDO discards error message text from prepared statement --- NEWS | 3 +++ ext/pdo_mysql/mysql_driver.c | 6 +++++- ext/pdo_mysql/tests/bug81037.phpt | 35 +++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 ext/pdo_mysql/tests/bug81037.phpt diff --git a/NEWS b/NEWS index 77d14f0538b6f..43f1e76142e33 100644 --- a/NEWS +++ b/NEWS @@ -28,6 +28,9 @@ PHP NEWS - ODBC: . Fixed bug #80460 (ODBC doesn't account for SQL_NO_TOTAL indicator). (cmb) +- PDO_MySQL: + . Fixed bug #81037 PDO discards error message text from prepared statement. (Kamil Tekiela) + - PDO_ODBC: . Fixed bug #44643 (bound parameters ignore explicit type definitions). (cmb) diff --git a/ext/pdo_mysql/mysql_driver.c b/ext/pdo_mysql/mysql_driver.c index c5c04adf8a7a1..eb7a55a7ee720 100644 --- a/ext/pdo_mysql/mysql_driver.c +++ b/ext/pdo_mysql/mysql_driver.c @@ -88,7 +88,11 @@ int _pdo_mysql_error(pdo_dbh_t *dbh, pdo_stmt_t *stmt, const char *file, int lin dbh->is_persistent); } else { - einfo->errmsg = pestrdup(mysql_error(H->server), dbh->is_persistent); + if (S && S->stmt) { + einfo->errmsg = pestrdup(mysql_stmt_error(S->stmt), dbh->is_persistent); + } else { + einfo->errmsg = pestrdup(mysql_error(H->server), dbh->is_persistent); + } } } else { /* no error */ strcpy(*pdo_err, PDO_ERR_NONE); diff --git a/ext/pdo_mysql/tests/bug81037.phpt b/ext/pdo_mysql/tests/bug81037.phpt new file mode 100644 index 0000000000000..67c90a4458ba6 --- /dev/null +++ b/ext/pdo_mysql/tests/bug81037.phpt @@ -0,0 +1,35 @@ +--TEST-- +Bug #81037 PDO discards error message text from prepared statement +--SKIPIF-- + +--FILE-- +setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); +$pdo->setAttribute(PDO::ATTR_EMULATE_PREPARES, false); +MySQLPDOTest::createTestTable($pdo); + +$sql = "SELECT id FROM test WHERE label = :par"; +$stmt = $pdo->prepare($sql); +try { + $stmt->execute(); +} catch (PDOException $e) { + echo $e->getMessage(), "\n"; +} +$data = $stmt->fetchAll(PDO::FETCH_ASSOC); + +?> +--CLEAN-- + +--EXPECT-- +SQLSTATE[HY000]: General error: 2031 No data supplied for parameters in prepared statement \ No newline at end of file From ead72aabfbb2e1284bed700d522bb98c76c55544 Mon Sep 17 00:00:00 2001 From: Kamil Tekiela Date: Fri, 14 May 2021 12:33:32 +0100 Subject: [PATCH 010/229] Fix NEWS --- NEWS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 43f1e76142e33..cd37e29e55a66 100644 --- a/NEWS +++ b/NEWS @@ -29,7 +29,8 @@ PHP NEWS . Fixed bug #80460 (ODBC doesn't account for SQL_NO_TOTAL indicator). (cmb) - PDO_MySQL: - . Fixed bug #81037 PDO discards error message text from prepared statement. (Kamil Tekiela) + . Fixed bug #81037 (PDO discards error message text from prepared + statement). (Kamil Tekiela) - PDO_ODBC: . Fixed bug #44643 (bound parameters ignore explicit type definitions). (cmb) From aca6aefd850d7f25b3730a3990826a25ebf02e37 Mon Sep 17 00:00:00 2001 From: George Peter Banyard Date: Fri, 14 May 2021 13:38:01 +0100 Subject: [PATCH 011/229] Remove 'register' type qualifier (#6980) The compiler should be smart enough to optimize this on its own --- Zend/zend_compile.c | 2 +- Zend/zend_hash.c | 2 +- Zend/zend_ini_scanner.l | 2 +- Zend/zend_language_scanner.l | 4 ++-- Zend/zend_operators.c | 14 +++++++------- Zend/zend_virtual_cwd.c | 4 ++-- ext/mbstring/mbstring.c | 6 +++--- ext/mysqli/mysqli_api.c | 2 +- ext/mysqlnd/mysqlnd_wireprotocol.c | 8 ++++---- ext/pcre/php_pcre.c | 4 ++-- ext/pdo_pgsql/pgsql_driver.c | 2 +- ext/standard/exec.c | 2 +- ext/standard/file.c | 2 +- ext/standard/flock_compat.c | 4 ++-- ext/standard/formatted_print.c | 20 ++++++++++---------- ext/standard/mt_rand.c | 14 +++++++------- ext/standard/quot_print.c | 8 ++++---- ext/standard/string.c | 16 ++++++++-------- ext/standard/url.c | 2 +- ext/zlib/zlib.c | 2 +- main/fopen_wrappers.c | 2 +- 21 files changed, 61 insertions(+), 61 deletions(-) diff --git a/Zend/zend_compile.c b/Zend/zend_compile.c index c41a501d0fc0d..1a884206fd7c8 100644 --- a/Zend/zend_compile.c +++ b/Zend/zend_compile.c @@ -1973,7 +1973,7 @@ void zend_verify_namespace(void) /* {{{ */ Returns directory name component of path */ ZEND_API size_t zend_dirname(char *path, size_t len) { - register char *end = path + len - 1; + char *end = path + len - 1; unsigned int len_adjust = 0; #ifdef ZEND_WIN32 diff --git a/Zend/zend_hash.c b/Zend/zend_hash.c index 1944e60606b74..a81af58b63e8a 100644 --- a/Zend/zend_hash.c +++ b/Zend/zend_hash.c @@ -2731,7 +2731,7 @@ ZEND_API zval* ZEND_FASTCALL zend_hash_minmax(const HashTable *ht, bucket_compar ZEND_API bool ZEND_FASTCALL _zend_handle_numeric_str_ex(const char *key, size_t length, zend_ulong *idx) { - register const char *tmp = key; + const char *tmp = key; const char *end = key + length; diff --git a/Zend/zend_ini_scanner.l b/Zend/zend_ini_scanner.l index 16b58ac00afc0..73853f6ec1171 100644 --- a/Zend/zend_ini_scanner.l +++ b/Zend/zend_ini_scanner.l @@ -305,7 +305,7 @@ zend_result zend_ini_prepare_string_for_scanning(char *str, int scanner_mode) /* {{{ zend_ini_escape_string() */ static void zend_ini_escape_string(zval *lval, char *str, int len, char quote_type) { - register char *s, *t; + char *s, *t; char *end; zend_ini_copy_value(lval, str, len); diff --git a/Zend/zend_language_scanner.l b/Zend/zend_language_scanner.l index 02f67d02dc77b..02d79633480c4 100644 --- a/Zend/zend_language_scanner.l +++ b/Zend/zend_language_scanner.l @@ -905,7 +905,7 @@ ZEND_API void zend_multibyte_yyinput_again(zend_encoding_filter old_input_filter static zend_result zend_scan_escape_string(zval *zendlval, char *str, int len, char quote_type) { - register char *s, *t; + char *s, *t; char *end; if (len <= 1) { @@ -2427,7 +2427,7 @@ inline_char_handler: b?['] { - register char *s, *t; + char *s, *t; char *end; int bprefix = (yytext[0] != '\'') ? 1 : 0; diff --git a/Zend/zend_operators.c b/Zend/zend_operators.c index 8b0fed8e90a24..2df256cc52c81 100644 --- a/Zend/zend_operators.c +++ b/Zend/zend_operators.c @@ -2683,13 +2683,13 @@ ZEND_API void ZEND_FASTCALL zend_str_tolower(char *str, size_t length) /* {{{ */ ZEND_API char* ZEND_FASTCALL zend_str_tolower_dup_ex(const char *source, size_t length) /* {{{ */ { - register const unsigned char *p = (const unsigned char*)source; - register const unsigned char *end = p + length; + const unsigned char *p = (const unsigned char*)source; + const unsigned char *end = p + length; while (p < end) { if (*p != zend_tolower_ascii(*p)) { char *res = (char*)emalloc(length + 1); - register unsigned char *r; + unsigned char *r; if (p != (const unsigned char*)source) { memcpy(res, source, p - (const unsigned char*)source); @@ -3280,8 +3280,8 @@ static zend_always_inline void zend_memnstr_ex_pre(unsigned int td[], const char ZEND_API const char* ZEND_FASTCALL zend_memnstr_ex(const char *haystack, const char *needle, size_t needle_len, const char *end) /* {{{ */ { unsigned int td[256]; - register size_t i; - register const char *p; + size_t i; + const char *p; if (needle_len == 0 || (end - haystack) < needle_len) { return NULL; @@ -3314,8 +3314,8 @@ ZEND_API const char* ZEND_FASTCALL zend_memnstr_ex(const char *haystack, const c ZEND_API const char* ZEND_FASTCALL zend_memnrstr_ex(const char *haystack, const char *needle, size_t needle_len, const char *end) /* {{{ */ { unsigned int td[256]; - register size_t i; - register const char *p; + size_t i; + const char *p; if (needle_len == 0 || (end - haystack) < needle_len) { return NULL; diff --git a/Zend/zend_virtual_cwd.c b/Zend/zend_virtual_cwd.c index c444bafcdce24..6e988ad963366 100644 --- a/Zend/zend_virtual_cwd.c +++ b/Zend/zend_virtual_cwd.c @@ -301,7 +301,7 @@ CWD_API char *virtual_getcwd(char *buf, size_t size) /* {{{ */ #ifdef ZEND_WIN32 static inline zend_ulong realpath_cache_key(const char *path, size_t path_len) /* {{{ */ { - register zend_ulong h; + zend_ulong h; size_t bucket_key_len; const char *bucket_key_start = tsrm_win32_get_path_sid_key(path, path_len, &bucket_key_len); const char *bucket_key = bucket_key_start; @@ -325,7 +325,7 @@ static inline zend_ulong realpath_cache_key(const char *path, size_t path_len) / #else static inline zend_ulong realpath_cache_key(const char *path, size_t path_len) /* {{{ */ { - register zend_ulong h; + zend_ulong h; const char *e = path + path_len; for (h = Z_UL(2166136261); path < e;) { diff --git a/ext/mbstring/mbstring.c b/ext/mbstring/mbstring.c index 3da816450491e..e43b8564fd775 100644 --- a/ext/mbstring/mbstring.c +++ b/ext/mbstring/mbstring.c @@ -4232,7 +4232,7 @@ MBSTRING_API size_t php_mb_mbchar_bytes(const char *s) /* {{{ MBSTRING_API char *php_mb_safe_strrchr_ex() */ MBSTRING_API char *php_mb_safe_strrchr_ex(const char *s, unsigned int c, size_t nbytes, const mbfl_encoding *enc) { - register const char *p = s; + const char *p = s; char *last=NULL; if (nbytes == (size_t)-1) { @@ -4252,8 +4252,8 @@ MBSTRING_API char *php_mb_safe_strrchr_ex(const char *s, unsigned int c, size_t ++p; } } else { - register size_t bcnt = nbytes; - register size_t nbytes_char; + size_t bcnt = nbytes; + size_t nbytes_char; while (bcnt > 0) { if ((unsigned char)*p == (unsigned char)c) { last = (char *)p; diff --git a/ext/mysqli/mysqli_api.c b/ext/mysqli/mysqli_api.c index 7ed90a3cfb585..786ca60906e4c 100644 --- a/ext/mysqli/mysqli_api.c +++ b/ext/mysqli/mysqli_api.c @@ -79,7 +79,7 @@ mysqli_escape_string_for_tx_name_in_comment(const char * const name) *p_copy++ = '/'; *p_copy++ = '*'; while (1) { - register char v = *p_orig; + char v = *p_orig; if (v == 0) { break; } diff --git a/ext/mysqlnd/mysqlnd_wireprotocol.c b/ext/mysqlnd/mysqlnd_wireprotocol.c index 1f4e5a63f4dc3..14b7206c30b73 100644 --- a/ext/mysqlnd/mysqlnd_wireprotocol.c +++ b/ext/mysqlnd/mysqlnd_wireprotocol.c @@ -91,7 +91,7 @@ static enum_mysqlnd_collected_stats packet_type_to_statistic_packet_count[PROT_L zend_ulong php_mysqlnd_net_field_length(const zend_uchar **packet) { - register const zend_uchar *p= (const zend_uchar *)*packet; + const zend_uchar *p= (const zend_uchar *)*packet; if (*p < 251) { (*packet)++; @@ -121,7 +121,7 @@ php_mysqlnd_net_field_length(const zend_uchar **packet) uint64_t php_mysqlnd_net_field_length_ll(const zend_uchar **packet) { - register const zend_uchar *p = (zend_uchar *)*packet; + const zend_uchar *p = (zend_uchar *)*packet; if (*p < 251) { (*packet)++; @@ -639,7 +639,7 @@ size_t php_mysqlnd_auth_write(MYSQLND_CONN_DATA * conn, void * _packet) static enum_func_status php_mysqlnd_auth_response_read(MYSQLND_CONN_DATA * conn, void * _packet) { - register MYSQLND_PACKET_AUTH_RESPONSE * packet= (MYSQLND_PACKET_AUTH_RESPONSE *) _packet; + MYSQLND_PACKET_AUTH_RESPONSE * packet= (MYSQLND_PACKET_AUTH_RESPONSE *) _packet; MYSQLND_ERROR_INFO * error_info = conn->error_info; MYSQLND_PFC * pfc = conn->protocol_frame_codec; MYSQLND_VIO * vio = conn->vio; @@ -802,7 +802,7 @@ php_mysqlnd_change_auth_response_write(MYSQLND_CONN_DATA * conn, void * _packet) static enum_func_status php_mysqlnd_ok_read(MYSQLND_CONN_DATA * conn, void * _packet) { - register MYSQLND_PACKET_OK *packet= (MYSQLND_PACKET_OK *) _packet; + MYSQLND_PACKET_OK *packet= (MYSQLND_PACKET_OK *) _packet; MYSQLND_ERROR_INFO * error_info = conn->error_info; MYSQLND_PFC * pfc = conn->protocol_frame_codec; MYSQLND_VIO * vio = conn->vio; diff --git a/ext/pcre/php_pcre.c b/ext/pcre/php_pcre.c index 4708e643b26be..4249980e5586e 100644 --- a/ext/pcre/php_pcre.c +++ b/ext/pcre/php_pcre.c @@ -1500,8 +1500,8 @@ PHP_FUNCTION(preg_match_all) /* {{{ preg_get_backref */ static int preg_get_backref(char **str, int *backref) { - register char in_brace = 0; - register char *walk = *str; + char in_brace = 0; + char *walk = *str; if (walk[1] == 0) return 0; diff --git a/ext/pdo_pgsql/pgsql_driver.c b/ext/pdo_pgsql/pgsql_driver.c index 6ebe8407093bf..806ba55840f86 100644 --- a/ext/pdo_pgsql/pgsql_driver.c +++ b/ext/pdo_pgsql/pgsql_driver.c @@ -37,7 +37,7 @@ static char * _pdo_pgsql_trim_message(const char *message, int persistent) { - register int i = strlen(message)-1; + size_t i = strlen(message)-1; char *tmp; if (i>1 && (message[i-1] == '\r' || message[i-1] == '\n') && message[i] == '.') { diff --git a/ext/standard/exec.c b/ext/standard/exec.c index d5e4542caad05..1831b8eaa54d7 100644 --- a/ext/standard/exec.c +++ b/ext/standard/exec.c @@ -281,7 +281,7 @@ PHP_FUNCTION(passthru) */ PHPAPI zend_string *php_escape_shell_cmd(const char *str) { - register size_t x, y; + size_t x, y; size_t l = strlen(str); uint64_t estimate = (2 * (uint64_t)l) + 1; zend_string *cmd; diff --git a/ext/standard/file.c b/ext/standard/file.c index 3010cd860931b..1afa4c648f114 100644 --- a/ext/standard/file.c +++ b/ext/standard/file.c @@ -722,7 +722,7 @@ PHP_FUNCTION(file) char *filename; size_t filename_len; char *p, *s, *e; - register int i = 0; + int i = 0; char eol_marker = '\n'; zend_long flags = 0; bool use_include_path; diff --git a/ext/standard/flock_compat.c b/ext/standard/flock_compat.c index 47511379d1f0f..34a790b6b8484 100644 --- a/ext/standard/flock_compat.c +++ b/ext/standard/flock_compat.c @@ -162,10 +162,10 @@ PHPAPI int php_flock(int fd, int operation) int inet_aton(const char *cp, struct in_addr *ap) { int dots = 0; - register unsigned long acc = 0, addr = 0; + unsigned long acc = 0, addr = 0; do { - register char cc = *cp; + char cc = *cp; switch (cc) { case '0': diff --git a/ext/standard/formatted_print.c b/ext/standard/formatted_print.c index e500a95734520..0990b390d6b29 100644 --- a/ext/standard/formatted_print.c +++ b/ext/standard/formatted_print.c @@ -85,7 +85,7 @@ php_sprintf_appendstring(zend_string **buffer, size_t *pos, char *add, size_t min_width, size_t max_width, char padding, size_t alignment, size_t len, int neg, int expprec, int always_sign) { - register size_t npad; + size_t npad; size_t req_size; size_t copy_len; size_t m_width; @@ -143,8 +143,8 @@ php_sprintf_appendint(zend_string **buffer, size_t *pos, zend_long number, int always_sign) { char numbuf[NUM_BUF_SIZE]; - register zend_ulong magn, nmagn; - register unsigned int i = NUM_BUF_SIZE - 1, neg = 0; + zend_ulong magn, nmagn; + unsigned int i = NUM_BUF_SIZE - 1, neg = 0; PRINTF_DEBUG(("sprintf: appendint(%x, %x, %x, %d, %d, '%c', %d)\n", *buffer, pos, &ZSTR_LEN(*buffer), number, width, padding, alignment)); @@ -187,8 +187,8 @@ php_sprintf_appenduint(zend_string **buffer, size_t *pos, size_t width, char padding, size_t alignment) { char numbuf[NUM_BUF_SIZE]; - register zend_ulong magn, nmagn; - register unsigned int i = NUM_BUF_SIZE - 1; + zend_ulong magn, nmagn; + unsigned int i = NUM_BUF_SIZE - 1; PRINTF_DEBUG(("sprintf: appenduint(%x, %x, %x, %d, %d, '%c', %d)\n", *buffer, pos, &ZSTR_LEN(*buffer), number, width, padding, alignment)); @@ -326,9 +326,9 @@ php_sprintf_append2n(zend_string **buffer, size_t *pos, zend_long number, const char *chartable, int expprec) { char numbuf[NUM_BUF_SIZE]; - register zend_ulong num; - register zend_ulong i = NUM_BUF_SIZE - 1; - register int andbits = (1 << n) - 1; + zend_ulong num; + zend_ulong i = NUM_BUF_SIZE - 1; + int andbits = (1 << n) - 1; PRINTF_DEBUG(("sprintf: append2n(%x, %x, %x, %d, %d, '%c', %d, %d, %x)\n", *buffer, pos, &ZSTR_LEN(*buffer), number, width, padding, alignment, n, @@ -355,8 +355,8 @@ inline static int php_sprintf_getnumber(char **buffer, size_t *len) { char *endptr; - register zend_long num = ZEND_STRTOL(*buffer, &endptr, 10); - register size_t i; + zend_long num = ZEND_STRTOL(*buffer, &endptr, 10); + size_t i; if (endptr != NULL) { i = (endptr - *buffer); diff --git a/ext/standard/mt_rand.c b/ext/standard/mt_rand.c index 366221d953443..25f6a431a57a0 100644 --- a/ext/standard/mt_rand.c +++ b/ext/standard/mt_rand.c @@ -100,9 +100,9 @@ static inline void php_mt_initialize(uint32_t seed, uint32_t *state) In previous versions, most significant bits (MSBs) of the seed affect only MSBs of the state array. Modified 9 Jan 2002 by Makoto Matsumoto. */ - register uint32_t *s = state; - register uint32_t *r = state; - register int i = 1; + uint32_t *s = state; + uint32_t *r = state; + int i = 1; *s++ = seed & 0xffffffffU; for( ; i < N; ++i ) { @@ -118,9 +118,9 @@ static inline void php_mt_reload(void) /* Generate N new values in state Made clearer and faster by Matthew Bellew (matthew.bellew@home.com) */ - register uint32_t *state = BG(state); - register uint32_t *p = state; - register int i; + uint32_t *state = BG(state); + uint32_t *p = state; + int i; if (BG(mt_rand_mode) == MT_RAND_MT19937) { for (i = N - M; i--; ++p) @@ -159,7 +159,7 @@ PHPAPI uint32_t php_mt_rand(void) /* Pull a 32-bit integer from the generator state Every other access function simply transforms the numbers extracted here */ - register uint32_t s1; + uint32_t s1; if (UNEXPECTED(!BG(mt_rand_is_seeded))) { zend_long bytes; diff --git a/ext/standard/quot_print.c b/ext/standard/quot_print.c index 3cdf7c35016c0..f5472fc386d14 100644 --- a/ext/standard/quot_print.c +++ b/ext/standard/quot_print.c @@ -49,10 +49,10 @@ static char php_hex2int(int c) /* {{{ */ PHPAPI zend_string *php_quot_print_decode(const unsigned char *str, size_t length, int replace_us_by_ws) /* {{{ */ { - register size_t i; - register unsigned const char *p1; - register unsigned char *p2; - register unsigned int h_nbl, l_nbl; + size_t i; + unsigned const char *p1; + unsigned char *p2; + unsigned int h_nbl, l_nbl; size_t decoded_len, buf_size; zend_string *retval; diff --git a/ext/standard/string.c b/ext/standard/string.c index efb13686a9edb..e97c02eef60f7 100644 --- a/ext/standard/string.c +++ b/ext/standard/string.c @@ -1367,7 +1367,7 @@ PHPAPI zend_string *php_string_toupper(zend_string *s) while (c < e) { if (islower(*c)) { - register unsigned char *r; + unsigned char *r; zend_string *res = zend_string_alloc(ZSTR_LEN(s), 0); if (c != (unsigned char*)ZSTR_VAL(s)) { @@ -1432,7 +1432,7 @@ PHPAPI zend_string *php_string_tolower(zend_string *s) while (c < e) { if (isupper(*c)) { - register unsigned char *r; + unsigned char *r; zend_string *res = zend_string_alloc(ZSTR_LEN(s), 0); if (c != (unsigned char*)ZSTR_VAL(s)) { @@ -1764,8 +1764,8 @@ PHPAPI char *php_stristr(char *s, char *t, size_t s_len, size_t t_len) /* {{{ php_strspn */ PHPAPI size_t php_strspn(const char *s1, const char *s2, const char *s1_end, const char *s2_end) { - register const char *p = s1, *spanp; - register char c = *p; + const char *p = s1, *spanp; + char c = *p; cont: for (spanp = s2; p != s1_end && spanp != s2_end;) { @@ -1781,8 +1781,8 @@ PHPAPI size_t php_strspn(const char *s1, const char *s2, const char *s1_end, con /* {{{ php_strcspn */ PHPAPI size_t php_strcspn(const char *s1, const char *s2, const char *s1_end, const char *s2_end) { - register const char *p, *spanp; - register char c = *s1; + const char *p, *spanp; + char c = *s1; for (p = s1;;) { spanp = s2; @@ -2667,8 +2667,8 @@ PHP_FUNCTION(ucwords) { zend_string *str; char *delims = " \t\r\n\f\v"; - register char *r; - register const char *r_end; + char *r; + const char *r_end; size_t delims_len = 6; char mask[256]; diff --git a/ext/standard/url.c b/ext/standard/url.c index 0614095b8f2d7..efad0d77c2c36 100644 --- a/ext/standard/url.c +++ b/ext/standard/url.c @@ -459,7 +459,7 @@ static int php_htoi(char *s) static const unsigned char hexchars[] = "0123456789ABCDEF"; static zend_always_inline zend_string *php_url_encode_impl(const char *s, size_t len, bool raw) /* {{{ */ { - register unsigned char c; + unsigned char c; unsigned char *to; unsigned char const *from, *end; zend_string *start; diff --git a/ext/zlib/zlib.c b/ext/zlib/zlib.c index 4c82e5db80ce6..2ed24b79b9459 100644 --- a/ext/zlib/zlib.c +++ b/ext/zlib/zlib.c @@ -612,7 +612,7 @@ PHP_FUNCTION(gzfile) size_t filename_len; int flags = REPORT_ERRORS; char buf[8192] = {0}; - register int i = 0; + int i = 0; zend_long use_include_path = 0; php_stream *stream; diff --git a/main/fopen_wrappers.c b/main/fopen_wrappers.c index 7b11ebd5492cc..6b05da6117a51 100644 --- a/main/fopen_wrappers.c +++ b/main/fopen_wrappers.c @@ -688,7 +688,7 @@ PHPAPI FILE *php_fopen_with_path(const char *filename, const char *mode, const c /* {{{ php_strip_url_passwd */ PHPAPI char *php_strip_url_passwd(char *url) { - register char *p, *url_start; + char *p, *url_start; if (url == NULL) { return ""; From e7135cb8170f25fc55c0c63bc4c55e864d657055 Mon Sep 17 00:00:00 2001 From: George Peter Banyard Date: Fri, 14 May 2021 13:45:17 +0100 Subject: [PATCH 012/229] Use zend_string_equals_* API in a couple of more place Closes GH-6979 --- ext/dom/node.c | 16 +++---- ext/gettext/gettext.c | 26 +++++----- ext/mbstring/mbstring.c | 2 +- ext/phar/phar_object.c | 20 +++----- ext/soap/soap.c | 14 ++---- ext/standard/basic_functions.c | 11 +++-- sapi/fpm/fpm/fpm_conf.c | 86 +++++++++++++++++----------------- 7 files changed, 82 insertions(+), 93 deletions(-) diff --git a/ext/dom/node.c b/ext/dom/node.c index 97ab61f71b3fd..67c6b768be0c3 100644 --- a/ext/dom/node.c +++ b/ext/dom/node.c @@ -603,7 +603,7 @@ int dom_node_prefix_read(dom_object *obj, zval *retval) int dom_node_prefix_write(dom_object *obj, zval *newval) { - zend_string *str; + zend_string *prefix_str; xmlNode *nodep, *nsnode = NULL; xmlNsPtr ns = NULL, curns; char *strURI; @@ -627,17 +627,17 @@ int dom_node_prefix_write(dom_object *obj, zval *newval) nsnode = xmlDocGetRootElement(nodep->doc); } } - str = zval_try_get_string(newval); - if (UNEXPECTED(!str)) { + prefix_str = zval_try_get_string(newval); + if (UNEXPECTED(!prefix_str)) { return FAILURE; } - prefix = ZSTR_VAL(str); + prefix = ZSTR_VAL(prefix_str); if (nsnode && nodep->ns != NULL && !xmlStrEqual(nodep->ns->prefix, (xmlChar *)prefix)) { strURI = (char *) nodep->ns->href; if (strURI == NULL || - (!strcmp(prefix, "xml") && strcmp(strURI, (char *) XML_XML_NAMESPACE)) || - (nodep->type == XML_ATTRIBUTE_NODE && !strcmp(prefix, "xmlns") && + (zend_string_equals_literal(prefix_str, "xml") && strcmp(strURI, (char *) XML_XML_NAMESPACE)) || + (nodep->type == XML_ATTRIBUTE_NODE && zend_string_equals_literal(prefix_str, "xmlns") && strcmp(strURI, (char *) DOM_XMLNS_NAMESPACE)) || (nodep->type == XML_ATTRIBUTE_NODE && !strcmp((char *) nodep->name, "xmlns"))) { ns = NULL; @@ -656,14 +656,14 @@ int dom_node_prefix_write(dom_object *obj, zval *newval) } if (ns == NULL) { - zend_string_release_ex(str, 0); + zend_string_release_ex(prefix_str, 0); php_dom_throw_error(NAMESPACE_ERR, dom_get_strict_error(obj->document)); return FAILURE; } xmlSetNs(nodep, ns); } - zend_string_release_ex(str, 0); + zend_string_release_ex(prefix_str, 0); break; default: break; diff --git a/ext/gettext/gettext.c b/ext/gettext/gettext.c index 723a1a6d20252..05f41552c63fc 100644 --- a/ext/gettext/gettext.c +++ b/ext/gettext/gettext.c @@ -71,19 +71,16 @@ PHP_MINFO_FUNCTION(php_gettext) /* {{{ Set the textdomain to "domain". Returns the current domain */ PHP_FUNCTION(textdomain) { - char *domain = NULL, *domain_name, *retval; - size_t domain_len = 0; + char *domain_name = NULL, *retval; + zend_string *domain = NULL; - if (zend_parse_parameters(ZEND_NUM_ARGS(), "s!", &domain, &domain_len) == FAILURE) { + if (zend_parse_parameters(ZEND_NUM_ARGS(), "S!", &domain) == FAILURE) { RETURN_THROWS(); } - PHP_GETTEXT_DOMAIN_LENGTH_CHECK(1, domain_len) - - if (domain != NULL && strcmp(domain, "") && strcmp(domain, "0")) { - domain_name = domain; - } else { - domain_name = NULL; + if (domain != NULL && ZSTR_LEN(domain) != 0 && !zend_string_equals_literal(domain, "0")) { + PHP_GETTEXT_DOMAIN_LENGTH_CHECK(1, ZSTR_LEN(domain)) + domain_name = ZSTR_VAL(domain); } retval = textdomain(domain_name); @@ -163,11 +160,12 @@ PHP_FUNCTION(dcgettext) /* {{{ Bind to the text domain domain_name, looking for translations in dir. Returns the current domain */ PHP_FUNCTION(bindtextdomain) { - char *domain, *dir = NULL; - size_t domain_len, dir_len; + char *domain; + size_t domain_len; + zend_string *dir = NULL; char *retval, dir_name[MAXPATHLEN]; - if (zend_parse_parameters(ZEND_NUM_ARGS(), "ss!", &domain, &domain_len, &dir, &dir_len) == FAILURE) { + if (zend_parse_parameters(ZEND_NUM_ARGS(), "sS!", &domain, &domain_len, &dir) == FAILURE) { RETURN_THROWS(); } @@ -182,8 +180,8 @@ PHP_FUNCTION(bindtextdomain) RETURN_STRING(bindtextdomain(domain, NULL)); } - if (dir[0] != '\0' && strcmp(dir, "0")) { - if (!VCWD_REALPATH(dir, dir_name)) { + if (ZSTR_LEN(dir) != 0 && !zend_string_equals_literal(dir, "0")) { + if (!VCWD_REALPATH(ZSTR_VAL(dir), dir_name)) { RETURN_FALSE; } } else if (!VCWD_GETCWD(dir_name, MAXPATHLEN)) { diff --git a/ext/mbstring/mbstring.c b/ext/mbstring/mbstring.c index e43b8564fd775..ce7c217cd2e60 100644 --- a/ext/mbstring/mbstring.c +++ b/ext/mbstring/mbstring.c @@ -357,7 +357,7 @@ static int php_mb_parse_encoding_array(HashTable *target_hash, const mbfl_encodi return FAILURE; } - if (strcasecmp(ZSTR_VAL(encoding_str), "auto") == 0) { + if (zend_string_equals_literal_ci(encoding_str, "auto")) { if (!included_auto) { const enum mbfl_no_encoding *src = MBSTRG(default_detect_order_list); const size_t identify_list_size = MBSTRG(default_detect_order_list_size); diff --git a/ext/phar/phar_object.c b/ext/phar/phar_object.c index ae601bf92d441..c012ac763b841 100644 --- a/ext/phar/phar_object.c +++ b/ext/phar/phar_object.c @@ -885,22 +885,16 @@ PHP_METHOD(Phar, mungServer) RETURN_THROWS(); } - if (Z_STRLEN_P(data) == sizeof("PHP_SELF")-1 && !strncmp(Z_STRVAL_P(data), "PHP_SELF", sizeof("PHP_SELF")-1)) { + if (zend_string_equals_literal(Z_STR_P(data), "PHP_SELF")) { PHAR_G(phar_SERVER_mung_list) |= PHAR_MUNG_PHP_SELF; - } - - if (Z_STRLEN_P(data) == sizeof("REQUEST_URI")-1) { - if (!strncmp(Z_STRVAL_P(data), "REQUEST_URI", sizeof("REQUEST_URI")-1)) { - PHAR_G(phar_SERVER_mung_list) |= PHAR_MUNG_REQUEST_URI; - } - if (!strncmp(Z_STRVAL_P(data), "SCRIPT_NAME", sizeof("SCRIPT_NAME")-1)) { - PHAR_G(phar_SERVER_mung_list) |= PHAR_MUNG_SCRIPT_NAME; - } - } - - if (Z_STRLEN_P(data) == sizeof("SCRIPT_FILENAME")-1 && !strncmp(Z_STRVAL_P(data), "SCRIPT_FILENAME", sizeof("SCRIPT_FILENAME")-1)) { + } else if (zend_string_equals_literal(Z_STR_P(data), "REQUEST_URI")) { + PHAR_G(phar_SERVER_mung_list) |= PHAR_MUNG_REQUEST_URI; + } else if (zend_string_equals_literal(Z_STR_P(data), "SCRIPT_NAME")) { + PHAR_G(phar_SERVER_mung_list) |= PHAR_MUNG_SCRIPT_NAME; + } else if (zend_string_equals_literal(Z_STR_P(data), "SCRIPT_FILENAME")) { PHAR_G(phar_SERVER_mung_list) |= PHAR_MUNG_SCRIPT_FILENAME; } + // TODO Warning for invalid value? } ZEND_HASH_FOREACH_END(); } /* }}} */ diff --git a/ext/soap/soap.c b/ext/soap/soap.c index 8564e1e4dcd82..ef8b721e93b60 100644 --- a/ext/soap/soap.c +++ b/ext/soap/soap.c @@ -731,23 +731,19 @@ static HashTable* soap_create_typemap(sdlPtr sdl, HashTable *ht) /* {{{ */ ZEND_HASH_FOREACH_STR_KEY_VAL(ht2, name, tmp) { if (name) { - if (ZSTR_LEN(name) == sizeof("type_name")-1 && - strncmp(ZSTR_VAL(name), "type_name", sizeof("type_name")-1) == 0) { + if (zend_string_equals_literal(name, "type_name")) { if (Z_TYPE_P(tmp) == IS_STRING) { type_name = Z_STRVAL_P(tmp); } else if (Z_TYPE_P(tmp) != IS_NULL) { } - } else if (ZSTR_LEN(name) == sizeof("type_ns")-1 && - strncmp(ZSTR_VAL(name), "type_ns", sizeof("type_ns")-1) == 0) { + } else if (zend_string_equals_literal(name, "type_ns")) { if (Z_TYPE_P(tmp) == IS_STRING) { type_ns = Z_STRVAL_P(tmp); } else if (Z_TYPE_P(tmp) != IS_NULL) { } - } else if (ZSTR_LEN(name) == sizeof("to_xml")-1 && - strncmp(ZSTR_VAL(name), "to_xml", sizeof("to_xml")-1) == 0) { + } else if (zend_string_equals_literal(name, "to_xml")) { to_xml = tmp; - } else if (ZSTR_LEN(name) == sizeof("from_xml")-1 && - strncmp(ZSTR_VAL(name), "from_xml", sizeof("from_xml")-1) == 0) { + } else if (zend_string_equals_literal(name, "from_xml")) { to_zval = tmp; } } @@ -1756,7 +1752,7 @@ static void soap_server_fault_ex(sdlFunctionPtr function, zval* fault, soapHeade if ((Z_TYPE(PG(http_globals)[TRACK_VARS_SERVER]) == IS_ARRAY || zend_is_auto_global(ZSTR_KNOWN(ZEND_STR_AUTOGLOBAL_SERVER))) && (agent_name = zend_hash_str_find(Z_ARRVAL(PG(http_globals)[TRACK_VARS_SERVER]), "HTTP_USER_AGENT", sizeof("HTTP_USER_AGENT")-1)) != NULL && Z_TYPE_P(agent_name) == IS_STRING) { - if (strncmp(Z_STRVAL_P(agent_name), "Shockwave Flash", sizeof("Shockwave Flash")-1) == 0) { + if (zend_string_equals_literal(Z_STR_P(agent_name), "Shockwave Flash")) { use_http_error_status = 0; } } diff --git a/ext/standard/basic_functions.c b/ext/standard/basic_functions.c index 1e63b16289770..0dfcdaf33c3c3 100755 --- a/ext/standard/basic_functions.c +++ b/ext/standard/basic_functions.c @@ -2246,12 +2246,13 @@ PHP_FUNCTION(ignore_user_abort) /* {{{ Returns port associated with service. Protocol must be "tcp" or "udp" */ PHP_FUNCTION(getservbyname) { - char *name, *proto; - size_t name_len, proto_len; + zend_string *name; + char *proto; + size_t proto_len; struct servent *serv; ZEND_PARSE_PARAMETERS_START(2, 2) - Z_PARAM_STRING(name, name_len) + Z_PARAM_STR(name) Z_PARAM_STRING(proto, proto_len) ZEND_PARSE_PARAMETERS_END(); @@ -2264,14 +2265,14 @@ PHP_FUNCTION(getservbyname) } #endif - serv = getservbyname(name, proto); + serv = getservbyname(ZSTR_VAL(name), proto); #if defined(_AIX) /* On AIX, imap is only known as imap2 in /etc/services, while on Linux imap is an alias for imap2. If a request for imap gives no result, we try again with imap2. */ - if (serv == NULL && strcmp(name, "imap") == 0) { + if (serv == NULL && zend_string_equals_literal(name, "imap")) { serv = getservbyname("imap2", proto); } #endif diff --git a/sapi/fpm/fpm/fpm_conf.c b/sapi/fpm/fpm/fpm_conf.c index bc582fe9cecce..0464a36f16801 100644 --- a/sapi/fpm/fpm/fpm_conf.c +++ b/sapi/fpm/fpm/fpm_conf.c @@ -208,9 +208,9 @@ static int fpm_conf_expand_pool_name(char **value) { static char *fpm_conf_set_boolean(zval *value, void **config, intptr_t offset) /* {{{ */ { - char *val = Z_STRVAL_P(value); - long value_y = !strcasecmp(val, "1"); - long value_n = !strcasecmp(val, ""); + zend_string *val = Z_STR_P(value); + bool value_y = zend_string_equals_literal(val, "1"); + bool value_n = ZSTR_LEN(val) == 0; /* Empty string is the only valid false value */ if (!value_y && !value_n) { return "invalid boolean value"; @@ -324,18 +324,18 @@ static char *fpm_conf_set_time(zval *value, void **config, intptr_t offset) /* { static char *fpm_conf_set_log_level(zval *value, void **config, intptr_t offset) /* {{{ */ { - char *val = Z_STRVAL_P(value); + zend_string *val = Z_STR_P(value); int log_level; - if (!strcasecmp(val, "debug")) { + if (zend_string_equals_literal_ci(val, "debug")) { log_level = ZLOG_DEBUG; - } else if (!strcasecmp(val, "notice")) { + } else if (zend_string_equals_literal_ci(val, "notice")) { log_level = ZLOG_NOTICE; - } else if (!strcasecmp(val, "warning") || !strcasecmp(val, "warn")) { + } else if (zend_string_equals_literal_ci(val, "warning") || zend_string_equals_literal_ci(val, "warn")) { log_level = ZLOG_WARNING; - } else if (!strcasecmp(val, "error")) { + } else if (zend_string_equals_literal_ci(val, "error")) { log_level = ZLOG_ERROR; - } else if (!strcasecmp(val, "alert")) { + } else if (zend_string_equals_literal_ci(val, "alert")) { log_level = ZLOG_ALERT; } else { return "invalid value for 'log_level'"; @@ -349,144 +349,144 @@ static char *fpm_conf_set_log_level(zval *value, void **config, intptr_t offset) #ifdef HAVE_SYSLOG_H static char *fpm_conf_set_syslog_facility(zval *value, void **config, intptr_t offset) /* {{{ */ { - char *val = Z_STRVAL_P(value); + zend_string *val = Z_STR_P(value); int *conf = (int *) ((char *) *config + offset); #ifdef LOG_AUTH - if (!strcasecmp(val, "AUTH")) { + if (zend_string_equals_literal_ci(val, "AUTH")) { *conf = LOG_AUTH; return NULL; } #endif #ifdef LOG_AUTHPRIV - if (!strcasecmp(val, "AUTHPRIV")) { + if (zend_string_equals_literal_ci(val, "AUTHPRIV")) { *conf = LOG_AUTHPRIV; return NULL; } #endif #ifdef LOG_CRON - if (!strcasecmp(val, "CRON")) { + if (zend_string_equals_literal_ci(val, "CRON")) { *conf = LOG_CRON; return NULL; } #endif #ifdef LOG_DAEMON - if (!strcasecmp(val, "DAEMON")) { + if (zend_string_equals_literal_ci(val, "DAEMON")) { *conf = LOG_DAEMON; return NULL; } #endif #ifdef LOG_FTP - if (!strcasecmp(val, "FTP")) { + if (zend_string_equals_literal_ci(val, "FTP")) { *conf = LOG_FTP; return NULL; } #endif #ifdef LOG_KERN - if (!strcasecmp(val, "KERN")) { + if (zend_string_equals_literal_ci(val, "KERN")) { *conf = LOG_KERN; return NULL; } #endif #ifdef LOG_LPR - if (!strcasecmp(val, "LPR")) { + if (zend_string_equals_literal_ci(val, "LPR")) { *conf = LOG_LPR; return NULL; } #endif #ifdef LOG_MAIL - if (!strcasecmp(val, "MAIL")) { + if (zend_string_equals_literal_ci(val, "MAIL")) { *conf = LOG_MAIL; return NULL; } #endif #ifdef LOG_NEWS - if (!strcasecmp(val, "NEWS")) { + if (zend_string_equals_literal_ci(val, "NEWS")) { *conf = LOG_NEWS; return NULL; } #endif #ifdef LOG_SYSLOG - if (!strcasecmp(val, "SYSLOG")) { + if (zend_string_equals_literal_ci(val, "SYSLOG")) { *conf = LOG_SYSLOG; return NULL; } #endif #ifdef LOG_USER - if (!strcasecmp(val, "USER")) { + if (zend_string_equals_literal_ci(val, "USER")) { *conf = LOG_USER; return NULL; } #endif #ifdef LOG_UUCP - if (!strcasecmp(val, "UUCP")) { + if (zend_string_equals_literal_ci(val, "UUCP")) { *conf = LOG_UUCP; return NULL; } #endif #ifdef LOG_LOCAL0 - if (!strcasecmp(val, "LOCAL0")) { + if (zend_string_equals_literal_ci(val, "LOCAL0")) { *conf = LOG_LOCAL0; return NULL; } #endif #ifdef LOG_LOCAL1 - if (!strcasecmp(val, "LOCAL1")) { + if (zend_string_equals_literal_ci(val, "LOCAL1")) { *conf = LOG_LOCAL1; return NULL; } #endif #ifdef LOG_LOCAL2 - if (!strcasecmp(val, "LOCAL2")) { + if (zend_string_equals_literal_ci(val, "LOCAL2")) { *conf = LOG_LOCAL2; return NULL; } #endif #ifdef LOG_LOCAL3 - if (!strcasecmp(val, "LOCAL3")) { + if (zend_string_equals_literal_ci(val, "LOCAL3")) { *conf = LOG_LOCAL3; return NULL; } #endif #ifdef LOG_LOCAL4 - if (!strcasecmp(val, "LOCAL4")) { + if (zend_string_equals_literal_ci(val, "LOCAL4")) { *conf = LOG_LOCAL4; return NULL; } #endif #ifdef LOG_LOCAL5 - if (!strcasecmp(val, "LOCAL5")) { + if (zend_string_equals_literal_ci(val, "LOCAL5")) { *conf = LOG_LOCAL5; return NULL; } #endif #ifdef LOG_LOCAL6 - if (!strcasecmp(val, "LOCAL6")) { + if (zend_string_equals_literal_ci(val, "LOCAL6")) { *conf = LOG_LOCAL6; return NULL; } #endif #ifdef LOG_LOCAL7 - if (!strcasecmp(val, "LOCAL7")) { + if (zend_string_equals_literal_ci(val, "LOCAL7")) { *conf = LOG_LOCAL7; return NULL; } @@ -499,10 +499,10 @@ static char *fpm_conf_set_syslog_facility(zval *value, void **config, intptr_t o static char *fpm_conf_set_rlimit_core(zval *value, void **config, intptr_t offset) /* {{{ */ { - char *val = Z_STRVAL_P(value); + zend_string *val = Z_STR_P(value); int *ptr = (int *) ((char *) *config + offset); - if (!strcasecmp(val, "unlimited")) { + if (zend_string_equals_literal_ci(val, "unlimited")) { *ptr = -1; } else { int int_value; @@ -528,13 +528,13 @@ static char *fpm_conf_set_rlimit_core(zval *value, void **config, intptr_t offse static char *fpm_conf_set_pm(zval *value, void **config, intptr_t offset) /* {{{ */ { - char *val = Z_STRVAL_P(value); + zend_string *val = Z_STR_P(value); struct fpm_worker_pool_config_s *c = *config; - if (!strcasecmp(val, "static")) { + if (zend_string_equals_literal_ci(val, "static")) { c->pm = PM_STYLE_STATIC; - } else if (!strcasecmp(val, "dynamic")) { + } else if (zend_string_equals_literal_ci(val, "dynamic")) { c->pm = PM_STYLE_DYNAMIC; - } else if (!strcasecmp(val, "ondemand")) { + } else if (zend_string_equals_literal_ci(val, "ondemand")) { c->pm = PM_STYLE_ONDEMAND; } else { return "invalid process manager (static, dynamic or ondemand)"; @@ -1403,7 +1403,7 @@ static void fpm_conf_ini_parser_section(zval *section, void *arg) /* {{{ */ int *error = (int *)arg; /* switch to global conf */ - if (!strcasecmp(Z_STRVAL_P(section), "global")) { + if (zend_string_equals_literal_ci(Z_STR_P(section), "global")) { current_wp = NULL; return; } @@ -1446,7 +1446,7 @@ static void fpm_conf_ini_parser_entry(zval *name, zval *value, void *arg) /* {{{ return; } - if (!strcmp(Z_STRVAL_P(name), "include")) { + if (zend_string_equals_literal(Z_STR_P(name), "include")) { if (ini_include) { zlog(ZLOG_ERROR, "[%s:%d] two includes at the same time !", ini_filename, ini_lineno); *error = 1; @@ -1508,7 +1508,7 @@ static void fpm_conf_ini_parser_array(zval *name, zval *key, zval *value, void * return; } - if (!strcmp("env", Z_STRVAL_P(name))) { + if (zend_string_equals_literal(Z_STR_P(name), "env")) { if (!*Z_STRVAL_P(value)) { zlog(ZLOG_ERROR, "[%s:%d] empty value", ini_filename, ini_lineno); *error = 1; @@ -1517,19 +1517,19 @@ static void fpm_conf_ini_parser_array(zval *name, zval *key, zval *value, void * config = (char *)current_wp->config + WPO(env); err = fpm_conf_set_array(key, value, &config, 0); - } else if (!strcmp("php_value", Z_STRVAL_P(name))) { + } else if (zend_string_equals_literal(Z_STR_P(name), "php_value")) { config = (char *)current_wp->config + WPO(php_values); err = fpm_conf_set_array(key, value, &config, 0); - } else if (!strcmp("php_admin_value", Z_STRVAL_P(name))) { + } else if (zend_string_equals_literal(Z_STR_P(name), "php_admin_value")) { config = (char *)current_wp->config + WPO(php_admin_values); err = fpm_conf_set_array(key, value, &config, 0); - } else if (!strcmp("php_flag", Z_STRVAL_P(name))) { + } else if (zend_string_equals_literal(Z_STR_P(name), "php_flag")) { config = (char *)current_wp->config + WPO(php_values); err = fpm_conf_set_array(key, value, &config, 1); - } else if (!strcmp("php_admin_flag", Z_STRVAL_P(name))) { + } else if (zend_string_equals_literal(Z_STR_P(name), "php_admin_flag")) { config = (char *)current_wp->config + WPO(php_admin_values); err = fpm_conf_set_array(key, value, &config, 1); From 008bfcc7ba7b47df4e4f6a920a299fdb58295e57 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Fri, 14 May 2021 14:47:40 +0200 Subject: [PATCH 013/229] Use NO_DYNAMIC_PROPERTIES for Closure Instead of manually implementing this, use the standard mechanism. This has minor behavior changes (e.g. doing an isset() will now return false instead of throwing) which are more in line with typical behavior. --- Zend/tests/bug50146.phpt | 8 ++---- Zend/tests/closure_022.phpt | 2 +- Zend/tests/closure_031.phpt | 5 ++-- Zend/tests/closure_write_prop.phpt | 2 +- Zend/zend_closures.c | 46 ------------------------------ Zend/zend_closures.stub.php | 1 + Zend/zend_closures_arginfo.h | 4 +-- 7 files changed, 10 insertions(+), 58 deletions(-) diff --git a/Zend/tests/bug50146.phpt b/Zend/tests/bug50146.phpt index 0ef9048b0613b..3d3b54ffc1e9e 100644 --- a/Zend/tests/bug50146.phpt +++ b/Zend/tests/bug50146.phpt @@ -13,11 +13,7 @@ var_dump($ref->hasProperty('b')); var_dump(isset($obj->a)); ?> ---EXPECTF-- +--EXPECT-- +bool(false) bool(false) bool(false) - -Fatal error: Uncaught Error: Closure object cannot have properties in %s:%d -Stack trace: -#0 {main} - thrown in %s on line %d diff --git a/Zend/tests/closure_022.phpt b/Zend/tests/closure_022.phpt index 8977e963dface..8e0fd06c1d329 100644 --- a/Zend/tests/closure_022.phpt +++ b/Zend/tests/closure_022.phpt @@ -8,7 +8,7 @@ $foo = function() use ($a) { $foo->a = 1; ?> --EXPECTF-- -Fatal error: Uncaught Error: Closure object cannot have properties in %sclosure_022.php:5 +Fatal error: Uncaught Error: Cannot create dynamic property Closure::$a in %s:%d Stack trace: #0 {main} thrown in %sclosure_022.php on line 5 diff --git a/Zend/tests/closure_031.phpt b/Zend/tests/closure_031.phpt index e757f677488ef..19f3dc6e3212e 100644 --- a/Zend/tests/closure_031.phpt +++ b/Zend/tests/closure_031.phpt @@ -3,7 +3,7 @@ Closure 031: Closure properties with custom error handlers --FILE-- --EXPECT-- -Error: Closure object cannot have properties +Warning: Undefined property: Closure::$a +NULL diff --git a/Zend/tests/closure_write_prop.phpt b/Zend/tests/closure_write_prop.phpt index 38bebf4e1b786..8dbf18e670418 100644 --- a/Zend/tests/closure_write_prop.phpt +++ b/Zend/tests/closure_write_prop.phpt @@ -19,4 +19,4 @@ try { ?> --EXPECT-- -Closure object cannot have properties +Cannot create dynamic property Closure::$b diff --git a/Zend/zend_closures.c b/Zend/zend_closures.c index 52d5c5d1ba059..ad8e68b9c8f40 100644 --- a/Zend/zend_closures.c +++ b/Zend/zend_closures.c @@ -28,11 +28,6 @@ #include "zend_globals.h" #include "zend_closures_arginfo.h" -#define ZEND_CLOSURE_PRINT_NAME "Closure object" - -#define ZEND_CLOSURE_PROPERTY_ERROR() \ - zend_throw_error(NULL, "Closure object cannot have properties") - typedef struct _zend_closure { zend_object std; zend_function func; @@ -442,42 +437,6 @@ static zend_function *zend_closure_get_method(zend_object **object, zend_string } /* }}} */ -static ZEND_COLD zval *zend_closure_read_property(zend_object *object, zend_string *member, int type, void **cache_slot, zval *rv) /* {{{ */ -{ - ZEND_CLOSURE_PROPERTY_ERROR(); - return &EG(uninitialized_zval); -} -/* }}} */ - -static ZEND_COLD zval *zend_closure_write_property(zend_object *object, zend_string *member, zval *value, void **cache_slot) /* {{{ */ -{ - ZEND_CLOSURE_PROPERTY_ERROR(); - return &EG(error_zval); -} -/* }}} */ - -static ZEND_COLD zval *zend_closure_get_property_ptr_ptr(zend_object *object, zend_string *member, int type, void **cache_slot) /* {{{ */ -{ - ZEND_CLOSURE_PROPERTY_ERROR(); - return NULL; -} -/* }}} */ - -static ZEND_COLD int zend_closure_has_property(zend_object *object, zend_string *member, int has_set_exists, void **cache_slot) /* {{{ */ -{ - if (has_set_exists != ZEND_PROPERTY_EXISTS) { - ZEND_CLOSURE_PROPERTY_ERROR(); - } - return 0; -} -/* }}} */ - -static ZEND_COLD void zend_closure_unset_property(zend_object *object, zend_string *member, void **cache_slot) /* {{{ */ -{ - ZEND_CLOSURE_PROPERTY_ERROR(); -} -/* }}} */ - static void zend_closure_free_storage(zend_object *object) /* {{{ */ { zend_closure *closure = (zend_closure *)object; @@ -645,11 +604,6 @@ void zend_register_closure_ce(void) /* {{{ */ closure_handlers.free_obj = zend_closure_free_storage; closure_handlers.get_constructor = zend_closure_get_constructor; closure_handlers.get_method = zend_closure_get_method; - closure_handlers.write_property = zend_closure_write_property; - closure_handlers.read_property = zend_closure_read_property; - closure_handlers.get_property_ptr_ptr = zend_closure_get_property_ptr_ptr; - closure_handlers.has_property = zend_closure_has_property; - closure_handlers.unset_property = zend_closure_unset_property; closure_handlers.compare = zend_closure_compare; closure_handlers.clone_obj = zend_closure_clone; closure_handlers.get_debug_info = zend_closure_get_debug_info; diff --git a/Zend/zend_closures.stub.php b/Zend/zend_closures.stub.php index 4bd93e241fa6e..3d451e58b69b2 100644 --- a/Zend/zend_closures.stub.php +++ b/Zend/zend_closures.stub.php @@ -2,6 +2,7 @@ /** @generate-class-entries */ +/** @strict-properties */ final class Closure { private function __construct() {} diff --git a/Zend/zend_closures_arginfo.h b/Zend/zend_closures_arginfo.h index 56bd16ffb65b1..888e4994a08c6 100644 --- a/Zend/zend_closures_arginfo.h +++ b/Zend/zend_closures_arginfo.h @@ -1,5 +1,5 @@ /* This is a generated file, edit the .stub.php file instead. - * Stub hash: 62da9b1e75331f30a0c63e82c9fd366e26b5724d */ + * Stub hash: 7c4df531cdb30ac4206f43f0d40098666466b9a6 */ ZEND_BEGIN_ARG_INFO_EX(arginfo_class_Closure___construct, 0, 0, 0) ZEND_END_ARG_INFO() @@ -47,7 +47,7 @@ static zend_class_entry *register_class_Closure(void) INIT_CLASS_ENTRY(ce, "Closure", class_Closure_methods); class_entry = zend_register_internal_class_ex(&ce, NULL); - class_entry->ce_flags |= ZEND_ACC_FINAL; + class_entry->ce_flags |= ZEND_ACC_FINAL|ZEND_ACC_NO_DYNAMIC_PROPERTIES; return class_entry; } From 532c60cb921757151f4a60b9f56e2805ce23d819 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1t=C3=A9=20Kocsis?= Date: Thu, 6 May 2021 15:05:21 +0200 Subject: [PATCH 014/229] Add support for tentative return types of internal methods RFC: https://wiki.php.net/rfc/internal_method_return_types Closses GH-6971 --- UPGRADING | 4 + Zend/Optimizer/zend_func_info.c | 4 +- Zend/Optimizer/zend_inference.c | 24 +- Zend/Optimizer/zend_inference.h | 2 +- ...nternal_declaration_error_class_const.phpt | 4 +- .../internal_declaration_error_const.phpt | 4 +- .../internal_declaration_error_false.phpt | 4 +- .../internal_declaration_error_int.phpt | 4 +- .../internal_declaration_error_null.phpt | 4 +- .../variance/internal_parent.phpt | 12 - .../compatible_return_type.phpt | 17 ++ .../incompatible_return_type.phpt | 17 ++ .../internal_parent/missing_return_type.phpt | 13 + .../unresolvable_inheritance_check_param.phpt | 12 + ...unresolvable_inheritance_check_return.phpt | 12 + .../return_type_will_change_class_error.phpt | 13 + ...eturn_type_will_change_function_error.phpt | 11 + ...eturn_type_will_change_property_error.phpt | 14 + .../suppressed_incompatible_return_type.phpt | 21 ++ Zend/zend_API.h | 73 ++++-- Zend/zend_attributes.c | 9 + Zend/zend_attributes.stub.php | 5 + Zend/zend_attributes_arginfo.h | 23 +- Zend/zend_compile.c | 4 +- Zend/zend_compile.h | 5 +- Zend/zend_execute.c | 1 + Zend/zend_inheritance.c | 71 +++-- build/gen_stub.php | 30 ++- ext/com_dotnet/com_handlers.c | 2 +- ext/date/php_date.stub.php | 243 +++++++++--------- ext/date/php_date_arginfo.h | 149 +++++++---- ext/date/tests/DateTime_extends_basic3.phpt | 2 +- ext/date/tests/bug55407.phpt | 2 +- ext/date/tests/bug62852_var2.phpt | 2 +- ext/date/tests/bug62852_var3.phpt | 2 +- ext/reflection/php_reflection.c | 44 +++- ext/reflection/php_reflection.stub.php | 4 + ext/reflection/php_reflection_arginfo.h | 30 ++- ...eflectionMethod_tentative_return_type.phpt | 62 +++++ 39 files changed, 668 insertions(+), 291 deletions(-) delete mode 100644 Zend/tests/type_declarations/variance/internal_parent.phpt create mode 100644 Zend/tests/type_declarations/variance/internal_parent/compatible_return_type.phpt create mode 100644 Zend/tests/type_declarations/variance/internal_parent/incompatible_return_type.phpt create mode 100644 Zend/tests/type_declarations/variance/internal_parent/missing_return_type.phpt create mode 100644 Zend/tests/type_declarations/variance/internal_parent/unresolvable_inheritance_check_param.phpt create mode 100644 Zend/tests/type_declarations/variance/internal_parent/unresolvable_inheritance_check_return.phpt create mode 100644 Zend/tests/type_declarations/variance/return_type_will_change_class_error.phpt create mode 100644 Zend/tests/type_declarations/variance/return_type_will_change_function_error.phpt create mode 100644 Zend/tests/type_declarations/variance/return_type_will_change_property_error.phpt create mode 100644 Zend/tests/type_declarations/variance/suppressed_incompatible_return_type.phpt create mode 100644 ext/reflection/tests/ReflectionMethod_tentative_return_type.phpt diff --git a/UPGRADING b/UPGRADING index 6d19c4281813d..25d82303b30ce 100644 --- a/UPGRADING +++ b/UPGRADING @@ -57,6 +57,10 @@ PHP 8.1 UPGRADE NOTES This means that static variables in methods now behave the same way as static properties. RFC: https://wiki.php.net/rfc/static_variable_inheritance + . Most non-final internal methods now require overriding methods to declare a + compatible return type, otherwise a deprecated notice is emitted during + inheritance validation. + RFC: https://wiki.php.net/rfc/internal_method_return_types - Fileinfo: . The fileinfo functions now accept and return, respectively, finfo objects diff --git a/Zend/Optimizer/zend_func_info.c b/Zend/Optimizer/zend_func_info.c index 3542458f762c6..4ca4dc3810ae0 100644 --- a/Zend/Optimizer/zend_func_info.c +++ b/Zend/Optimizer/zend_func_info.c @@ -844,7 +844,7 @@ ZEND_API uint32_t zend_get_func_info( #endif ret = zend_get_return_info_from_signature_only( - callee_func, /* script */ NULL, ce, ce_is_instanceof); + callee_func, /* script */ NULL, ce, ce_is_instanceof, /* use_tentative_return_info */ !call_info->is_prototype); #if ZEND_DEBUG if (internal_ret) { @@ -884,7 +884,7 @@ ZEND_API uint32_t zend_get_func_info( } if (!ret) { ret = zend_get_return_info_from_signature_only( - callee_func, /* TODO: script */ NULL, ce, ce_is_instanceof); + callee_func, /* TODO: script */ NULL, ce, ce_is_instanceof, /* use_tentative_return_info */ !call_info->is_prototype); } } return ret; diff --git a/Zend/Optimizer/zend_inference.c b/Zend/Optimizer/zend_inference.c index 2326b8a21f03d..c582a8763ad98 100644 --- a/Zend/Optimizer/zend_inference.c +++ b/Zend/Optimizer/zend_inference.c @@ -4001,9 +4001,11 @@ static int is_recursive_tail_call(const zend_op_array *op_array, uint32_t zend_get_return_info_from_signature_only( const zend_function *func, const zend_script *script, - zend_class_entry **ce, bool *ce_is_instanceof) { + zend_class_entry **ce, bool *ce_is_instanceof, bool use_tentative_return_info) { uint32_t type; - if (func->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE) { + if (func->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE && + (use_tentative_return_info || !ZEND_ARG_TYPE_IS_TENTATIVE(func->common.arg_info - 1)) + ) { zend_arg_info *ret_info = func->common.arg_info - 1; type = zend_fetch_arg_info_type(script, ret_info, ce); *ce_is_instanceof = ce != NULL; @@ -4025,15 +4027,15 @@ uint32_t zend_get_return_info_from_signature_only( ZEND_API void zend_init_func_return_info( const zend_op_array *op_array, const zend_script *script, zend_ssa_var_info *ret) { - if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) { - zend_ssa_range tmp_range = {0, 0, 0, 0}; - bool is_instanceof = false; - ret->type = zend_get_return_info_from_signature_only( - (zend_function *) op_array, script, &ret->ce, &is_instanceof); - ret->is_instanceof = is_instanceof; - ret->range = tmp_range; - ret->has_range = 0; - } + ZEND_ASSERT((op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE)); + + zend_ssa_range tmp_range = {0, 0, 0, 0}; + bool is_instanceof = false; + ret->type = zend_get_return_info_from_signature_only( + (zend_function *) op_array, script, &ret->ce, &is_instanceof, /* use_tentative_return_info */ 1); + ret->is_instanceof = is_instanceof; + ret->range = tmp_range; + ret->has_range = 0; } void zend_func_return_info(const zend_op_array *op_array, diff --git a/Zend/Optimizer/zend_inference.h b/Zend/Optimizer/zend_inference.h index 861262968111b..f9dbb405cc23c 100644 --- a/Zend/Optimizer/zend_inference.h +++ b/Zend/Optimizer/zend_inference.h @@ -271,7 +271,7 @@ ZEND_API void zend_init_func_return_info( const zend_op_array *op_array, const zend_script *script, zend_ssa_var_info *ret); uint32_t zend_get_return_info_from_signature_only( const zend_function *func, const zend_script *script, - zend_class_entry **ce, bool *ce_is_instanceof); + zend_class_entry **ce, bool *ce_is_instanceof, bool use_tentative_return_info); void zend_func_return_info(const zend_op_array *op_array, const zend_script *script, int recursive, diff --git a/Zend/tests/parameter_default_values/internal_declaration_error_class_const.phpt b/Zend/tests/parameter_default_values/internal_declaration_error_class_const.phpt index 69e7607bb350c..fc354a3855715 100644 --- a/Zend/tests/parameter_default_values/internal_declaration_error_class_const.phpt +++ b/Zend/tests/parameter_default_values/internal_declaration_error_class_const.phpt @@ -4,10 +4,10 @@ The default value is a class constant in the parent class method's signature. --EXPECTF-- -Fatal error: Declaration of MyDateTimeZone::listIdentifiers() must be compatible with DateTimeZone::listIdentifiers(int $timezoneGroup = DateTimeZone::ALL, ?string $countryCode = null) in %s on line %d +Fatal error: Declaration of MyDateTimeZone::listIdentifiers(): array must be compatible with DateTimeZone::listIdentifiers(int $timezoneGroup = DateTimeZone::ALL, ?string $countryCode = null): array in %s on line %d diff --git a/Zend/tests/parameter_default_values/internal_declaration_error_const.phpt b/Zend/tests/parameter_default_values/internal_declaration_error_const.phpt index 9335abd5d6a91..5b0a9bf05f0f0 100644 --- a/Zend/tests/parameter_default_values/internal_declaration_error_const.phpt +++ b/Zend/tests/parameter_default_values/internal_declaration_error_const.phpt @@ -4,10 +4,10 @@ The default value is a constant in the parent class method's signature. --EXPECTF-- -Fatal error: Declaration of MyDateTimeZone::getTransitions() must be compatible with DateTimeZone::getTransitions(int $timestampBegin = PHP_INT_MIN, int $timestampEnd = PHP_INT_MAX) in %s on line %d +Fatal error: Declaration of MyDateTimeZone::getTransitions(): array|false must be compatible with DateTimeZone::getTransitions(int $timestampBegin = PHP_INT_MIN, int $timestampEnd = PHP_INT_MAX): array|false in %s on line %d diff --git a/Zend/tests/parameter_default_values/internal_declaration_error_false.phpt b/Zend/tests/parameter_default_values/internal_declaration_error_false.phpt index 80c98a405253a..c01d7d256a66d 100644 --- a/Zend/tests/parameter_default_values/internal_declaration_error_false.phpt +++ b/Zend/tests/parameter_default_values/internal_declaration_error_false.phpt @@ -5,8 +5,8 @@ The default value is false in the parent class method's signature. interface MyDateTimeInterface extends DateTimeInterface { - public function diff(); + public function diff(): DateInterval; } ?> --EXPECTF-- -Fatal error: Declaration of MyDateTimeInterface::diff() must be compatible with DateTimeInterface::diff(DateTimeInterface $targetObject, bool $absolute = false) in %s on line %d +Fatal error: Declaration of MyDateTimeInterface::diff(): DateInterval must be compatible with DateTimeInterface::diff(DateTimeInterface $targetObject, bool $absolute = false): DateInterval in %s on line %d diff --git a/Zend/tests/parameter_default_values/internal_declaration_error_int.phpt b/Zend/tests/parameter_default_values/internal_declaration_error_int.phpt index c32cc9e41fbc1..edf71c0263fea 100644 --- a/Zend/tests/parameter_default_values/internal_declaration_error_int.phpt +++ b/Zend/tests/parameter_default_values/internal_declaration_error_int.phpt @@ -4,10 +4,10 @@ The default value is an integer in the parent class method's signature. --EXPECTF-- -Fatal error: Declaration of MyDateTime::setTime(int $hour, int $minute, int $second = 0, bool $microsecond = false) must be compatible with DateTime::setTime(int $hour, int $minute, int $second = 0, int $microsecond = 0) in %s on line %d +Fatal error: Declaration of MyDateTime::setTime(int $hour, int $minute, int $second = 0, bool $microsecond = false): DateTime must be compatible with DateTime::setTime(int $hour, int $minute, int $second = 0, int $microsecond = 0): DateTime in %s on line %d diff --git a/Zend/tests/parameter_default_values/internal_declaration_error_null.phpt b/Zend/tests/parameter_default_values/internal_declaration_error_null.phpt index 3804c2a6e1557..ebe5e142155ac 100644 --- a/Zend/tests/parameter_default_values/internal_declaration_error_null.phpt +++ b/Zend/tests/parameter_default_values/internal_declaration_error_null.phpt @@ -4,10 +4,10 @@ The default value is null in the parent class method's signature. --EXPECTF-- -Fatal error: Declaration of MyDateTime::createFromFormat() must be compatible with DateTime::createFromFormat(string $format, string $datetime, ?DateTimeZone $timezone = null) in %s on line %d +Fatal error: Declaration of MyDateTime::createFromFormat(): DateTime|false must be compatible with DateTime::createFromFormat(string $format, string $datetime, ?DateTimeZone $timezone = null): DateTime|false in %s on line %d diff --git a/Zend/tests/type_declarations/variance/internal_parent.phpt b/Zend/tests/type_declarations/variance/internal_parent.phpt deleted file mode 100644 index c82e10ffc34d5..0000000000000 --- a/Zend/tests/type_declarations/variance/internal_parent.phpt +++ /dev/null @@ -1,12 +0,0 @@ ---TEST-- -Internal class as parent ---FILE-- - ---EXPECTF-- -Fatal error: Could not check compatibility between Test::createFromFormat($format, $datetime, ?Wrong $timezone = null) and DateTime::createFromFormat(string $format, string $datetime, ?DateTimeZone $timezone = null), because class Wrong is not available in %s on line %d diff --git a/Zend/tests/type_declarations/variance/internal_parent/compatible_return_type.phpt b/Zend/tests/type_declarations/variance/internal_parent/compatible_return_type.phpt new file mode 100644 index 0000000000000..a45e3f4eb942f --- /dev/null +++ b/Zend/tests/type_declarations/variance/internal_parent/compatible_return_type.phpt @@ -0,0 +1,17 @@ +--TEST-- +Test that no notice is emitted when the return type/value of the overriding method is compatible with the tentative return type/value of the overridden method +--FILE-- + +--EXPECT-- +array(0) { +} diff --git a/Zend/tests/type_declarations/variance/internal_parent/incompatible_return_type.phpt b/Zend/tests/type_declarations/variance/internal_parent/incompatible_return_type.phpt new file mode 100644 index 0000000000000..8c40a46a86608 --- /dev/null +++ b/Zend/tests/type_declarations/variance/internal_parent/incompatible_return_type.phpt @@ -0,0 +1,17 @@ +--TEST-- +Test that a notice is emitted when the return type/value of the overriding method is incompatible with the tentative return type/value of the overridden method +--FILE-- + +--EXPECTF-- +Deprecated: Declaration of MyDateTimeZone::listIdentifiers(int $timezoneGroup = DateTimeZone::ALL, ?string $countryCode = null): string should be compatible with DateTimeZone::listIdentifiers(int $timezoneGroup = DateTimeZone::ALL, ?string $countryCode = null): array in %s on line %d +string(0) "" diff --git a/Zend/tests/type_declarations/variance/internal_parent/missing_return_type.phpt b/Zend/tests/type_declarations/variance/internal_parent/missing_return_type.phpt new file mode 100644 index 0000000000000..c20255df15c56 --- /dev/null +++ b/Zend/tests/type_declarations/variance/internal_parent/missing_return_type.phpt @@ -0,0 +1,13 @@ +--TEST-- +Test that a notice is emitted when the tentative return type of the overridden method is omitted +--FILE-- + +--EXPECTF-- +Deprecated: Declaration of MyDateTimeZone::listIdentifiers(int $timezoneGroup = DateTimeZone::ALL, ?string $countryCode = null) should be compatible with DateTimeZone::listIdentifiers(int $timezoneGroup = DateTimeZone::ALL, ?string $countryCode = null): array in %s on line %d diff --git a/Zend/tests/type_declarations/variance/internal_parent/unresolvable_inheritance_check_param.phpt b/Zend/tests/type_declarations/variance/internal_parent/unresolvable_inheritance_check_param.phpt new file mode 100644 index 0000000000000..d1a1ad78ce30a --- /dev/null +++ b/Zend/tests/type_declarations/variance/internal_parent/unresolvable_inheritance_check_param.phpt @@ -0,0 +1,12 @@ +--TEST-- +Test unresolvable inheritance check due to unavailable parameter type when the parent has a tentative return type. +--FILE-- + +--EXPECTF-- +Fatal error: Could not check compatibility between Test::createFromFormat($format, $datetime, ?Wrong $timezone = null): DateTime|false and DateTime::createFromFormat(string $format, string $datetime, ?DateTimeZone $timezone = null): DateTime|false, because class Wrong is not available in %s on line %d diff --git a/Zend/tests/type_declarations/variance/internal_parent/unresolvable_inheritance_check_return.phpt b/Zend/tests/type_declarations/variance/internal_parent/unresolvable_inheritance_check_return.phpt new file mode 100644 index 0000000000000..5cc1730741920 --- /dev/null +++ b/Zend/tests/type_declarations/variance/internal_parent/unresolvable_inheritance_check_return.phpt @@ -0,0 +1,12 @@ +--TEST-- +Test unresolvable inheritance check due to unavailable return type when the parent has a tentative return type. +--FILE-- + +--EXPECTF-- +Fatal error: Could not check compatibility between Test::createFromFormat($format, $datetime, $timezone = null): Wrong and DateTime::createFromFormat(string $format, string $datetime, ?DateTimeZone $timezone = null): DateTime|false, because class Wrong is not available in %s on line %d diff --git a/Zend/tests/type_declarations/variance/return_type_will_change_class_error.phpt b/Zend/tests/type_declarations/variance/return_type_will_change_class_error.phpt new file mode 100644 index 0000000000000..f2698bb7a9563 --- /dev/null +++ b/Zend/tests/type_declarations/variance/return_type_will_change_class_error.phpt @@ -0,0 +1,13 @@ +--TEST-- +Test that the ReturnTypeWillChange attribute cannot target classes +--FILE-- + +--EXPECTF-- +Fatal error: Attribute "ReturnTypeWillChange" cannot target class (allowed targets: method) in %s on line %d diff --git a/Zend/tests/type_declarations/variance/return_type_will_change_function_error.phpt b/Zend/tests/type_declarations/variance/return_type_will_change_function_error.phpt new file mode 100644 index 0000000000000..e539275c3e862 --- /dev/null +++ b/Zend/tests/type_declarations/variance/return_type_will_change_function_error.phpt @@ -0,0 +1,11 @@ +--TEST-- +Test that the ReturnTypeWillChange attribute cannot target functions +--FILE-- + +--EXPECTF-- +Fatal error: Attribute "ReturnTypeWillChange" cannot target function (allowed targets: method) in %s on line %d diff --git a/Zend/tests/type_declarations/variance/return_type_will_change_property_error.phpt b/Zend/tests/type_declarations/variance/return_type_will_change_property_error.phpt new file mode 100644 index 0000000000000..ebb2b49c2f8c3 --- /dev/null +++ b/Zend/tests/type_declarations/variance/return_type_will_change_property_error.phpt @@ -0,0 +1,14 @@ +--TEST-- +Test that the ReturnTypeWillChange attribute cannot be used with functions +--FILE-- + +--EXPECTF-- +Fatal error: Attribute "ReturnTypeWillChange" cannot target property (allowed targets: method) in %s on line %d diff --git a/Zend/tests/type_declarations/variance/suppressed_incompatible_return_type.phpt b/Zend/tests/type_declarations/variance/suppressed_incompatible_return_type.phpt new file mode 100644 index 0000000000000..5da5f8d513032 --- /dev/null +++ b/Zend/tests/type_declarations/variance/suppressed_incompatible_return_type.phpt @@ -0,0 +1,21 @@ +--TEST-- +Test that the notice can be suppressed when the return type/value of the overriding method is incompatible with the tentative return type/value of the overridden method +--FILE-- +modify("+1 sec")); +?> +--EXPECT-- +bool(false) diff --git a/Zend/zend_API.h b/Zend/zend_API.h index 50764faeca367..133791f300ecd 100644 --- a/Zend/zend_API.h +++ b/Zend/zend_API.h @@ -107,66 +107,91 @@ typedef struct _zend_fcall_info_cache { #define ZEND_FE_END { NULL, NULL, NULL, 0, 0 } -#define _ZEND_ARG_INFO_FLAGS(pass_by_ref, is_variadic) \ - (((pass_by_ref) << _ZEND_SEND_MODE_SHIFT) | ((is_variadic) ? _ZEND_IS_VARIADIC_BIT : 0)) +#define _ZEND_ARG_INFO_FLAGS(pass_by_ref, is_variadic, is_tentative) \ + (((pass_by_ref) << _ZEND_SEND_MODE_SHIFT) | ((is_variadic) ? _ZEND_IS_VARIADIC_BIT : 0) | ((is_tentative) ? _ZEND_IS_TENTATIVE_BIT : 0)) /* Arginfo structures without type information */ #define ZEND_ARG_INFO(pass_by_ref, name) \ - { #name, ZEND_TYPE_INIT_NONE(_ZEND_ARG_INFO_FLAGS(pass_by_ref, 0)), NULL }, + { #name, ZEND_TYPE_INIT_NONE(_ZEND_ARG_INFO_FLAGS(pass_by_ref, 0, 0)), NULL }, #define ZEND_ARG_INFO_WITH_DEFAULT_VALUE(pass_by_ref, name, default_value) \ - { #name, ZEND_TYPE_INIT_NONE(_ZEND_ARG_INFO_FLAGS(pass_by_ref, 0)), default_value }, + { #name, ZEND_TYPE_INIT_NONE(_ZEND_ARG_INFO_FLAGS(pass_by_ref, 0, 0)), default_value }, #define ZEND_ARG_VARIADIC_INFO(pass_by_ref, name) \ - { #name, ZEND_TYPE_INIT_NONE(_ZEND_ARG_INFO_FLAGS(pass_by_ref, 1)), NULL }, + { #name, ZEND_TYPE_INIT_NONE(_ZEND_ARG_INFO_FLAGS(pass_by_ref, 1, 0)), NULL }, /* Arginfo structures with simple type information */ #define ZEND_ARG_TYPE_INFO(pass_by_ref, name, type_hint, allow_null) \ - { #name, ZEND_TYPE_INIT_CODE(type_hint, allow_null, _ZEND_ARG_INFO_FLAGS(pass_by_ref, 0)), NULL }, + { #name, ZEND_TYPE_INIT_CODE(type_hint, allow_null, _ZEND_ARG_INFO_FLAGS(pass_by_ref, 0, 0)), NULL }, #define ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(pass_by_ref, name, type_hint, allow_null, default_value) \ - { #name, ZEND_TYPE_INIT_CODE(type_hint, allow_null, _ZEND_ARG_INFO_FLAGS(pass_by_ref, 0)), default_value }, + { #name, ZEND_TYPE_INIT_CODE(type_hint, allow_null, _ZEND_ARG_INFO_FLAGS(pass_by_ref, 0, 0)), default_value }, #define ZEND_ARG_VARIADIC_TYPE_INFO(pass_by_ref, name, type_hint, allow_null) \ - { #name, ZEND_TYPE_INIT_CODE(type_hint, allow_null, _ZEND_ARG_INFO_FLAGS(pass_by_ref, 1)), NULL }, + { #name, ZEND_TYPE_INIT_CODE(type_hint, allow_null, _ZEND_ARG_INFO_FLAGS(pass_by_ref, 1, 0)), NULL }, /* Arginfo structures with complex type information */ #define ZEND_ARG_TYPE_MASK(pass_by_ref, name, type_mask, default_value) \ - { #name, ZEND_TYPE_INIT_MASK(type_mask | _ZEND_ARG_INFO_FLAGS(pass_by_ref, 0)), default_value }, + { #name, ZEND_TYPE_INIT_MASK(type_mask | _ZEND_ARG_INFO_FLAGS(pass_by_ref, 0, 0)), default_value }, #define ZEND_ARG_OBJ_TYPE_MASK(pass_by_ref, name, class_name, type_mask, default_value) \ - { #name, ZEND_TYPE_INIT_CLASS_CONST_MASK(#class_name, type_mask | _ZEND_ARG_INFO_FLAGS(pass_by_ref, 0)), default_value }, + { #name, ZEND_TYPE_INIT_CLASS_CONST_MASK(#class_name, type_mask | _ZEND_ARG_INFO_FLAGS(pass_by_ref, 0, 0)), default_value }, /* Arginfo structures with object type information */ #define ZEND_ARG_OBJ_INFO(pass_by_ref, name, classname, allow_null) \ - { #name, ZEND_TYPE_INIT_CLASS_CONST(#classname, allow_null, _ZEND_ARG_INFO_FLAGS(pass_by_ref, 0)), NULL }, + { #name, ZEND_TYPE_INIT_CLASS_CONST(#classname, allow_null, _ZEND_ARG_INFO_FLAGS(pass_by_ref, 0, 0)), NULL }, #define ZEND_ARG_OBJ_INFO_WITH_DEFAULT_VALUE(pass_by_ref, name, classname, allow_null, default_value) \ - { #name, ZEND_TYPE_INIT_CLASS_CONST(#classname, allow_null, _ZEND_ARG_INFO_FLAGS(pass_by_ref, 0)), default_value }, + { #name, ZEND_TYPE_INIT_CLASS_CONST(#classname, allow_null, _ZEND_ARG_INFO_FLAGS(pass_by_ref, 0, 0)), default_value }, #define ZEND_ARG_VARIADIC_OBJ_INFO(pass_by_ref, name, classname, allow_null) \ - { #name, ZEND_TYPE_INIT_CLASS_CONST(#classname, allow_null, _ZEND_ARG_INFO_FLAGS(pass_by_ref, 1)), NULL }, + { #name, ZEND_TYPE_INIT_CLASS_CONST(#classname, allow_null, _ZEND_ARG_INFO_FLAGS(pass_by_ref, 1, 0)), NULL }, /* Legacy arginfo structures */ #define ZEND_ARG_ARRAY_INFO(pass_by_ref, name, allow_null) \ - { #name, ZEND_TYPE_INIT_CODE(IS_ARRAY, allow_null, _ZEND_ARG_INFO_FLAGS(pass_by_ref, 0)), NULL }, + { #name, ZEND_TYPE_INIT_CODE(IS_ARRAY, allow_null, _ZEND_ARG_INFO_FLAGS(pass_by_ref, 0, 0)), NULL }, #define ZEND_ARG_CALLABLE_INFO(pass_by_ref, name, allow_null) \ - { #name, ZEND_TYPE_INIT_CODE(IS_CALLABLE, allow_null, _ZEND_ARG_INFO_FLAGS(pass_by_ref, 0)), NULL }, + { #name, ZEND_TYPE_INIT_CODE(IS_CALLABLE, allow_null, _ZEND_ARG_INFO_FLAGS(pass_by_ref, 0, 0)), NULL }, -#define ZEND_BEGIN_ARG_WITH_RETURN_OBJ_INFO_EX(name, return_reference, required_num_args, class_name, allow_null) \ +#define ZEND_BEGIN_ARG_WITH_RETURN_OBJ_INFO_EX2(name, return_reference, required_num_args, class_name, allow_null, is_tentative_return_type) \ static const zend_internal_arg_info name[] = { \ { (const char*)(zend_uintptr_t)(required_num_args), \ - ZEND_TYPE_INIT_CLASS_CONST(#class_name, allow_null, _ZEND_ARG_INFO_FLAGS(return_reference, 0)), NULL }, + ZEND_TYPE_INIT_CLASS_CONST(#class_name, allow_null, _ZEND_ARG_INFO_FLAGS(return_reference, 0, is_tentative_return_type)), NULL }, + +#define ZEND_BEGIN_ARG_WITH_RETURN_OBJ_INFO_EX(name, return_reference, required_num_args, class_name, allow_null) \ + ZEND_BEGIN_ARG_WITH_RETURN_OBJ_INFO_EX2(name, return_reference, required_num_args, class_name, allow_null, 0) + +#define ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(name, return_reference, required_num_args, class_name, allow_null) \ + ZEND_BEGIN_ARG_WITH_RETURN_OBJ_INFO_EX2(name, return_reference, required_num_args, class_name, allow_null, 1) #define ZEND_BEGIN_ARG_WITH_RETURN_OBJ_INFO(name, class_name, allow_null) \ - ZEND_BEGIN_ARG_WITH_RETURN_OBJ_INFO_EX(name, 0, -1, class_name, allow_null) + ZEND_BEGIN_ARG_WITH_RETURN_OBJ_INFO_EX2(name, 0, -1, class_name, allow_null, 0) + +#define ZEND_BEGIN_ARG_WITH_RETURN_TYPE_MASK_EX2(name, return_reference, required_num_args, type, is_tentative_return_type) \ + static const zend_internal_arg_info name[] = { \ + { (const char*)(zend_uintptr_t)(required_num_args), ZEND_TYPE_INIT_MASK(type | _ZEND_ARG_INFO_FLAGS(return_reference, 0, is_tentative_return_type)), NULL }, #define ZEND_BEGIN_ARG_WITH_RETURN_TYPE_MASK_EX(name, return_reference, required_num_args, type) \ + ZEND_BEGIN_ARG_WITH_RETURN_TYPE_MASK_EX2(name, return_reference, required_num_args, type, 0) + +#define ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_TYPE_MASK_EX(name, return_reference, required_num_args, type) \ + ZEND_BEGIN_ARG_WITH_RETURN_TYPE_MASK_EX2(name, return_reference, required_num_args, type, 1) + +#define ZEND_BEGIN_ARG_WITH_RETURN_OBJ_TYPE_MASK_EX2(name, return_reference, required_num_args, class_name, type, is_tentative_return_type) \ static const zend_internal_arg_info name[] = { \ - { (const char*)(zend_uintptr_t)(required_num_args), ZEND_TYPE_INIT_MASK(type | _ZEND_ARG_INFO_FLAGS(return_reference, 0)), NULL }, + { (const char*)(zend_uintptr_t)(required_num_args), ZEND_TYPE_INIT_CLASS_CONST_MASK(#class_name, type | _ZEND_ARG_INFO_FLAGS(return_reference, 0, is_tentative_return_type)), NULL }, #define ZEND_BEGIN_ARG_WITH_RETURN_OBJ_TYPE_MASK_EX(name, return_reference, required_num_args, class_name, type) \ + ZEND_BEGIN_ARG_WITH_RETURN_OBJ_TYPE_MASK_EX2(name, return_reference, required_num_args, class_name, type, 0) + +#define ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_TYPE_MASK_EX(name, return_reference, required_num_args, class_name, type) \ + ZEND_BEGIN_ARG_WITH_RETURN_OBJ_TYPE_MASK_EX2(name, return_reference, required_num_args, class_name, type, 1) + +#define ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX2(name, return_reference, required_num_args, type, allow_null, is_tentative_return_type) \ static const zend_internal_arg_info name[] = { \ - { (const char*)(zend_uintptr_t)(required_num_args), ZEND_TYPE_INIT_CLASS_CONST_MASK(#class_name, type | _ZEND_ARG_INFO_FLAGS(return_reference, 0)), NULL }, + { (const char*)(zend_uintptr_t)(required_num_args), ZEND_TYPE_INIT_CODE(type, allow_null, _ZEND_ARG_INFO_FLAGS(return_reference, 0, is_tentative_return_type)), NULL }, #define ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(name, return_reference, required_num_args, type, allow_null) \ - static const zend_internal_arg_info name[] = { \ - { (const char*)(zend_uintptr_t)(required_num_args), ZEND_TYPE_INIT_CODE(type, allow_null, _ZEND_ARG_INFO_FLAGS(return_reference, 0)), NULL }, + ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX2(name, return_reference, required_num_args, type, allow_null, 0) + +#define ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_TYPE_INFO_EX(name, return_reference, required_num_args, type, allow_null) \ + ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX2(name, return_reference, required_num_args, type, allow_null, 1) + #define ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO(name, type, allow_null) \ - ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(name, 0, -1, type, allow_null) + ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX2(name, 0, -1, type, allow_null, 0) #define ZEND_BEGIN_ARG_INFO_EX(name, _unused, return_reference, required_num_args) \ static const zend_internal_arg_info name[] = { \ - { (const char*)(zend_uintptr_t)(required_num_args), ZEND_TYPE_INIT_NONE(_ZEND_ARG_INFO_FLAGS(return_reference, 0)), NULL }, + { (const char*)(zend_uintptr_t)(required_num_args), ZEND_TYPE_INIT_NONE(_ZEND_ARG_INFO_FLAGS(return_reference, 0, 0)), NULL }, #define ZEND_BEGIN_ARG_INFO(name, _unused) \ ZEND_BEGIN_ARG_INFO_EX(name, {}, ZEND_RETURN_VALUE, -1) #define ZEND_END_ARG_INFO() }; diff --git a/Zend/zend_attributes.c b/Zend/zend_attributes.c index 2823d3bafb3ea..e04547300f75f 100644 --- a/Zend/zend_attributes.c +++ b/Zend/zend_attributes.c @@ -24,6 +24,7 @@ #include "zend_smart_str.h" ZEND_API zend_class_entry *zend_ce_attribute; +ZEND_API zend_class_entry *zend_ce_return_type_will_change_attribute; static HashTable internal_attributes; @@ -67,6 +68,11 @@ ZEND_METHOD(Attribute, __construct) ZVAL_LONG(OBJ_PROP_NUM(Z_OBJ_P(ZEND_THIS), 0), flags); } +ZEND_METHOD(ReturnTypeWillChange, __construct) +{ + ZEND_PARSE_PARAMETERS_NONE(); +} + static zend_attribute *get_attribute(HashTable *attributes, zend_string *lcname, uint32_t offset) { if (attributes) { @@ -278,6 +284,9 @@ void zend_register_attribute_ce(void) zend_declare_class_constant_long(zend_ce_attribute, ZEND_STRL("TARGET_PARAMETER"), ZEND_ATTRIBUTE_TARGET_PARAMETER); zend_declare_class_constant_long(zend_ce_attribute, ZEND_STRL("TARGET_ALL"), ZEND_ATTRIBUTE_TARGET_ALL); zend_declare_class_constant_long(zend_ce_attribute, ZEND_STRL("IS_REPEATABLE"), ZEND_ATTRIBUTE_IS_REPEATABLE); + + zend_ce_return_type_will_change_attribute = register_class_ReturnTypeWillChange(); + zend_internal_attribute_register(zend_ce_return_type_will_change_attribute, ZEND_ATTRIBUTE_TARGET_METHOD); } void zend_attributes_shutdown(void) diff --git a/Zend/zend_attributes.stub.php b/Zend/zend_attributes.stub.php index 26defa8d4db60..70b0c49aeb9f2 100644 --- a/Zend/zend_attributes.stub.php +++ b/Zend/zend_attributes.stub.php @@ -8,3 +8,8 @@ final class Attribute public function __construct(int $flags = Attribute::TARGET_ALL) {} } + +final class ReturnTypeWillChange +{ + public function __construct() {} +} diff --git a/Zend/zend_attributes_arginfo.h b/Zend/zend_attributes_arginfo.h index a09f9161fd61e..5f62eb8fd057d 100644 --- a/Zend/zend_attributes_arginfo.h +++ b/Zend/zend_attributes_arginfo.h @@ -1,12 +1,16 @@ /* This is a generated file, edit the .stub.php file instead. - * Stub hash: 0183e750e66999862a7688ecb251017110d06d1f */ + * Stub hash: 3fd949e1b9f49666bed3081ed1e8e711acd9f49c */ ZEND_BEGIN_ARG_INFO_EX(arginfo_class_Attribute___construct, 0, 0, 0) ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, flags, IS_LONG, 0, "Attribute::TARGET_ALL") ZEND_END_ARG_INFO() +ZEND_BEGIN_ARG_INFO_EX(arginfo_class_ReturnTypeWillChange___construct, 0, 0, 0) +ZEND_END_ARG_INFO() + ZEND_METHOD(Attribute, __construct); +ZEND_METHOD(ReturnTypeWillChange, __construct); static const zend_function_entry class_Attribute_methods[] = { @@ -14,6 +18,12 @@ static const zend_function_entry class_Attribute_methods[] = { ZEND_FE_END }; + +static const zend_function_entry class_ReturnTypeWillChange_methods[] = { + ZEND_ME(ReturnTypeWillChange, __construct, arginfo_class_ReturnTypeWillChange___construct, ZEND_ACC_PUBLIC) + ZEND_FE_END +}; + static zend_class_entry *register_class_Attribute(void) { zend_class_entry ce, *class_entry; @@ -30,3 +40,14 @@ static zend_class_entry *register_class_Attribute(void) return class_entry; } + +static zend_class_entry *register_class_ReturnTypeWillChange(void) +{ + zend_class_entry ce, *class_entry; + + INIT_CLASS_ENTRY(ce, "ReturnTypeWillChange", class_ReturnTypeWillChange_methods); + class_entry = zend_register_internal_class_ex(&ce, NULL); + class_entry->ce_flags |= ZEND_ACC_FINAL; + + return class_entry; +} diff --git a/Zend/zend_compile.c b/Zend/zend_compile.c index 1a884206fd7c8..6a16700d2c60e 100644 --- a/Zend/zend_compile.c +++ b/Zend/zend_compile.c @@ -6476,7 +6476,7 @@ void zend_compile_params(zend_ast *ast, zend_ast *return_type_ast, uint32_t fall arg_infos->type = zend_compile_typename( return_type_ast, /* force_allow_null */ 0); ZEND_TYPE_FULL_MASK(arg_infos->type) |= _ZEND_ARG_INFO_FLAGS( - (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) != 0, /* is_variadic */ 0); + (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) != 0, /* is_variadic */ 0, /* is_tentative */ 0); } else { arg_infos->type = (zend_type) ZEND_TYPE_INIT_CODE(fallback_return_type, 0, 0); } @@ -6606,7 +6606,7 @@ void zend_compile_params(zend_ast *ast, zend_ast *return_type_ast, uint32_t fall zend_alloc_cache_slots(zend_type_get_num_classes(arg_info->type)); } - uint32_t arg_info_flags = _ZEND_ARG_INFO_FLAGS(is_ref, is_variadic) + uint32_t arg_info_flags = _ZEND_ARG_INFO_FLAGS(is_ref, is_variadic, /* is_tentative */ 0) | (visibility ? _ZEND_IS_PROMOTED_BIT : 0); ZEND_TYPE_FULL_MASK(arg_info->type) |= arg_info_flags; if (opcode == ZEND_RECV) { diff --git a/Zend/zend_compile.h b/Zend/zend_compile.h index 6291e4397b883..5c60a22086730 100644 --- a/Zend/zend_compile.h +++ b/Zend/zend_compile.h @@ -983,16 +983,19 @@ ZEND_API zend_string *zend_type_to_string(zend_type type); #define ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS 1 -/* The send mode and is_variadic flag are stored as part of zend_type */ +/* The send mode, the is_variadic, the is_promoted, and the is_tentative flags are stored as part of zend_type */ #define _ZEND_SEND_MODE_SHIFT _ZEND_TYPE_EXTRA_FLAGS_SHIFT #define _ZEND_IS_VARIADIC_BIT (1 << (_ZEND_TYPE_EXTRA_FLAGS_SHIFT + 2)) #define _ZEND_IS_PROMOTED_BIT (1 << (_ZEND_TYPE_EXTRA_FLAGS_SHIFT + 3)) +#define _ZEND_IS_TENTATIVE_BIT (1 << (_ZEND_TYPE_EXTRA_FLAGS_SHIFT + 4)) #define ZEND_ARG_SEND_MODE(arg_info) \ ((ZEND_TYPE_FULL_MASK((arg_info)->type) >> _ZEND_SEND_MODE_SHIFT) & 3) #define ZEND_ARG_IS_VARIADIC(arg_info) \ ((ZEND_TYPE_FULL_MASK((arg_info)->type) & _ZEND_IS_VARIADIC_BIT) != 0) #define ZEND_ARG_IS_PROMOTED(arg_info) \ ((ZEND_TYPE_FULL_MASK((arg_info)->type) & _ZEND_IS_PROMOTED_BIT) != 0) +#define ZEND_ARG_TYPE_IS_TENTATIVE(arg_info) \ + ((ZEND_TYPE_FULL_MASK((arg_info)->type) & _ZEND_IS_TENTATIVE_BIT) != 0) #define ZEND_DIM_IS (1 << 0) /* isset fetch needed for null coalesce */ #define ZEND_DIM_ALTERNATIVE_SYNTAX (1 << 1) /* deprecated curly brace usage */ diff --git a/Zend/zend_execute.c b/Zend/zend_execute.c index 151eb4ecc59a3..18868dcad2a70 100644 --- a/Zend/zend_execute.c +++ b/Zend/zend_execute.c @@ -1294,6 +1294,7 @@ static ZEND_COLD void zend_verify_void_return_error(const zend_function *zf, con static bool zend_verify_internal_return_type(zend_function *zf, zval *ret) { zend_internal_arg_info *ret_info = zf->internal_function.arg_info - 1; + if (ZEND_TYPE_FULL_MASK(ret_info->type) & MAY_BE_VOID) { if (UNEXPECTED(Z_TYPE_P(ret) != IS_NULL)) { zend_verify_void_return_error(zf, zend_zval_type_name(ret), ""); diff --git a/Zend/zend_inheritance.c b/Zend/zend_inheritance.c index 1df7d5016df69..8f110345be8ca 100644 --- a/Zend/zend_inheritance.c +++ b/Zend/zend_inheritance.c @@ -27,10 +27,21 @@ #include "zend_operators.h" #include "zend_exceptions.h" #include "zend_enum.h" +#include "zend_attributes.h" ZEND_API zend_class_entry* (*zend_inheritance_cache_get)(zend_class_entry *ce, zend_class_entry *parent, zend_class_entry **traits_and_interfaces) = NULL; ZEND_API zend_class_entry* (*zend_inheritance_cache_add)(zend_class_entry *ce, zend_class_entry *proto, zend_class_entry *parent, zend_class_entry **traits_and_interfaces, HashTable *dependencies) = NULL; +/* Unresolved means that class declarations that are currently not available are needed to + * determine whether the inheritance is valid or not. At runtime UNRESOLVED should be treated + * as an ERROR. */ +typedef enum { + INHERITANCE_UNRESOLVED = -1, + INHERITANCE_ERROR = 0, + INHERITANCE_WARNING = 1, + INHERITANCE_SUCCESS = 2, +} inheritance_status; + static void add_dependency_obligation(zend_class_entry *ce, zend_class_entry *dependency_ce); static void add_compatibility_obligation( zend_class_entry *ce, const zend_function *child_fn, zend_class_entry *child_scope, @@ -39,7 +50,12 @@ static void add_property_compatibility_obligation( zend_class_entry *ce, const zend_property_info *child_prop, const zend_property_info *parent_prop); -static void zend_type_copy_ctor(zend_type *type, bool persistent) { +static void ZEND_COLD emit_incompatible_method_error( + const zend_function *child, zend_class_entry *child_scope, + const zend_function *parent, zend_class_entry *parent_scope, + inheritance_status status); + +static void zend_type_copy_ctor(zend_type *type, zend_bool persistent) { if (ZEND_TYPE_HAS_LIST(*type)) { zend_type_list *old_list = ZEND_TYPE_LIST(*type); size_t size = ZEND_TYPE_LIST_SIZE(old_list->num_types); @@ -343,16 +359,6 @@ static bool zend_type_permits_self( return 0; } -/* Unresolved means that class declarations that are currently not available are needed to - * determine whether the inheritance is valid or not. At runtime UNRESOLVED should be treated - * as an ERROR. */ -typedef enum { - INHERITANCE_UNRESOLVED = -1, - INHERITANCE_ERROR = 0, - INHERITANCE_SUCCESS = 1, -} inheritance_status; - - static void track_class_dependency(zend_class_entry *ce, zend_string *class_name) { HashTable *ht; @@ -455,7 +461,7 @@ static inheritance_status zend_perform_covariant_class_type_check( static inheritance_status zend_perform_covariant_type_check( zend_class_entry *fe_scope, zend_type fe_type, - zend_class_entry *proto_scope, zend_type proto_type) /* {{{ */ + zend_class_entry *proto_scope, zend_type proto_type, bool tentative) /* {{{ */ { ZEND_ASSERT(ZEND_TYPE_IS_SET(fe_type) && ZEND_TYPE_IS_SET(proto_type)); @@ -496,7 +502,7 @@ static inheritance_status zend_perform_covariant_type_check( if (added_types) { /* Otherwise adding new types is illegal */ - return INHERITANCE_ERROR; + return tentative ? INHERITANCE_WARNING : INHERITANCE_ERROR; } } @@ -522,7 +528,7 @@ static inheritance_status zend_perform_covariant_type_check( } if (status == INHERITANCE_ERROR) { - return INHERITANCE_ERROR; + return tentative ? INHERITANCE_WARNING : INHERITANCE_ERROR; } if (status != INHERITANCE_SUCCESS) { all_success = 0; @@ -570,7 +576,7 @@ static inheritance_status zend_do_perform_arg_type_hint_check( /* Contravariant type check is performed as a covariant type check with swapped * argument order. */ return zend_perform_covariant_type_check( - proto_scope, proto_arg_info->type, fe_scope, fe_arg_info->type); + proto_scope, proto_arg_info->type, fe_scope, fe_arg_info->type, 0); } /* }}} */ @@ -660,18 +666,25 @@ static inheritance_status zend_do_perform_implementation_check( /* Check return type compatibility, but only if the prototype already specifies * a return type. Adding a new return type is always valid. */ if (proto->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE) { - /* Removing a return type is not valid. */ + /* Removing a return type is not valid, unless the parent return type is tentative. */ if (!(fe->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE)) { - return INHERITANCE_ERROR; + if (ZEND_ARG_TYPE_IS_TENTATIVE(&proto->common.arg_info[-1])) { + if (status == INHERITANCE_SUCCESS) { + emit_incompatible_method_error(fe, fe_scope, proto, proto_scope, INHERITANCE_WARNING); + } + return status; + } else { + return INHERITANCE_ERROR; + } } local_status = zend_perform_covariant_type_check( fe_scope, fe->common.arg_info[-1].type, - proto_scope, proto->common.arg_info[-1].type); + proto_scope, proto->common.arg_info[-1].type, ZEND_ARG_TYPE_IS_TENTATIVE(&proto->common.arg_info[-1])); if (UNEXPECTED(local_status != INHERITANCE_SUCCESS)) { - if (UNEXPECTED(local_status == INHERITANCE_ERROR)) { - return INHERITANCE_ERROR; + if (UNEXPECTED(local_status == INHERITANCE_ERROR || local_status == INHERITANCE_WARNING)) { + return local_status; } ZEND_ASSERT(local_status == INHERITANCE_UNRESOLVED); status = INHERITANCE_UNRESOLVED; @@ -854,6 +867,18 @@ static void ZEND_COLD emit_incompatible_method_error( zend_error_at(E_COMPILE_ERROR, NULL, func_lineno(child), "Could not check compatibility between %s and %s, because class %s is not available", ZSTR_VAL(child_prototype), ZSTR_VAL(parent_prototype), ZSTR_VAL(unresolved_class)); + } else if (status == INHERITANCE_WARNING) { + zend_attribute *return_type_will_change_attribute = zend_get_attribute_str( + child->common.attributes, + "returntypewillchange", + sizeof("returntypewillchange")-1 + ); + + if (!return_type_will_change_attribute) { + zend_error_at(E_DEPRECATED, NULL, func_lineno(child), + "Declaration of %s should be compatible with %s", + ZSTR_VAL(child_prototype), ZSTR_VAL(parent_prototype)); + } } else { zend_error_at(E_COMPILE_ERROR, NULL, func_lineno(child), "Declaration of %s must be compatible with %s", @@ -874,7 +899,7 @@ static void perform_delayable_implementation_check( if (EXPECTED(status == INHERITANCE_UNRESOLVED)) { add_compatibility_obligation(ce, fe, fe_scope, proto, proto_scope); } else { - ZEND_ASSERT(status == INHERITANCE_ERROR); + ZEND_ASSERT(status == INHERITANCE_ERROR || status == INHERITANCE_WARNING); emit_incompatible_method_error(fe, fe_scope, proto, proto_scope, status); } } @@ -1048,9 +1073,9 @@ inheritance_status property_types_compatible( /* Perform a covariant type check in both directions to determined invariance. */ inheritance_status status1 = zend_perform_covariant_type_check( - child_info->ce, child_info->type, parent_info->ce, parent_info->type); + child_info->ce, child_info->type, parent_info->ce, parent_info->type, 0); inheritance_status status2 = zend_perform_covariant_type_check( - parent_info->ce, parent_info->type, child_info->ce, child_info->type); + parent_info->ce, parent_info->type, child_info->ce, child_info->type, 0); if (status1 == INHERITANCE_SUCCESS && status2 == INHERITANCE_SUCCESS) { return INHERITANCE_SUCCESS; } diff --git a/build/gen_stub.php b/build/gen_stub.php index 084e142e34598..30d1eac28f0c9 100755 --- a/build/gen_stub.php +++ b/build/gen_stub.php @@ -599,16 +599,20 @@ class ReturnInfo { public $type; /** @var Type|null */ public $phpDocType; + /** @var bool */ + public $tentativeReturnType; - public function __construct(bool $byRef, ?Type $type, ?Type $phpDocType) { + public function __construct(bool $byRef, ?Type $type, ?Type $phpDocType, bool $tentativeReturnType) { $this->byRef = $byRef; $this->type = $type; $this->phpDocType = $phpDocType; + $this->tentativeReturnType = $tentativeReturnType; } public function equals(ReturnInfo $other): bool { return $this->byRef === $other->byRef - && Type::equals($this->type, $other->type); + && Type::equals($this->type, $other->type) + && $this->tentativeReturnType === $other->tentativeReturnType; } public function getMethodSynopsisType(): ?Type { @@ -1431,6 +1435,7 @@ function parseFunctionLike( $isDeprecated = false; $verify = true; $docReturnType = null; + $tentativeReturnType = false; $docParamTypes = []; if ($comment) { @@ -1452,8 +1457,10 @@ function parseFunctionLike( } } else if ($tag->name === 'deprecated') { $isDeprecated = true; - } else if ($tag->name === 'no-verify') { + } else if ($tag->name === 'no-verify') { $verify = false; + } else if ($tag->name === 'tentative-return-type') { + $tentativeReturnType = true; } else if ($tag->name === 'return') { $docReturnType = $tag->getType(); } else if ($tag->name === 'param') { @@ -1530,7 +1537,8 @@ function parseFunctionLike( $return = new ReturnInfo( $func->returnsByRef(), $returnType ? Type::fromNode($returnType) : null, - $docReturnType ? Type::fromPhpDoc($docReturnType) : null + $docReturnType ? Type::fromPhpDoc($docReturnType) : null, + $tentativeReturnType ); return new FuncInfo( @@ -1814,18 +1822,22 @@ protected function pName_FullyQualified(Name\FullyQualified $node) { function funcInfoToCode(FuncInfo $funcInfo): string { $code = ''; $returnType = $funcInfo->return->type; + $isTentativeReturnType = $funcInfo->return->tentativeReturnType; + if ($returnType !== null) { if (null !== $simpleReturnType = $returnType->tryToSimpleType()) { if ($simpleReturnType->isBuiltin) { $code .= sprintf( - "ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(%s, %d, %d, %s, %d)\n", + "%s(%s, %d, %d, %s, %d)\n", + $isTentativeReturnType ? "ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_TYPE_INFO_EX" : "ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX", $funcInfo->getArgInfoName(), $funcInfo->return->byRef, $funcInfo->numRequiredArgs, $simpleReturnType->toTypeCode(), $returnType->isNullable() ); } else { $code .= sprintf( - "ZEND_BEGIN_ARG_WITH_RETURN_OBJ_INFO_EX(%s, %d, %d, %s, %d)\n", + "%s(%s, %d, %d, %s, %d)\n", + $isTentativeReturnType ? "ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX" : "ZEND_BEGIN_ARG_WITH_RETURN_OBJ_INFO_EX", $funcInfo->getArgInfoName(), $funcInfo->return->byRef, $funcInfo->numRequiredArgs, $simpleReturnType->toEscapedName(), $returnType->isNullable() @@ -1835,14 +1847,16 @@ function funcInfoToCode(FuncInfo $funcInfo): string { $arginfoType = $returnType->toArginfoType(); if ($arginfoType->hasClassType()) { $code .= sprintf( - "ZEND_BEGIN_ARG_WITH_RETURN_OBJ_TYPE_MASK_EX(%s, %d, %d, %s, %s)\n", + "%s(%s, %d, %d, %s, %s)\n", + $isTentativeReturnType ? "ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_TYPE_MASK_EX" : "ZEND_BEGIN_ARG_WITH_RETURN_OBJ_TYPE_MASK_EX", $funcInfo->getArgInfoName(), $funcInfo->return->byRef, $funcInfo->numRequiredArgs, $arginfoType->toClassTypeString(), $arginfoType->toTypeMask() ); } else { $code .= sprintf( - "ZEND_BEGIN_ARG_WITH_RETURN_TYPE_MASK_EX(%s, %d, %d, %s)\n", + "%s(%s, %d, %d, %s)\n", + $isTentativeReturnType ? "ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_TYPE_MASK_EX" : "ZEND_BEGIN_ARG_WITH_RETURN_TYPE_MASK_EX", $funcInfo->getArgInfoName(), $funcInfo->return->byRef, $funcInfo->numRequiredArgs, $arginfoType->toTypeMask() diff --git a/ext/com_dotnet/com_handlers.c b/ext/com_dotnet/com_handlers.c index db00909d99b1a..afab1dc069ba0 100644 --- a/ext/com_dotnet/com_handlers.c +++ b/ext/com_dotnet/com_handlers.c @@ -336,7 +336,7 @@ static zend_function *com_method_get(zend_object **object_ptr, zend_string *name for (i = 0; i < bindptr.lpfuncdesc->cParams; i++) { bool by_ref = (bindptr.lpfuncdesc->lprgelemdescParam[i].paramdesc.wParamFlags & PARAMFLAG_FOUT) != 0; - f.arg_info[i].type = (zend_type) ZEND_TYPE_INIT_NONE(_ZEND_ARG_INFO_FLAGS(by_ref, 0)); + f.arg_info[i].type = (zend_type) ZEND_TYPE_INIT_NONE(_ZEND_ARG_INFO_FLAGS(by_ref, 0, 0)); } f.num_args = bindptr.lpfuncdesc->cParams; diff --git a/ext/date/php_date.stub.php b/ext/date/php_date.stub.php index 63d590d9c9909..f17acf1e7ebbb 100644 --- a/ext/date/php_date.stub.php +++ b/ext/date/php_date.stub.php @@ -114,214 +114,211 @@ function date_sunset( function date_sun_info(int $timestamp, float $latitude, float $longitude): array {} -// NB: Adding return types to methods is a BC break! -// For now only using @return annotations here. - interface DateTimeInterface { - /** @return string */ - public function format(string $format); + /** @tentative-return-type */ + public function format(string $format): string; - /** @return DateTimeZone|false */ - public function getTimezone(); + /** @tentative-return-type */ + public function getTimezone(): DateTimeZone|false; - /** @return int */ - public function getOffset(); + /** @tentative-return-type */ + public function getOffset(): int; - /** @return int|false */ - public function getTimestamp(); + /** @tentative-return-type */ + public function getTimestamp(): int|false; - /** @return DateInterval|false */ - public function diff(DateTimeInterface $targetObject, bool $absolute = false); + /** @tentative-return-type */ + public function diff(DateTimeInterface $targetObject, bool $absolute = false): DateInterval; - /** @return void */ - public function __wakeup(); + /** @tentative-return-type */ + public function __wakeup(): void; } class DateTime implements DateTimeInterface { public function __construct(string $datetime = "now", ?DateTimeZone $timezone = null) {} - /** @return void */ - public function __wakeup() {} + /** @tentative-return-type */ + public function __wakeup(): void {} - /** @return DateTime */ - public static function __set_state(array $array) {} + /** @tentative-return-type */ + public static function __set_state(array $array): DateTime {} - /** @return DateTime */ - public static function createFromImmutable(DateTimeImmutable $object) {} + /** @tentative-return-type */ + public static function createFromImmutable(DateTimeImmutable $object): DateTime {} public static function createFromInterface(DateTimeInterface $object): DateTime {} /** - * @return DateTime|false + * @tentative-return-type * @alias date_create_from_format */ - public static function createFromFormat(string $format, string $datetime, ?DateTimeZone $timezone = null) {} + public static function createFromFormat(string $format, string $datetime, ?DateTimeZone $timezone = null): DateTime|false {} /** - * @return array|false + * @tentative-return-type * @alias date_get_last_errors */ - public static function getLastErrors() {} + public static function getLastErrors(): array|false {} /** - * @return string + * @tentative-return-type * @alias date_format */ - public function format(string $format) {} + public function format(string $format): string {} /** - * @return DateTime|false + * @tentative-return-type * @alias date_modify */ - public function modify(string $modifier) {} + public function modify(string $modifier): DateTime|false {} /** - * @return DateTime + * @tentative-return-type * @alias date_add */ - public function add(DateInterval $interval) {} + public function add(DateInterval $interval): DateTime {} /** - * @return DateTime + * @tentative-return-type * @alias date_sub */ - public function sub(DateInterval $interval) {} + public function sub(DateInterval $interval): DateTime {} /** - * @return DateTimeZone|false + * @tentative-return-type * @alias date_timezone_get */ - public function getTimezone() {} + public function getTimezone(): DateTimeZone|false {} /** - * @return DateTime + * @tentative-return-type * @alias date_timezone_set */ - public function setTimezone(DateTimeZone $timezone) {} + public function setTimezone(DateTimeZone $timezone): DateTime {} /** - * @return int + * @tentative-return-type * @alias date_offset_get */ - public function getOffset() {} + public function getOffset(): int {} /** - * @return DateTime + * @tentative-return-type * @alias date_time_set */ - public function setTime(int $hour, int $minute, int $second = 0, int $microsecond = 0) {} + public function setTime(int $hour, int $minute, int $second = 0, int $microsecond = 0): DateTime {} /** - * @return DateTime + * @tentative-return-type * @alias date_date_set */ - public function setDate(int $year, int $month, int $day) {} + public function setDate(int $year, int $month, int $day): DateTime {} /** - * @return DateTime + * @tentative-return-type * @alias date_isodate_set */ - public function setISODate(int $year, int $week, int $dayOfWeek = 1) {} + public function setISODate(int $year, int $week, int $dayOfWeek = 1): DateTime {} /** - * @return DateTime + * @tentative-return-type * @alias date_timestamp_set */ - public function setTimestamp(int $timestamp) {} + public function setTimestamp(int $timestamp): DateTime {} /** - * @return int + * @tentative-return-type * @alias date_timestamp_get */ - public function getTimestamp() {} + public function getTimestamp(): int {} /** - * @return DateInterval + * @tentative-return-type * @alias date_diff */ - public function diff(DateTimeInterface $targetObject, bool $absolute = false) {} + public function diff(DateTimeInterface $targetObject, bool $absolute = false): DateInterval {} } class DateTimeImmutable implements DateTimeInterface { public function __construct(string $datetime = "now", ?DateTimeZone $timezone = null) {} - /** @return void */ - public function __wakeup() {} + /** @tentative-return-type */ + public function __wakeup(): void {} - /** @return DateTimeImmutable */ - public static function __set_state(array $array) {} + /** @tentative-return-type */ + public static function __set_state(array $array): DateTimeImmutable {} /** - * @return DateTimeImmutable|false + * @tentative-return-type * @alias date_create_immutable_from_format */ - public static function createFromFormat(string $format, string $datetime, ?DateTimeZone $timezone = null) {} + public static function createFromFormat(string $format, string $datetime, ?DateTimeZone $timezone = null): DateTimeImmutable|false {} /** - * @return array|false + * @tentative-return-type * @alias date_get_last_errors */ - public static function getLastErrors() {} + public static function getLastErrors(): array|false {} /** - * @return string + * @tentative-return-type * @alias date_format */ - public function format(string $format) {} + public function format(string $format): string {} /** - * @return DateTimeZone|false + * @tentative-return-type * @alias date_timezone_get */ - public function getTimezone() {} + public function getTimezone(): DateTimeZone|false {} /** - * @return int + * @tentative-return-type * @alias date_offset_get */ - public function getOffset() {} + public function getOffset(): int {} /** - * @return int + * @tentative-return-type * @alias date_timestamp_get */ - public function getTimestamp() {} + public function getTimestamp(): int {} /** - * @return DateInterval + * @tentative-return-type * @alias date_diff */ - public function diff(DateTimeInterface $targetObject, bool $absolute = false) {} + public function diff(DateTimeInterface $targetObject, bool $absolute = false): DateInterval {} - /** @return DateTimeImmutable|false */ - public function modify(string $modifier) {} + /** @tentative-return-type */ + public function modify(string $modifier): DateTimeImmutable|false {} - /** @return DateTimeImmutable */ - public function add(DateInterval $interval) {} + /** @tentative-return-type */ + public function add(DateInterval $interval): DateTimeImmutable {} - /** @return DateTimeImmutable */ - public function sub(DateInterval $interval) {} + /** @tentative-return-type */ + public function sub(DateInterval $interval): DateTimeImmutable {} - /** @return DateTimeImmutable */ - public function setTimezone(DateTimeZone $timezone) {} + /** @tentative-return-type */ + public function setTimezone(DateTimeZone $timezone): DateTimeImmutable {} - /** @return DateTimeImmutable */ - public function setTime(int $hour, int $minute, int $second = 0, int $microsecond = 0) {} + /** @tentative-return-type */ + public function setTime(int $hour, int $minute, int $second = 0, int $microsecond = 0): DateTimeImmutable {} - /** @return DateTimeImmutable */ - public function setDate(int $year, int $month, int $day) {} + /** @tentative-return-type */ + public function setDate(int $year, int $month, int $day): DateTimeImmutable {} - /** @return DateTimeImmutable */ - public function setISODate(int $year, int $week, int $dayOfWeek = 1) {} + /** @tentative-return-type */ + public function setISODate(int $year, int $week, int $dayOfWeek = 1): DateTimeImmutable {} - /** @return DateTimeImmutable */ - public function setTimestamp(int $timestamp) {} + /** @tentative-return-type */ + public function setTimestamp(int $timestamp): DateTimeImmutable {} - /** @return DateTimeImmutable */ - public static function createFromMutable(DateTime $object) {} + /** @tentative-return-type */ + public static function createFromMutable(DateTime $object): DateTimeImmutable {} public static function createFromInterface(DateTimeInterface $object): DateTimeImmutable {} } @@ -331,46 +328,46 @@ class DateTimeZone public function __construct(string $timezone) {} /** - * @return string + * @tentative-return-type * @alias timezone_name_get */ - public function getName() {} + public function getName(): string {} /** - * @return int + * @tentative-return-type * @alias timezone_offset_get */ - public function getOffset(DateTimeInterface $datetime) {} + public function getOffset(DateTimeInterface $datetime): int {} /** - * @return array|false + * @tentative-return-type * @alias timezone_transitions_get */ - public function getTransitions(int $timestampBegin = PHP_INT_MIN, int $timestampEnd = PHP_INT_MAX) {} + public function getTransitions(int $timestampBegin = PHP_INT_MIN, int $timestampEnd = PHP_INT_MAX): array|false {} /** - * @return array|false + * @tentative-return-type * @alias timezone_location_get */ - public function getLocation() {} + public function getLocation(): array|false {} /** - * @return array + * @tentative-return-type * @alias timezone_abbreviations_list */ - public static function listAbbreviations() {} + public static function listAbbreviations(): array {} /** - * @return array + * @tentative-return-type * @alias timezone_identifiers_list */ - public static function listIdentifiers(int $timezoneGroup = DateTimeZone::ALL, ?string $countryCode = null) {} + public static function listIdentifiers(int $timezoneGroup = DateTimeZone::ALL, ?string $countryCode = null): array {} - /** @return void */ - public function __wakeup() {} + /** @tentative-return-type */ + public function __wakeup(): void {} - /** @return DateTimeZone */ - public static function __set_state(array $array) {} + /** @tentative-return-type */ + public static function __set_state(array $array): DateTimeZone {} } class DateInterval @@ -378,22 +375,22 @@ class DateInterval public function __construct(string $duration) {} /** - * @return DateInterval|false + * @tentative-return-type * @alias date_interval_create_from_date_string */ - public static function createFromDateString(string $datetime) {} + public static function createFromDateString(string $datetime): DateInterval|false {} /** - * @return string + * @tentative-return-type * @alias date_interval_format */ - public function format(string $format) {} + public function format(string $format): string {} - /** @return void */ - public function __wakeup() {} + /** @tentative-return-type */ + public function __wakeup(): void {} - /** @return DateInterval */ - public static function __set_state(array $array) {} + /** @tentative-return-type */ + public static function __set_state(array $array): DateInterval {} } class DatePeriod implements IteratorAggregate @@ -406,23 +403,23 @@ class DatePeriod implements IteratorAggregate */ public function __construct($start, $interval = UNKNOWN, $end = UNKNOWN, $options = UNKNOWN) {} - /** @return DateTimeInterface */ - public function getStartDate() {} + /** @tentative-return-type */ + public function getStartDate(): DateTimeInterface {} - /** @return DateTimeInterface|null */ - public function getEndDate() {} + /** @tentative-return-type */ + public function getEndDate(): ?DateTimeInterface {} - /** @return DateInterval */ - public function getDateInterval() {} + /** @tentative-return-type */ + public function getDateInterval(): DateInterval {} - /** @return int|null */ - public function getRecurrences() {} + /** @tentative-return-type */ + public function getRecurrences(): ?int {} - /** @return void */ - public function __wakeup() {} + /** @tentative-return-type */ + public function __wakeup(): void {} - /** @return DatePeriod */ - public static function __set_state(array $array) {} + /** @tentative-return-type */ + public static function __set_state(array $array): DatePeriod {} public function getIterator(): Iterator {} } diff --git a/ext/date/php_date_arginfo.h b/ext/date/php_date_arginfo.h index 8f90dd8cbadd6..095fc486b78e2 100644 --- a/ext/date/php_date_arginfo.h +++ b/ext/date/php_date_arginfo.h @@ -1,5 +1,5 @@ /* This is a generated file, edit the .stub.php file instead. - * Stub hash: 108136459e578cc699cffcb84d3335a11f8d5c9d */ + * Stub hash: e8bc76a5db3a225746daffe29c0b8404cf971452 */ ZEND_BEGIN_ARG_WITH_RETURN_TYPE_MASK_EX(arginfo_strtotime, 0, 1, MAY_BE_LONG|MAY_BE_FALSE) ZEND_ARG_TYPE_INFO(0, datetime, IS_STRING, 0) @@ -225,36 +225,39 @@ ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_date_sun_info, 0, 3, IS_ARRAY, 0 ZEND_ARG_TYPE_INFO(0, longitude, IS_DOUBLE, 0) ZEND_END_ARG_INFO() -ZEND_BEGIN_ARG_INFO_EX(arginfo_class_DateTimeInterface_format, 0, 0, 1) +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_TYPE_INFO_EX(arginfo_class_DateTimeInterface_format, 0, 1, IS_STRING, 0) ZEND_ARG_TYPE_INFO(0, format, IS_STRING, 0) ZEND_END_ARG_INFO() -ZEND_BEGIN_ARG_INFO_EX(arginfo_class_DateTimeInterface_getTimezone, 0, 0, 0) +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_TYPE_MASK_EX(arginfo_class_DateTimeInterface_getTimezone, 0, 0, DateTimeZone, MAY_BE_FALSE) ZEND_END_ARG_INFO() -#define arginfo_class_DateTimeInterface_getOffset arginfo_class_DateTimeInterface_getTimezone +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_TYPE_INFO_EX(arginfo_class_DateTimeInterface_getOffset, 0, 0, IS_LONG, 0) +ZEND_END_ARG_INFO() -#define arginfo_class_DateTimeInterface_getTimestamp arginfo_class_DateTimeInterface_getTimezone +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_TYPE_MASK_EX(arginfo_class_DateTimeInterface_getTimestamp, 0, 0, MAY_BE_LONG|MAY_BE_FALSE) +ZEND_END_ARG_INFO() -ZEND_BEGIN_ARG_INFO_EX(arginfo_class_DateTimeInterface_diff, 0, 0, 1) +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(arginfo_class_DateTimeInterface_diff, 0, 1, DateInterval, 0) ZEND_ARG_OBJ_INFO(0, targetObject, DateTimeInterface, 0) ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, absolute, _IS_BOOL, 0, "false") ZEND_END_ARG_INFO() -#define arginfo_class_DateTimeInterface___wakeup arginfo_class_DateTimeInterface_getTimezone +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_TYPE_INFO_EX(arginfo_class_DateTimeInterface___wakeup, 0, 0, IS_VOID, 0) +ZEND_END_ARG_INFO() ZEND_BEGIN_ARG_INFO_EX(arginfo_class_DateTime___construct, 0, 0, 0) ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, datetime, IS_STRING, 0, "\"now\"") ZEND_ARG_OBJ_INFO_WITH_DEFAULT_VALUE(0, timezone, DateTimeZone, 1, "null") ZEND_END_ARG_INFO() -#define arginfo_class_DateTime___wakeup arginfo_class_DateTimeInterface_getTimezone +#define arginfo_class_DateTime___wakeup arginfo_class_DateTimeInterface___wakeup -ZEND_BEGIN_ARG_INFO_EX(arginfo_class_DateTime___set_state, 0, 0, 1) +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(arginfo_class_DateTime___set_state, 0, 1, DateTime, 0) ZEND_ARG_TYPE_INFO(0, array, IS_ARRAY, 0) ZEND_END_ARG_INFO() -ZEND_BEGIN_ARG_INFO_EX(arginfo_class_DateTime_createFromImmutable, 0, 0, 1) +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(arginfo_class_DateTime_createFromImmutable, 0, 1, DateTime, 0) ZEND_ARG_OBJ_INFO(0, object, DateTimeImmutable, 0) ZEND_END_ARG_INFO() @@ -262,21 +265,22 @@ ZEND_BEGIN_ARG_WITH_RETURN_OBJ_INFO_EX(arginfo_class_DateTime_createFromInterfac ZEND_ARG_OBJ_INFO(0, object, DateTimeInterface, 0) ZEND_END_ARG_INFO() -ZEND_BEGIN_ARG_INFO_EX(arginfo_class_DateTime_createFromFormat, 0, 0, 2) +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_TYPE_MASK_EX(arginfo_class_DateTime_createFromFormat, 0, 2, DateTime, MAY_BE_FALSE) ZEND_ARG_TYPE_INFO(0, format, IS_STRING, 0) ZEND_ARG_TYPE_INFO(0, datetime, IS_STRING, 0) ZEND_ARG_OBJ_INFO_WITH_DEFAULT_VALUE(0, timezone, DateTimeZone, 1, "null") ZEND_END_ARG_INFO() -#define arginfo_class_DateTime_getLastErrors arginfo_class_DateTimeInterface_getTimezone +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_TYPE_MASK_EX(arginfo_class_DateTime_getLastErrors, 0, 0, MAY_BE_ARRAY|MAY_BE_FALSE) +ZEND_END_ARG_INFO() #define arginfo_class_DateTime_format arginfo_class_DateTimeInterface_format -ZEND_BEGIN_ARG_INFO_EX(arginfo_class_DateTime_modify, 0, 0, 1) +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_TYPE_MASK_EX(arginfo_class_DateTime_modify, 0, 1, DateTime, MAY_BE_FALSE) ZEND_ARG_TYPE_INFO(0, modifier, IS_STRING, 0) ZEND_END_ARG_INFO() -ZEND_BEGIN_ARG_INFO_EX(arginfo_class_DateTime_add, 0, 0, 1) +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(arginfo_class_DateTime_add, 0, 1, DateTime, 0) ZEND_ARG_OBJ_INFO(0, interval, DateInterval, 0) ZEND_END_ARG_INFO() @@ -284,76 +288,103 @@ ZEND_END_ARG_INFO() #define arginfo_class_DateTime_getTimezone arginfo_class_DateTimeInterface_getTimezone -ZEND_BEGIN_ARG_INFO_EX(arginfo_class_DateTime_setTimezone, 0, 0, 1) +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(arginfo_class_DateTime_setTimezone, 0, 1, DateTime, 0) ZEND_ARG_OBJ_INFO(0, timezone, DateTimeZone, 0) ZEND_END_ARG_INFO() -#define arginfo_class_DateTime_getOffset arginfo_class_DateTimeInterface_getTimezone +#define arginfo_class_DateTime_getOffset arginfo_class_DateTimeInterface_getOffset -ZEND_BEGIN_ARG_INFO_EX(arginfo_class_DateTime_setTime, 0, 0, 2) +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(arginfo_class_DateTime_setTime, 0, 2, DateTime, 0) ZEND_ARG_TYPE_INFO(0, hour, IS_LONG, 0) ZEND_ARG_TYPE_INFO(0, minute, IS_LONG, 0) ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, second, IS_LONG, 0, "0") ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, microsecond, IS_LONG, 0, "0") ZEND_END_ARG_INFO() -ZEND_BEGIN_ARG_INFO_EX(arginfo_class_DateTime_setDate, 0, 0, 3) +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(arginfo_class_DateTime_setDate, 0, 3, DateTime, 0) ZEND_ARG_TYPE_INFO(0, year, IS_LONG, 0) ZEND_ARG_TYPE_INFO(0, month, IS_LONG, 0) ZEND_ARG_TYPE_INFO(0, day, IS_LONG, 0) ZEND_END_ARG_INFO() -ZEND_BEGIN_ARG_INFO_EX(arginfo_class_DateTime_setISODate, 0, 0, 2) +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(arginfo_class_DateTime_setISODate, 0, 2, DateTime, 0) ZEND_ARG_TYPE_INFO(0, year, IS_LONG, 0) ZEND_ARG_TYPE_INFO(0, week, IS_LONG, 0) ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, dayOfWeek, IS_LONG, 0, "1") ZEND_END_ARG_INFO() -ZEND_BEGIN_ARG_INFO_EX(arginfo_class_DateTime_setTimestamp, 0, 0, 1) +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(arginfo_class_DateTime_setTimestamp, 0, 1, DateTime, 0) ZEND_ARG_TYPE_INFO(0, timestamp, IS_LONG, 0) ZEND_END_ARG_INFO() -#define arginfo_class_DateTime_getTimestamp arginfo_class_DateTimeInterface_getTimezone +#define arginfo_class_DateTime_getTimestamp arginfo_class_DateTimeInterface_getOffset #define arginfo_class_DateTime_diff arginfo_class_DateTimeInterface_diff #define arginfo_class_DateTimeImmutable___construct arginfo_class_DateTime___construct -#define arginfo_class_DateTimeImmutable___wakeup arginfo_class_DateTimeInterface_getTimezone +#define arginfo_class_DateTimeImmutable___wakeup arginfo_class_DateTimeInterface___wakeup -#define arginfo_class_DateTimeImmutable___set_state arginfo_class_DateTime___set_state +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(arginfo_class_DateTimeImmutable___set_state, 0, 1, DateTimeImmutable, 0) + ZEND_ARG_TYPE_INFO(0, array, IS_ARRAY, 0) +ZEND_END_ARG_INFO() -#define arginfo_class_DateTimeImmutable_createFromFormat arginfo_class_DateTime_createFromFormat +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_TYPE_MASK_EX(arginfo_class_DateTimeImmutable_createFromFormat, 0, 2, DateTimeImmutable, MAY_BE_FALSE) + ZEND_ARG_TYPE_INFO(0, format, IS_STRING, 0) + ZEND_ARG_TYPE_INFO(0, datetime, IS_STRING, 0) + ZEND_ARG_OBJ_INFO_WITH_DEFAULT_VALUE(0, timezone, DateTimeZone, 1, "null") +ZEND_END_ARG_INFO() -#define arginfo_class_DateTimeImmutable_getLastErrors arginfo_class_DateTimeInterface_getTimezone +#define arginfo_class_DateTimeImmutable_getLastErrors arginfo_class_DateTime_getLastErrors #define arginfo_class_DateTimeImmutable_format arginfo_class_DateTimeInterface_format #define arginfo_class_DateTimeImmutable_getTimezone arginfo_class_DateTimeInterface_getTimezone -#define arginfo_class_DateTimeImmutable_getOffset arginfo_class_DateTimeInterface_getTimezone +#define arginfo_class_DateTimeImmutable_getOffset arginfo_class_DateTimeInterface_getOffset -#define arginfo_class_DateTimeImmutable_getTimestamp arginfo_class_DateTimeInterface_getTimezone +#define arginfo_class_DateTimeImmutable_getTimestamp arginfo_class_DateTimeInterface_getOffset #define arginfo_class_DateTimeImmutable_diff arginfo_class_DateTimeInterface_diff -#define arginfo_class_DateTimeImmutable_modify arginfo_class_DateTime_modify +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_TYPE_MASK_EX(arginfo_class_DateTimeImmutable_modify, 0, 1, DateTimeImmutable, MAY_BE_FALSE) + ZEND_ARG_TYPE_INFO(0, modifier, IS_STRING, 0) +ZEND_END_ARG_INFO() -#define arginfo_class_DateTimeImmutable_add arginfo_class_DateTime_add +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(arginfo_class_DateTimeImmutable_add, 0, 1, DateTimeImmutable, 0) + ZEND_ARG_OBJ_INFO(0, interval, DateInterval, 0) +ZEND_END_ARG_INFO() -#define arginfo_class_DateTimeImmutable_sub arginfo_class_DateTime_add +#define arginfo_class_DateTimeImmutable_sub arginfo_class_DateTimeImmutable_add -#define arginfo_class_DateTimeImmutable_setTimezone arginfo_class_DateTime_setTimezone +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(arginfo_class_DateTimeImmutable_setTimezone, 0, 1, DateTimeImmutable, 0) + ZEND_ARG_OBJ_INFO(0, timezone, DateTimeZone, 0) +ZEND_END_ARG_INFO() -#define arginfo_class_DateTimeImmutable_setTime arginfo_class_DateTime_setTime +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(arginfo_class_DateTimeImmutable_setTime, 0, 2, DateTimeImmutable, 0) + ZEND_ARG_TYPE_INFO(0, hour, IS_LONG, 0) + ZEND_ARG_TYPE_INFO(0, minute, IS_LONG, 0) + ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, second, IS_LONG, 0, "0") + ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, microsecond, IS_LONG, 0, "0") +ZEND_END_ARG_INFO() -#define arginfo_class_DateTimeImmutable_setDate arginfo_class_DateTime_setDate +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(arginfo_class_DateTimeImmutable_setDate, 0, 3, DateTimeImmutable, 0) + ZEND_ARG_TYPE_INFO(0, year, IS_LONG, 0) + ZEND_ARG_TYPE_INFO(0, month, IS_LONG, 0) + ZEND_ARG_TYPE_INFO(0, day, IS_LONG, 0) +ZEND_END_ARG_INFO() -#define arginfo_class_DateTimeImmutable_setISODate arginfo_class_DateTime_setISODate +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(arginfo_class_DateTimeImmutable_setISODate, 0, 2, DateTimeImmutable, 0) + ZEND_ARG_TYPE_INFO(0, year, IS_LONG, 0) + ZEND_ARG_TYPE_INFO(0, week, IS_LONG, 0) + ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, dayOfWeek, IS_LONG, 0, "1") +ZEND_END_ARG_INFO() -#define arginfo_class_DateTimeImmutable_setTimestamp arginfo_class_DateTime_setTimestamp +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(arginfo_class_DateTimeImmutable_setTimestamp, 0, 1, DateTimeImmutable, 0) + ZEND_ARG_TYPE_INFO(0, timestamp, IS_LONG, 0) +ZEND_END_ARG_INFO() -ZEND_BEGIN_ARG_INFO_EX(arginfo_class_DateTimeImmutable_createFromMutable, 0, 0, 1) +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(arginfo_class_DateTimeImmutable_createFromMutable, 0, 1, DateTimeImmutable, 0) ZEND_ARG_OBJ_INFO(0, object, DateTime, 0) ZEND_END_ARG_INFO() @@ -365,43 +396,49 @@ ZEND_BEGIN_ARG_INFO_EX(arginfo_class_DateTimeZone___construct, 0, 0, 1) ZEND_ARG_TYPE_INFO(0, timezone, IS_STRING, 0) ZEND_END_ARG_INFO() -#define arginfo_class_DateTimeZone_getName arginfo_class_DateTimeInterface_getTimezone +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_TYPE_INFO_EX(arginfo_class_DateTimeZone_getName, 0, 0, IS_STRING, 0) +ZEND_END_ARG_INFO() -ZEND_BEGIN_ARG_INFO_EX(arginfo_class_DateTimeZone_getOffset, 0, 0, 1) +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_TYPE_INFO_EX(arginfo_class_DateTimeZone_getOffset, 0, 1, IS_LONG, 0) ZEND_ARG_OBJ_INFO(0, datetime, DateTimeInterface, 0) ZEND_END_ARG_INFO() -ZEND_BEGIN_ARG_INFO_EX(arginfo_class_DateTimeZone_getTransitions, 0, 0, 0) +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_TYPE_MASK_EX(arginfo_class_DateTimeZone_getTransitions, 0, 0, MAY_BE_ARRAY|MAY_BE_FALSE) ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, timestampBegin, IS_LONG, 0, "PHP_INT_MIN") ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, timestampEnd, IS_LONG, 0, "PHP_INT_MAX") ZEND_END_ARG_INFO() -#define arginfo_class_DateTimeZone_getLocation arginfo_class_DateTimeInterface_getTimezone +#define arginfo_class_DateTimeZone_getLocation arginfo_class_DateTime_getLastErrors -#define arginfo_class_DateTimeZone_listAbbreviations arginfo_class_DateTimeInterface_getTimezone +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_TYPE_INFO_EX(arginfo_class_DateTimeZone_listAbbreviations, 0, 0, IS_ARRAY, 0) +ZEND_END_ARG_INFO() -ZEND_BEGIN_ARG_INFO_EX(arginfo_class_DateTimeZone_listIdentifiers, 0, 0, 0) +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_TYPE_INFO_EX(arginfo_class_DateTimeZone_listIdentifiers, 0, 0, IS_ARRAY, 0) ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, timezoneGroup, IS_LONG, 0, "DateTimeZone::ALL") ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, countryCode, IS_STRING, 1, "null") ZEND_END_ARG_INFO() -#define arginfo_class_DateTimeZone___wakeup arginfo_class_DateTimeInterface_getTimezone +#define arginfo_class_DateTimeZone___wakeup arginfo_class_DateTimeInterface___wakeup -#define arginfo_class_DateTimeZone___set_state arginfo_class_DateTime___set_state +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(arginfo_class_DateTimeZone___set_state, 0, 1, DateTimeZone, 0) + ZEND_ARG_TYPE_INFO(0, array, IS_ARRAY, 0) +ZEND_END_ARG_INFO() ZEND_BEGIN_ARG_INFO_EX(arginfo_class_DateInterval___construct, 0, 0, 1) ZEND_ARG_TYPE_INFO(0, duration, IS_STRING, 0) ZEND_END_ARG_INFO() -ZEND_BEGIN_ARG_INFO_EX(arginfo_class_DateInterval_createFromDateString, 0, 0, 1) +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_TYPE_MASK_EX(arginfo_class_DateInterval_createFromDateString, 0, 1, DateInterval, MAY_BE_FALSE) ZEND_ARG_TYPE_INFO(0, datetime, IS_STRING, 0) ZEND_END_ARG_INFO() #define arginfo_class_DateInterval_format arginfo_class_DateTimeInterface_format -#define arginfo_class_DateInterval___wakeup arginfo_class_DateTimeInterface_getTimezone +#define arginfo_class_DateInterval___wakeup arginfo_class_DateTimeInterface___wakeup -#define arginfo_class_DateInterval___set_state arginfo_class_DateTime___set_state +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(arginfo_class_DateInterval___set_state, 0, 1, DateInterval, 0) + ZEND_ARG_TYPE_INFO(0, array, IS_ARRAY, 0) +ZEND_END_ARG_INFO() ZEND_BEGIN_ARG_INFO_EX(arginfo_class_DatePeriod___construct, 0, 0, 1) ZEND_ARG_INFO(0, start) @@ -410,17 +447,23 @@ ZEND_BEGIN_ARG_INFO_EX(arginfo_class_DatePeriod___construct, 0, 0, 1) ZEND_ARG_INFO(0, options) ZEND_END_ARG_INFO() -#define arginfo_class_DatePeriod_getStartDate arginfo_class_DateTimeInterface_getTimezone +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(arginfo_class_DatePeriod_getStartDate, 0, 0, DateTimeInterface, 0) +ZEND_END_ARG_INFO() -#define arginfo_class_DatePeriod_getEndDate arginfo_class_DateTimeInterface_getTimezone +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(arginfo_class_DatePeriod_getEndDate, 0, 0, DateTimeInterface, 1) +ZEND_END_ARG_INFO() -#define arginfo_class_DatePeriod_getDateInterval arginfo_class_DateTimeInterface_getTimezone +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(arginfo_class_DatePeriod_getDateInterval, 0, 0, DateInterval, 0) +ZEND_END_ARG_INFO() -#define arginfo_class_DatePeriod_getRecurrences arginfo_class_DateTimeInterface_getTimezone +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_TYPE_INFO_EX(arginfo_class_DatePeriod_getRecurrences, 0, 0, IS_LONG, 1) +ZEND_END_ARG_INFO() -#define arginfo_class_DatePeriod___wakeup arginfo_class_DateTimeInterface_getTimezone +#define arginfo_class_DatePeriod___wakeup arginfo_class_DateTimeInterface___wakeup -#define arginfo_class_DatePeriod___set_state arginfo_class_DateTime___set_state +ZEND_BEGIN_ARG_WITH_TENTATIVE_RETURN_OBJ_INFO_EX(arginfo_class_DatePeriod___set_state, 0, 1, DatePeriod, 0) + ZEND_ARG_TYPE_INFO(0, array, IS_ARRAY, 0) +ZEND_END_ARG_INFO() ZEND_BEGIN_ARG_WITH_RETURN_OBJ_INFO_EX(arginfo_class_DatePeriod_getIterator, 0, 0, Iterator, 0) ZEND_END_ARG_INFO() diff --git a/ext/date/tests/DateTime_extends_basic3.phpt b/ext/date/tests/DateTime_extends_basic3.phpt index 27fc602e00506..a1e72cf1f2c32 100644 --- a/ext/date/tests/DateTime_extends_basic3.phpt +++ b/ext/date/tests/DateTime_extends_basic3.phpt @@ -9,7 +9,7 @@ echo "*** Testing new DateTime() : with user format() method ***\n"; class DateTimeExt extends DateTime { - public function format($format = "F j, Y, g:i:s a") + public function format($format = "F j, Y, g:i:s a"): string { return parent::format($format); } diff --git a/ext/date/tests/bug55407.phpt b/ext/date/tests/bug55407.phpt index c4638c94b3352..f478a52129826 100644 --- a/ext/date/tests/bug55407.phpt +++ b/ext/date/tests/bug55407.phpt @@ -6,7 +6,7 @@ error_reporting=-1 op_array.fn_flags & ZEND_ACC_HAS_RETURN_TYPE) { - smart_str_append_printf(str, " %s- Return [ ", indent); + if ((fptr->op_array.fn_flags & ZEND_ACC_HAS_RETURN_TYPE)) { + smart_str_append_printf(str, " %s- %s [ ", indent, ZEND_ARG_TYPE_IS_TENTATIVE(&fptr->common.arg_info[-1]) ? "Tentative return" : "Return"); if (ZEND_TYPE_IS_SET(fptr->common.arg_info[-1].type)) { zend_string *type_str = zend_type_to_string(fptr->common.arg_info[-1].type); smart_str_append_printf(str, "%s ", ZSTR_VAL(type_str)); @@ -3449,7 +3449,7 @@ ZEND_METHOD(ReflectionFunctionAbstract, hasReturnType) GET_REFLECTION_OBJECT_PTR(fptr); - RETVAL_BOOL(fptr->op_array.fn_flags & ZEND_ACC_HAS_RETURN_TYPE); + RETVAL_BOOL((fptr->op_array.fn_flags & ZEND_ACC_HAS_RETURN_TYPE) && !ZEND_ARG_TYPE_IS_TENTATIVE(&fptr->common.arg_info[-1])); } /* }}} */ @@ -3465,7 +3465,43 @@ ZEND_METHOD(ReflectionFunctionAbstract, getReturnType) GET_REFLECTION_OBJECT_PTR(fptr); - if (!(fptr->op_array.fn_flags & ZEND_ACC_HAS_RETURN_TYPE)) { + if (!(fptr->op_array.fn_flags & ZEND_ACC_HAS_RETURN_TYPE) || ZEND_ARG_TYPE_IS_TENTATIVE(&fptr->common.arg_info[-1])) { + RETURN_NULL(); + } + + reflection_type_factory(fptr->common.arg_info[-1].type, return_value, 1); +} +/* }}} */ + +/* {{{ Return whether the function has a return type */ +ZEND_METHOD(ReflectionFunctionAbstract, hasTentativeReturnType) +{ + reflection_object *intern; + zend_function *fptr; + + if (zend_parse_parameters_none() == FAILURE) { + RETURN_THROWS(); + } + + GET_REFLECTION_OBJECT_PTR(fptr); + + RETVAL_BOOL(fptr->op_array.fn_flags & ZEND_ACC_HAS_RETURN_TYPE && ZEND_ARG_TYPE_IS_TENTATIVE(&fptr->common.arg_info[-1])); +} +/* }}} */ + +/* {{{ Returns the return type associated with the function */ +ZEND_METHOD(ReflectionFunctionAbstract, getTentativeReturnType) +{ + reflection_object *intern; + zend_function *fptr; + + if (zend_parse_parameters_none() == FAILURE) { + RETURN_THROWS(); + } + + GET_REFLECTION_OBJECT_PTR(fptr); + + if (!(fptr->op_array.fn_flags & ZEND_ACC_HAS_RETURN_TYPE) || !ZEND_ARG_TYPE_IS_TENTATIVE(&fptr->common.arg_info[-1])) { RETURN_NULL(); } diff --git a/ext/reflection/php_reflection.stub.php b/ext/reflection/php_reflection.stub.php index 69384f88bb320..7bd22b6d839d9 100644 --- a/ext/reflection/php_reflection.stub.php +++ b/ext/reflection/php_reflection.stub.php @@ -98,6 +98,10 @@ public function hasReturnType() {} /** @return ReflectionType|null */ public function getReturnType() {} + public function hasTentativeReturnType(): bool {} + + public function getTentativeReturnType(): ?ReflectionType {} + /** @return ReflectionAttribute[] */ public function getAttributes(?string $name = null, int $flags = 0): array {} } diff --git a/ext/reflection/php_reflection_arginfo.h b/ext/reflection/php_reflection_arginfo.h index b1ea070d2ac3a..fa48db957e96c 100644 --- a/ext/reflection/php_reflection_arginfo.h +++ b/ext/reflection/php_reflection_arginfo.h @@ -1,5 +1,5 @@ /* This is a generated file, edit the .stub.php file instead. - * Stub hash: 47ac64b027cdeb0e9996147277f79fa9d6b876bd */ + * Stub hash: 0b2b52d4f891a594ccfcbcc0edeec97a9e0f80e6 */ ZEND_BEGIN_ARG_INFO_EX(arginfo_class_Reflection_getModifierNames, 0, 0, 1) ZEND_ARG_TYPE_INFO(0, modifiers, IS_LONG, 0) @@ -59,6 +59,12 @@ ZEND_END_ARG_INFO() #define arginfo_class_ReflectionFunctionAbstract_getReturnType arginfo_class_ReflectionFunctionAbstract_inNamespace +ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_class_ReflectionFunctionAbstract_hasTentativeReturnType, 0, 0, _IS_BOOL, 0) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_WITH_RETURN_OBJ_INFO_EX(arginfo_class_ReflectionFunctionAbstract_getTentativeReturnType, 0, 0, ReflectionType, 1) +ZEND_END_ARG_INFO() + ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_class_ReflectionFunctionAbstract_getAttributes, 0, 0, IS_ARRAY, 0) ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, name, IS_STRING, 1, "null") ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, flags, IS_LONG, 0, "0") @@ -218,8 +224,7 @@ ZEND_END_ARG_INFO() #define arginfo_class_ReflectionClass_isTrait arginfo_class_ReflectionFunctionAbstract_inNamespace -ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_class_ReflectionClass_isEnum, 0, 0, _IS_BOOL, 0) -ZEND_END_ARG_INFO() +#define arginfo_class_ReflectionClass_isEnum arginfo_class_ReflectionFunctionAbstract_hasTentativeReturnType #define arginfo_class_ReflectionClass_isAbstract arginfo_class_ReflectionFunctionAbstract_inNamespace @@ -311,7 +316,7 @@ ZEND_END_ARG_INFO() #define arginfo_class_ReflectionProperty_isDefault arginfo_class_ReflectionFunctionAbstract_inNamespace -#define arginfo_class_ReflectionProperty_isPromoted arginfo_class_ReflectionClass_isEnum +#define arginfo_class_ReflectionProperty_isPromoted arginfo_class_ReflectionFunctionAbstract_hasTentativeReturnType #define arginfo_class_ReflectionProperty_getModifiers arginfo_class_ReflectionFunctionAbstract_inNamespace @@ -325,7 +330,7 @@ ZEND_END_ARG_INFO() #define arginfo_class_ReflectionProperty_hasType arginfo_class_ReflectionFunctionAbstract_inNamespace -#define arginfo_class_ReflectionProperty_hasDefaultValue arginfo_class_ReflectionClass_isEnum +#define arginfo_class_ReflectionProperty_hasDefaultValue arginfo_class_ReflectionFunctionAbstract_hasTentativeReturnType #define arginfo_class_ReflectionProperty_getDefaultValue arginfo_class_ReflectionFunctionAbstract_inNamespace @@ -358,7 +363,7 @@ ZEND_END_ARG_INFO() #define arginfo_class_ReflectionClassConstant_getAttributes arginfo_class_ReflectionFunctionAbstract_getAttributes -#define arginfo_class_ReflectionClassConstant_isEnumCase arginfo_class_ReflectionClass_isEnum +#define arginfo_class_ReflectionClassConstant_isEnumCase arginfo_class_ReflectionFunctionAbstract_hasTentativeReturnType #define arginfo_class_ReflectionParameter___clone arginfo_class_ReflectionFunctionAbstract___clone @@ -405,7 +410,7 @@ ZEND_END_ARG_INFO() #define arginfo_class_ReflectionParameter_isVariadic arginfo_class_ReflectionFunctionAbstract_inNamespace -#define arginfo_class_ReflectionParameter_isPromoted arginfo_class_ReflectionClass_isEnum +#define arginfo_class_ReflectionParameter_isPromoted arginfo_class_ReflectionFunctionAbstract_hasTentativeReturnType #define arginfo_class_ReflectionParameter_getAttributes arginfo_class_ReflectionFunctionAbstract_getAttributes @@ -482,7 +487,7 @@ ZEND_END_ARG_INFO() ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_class_ReflectionAttribute_getTarget, 0, 0, IS_LONG, 0) ZEND_END_ARG_INFO() -#define arginfo_class_ReflectionAttribute_isRepeated arginfo_class_ReflectionClass_isEnum +#define arginfo_class_ReflectionAttribute_isRepeated arginfo_class_ReflectionFunctionAbstract_hasTentativeReturnType #define arginfo_class_ReflectionAttribute_getArguments arginfo_class_ReflectionUnionType_getTypes @@ -505,10 +510,9 @@ ZEND_END_ARG_INFO() #define arginfo_class_ReflectionEnum_getCases arginfo_class_ReflectionUnionType_getTypes -#define arginfo_class_ReflectionEnum_isBacked arginfo_class_ReflectionClass_isEnum +#define arginfo_class_ReflectionEnum_isBacked arginfo_class_ReflectionFunctionAbstract_hasTentativeReturnType -ZEND_BEGIN_ARG_WITH_RETURN_OBJ_INFO_EX(arginfo_class_ReflectionEnum_getBackingType, 0, 0, ReflectionType, 1) -ZEND_END_ARG_INFO() +#define arginfo_class_ReflectionEnum_getBackingType arginfo_class_ReflectionFunctionAbstract_getTentativeReturnType #define arginfo_class_ReflectionEnumUnitCase___construct arginfo_class_ReflectionClassConstant___construct @@ -569,6 +573,8 @@ ZEND_METHOD(ReflectionFunctionAbstract, getStaticVariables); ZEND_METHOD(ReflectionFunctionAbstract, returnsReference); ZEND_METHOD(ReflectionFunctionAbstract, hasReturnType); ZEND_METHOD(ReflectionFunctionAbstract, getReturnType); +ZEND_METHOD(ReflectionFunctionAbstract, hasTentativeReturnType); +ZEND_METHOD(ReflectionFunctionAbstract, getTentativeReturnType); ZEND_METHOD(ReflectionFunctionAbstract, getAttributes); ZEND_METHOD(ReflectionFunction, __construct); ZEND_METHOD(ReflectionFunction, __toString); @@ -805,6 +811,8 @@ static const zend_function_entry class_ReflectionFunctionAbstract_methods[] = { ZEND_ME(ReflectionFunctionAbstract, returnsReference, arginfo_class_ReflectionFunctionAbstract_returnsReference, ZEND_ACC_PUBLIC) ZEND_ME(ReflectionFunctionAbstract, hasReturnType, arginfo_class_ReflectionFunctionAbstract_hasReturnType, ZEND_ACC_PUBLIC) ZEND_ME(ReflectionFunctionAbstract, getReturnType, arginfo_class_ReflectionFunctionAbstract_getReturnType, ZEND_ACC_PUBLIC) + ZEND_ME(ReflectionFunctionAbstract, hasTentativeReturnType, arginfo_class_ReflectionFunctionAbstract_hasTentativeReturnType, ZEND_ACC_PUBLIC) + ZEND_ME(ReflectionFunctionAbstract, getTentativeReturnType, arginfo_class_ReflectionFunctionAbstract_getTentativeReturnType, ZEND_ACC_PUBLIC) ZEND_ME(ReflectionFunctionAbstract, getAttributes, arginfo_class_ReflectionFunctionAbstract_getAttributes, ZEND_ACC_PUBLIC) ZEND_FE_END }; diff --git a/ext/reflection/tests/ReflectionMethod_tentative_return_type.phpt b/ext/reflection/tests/ReflectionMethod_tentative_return_type.phpt new file mode 100644 index 0000000000000..576b87172292a --- /dev/null +++ b/ext/reflection/tests/ReflectionMethod_tentative_return_type.phpt @@ -0,0 +1,62 @@ +--TEST-- +ReflectionMethod returns tentative return type information correctly +--FILE-- +hasReturnType()); +var_dump($methodInfo->hasTentativeReturnType()); +var_dump($methodInfo->getReturnType()); +var_dump((string) $methodInfo->getTentativeReturnType()); +var_dump((string) $methodInfo); +echo "\n"; + +$methodInfo = new ReflectionMethod(MyDateTimeZone::class, 'listIdentifiers'); + +var_dump($methodInfo->hasReturnType()); +var_dump($methodInfo->hasTentativeReturnType()); +var_dump((string) $methodInfo->getReturnType()); +var_dump($methodInfo->getTentativeReturnType()); +var_dump((string) $methodInfo); +echo "\n"; + +?> +--EXPECTF-- +bool(false) +bool(true) +NULL +string(5) "array" +string(%d) "Method [ static public method listIdentifiers ] { + + - Parameters [2] { + Parameter #0 [ int $timezoneGroup = DateTimeZone::ALL ] + Parameter #1 [ ?string $countryCode = null ] + } + - Tentative return [ array ] +} +" + +bool(true) +bool(false) +string(6) "string" +NULL +string(%d) "Method [ static public method listIdentifiers ] { + @@ %s + + - Parameters [2] { + Parameter #0 [ int $timezoneGroup = %d ] + Parameter #1 [ ?string $countryCode = NULL ] + } + - Return [ string ] +} +" From 87e4970ebcfc579067cbdf16add084d71dbca495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1t=C3=A9=20Kocsis?= Date: Fri, 14 May 2021 17:03:05 +0200 Subject: [PATCH 015/229] Declare SNMP properties Additionally, convert them to typed properties. Closes GH-6742 --- ext/snmp/snmp.c | 65 ++++++++------ ext/snmp/snmp.stub.php | 10 +++ ext/snmp/snmp_arginfo.h | 50 ++++++++++- ext/snmp/tests/snmp-object-error.phpt | 6 +- .../tests/snmp-object-properties-error.phpt | 84 +++++++++++++++++++ ext/snmp/tests/snmp-object-properties.phpt | 2 +- 6 files changed, 186 insertions(+), 31 deletions(-) create mode 100644 ext/snmp/tests/snmp-object-properties-error.phpt diff --git a/ext/snmp/snmp.c b/ext/snmp/snmp.c index 017c88ce51198..699e6fa97fcb8 100644 --- a/ext/snmp/snmp.c +++ b/ext/snmp/snmp.c @@ -1667,29 +1667,36 @@ zval *php_snmp_read_property(zend_object *object, zend_string *name, int type, v } /* }}} */ -/* {{{ php_snmp_write_property(zval *object, zval *member, zval *value[, const zend_literal *key]) - Generic object property writer */ +/* {{{ Generic object property writer */ zval *php_snmp_write_property(zend_object *object, zend_string *name, zval *value, void **cache_slot) { - php_snmp_object *obj; - php_snmp_prop_handler *hnd; + php_snmp_object *obj = php_snmp_fetch_object(object); + php_snmp_prop_handler *hnd = zend_hash_find_ptr(&php_snmp_properties, name); - obj = php_snmp_fetch_object(object); - hnd = zend_hash_find_ptr(&php_snmp_properties, name); + if (hnd) { + if (!hnd->write_func) { + zend_throw_error(NULL, "Cannot write read-only property %s::$%s", ZSTR_VAL(object->ce->name), ZSTR_VAL(name)); + return &EG(error_zval); + } - if (hnd && hnd->write_func) { - hnd->write_func(obj, value); - /* - if (!PZVAL_IS_REF(value) && Z_REFCOUNT_P(value) == 0) { - Z_ADDREF_P(value); - zval_ptr_dtor(&value); + zend_property_info *prop = zend_get_property_info(object->ce, name, /* silent */ true); + if (prop && ZEND_TYPE_IS_SET(prop->type)) { + zval tmp; + ZVAL_COPY(&tmp, value); + if (!zend_verify_property_type(prop, &tmp, + ZEND_CALL_USES_STRICT_TYPES(EG(current_execute_data)))) { + zval_ptr_dtor(&tmp); + return &EG(error_zval); + } + hnd->write_func(obj, &tmp); + zval_ptr_dtor(&tmp); + } else { + hnd->write_func(obj, value); } - */ - } else { - value = zend_std_write_property(object, name, value, cache_slot); + return value; } - return value; + return zend_std_write_property(object, name, value, cache_slot); } /* }}} */ @@ -1762,6 +1769,16 @@ static HashTable *php_snmp_get_properties(zend_object *object) } /* }}} */ +static zval *php_snmp_get_property_ptr_ptr(zend_object *object, zend_string *name, int type, void **cache_slot) +{ + php_snmp_prop_handler *hnd = zend_hash_find_ptr(&php_snmp_properties, name); + if (hnd == NULL) { + return zend_std_get_property_ptr_ptr(object, name, type, cache_slot); + } + + return NULL; +} + /* {{{ */ static int php_snmp_read_info(php_snmp_object *snmp_object, zval *retval) { @@ -1820,14 +1837,6 @@ PHP_SNMP_LONG_PROPERTY_READER_FUNCTION(valueretrieval) PHP_SNMP_LONG_PROPERTY_READER_FUNCTION(oid_output_format) PHP_SNMP_LONG_PROPERTY_READER_FUNCTION(exceptions_enabled) -/* {{{ */ -static int php_snmp_write_info(php_snmp_object *snmp_object, zval *newval) -{ - zend_throw_error(NULL, "SNMP::$info property is read-only"); - return FAILURE; -} -/* }}} */ - /* {{{ */ static int php_snmp_write_max_oids(php_snmp_object *snmp_object, zval *newval) { @@ -1841,7 +1850,7 @@ static int php_snmp_write_max_oids(php_snmp_object *snmp_object, zval *newval) lval = zval_get_long(newval); if (lval <= 0) { - zend_value_error("max_oids must be greater than 0 or null"); + zend_value_error("SNMP::$max_oids must be greater than 0 or null"); return FAILURE; } snmp_object->max_oids = lval; @@ -1924,8 +1933,11 @@ static void free_php_snmp_properties(zval *el) /* {{{ */ #define PHP_SNMP_PROPERTY_ENTRY_RECORD(name) \ { "" #name "", sizeof("" #name "") - 1, php_snmp_read_##name, php_snmp_write_##name } +#define PHP_SNMP_READONLY_PROPERTY_ENTRY_RECORD(name) \ + { "" #name "", sizeof("" #name "") - 1, php_snmp_read_##name, NULL } + const php_snmp_prop_handler php_snmp_property_entries[] = { - PHP_SNMP_PROPERTY_ENTRY_RECORD(info), + PHP_SNMP_READONLY_PROPERTY_ENTRY_RECORD(info), PHP_SNMP_PROPERTY_ENTRY_RECORD(max_oids), PHP_SNMP_PROPERTY_ENTRY_RECORD(valueretrieval), PHP_SNMP_PROPERTY_ENTRY_RECORD(quick_print), @@ -1961,6 +1973,7 @@ PHP_MINIT_FUNCTION(snmp) memcpy(&php_snmp_object_handlers, &std_object_handlers, sizeof(zend_object_handlers)); php_snmp_object_handlers.read_property = php_snmp_read_property; php_snmp_object_handlers.write_property = php_snmp_write_property; + php_snmp_object_handlers.get_property_ptr_ptr = php_snmp_get_property_ptr_ptr; php_snmp_object_handlers.has_property = php_snmp_has_property; php_snmp_object_handlers.get_properties = php_snmp_get_properties; php_snmp_object_handlers.get_gc = php_snmp_get_gc; diff --git a/ext/snmp/snmp.stub.php b/ext/snmp/snmp.stub.php index 1378f34ff9989..db61d1c0d6a1a 100644 --- a/ext/snmp/snmp.stub.php +++ b/ext/snmp/snmp.stub.php @@ -75,6 +75,16 @@ function snmp_read_mib(string $filename): bool {} class SNMP { + /** @readonly */ + public array $info; + public ?int $max_oids; + public int $valueretrieval; + public bool $quick_print; + public bool $enum_print; + public int $oid_output_format; + public bool $oid_increasing_check; + public int $exceptions_enabled; + public function __construct(int $version, string $hostname, string $community, int $timeout = -1, int $retries = -1) {} /** @return bool */ diff --git a/ext/snmp/snmp_arginfo.h b/ext/snmp/snmp_arginfo.h index 0f076ca2a4b72..8088755169fb9 100644 --- a/ext/snmp/snmp_arginfo.h +++ b/ext/snmp/snmp_arginfo.h @@ -1,5 +1,5 @@ /* This is a generated file, edit the .stub.php file instead. - * Stub hash: 08192d87d2ac5d35092cfcf4a2cdcc50f7ec4ada */ + * Stub hash: 5258c5796aca15e369dd72c0a8ed4dc1df31ce9d */ ZEND_BEGIN_ARG_WITH_RETURN_OBJ_TYPE_MASK_EX(arginfo_snmpget, 0, 3, stdClass, MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_BOOL) ZEND_ARG_TYPE_INFO(0, hostname, IS_STRING, 0) @@ -249,6 +249,54 @@ static zend_class_entry *register_class_SNMP(void) INIT_CLASS_ENTRY(ce, "SNMP", class_SNMP_methods); class_entry = zend_register_internal_class_ex(&ce, NULL); + zval property_info_default_value; + ZVAL_UNDEF(&property_info_default_value); + zend_string *property_info_name = zend_string_init("info", sizeof("info") - 1, 1); + zend_declare_typed_property(class_entry, property_info_name, &property_info_default_value, ZEND_ACC_PUBLIC, NULL, (zend_type) ZEND_TYPE_INIT_MASK(MAY_BE_ARRAY)); + zend_string_release(property_info_name); + + zval property_max_oids_default_value; + ZVAL_UNDEF(&property_max_oids_default_value); + zend_string *property_max_oids_name = zend_string_init("max_oids", sizeof("max_oids") - 1, 1); + zend_declare_typed_property(class_entry, property_max_oids_name, &property_max_oids_default_value, ZEND_ACC_PUBLIC, NULL, (zend_type) ZEND_TYPE_INIT_MASK(MAY_BE_LONG|MAY_BE_NULL)); + zend_string_release(property_max_oids_name); + + zval property_valueretrieval_default_value; + ZVAL_UNDEF(&property_valueretrieval_default_value); + zend_string *property_valueretrieval_name = zend_string_init("valueretrieval", sizeof("valueretrieval") - 1, 1); + zend_declare_typed_property(class_entry, property_valueretrieval_name, &property_valueretrieval_default_value, ZEND_ACC_PUBLIC, NULL, (zend_type) ZEND_TYPE_INIT_MASK(MAY_BE_LONG)); + zend_string_release(property_valueretrieval_name); + + zval property_quick_print_default_value; + ZVAL_UNDEF(&property_quick_print_default_value); + zend_string *property_quick_print_name = zend_string_init("quick_print", sizeof("quick_print") - 1, 1); + zend_declare_typed_property(class_entry, property_quick_print_name, &property_quick_print_default_value, ZEND_ACC_PUBLIC, NULL, (zend_type) ZEND_TYPE_INIT_MASK(MAY_BE_BOOL)); + zend_string_release(property_quick_print_name); + + zval property_enum_print_default_value; + ZVAL_UNDEF(&property_enum_print_default_value); + zend_string *property_enum_print_name = zend_string_init("enum_print", sizeof("enum_print") - 1, 1); + zend_declare_typed_property(class_entry, property_enum_print_name, &property_enum_print_default_value, ZEND_ACC_PUBLIC, NULL, (zend_type) ZEND_TYPE_INIT_MASK(MAY_BE_BOOL)); + zend_string_release(property_enum_print_name); + + zval property_oid_output_format_default_value; + ZVAL_UNDEF(&property_oid_output_format_default_value); + zend_string *property_oid_output_format_name = zend_string_init("oid_output_format", sizeof("oid_output_format") - 1, 1); + zend_declare_typed_property(class_entry, property_oid_output_format_name, &property_oid_output_format_default_value, ZEND_ACC_PUBLIC, NULL, (zend_type) ZEND_TYPE_INIT_MASK(MAY_BE_LONG)); + zend_string_release(property_oid_output_format_name); + + zval property_oid_increasing_check_default_value; + ZVAL_UNDEF(&property_oid_increasing_check_default_value); + zend_string *property_oid_increasing_check_name = zend_string_init("oid_increasing_check", sizeof("oid_increasing_check") - 1, 1); + zend_declare_typed_property(class_entry, property_oid_increasing_check_name, &property_oid_increasing_check_default_value, ZEND_ACC_PUBLIC, NULL, (zend_type) ZEND_TYPE_INIT_MASK(MAY_BE_BOOL)); + zend_string_release(property_oid_increasing_check_name); + + zval property_exceptions_enabled_default_value; + ZVAL_UNDEF(&property_exceptions_enabled_default_value); + zend_string *property_exceptions_enabled_name = zend_string_init("exceptions_enabled", sizeof("exceptions_enabled") - 1, 1); + zend_declare_typed_property(class_entry, property_exceptions_enabled_name, &property_exceptions_enabled_default_value, ZEND_ACC_PUBLIC, NULL, (zend_type) ZEND_TYPE_INIT_MASK(MAY_BE_LONG)); + zend_string_release(property_exceptions_enabled_name); + return class_entry; } diff --git a/ext/snmp/tests/snmp-object-error.phpt b/ext/snmp/tests/snmp-object-error.phpt index b170f9a08323b..77e2d6bf4f7e8 100644 --- a/ext/snmp/tests/snmp-object-error.phpt +++ b/ext/snmp/tests/snmp-object-error.phpt @@ -75,7 +75,7 @@ $session = new SNMP(SNMP::VERSION_2c, $hostname, $community, $timeout, $retries) var_dump($session->max_oids); try { $session->max_oids = "ttt"; -} catch (\ValueError $e) { +} catch (TypeError $e) { echo $e->getMessage() . \PHP_EOL; } try { @@ -103,6 +103,6 @@ Closing session bool(true) Invalid or uninitialized SNMP object NULL -max_oids must be greater than 0 or null -max_oids must be greater than 0 or null +Cannot assign string to property SNMP::$max_oids of type ?int +SNMP::$max_oids must be greater than 0 or null NULL diff --git a/ext/snmp/tests/snmp-object-properties-error.phpt b/ext/snmp/tests/snmp-object-properties-error.phpt new file mode 100644 index 0000000000000..4a8e8edab723d --- /dev/null +++ b/ext/snmp/tests/snmp-object-properties-error.phpt @@ -0,0 +1,84 @@ +--TEST-- +Test SNMP object property errors +--SKIPIF-- + +--FILE-- +info = []; +} catch (Error $exception) { + echo $exception->getMessage() . "\n"; +} + +try { + $session->info += []; +} catch (Error $exception) { + echo $exception->getMessage() . "\n"; +} + +try { + $session->max_oids = []; +} catch (TypeError $exception) { + echo $exception->getMessage() . "\n"; +} + +try { + $session->max_oids = -1; +} catch (ValueError $exception) { + echo $exception->getMessage() . "\n"; +} + +try { + $session->valueretrieval = []; +} catch (TypeError $exception) { + echo $exception->getMessage() . "\n"; +} + +try { + $session->quick_print = []; +} catch (TypeError $exception) { + echo $exception->getMessage() . "\n"; +} + +try { + $session->enum_print = []; +} catch (TypeError $exception) { + echo $exception->getMessage() . "\n"; +} + +try { + $session->oid_output_format = []; +} catch (TypeError $exception) { + echo $exception->getMessage() . "\n"; +} + +try { + $session->oid_increasing_check = []; +} catch (TypeError $exception) { + echo $exception->getMessage() . "\n"; +} + +try { + $session->exceptions_enabled = []; +} catch (TypeError $exception) { + echo $exception->getMessage() . "\n"; +} + +?> +--EXPECT-- +Cannot write read-only property SNMP::$info +Cannot write read-only property SNMP::$info +Cannot assign array to property SNMP::$max_oids of type ?int +SNMP::$max_oids must be greater than 0 or null +Cannot assign array to property SNMP::$valueretrieval of type int +Cannot assign array to property SNMP::$quick_print of type bool +Cannot assign array to property SNMP::$enum_print of type bool +Cannot assign array to property SNMP::$oid_output_format of type int +Cannot assign array to property SNMP::$oid_increasing_check of type bool +Cannot assign array to property SNMP::$exceptions_enabled of type int diff --git a/ext/snmp/tests/snmp-object-properties.phpt b/ext/snmp/tests/snmp-object-properties.phpt index 47c6dc663870c..af37eaac565df 100644 --- a/ext/snmp/tests/snmp-object-properties.phpt +++ b/ext/snmp/tests/snmp-object-properties.phpt @@ -192,5 +192,5 @@ NULL bool(false) SNMP retrieval method must be a bitmask of SNMP_VALUE_LIBRARY, SNMP_VALUE_PLAIN, and SNMP_VALUE_OBJECT SNMP output print format must be an SNMP_OID_OUTPUT_* constant -SNMP::$info property is read-only +Cannot write read-only property SNMP::$info NULL From d60bc0e2d91d299c6cb03cb48672479bcffd03a7 Mon Sep 17 00:00:00 2001 From: Tyson Andre Date: Fri, 14 May 2021 15:25:03 -0400 Subject: [PATCH 016/229] Support doc comments on enum cases (#6984) Because php supports doc comments on class constants, I believe it would also make sense to support them on enum cases. I don't have strong opinions about whether attributes should be moved to be the last element or whether the doc comment should go after the attribute, but the ast will likely change again before php 8.1 is stable. So far, all attributes are the last ast child node. I didn't notice that doc comments weren't implemented due to https://github.com/php/php-src/pull/6489 being a large change. https://wiki.php.net/rfc/enumerations did not mention whether or not doc comments were meant to be supported --- Zend/zend_ast.c | 8 +++---- Zend/zend_ast.h | 2 +- Zend/zend_compile.c | 12 ++++++++-- Zend/zend_language_parser.y | 4 ++-- .../ReflectionEnumUnitCase_getDocComment.phpt | 23 +++++++++++++++++++ 5 files changed, 39 insertions(+), 10 deletions(-) create mode 100644 ext/reflection/tests/ReflectionEnumUnitCase_getDocComment.phpt diff --git a/Zend/zend_ast.c b/Zend/zend_ast.c index a09c13fd89ece..0d4a9dff5aaff 100644 --- a/Zend/zend_ast.c +++ b/Zend/zend_ast.c @@ -2201,8 +2201,8 @@ static ZEND_COLD void zend_ast_export_ex(smart_str *str, zend_ast *ast, int prio zend_ast_export_name(str, ast->child[1], 0, indent); APPEND_DEFAULT_VALUE(2); case ZEND_AST_ENUM_CASE: - if (ast->child[2]) { - zend_ast_export_attributes(str, ast->child[2], indent, 1); + if (ast->child[3]) { + zend_ast_export_attributes(str, ast->child[3], indent, 1); } smart_str_appends(str, "case "); zend_ast_export_name(str, ast->child[0], 0, indent); @@ -2329,14 +2329,12 @@ zend_ast * ZEND_FASTCALL zend_ast_with_attributes(zend_ast *ast, zend_ast *attr) ast->child[2] = attr; break; case ZEND_AST_PARAM: + case ZEND_AST_ENUM_CASE: ast->child[3] = attr; break; case ZEND_AST_CLASS_CONST_GROUP: ast->child[1] = attr; break; - case ZEND_AST_ENUM_CASE: - ast->child[2] = attr; - break; EMPTY_SWITCH_DEFAULT_CASE() } diff --git a/Zend/zend_ast.h b/Zend/zend_ast.h index fb6587b48cd31..0e3468ebde110 100644 --- a/Zend/zend_ast.h +++ b/Zend/zend_ast.h @@ -159,7 +159,6 @@ enum _zend_ast_kind { ZEND_AST_PROP_GROUP, ZEND_AST_PROP_ELEM, ZEND_AST_CONST_ELEM, - ZEND_AST_ENUM_CASE, // Pseudo node for initializing enums ZEND_AST_CONST_ENUM_INIT, @@ -167,6 +166,7 @@ enum _zend_ast_kind { /* 4 child nodes */ ZEND_AST_FOR = 4 << ZEND_AST_NUM_CHILDREN_SHIFT, ZEND_AST_FOREACH, + ZEND_AST_ENUM_CASE, /* 5 child nodes */ ZEND_AST_PARAM = 5 << ZEND_AST_NUM_CHILDREN_SHIFT, diff --git a/Zend/zend_compile.c b/Zend/zend_compile.c index 6a16700d2c60e..fbb680be2f716 100644 --- a/Zend/zend_compile.c +++ b/Zend/zend_compile.c @@ -7748,11 +7748,19 @@ static void zend_compile_enum_case(zend_ast *ast) zval value_zv; zend_const_expr_to_zval(&value_zv, &const_enum_init_ast); - zend_class_constant *c = zend_declare_class_constant_ex(enum_class, enum_case_name, &value_zv, ZEND_ACC_PUBLIC, NULL); + + /* Doc comment has been appended as second last element in ZEND_AST_ENUM ast - attributes are conventionally last */ + zend_ast *doc_comment_ast = ast->child[2]; + zend_string *doc_comment = NULL; + if (doc_comment_ast) { + doc_comment = zend_string_copy(zend_ast_get_str(doc_comment_ast)); + } + + zend_class_constant *c = zend_declare_class_constant_ex(enum_class, enum_case_name, &value_zv, ZEND_ACC_PUBLIC, doc_comment); ZEND_CLASS_CONST_FLAGS(c) |= ZEND_CLASS_CONST_IS_CASE; zend_ast_destroy(const_enum_init_ast); - zend_ast *attr_ast = ast->child[2]; + zend_ast *attr_ast = ast->child[3]; if (attr_ast) { zend_compile_attributes(&c->attributes, attr_ast, 0, ZEND_ATTRIBUTE_TARGET_CLASS_CONST); } diff --git a/Zend/zend_language_parser.y b/Zend/zend_language_parser.y index fcbdcb622261b..917a34b8c3c29 100644 --- a/Zend/zend_language_parser.y +++ b/Zend/zend_language_parser.y @@ -607,8 +607,8 @@ enum_backing_type: ; enum_case: - T_CASE identifier enum_case_expr ';' - { $$ = zend_ast_create(ZEND_AST_ENUM_CASE, $2, $3, NULL); } + T_CASE backup_doc_comment identifier enum_case_expr ';' + { $$ = zend_ast_create(ZEND_AST_ENUM_CASE, $3, $4, ($2 ? zend_ast_create_zval_from_str($2) : NULL), NULL); } ; enum_case_expr: diff --git a/ext/reflection/tests/ReflectionEnumUnitCase_getDocComment.phpt b/ext/reflection/tests/ReflectionEnumUnitCase_getDocComment.phpt new file mode 100644 index 0000000000000..d51543601d917 --- /dev/null +++ b/ext/reflection/tests/ReflectionEnumUnitCase_getDocComment.phpt @@ -0,0 +1,23 @@ +--TEST-- +ReflectionEnumUnitCase::getDocComment() +--FILE-- +getDocComment()); +var_dump((new ReflectionEnumUnitCase(Foo::class, 'Baz'))->getDocComment()); +var_dump((new ReflectionClassConstant(Foo::class, 'Bar'))->getDocComment()); +var_dump((new ReflectionClassConstant(Foo::class, 'Baz'))->getDocComment()); + +?> +--EXPECT-- +string(26) "/** Example doc comment */" +bool(false) +string(26) "/** Example doc comment */" +bool(false) From c4d69c2e609a8273715d20f29fe996d6f4b1679e Mon Sep 17 00:00:00 2001 From: George Peter Banyard Date: Sat, 15 May 2021 02:39:11 +0100 Subject: [PATCH 017/229] Fix test expectation now that -a doesn't work without readline Follow-up upon 959e5787bdf7c088a57dce5f4f7570abd7fe35f8 --- Zend/tests/bug40236.phpt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Zend/tests/bug40236.phpt b/Zend/tests/bug40236.phpt index 204b117099f03..35ed84066b8b0 100644 --- a/Zend/tests/bug40236.phpt +++ b/Zend/tests/bug40236.phpt @@ -10,7 +10,5 @@ $php = getenv('TEST_PHP_EXECUTABLE'); $cmd = "\"$php\" -n -d memory_limit=4M -a \"".__DIR__."\"/bug40236.inc"; echo `$cmd`; ?> ---EXPECTF-- -Interactive %s - -ok +--EXPECT-- +Interactive shell (-a) requires the readline extension. From fb86a176ba24b84ae8192d3b81e2083c6fbd723c Mon Sep 17 00:00:00 2001 From: "Christoph M. Becker" Date: Sun, 16 May 2021 14:03:43 +0200 Subject: [PATCH 018/229] Properly push test artifacts Cf. https://www.appveyor.com/docs/packaging-artifacts/#pushing-artifacts-from-scripts. Closes GH-6990. --- appveyor/test_task.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor/test_task.bat b/appveyor/test_task.bat index c1991d88365c2..93612292bb996 100644 --- a/appveyor/test_task.bat +++ b/appveyor/test_task.bat @@ -104,6 +104,6 @@ nmake test TESTS="%OPCACHE_OPTS% -q --offline --show-diff --show-slow 1000 --set set EXIT_CODE=%errorlevel% -powershell -Command "$wc = New-Object 'System.Net.WebClient'; $wc.UploadFile('https://ci.appveyor.com/api/testresults/junit/%APPVEYOR_JOB_ID%', 'c:\junit.out.xml')" +appveyor PushArtifact %TEST_PHP_JUNIT% exit /b %EXIT_CODE% From f0736631d9ffb4b0d1f806d7b5ef67a2a7215339 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1t=C3=A9=20Kocsis?= Date: Sun, 16 May 2021 15:24:26 +0200 Subject: [PATCH 019/229] Fix some types in ext/oci8 Closes GH-6992 --- ext/oci8/oci8.stub.php | 9 ++++++--- ext/oci8/oci8_arginfo.h | 6 +++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ext/oci8/oci8.stub.php b/ext/oci8/oci8.stub.php index ee8ea98b5430b..330c445b7b950 100644 --- a/ext/oci8/oci8.stub.php +++ b/ext/oci8/oci8.stub.php @@ -22,8 +22,11 @@ function oci_bind_by_name($statement, string $param, mixed &$var, int $max_lengt */ function ocibindbyname($statement, string $param, mixed &$var, int $max_length = -1, int $type = 0): bool {} -/** @param resource $statement */ -function oci_bind_array_by_name($statement, string $param, mixed &$var, int $max_array_length, int $max_item_length = -1, int $type = SQLT_AFC): bool {} +/** + * @param resource $statement + * @param array $var + */ +function oci_bind_array_by_name($statement, string $param, &$var, int $max_array_length, int $max_item_length = -1, int $type = SQLT_AFC): bool {} function oci_free_descriptor(OCILob $lob): bool {} @@ -247,7 +250,7 @@ function oci_fetch_all($statement, &$output, int $offset = 0, int $limit = -1, i function ocifetchstatement($statement, &$output, int $offset = 0, int $limit = -1, int $flags = 0): int {} /** @param resource $statement */ -function oci_fetch_object($statement, int $mode = PHP_OCI_ASSOC | PHP_OCI_RETURN_NULLS): stdClass|null|false {} +function oci_fetch_object($statement, int $mode = PHP_OCI_ASSOC | PHP_OCI_RETURN_NULLS): stdClass|false {} /** @param resource $statement */ function oci_fetch_row($statement): array|false {} diff --git a/ext/oci8/oci8_arginfo.h b/ext/oci8/oci8_arginfo.h index 4486036c4905e..2eba00ade71e2 100644 --- a/ext/oci8/oci8_arginfo.h +++ b/ext/oci8/oci8_arginfo.h @@ -1,5 +1,5 @@ /* This is a generated file, edit the .stub.php file instead. - * Stub hash: e7a7a9402b2668136f9f47d6e547e3af46a78a50 */ + * Stub hash: 7355ccbef6b490db7ddec94d485fbfb4597c3150 */ ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_oci_define_by_name, 0, 3, _IS_BOOL, 0) ZEND_ARG_INFO(0, statement) @@ -23,7 +23,7 @@ ZEND_END_ARG_INFO() ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_oci_bind_array_by_name, 0, 4, _IS_BOOL, 0) ZEND_ARG_INFO(0, statement) ZEND_ARG_TYPE_INFO(0, param, IS_STRING, 0) - ZEND_ARG_TYPE_INFO(1, var, IS_MIXED, 0) + ZEND_ARG_INFO(1, var) ZEND_ARG_TYPE_INFO(0, max_array_length, IS_LONG, 0) ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, max_item_length, IS_LONG, 0, "-1") ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, type, IS_LONG, 0, "SQLT_AFC") @@ -221,7 +221,7 @@ ZEND_END_ARG_INFO() #define arginfo_ocifetchstatement arginfo_oci_fetch_all -ZEND_BEGIN_ARG_WITH_RETURN_OBJ_TYPE_MASK_EX(arginfo_oci_fetch_object, 0, 1, stdClass, MAY_BE_NULL|MAY_BE_FALSE) +ZEND_BEGIN_ARG_WITH_RETURN_OBJ_TYPE_MASK_EX(arginfo_oci_fetch_object, 0, 1, stdClass, MAY_BE_FALSE) ZEND_ARG_INFO(0, statement) ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, mode, IS_LONG, 0, "PHP_OCI_ASSOC | PHP_OCI_RETURN_NULLS") ZEND_END_ARG_INFO() From 9939b2b71f46dd807d86578941a19378c6e721a6 Mon Sep 17 00:00:00 2001 From: Matteo Beccati Date: Mon, 17 May 2021 09:24:49 +0200 Subject: [PATCH 020/229] Fix test on non-UTC setups --- ext/intl/tests/dateformat_create_default.phpt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ext/intl/tests/dateformat_create_default.phpt b/ext/intl/tests/dateformat_create_default.phpt index cc130718f5489..eddac36f5ea39 100644 --- a/ext/intl/tests/dateformat_create_default.phpt +++ b/ext/intl/tests/dateformat_create_default.phpt @@ -2,6 +2,8 @@ IntlDateFormatter::create() with default date and time types --EXTENSIONS-- intl +--INI-- +date.timezone=UTC --FILE-- Date: Mon, 17 May 2021 10:16:47 +0200 Subject: [PATCH 021/229] Don't directly throw warning during method compatibility check Only return the INHERITANCE_WARNING status. --- Zend/zend_inheritance.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Zend/zend_inheritance.c b/Zend/zend_inheritance.c index 8f110345be8ca..60e60c1fca93a 100644 --- a/Zend/zend_inheritance.c +++ b/Zend/zend_inheritance.c @@ -668,14 +668,13 @@ static inheritance_status zend_do_perform_implementation_check( if (proto->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE) { /* Removing a return type is not valid, unless the parent return type is tentative. */ if (!(fe->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE)) { - if (ZEND_ARG_TYPE_IS_TENTATIVE(&proto->common.arg_info[-1])) { - if (status == INHERITANCE_SUCCESS) { - emit_incompatible_method_error(fe, fe_scope, proto, proto_scope, INHERITANCE_WARNING); - } - return status; - } else { + if (!ZEND_ARG_TYPE_IS_TENTATIVE(&proto->common.arg_info[-1])) { return INHERITANCE_ERROR; } + if (status == INHERITANCE_SUCCESS) { + return INHERITANCE_WARNING; + } + return status; } local_status = zend_perform_covariant_type_check( From f0858d891fb4bfb95be11b88d6afbe4ae48d1f53 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 17 May 2021 10:18:26 +0200 Subject: [PATCH 022/229] Record warnings during early binding These should also get replayed when the class is loaded from opcache. --- Zend/zend_inheritance.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Zend/zend_inheritance.c b/Zend/zend_inheritance.c index 60e60c1fca93a..05ee34da2b52c 100644 --- a/Zend/zend_inheritance.c +++ b/Zend/zend_inheritance.c @@ -2960,6 +2960,10 @@ zend_class_entry *zend_try_early_bind(zend_class_entry *ce, zend_class_entry *pa orig_linking_class = CG(current_linking_class); CG(current_linking_class) = is_cacheable ? ce : NULL; + if (is_cacheable) { + zend_begin_record_errors(); + } + zend_do_inheritance_ex(ce, parent_ce, status == INHERITANCE_SUCCESS); if (parent_ce && parent_ce->num_interfaces) { zend_do_inherit_interfaces(ce, parent_ce); @@ -2972,6 +2976,7 @@ zend_class_entry *zend_try_early_bind(zend_class_entry *ce, zend_class_entry *pa ce->ce_flags |= ZEND_ACC_LINKED; CG(current_linking_class) = orig_linking_class; + EG(record_errors) = false; if (is_cacheable) { HashTable *ht = (HashTable*)ce->inheritance_cache; From a39edab96eca5d4a69a40bda6bfc549d81515659 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 17 May 2021 10:25:52 +0200 Subject: [PATCH 023/229] Skip float truncation test under libmysql We don't return native types for libmysql, so this test doesn't make sense. --- ext/pdo_mysql/tests/bug79596.phpt | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/pdo_mysql/tests/bug79596.phpt b/ext/pdo_mysql/tests/bug79596.phpt index 185b2e60b14f8..b71bbd5d8c954 100644 --- a/ext/pdo_mysql/tests/bug79596.phpt +++ b/ext/pdo_mysql/tests/bug79596.phpt @@ -6,6 +6,7 @@ require_once(__DIR__ . DIRECTORY_SEPARATOR . 'skipif.inc'); require_once(__DIR__ . DIRECTORY_SEPARATOR . 'mysql_pdo_test.inc'); MySQLPDOTest::skip(); if (!setlocale(LC_ALL, 'de_DE', 'de-DE')) die('skip German locale not available'); +if (!MySQLPDOTest::isPDOMySQLnd()) die('skip libmysql returns result as string'); ?> --FILE-- Date: Mon, 17 May 2021 14:24:05 +0200 Subject: [PATCH 024/229] Fix line endings in test --- ext/pdo_mysql/tests/bug81037.phpt | 68 +++++++++++++++---------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/ext/pdo_mysql/tests/bug81037.phpt b/ext/pdo_mysql/tests/bug81037.phpt index 67c90a4458ba6..4bf33b46853b8 100644 --- a/ext/pdo_mysql/tests/bug81037.phpt +++ b/ext/pdo_mysql/tests/bug81037.phpt @@ -1,35 +1,35 @@ ---TEST-- -Bug #81037 PDO discards error message text from prepared statement ---SKIPIF-- - ---FILE-- -setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); -$pdo->setAttribute(PDO::ATTR_EMULATE_PREPARES, false); -MySQLPDOTest::createTestTable($pdo); - -$sql = "SELECT id FROM test WHERE label = :par"; -$stmt = $pdo->prepare($sql); -try { - $stmt->execute(); -} catch (PDOException $e) { - echo $e->getMessage(), "\n"; -} -$data = $stmt->fetchAll(PDO::FETCH_ASSOC); - -?> ---CLEAN-- - ---EXPECT-- +--TEST-- +Bug #81037 PDO discards error message text from prepared statement +--SKIPIF-- + +--FILE-- +setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); +$pdo->setAttribute(PDO::ATTR_EMULATE_PREPARES, false); +MySQLPDOTest::createTestTable($pdo); + +$sql = "SELECT id FROM test WHERE label = :par"; +$stmt = $pdo->prepare($sql); +try { + $stmt->execute(); +} catch (PDOException $e) { + echo $e->getMessage(), "\n"; +} +$data = $stmt->fetchAll(PDO::FETCH_ASSOC); + +?> +--CLEAN-- + +--EXPECT-- SQLSTATE[HY000]: General error: 2031 No data supplied for parameters in prepared statement \ No newline at end of file From ec566508018c03d56c8d0612619ce91e1bbea258 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 17 May 2021 14:29:47 +0200 Subject: [PATCH 025/229] PDO MySQL: Check number of bounds params even if none are bound The check for the number of bound parameters was only executed if at least one was bound. We should also error if nothing was bound. With mysqlnd, mysqlnd itself ended up emitting an error, but with libmysqlclient this error condition would not be detected. --- ext/pdo_mysql/mysql_statement.c | 16 +++++++++------- ext/pdo_mysql/tests/bug81037.phpt | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/ext/pdo_mysql/mysql_statement.c b/ext/pdo_mysql/mysql_statement.c index 3f08980d54c6a..401031e86b42e 100644 --- a/ext/pdo_mysql/mysql_statement.c +++ b/ext/pdo_mysql/mysql_statement.c @@ -315,6 +315,15 @@ static int pdo_mysql_stmt_execute(pdo_stmt_t *stmt) /* {{{ */ S->done = 0; if (S->stmt) { + uint32_t num_bound_params = + stmt->bound_params ? zend_hash_num_elements(stmt->bound_params) : 0; + if (num_bound_params < (uint32_t) S->num_params) { + /* too few parameter bound */ + PDO_DBG_ERR("too few parameters bound"); + strcpy(stmt->error_code, "HY093"); + PDO_DBG_RETURN(0); + } + PDO_DBG_RETURN(pdo_mysql_stmt_execute_prepared(stmt)); } @@ -403,13 +412,6 @@ static int pdo_mysql_stmt_param_hook(pdo_stmt_t *stmt, struct pdo_bound_param_da PDO_DBG_RETURN(1); case PDO_PARAM_EVT_EXEC_PRE: - if (zend_hash_num_elements(stmt->bound_params) < (unsigned int) S->num_params) { - /* too few parameter bound */ - PDO_DBG_ERR("too few parameters bound"); - strcpy(stmt->error_code, "HY093"); - PDO_DBG_RETURN(0); - } - if (!Z_ISREF(param->parameter)) { parameter = ¶m->parameter; } else { diff --git a/ext/pdo_mysql/tests/bug81037.phpt b/ext/pdo_mysql/tests/bug81037.phpt index 4bf33b46853b8..1452839652033 100644 --- a/ext/pdo_mysql/tests/bug81037.phpt +++ b/ext/pdo_mysql/tests/bug81037.phpt @@ -32,4 +32,4 @@ require __DIR__ . '/mysql_pdo_test.inc'; MySQLPDOTest::dropTestTable(); ?> --EXPECT-- -SQLSTATE[HY000]: General error: 2031 No data supplied for parameters in prepared statement \ No newline at end of file +SQLSTATE[HY093]: Invalid parameter number From 671fd3f25d1cc20bae116872f0f920fc3c2af4a8 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 17 May 2021 14:36:40 +0200 Subject: [PATCH 026/229] Fix test expectation for libmysql-only test This test now correctly displays an error message, yay. --- ext/pdo_mysql/tests/pdo_mysql_stmt_unbuffered_2050.phpt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/pdo_mysql/tests/pdo_mysql_stmt_unbuffered_2050.phpt b/ext/pdo_mysql/tests/pdo_mysql_stmt_unbuffered_2050.phpt index 683c6e0bbee07..a2df9b4597c52 100644 --- a/ext/pdo_mysql/tests/pdo_mysql_stmt_unbuffered_2050.phpt +++ b/ext/pdo_mysql/tests/pdo_mysql_stmt_unbuffered_2050.phpt @@ -138,11 +138,11 @@ array(1) { } Unbuffered... -Warning: PDOStatement::fetchAll(): SQLSTATE[HY000]: General error: 2050 in %s on line %d +Warning: PDOStatement::fetchAll(): SQLSTATE[HY000]: General error: 2050 Row retrieval was canceled by mysql_stmt_close() call in %s on line %d array(0) { } -Warning: PDOStatement::fetchAll(): SQLSTATE[HY000]: General error: 2050 in %s on line %d +Warning: PDOStatement::fetchAll(): SQLSTATE[HY000]: General error: 2050 Row retrieval was canceled by mysql_stmt_close() call in %s on line %d array(0) { } array(1) { From c446d68f7c9de6da8dc614ca4e065d81130d3956 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 17 May 2021 15:43:31 +0200 Subject: [PATCH 027/229] Fixed bug #81046 Literal compaction was incorrectly assuming that literals with the same base literal and the same number of related literals would be equal. Maybe that was the case historically, but at least it isn't true in PHP 8, where FETCH_CONSTANT and INIT_METHOD have distinct literals at the second position. Fix this by making the cache key a concatenation of all literals, rather than just the base literal. We still distinguish the number of related literals based on a bias added to the string hash. --- NEWS | 2 ++ ext/opcache/Optimizer/compact_literals.c | 46 ++++++++++++++++++------ ext/opcache/tests/bug81046.phpt | 19 ++++++++++ 3 files changed, 57 insertions(+), 10 deletions(-) create mode 100644 ext/opcache/tests/bug81046.phpt diff --git a/NEWS b/NEWS index 92d0d8ff05afc..8c5c599a0c766 100644 --- a/NEWS +++ b/NEWS @@ -31,6 +31,8 @@ PHP NEWS (Nikita) . Fixed bug #81015 (Opcache optimization assumes wrong part of ternary operator in if-condition). (Nikita) + . Fixed bug #81046 (Literal compaction merges non-equal related literals). + (Nikita) - PDO_MySQL: . Fixed bug #81037 (PDO discards error message text from prepared diff --git a/ext/opcache/Optimizer/compact_literals.c b/ext/opcache/Optimizer/compact_literals.c index 0e1529d2bd195..9340f2b055bbd 100644 --- a/ext/opcache/Optimizer/compact_literals.c +++ b/ext/opcache/Optimizer/compact_literals.c @@ -113,6 +113,41 @@ static uint32_t add_static_slot(HashTable *hash, return ret; } +static zend_string *create_str_cache_key(zval *literal, uint32_t flags) +{ + ZEND_ASSERT(Z_TYPE_P(literal) == IS_STRING); + uint32_t num_related = LITERAL_NUM_RELATED(flags); + if (num_related == 1) { + return zend_string_copy(Z_STR_P(literal)); + } + if ((flags & LITERAL_KIND_MASK) == LITERAL_VALUE) { + /* Don't merge LITERAL_VALUE that has related literals */ + return NULL; + } + + /* Concatenate all the related literals for the cache key. */ + zend_string *key; + if (num_related == 2) { + ZEND_ASSERT(Z_TYPE_P(literal + 1) == IS_STRING); + key = zend_string_concat2( + Z_STRVAL_P(literal), Z_STRLEN_P(literal), + Z_STRVAL_P(literal + 1), Z_STRLEN_P(literal + 1)); + } else if (num_related == 3) { + ZEND_ASSERT(Z_TYPE_P(literal + 1) == IS_STRING && Z_TYPE_P(literal + 2) == IS_STRING); + key = zend_string_concat3( + Z_STRVAL_P(literal), Z_STRLEN_P(literal), + Z_STRVAL_P(literal + 1), Z_STRLEN_P(literal + 1), + Z_STRVAL_P(literal + 2), Z_STRLEN_P(literal + 2)); + } else { + ZEND_ASSERT(0 && "Currently not needed"); + } + + /* Add a bias to the hash so we can distinguish keys + * that would otherwise be the same after concatenation. */ + ZSTR_H(key) = zend_string_hash_val(key) + num_related - 1; + return key; +} + void zend_optimizer_compact_literals(zend_op_array *op_array, zend_optimizer_ctx *ctx) { zend_op *opline, *end; @@ -407,16 +442,7 @@ void zend_optimizer_compact_literals(zend_op_array *op_array, zend_optimizer_ctx } break; case IS_STRING: { - if (LITERAL_NUM_RELATED(info[i].flags) == 1) { - key = zend_string_copy(Z_STR(op_array->literals[i])); - } else if ((info[i].flags & LITERAL_KIND_MASK) != LITERAL_VALUE) { - key = zend_string_init(Z_STRVAL(op_array->literals[i]), Z_STRLEN(op_array->literals[i]), 0); - ZSTR_H(key) = ZSTR_HASH(Z_STR(op_array->literals[i])) + - LITERAL_NUM_RELATED(info[i].flags) - 1; - } else { - /* Don't merge LITERAL_VALUE that has related literals */ - key = NULL; - } + key = create_str_cache_key(&op_array->literals[i], info[i].flags); if (key && (pos = zend_hash_find(&hash, key)) != NULL && Z_TYPE(op_array->literals[Z_LVAL_P(pos)]) == IS_STRING && LITERAL_NUM_RELATED(info[i].flags) == LITERAL_NUM_RELATED(info[Z_LVAL_P(pos)].flags) && diff --git a/ext/opcache/tests/bug81046.phpt b/ext/opcache/tests/bug81046.phpt new file mode 100644 index 0000000000000..f01890cb781ba --- /dev/null +++ b/ext/opcache/tests/bug81046.phpt @@ -0,0 +1,19 @@ +--TEST-- +Bug #81046: Literal compaction merges non-equal related literals +--FILE-- + +--EXPECT-- +int(1) +Method called From 0bff67c0ab37067b67623766740e84440e933958 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 17 May 2021 16:31:58 +0200 Subject: [PATCH 028/229] Fixed bug #81019 Before the zval -> zend_object migration, this code used macros like FORMATTER_METHOD_FETCH_OBJECT_NO_CHECK, which internally clear the error. Now that they are no longer used, we need to manually clear the error. --- NEWS | 4 ++++ ext/intl/dateformat/dateformat_class.c | 1 + ext/intl/formatter/formatter_class.c | 2 ++ ext/intl/msgformat/msgformat_class.c | 2 ++ ext/intl/tests/bug81019.phpt | 21 +++++++++++++++++++++ 5 files changed, 30 insertions(+) create mode 100644 ext/intl/tests/bug81019.phpt diff --git a/NEWS b/NEWS index 8c5c599a0c766..ebf1e2f6175ed 100644 --- a/NEWS +++ b/NEWS @@ -20,6 +20,10 @@ PHP NEWS . Fixed bug #81032 (GD install is affected by external libgd installation). (Flavio Heleno, cmb) +- Intl: + . Fixed bug #81019 (Unable to clone NumberFormatter after failed parse()). + (Nikita) + - MBString: . Fixed bug #81011 (mb_convert_encoding removes references from arrays). (cmb) diff --git a/ext/intl/dateformat/dateformat_class.c b/ext/intl/dateformat/dateformat_class.c index 7d9dde20b056b..76755030626e9 100644 --- a/ext/intl/dateformat/dateformat_class.c +++ b/ext/intl/dateformat/dateformat_class.c @@ -77,6 +77,7 @@ zend_object *IntlDateFormatter_object_clone(zend_object *object) zend_object *new_obj; dfo = php_intl_dateformatter_fetch_object(object); + intl_error_reset(INTL_DATA_ERROR_P(dfo)); new_obj = IntlDateFormatter_ce_ptr->create_object(object->ce); new_dfo = php_intl_dateformatter_fetch_object(new_obj); diff --git a/ext/intl/formatter/formatter_class.c b/ext/intl/formatter/formatter_class.c index f7965ec7d11e5..449e8c271ad5c 100644 --- a/ext/intl/formatter/formatter_class.c +++ b/ext/intl/formatter/formatter_class.c @@ -64,6 +64,8 @@ zend_object *NumberFormatter_object_clone(zend_object *object) zend_object *new_obj; nfo = php_intl_number_format_fetch_object(object); + intl_error_reset(INTL_DATA_ERROR_P(nfo)); + new_obj = NumberFormatter_ce_ptr->create_object(object->ce); new_nfo = php_intl_number_format_fetch_object(new_obj); /* clone standard parts */ diff --git a/ext/intl/msgformat/msgformat_class.c b/ext/intl/msgformat/msgformat_class.c index d8bf9a006c050..6c59c967a1906 100644 --- a/ext/intl/msgformat/msgformat_class.c +++ b/ext/intl/msgformat/msgformat_class.c @@ -62,6 +62,8 @@ zend_object *MessageFormatter_object_clone(zend_object *object) zend_object *new_obj; mfo = php_intl_messageformatter_fetch_object(object); + intl_error_reset(INTL_DATA_ERROR_P(mfo)); + new_obj = MessageFormatter_ce_ptr->create_object(object->ce); new_mfo = php_intl_messageformatter_fetch_object(new_obj); /* clone standard parts */ diff --git a/ext/intl/tests/bug81019.phpt b/ext/intl/tests/bug81019.phpt new file mode 100644 index 0000000000000..6703dae1bddc1 --- /dev/null +++ b/ext/intl/tests/bug81019.phpt @@ -0,0 +1,21 @@ +--TEST-- +Bug #81019: Unable to clone NumberFormatter after failed parse() +--FILE-- +parse('abc'); +$fmt2 = clone $fmt; + +$datefmt = new IntlDateFormatter('en_US', IntlDateFormatter::FULL, IntlDateFormatter::FULL); +$datefmt->parse('abc'); +$datefmt2 = clone $datefmt; + +$msgfmt = new MessageFormatter('en_US', '{0,number,integer}'); +$msgfmt->parse('abc'); +$msgfmt2 = clone $msgfmt; + +?> +===DONE=== +--EXPECT-- +===DONE=== From 941b55fe6397cd7aa9e843c9209a782172518386 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 17 May 2021 18:03:22 +0300 Subject: [PATCH 029/229] Fixed JIT failure after overflow detection (bench.php failure in 32-bit build with -d opcache.jit=1254 -d opcache.jit_hot_loop=1 -d opcache.jit_hot_func=1 -d opcache.jit_hot_return=1 -d opcache.jit_hot_side_exit=1) --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 01910e4c0b306..81c9dd20e9bde 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4233,7 +4233,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, } else { result_reg = Z_REG(res_addr); } - } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { + } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr) && !may_overflow) { result_reg = Z_REG(op1_addr); } else if (Z_REG(res_addr) != ZREG_R0) { result_reg = ZREG_R0; From 1c8bb6d6818c10a691d6836b336bcee003478b44 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Mon, 17 May 2021 18:44:25 +0200 Subject: [PATCH 030/229] Remove unnecessary LIBZEND_MM_ALIGN assignment We're not using the variable in this branch. --- Zend/Zend.m4 | 1 - 1 file changed, 1 deletion(-) diff --git a/Zend/Zend.m4 b/Zend/Zend.m4 index ca6768d2d6c54..d4dcd5459078b 100644 --- a/Zend/Zend.m4 +++ b/Zend/Zend.m4 @@ -278,7 +278,6 @@ int main() AC_DEFINE_UNQUOTED(ZEND_MM_ALIGNMENT_LOG2, $LIBZEND_MM_ALIGN_LOG2, [ ]) ], [], [ dnl Cross compilation needs something here. - LIBZEND_MM_ALIGN=8 AC_DEFINE_UNQUOTED(ZEND_MM_ALIGNMENT, 8, [ ]) AC_DEFINE_UNQUOTED(ZEND_MM_ALIGNMENT_LOG2, 3, [ ]) ]) From de6e401e05e65055403a39fa603fefe20b602aa9 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Tue, 18 May 2021 11:43:37 +0200 Subject: [PATCH 031/229] Use common formatting for backtraces (#6977) This makes debug_print_backtrace() use the same formatting as exception backtraces. The only difference is that the final #{main} is omitted, because it wouldn't make sense for limited backtraces, and wasn't there previously either. --- Zend/tests/attributes/031_backtrace.phpt | 4 +- Zend/tests/bug29896.phpt | 6 +- Zend/tests/bug30828.phpt | 12 ++-- Zend/tests/bug64239_3.phpt | 8 +-- Zend/tests/bug64239_4.phpt | 8 +-- Zend/tests/bug70156.phpt | 6 +- Zend/tests/bug73916.phpt | 2 +- Zend/tests/bug78973.phpt | 2 +- Zend/tests/bug_debug_backtrace.phpt | 8 +-- Zend/tests/closure_032.phpt | 6 +- Zend/tests/debug_backtrace_options.phpt | 36 +++++----- .../debug_print_backtrace_from_main.phpt | 1 + Zend/tests/debug_print_backtrace_limit.phpt | 24 ++++--- Zend/tests/fibers/debug-backtrace.phpt | 6 +- Zend/tests/generators/backtrace.phpt | 8 +-- .../backtrace_multi_yield_from.phpt | 6 +- .../generators/yield_from_backtrace.phpt | 22 +++--- Zend/tests/named_params/backtrace.phpt | 2 +- Zend/zend_builtin_functions.c | 68 ++----------------- Zend/zend_exceptions.c | 53 ++++++++------- Zend/zend_exceptions.h | 1 + tests/basic/bug73969.phpt | 4 +- tests/lang/bug28213.phpt | 7 +- 23 files changed, 129 insertions(+), 171 deletions(-) diff --git a/Zend/tests/attributes/031_backtrace.phpt b/Zend/tests/attributes/031_backtrace.phpt index b0374e67e2fc8..504e356e7cc8d 100644 --- a/Zend/tests/attributes/031_backtrace.phpt +++ b/Zend/tests/attributes/031_backtrace.phpt @@ -19,8 +19,8 @@ class Test {} ?> --EXPECTF-- -#0 MyAttribute->__construct() called at [%s031_backtrace.php:12] -#1 ReflectionAttribute->newInstance() called at [%s:%d] +#0 %s031_backtrace.php(12): MyAttribute->__construct() +#1 %s(%d): ReflectionAttribute->newInstance() array(2) { [0]=> array(7) { diff --git a/Zend/tests/bug29896.phpt b/Zend/tests/bug29896.phpt index b32a25e8d94c4..f0e25b716a4a4 100644 --- a/Zend/tests/bug29896.phpt +++ b/Zend/tests/bug29896.phpt @@ -22,6 +22,6 @@ function GenerateError2($A1) GenerateError2("Test2"); ?> --EXPECTF-- -#0 userErrorHandler(2, Undefined variable $b, %s, %d) called at [%s:%d] -#1 GenerateError1(Test1) called at [%sbug29896.php:16] -#2 GenerateError2(Test2) called at [%sbug29896.php:19] +#0 %s(%d): userErrorHandler(2, 'Undefined varia...', '%s', %d) +#1 %s(%d): GenerateError1('Test1') +#2 %s(%d): GenerateError2('Test2') diff --git a/Zend/tests/bug30828.phpt b/Zend/tests/bug30828.phpt index 4e921403b7f84..9b7e6eac3ca97 100644 --- a/Zend/tests/bug30828.phpt +++ b/Zend/tests/bug30828.phpt @@ -47,15 +47,15 @@ $b->foo(); B::bar(); ?> --EXPECTF-- -#0 A->__construct() called at [%sbug30828.php:30] -#1 B->__construct() called at [%sbug30828.php:42] +#0 %sbug30828.php(30): A->__construct() +#1 %sbug30828.php(42): B->__construct() A->__construct B->__construct -#0 A->foo() called at [%sbug30828.php:34] -#1 B->foo() called at [%sbug30828.php:43] +#0 %sbug30828.php(34): A->foo() +#1 %sbug30828.php(43): B->foo() A->foo B->foo -#0 A::bar() called at [%sbug30828.php:38] -#1 B::bar() called at [%sbug30828.php:44] +#0 %sbug30828.php(38): A::bar() +#1 %sbug30828.php(44): B::bar() A::bar B::bar diff --git a/Zend/tests/bug64239_3.phpt b/Zend/tests/bug64239_3.phpt index 1a7da60e3c656..1de8cede02f70 100644 --- a/Zend/tests/bug64239_3.phpt +++ b/Zend/tests/bug64239_3.phpt @@ -27,7 +27,7 @@ $c->Bmethod(); $c->t2method(); ?> --EXPECTF-- -#0 A->Bmethod() called at [%sbug64239_3.php:%d] -#0 A->t2method() called at [%sbug64239_3.php:%d] -#0 C->Bmethod() called at [%sbug64239_3.php:%d] -#0 A->t2method() called at [%sbug64239_3.php:%d] +#0 %s(%d): A->Bmethod() +#0 %s(%d): A->t2method() +#0 %s(%d): C->Bmethod() +#0 %s(%d): A->t2method() diff --git a/Zend/tests/bug64239_4.phpt b/Zend/tests/bug64239_4.phpt index 8204a5b8432bc..086a33bb48d9b 100644 --- a/Zend/tests/bug64239_4.phpt +++ b/Zend/tests/bug64239_4.phpt @@ -25,7 +25,7 @@ C::Bmethod(); C::t2method(); ?> --EXPECTF-- -#0 A::Bmethod() called at [%sbug64239_4.php:%d] -#0 A::t2method() called at [%sbug64239_4.php:%d] -#0 C::Bmethod() called at [%sbug64239_4.php:%d] -#0 A::t2method() called at [%sbug64239_4.php:%d] +#0 %s(%d): A::Bmethod() +#0 %s(%d): A::t2method() +#0 %s(%d): C::Bmethod() +#0 %s(%d): A::t2method() diff --git a/Zend/tests/bug70156.phpt b/Zend/tests/bug70156.phpt index 0d20b38181255..be98277548275 100644 --- a/Zend/tests/bug70156.phpt +++ b/Zend/tests/bug70156.phpt @@ -32,6 +32,6 @@ class dummy { new dummy(); ?> --EXPECTF-- -#0 dummy->bar() called at [%sbug70156.php:%d] -#1 dummy->foo1() called at [%sbug70156.php:%d] -#2 dummy->__construct() called at [%sbug70156.php:%d] +#0 %s(%d): dummy->bar() +#1 %s(%d): dummy->foo1() +#2 %s(%d): dummy->__construct() diff --git a/Zend/tests/bug73916.phpt b/Zend/tests/bug73916.phpt index 7b51f8f31a38a..aa765a96aac2e 100644 --- a/Zend/tests/bug73916.phpt +++ b/Zend/tests/bug73916.phpt @@ -13,4 +13,4 @@ function test() { } ?> --EXPECTF-- -#0 test(Array ([0] => Array ([0] => a),[1] => b Object ())) called at [%sbug73916.php:%d] +#0 %s(%d): test(Array) diff --git a/Zend/tests/bug78973.phpt b/Zend/tests/bug78973.phpt index 688f4e6cc54e0..f786636bfddfe 100644 --- a/Zend/tests/bug78973.phpt +++ b/Zend/tests/bug78973.phpt @@ -15,4 +15,4 @@ test(new class { ?> --EXPECTF-- -#0 class@anonymous->__destruct() called at [%s:%d] +#0 %s(%d): class@anonymous->__destruct() diff --git a/Zend/tests/bug_debug_backtrace.phpt b/Zend/tests/bug_debug_backtrace.phpt index 95530e01e46ba..53f108ced76d5 100644 --- a/Zend/tests/bug_debug_backtrace.phpt +++ b/Zend/tests/bug_debug_backtrace.phpt @@ -19,8 +19,8 @@ eval("foo();"); echo "Done\n"; ?> --EXPECTF-- -#0 boo() called at [%s:%d] -#1 bar() called at [%s:%d] -#2 foo() called at [%s(%d) : eval()'d code:1] -#3 eval() called at [%s:%d] +#0 %s(%d): boo() +#1 %s(%d): bar() +#2 %s(%d) : eval()'d code(1): foo() +#3 %s(%d): eval() Done diff --git a/Zend/tests/closure_032.phpt b/Zend/tests/closure_032.phpt index 601db15989b31..bf055c22856e2 100644 --- a/Zend/tests/closure_032.phpt +++ b/Zend/tests/closure_032.phpt @@ -29,7 +29,7 @@ Array ) ) -#0 {closure}(23) called at [%s:%d] +#0 %s(%d): {closure}(23) Array ( [0] => Array @@ -65,5 +65,5 @@ Array ) ) -#0 {closure}(23) called at [%s:%d] -#1 test(Closure Object ()) called at [%s:%d] +#0 %s(%d): {closure}(23) +#1 %s(%d): test(Object(Closure)) diff --git a/Zend/tests/debug_backtrace_options.phpt b/Zend/tests/debug_backtrace_options.phpt index 94842279ea979..460bbe3936ff7 100644 --- a/Zend/tests/debug_backtrace_options.phpt +++ b/Zend/tests/debug_backtrace_options.phpt @@ -45,29 +45,29 @@ foo::statCall("doit", "backtrace_print"); ?> --EXPECTF-- ==default -#0 doit(a, b, debug_print_backtrace) called at [%sdebug_backtrace_options.php:%d] -#1 foo->doCall(doit, debug_print_backtrace) called at [%sdebug_backtrace_options.php:%d] -#2 foo::statCall(doit, debug_print_backtrace) called at [%sdebug_backtrace_options.php:%d] +#0 %sdebug_backtrace_options.php(%d): doit('a', 'b', 'debug_print_bac...') +#1 %sdebug_backtrace_options.php(%d): foo->doCall('doit', 'debug_print_bac...') +#2 %sdebug_backtrace_options.php(%d): foo::statCall('doit', 'debug_print_bac...') ==true -#0 doit(a, b, debug_print_backtrace) called at [%sdebug_backtrace_options.php:%d] -#1 foo->doCall(doit, debug_print_backtrace) called at [%sdebug_backtrace_options.php:%d] -#2 foo::statCall(doit, debug_print_backtrace) called at [%sdebug_backtrace_options.php:%d] +#0 %sdebug_backtrace_options.php(%d): doit('a', 'b', 'debug_print_bac...') +#1 %sdebug_backtrace_options.php(%d): foo->doCall('doit', 'debug_print_bac...') +#2 %sdebug_backtrace_options.php(%d): foo::statCall('doit', 'debug_print_bac...') ==false -#0 doit(a, b, debug_print_backtrace) called at [%sdebug_backtrace_options.php:%d] -#1 foo->doCall(doit, debug_print_backtrace) called at [%sdebug_backtrace_options.php:%d] -#2 foo::statCall(doit, debug_print_backtrace) called at [%sdebug_backtrace_options.php:%d] +#0 %sdebug_backtrace_options.php(%d): doit('a', 'b', 'debug_print_bac...') +#1 %sdebug_backtrace_options.php(%d): foo->doCall('doit', 'debug_print_bac...') +#2 %sdebug_backtrace_options.php(%d): foo::statCall('doit', 'debug_print_bac...') ==DEBUG_BACKTRACE_PROVIDE_OBJECT -#0 doit(a, b, debug_print_backtrace) called at [%sdebug_backtrace_options.php:%d] -#1 foo->doCall(doit, debug_print_backtrace) called at [%sdebug_backtrace_options.php:%d] -#2 foo::statCall(doit, debug_print_backtrace) called at [%sdebug_backtrace_options.php:%d] +#0 %sdebug_backtrace_options.php(%d): doit('a', 'b', 'debug_print_bac...') +#1 %sdebug_backtrace_options.php(%d): foo->doCall('doit', 'debug_print_bac...') +#2 %sdebug_backtrace_options.php(%d): foo::statCall('doit', 'debug_print_bac...') ==DEBUG_BACKTRACE_IGNORE_ARGS -#0 doit() called at [%sdebug_backtrace_options.php:%d] -#1 foo->doCall() called at [%sdebug_backtrace_options.php:%d] -#2 foo::statCall() called at [%sdebug_backtrace_options.php:%d] +#0 %sdebug_backtrace_options.php(%d): doit() +#1 %sdebug_backtrace_options.php(%d): foo->doCall() +#2 %sdebug_backtrace_options.php(%d): foo::statCall() ==both -#0 doit() called at [%sdebug_backtrace_options.php:%d] -#1 foo->doCall() called at [%sdebug_backtrace_options.php:%d] -#2 foo::statCall() called at [%sdebug_backtrace_options.php:%d] +#0 %sdebug_backtrace_options.php(%d): doit() +#1 %sdebug_backtrace_options.php(%d): foo->doCall() +#2 %sdebug_backtrace_options.php(%d): foo::statCall() ==default Array ( diff --git a/Zend/tests/debug_print_backtrace_from_main.phpt b/Zend/tests/debug_print_backtrace_from_main.phpt index 82f75742c774f..f6ea383c397e4 100644 --- a/Zend/tests/debug_print_backtrace_from_main.phpt +++ b/Zend/tests/debug_print_backtrace_from_main.phpt @@ -5,3 +5,4 @@ Calling debug_print_backtrace() from main script debug_print_backtrace(); ?> --EXPECT-- + diff --git a/Zend/tests/debug_print_backtrace_limit.phpt b/Zend/tests/debug_print_backtrace_limit.phpt index 5100387bd6eed..e0d791cc7ba57 100644 --- a/Zend/tests/debug_print_backtrace_limit.phpt +++ b/Zend/tests/debug_print_backtrace_limit.phpt @@ -12,20 +12,26 @@ function b() { function c() { debug_print_backtrace(0, 1); + echo "\n"; debug_print_backtrace(0, 2); + echo "\n"; debug_print_backtrace(0, 0); + echo "\n"; debug_print_backtrace(0, 4); } a(); ?> --EXPECTF-- -#0 c() called at [%sdebug_print_backtrace_limit.php:7] -#0 c() called at [%sdebug_print_backtrace_limit.php:7] -#1 b() called at [%sdebug_print_backtrace_limit.php:3] -#0 c() called at [%sdebug_print_backtrace_limit.php:7] -#1 b() called at [%sdebug_print_backtrace_limit.php:3] -#2 a() called at [%sdebug_print_backtrace_limit.php:17] -#0 c() called at [%sdebug_print_backtrace_limit.php:7] -#1 b() called at [%sdebug_print_backtrace_limit.php:3] -#2 a() called at [%sdebug_print_backtrace_limit.php:17] +#0 %sdebug_print_backtrace_limit.php(7): c() + +#0 %sdebug_print_backtrace_limit.php(7): c() +#1 %sdebug_print_backtrace_limit.php(3): b() + +#0 %sdebug_print_backtrace_limit.php(7): c() +#1 %sdebug_print_backtrace_limit.php(3): b() +#2 %sdebug_print_backtrace_limit.php(20): a() + +#0 %sdebug_print_backtrace_limit.php(7): c() +#1 %sdebug_print_backtrace_limit.php(3): b() +#2 %sdebug_print_backtrace_limit.php(20): a() diff --git a/Zend/tests/fibers/debug-backtrace.phpt b/Zend/tests/fibers/debug-backtrace.phpt index bb37d4f5d0ec0..ef5ae8c9f0902 100644 --- a/Zend/tests/fibers/debug-backtrace.phpt +++ b/Zend/tests/fibers/debug-backtrace.phpt @@ -16,6 +16,6 @@ $fiber->start(); ?> --EXPECTF-- -#0 inner_function() called at [%sdebug-backtrace.php:9] -#1 {closure}() -#2 Fiber->start() called at [%sdebug-backtrace.php:12] +#0 %sdebug-backtrace.php(9): inner_function() +#1 [internal function]: {closure}() +#2 %sdebug-backtrace.php(12): Fiber->start() diff --git a/Zend/tests/generators/backtrace.phpt b/Zend/tests/generators/backtrace.phpt index 5fed1d467e676..816d06ba12d9b 100644 --- a/Zend/tests/generators/backtrace.phpt +++ b/Zend/tests/generators/backtrace.phpt @@ -21,7 +21,7 @@ f3($gen); ?> --EXPECTF-- -#0 f1() called at [%s:%d] -#1 f2(foo, bar) -#2 Generator->rewind() called at [%s:%d] -#3 f3(Generator Object ()) called at [%s:%d] +#0 %s(%d): f1() +#1 [internal function]: f2('foo', 'bar') +#2 %s(%d): Generator->rewind() +#3 %s(%d): f3(Object(Generator)) diff --git a/Zend/tests/generators/backtrace_multi_yield_from.phpt b/Zend/tests/generators/backtrace_multi_yield_from.phpt index 6627fe8458a42..4fbaa2f05bb49 100644 --- a/Zend/tests/generators/backtrace_multi_yield_from.phpt +++ b/Zend/tests/generators/backtrace_multi_yield_from.phpt @@ -26,7 +26,7 @@ var_dump($gen2->current()); --EXPECTF-- int(1) int(1) -#0 gen() called at [%s:10] -#1 from(Generator Object ()) -#2 Generator->next() called at [%s:19] +#0 %s(10): gen() +#1 [internal function]: from(Object(Generator)) +#2 %s(19): Generator->next() int(2) diff --git a/Zend/tests/generators/yield_from_backtrace.phpt b/Zend/tests/generators/yield_from_backtrace.phpt index 8fb1aeef8cb19..c92f80f4ca736 100644 --- a/Zend/tests/generators/yield_from_backtrace.phpt +++ b/Zend/tests/generators/yield_from_backtrace.phpt @@ -28,21 +28,21 @@ for ($gen = gen(); $gen->valid(); $gen->next()) { --EXPECTF-- Implicit foreach: int(1) -#0 gen() called at [%s:%d] +#0 %s(%d): gen() int(2) -#0 from(2) called at [%s:%d] -#1 gen() called at [%s:%d] +#0 %s(%d): from(2) +#1 %s(%d): gen() int(3) -#0 gen() called at [%s:%d] +#0 %s(%d): gen() Explicit iterator: int(1) -#0 gen() -#1 Generator->next() called at [%s:%d] +#0 [internal function]: gen() +#1 %s(%d): Generator->next() int(2) -#0 from(2) called at [%s:%d] -#1 gen() -#2 Generator->next() called at [%s:%d] +#0 %s(%d): from(2) +#1 [internal function]: gen() +#2 %s(%d): Generator->next() int(3) -#0 gen() -#1 Generator->next() called at [%s:%d] +#0 [internal function]: gen() +#1 %s(%d): Generator->next() diff --git a/Zend/tests/named_params/backtrace.phpt b/Zend/tests/named_params/backtrace.phpt index 1ccae8503de6f..89d3c00ee8c3f 100644 --- a/Zend/tests/named_params/backtrace.phpt +++ b/Zend/tests/named_params/backtrace.phpt @@ -40,7 +40,7 @@ array(1) { } } } -#0 test(1, 2, x: 3, y: 4) called at [%s:10] +#0 %s(10): test(1, 2, x: 3, y: 4) array(1) { [0]=> array(4) { diff --git a/Zend/zend_builtin_functions.c b/Zend/zend_builtin_functions.c index 53f64b3470bc9..f426c5fcb1ad2 100644 --- a/Zend/zend_builtin_functions.c +++ b/Zend/zend_builtin_functions.c @@ -1637,33 +1637,12 @@ static void debug_backtrace_get_args(zend_execute_data *call, zval *arg_array) / } /* }}} */ -void debug_print_backtrace_args(smart_str *str, zval *arg_array) /* {{{ */ -{ - zend_string *name; - zval *tmp; - int i = 0; - - ZEND_HASH_FOREACH_STR_KEY_VAL(Z_ARRVAL_P(arg_array), name, tmp) { - if (i++) { - smart_str_appends(str, ", "); - } - if (name) { - smart_str_append(str, name); - smart_str_appends(str, ": "); - } - zend_print_flat_zval_r_to_buf(str, tmp); - } ZEND_HASH_FOREACH_END(); -} -/* }}} */ - /* {{{ */ ZEND_FUNCTION(debug_print_backtrace) { zend_long options = 0; zend_long limit = 0; - zval backtrace, *frame; - zend_long frame_no; - smart_str str = {0}; + zval backtrace; if (zend_parse_parameters(ZEND_NUM_ARGS(), "|ll", &options, &limit) == FAILURE) { RETURN_THROWS(); @@ -1671,48 +1650,11 @@ ZEND_FUNCTION(debug_print_backtrace) zend_fetch_debug_backtrace(&backtrace, 1, options, limit); ZEND_ASSERT(Z_TYPE(backtrace) == IS_ARRAY); - ZEND_HASH_FOREACH_NUM_KEY_VAL(Z_ARR(backtrace), frame_no, frame) { - ZEND_ASSERT(Z_TYPE_P(frame) == IS_ARRAY); - zval *function = zend_hash_find_ex(Z_ARR_P(frame), ZSTR_KNOWN(ZEND_STR_FUNCTION), 1); - zval *class = zend_hash_find_ex(Z_ARR_P(frame), ZSTR_KNOWN(ZEND_STR_CLASS), 1); - zval *type = zend_hash_find_ex(Z_ARR_P(frame), ZSTR_KNOWN(ZEND_STR_TYPE), 1); - zval *file = zend_hash_find_ex(Z_ARR_P(frame), ZSTR_KNOWN(ZEND_STR_FILE), 1); - zval *line = zend_hash_find_ex(Z_ARR_P(frame), ZSTR_KNOWN(ZEND_STR_LINE), 1); - zval *args = zend_hash_find_ex(Z_ARR_P(frame), ZSTR_KNOWN(ZEND_STR_ARGS), 1); - - smart_str_append_printf(&str, "#%-2d ", (int) frame_no); - if (class) { - ZEND_ASSERT(Z_TYPE_P(class) == IS_STRING); - ZEND_ASSERT(type && Z_TYPE_P(type) == IS_STRING); - /* Cut off anonymous class names at null byte. */ - smart_str_appends(&str, Z_STRVAL_P(class)); - smart_str_append(&str, Z_STR_P(type)); - } - smart_str_append(&str, Z_STR_P(function)); - smart_str_appendc(&str, '('); - if (args) { - ZEND_ASSERT(Z_TYPE_P(args) == IS_ARRAY); - debug_print_backtrace_args(&str, args); - } - smart_str_appendc(&str, ')'); - if (file) { - ZEND_ASSERT(Z_TYPE_P(file) == IS_STRING); - ZEND_ASSERT(line && Z_TYPE_P(line) == IS_LONG); - smart_str_appends(&str, " called at ["); - smart_str_append(&str, Z_STR_P(file)); - smart_str_appendc(&str, ':'); - smart_str_append_long(&str, Z_LVAL_P(line)); - smart_str_appendc(&str, ']'); - } - smart_str_appendc(&str, '\n'); - } ZEND_HASH_FOREACH_END(); - zval_ptr_dtor(&backtrace); - smart_str_0(&str); - if (str.s) { - ZEND_WRITE(ZSTR_VAL(str.s), ZSTR_LEN(str.s)); - } - smart_str_free(&str); + zend_string *str = zend_trace_to_string(Z_ARRVAL(backtrace), /* include_main */ false); + ZEND_WRITE(ZSTR_VAL(str), ZSTR_LEN(str)); + zend_string_release(str); + zval_ptr_dtor(&backtrace); } /* }}} */ diff --git a/Zend/zend_exceptions.c b/Zend/zend_exceptions.c index 4cf49320a732a..457f4aa7ce18a 100644 --- a/Zend/zend_exceptions.c +++ b/Zend/zend_exceptions.c @@ -600,29 +600,13 @@ static void _build_trace_string(smart_str *str, HashTable *ht, uint32_t num) /* } /* }}} */ -/* {{{ Obtain the backtrace for the exception as a string (instead of an array) */ -ZEND_METHOD(Exception, getTraceAsString) -{ - zval *trace, *frame, rv; +ZEND_API zend_string *zend_trace_to_string(HashTable *trace, bool include_main) { zend_ulong index; - zval *object; - zend_class_entry *base_ce; - smart_str str = {0}; + zval *frame; uint32_t num = 0; + smart_str str = {0}; - ZEND_PARSE_PARAMETERS_NONE(); - - object = ZEND_THIS; - base_ce = i_get_exception_base(Z_OBJ_P(object)); - - trace = zend_read_property_ex(base_ce, Z_OBJ_P(object), ZSTR_KNOWN(ZEND_STR_TRACE), 1, &rv); - if (EG(exception)) { - RETURN_THROWS(); - } - - /* Type should be guaranteed by property type. */ - ZEND_ASSERT(Z_TYPE_P(trace) == IS_ARRAY); - ZEND_HASH_FOREACH_NUM_KEY_VAL(Z_ARRVAL_P(trace), index, frame) { + ZEND_HASH_FOREACH_NUM_KEY_VAL(trace, index, frame) { if (Z_TYPE_P(frame) != IS_ARRAY) { zend_error(E_WARNING, "Expected array for frame " ZEND_ULONG_FMT, index); continue; @@ -631,12 +615,33 @@ ZEND_METHOD(Exception, getTraceAsString) _build_trace_string(&str, Z_ARRVAL_P(frame), num++); } ZEND_HASH_FOREACH_END(); - smart_str_appendc(&str, '#'); - smart_str_append_long(&str, num); - smart_str_appends(&str, " {main}"); + if (include_main) { + smart_str_appendc(&str, '#'); + smart_str_append_long(&str, num); + smart_str_appends(&str, " {main}"); + } + smart_str_0(&str); + return str.s ? str.s : ZSTR_EMPTY_ALLOC(); +} + +/* {{{ Obtain the backtrace for the exception as a string (instead of an array) */ +ZEND_METHOD(Exception, getTraceAsString) +{ + + ZEND_PARSE_PARAMETERS_NONE(); - RETURN_NEW_STR(str.s); + zval *object = ZEND_THIS; + zend_class_entry *base_ce = i_get_exception_base(Z_OBJ_P(object)); + zval rv; + zval *trace = zend_read_property_ex(base_ce, Z_OBJ_P(object), ZSTR_KNOWN(ZEND_STR_TRACE), 1, &rv); + if (EG(exception)) { + RETURN_THROWS(); + } + + /* Type should be guaranteed by property type. */ + ZEND_ASSERT(Z_TYPE_P(trace) == IS_ARRAY); + RETURN_NEW_STR(zend_trace_to_string(Z_ARRVAL_P(trace), /* include_main */ true)); } /* }}} */ diff --git a/Zend/zend_exceptions.h b/Zend/zend_exceptions.h index f34d220adab85..2bfafba1359bd 100644 --- a/Zend/zend_exceptions.h +++ b/Zend/zend_exceptions.h @@ -68,6 +68,7 @@ extern ZEND_API void (*zend_throw_exception_hook)(zend_object *ex); /* show an exception using zend_error(severity,...), severity should be E_ERROR */ ZEND_API ZEND_COLD zend_result zend_exception_error(zend_object *exception, int severity); +ZEND_API zend_string *zend_trace_to_string(HashTable *trace, bool include_main); ZEND_API ZEND_COLD void zend_throw_unwind_exit(void); ZEND_API ZEND_COLD void zend_throw_graceful_exit(void); diff --git a/tests/basic/bug73969.phpt b/tests/basic/bug73969.phpt index 11cfecd16b241..571bc33b1bb5b 100644 --- a/tests/basic/bug73969.phpt +++ b/tests/basic/bug73969.phpt @@ -26,5 +26,5 @@ class c1 c1::go(); ?> --EXPECTF-- -#0 require() called at [%s:19] -#1 c1::go() called at [%s:23] +#0 %s(19): require() +#1 %s(23): c1::go() diff --git a/tests/lang/bug28213.phpt b/tests/lang/bug28213.phpt index 3677d4c6f3557..abd965cbb935a 100644 --- a/tests/lang/bug28213.phpt +++ b/tests/lang/bug28213.phpt @@ -6,5 +6,8 @@ class FooBar { static function error() { debug_print_backtrace(); } } set_error_handler(array('FooBar', 'error')); include('foobar.php'); ?> ---EXPECTREGEX-- -.*#1\s*include.* +--EXPECTF-- +#0 %s(%d): FooBar::error(2, 'include(foobar....', '%s', 4) +#1 %s(%d): include() +#0 %s(%d): FooBar::error(2, 'include(): Fail...', '%s', 4) +#1 %s(%d): include() From 10a5e506ed55aa384ec9959d100597f3ca0c7690 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Tue, 18 May 2021 11:44:50 +0200 Subject: [PATCH 032/229] Drop SXE_METHOD() macro Don't break my grep. --- ext/simplexml/simplexml.c | 45 +++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/ext/simplexml/simplexml.c b/ext/simplexml/simplexml.c index ac19efb1ac45a..9b5305a509f26 100644 --- a/ext/simplexml/simplexml.c +++ b/ext/simplexml/simplexml.c @@ -43,11 +43,6 @@ PHP_SXE_API zend_class_entry *sxe_get_element_class_entry(void) /* {{{ */ } /* }}} */ -#define SXE_ME(func, arg_info, flags) PHP_ME(simplexml_element, func, arg_info, flags) -#define SXE_MALIAS(func, alias, arg_info, flags) PHP_MALIAS(simplexml_element, func, alias, arg_info, flags) - -#define SXE_METHOD(func) PHP_METHOD(SimpleXMLElement, func) - static php_sxe_object* php_sxe_object_new(zend_class_entry *ce, zend_function *fptr_count); static xmlNodePtr php_sxe_reset_iterator(php_sxe_object *sxe, int use_data); static xmlNodePtr php_sxe_iterator_fetch(php_sxe_object *sxe, xmlNodePtr node, int use_data); @@ -1265,7 +1260,7 @@ static int sxe_objects_compare(zval *object1, zval *object2) /* {{{ */ /* }}} */ /* {{{ Runs XPath query on the XML data */ -SXE_METHOD(xpath) +PHP_METHOD(SimpleXMLElement, xpath) { php_sxe_object *sxe; zval value; @@ -1353,7 +1348,7 @@ SXE_METHOD(xpath) /* }}} */ /* {{{ Creates a prefix/ns context for the next XPath query */ -SXE_METHOD(registerXPathNamespace) +PHP_METHOD(SimpleXMLElement, registerXPathNamespace) { php_sxe_object *sxe; size_t prefix_len, ns_uri_len; @@ -1382,7 +1377,7 @@ SXE_METHOD(registerXPathNamespace) /* }}} */ /* {{{ Return a well-formed XML string based on SimpleXML element */ -SXE_METHOD(asXML) +PHP_METHOD(SimpleXMLElement, asXML) { php_sxe_object *sxe; xmlNodePtr node; @@ -1507,7 +1502,7 @@ static void sxe_add_namespaces(php_sxe_object *sxe, xmlNodePtr node, bool recurs } /* }}} */ /* {{{ Return all namespaces in use */ -SXE_METHOD(getNamespaces) +PHP_METHOD(SimpleXMLElement, getNamespaces) { bool recursive = 0; php_sxe_object *sxe; @@ -1555,7 +1550,7 @@ static void sxe_add_registered_namespaces(php_sxe_object *sxe, xmlNodePtr node, /* }}} */ /* {{{ Return all namespaces registered with document */ -SXE_METHOD(getDocNamespaces) +PHP_METHOD(SimpleXMLElement, getDocNamespaces) { bool recursive = 0, from_root = 1; php_sxe_object *sxe; @@ -1587,7 +1582,7 @@ SXE_METHOD(getDocNamespaces) /* }}} */ /* {{{ Finds children of given node */ -SXE_METHOD(children) +PHP_METHOD(SimpleXMLElement, children) { php_sxe_object *sxe; char *nsprefix = NULL; @@ -1617,7 +1612,7 @@ SXE_METHOD(children) /* }}} */ /* {{{ Finds children of given node */ -SXE_METHOD(getName) +PHP_METHOD(SimpleXMLElement, getName) { php_sxe_object *sxe; xmlNodePtr node; @@ -1641,7 +1636,7 @@ SXE_METHOD(getName) /* }}} */ /* {{{ Identifies an element's attributes */ -SXE_METHOD(attributes) +PHP_METHOD(SimpleXMLElement, attributes) { php_sxe_object *sxe; char *nsprefix = NULL; @@ -1669,7 +1664,7 @@ SXE_METHOD(attributes) /* }}} */ /* {{{ Add Element with optional namespace information */ -SXE_METHOD(addChild) +PHP_METHOD(SimpleXMLElement, addChild) { php_sxe_object *sxe; char *qname, *value = NULL, *nsuri = NULL; @@ -1733,7 +1728,7 @@ SXE_METHOD(addChild) /* }}} */ /* {{{ Add Attribute with optional namespace information */ -SXE_METHOD(addAttribute) +PHP_METHOD(SimpleXMLElement, addAttribute) { php_sxe_object *sxe; char *qname, *value = NULL, *nsuri = NULL; @@ -1900,7 +1895,7 @@ static int sxe_object_cast(zend_object *readobj, zval *writeobj, int type) /* }}} */ /* {{{ Returns the string content */ -SXE_METHOD(__toString) +PHP_METHOD(SimpleXMLElement, __toString) { if (zend_parse_parameters_none() == FAILURE) { RETURN_THROWS(); @@ -1959,7 +1954,7 @@ static int sxe_count_elements(zend_object *object, zend_long *count) /* {{{ */ /* }}} */ /* {{{ Get number of child elements */ -SXE_METHOD(count) +PHP_METHOD(SimpleXMLElement, count) { zend_long count = 0; php_sxe_object *sxe = Z_SXEOBJ_P(ZEND_THIS); @@ -1976,7 +1971,7 @@ SXE_METHOD(count) /* {{{ Rewind to first element */ -SXE_METHOD(rewind) +PHP_METHOD(SimpleXMLElement, rewind) { if (zend_parse_parameters_none() == FAILURE) { RETURN_THROWS(); @@ -1987,7 +1982,7 @@ SXE_METHOD(rewind) /* }}} */ /* {{{ Check whether iteration is valid */ -SXE_METHOD(valid) +PHP_METHOD(SimpleXMLElement, valid) { php_sxe_object *sxe = Z_SXEOBJ_P(ZEND_THIS); @@ -2000,7 +1995,7 @@ SXE_METHOD(valid) /* }}} */ /* {{{ Get current element */ -SXE_METHOD(current) +PHP_METHOD(SimpleXMLElement, current) { php_sxe_object *sxe = Z_SXEOBJ_P(ZEND_THIS); zval *data; @@ -2019,7 +2014,7 @@ SXE_METHOD(current) /* }}} */ /* {{{ Get name of current child element */ -SXE_METHOD(key) +PHP_METHOD(SimpleXMLElement, key) { xmlNodePtr curnode; php_sxe_object *intern; @@ -2044,7 +2039,7 @@ SXE_METHOD(key) /* }}} */ /* {{{ Move to next element */ -SXE_METHOD(next) +PHP_METHOD(SimpleXMLElement, next) { if (zend_parse_parameters_none() == FAILURE) { RETURN_THROWS(); @@ -2055,7 +2050,7 @@ SXE_METHOD(next) /* }}} */ /* {{{ Check whether element has children (elements) */ -SXE_METHOD(hasChildren) +PHP_METHOD(SimpleXMLElement, hasChildren) { php_sxe_object *sxe = Z_SXEOBJ_P(ZEND_THIS); php_sxe_object *child; @@ -2082,7 +2077,7 @@ SXE_METHOD(hasChildren) /* }}} */ /* {{{ Get child element iterator */ -SXE_METHOD(getChildren) +PHP_METHOD(SimpleXMLElement, getChildren) { php_sxe_object *sxe = Z_SXEOBJ_P(ZEND_THIS); zval *data; @@ -2354,7 +2349,7 @@ PHP_FUNCTION(simplexml_load_string) /* }}} */ /* {{{ SimpleXMLElement constructor */ -SXE_METHOD(__construct) +PHP_METHOD(SimpleXMLElement, __construct) { php_sxe_object *sxe = Z_SXEOBJ_P(ZEND_THIS); char *data, *ns = NULL; From 66c3a174f57ee702d278277436045c91f5a89faf Mon Sep 17 00:00:00 2001 From: Graham Campbell Date: Tue, 18 May 2021 11:15:42 +0100 Subject: [PATCH 033/229] [ci skip] Removed incorrect news entry Closes GH-7007. --- NEWS | 4 ---- 1 file changed, 4 deletions(-) diff --git a/NEWS b/NEWS index cd37e29e55a66..203471a69b405 100644 --- a/NEWS +++ b/NEWS @@ -35,10 +35,6 @@ PHP NEWS - PDO_ODBC: . Fixed bug #44643 (bound parameters ignore explicit type definitions). (cmb) -- PDO_pgsql: - . Reverted bug fix for #80892 (PDO::PARAM_INT is treated the same as - PDO::PARAM_STR). (Matteo) - - pgsql: . Fixed php_pgsql_fd_cast() wrt. php_stream_can_cast(). (cmb) From afc4d67c8b4e02a985a4cd27b8e79b343eb3c0ad Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Tue, 18 May 2021 12:28:20 +0200 Subject: [PATCH 034/229] Consistently treat optional-before-required as required There was a loophole here when it came to usage with named arguments, which was not intended. Close the loophole thoroughly by actually dropping the default value from the signature entirely. The default is still used to make the type nullable, but not for anything else. --- Zend/tests/call_user_func_005.phpt | 2 +- Zend/tests/required_param_after_optional.phpt | 6 ++-- ...uired_param_after_optional_named_args.phpt | 17 ++++++++++ Zend/zend_compile.c | 31 ++++++++++++++----- ext/reflection/tests/bug62715.phpt | 8 ++--- 5 files changed, 49 insertions(+), 15 deletions(-) create mode 100644 Zend/tests/required_param_after_optional_named_args.phpt diff --git a/Zend/tests/call_user_func_005.phpt b/Zend/tests/call_user_func_005.phpt index 2f5220db6291b..6a32f1970b563 100644 --- a/Zend/tests/call_user_func_005.phpt +++ b/Zend/tests/call_user_func_005.phpt @@ -18,7 +18,7 @@ var_dump(call_user_func(array('foo', 'teste'))); ?> --EXPECTF-- -Deprecated: Required parameter $b follows optional parameter $a in %s on line %d +Deprecated: Optional parameter $a declared before required parameter $b is implicitly treated as a required parameter in %s on line %d string(1) "x" array(1) { [0]=> diff --git a/Zend/tests/required_param_after_optional.phpt b/Zend/tests/required_param_after_optional.phpt index cd715e77d4819..93b98acfd9f4d 100644 --- a/Zend/tests/required_param_after_optional.phpt +++ b/Zend/tests/required_param_after_optional.phpt @@ -9,6 +9,8 @@ function test3(Type $test3A = null, Type2 $test3B = null, $test3C) {} ?> --EXPECTF-- -Deprecated: Required parameter $testC follows optional parameter $testA in %s on line %d +Deprecated: Optional parameter $testA declared before required parameter $testC is implicitly treated as a required parameter in %s on line %d -Deprecated: Required parameter $test2C follows optional parameter $test2B in %s on line %d +Deprecated: Optional parameter $testB declared before required parameter $testC is implicitly treated as a required parameter in %s on line %d + +Deprecated: Optional parameter $test2B declared before required parameter $test2C is implicitly treated as a required parameter in %s on line %d diff --git a/Zend/tests/required_param_after_optional_named_args.phpt b/Zend/tests/required_param_after_optional_named_args.phpt new file mode 100644 index 0000000000000..e504d76c7efed --- /dev/null +++ b/Zend/tests/required_param_after_optional_named_args.phpt @@ -0,0 +1,17 @@ +--TEST-- +Optional param before required should be treated as required for named args as well +--FILE-- +getMessage(), "\n"; +} + +?> +--EXPECTF-- +Deprecated: Optional parameter $a declared before required parameter $b is implicitly treated as a required parameter in %s on line %d +test(): Argument #1 ($a) not passed diff --git a/Zend/zend_compile.c b/Zend/zend_compile.c index fbb680be2f716..81c7009bda0d4 100644 --- a/Zend/zend_compile.c +++ b/Zend/zend_compile.c @@ -6466,7 +6466,6 @@ void zend_compile_params(zend_ast *ast, zend_ast *return_type_ast, uint32_t fall uint32_t i; zend_op_array *op_array = CG(active_op_array); zend_arg_info *arg_infos; - zend_string *optional_param = NULL; if (return_type_ast || fallback_return_type) { /* Use op_array->arg_info[-1] for return type */ @@ -6489,6 +6488,17 @@ void zend_compile_params(zend_ast *ast, zend_ast *return_type_ast, uint32_t fall arg_infos = safe_emalloc(sizeof(zend_arg_info), list->children, 0); } + /* Find last required parameter number for deprecation message. */ + uint32_t last_required_param = (uint32_t) -1; + for (i = 0; i < list->children; ++i) { + zend_ast *param_ast = list->child[i]; + zend_ast *default_ast_ptr = param_ast->child[2]; + bool is_variadic = (param_ast->attr & ZEND_PARAM_VARIADIC) != 0; + if (!default_ast_ptr && !is_variadic) { + last_required_param = i; + } + } + for (i = 0; i < list->children; ++i) { zend_ast *param_ast = list->child[i]; zend_ast *type_ast = param_ast->child[0]; @@ -6544,23 +6554,30 @@ void zend_compile_params(zend_ast *ast, zend_ast *return_type_ast, uint32_t fall zend_const_expr_to_zval(&default_node.u.constant, default_ast_ptr); CG(compiler_options) = cops; - if (!optional_param) { + if (last_required_param != (uint32_t) -1 && i < last_required_param) { /* Ignore parameters of the form "Type $param = null". * This is the PHP 5 style way of writing "?Type $param", so allow it for now. */ bool is_implicit_nullable = type_ast && Z_TYPE(default_node.u.constant) == IS_NULL; if (!is_implicit_nullable) { - optional_param = name; + zend_ast *required_param_ast = list->child[last_required_param]; + zend_error(E_DEPRECATED, + "Optional parameter $%s declared before required parameter $%s " + "is implicitly treated as a required parameter", + ZSTR_VAL(name), ZSTR_VAL(zend_ast_get_str(required_param_ast->child[1]))); } + + /* Regardless of whether we issue a deprecation, convert this parameter into + * a required parameter without a default value. This ensures that it cannot be + * used as an optional parameter even with named parameters. */ + opcode = ZEND_RECV; + default_node.op_type = IS_UNUSED; + zval_ptr_dtor(&default_node.u.constant); } } else { opcode = ZEND_RECV; default_node.op_type = IS_UNUSED; op_array->required_num_args = i + 1; - if (optional_param) { - zend_error(E_DEPRECATED, "Required parameter $%s follows optional parameter $%s", - ZSTR_VAL(name), ZSTR_VAL(optional_param)); - } } arg_info = &arg_infos[i]; diff --git a/ext/reflection/tests/bug62715.phpt b/ext/reflection/tests/bug62715.phpt index 63339cddc1ec1..b0080b6da63b4 100644 --- a/ext/reflection/tests/bug62715.phpt +++ b/ext/reflection/tests/bug62715.phpt @@ -17,9 +17,7 @@ foreach ($r->getParameters() as $p) { } ?> --EXPECTF-- -Deprecated: Required parameter $c follows optional parameter $b in %s on line %d -bool(true) -bool(true) +Deprecated: Optional parameter $b declared before required parameter $c is implicitly treated as a required parameter in %s on line %d +bool(false) +bool(false) bool(false) -NULL -int(0) From c939bd2f10b41bced49eb5bf12d48c3cf64f984a Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Tue, 18 May 2021 14:00:25 +0200 Subject: [PATCH 035/229] Handle explicitly nullable types in optional-before-required deprecation The exception for null default values here exists to keep compatibility with PHP < 7.1 where "Foo $bar = null" was the canonical way to create a nullable parameter. If the parameter is actually "?Foo $bar = null", then clearly compatibility with PHP < 7.1 is not a concern, and we can throw a deprecation notice. --- Zend/tests/required_param_after_optional.phpt | 4 +++- Zend/zend_compile.c | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Zend/tests/required_param_after_optional.phpt b/Zend/tests/required_param_after_optional.phpt index 93b98acfd9f4d..c3a925efe88ac 100644 --- a/Zend/tests/required_param_after_optional.phpt +++ b/Zend/tests/required_param_after_optional.phpt @@ -5,7 +5,7 @@ Required parameter after optional is deprecated function test($testA = null, $testB = null, $testC) {} function test2(Type $test2A = null, $test2B = null, $test2C) {} -function test3(Type $test3A = null, Type2 $test3B = null, $test3C) {} +function test3(Type $test3A = null, ?Type2 $test3B = null, $test3C) {} ?> --EXPECTF-- @@ -14,3 +14,5 @@ Deprecated: Optional parameter $testA declared before required parameter $testC Deprecated: Optional parameter $testB declared before required parameter $testC is implicitly treated as a required parameter in %s on line %d Deprecated: Optional parameter $test2B declared before required parameter $test2C is implicitly treated as a required parameter in %s on line %d + +Deprecated: Optional parameter $test3B declared before required parameter $test3C is implicitly treated as a required parameter in %s on line %d diff --git a/Zend/zend_compile.c b/Zend/zend_compile.c index 81c7009bda0d4..07e8679885163 100644 --- a/Zend/zend_compile.c +++ b/Zend/zend_compile.c @@ -6558,7 +6558,8 @@ void zend_compile_params(zend_ast *ast, zend_ast *return_type_ast, uint32_t fall /* Ignore parameters of the form "Type $param = null". * This is the PHP 5 style way of writing "?Type $param", so allow it for now. */ bool is_implicit_nullable = - type_ast && Z_TYPE(default_node.u.constant) == IS_NULL; + type_ast && !(type_ast->attr & ZEND_TYPE_NULLABLE) + && Z_TYPE(default_node.u.constant) == IS_NULL; if (!is_implicit_nullable) { zend_ast *required_param_ast = list->child[last_required_param]; zend_error(E_DEPRECATED, From 0de9494464f17379db73817458db047eab378238 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Thu, 1 Apr 2021 09:05:35 +0000 Subject: [PATCH 036/229] Initial support of JIT/arm64 SUMMARY We implemented a prototype of PHP JIT/arm64. Briefly speaking, 1. build system Changes to the build system are made so that PHP JIT can be successfully built and run on ARM-based machine. Major change lies in file zend_jit_arm64.dasc, where the handler for each opcode is generated into machine code. Note that this file is just copied from zend_jit_x86.dasc and the *unimplemented* parts are substitued with 'brk' instruction for future work. 2. registers AArch64 registers are defined in file zend_jit_arm64.h. From our perspectives, the register usage is quite different from the x86 implementation due to the different ABI, number of registers and addressing modes. We had many confusions on this part, and will discuss it in details in the final section. 3. opcodes Several opcodes are partially supported, including INIT_FCALL, DO_UCALL, DO_ICALL, RETURN, ADD, PRE_INC, JMP, QM_ASSIGN, etc. Hence, simple use scenarios such as user function call, loops, addition with integer and floating point numbers can be supported. 18 micro test cases are added under 'ext/opcache/tests/jit/arm64/'. Note that majority of these test cases are design for functional JIT, and cases 'hot_func_*.phpt' and 'loop_002.phpt' can trigger tracing JIT. 4. test Our local test environment is an ARM-based server with Ubuntu 20.04 and GCC-10. Note that both HYBRID and CALL VM modes are supported. We suggest running the JIT test cases using the following command. Out of all 130 test cases, 66 cases can be passed currently. ``` $ make test TESTS='-d opcache.jit=1203 ext/opcache/tests/jit/' ``` DETAILS 1. I-cache flush Instruction cache must be flushed for the JIT-ed code on AArch64. See macro JIT_CACHE_FLUSH in file 'zend_jit_internal.h'. 2. Disassembler Add initialization and jump target parse operations for AArch64 backed. See the updates in file 'zend_jit_disasm.c'. 3. redzone Enable redzone for AArch64. See the update in zend_vm_opcodes.h. Redzone is designated to prevent 'vm_stack_data' from being optimized out by compilers. It's worth noting that this 16-byte redzone might be reused as temporary use(treated as extra stack space) for HYBRID mode. 4. stack space reservation The definitions of HYBRID_SPAD, SPAD and NR_SPAD are a bit tricky for x86/64. In AArch64, HYBRID_SPAD and SPAD are both defined as 16. These 16 bytes are pre-allocated for tempoerary usage along the exuection of JIT-ed code. Take line 4185 in file zend_jit_arm64.dasc as an example. NR_SPAD is defined as 48, out of which 32 bytes to save FP/IP/LR registers. Note that we choose to always reserve HYBRID_SPAD bytes in HYBRID mode, no matter whether redzone is used or not, for the sake of safety. 5. stack alignment In AArch64 the stack pointer should be 16-byte aligned. Since shadow stack is used for JIT, it's easy to guarantee the stack alignment, via simply moving SP with an offset like 16 or a multiple of 16. That's why NR_SPAD is defined as 48 and we use 32 of them to save FP/IP/LR registers which only occupies 24 bytes. 6. global registers x27 and x28 are reserved as global registers. See the updates in file zend_jit_vm_helpers.c 7. function prologue for CALL mode Two callee-saved registers x27 and x28 should saved in function zend_jit_prologue() in file zend_jit_arm64.dasc. Besides the LR, i.e. x30, should also be saved since runtime C helper functions(such as zend_jit_find_func_helper) might be invoked along the execution of JIT-ed code. 8. regset Minor changes are done to regset operations particularly for AArch64. See the updates in file zend_jit_internal.h. REGISTER USAGE In this section, we will first talk about our understanding on register usage and then demonstrate our design. 1. Register usage for HYBRID/CALL modes Registers are used similarly between HYBRID mode and CALL mode. One difference is how FP and IP are saved. In HYBRID mode, they are assigned to global registers, while in CALL mode they are saved/restored on the VM stack explicitly in prologue/epilogue. The other difference is that LR register should also be saved/restored in CALL mode since JIT-ed code are invoked as normal functions. 2. Register usage for functional/tracing JIT The way registers are used differs a lot between functional JIT and tracing JIT. For functional JIT, runtime C code (e.g. helper functions) would be invoked along the execution of JIT-ed code. As the operands for *most* opcodes are accessed via the stack slot, i.e. FP + offset. Hence there is no need to save/restore local(caller-saved) registers before/after invoking runtime C code. Exception lies in Phi node and registers might be allocated for these nodes. Currently I don't fully understand the reason, why registers are allocated for Phi functions, because I suppose for different versions of SSA variables at the Phi function, their postions on the stack slot should be identical(in other words, access via the stack slot is enough and there is no need to allocate registers). For tracing JIT, runtime information are recorded for traces(before the JIT compilation), and the data types and control flows are concrete as well. Hence it's would be faster to conduct operations and computations via registers rather than stack slots(as functional JIT does) for these collected hot paths. Besides, runtime C code can be invoked for tracing JIT, however this only happends for deoptimization and all registers are saved to stack in advance. 3. Candidates for register allocator 1) opcode candidates Function zend_jit_opline_supports_reg() determines the candidate opcodes which can use CPU registers. 2) register candidates Registers in set "ZEND_REGSET_FP + ZEND_REGSET_GP - ZEND_REGSET_FIXED - ZEND_REGSET_PRESERVED" are available for register allocator. Note that registers from ZEND_REGSET_FIXED are reserved for special purpose, such as the stack pointer, and they are excluded from register allocation process. Note that registers from ZEND_REGSET_PRESERVED are callee-saved based on the ABI and it's safe to not use them either. 4. Temporary registers Temporary registers are needed by some opcodes to save intermediate computation results. 1) Functions zend_jit_get_def_scratch_regset() and zend_jit_get_scratch_regset() return which registers might be clobbered by some opcodes. Hence register allocator would spill these scratch registers if necessary when encountering these opcodes. 2) Macro ZEND_REGSET_LOW_PRIORITY denotes a set of registers which would be allocated with low priority, and these registers can be used as temporary usage to avoid conflicts to its best. 5. Compared to the x86 implementation, in JIT/arm64 1) Called-saved FP registers are included into ZEND_REGSET_PRESERVED for AArch64. 2) We follow the logic of function zend_jit_opline_supports_reg(). 3) We reserve 4 GPRs and 2 FPRs out from register allocator and use them as temporary registers in particular. Note that these 6 registers are included in set ZEND_REGSET_FIXED. Since they are reserved, may-clobbered registers can be removed for most opcodes except for function calls. Besides, low-priority registers are defined as empty since all candidate registers are of the same priority. See the updates in function zend_jit_get_scratch_regset() and macro ZEND_REGSET_LOW_PRIORITY. 6. Why we reserve registers for temporary usage? 1) Addressing mode in AArch64 needs more temporary registers. The addressing mode is different from x86 and tempory registers might be *always* needed for most opcodes. For instance, an immediate must be first moved into one register before storing into memory in AArch64, whereas in x86 this immediate can be stored directly. 2) There are more registers in AArch64. Compared to the solution in JIT/x86(that is, temporary registers are reserved on demand, i.e. different registers for different opcodes under different conditions), our solution seems a coarse-granularity and brute-force solution, and the execution performance might be downgraded to some extent since the number of candidate registers used for allocation becomes less. We suppose the performance loss might be acceptable since there are more registers in AArch64. 3) Based on my understanding, scratch registers defined in x86 are excluded from candidates for register allocator with *low possibility*, and it can still allocate these registers. Special handling should be conducted, such as checking 'reg != ZREG_R0'. Hence, as we see it, it's simpler to reserve some temporary registers exclusively. See the updates in function zend_jit_math_long_long() for instance. TMP1 can be used directly without checking. Co-Developed-by: Nick Gasson --- Zend/zend_vm_opcodes.h | 3 +- build/Makefile.global | 2 + ext/opcache/config.m4 | 3 +- ext/opcache/config.w32 | 1 + ext/opcache/jit/Makefile.frag | 6 +- ext/opcache/jit/zend_jit.c | 36 + ext/opcache/jit/zend_jit_arm64.dasc | 6050 +++++++++++++++++ ext/opcache/jit/zend_jit_arm64.h | 142 + ext/opcache/jit/zend_jit_disasm.c | 18 + ext/opcache/jit/zend_jit_gdb.c | 5 + ext/opcache/jit/zend_jit_internal.h | 33 +- ext/opcache/jit/zend_jit_perf_dump.c | 3 + ext/opcache/jit/zend_jit_vm_helpers.c | 10 +- ext/opcache/tests/jit/arm64/add_001.phpt | 20 + ext/opcache/tests/jit/arm64/add_002.phpt | 20 + ext/opcache/tests/jit/arm64/add_003.phpt | 20 + ext/opcache/tests/jit/arm64/add_004.phpt | 20 + ext/opcache/tests/jit/arm64/add_005.phpt | 24 + ext/opcache/tests/jit/arm64/hot_func_001.phpt | 24 + ext/opcache/tests/jit/arm64/hot_func_002.phpt | 25 + ext/opcache/tests/jit/arm64/icall_001.phpt | 24 + ext/opcache/tests/jit/arm64/loop_001.phpt | 22 + ext/opcache/tests/jit/arm64/loop_002.phpt | 26 + ext/opcache/tests/jit/arm64/recv_001.phpt | 26 + ext/opcache/tests/jit/arm64/ret_001.phpt | 20 + ext/opcache/tests/jit/arm64/ret_002.phpt | 20 + ext/opcache/tests/jit/arm64/ret_003.phpt | 20 + ext/opcache/tests/jit/arm64/skipif.inc | 3 + ext/opcache/tests/jit/arm64/ucall_001.phpt | 19 + ext/opcache/tests/jit/arm64/ucall_002.phpt | 21 + ext/opcache/tests/jit/arm64/ucall_003.phpt | 22 + ext/opcache/tests/jit/arm64/ucall_004.phpt | 23 + 32 files changed, 6704 insertions(+), 7 deletions(-) create mode 100644 ext/opcache/jit/zend_jit_arm64.dasc create mode 100644 ext/opcache/jit/zend_jit_arm64.h create mode 100644 ext/opcache/tests/jit/arm64/add_001.phpt create mode 100644 ext/opcache/tests/jit/arm64/add_002.phpt create mode 100644 ext/opcache/tests/jit/arm64/add_003.phpt create mode 100644 ext/opcache/tests/jit/arm64/add_004.phpt create mode 100644 ext/opcache/tests/jit/arm64/add_005.phpt create mode 100644 ext/opcache/tests/jit/arm64/hot_func_001.phpt create mode 100644 ext/opcache/tests/jit/arm64/hot_func_002.phpt create mode 100644 ext/opcache/tests/jit/arm64/icall_001.phpt create mode 100644 ext/opcache/tests/jit/arm64/loop_001.phpt create mode 100644 ext/opcache/tests/jit/arm64/loop_002.phpt create mode 100644 ext/opcache/tests/jit/arm64/recv_001.phpt create mode 100644 ext/opcache/tests/jit/arm64/ret_001.phpt create mode 100644 ext/opcache/tests/jit/arm64/ret_002.phpt create mode 100644 ext/opcache/tests/jit/arm64/ret_003.phpt create mode 100644 ext/opcache/tests/jit/arm64/skipif.inc create mode 100644 ext/opcache/tests/jit/arm64/ucall_001.phpt create mode 100644 ext/opcache/tests/jit/arm64/ucall_002.phpt create mode 100644 ext/opcache/tests/jit/arm64/ucall_003.phpt create mode 100644 ext/opcache/tests/jit/arm64/ucall_004.phpt diff --git a/Zend/zend_vm_opcodes.h b/Zend/zend_vm_opcodes.h index 94e74c0a57f12..d6cc6e28edb14 100644 --- a/Zend/zend_vm_opcodes.h +++ b/Zend/zend_vm_opcodes.h @@ -35,7 +35,8 @@ #endif #if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) && !defined(__SANITIZE_ADDRESS__) -# if ((defined(i386) && !defined(__PIC__)) || defined(__x86_64__) || defined(_M_X64)) +# if ((defined(i386) && !defined(__PIC__)) || defined(__x86_64__) || \ + defined(_M_X64) || defined(__aarch64__)) # define ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 16 # endif #endif diff --git a/build/Makefile.global b/build/Makefile.global index 2ff838cb3318d..41151163fb72a 100644 --- a/build/Makefile.global +++ b/build/Makefile.global @@ -125,6 +125,8 @@ distclean: clean rm -f scripts/man1/phpize.1 scripts/php-config scripts/man1/php-config.1 sapi/cli/php.1 sapi/cgi/php-cgi.1 sapi/phpdbg/phpdbg.1 ext/phar/phar.1 ext/phar/phar.phar.1 rm -f sapi/fpm/php-fpm.conf sapi/fpm/init.d.php-fpm sapi/fpm/php-fpm.service sapi/fpm/php-fpm.8 sapi/fpm/status.html rm -f ext/phar/phar.phar ext/phar/phar.php + rm -f ext/opcache/jit/zend_jit_x86.c + rm -f ext/opcache/jit/zend_jit_arm64.c if test "$(srcdir)" != "$(builddir)"; then \ rm -f ext/phar/phar/phar.inc; \ fi diff --git a/ext/opcache/config.m4 b/ext/opcache/config.m4 index 633618e885498..f49480117ba2e 100644 --- a/ext/opcache/config.m4 +++ b/ext/opcache/config.m4 @@ -29,7 +29,7 @@ if test "$PHP_OPCACHE" != "no"; then if test "$PHP_OPCACHE_JIT" = "yes"; then case $host_cpu in - i[[34567]]86*|x86*) + i[[34567]]86*|x86*|aarch64) ;; *) AC_MSG_WARN([JIT not supported by host architecture]) @@ -77,6 +77,7 @@ if test "$PHP_OPCACHE" != "no"; then fi PHP_SUBST(DASM_FLAGS) + PHP_SUBST(DASM_ARCH) AC_MSG_CHECKING(for opagent in default path) for i in /usr/local /usr; do diff --git a/ext/opcache/config.w32 b/ext/opcache/config.w32 index a7f292ee7625f..764a2edaab146 100644 --- a/ext/opcache/config.w32 +++ b/ext/opcache/config.w32 @@ -25,6 +25,7 @@ if (PHP_OPCACHE != "no") { dasm_flags += " -D ZTS=1"; } DEFINE("DASM_FLAGS", dasm_flags); + DEFINE("DASM_ARCH", "x86"); AC_DEFINE('HAVE_JIT', 1, 'Define to enable JIT'); /* XXX read this dynamically */ diff --git a/ext/opcache/jit/Makefile.frag b/ext/opcache/jit/Makefile.frag index d44e06a3ad91b..98c5cdaea2494 100644 --- a/ext/opcache/jit/Makefile.frag +++ b/ext/opcache/jit/Makefile.frag @@ -2,11 +2,11 @@ $(builddir)/minilua: $(srcdir)/jit/dynasm/minilua.c $(BUILD_CC) $(srcdir)/jit/dynasm/minilua.c -lm -o $@ -$(builddir)/jit/zend_jit_x86.c: $(srcdir)/jit/zend_jit_x86.dasc $(srcdir)/jit/dynasm/*.lua $(builddir)/minilua - $(builddir)/minilua $(srcdir)/jit/dynasm/dynasm.lua $(DASM_FLAGS) -o $@ $(srcdir)/jit/zend_jit_x86.dasc +$(builddir)/jit/zend_jit_$(DASM_ARCH).c: $(srcdir)/jit/zend_jit_$(DASM_ARCH).dasc $(srcdir)/jit/dynasm/*.lua $(builddir)/minilua + $(builddir)/minilua $(srcdir)/jit/dynasm/dynasm.lua $(DASM_FLAGS) -o $@ $(srcdir)/jit/zend_jit_$(DASM_ARCH).dasc $(builddir)/jit/zend_jit.lo: \ - $(builddir)/jit/zend_jit_x86.c \ + $(builddir)/jit/zend_jit_$(DASM_ARCH).c \ $(srcdir)/jit/zend_jit_helpers.c \ $(srcdir)/jit/zend_jit_disasm.c \ $(srcdir)/jit/zend_jit_gdb.c \ diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 0cb9fc946df5c..221902445f8db 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -39,7 +39,14 @@ #include "Optimizer/zend_call_graph.h" #include "Optimizer/zend_dump.h" +#if defined(__x86_64__) || defined(i386) #include "jit/zend_jit_x86.h" +#elif defined (__aarch64__) +#include "jit/zend_jit_arm64.h" +#else +#error "JIT not supported on this platform" +#endif + #include "jit/zend_jit_internal.h" #ifdef ZTS @@ -204,7 +211,12 @@ static bool zend_long_is_power_of_two(zend_long x) #define OP2_RANGE() OP_RANGE(ssa_op, op2) #define OP1_DATA_RANGE() OP_RANGE(ssa_op + 1, op1) +#if defined(__x86_64__) || defined(i386) #include "dynasm/dasm_x86.h" +#elif defined(__aarch64__) +#include "dynasm/dasm_arm64.h" +#endif + #include "jit/zend_jit_helpers.c" #include "jit/zend_jit_disasm.c" #ifndef _WIN32 @@ -216,7 +228,11 @@ static bool zend_long_is_power_of_two(zend_long x) #endif #include "jit/zend_jit_vtune.c" +#if defined(__x86_64__) || defined(i386) #include "jit/zend_jit_x86.c" +#elif defined(__aarch64__) +#include "jit/zend_jit_arm64.c" +#endif #if _WIN32 # include @@ -298,15 +314,32 @@ static void handle_dasm_error(int ret) { case DASM_S_RANGE_PC: fprintf(stderr, "DASM_S_RANGE_PC %d\n", ret & 0xffffffu); break; +#ifdef DASM_S_RANGE_VREG case DASM_S_RANGE_VREG: fprintf(stderr, "DASM_S_RANGE_VREG\n"); break; +#endif +#ifdef DASM_S_UNDEF_L case DASM_S_UNDEF_L: fprintf(stderr, "DASM_S_UNDEF_L\n"); break; +#endif +#ifdef DASM_S_UNDEF_LG + case DASM_S_UNDEF_LG: + fprintf(stderr, "DASM_S_UNDEF_LG\n"); + break; +#endif +#ifdef DASM_S_RANGE_REL + case DASM_S_RANGE_REL: + fprintf(stderr, "DASM_S_RANGE_REL\n"); + break; +#endif case DASM_S_UNDEF_PC: fprintf(stderr, "DASM_S_UNDEF_PC\n"); break; + default: + fprintf(stderr, "DASM_S_%0x\n", ret & 0xff000000u); + break; } ZEND_UNREACHABLE(); } @@ -391,6 +424,9 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, entry = *dasm_ptr; *dasm_ptr = (void*)((char*)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT)); + /* flush the hardware I-cache */ + JIT_CACHE_FLUSH(entry, entry + size); + if (trace_num) { zend_jit_trace_add_code(entry, size); } diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc new file mode 100644 index 0000000000000..b4a83e4b8930b --- /dev/null +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -0,0 +1,6050 @@ +/* + * +----------------------------------------------------------------------+ + * | Zend JIT | + * +----------------------------------------------------------------------+ + * | Copyright (c) The PHP Group | + * +----------------------------------------------------------------------+ + * | This source file is subject to version 3.01 of the PHP license, | + * | that is bundled with this package in the file LICENSE, and is | + * | available through the world-wide-web at the following url: | + * | http://www.php.net/license/3_01.txt | + * | If you did not receive a copy of the PHP license and are unable to | + * | obtain it through the world-wide-web, please send a note to | + * | license@php.net so we can mail you a copy immediately. | + * +----------------------------------------------------------------------+ + * | Authors: Dmitry Stogov | + * | Xinchen Hui | + * | Hao Sun | + * +----------------------------------------------------------------------+ + */ + +|.arch arm64 + +|.define FP, x27 +|.define IP, x28 +|.define IPl, w28 +|.define RX, x28 // the same as VM IP reused as a general purpose reg +|.define LR, x30 +|.define CARG1, x0 +|.define CARG2, x1 +|.define CARG3, x2 +|.define CARG4, x3 +|.define CARG5, x4 +|.define CARG6, x5 +|.define RETVALx, x0 +|.define RETVALw, w0 +|.define FCARG1x, x0 +|.define FCARG1w, w0 +|.define FCARG2x, x1 +|.define SPAD, #0x10 // padding for CPU stack alignment +|.define NR_SPAD, #0x30 // padding for CPU stack alignment +|.define T4, [sp, #0x20] // Used to store old value of LR (CALL VM only) +|.define T3, [sp, #0x18] // Used to store old value of IP (CALL VM only) +|.define T2, [sp, #0x10] // Used to store old value of FP (CALL VM only) +|.define T1, [sp] +|.define A4, [r4+0xC] // preallocated slots for arguments of "cdecl" functions (intersect with T1) +|.define A3, [r4+0x8] +|.define A2, [r4+0x4] +|.define A1, [r4] + +// Temporaries, not preserved across calls +|.define TMP1, x8 +|.define TMP1w, w8 +|.define TMP2, x9 +|.define TMP2w, w9 +|.define TMP3, x10 +|.define TMP3w, w10 +|.define TMP4, x11 +|.define TMP4w, w11 +|.define FPTMP1, v16 +|.define FPTMP2, v17 + +// Temporary register index in _zend_reg +|.define ZREG_TMP1, ZREG_X8 +|.define ZREG_TMP2, ZREG_X9 +|.define ZREG_TMP3, ZREG_X10 +|.define ZREG_TMP4, ZREG_X11 +|.define ZREG_FPTMP1, ZREG_V16 +|.define ZREG_FPTMP2, ZREG_V17 + +#define ZREG_TMP1 ZREG_X8 +#define ZREG_TMP2 ZREG_X9 +#define ZREG_TMP3 ZREG_X10 +#define ZREG_TMP4 ZREG_X11 +#define ZREG_FPTMP1 ZREG_V16 +#define ZREG_FPTMP2 ZREG_V17 + +|.define HYBRID_SPAD, #16 // padding for stack alignment + +#define TMP_ZVAL_OFFSET 0 +#define DASM_ALIGNMENT 16 +#define MAX_IMM12 0xfff // maximum value for imm12 + +#include "Zend/zend_cpuinfo.h" + +#ifdef HAVE_VALGRIND +# include +#endif + +/* The generated code may contain tautological comparisons, ignore them. */ +#if defined(__clang__) +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wtautological-compare" +# pragma clang diagnostic ignored "-Wstring-compare" +#endif + +const char* zend_reg_name[] = { + "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", + "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", + "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23", + "x24", "x25", "x26", "x27", "x28", "x29", "x30", "sp", + "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", + "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", + "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31" +}; + +#ifdef HAVE_GCC_GLOBAL_REGS +# define GCC_GLOBAL_REGS 1 +#else +# define GCC_GLOBAL_REGS 0 +#endif + +# define ZREG_FCARG1x ZREG_X0 +# define ZREG_FCARG2x ZREG_X1 + +#if ZTS +static size_t tsrm_ls_cache_tcb_offset = 0; +static size_t tsrm_tls_index; +static size_t tsrm_tls_offset; +#endif + +/* By default avoid JITing inline handlers if it does not seem profitable due to lack of + * type information. Disabling this option allows testing some JIT handlers in the + * presence of try/catch blocks, which prevent SSA construction. */ +#ifndef PROFITABILITY_CHECKS +# define PROFITABILITY_CHECKS 1 +#endif + +|.type EX, zend_execute_data, FP +|.type OP, zend_op +|.type ZVAL, zval + +|.actionlist dasm_actions + +|.globals zend_lb +static void* dasm_labels[zend_lb_MAX]; + +|.section code, cold_code, jmp_table + +#define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0x7fffffff) + +#define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= (-2147483647 - 1))) + +#define BP_JIT_IS 6 + +/* In x86/64, HYBRID_SPAD bytes are reserved on the stack only if flag ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE + * is not defined, because the 16-byte redzone, allocated on the stack when the flag is defined, can be + * reused. In AArch64, it's safe that these bytes are always reserved because the stack layout might + * change along software evolution, making the redzone not reusable any longer. */ +|.macro ADD_HYBRID_SPAD +| add sp, sp, HYBRID_SPAD +|.endmacro + +|.macro SUB_HYBRID_SPAD +| sub sp, sp, HYBRID_SPAD +|.endmacro + +|.macro LOAD_ADDR, reg, addr +| // 48-bit virtual address +| mov reg, #((uintptr_t)(addr) & 0xffff) +| movk reg, #(((uintptr_t)(addr) >> 16) & 0xffff), lsl #16 +| movk reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32 +|.endmacro + +// Type cast to unsigned is used to avoid undefined behavior. +|.macro LOAD_32BIT_VAL, reg, val +| mov reg, #((uint32_t)(val) & 0xffff) +| movk reg, #(((uint32_t)(val) >> 16) & 0xffff), lsl #16 +|.endmacro + +|.macro LOAD_64BIT_VAL, reg, val +| mov reg, #((uint64_t)(val) & 0xffff) +| movk reg, #(((uint64_t)(val) >> 16) & 0xffff), lsl #16 +| movk reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 +| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 +|.endmacro + +// Safe memory load/store with an unsigned immediate offset. +// When using Z_OFFSET(addr), which is 24-bit long, as the unsigned offset to compute a memory address, +// we should firstly check whether it's greater than MAX_IMM12. +|.macro SAFE_MEM_ACC_WITH_UOFFSET, ldr_str_ins, op, base_reg, offset, tmp_reg +|| if (offset > MAX_IMM12) { +| LOAD_32BIT_VAL tmp_reg, offset +| ldr_str_ins op, [base_reg, tmp_reg] +|| } else { +| ldr_str_ins op, [base_reg, #(offset)] +|| } +|.endmacro + +|.macro LOAD_TSRM_CACHE, reg +| brk #0 // TODO +|.endmacro + +|.macro LOAD_ADDR_ZTS, reg, struct, field +| brk #0 // TODO +|.endmacro + +|.macro ADDR_OP1, addr_ins, addr, tmp_reg +| brk #0 // TODO +|.endmacro + +// Move the 48-bit address 'addr' into 'tmp_reg' and store it into the dest addr 'op1' +|.macro ADDR_STORE, op1, addr, tmp_reg +| LOAD_ADDR tmp_reg, addr +| str tmp_reg, op1 +|.endmacro + +// Move the 48-bit address 'addr' into 'tmp_reg1' and compare with the value inside address 'op1' +|.macro ADDR_CMP, op1, addr, tmp_reg1, tmp_reg2 +| LOAD_ADDR tmp_reg1, addr +| ldr tmp_reg2, op1 +| cmp tmp_reg2, tmp_reg1 +|.endmacro + +|.macro PUSH_ADDR, addr, tmp_reg +| ADDR_OP1 push, addr, tmp_reg +|.endmacro + +|.macro PUSH_ADDR_ZTS, struct, field, tmp_reg +| brk #0 // TODO +|.endmacro + +// Store the value from a register 'op' into memory 'addr' +|.macro MEM_STORE, str_ins, op, addr, tmp_reg +| LOAD_ADDR tmp_reg, addr +| str_ins op, [tmp_reg] +|.endmacro + +|.macro MEM_STORE_ZTS, str_ins, op, struct, field, tmp_reg +| .if ZTS +| brk #0 // TODO: test +| LOAD_TSRM_CACHE tmp_reg +| str_ins op, [tmp_reg, #(struct.._offset+offsetof(zend_..struct, field))] +| .else +| MEM_STORE str_ins, op, &struct.field, tmp_reg +| .endif +|.endmacro + +// Load the value from memory 'addr' into a register 'op' +|.macro MEM_LOAD, ldr_ins, op, addr, tmp_reg +| LOAD_ADDR tmp_reg, addr +| ldr_ins op, [tmp_reg] +|.endmacro + +|.macro MEM_LOAD_ZTS, ldr_ins, op, struct, field, tmp_reg +| .if ZTS +| brk #0 // TODO: test +| LOAD_TSRM_CACHE tmp_reg +| ldr_ins op, [tmp_reg, #(struct.._offset+offsetof(zend_..struct, field))] +| .else +| MEM_LOAD ldr_ins, op, &struct.field, tmp_reg +| .endif +|.endmacro + +// Load the value from memory 'addr' into a tmp register 'tmp_reg1', +// and conduct arithmetic operations with 'op'. +// Operations can be add/sub/div/mul, and the computation result is stored into 'op'. +|.macro MEM_LOAD_OP, mem_ins, ldr_ins, op, addr, tmp_reg1, tmp_reg2 +| MEM_LOAD ldr_ins, tmp_reg1, addr, tmp_reg2 +| mem_ins op, op, tmp_reg1 +|.endmacro + +|.macro MEM_LOAD_OP_ZTS, mem_ins, ldr_ins, op, struct, field, tmp_reg1, tmp_reg2 +| .if ZTS +| brk #0 // TODO: test +| LOAD_TSRM_CACHE tmp_reg1 +| ldr_ins tmp_reg2, [tmp_reg1, #(struct.._offset+offsetof(zend_..struct, field))] +| mem_ins op, op, tmp_reg2 +| .else +| MEM_LOAD_OP mem_ins, ldr_ins, op, &struct.field, tmp_reg1, tmp_reg2 +| .endif +|.endmacro + +// Similar to MEM_LOAD_OP/_ZTS, but operations are compare instructions. +// Note that 'op' can be imm12. +|.macro MEM_LOAD_CMP, ldr_ins, op, addr, tmp_reg1, tmp_reg2 +| MEM_LOAD ldr_ins, tmp_reg1, addr, tmp_reg2 +| cmp tmp_reg1, op +|.endmacro + +|.macro MEM_LOAD_CMP_ZTS, ldr_ins, op, struct, field, tmp_reg1, tmp_reg2 +| .if ZTS +| brk #0 // TODO: test +| LOAD_TSRM_CACHE tmp_reg1 +| ldr_ins tmp_reg2, [tmp_reg1, #(struct.._offset+offsetof(zend_..struct, field))] +| cmp tmp_reg2, op +| .else +| MEM_LOAD_CMP ldr_ins, op, &struct.field, tmp_reg1, tmp_reg2 +| .endif +|.endmacro + +// Load the value from memory 'addr' into a tmp register 'tmp_reg1' and conduct arithmetic operations with 'op'. +// The computation result is stored back to memory 'addr'. 'op' can be either imm12 or register. +// For constant case, it should be guaranteed that 'op' can be represented by imm12 before using this macro. +|.macro MEM_LOAD_OP_STORE, mem_ins, ldr_ins, str_ins, op, addr, tmp_reg1, tmp_reg2 +| MEM_LOAD ldr_ins, tmp_reg1, addr, tmp_reg2 +| mem_ins tmp_reg1, tmp_reg1, op +| str_ins tmp_reg1, [tmp_reg2] +|.endmacro + +|.macro MEM_LOAD_OP_STORE_ZTS, mem_ins, ldr_ins, str_ins, op, struct, field, tmp_reg1, tmp_reg2 +| .if ZTS +| brk #0 // TODO: test +| LOAD_TSRM_CACHE tmp_reg1 +| ldr_ins tmp_reg2, [tmp_reg1, #(struct.._offset+offsetof(zend_..struct, field))] +| mem_ins tmp_reg2, tmp_reg2, op +| str_ins tmp_reg2, [tmp_reg1, #(struct.._offset+offsetof(zend_..struct, field))] +| .else +| MEM_LOAD_OP_STORE mem_ins, ldr_ins, str_ins, op, &struct.field, tmp_reg1, tmp_reg2 +| .endif +|.endmacro + +|.macro MEM_OP3_3, mem_ins, op1, op2, prefix, addr, tmp_reg +| brk #0 // TODO +|.endmacro + +|.macro LOAD_BASE_ADDR, reg, base, offset +|| if (offset) { +|| if (offset > MAX_IMM12) { +| LOAD_32BIT_VAL reg, offset +| add reg, Rx(base), reg +|| } else { +| add reg, Rx(base), #offset +|| } +|| } else { +|| if (base == ZREG_RSP) { +| mov reg, sp +|| } else { +| mov reg, Rx(base) +|| } +|| } +|.endmacro + +|.macro PUSH_BASE_ADDR, base, offset, tmp_reg +| brk #0 // TODO +|.endmacro + +|.macro EXT_CALL, func, tmp_reg +| LOAD_ADDR tmp_reg, func +| blr tmp_reg +|.endmacro + +|.macro EXT_JMP, func, tmp_reg +| LOAD_ADDR tmp_reg, func +| br tmp_reg +|.endmacro + +|.macro SAVE_IP +|| if (GCC_GLOBAL_REGS) { +| str IP, EX->opline +|| } +|.endmacro + +|.macro LOAD_IP +|| if (GCC_GLOBAL_REGS) { +| ldr IP, EX->opline +|| } +|.endmacro + +|.macro LOAD_IP_ADDR, addr +|| if (GCC_GLOBAL_REGS) { +| LOAD_ADDR IP, addr +|| } else { +| ADDR_STORE EX->opline, addr, RX +|| } +|.endmacro + +|.macro LOAD_IP_ADDR_ZTS, struct, field +| brk #0 // TODO +|.endmacro + +|.macro GET_IP, reg +|| if (GCC_GLOBAL_REGS) { +| mov reg, IP +|| } else { +| ldr reg, EX->opline +|| } +|.endmacro + +// In x86 implementation, 'val' can be either a constant or a register. +// In AArch64, use ADD_IP for register case, +// and use ADD_IP_FROM_CST for constant case, where the value can be represented by imm12. +|.macro ADD_IP, val, tmp_reg +|| if (GCC_GLOBAL_REGS) { +| add IP, IP, val +|| } else { +| ldr tmp_reg, EX->opline +| add tmp_reg, tmp_reg, val +| str tmp_reg, EX->opline +|| } +|.endmacro + +|.macro ADD_IP_FROM_CST, val, tmp_reg +|| ZEND_ASSERT(val >=0 && val <= MAX_IMM12); +|| if (GCC_GLOBAL_REGS) { +| add IP, IP, #val +|| } else { +| ldr tmp_reg, EX->opline +| add tmp_reg, tmp_reg, #val +| str tmp_reg, EX->opline +|| } +|.endmacro + +|.macro JMP_IP, tmp_reg +|| if (GCC_GLOBAL_REGS) { +| ldr tmp_reg, [IP] +| br tmp_reg +|| } else { +| ldr tmp_reg, EX:CARG1->opline +| br tmp_reg +|| } +|.endmacro + +|.macro CMP_IP, addr +| brk #0 // TODO +|.endmacro + +|.macro LOAD_ZVAL_ADDR, reg, addr +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| LOAD_ADDR reg, Z_ZV(addr) +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| LOAD_BASE_ADDR reg, Z_REG(addr), Z_OFFSET(addr) +|| } else { +|| ZEND_UNREACHABLE(); +|| } +|.endmacro + +|.macro PUSH_ZVAL_ADDR, addr, tmp_reg +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| PUSH_ADDR Z_ZV(addr), tmp_reg +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| PUSH_BASE_ADDR Z_REG(addr), Z_OFFSET(addr), tmp_reg +|| } else { +|| ZEND_UNREACHABLE(); +|| } +|.endmacro + +|.macro GET_Z_TYPE_INFO, reg, zv +| mov reg, dword [zv+offsetof(zval,u1.type_info)] +|.endmacro + +|.macro SET_Z_TYPE_INFO, zv, type, tmp_reg +| LOAD_32BIT_VAL tmp_reg, type +| str tmp_reg, [zv, #offsetof(zval,u1.type_info)] +|.endmacro + +|.macro GET_ZVAL_TYPE, reg, addr +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| mov reg, byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.v.type)] +|.endmacro + +|.macro GET_ZVAL_TYPE_INFO, reg, addr, tmp_reg +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| SAFE_MEM_ACC_WITH_UOFFSET ldr, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg +|.endmacro + +|.macro SET_ZVAL_TYPE_INFO, addr, type, tmp_reg1, tmp_reg2 +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| LOAD_32BIT_VAL tmp_reg1, type +| SAFE_MEM_ACC_WITH_UOFFSET str, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg2 +|.endmacro + +|.macro SET_ZVAL_TYPE_INFO_FROM_REG, addr, type, tmp_reg +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| SAFE_MEM_ACC_WITH_UOFFSET str, type, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg +|.endmacro + +|.macro GET_Z_PTR, reg, zv +| mov reg, aword [zv] +|.endmacro + +|.macro SET_Z_PTR, zv, val +| mov aword [zv], val +|.endmacro + +|.macro GET_Z_W2, reg, zv +| mov reg, dword [zv+4] +|.endmacro + +|.macro SET_Z_W2, zv, reg +| mov dword [zv+4], reg +|.endmacro + +|.macro GET_ZVAL_PTR, reg, addr, tmp_reg +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| SAFE_MEM_ACC_WITH_UOFFSET ldr, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg +|.endmacro + +|.macro SET_ZVAL_PTR, addr, val, tmp_reg +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| SAFE_MEM_ACC_WITH_UOFFSET str, val, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg +|.endmacro + +|.macro GET_ZVAL_W2, reg, addr +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| brk #0 // TODO +|.endmacro + +|.macro SET_ZVAL_W2, addr, val +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| brk #0 // TODO +|.endmacro + +|.macro UNDEF_OPLINE_RESULT +| brk #0 // TODO +|.endmacro + +// Define DOUBLE_GET_LONG to replace SSE_GET_LONG in x86 implementation. +// Convert the LONG value 'lval' into DOUBLE type, and move it into 'reg' +|.macro DOUBLE_GET_LONG, reg, lval, tmp_reg +|| if (lval == 0) { +| brk #0 // TODO: test +| // vxorps xmm(reg-ZREG_V0), xmm(reg-ZREG_V0), xmm(reg-ZREG_V0) +|| } else { +| LOAD_64BIT_VAL Rx(tmp_reg), lval +| scvtf Rd(reg-ZREG_V0), Rx(tmp_reg) +|| } +|.endmacro + +// Define DOUBLE_GET_ZVAL_LVAL to replace SSE_GET_ZVAL_LVAL in x86 implementation. +// Convert the LONG value in 'addr' into DOUBLE type, and move it into 'reg' +|.macro DOUBLE_GET_ZVAL_LVAL, reg, addr, tmp_reg1, tmp_reg2 +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| DOUBLE_GET_LONG reg, Z_LVAL_P(Z_ZV(addr)), tmp_reg1 +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| SAFE_MEM_ACC_WITH_UOFFSET ldr, Rx(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg2) +| scvtf Rd(reg-ZREG_V0), Rx(tmp_reg1) +|| } else if (Z_MODE(addr) == IS_REG) { +| brk #0 // TODO: test +| scvtf Rd(reg-ZREG_V0), Rx(Z_REG(addr)) +|| } else { +|| ZEND_UNREACHABLE(); +|| } +|.endmacro + +// Define DOUBLE_MATH_REG to replace AVX_MATH_REG in x86 implementation. +|.macro DOUBLE_MATH_REG, opcode, dst_reg, op1_reg, src_reg +|| switch (opcode) { +|| case ZEND_ADD: +| fadd Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(src_reg-ZREG_V0) +|| break; +|| case ZEND_SUB: +| brk #0 // vsubsd xmm(dst_reg-ZREG_V0), xmm(op1_reg-ZREG_V0), xmm(src_reg-ZREG_V0) +|| break; +|| case ZEND_MUL: +| brk #0 // vmulsd xmm(dst_reg-ZREG_V0), xmm(op1_reg-ZREG_V0), xmm(src_reg-ZREG_V0) +|| break; +|| case ZEND_DIV: +| brk #0 // vdivsd xmm(dst_reg-ZREG_V0), xmm(op1_reg-ZREG_V0), xmm(src_reg-ZREG_V0) +|| break; +|| } +|.endmacro + +|.macro LONG_OP, long_ins, reg, addr, tmp_reg1, tmp_reg2 +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +|| if(Z_LVAL_P(Z_ZV(addr)) >= 0 && Z_LVAL_P(Z_ZV(addr)) <= MAX_IMM12) { +| long_ins Rx(reg), Rx(reg), #(Z_LVAL_P(Z_ZV(addr))) +|| } else { +| LOAD_64BIT_VAL tmp_reg1, Z_LVAL_P(Z_ZV(addr)) +| long_ins Rx(reg), Rx(reg), tmp_reg1 +|| } +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| SAFE_MEM_ACC_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg2 +| long_ins Rx(reg), Rx(reg), tmp_reg1 +|| } else if (Z_MODE(addr) == IS_REG) { +| long_ins Rx(reg), Rx(reg), Rx(Z_REG(addr)) +|| } else { +|| ZEND_UNREACHABLE(); +|| } +|.endmacro + +|.macro LONG_CMP, reg, addr, tmp_reg1, tmp_reg2 +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| brk #0 // TODO: test +|| if(Z_LVAL_P(Z_ZV(addr)) >= 0 && Z_LVAL_P(Z_ZV(addr)) <= MAX_IMM12) { +| cmp Rx(reg), #(Z_LVAL_P(Z_ZV(addr))) +|| } else { +| LOAD_64BIT_VAL tmp_reg1, Z_LVAL_P(Z_ZV(addr)) +| cmp Rx(reg), tmp_reg1 +|| } +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| SAFE_MEM_ACC_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg2 +| cmp Rx(reg), tmp_reg1 +|| } else if (Z_MODE(addr) == IS_REG) { +| cmp Rx(reg), Rx(Z_REG(addr)) +|| } else { +|| ZEND_UNREACHABLE(); +|| } +|.endmacro + +|.macro LONG_OP_WITH_CONST_IMM12, long_ins, op1_addr, lval, tmp_reg1, tmp_reg2 +|| ZEND_ASSERT(lval >=0 && lval <= MAX_IMM12); +|| if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { +| SAFE_MEM_ACC_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(op1_addr)), Z_OFFSET(op1_addr), tmp_reg2 +| long_ins tmp_reg1, tmp_reg1, #lval +| SAFE_MEM_ACC_WITH_UOFFSET str, tmp_reg1, Rx(Z_REG(op1_addr)), Z_OFFSET(op1_addr), tmp_reg2 +|| } else if (Z_MODE(op1_addr) == IS_REG) { +| long_ins Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)), #lval +|| } else { +|| ZEND_UNREACHABLE(); +|| } +|.endmacro + +|.macro LONG_OP_WITH_32BIT_CONST, long_ins, op1_addr, lval +| brk #0 // TODO +|.endmacro + +|.macro LONG_OP_WITH_CONST, long_ins, op1_addr, lval +| brk #0 // TODO +|.endmacro + +|.macro GET_ZVAL_LVAL, reg, addr, tmp_reg +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +|| if (Z_LVAL_P(Z_ZV(addr)) == 0) { +| brk #0 // TODO: test +| mov Rx(reg), xzr +|| } else { +| brk #0 // TODO: test +| LOAD_64BIT_VAL Rx(reg), Z_LVAL_P(Z_ZV(addr)) +|| } +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| SAFE_MEM_ACC_WITH_UOFFSET ldr, Rx(reg), Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg +|| } else if (Z_MODE(addr) == IS_REG) { +|| if (reg != Z_REG(addr)) { +| mov Rx(reg), Rx(Z_REG(addr)) +|| } +|| } else { +|| ZEND_UNREACHABLE(); +|| } +|.endmacro + +|.macro LONG_MATH, opcode, reg, addr, tmp_reg1, tmp_reg2 +|| switch (opcode) { +|| case ZEND_ADD: +| LONG_OP adds, reg, addr, tmp_reg1, tmp_reg2 +|| break; +|| case ZEND_SUB: +| brk #0 // LONG_OP sub, reg, addr +|| break; +|| case ZEND_MUL: +| brk #0 // LONG_OP imul, reg, addr +|| break; +|| case ZEND_BW_OR: +| brk #0 // LONG_OP or, reg, addr +|| break; +|| case ZEND_BW_AND: +| brk #0 // LONG_OP and, reg, addr +|| break; +|| case ZEND_BW_XOR: +| brk #0 // LONG_OP xor, reg, addr +|| break; +|| default: +|| ZEND_UNREACHABLE(); +|| } +|.endmacro + +|.macro LONG_MATH_REG, opcode, dst_reg, src_reg +| brk #0 // TODO +|.endmacro + +// In x86 implementation, argument 'lval' of SET_ZVAL_LVAL can be either a LONG constant +// or a register. Here, we separate it into two macros, SET_ZVAL_LVAL for the consant case, +// and SET_ZVAL_LVAL_FROM_REG for the register case. +|.macro SET_ZVAL_LVAL_FROM_REG, addr, reg, tmp_reg +|| if (Z_MODE(addr) == IS_REG) { +| mov Rx(Z_REG(addr)), reg +|| } else { +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| SAFE_MEM_ACC_WITH_UOFFSET str, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg +|| } +|.endmacro + +|.macro SET_ZVAL_LVAL, addr, lval, tmp_reg1, tmp_reg2 +|| if (lval == 0) { +| SET_ZVAL_LVAL_FROM_REG addr, xzr, tmp_reg2 +|| } else { +| LOAD_64BIT_VAL tmp_reg1, lval +| SET_ZVAL_LVAL_FROM_REG addr, tmp_reg1, tmp_reg2 +|| } +|.endmacro + +// Define SET_ZVAL_DVAL to replace SSE_SET_ZVAL_DVAL in x86 implementation. +|.macro SET_ZVAL_DVAL, addr, reg, tmp_reg +|| if (Z_MODE(addr) == IS_REG) { +|| if (reg != Z_REG(addr)) { +| brk #0 // TODO: test +| fmov Rd(Z_REG(addr)-ZREG_V0), Rd(reg-ZREG_V0) +|| } +|| } else { +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| SAFE_MEM_ACC_WITH_UOFFSET str, Rd(reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) +|| } +|.endmacro + +// Define GET_ZVAL_DVAL to replace SSE_GET_ZVAL_DVAL in x86 implementation. +|.macro GET_ZVAL_DVAL, reg, addr, tmp_reg +| brk #0 // TODO: test +|| if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) { +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| brk #0 // TODO: test +| LOAD_ADDR Rx(tmp_reg), Z_ZV(addr) +| ldr Rd(reg-ZREG_V0), [Rx(tmp_reg)] +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| brk #0 // TODO: test +| SAFE_MEM_ACC_WITH_UOFFSET ldr, Rd(reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) +|| } else if (Z_MODE(addr) == IS_REG) { +| brk #0 // TODO: test +| fmov Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) +|| } else { +|| ZEND_UNREACHABLE(); +|| } +|| } +|.endmacro + +|.macro ZVAL_COPY_CONST, dst_addr, dst_info, dst_def_info, zv, tmp_reg1, tmp_reg2, fp_tmp_reg +|| if (Z_TYPE_P(zv) > IS_TRUE) { +|| if (Z_TYPE_P(zv) == IS_DOUBLE) { +|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg; +| LOAD_ADDR Rx(tmp_reg1), zv +| ldr Rd(dst_reg-ZREG_V0), [Rx(tmp_reg1)] +| SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 +|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { +|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg; +| brk #0 // TODO: test +|| } else { +| // In x64, if the range of this LONG value can be represented via INT type, only move the low 32 bits into dst_addr. +| // Note that imm32 is signed extended to 64 bits during mov. +| // In aarch64, we choose to handle both cases in the same way. Even though 4 mov's are used for 64-bit value and 2 mov's are +| // needed for 32-bit value, an extra ext insn is needed for 32-bit vlaue. +| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) +|| } +|| } +|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { +|| if (dst_def_info == MAY_BE_DOUBLE) { +| brk #0 // TODO: test +|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { +| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2) +|| } +|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<=0 && val <= MAX_IMM12); +| ldrb tmp_reg, [zv, #offsetof(zval, u1.v.type)] +| IF_NOT_TYPE tmp_reg, val, label +|.endmacro + +|.macro CMP_ZVAL_TYPE, addr, val +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| cmp byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val +|.endmacro + +|.macro IF_ZVAL_TYPE, addr, val, label, tmp_reg1, tmp_reg2 +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +|| ZEND_ASSERT(val <= MAX_IMM12); +| SAFE_MEM_ACC_WITH_UOFFSET ldrb, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), tmp_reg2 +| IF_TYPE tmp_reg1, val, label +|.endmacro + +|.macro IF_NOT_ZVAL_TYPE, addr, val, label, tmp_reg1, tmp_reg2 +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +|| ZEND_ASSERT(val <= MAX_IMM12); +| SAFE_MEM_ACC_WITH_UOFFSET ldrb, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), tmp_reg2 +| IF_NOT_TYPE tmp_reg1, val, label +|.endmacro + +|.macro IF_FLAGS, type_flags, mask, label +| tst type_flags, #mask +| bne label +|.endmacro + +|.macro IF_NOT_FLAGS, type_flags, mask, label +| tst type_flags, #mask +| beq label +|.endmacro + +|.macro IF_REFCOUNTED, type_flags, label +| IF_FLAGS type_flags, IS_TYPE_REFCOUNTED, label +|.endmacro + +|.macro IF_NOT_REFCOUNTED, type_flags, label +| //IF_NOT_FLAGS type_flags, IS_TYPE_REFCOUNTED, label +| tst type_flags, type_flags +| beq label +|.endmacro + +|.macro IF_ZVAL_FLAGS, addr, mask, label, tmp_reg1, tmp_reg2 +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| SAFE_MEM_ACC_WITH_UOFFSET ldrb, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags), tmp_reg2 +| IF_FLAGS tmp_reg1, mask, label +|.endmacro + +|.macro IF_NOT_ZVAL_FLAGS, addr, mask, label, tmp_reg1, tmp_reg2 +|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); +| SAFE_MEM_ACC_WITH_UOFFSET ldrb, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags), tmp_reg2 +| IF_NOT_FLAGS tmp_reg1, mask, label +|.endmacro + +|.macro IF_ZVAL_REFCOUNTED, addr, label, tmp_reg1, tmp_reg2 +| IF_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label, tmp_reg1, tmp_reg2 +|.endmacro + +|.macro IF_NOT_ZVAL_REFCOUNTED, addr, label, tmp_reg1, tmp_reg2 +| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label, tmp_reg1, tmp_reg2 +|.endmacro + +|.macro IF_NOT_ZVAL_COLLECTABLE, addr, label, tmp_reg1, tmp_reg2 +| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_COLLECTABLE, label, tmp_reg1, tmp_reg2 +|.endmacro + +|.macro GC_ADDREF, zv, tmp_reg +| brk #0 // TODO: test +| ldr tmp_reg, [zv] +| add tmp_reg, tmp_reg, #1 +| str tmp_reg, [zv] +|.endmacro + +|.macro GC_DELREF, zv, tmp_reg +| ldr tmp_reg, [zv] +| sub tmp_reg, tmp_reg, #1 +| str tmp_reg, [zv] +|.endmacro + +|.macro IF_GC_MAY_NOT_LEAK, ptr, label, tmp_reg +| ldrh tmp_reg, [ptr, #4] +| tst tmp_reg, #(GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT)) +| bne label +|.endmacro + +|.macro ADDREF_CONST, zv, tmp_reg +| .if X64 +|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) { +| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv)) +| add dword [tmp_reg], 1 +|| } else { +| add dword [Z_LVAL_P(zv)], 1 +|| } +| .else +| add dword [Z_LVAL_P(zv)], 1 +| .endif +|.endmacro + +|.macro ADDREF_CONST_2, zv, tmp_reg +| brk #0 // TODO +|.endmacro + +|.macro TRY_ADDREF, val_info, type_flags_reg, value_ptr_reg, tmp_reg +|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { +|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { +| IF_NOT_REFCOUNTED type_flags_reg, >1 +|| } +| // brk #0 // TODO: test +| GC_ADDREF value_ptr_reg, tmp_reg +|1: +|| } +|.endmacro + +|.macro TRY_ADDREF_2, val_info, type_flags_reg, value_ptr_reg +|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { +|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { +| IF_NOT_REFCOUNTED type_flags_reg, >1 +|| } +| add dword [value_ptr_reg], 2 +|1: +|| } +|.endmacro + +|.macro ZVAL_DEREF, reg, info, tmp_reg +| brk #0 // TODO +|| if (info & MAY_BE_REF) { +| IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1, tmp_reg +| GET_Z_PTR reg, reg +| add reg, offsetof(zend_reference, val) +|1: +|| } +|.endmacro + +|.macro SET_EX_OPLINE, op, tmp_reg +|| if (op == last_valid_opline) { +|| zend_jit_use_last_valid_opline(); +| SAVE_IP +|| } else { +| ADDR_STORE EX->opline, op, tmp_reg +|| if (!GCC_GLOBAL_REGS) { +|| zend_jit_reset_last_valid_opline(); +|| } +|| } +|.endmacro + +// arg1 "zval" should be in FCARG1x +|.macro ZVAL_DTOR_FUNC, var_info, opline, tmp_reg +|| do { +|| if (has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_INDIRECT))) { +| brk #0 // TODO: test +|| } +|| if (opline) { +| SET_EX_OPLINE opline, tmp_reg +|| } +| EXT_CALL rc_dtor_func, tmp_reg +|| } while(0); +|.endmacro + +|.macro ZVAL_PTR_DTOR, addr, op_info, gc, cold, opline, tmp_reg1, tmp_reg2 +|| if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { +|| if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { +| // if (Z_REFCOUNTED_P(cv)) { +|| if (cold) { +| IF_ZVAL_REFCOUNTED addr, >1, Rw(tmp_reg1), Rx(tmp_reg2) +|.cold_code +|1: +|| } else { +| brk #0 // TODO: test. +| IF_NOT_ZVAL_REFCOUNTED addr, >4, Rw(tmp_reg1), Rx(tmp_reg2) +|| } +|| } +| // if (!Z_DELREF_P(cv)) { +| GET_ZVAL_PTR FCARG1x, addr, Rx(tmp_reg2) +| GC_DELREF FCARG1x, Rw(tmp_reg1) +|| if (RC_MAY_BE_1(op_info)) { +|| if (RC_MAY_BE_N(op_info)) { +|| if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { +| bne >3 +|| } else { +| brk #0 // TODO: test +| bne >4 +|| } +|| } +| // zval_dtor_func(r); +| ZVAL_DTOR_FUNC op_info, opline, Rx(tmp_reg1) +|| if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { +| b >4 +|| } +|3: +|| } +|| if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { +|| if ((op_info) & MAY_BE_REF) { +|| zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, offsetof(zend_reference, val)); +| IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1, Rw(tmp_reg1), Rx(tmp_reg2) +| IF_NOT_ZVAL_COLLECTABLE ref_addr, >4, Rw(tmp_reg1), Rx(tmp_reg2) +| GET_ZVAL_PTR FCARG1x, ref_addr, Rx(tmp_reg2) +|1: +|| } +| IF_GC_MAY_NOT_LEAK FCARG1x, >4, Rw(tmp_reg1) +| // gc_possible_root(Z_COUNTED_P(z)) +| EXT_CALL gc_possible_root, Rx(tmp_reg1) +|| } +|| if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0) { +| b >4 +|.code +|| } +|4: +|| } +|.endmacro + +|.macro FREE_OP, op_type, op, op_info, cold, opline +|| if (op_type & (IS_VAR|IS_TMP_VAR)) { +| brk #0 // TODO: test +| // ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var), op_info, 0, cold, opline +|| } +|.endmacro + +|.macro SEPARATE_ARRAY, addr, op_info, cold +| brk #0 // TODO +|.endmacro + +|.macro EFREE_REG_REFERENCE +| brk #0 // TODO +|.endmacro + +|.macro EFREE_REFERENCE, ptr +| brk #0 // TODO +|.endmacro + +|.macro EMALLOC, size, op_array, opline +| brk #0 // TODO +|.endmacro + +|.macro OBJ_RELEASE, reg, exit_label +| brk #0 // TODO +|.endmacro + +|.macro UNDEFINED_OFFSET, opline +|| if (opline == last_valid_opline) { +|| zend_jit_use_last_valid_opline(); +| call ->undefined_offset_ex +|| } else { +| SET_EX_OPLINE opline, r0 +| call ->undefined_offset +|| } +|.endmacro + +|.macro UNDEFINED_INDEX, opline +|| if (opline == last_valid_opline) { +|| zend_jit_use_last_valid_opline(); +| call ->undefined_index_ex +|| } else { +| SET_EX_OPLINE opline, r0 +| call ->undefined_index +|| } +|.endmacro + +|.macro CANNOT_ADD_ELEMENT, opline +|| if (opline == last_valid_opline) { +|| zend_jit_use_last_valid_opline(); +| call ->cannot_add_element_ex +|| } else { +| SET_EX_OPLINE opline, r0 +| call ->cannot_add_element +|| } +|.endmacro + +static bool reuse_ip = 0; +static bool delayed_call_chain = 0; +static uint32_t delayed_call_level = 0; +static const zend_op *last_valid_opline = NULL; +static bool use_last_vald_opline = 0; +static bool track_last_valid_opline = 0; +static int jit_return_label = -1; +static uint32_t current_trace_num = 0; +static uint32_t allowed_opt_flags = 0; + +static void zend_jit_track_last_valid_opline(void) +{ + use_last_vald_opline = 0; + track_last_valid_opline = 1; +} + +static void zend_jit_use_last_valid_opline(void) +{ + if (track_last_valid_opline) { + use_last_vald_opline = 1; + track_last_valid_opline = 0; + } +} + +static bool zend_jit_trace_uses_initial_ip(void) +{ + return use_last_vald_opline; +} + +static void zend_jit_set_last_valid_opline(const zend_op *target_opline) +{ + if (!reuse_ip) { + track_last_valid_opline = 0; + last_valid_opline = target_opline; + } +} + +static void zend_jit_reset_last_valid_opline(void) +{ + track_last_valid_opline = 0; + last_valid_opline = NULL; +} + +static void zend_jit_start_reuse_ip(void) +{ + zend_jit_reset_last_valid_opline(); + reuse_ip = 1; +} + +static int zend_jit_reuse_ip(dasm_State **Dst) +{ + if (!reuse_ip) { + zend_jit_start_reuse_ip(); + | // call = EX(call); + | ldr RX, EX->call + } + return 1; +} + +static void zend_jit_stop_reuse_ip(void) +{ + reuse_ip = 0; +} + +/* bit helpers */ + +/* from http://aggregate.org/MAGIC/ */ +static uint32_t ones32(uint32_t x) +{ + x -= ((x >> 1) & 0x55555555); + x = (((x >> 2) & 0x33333333) + (x & 0x33333333)); + x = (((x >> 4) + x) & 0x0f0f0f0f); + x += (x >> 8); + x += (x >> 16); + return x & 0x0000003f; +} + +static uint32_t floor_log2(uint32_t x) +{ + ZEND_ASSERT(x != 0); + x |= (x >> 1); + x |= (x >> 2); + x |= (x >> 4); + x |= (x >> 8); + x |= (x >> 16); + return ones32(x) - 1; +} + +static bool is_power_of_two(uint32_t x) +{ + return !(x & (x - 1)) && x != 0; +} + +static bool has_concrete_type(uint32_t value_type) +{ + return is_power_of_two (value_type & (MAY_BE_ANY|MAY_BE_UNDEF)); +} + +static uint32_t concrete_type(uint32_t value_type) +{ + return floor_log2(value_type & (MAY_BE_ANY|MAY_BE_UNDEF)); +} + +static inline bool is_signed(double d) +{ + return (((unsigned char*)&d)[sizeof(double)-1] & 0x80) != 0; +} + +static int zend_jit_interrupt_handler_stub(dasm_State **Dst) +{ + |->interrupt_handler: + | brk #0 // TODO + + return 1; +} + +static int zend_jit_exception_handler_stub(dasm_State **Dst) +{ + |->exception_handler: + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + const void *handler = zend_get_opcode_handler_func(EG(exception_op)); + + | ADD_HYBRID_SPAD + | EXT_CALL handler, TMP1 + | JMP_IP TMP1 + } else { + const void *handler = EG(exception_op)->handler; + + if (GCC_GLOBAL_REGS) { + | add sp, sp, SPAD // stack alignment + | EXT_JMP handler, TMP1 + } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + | brk #0 // TODO: test + } else { + | mov FCARG1x, FP + | ldp FP, RX, T2 // retore FP and IP + | ldr LR, T4 // retore LR + | add sp, sp, NR_SPAD // stack alignment + | EXT_JMP handler, TMP1 + } + } + + return 1; +} + +static int zend_jit_exception_handler_undef_stub(dasm_State **Dst) +{ + |->exception_handler_undef: + | brk #0 // TODO + + return 1; +} + +static int zend_jit_leave_function_stub(dasm_State **Dst) +{ + |->leave_function_handler: + | brk #0 // TODO: test + + return 1; +} + +static int zend_jit_leave_throw_stub(dasm_State **Dst) +{ + |->leave_throw_handler: + | brk #0 // TODO: test + + return 1; +} + +static int zend_jit_icall_throw_stub(dasm_State **Dst) +{ + |->icall_throw_handler: + | brk #0 // TODO + + return 1; +} + +static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) +{ + |->throw_cannot_pass_by_ref: + | brk #0 // TODO + + return 1; +} + +static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) +{ + |->undefined_offset_ex: + | brk #0 // TODO + + return 1; +} + +static int zend_jit_undefined_offset_stub(dasm_State **Dst) +{ + |->undefined_offset: + | brk #0 // TODO + + return 1; +} + +static int zend_jit_undefined_index_ex_stub(dasm_State **Dst) +{ + |->undefined_index_ex: + | SAVE_IP + | b ->undefined_index + + return 1; +} + +static int zend_jit_undefined_index_stub(dasm_State **Dst) +{ + |->undefined_index: + | brk #0 // TODO + + return 1; +} + +static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) +{ + |->cannot_add_element_ex: + | brk #0 // TODO + + return 1; +} + +static int zend_jit_cannot_add_element_stub(dasm_State **Dst) +{ + |->cannot_add_element: + | brk #0 // TODO + + return 1; +} + +static int zend_jit_undefined_function_stub(dasm_State **Dst) +{ + |->undefined_function: + | brk #0 // TODO + return 1; +} + +static int zend_jit_negative_shift_stub(dasm_State **Dst) +{ + |->negative_shift: + | brk #0 // TODO + return 1; +} + +static int zend_jit_mod_by_zero_stub(dasm_State **Dst) +{ + |->mod_by_zero: + | brk #0 // TODO + return 1; +} + +static int zend_jit_invalid_this_stub(dasm_State **Dst) +{ + |->invalid_this: + | brk #0 // TODO + return 1; +} + +static int zend_jit_double_one_stub(dasm_State **Dst) +{ + |->one: + | brk #0 // TODO + return 1; +} + +static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst) +{ + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { + return 1; + } + + |->hybrid_runtime_jit: + | EXT_CALL zend_runtime_jit, TMP1 + | JMP_IP TMP1 + return 1; +} + +static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) +{ + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { + return 1; + } + + |->hybrid_profile_jit: + | brk #0 // TODO + return 1; +} + +static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst) +{ + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { + return 1; + } + + |->hybrid_hot_code: + | brk #0 // TODO + return 1; +} + +/* + * This code is based Mike Pall's "Hashed profile counters" idea, implemented + * in LuaJIT. The full description may be found in "LuaJIT 2.0 intellectual + * property disclosure and research opportunities" email + * at http://lua-users.org/lists/lua-l/2009-11/msg00089.html + * + * In addition we use a variation of Knuth's multiplicative hash function + * described at https://code.i-harness.com/en/q/a21ce + * + * uint64_t hash(uint64_t x) { + * x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9; + * x = (x ^ (x >> 27)) * 0x94d049bb133111eb; + * x = x ^ (x >> 31); + * return x; + * } + * + * uint_32_t hash(uint32_t x) { + * x = ((x >> 16) ^ x) * 0x45d9f3b; + * x = ((x >> 16) ^ x) * 0x45d9f3b; + * x = (x >> 16) ^ x; + * return x; + * } + * + */ +static int zend_jit_hybrid_hot_counter_stub(dasm_State **Dst, uint32_t cost) +{ + | brk #0 // TODO + return 1; +} + +static int zend_jit_hybrid_func_hot_counter_stub(dasm_State **Dst) +{ + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { + return 1; + } + + |->hybrid_func_hot_counter: + + return zend_jit_hybrid_hot_counter_stub(Dst, + ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); +} + +static int zend_jit_hybrid_loop_hot_counter_stub(dasm_State **Dst) +{ + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { + return 1; + } + + |->hybrid_loop_hot_counter: + + return zend_jit_hybrid_hot_counter_stub(Dst, + ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); +} + +static int zend_jit_hybrid_hot_trace_stub(dasm_State **Dst) +{ + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { + return 1; + } + + // On entry from counter stub: + // TMP4 -> zend_op_trace_info.counter + + |->hybrid_hot_trace: + | mov TMP1w, #ZEND_JIT_COUNTER_INIT + | strh TMP1w, [TMP4] + | mov FCARG1x, FP + | GET_IP FCARG2x + | EXT_CALL zend_jit_trace_hot_root, TMP1 + | cmp RETVALw, #0 // Result is < 0 on failure. + | blt >1 + | MEM_LOAD_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 + | LOAD_IP + | JMP_IP TMP1 + |1: + | EXT_JMP zend_jit_halt_op->handler, TMP1 + + return 1; +} + +static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost) +{ + | ldr TMP1, EX->func + | ldr TMP2, [TMP1, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | ldr TMP2, [TMP2, #offsetof(zend_jit_op_array_trace_extension, offset)] + | add TMP3, TMP2, IP + | ldr TMP4, [TMP3, #offsetof(zend_op_trace_info, counter)] + | ldrh TMP1w, [TMP4] + | LOAD_32BIT_VAL TMP2w, cost + | sub TMP1w, TMP1w, TMP2w + | strh TMP1w, [TMP4] + | cmp TMP1w, #0 + | ble ->hybrid_hot_trace + | ldr TMP1, [TMP3, #offsetof(zend_op_trace_info, orig_handler)] + | br TMP1 + + return 1; +} + +static int zend_jit_hybrid_func_trace_counter_stub(dasm_State **Dst) +{ + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { + return 1; + } + + |->hybrid_func_trace_counter: + + return zend_jit_hybrid_trace_counter_stub(Dst, + ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); +} + +static int zend_jit_hybrid_ret_trace_counter_stub(dasm_State **Dst) +{ + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) { + return 1; + } + + |->hybrid_ret_trace_counter: + + return zend_jit_hybrid_trace_counter_stub(Dst, + ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return))); +} + +static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst) +{ + if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { + return 1; + } + + |->hybrid_loop_trace_counter: + + return zend_jit_hybrid_trace_counter_stub(Dst, + ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); +} + +static int zend_jit_trace_halt_stub(dasm_State **Dst) +{ + |->trace_halt: + | brk #0 // TODO + return 1; +} + +static int zend_jit_trace_exit_stub(dasm_State **Dst) +{ + |->trace_exit: + | + | // Save CPU registers(32 GP regs + 32 FP regs) on stack in the order of d31 to x0 + | + | stp d30, d31, [sp, #-16]! + | stp d28, d29, [sp, #-16]! + | stp d26, d27, [sp, #-16]! + | stp d24, d25, [sp, #-16]! + | stp d22, d23, [sp, #-16]! + | stp d20, d21, [sp, #-16]! + | stp d18, d19, [sp, #-16]! + | stp d16, d17, [sp, #-16]! + | stp d14, d15, [sp, #-16]! + | stp d12, d13, [sp, #-16]! + | stp d10, d11, [sp, #-16]! + | stp d8, d9, [sp, #-16]! + | stp d6, d7, [sp, #-16]! + | stp d4, d5, [sp, #-16]! + | stp d2, d3, [sp, #-16]! + | stp d0, d1, [sp, #-16]! + | + | str x30, [sp, #-16]! // x31 can be omitted + | stp x28, x29, [sp, #-16]! + | stp x26, x27, [sp, #-16]! + | stp x24, x25, [sp, #-16]! + | stp x22, x23, [sp, #-16]! + | stp x20, x21, [sp, #-16]! + | stp x18, x19, [sp, #-16]! + | stp x16, x17, [sp, #-16]! + | stp x14, x15, [sp, #-16]! + | stp x12, x13, [sp, #-16]! + | stp x10, x11, [sp, #-16]! + | stp x8, x9, [sp, #-16]! + | stp x6, x7, [sp, #-16]! + | stp x4, x5, [sp, #-16]! + | stp x2, x3, [sp, #-16]! + | stp x0, x1, [sp, #-16]! + | + | ldr FCARG1w, [sp, #(32 * 16)] // exit_num = POP + | mov FCARG2x, sp + | + | // EX(opline) = opline + | SAVE_IP + | // zend_jit_trace_exit(trace_num, exit_num) + | EXT_CALL zend_jit_trace_exit, TMP1 + | + | add sp, sp, #(33 * 16) // including the pre-allocated 16 bytes + | + + | tst RETVALw, RETVALw + | bne >1 // not zero + + | // execute_data = EG(current_execute_data) + | MEM_LOAD_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 + | // opline = EX(opline) + | LOAD_IP + + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | ADD_HYBRID_SPAD + | JMP_IP TMP1 + } else if (GCC_GLOBAL_REGS) { + | add sp, sp, SPAD // stack alignment + | JMP_IP TMP1 + } else { + | ldp FP, RX, T2 // retore FP and IP + | ldr LR, T4 // retore LR + | add sp, sp, NR_SPAD // stack alignment + | mov RETVALx, #1 // ZEND_VM_ENTER + | ret + } + + |1: + | brk #0 // TODO: test + | blt ->trace_halt + + | // execute_data = EG(current_execute_data) + | MEM_LOAD_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 + | // opline = EX(opline) + | LOAD_IP + + | // check for interrupt (try to avoid this ???) + | MEM_LOAD_CMP_ZTS ldrb, wzr, executor_globals, vm_interrupt, TMP1w, TMP2 + | bne ->interrupt_handler + + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | ADD_HYBRID_SPAD + | ldr TMP1, EX->func + | ldr TMP1, [TMP1, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | ldr TMP1, [TMP1, #offsetof(zend_jit_op_array_trace_extension, offset)] + | add TMP1, IP, TMP1 + | ldr TMP1, [TMP1] + | br TMP1 + } else if (GCC_GLOBAL_REGS) { + | add sp, sp, SPAD // stack alignment + | ldr TMP1, EX->func + | ldr TMP1, [TMP1, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | ldr TMP1, [TMP1, #offsetof(zend_jit_op_array_trace_extension, offset)] + | add TMP1, IP, TMP1 + | ldr TMP1, [TMP1] + | br TMP1 + | + | tst RETVALw, RETVALw + | blt ->trace_halt + | + | ldp FP, RX, T2 // retore FP and IP + | ldr LR, T4 // retore LR + | add sp, sp, NR_SPAD // stack alignment + | mov RETVALx, #1 // ZEND_VM_ENTER + | ret + } + + return 1; +} + +static int zend_jit_trace_escape_stub(dasm_State **Dst) +{ + |->trace_escape: + | + | brk #0 // TODO + + return 1; +} + +/* Keep 32 exit points in a single code block */ +#define ZEND_JIT_EXIT_POINTS_SPACING 12 // mov + strb + b = bytes +#define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points + +static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n) +{ + uint32_t i; + + for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP - 1; i++) { + | mov TMP1w, #i + | strb TMP1w, [sp, #-16]! + | b >1 + } + | mov TMP1w, #i + | strb TMP1w, [sp, #-16]! + |1: + | ldrb TMP1w, [sp] + | LOAD_32BIT_VAL TMP2w, n + | add TMP2w, TMP2w, TMP1w + | str TMP2w, [sp] + | b ->trace_exit + + return 1; +} + +#ifdef CONTEXT_THREADED_JIT +static int zend_jit_context_threaded_call_stub(dasm_State **Dst) +{ + |->context_threaded_call: + | brk #0 // TODO + return 1; +} +#endif + +static int zend_jit_assign_to_variable(dasm_State **Dst, + const zend_op *opline, + zend_jit_addr var_use_addr, + zend_jit_addr var_addr, + uint32_t var_info, + uint32_t var_def_info, + zend_uchar val_type, + zend_jit_addr val_addr, + uint32_t val_info, + zend_jit_addr res_addr, + bool check_exception); + +static int zend_jit_assign_const_stub(dasm_State **Dst) +{ + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2x, 0); + uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; + + |->assign_const: + | brk #0 // TODO + if (!zend_jit_assign_to_variable( + Dst, NULL, + var_addr, var_addr, -1, -1, + IS_CONST, val_addr, val_info, + 0, 0)) { + return 0; + } + | ret + return 1; +} + +static int zend_jit_assign_tmp_stub(dasm_State **Dst) +{ + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2x, 0); + uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; + + |->assign_tmp: + | brk #0 // TODO + if (!zend_jit_assign_to_variable( + Dst, NULL, + var_addr, var_addr, -1, -1, + IS_TMP_VAR, val_addr, val_info, + 0, 0)) { + return 0; + } + | ret + return 1; +} + +static int zend_jit_assign_var_stub(dasm_State **Dst) +{ + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2x, 0); + uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF; + + |->assign_var: + | brk #0 // TODOa + | ret + return 1; +} + +static int zend_jit_assign_cv_noref_stub(dasm_State **Dst) +{ + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2x, 0); + uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/; + + |->assign_cv_noref: + | brk #0 // TODO + | ret + return 1; +} + +static int zend_jit_assign_cv_stub(dasm_State **Dst) +{ + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2x, 0); + uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/; + + |->assign_cv: + | brk #0 // TODO + | ret + return 1; +} + +static const zend_jit_stub zend_jit_stubs[] = { + JIT_STUB(interrupt_handler), + JIT_STUB(exception_handler), + JIT_STUB(exception_handler_undef), + JIT_STUB(leave_function), + JIT_STUB(leave_throw), + JIT_STUB(icall_throw), + JIT_STUB(throw_cannot_pass_by_ref), + JIT_STUB(undefined_offset), + JIT_STUB(undefined_index), + JIT_STUB(cannot_add_element), + JIT_STUB(undefined_offset_ex), + JIT_STUB(undefined_index_ex), + JIT_STUB(cannot_add_element_ex), + JIT_STUB(undefined_function), + JIT_STUB(negative_shift), + JIT_STUB(mod_by_zero), + JIT_STUB(invalid_this), + JIT_STUB(trace_halt), + JIT_STUB(trace_exit), + JIT_STUB(trace_escape), + JIT_STUB(hybrid_runtime_jit), + JIT_STUB(hybrid_profile_jit), + JIT_STUB(hybrid_hot_code), + JIT_STUB(hybrid_func_hot_counter), + JIT_STUB(hybrid_loop_hot_counter), + JIT_STUB(hybrid_hot_trace), + JIT_STUB(hybrid_func_trace_counter), + JIT_STUB(hybrid_ret_trace_counter), + JIT_STUB(hybrid_loop_trace_counter), + JIT_STUB(assign_const), + JIT_STUB(assign_tmp), + JIT_STUB(assign_var), + JIT_STUB(assign_cv_noref), + JIT_STUB(assign_cv), + JIT_STUB(double_one), +#ifdef CONTEXT_THREADED_JIT + JIT_STUB(context_threaded_call), +#endif +}; + +#if ZTS && defined(ZEND_WIN32) +extern uint32_t _tls_index; +extern char *_tls_start; +extern char *_tls_end; +#endif + +static int zend_jit_setup(void) +{ + allowed_opt_flags = 0; + +#if ZTS +# ifdef _WIN64 + tsrm_tls_index = _tls_index * sizeof(void*); + + /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ + /* Probably, it might be better solution */ + do { + void ***tls_mem = ((void**)__readgsqword(0x58))[_tls_index]; + void *val = _tsrm_ls_cache; + size_t offset = 0; + size_t size = (char*)&_tls_end - (char*)&_tls_start; + + while (offset < size) { + if (*tls_mem == val) { + tsrm_tls_offset = offset; + break; + } + tls_mem++; + offset += sizeof(void*); + } + if (offset >= size) { + // TODO: error message ??? + return FAILURE; + } + } while(0); +# elif ZEND_WIN32 + tsrm_tls_index = _tls_index * sizeof(void*); + + /* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */ + /* Probably, it might be better solution */ + do { + void ***tls_mem = ((void***)__readfsdword(0x2c))[_tls_index]; + void *val = _tsrm_ls_cache; + size_t offset = 0; + size_t size = (char*)&_tls_end - (char*)&_tls_start; + + while (offset < size) { + if (*tls_mem == val) { + tsrm_tls_offset = offset; + break; + } + tls_mem++; + offset += sizeof(void*); + } + if (offset >= size) { + // TODO: error message ??? + return FAILURE; + } + } while(0); +# elif defined(__APPLE__) && defined(__x86_64__) + tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); + if (tsrm_ls_cache_tcb_offset == 0) { + size_t *ti; + __asm__( + "leaq __tsrm_ls_cache(%%rip),%0" + : "=r" (ti)); + tsrm_tls_offset = ti[2]; + tsrm_tls_index = ti[1] * 8; + } +# elif defined(__GNUC__) && defined(__x86_64__) + tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); + if (tsrm_ls_cache_tcb_offset == 0) { +#if defined(__has_attribute) && __has_attribute(tls_model) && !defined(__FreeBSD__) + size_t ret; + + asm ("movq _tsrm_ls_cache@gottpoff(%%rip),%0" + : "=r" (ret)); + tsrm_ls_cache_tcb_offset = ret; +#else + size_t *ti; + + __asm__( + "leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n" + : "=a" (ti)); + tsrm_tls_offset = ti[1]; + tsrm_tls_index = ti[0] * 16; +#endif + } +# elif defined(__GNUC__) && defined(__i386__) + tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); + if (tsrm_ls_cache_tcb_offset == 0) { +#if !defined(__FreeBSD__) + size_t ret; + + asm ("leal _tsrm_ls_cache@ntpoff,%0\n" + : "=a" (ret)); + tsrm_ls_cache_tcb_offset = ret; +#else + size_t *ti, _ebx, _ecx, _edx; + + __asm__( + "call 1f\n" + ".subsection 1\n" + "1:\tmovl (%%esp), %%ebx\n\t" + "ret\n" + ".previous\n\t" + "addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t" + "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n\t" + "call ___tls_get_addr@plt\n\t" + "leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n" + : "=a" (ti), "=&b" (_ebx), "=&c" (_ecx), "=&d" (_edx)); + tsrm_tls_offset = ti[1]; + tsrm_tls_index = ti[0] * 8; +#endif + } +# endif +#endif + + return SUCCESS; +} + +static ZEND_ATTRIBUTE_UNUSED int zend_jit_trap(dasm_State **Dst) +{ + | brk #0 + return 1; +} + +static int zend_jit_align_func(dasm_State **Dst) +{ + reuse_ip = 0; + delayed_call_chain = 0; + last_valid_opline = NULL; + use_last_vald_opline = 0; + track_last_valid_opline = 0; + jit_return_label = -1; + |.align 16 + return 1; +} + +static int zend_jit_prologue(dasm_State **Dst) +{ + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | SUB_HYBRID_SPAD + } else if (GCC_GLOBAL_REGS) { + | sub sp, sp, SPAD // stack alignment + } else { + | sub sp, sp, NR_SPAD // stack alignment + | stp FP, RX, T2 // save FP and IP + | str LR, T4 // save LR + | mov FP, FCARG1x + } + return 1; +} + +static int zend_jit_label(dasm_State **Dst, unsigned int label) +{ + |=>label: + return 1; +} + +static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level) +{ + | // call->prev_execute_data = EX(call); + if (call_level == 1) { + | str xzr, EX:RX->prev_execute_data + } else { + | brk #0 // TODO: test + | ldr TMP1, EX->call + | str TMP1, EX:RX->prev_execute_data + } + | // EX(call) = call; + | str RX, EX->call + + delayed_call_chain = 0; + + return 1; +} + +static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline) +{ + if (last_valid_opline == opline) { + zend_jit_use_last_valid_opline(); + } else if (GCC_GLOBAL_REGS && last_valid_opline) { + zend_jit_use_last_valid_opline(); + | LOAD_64BIT_VAL TMP1, (opline - last_valid_opline) * sizeof(zend_op) + | ADD_IP TMP1, TMP2 + } else { + | LOAD_IP_ADDR opline + } + zend_jit_set_last_valid_opline(opline); + + return 1; +} + +static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) +{ + if (delayed_call_chain) { + | brk #0 // TODO: test + if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { + return 0; + } + } + if (!zend_jit_set_ip(Dst, opline)) { + return 0; + } + reuse_ip = 0; + return 1; +} + +static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const void *exit_addr) +{ + // TODO: not implemented. + return 1; +} + +static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void *timeout_exit_addr) +{ + if (timeout_exit_addr) { + | MEM_LOAD_CMP_ZTS ldrb, wzr, executor_globals, vm_interrupt, TMP1w, TMP2 + | beq =>loop_label + | EXT_JMP timeout_exit_addr, TMP1 + } else { + | brk #0 // TODO + | b =>loop_label + } + return 1; +} + +static int zend_jit_check_exception(dasm_State **Dst) +{ + | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, TMP1, TMP2 + | bne ->exception_handler + return 1; +} + +static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline) +{ + if (opline->result_type & (IS_TMP_VAR|IS_VAR)) { + | brk #0 // TODO + return 1; + } + return zend_jit_check_exception(Dst); +} + +static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_trace_info *parent, uint32_t exit_num) +{ + zend_regset regset = ZEND_REGSET_SCRATCH; + + // In the x86 implementation, this clause would be conducted if ZTS is enabled or the addressing mode is 64-bit. + { + /* assignment to EG(jit_trace_num) shouldn't clober CPU register used by deoptimizer */ + if (parent) { + int i; + int parent_vars_count = parent->exit_info[exit_num].stack_size; + zend_jit_trace_stack *parent_stack = + parent->stack_map + + parent->exit_info[exit_num].stack_offset; + + for (i = 0; i < parent_vars_count; i++) { + if (STACK_REG(parent_stack, i) != ZREG_NONE) { + if (STACK_REG(parent_stack, i) < ZREG_NUM) { + ZEND_REGSET_EXCL(regset, STACK_REG(parent_stack, i)); + } else if (STACK_REG(parent_stack, i) == ZREG_ZVAL_COPY_GPR0) { + ZEND_REGSET_EXCL(regset, ZREG_X0); + } + } + } + } + } + + if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) { + ZEND_REGSET_EXCL(regset, ZREG_X0); + } + + current_trace_num = trace_num; + + | // EG(jit_trace_num) = trace_num; + if (regset == ZEND_REGSET_EMPTY || ZEND_REGSET_IS_SINGLETON(regset)) { + | sub sp, sp, #16 + | stp TMP1, TMP2, [sp] // save TMP1 and TMP2 + | LOAD_32BIT_VAL TMP1w, trace_num + | MEM_STORE_ZTS str, TMP1w, executor_globals, jit_trace_num, TMP2 + | ldp TMP1, TMP2, [sp] // retore TMP1 and TMP2 + | add sp, sp, #16 + } else { + zend_reg tmp1 = ZEND_REGSET_FIRST(regset); + zend_reg tmp2 = ZEND_REGSET_SECOND(regset); + + | LOAD_32BIT_VAL Rw(tmp1), trace_num + | MEM_STORE_ZTS str, Rw(tmp1), executor_globals, jit_trace_num, Rx(tmp2) + (void)tmp1; + (void)tmp2; + } + + return 1; +} + +typedef ZEND_SET_ALIGNED(1, uint16_t unaligned_uint16_t); +typedef ZEND_SET_ALIGNED(1, int32_t unaligned_int32_t); + +static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr) +{ + int ret = 0; + uint8_t *p, *end; + + abort(); // TODO + return ret; +} + +static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr) +{ + return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr); +} + +static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr) +{ + const void *link_addr; + size_t prologue_size; + + | brk #0 // TODO + return 1; +} + +static int zend_jit_trace_return(dasm_State **Dst, bool original_handler) +{ + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | ADD_HYBRID_SPAD + if (!original_handler) { + | JMP_IP TMP1 + } else { + | brk #0 // TODO: test + | ldr TMP1, EX->func + | ldr TMP1, [TMP1, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | ldr TMP1, [TMP1, #offsetof(zend_jit_op_array_trace_extension, offset)] + | add TMP1, IP, TMP1 + | ldr TMP1, [TMP1] + | br TMP1 + } + } else if (GCC_GLOBAL_REGS) { + | add sp, sp, SPAD // stack alignment + if (!original_handler) { + | JMP_IP TMP1 + } else { + | brk #0 // TODO: test + | ldr TMP1, EX->func + | ldr TMP1, [TMP1, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | ldr TMP1, [TMP1, #offsetof(zend_jit_op_array_trace_extension, offset)] + | add TMP1, IP, TMP1 + | ldr TMP1, [TMP1] + | br TMP1 + } + } else { + if (original_handler) { + | brk #0 // TODO: test + | mov FCARG1x, FP + | ldr TMP1, EX->func + | ldr TMP1, [TMP1, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | ldr TMP1, [TMP1, #offsetof(zend_jit_op_array_trace_extension, offset)] + | add TMP1, IP, TMP1 + | ldr TMP1, [TMP1] + | blr TMP1 + } + | ldp FP, RX, T2 // retore FP and IP + | ldr LR, T4 // retore LR + | add sp, sp, NR_SPAD // stack alignment + | mov RETVALx, #2 // ZEND_VM_LEAVE + | ret + } + return 1; +} + +static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint8_t type) +{ + int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + + if (var > MAX_IMM12) { + | LOAD_32BIT_VAL TMP1, var + | add TMP1, FP, TMP1 + } else { + | add TMP1, FP, #var + } + | IF_NOT_Z_TYPE TMP1, type, >1, TMP2w + |.cold_code + |1: + | EXT_JMP exit_addr, TMP1 + |.code + + return 1; +} + +static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint32_t op_info) +{ + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + + | brk #0 // TODO + + return 1; +} + +static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace) +{ + zend_jit_op_array_trace_extension *jit_extension = + (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + size_t offset = jit_extension->offset; + const void *handler = + (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; + + if (!zend_jit_set_valid_ip(Dst, opline)) { + return 0; + } + + | brk #0 // TODO + + return 1; +} + +static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw) +{ + const void *handler; + + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + handler = zend_get_opcode_handler_func(opline); + } else { + handler = opline->handler; + } + + if (!zend_jit_set_valid_ip(Dst, opline)) { + return 0; + } + if (!GCC_GLOBAL_REGS) { + | mov FCARG1x, FP + } + | EXT_CALL handler, TMP1 + if (may_throw) { + zend_jit_check_exception(Dst); + } + + /* Skip the following OP_DATA */ + switch (opline->opcode) { + case ZEND_ASSIGN_DIM: + case ZEND_ASSIGN_OBJ: + case ZEND_ASSIGN_STATIC_PROP: + case ZEND_ASSIGN_DIM_OP: + case ZEND_ASSIGN_OBJ_OP: + case ZEND_ASSIGN_STATIC_PROP_OP: + case ZEND_ASSIGN_STATIC_PROP_REF: + case ZEND_ASSIGN_OBJ_REF: + zend_jit_set_last_valid_opline(opline + 2); + break; + default: + zend_jit_set_last_valid_opline(opline + 1); + break; + } + + return 1; +} + +static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) +{ + if (!zend_jit_set_valid_ip(Dst, opline)) { + return 0; + } + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + if (opline->opcode == ZEND_DO_UCALL || + opline->opcode == ZEND_DO_FCALL_BY_NAME || + opline->opcode == ZEND_DO_FCALL || + opline->opcode == ZEND_RETURN) { + + /* Use inlined HYBRID VM handler */ + const void *handler = opline->handler; + + | ADD_HYBRID_SPAD + | EXT_JMP handler, TMP1 + } else { + const void *handler = zend_get_opcode_handler_func(opline); + + | brk #0 // TODO: test + } + } else { + const void *handler = opline->handler; + + if (GCC_GLOBAL_REGS) { + | add sp, sp, SPAD // stack alignment + } else { + | mov FCARG1x, FP + | ldp FP, RX, T2 // retore FP and IP + | ldr LR, T4 // retore LR + | add sp, sp, NR_SPAD // stack alignment + } + | EXT_JMP handler, TMP1 + } + zend_jit_reset_last_valid_opline(); + return 1; +} + +static int zend_jit_trace_opline_guard(dasm_State **Dst, const zend_op *opline) +{ + uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | brk #0 // TODO + + zend_jit_set_last_valid_opline(opline); + + return 1; +} + +static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) +{ + | b =>target_label + return 1; +} + +static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label) +{ + | brk #0 // TODO + + zend_jit_set_last_valid_opline(next_opline); + + return 1; +} + +#ifdef CONTEXT_THREADED_JIT +static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) +{ + | brk #0 // TODO + return 1; +} +#endif + +static int zend_jit_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) +{ +#ifdef CONTEXT_THREADED_JIT + return zend_jit_context_threaded_call(Dst, opline, next_block); +#else + return zend_jit_tail_handler(Dst, opline); +#endif +} + +static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, bool set_type) +{ + ZEND_ASSERT(Z_MODE(src) == IS_REG); + ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL); + + | brk #0 // TODO + return 1; +} + +static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info) +{ + ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL); + ZEND_ASSERT(Z_MODE(dst) == IS_REG); + + if ((info & MAY_BE_ANY) == MAY_BE_LONG) { + | GET_ZVAL_LVAL Z_REG(dst), src, TMP1 + } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { + | brk #0 // TODO + } else { + ZEND_UNREACHABLE(); + } + return 1; +} + +static int zend_jit_store_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg, bool set_type) +{ + zend_jit_addr src = ZEND_ADDR_REG(reg); + zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); + + return zend_jit_spill_store(Dst, src, dst, info, set_type); +} + +static int zend_jit_store_var_if_necessary(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info) +{ + if (Z_MODE(src) == IS_REG && Z_STORE(src)) { + | brk #0 // TODO: test + zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); + return zend_jit_spill_store(Dst, src, dst, info, 1); + } + return 1; +} + +static int zend_jit_store_var_if_necessary_ex(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info) +{ + if (Z_MODE(src) == IS_REG && Z_STORE(src)) { + zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); + bool set_type = 1; + + if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == + (old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) { + if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) { + set_type = 0; + } + } + return zend_jit_spill_store(Dst, src, dst, info, set_type); + } + return 1; +} + +static int zend_jit_load_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg) +{ + zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); + zend_jit_addr dst = ZEND_ADDR_REG(reg); + + return zend_jit_load_reg(Dst, src, dst, info); +} + +static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info) +{ + if (!zend_jit_same_addr(src, dst)) { + | brk #0 // TODO: test + } + return 1; +} + +static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags, const zend_op *opline) +{ + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_X0, 0); // COPY_GPR0, use X0 here + + | brk #0 // TODO + + return 1; +} + +static int zend_jit_store_const(dasm_State **Dst, int var, zend_reg reg) +{ + zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); + + | brk #0 // TODO + return 1; +} + +static int zend_jit_free_trampoline(dasm_State **Dst) +{ + | brk #0 // TODO + return 1; +} + +static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) +{ + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, TMP1w, TMP2 + } + if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { + | brk #0 // TODO: test + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_TMP1, ZREG_TMP2, ZREG_TMP3, ZREG_TMP4, ZREG_FPTMP1 + } + if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, MAY_BE_LONG)) { + return 0; + } + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + | LONG_OP_WITH_CONST_IMM12 adds, op1_def_addr, Z_L(1), TMP1, TMP2 + } else { + | brk #0 // TODO: test + | LONG_OP_WITH_CONST_IMM12 subs, op1_def_addr, Z_L(1), TMP1, TMP2 + } + + if (may_overflow && + (((op1_def_info & MAY_BE_GUARD) && (op1_def_info & MAY_BE_LONG)) || + ((opline->result_type != IS_UNUSED && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG))))) { + int32_t exit_point; + const void *exit_addr; + zend_jit_trace_stack *stack; + uint32_t old_op1_info, old_res_info = 0; + + | brk #0 // TODO: test + } else if (may_overflow) { + | bvs >1 + if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && + opline->result_type != IS_UNUSED) { + | brk #0 // TODO: test + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_TMP1, ZREG_TMP2, ZREG_TMP3, ZREG_TMP4, ZREG_FPTMP1 + } + |.cold_code + |1: + | brk #0 // TODO: test + | b >3 + |.code + } else { + if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && + opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_TMP1, ZREG_TMP2, ZREG_TMP3, ZREG_TMP4, ZREG_FPTMP1 + } + } + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { + |.cold_code + |2: + | brk #0 // TODO: test + | b >3 + |.code + } + |3: + if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) { + return 0; + } + if (opline->result_type != IS_UNUSED) { + if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { + return 0; + } + } + return 1; +} + +static int zend_jit_opline_uses_reg(const zend_op *opline, int8_t reg) +{ + if ((opline+1)->opcode == ZEND_OP_DATA + && ((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) + && JIT_G(current_frame)->stack[EX_VAR_TO_NUM((opline+1)->op1.var)].reg == reg) { + return 1; + } + return + ((opline->result_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && + JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->result.var)].reg == reg) || + ((opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && + JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op1.var)].reg == reg) || + ((opline->op2_type & (IS_VAR|IS_TMP_VAR|IS_CV)) && + JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op2.var)].reg == reg); +} + +static int zend_jit_math_long_long(dasm_State **Dst, + const zend_op *opline, + zend_uchar opcode, + zend_jit_addr op1_addr, + zend_jit_addr op2_addr, + zend_jit_addr res_addr, + uint32_t res_info, + uint32_t res_use_info, + int may_overflow) +{ + bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); + zend_reg result_reg; + + // x86 defines a 'tmp_reg' to handle integer overflow case. + // In AArch64, we directly use our reserved TMP1. + // zend_reg tmp_reg = ZREG_X0; + + if (Z_MODE(res_addr) == IS_REG && (res_info & MAY_BE_LONG)) { + if (may_overflow && (res_info & MAY_BE_GUARD) + && JIT_G(current_frame) + && zend_jit_opline_uses_reg(opline, Z_REG(res_addr))) { + result_reg = ZREG_TMP3; // to store the result temporarily. Use TMP3 + } else { + result_reg = Z_REG(res_addr); + } + } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { + | brk #0 // TODO: test + result_reg = Z_REG(op1_addr); + } else if (Z_REG(res_addr) != ZREG_X0) { + result_reg = ZREG_TMP3; // Use TMP3 + } else { + | brk #0 // TODO: test + /* ASSIGN_DIM_OP */ + result_reg = ZREG_FCARG1x; + } + + if (opcode == ZEND_MUL && + ((Z_MODE(op2_addr) == IS_CONST_ZVAL && + IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op2_addr))) && + is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) || + (Z_MODE(op1_addr) == IS_CONST_ZVAL && + IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr))) && + is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))))) { + | brk #0 // TODO: test + } else if (opcode == ZEND_DIV && + (Z_MODE(op2_addr) == IS_CONST_ZVAL && + is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { + | brk #0 // TODO: test + } else if (opcode == ZEND_ADD && + !may_overflow && + Z_MODE(op1_addr) == IS_REG && + Z_MODE(op2_addr) == IS_CONST_ZVAL) { + | brk #0 // TODO: test + } else if (opcode == ZEND_ADD && + !may_overflow && + Z_MODE(op2_addr) == IS_REG && + Z_MODE(op1_addr) == IS_CONST_ZVAL) { + | brk #0 // TODO: test + } else if (opcode == ZEND_SUB && + !may_overflow && + Z_MODE(op1_addr) == IS_REG && + Z_MODE(op2_addr) == IS_CONST_ZVAL) { + | brk #0 // TODO: test + } else { + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + if ((opcode == ZEND_ADD || opcode == ZEND_SUB) + && Z_MODE(op2_addr) == IS_CONST_ZVAL + && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { + /* +/- 0 */ + may_overflow = 0; + } else if (same_ops && opcode != ZEND_DIV) { + | brk #0 // TODO: test + } else { + | LONG_MATH opcode, result_reg, op2_addr, TMP1, TMP2 + } + } + if (may_overflow) { + if (res_info & MAY_BE_GUARD) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) { + | bvs >3 + |.cold_code + |3: + | EXT_JMP exit_addr, TMP1 + |.code + if (Z_MODE(res_addr) == IS_REG && result_reg != Z_REG(res_addr)) { + | mov Rx(Z_REG(res_addr)), Rx(result_reg) + } + } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { + | brk #0 // TODO + } else { + ZEND_UNREACHABLE(); + } + } else { + if (res_info & MAY_BE_LONG) { + | bvs >1 + } else { + | brk #0 // TODO: test + } + } + } + + if (Z_MODE(res_addr) == IS_MEM_ZVAL && (res_info & MAY_BE_LONG)) { + | SET_ZVAL_LVAL_FROM_REG res_addr, Rx(result_reg), TMP1 + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 + } + } + } + + if (may_overflow && (!(res_info & MAY_BE_GUARD) || (res_info & MAY_BE_ANY) == MAY_BE_DOUBLE)) { + if (res_info & MAY_BE_LONG) { + |.cold_code + |1: + } + + do { + if ((Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 1) || + (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1)) { + if (opcode == ZEND_ADD) { + if (Z_MODE(res_addr) == IS_REG) { + | brk #0 // TODO: test + } else { + | SET_ZVAL_LVAL res_addr, 0x43e0000000000000, TMP1, TMP2 + } + break; + } else if (opcode == ZEND_SUB) { + | brk #0 // TODO: test + break; + } + } + + | DOUBLE_GET_ZVAL_LVAL ZREG_FPTMP1, op1_addr, ZREG_TMP1, ZREG_TMP2 + | DOUBLE_GET_ZVAL_LVAL ZREG_FPTMP2, op2_addr, ZREG_TMP1, ZREG_TMP2 + | DOUBLE_MATH_REG opcode, ZREG_FPTMP1, ZREG_FPTMP1, ZREG_FPTMP2 + | SET_ZVAL_DVAL res_addr, ZREG_FPTMP1, ZREG_TMP1 + } while (0); + + if (Z_MODE(res_addr) == IS_MEM_ZVAL + && (res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { + | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 + } + if (res_info & MAY_BE_LONG) { + | b >2 + |.code + } + |2: + } + + return 1; +} + +static int zend_jit_math_long_double(dasm_State **Dst, + zend_uchar opcode, + zend_jit_addr op1_addr, + zend_jit_addr op2_addr, + zend_jit_addr res_addr, + uint32_t res_use_info) +{ + zend_reg result_reg = + (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_V0; + zend_reg tmp_reg; + + | brk #0 // TODO + + return 1; +} + +static int zend_jit_math_double_long(dasm_State **Dst, + zend_uchar opcode, + zend_jit_addr op1_addr, + zend_jit_addr op2_addr, + zend_jit_addr res_addr, + uint32_t res_use_info) +{ + zend_reg result_reg, tmp_reg; + + | brk #0 // TODO + return 1; +} + +static int zend_jit_math_double_double(dasm_State **Dst, + zend_uchar opcode, + zend_jit_addr op1_addr, + zend_jit_addr op2_addr, + zend_jit_addr res_addr, + uint32_t res_use_info) +{ + bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); + zend_reg result_reg; + + | brk #0 // TODO + return 1; +} + +static int zend_jit_math_helper(dasm_State **Dst, + const zend_op *opline, + zend_uchar opcode, + zend_uchar op1_type, + znode_op op1, + zend_jit_addr op1_addr, + uint32_t op1_info, + zend_uchar op2_type, + znode_op op2, + zend_jit_addr op2_addr, + uint32_t op2_info, + uint32_t res_var, + zend_jit_addr res_addr, + uint32_t res_info, + uint32_t res_use_info, + int may_overflow, + int may_throw) +/* Labels: 1,2,3,4,5,6 */ +{ + bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); + + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { + if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { + if (op1_info & MAY_BE_DOUBLE) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, TMP1w, TMP2 + } else { + | brk #0 // TODO: test + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, TMP1w, TMP2 + } + } + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { + if (op2_info & MAY_BE_DOUBLE) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1, TMP1w, TMP2 + |.cold_code + |1: + | brk #0 // TODO: test + if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, TMP1w, TMP2 + } + if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { + return 0; + } + | b >5 + |.code + } else { + | brk #0 // TODO: test + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, TMP1w, TMP2 + } + } + if (!zend_jit_math_long_long(Dst, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) { + return 0; + } + if (op1_info & MAY_BE_DOUBLE) { + |.cold_code + |3: + if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, TMP1w, TMP2 + } + | brk #0 // TODO: test + if (op2_info & MAY_BE_DOUBLE) { + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + if (!same_ops) { + | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >1, TMP1w, TMP2 + } else { + | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6, TMP1w, TMP2 + } + } + if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { + return 0; + } + | b >5 + } + if (!same_ops) { + |1: + if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, TMP1w, TMP2 + } + if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { + return 0; + } + | b >5 + } + |.code + } + } else if ((op1_info & MAY_BE_DOUBLE) && + !(op1_info & MAY_BE_LONG) && + (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + | brk #0 // TODO: test + if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, TMP1w, TMP2 + } + if (op2_info & MAY_BE_DOUBLE) { + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + if (!same_ops && (op2_info & MAY_BE_LONG)) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >1, TMP1w, TMP2 + } else { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, TMP1w, TMP2 + } + } + if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { + return 0; + } + } + if (!same_ops && (op2_info & MAY_BE_LONG)) { + if (op2_info & MAY_BE_DOUBLE) { + |.cold_code + } + |1: + if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, TMP1w, TMP2 + } + if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { + return 0; + } + if (op2_info & MAY_BE_DOUBLE) { + | b >5 + |.code + } + } + } else if ((op2_info & MAY_BE_DOUBLE) && + !(op2_info & MAY_BE_LONG) && + (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + | brk #0 // TODO: test + if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, TMP1w, TMP2 + } + if (op1_info & MAY_BE_DOUBLE) { + if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + if (!same_ops && (op1_info & MAY_BE_LONG)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >1, TMP1w, TMP2 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, TMP1w, TMP2 + } + } + if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { + return 0; + } + } + if (!same_ops && (op1_info & MAY_BE_LONG)) { + if (op1_info & MAY_BE_DOUBLE) { + |.cold_code + } + |1: + if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, TMP1w, TMP2 + } + if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { + return 0; + } + if (op1_info & MAY_BE_DOUBLE) { + | b >5 + |.code + } + } + } + + |5: + + if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || + (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { + if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && + (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + |.cold_code + } + |6: + if (Z_MODE(res_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); + | brk #0 // TODO: test + | LOAD_ZVAL_ADDR FCARG1x, real_addr + } else if (Z_REG(res_addr) != ZREG_FCARG1x || Z_OFFSET(res_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, res_addr + } + if (Z_MODE(op1_addr) == IS_REG) { + | brk #0 // TODO: test + } + | LOAD_ZVAL_ADDR FCARG2x, op1_addr + if (Z_MODE(op2_addr) == IS_REG) { + | brk #0 // TODO: test + } + | LOAD_ZVAL_ADDR CARG3, op2_addr + | SET_EX_OPLINE opline, TMP1 + if (opcode == ZEND_ADD) { + | EXT_CALL add_function, TMP1 + } else if (opcode == ZEND_SUB) { + | brk #0 // TODO: test + | EXT_CALL sub_function, TMP1 + } else if (opcode == ZEND_MUL) { + | brk #0 // TODO: test + | EXT_CALL mul_function, TMP1 + } else if (opcode == ZEND_DIV) { + | brk #0 // TODO: test + | EXT_CALL div_function, TMP1 + } else { + ZEND_UNREACHABLE(); + } + | FREE_OP op1_type, op1, op1_info, 0, opline + | FREE_OP op2_type, op2, op2_info, 0, opline + if (may_throw) { + zend_jit_check_exception(Dst); + } + if (Z_MODE(res_addr) == IS_REG) { + | brk #0 // TODO: test + } + if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && + (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + | b <5 + |.code + } + } + + return 1; +} + +static int zend_jit_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) +{ + ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); + ZEND_ASSERT((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && + (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))); + + if (!zend_jit_math_helper(Dst, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) { + return 0; + } + if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { + return 0; + } + return 1; +} + +static int zend_jit_add_arrays(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr) +{ + zend_jit_addr op1_addr = OP1_ADDR(); + zend_jit_addr op2_addr = OP2_ADDR(); + + | brk #0 // TODO + return 1; +} + +static int zend_jit_long_math_helper(dasm_State **Dst, + const zend_op *opline, + zend_uchar opcode, + zend_uchar op1_type, + znode_op op1, + zend_jit_addr op1_addr, + uint32_t op1_info, + zend_ssa_range *op1_range, + zend_uchar op2_type, + znode_op op2, + zend_jit_addr op2_addr, + uint32_t op2_info, + zend_ssa_range *op2_range, + uint32_t res_var, + zend_jit_addr res_addr, + uint32_t res_info, + uint32_t res_use_info, + int may_throw) +/* Labels: 6 */ +{ + bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); + zend_reg result_reg; + zval tmp; + + | brk #0 // TODO + + return 1; +} + +static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw) +{ + ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); + ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)); + + if (!zend_jit_long_math_helper(Dst, opline, opline->opcode, + opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, + opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, + opline->result.var, res_addr, res_info, res_use_info, may_throw)) { + return 0; + } + if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { + return 0; + } + return 1; +} + +static int zend_jit_concat_helper(dasm_State **Dst, + const zend_op *opline, + zend_uchar op1_type, + znode_op op1, + zend_jit_addr op1_addr, + uint32_t op1_info, + zend_uchar op2_type, + znode_op op2, + zend_jit_addr op2_addr, + uint32_t op2_info, + zend_jit_addr res_addr, + int may_throw) +{ + | brk #0 // TODO + + return 1; +} + +static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw) +{ + zend_jit_addr op1_addr, op2_addr; + + ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); + ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)); + + op1_addr = OP1_ADDR(); + op2_addr = OP2_ADDR(); + + return zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw); +} + +static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, uint32_t type, uint32_t op1_info, uint32_t op2_info, const void *found_exit_addr, const void *not_found_exit_addr, const void *exit_addr) +/* Labels: 1,2,3,4,5 */ +{ + zend_jit_addr op2_addr = OP2_ADDR(); + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + + | brk #0 // TODO + + return 1; +} + +static int zend_jit_simple_assign(dasm_State **Dst, + const zend_op *opline, + zend_jit_addr var_addr, + uint32_t var_info, + uint32_t var_def_info, + zend_uchar val_type, + zend_jit_addr val_addr, + uint32_t val_info, + zend_jit_addr res_addr, + int in_cold, + int save_r1) +/* Labels: 1,2,3 */ +{ + zend_reg tmp_reg; + + if (Z_MODE(var_addr) == IS_REG || Z_REG(var_addr) != ZREG_X0) { + tmp_reg = ZREG_TMP1; // TODO: same issue with zend_jit_math_long_long + } else { + /* ASSIGN_DIM */ + tmp_reg = ZREG_FCARG1x; + } + + if (Z_MODE(val_addr) == IS_CONST_ZVAL) { + zval *zv = Z_ZV(val_addr); + + if (!res_addr) { + | ZVAL_COPY_CONST var_addr, var_info, var_def_info, zv, tmp_reg, ZREG_TMP2, ZREG_FPTMP1 + } else { + | brk #0 // TODO + } + if (Z_REFCOUNTED_P(zv)) { + | brk #0 // TODO + } + } else { + | brk #0 // TODO + } + return 1; +} + +static int zend_jit_assign_to_typed_ref(dasm_State **Dst, + const zend_op *opline, + zend_uchar val_type, + zend_jit_addr val_addr, + bool check_exception) +{ + | brk #0 // TODO + + return 1; +} + +static int zend_jit_assign_to_variable_call(dasm_State **Dst, + const zend_op *opline, + zend_jit_addr __var_use_addr, + zend_jit_addr var_addr, + uint32_t __var_info, + uint32_t __var_def_info, + zend_uchar val_type, + zend_jit_addr val_addr, + uint32_t val_info, + zend_jit_addr __res_addr, + bool __check_exception) +{ + | brk #0 // TODO + + return 1; +} + +static int zend_jit_assign_to_variable(dasm_State **Dst, + const zend_op *opline, + zend_jit_addr var_use_addr, + zend_jit_addr var_addr, + uint32_t var_info, + uint32_t var_def_info, + zend_uchar val_type, + zend_jit_addr val_addr, + uint32_t val_info, + zend_jit_addr res_addr, + bool check_exception) +/* Labels: 1,2,3,4,5,8 */ +{ + int done = 0; + zend_reg ref_reg, tmp_reg; + + | brk #0 // TODO + return 1; +} + +static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, int may_throw) +{ + zend_jit_addr op2_addr, op3_addr, res_addr; + + | brk #0 // TODO + return 1; +} + +static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, int may_throw) +{ + zend_jit_addr op2_addr, op3_addr, var_addr; + + ZEND_ASSERT(opline->result_type == IS_UNUSED); + + | brk #0 // TODO + return 1; +} + +static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_ssa_range *op1_range, uint32_t op2_info, zend_ssa_range *op2_range, int may_overflow, int may_throw) +{ + zend_jit_addr op1_addr, op2_addr; + + ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED); + ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); + + | brk #0 // TODO + return 1; +} + +static int zend_jit_is_constant_cmp_long_long(const zend_op *opline, + zend_ssa_range *op1_range, + zend_jit_addr op1_addr, + zend_ssa_range *op2_range, + zend_jit_addr op2_addr, + bool *result) +{ + zend_long op1_min; + zend_long op1_max; + zend_long op2_min; + zend_long op2_max; + + if (op1_range) { + op1_min = op1_range->min; + op1_max = op1_range->max; + } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { + ZEND_ASSERT(Z_TYPE_P(Z_ZV(op1_addr)) == IS_LONG); + op1_min = op1_max = Z_LVAL_P(Z_ZV(op1_addr)); + } else { + return 0; + } + + if (op2_range) { + op2_min = op2_range->min; + op2_max = op2_range->max; + } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + ZEND_ASSERT(Z_TYPE_P(Z_ZV(op2_addr)) == IS_LONG); + op2_min = op2_max = Z_LVAL_P(Z_ZV(op2_addr)); + } else { + return 0; + } + + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) { + *result = 1; + return 1; + } else if (op1_max < op2_min || op1_min > op2_max) { + *result = 0; + return 1; + } + return 0; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) { + *result = 0; + return 1; + } else if (op1_max < op2_min || op1_min > op2_max) { + *result = 1; + return 1; + } + return 0; + case ZEND_IS_SMALLER: + if (op1_max < op2_min) { + *result = 1; + return 1; + } else if (op1_min >= op2_max) { + *result = 0; + return 1; + } + return 0; + case ZEND_IS_SMALLER_OR_EQUAL: + if (op1_max <= op2_min) { + *result = 1; + return 1; + } else if (op1_min > op2_max) { + *result = 0; + return 1; + } + return 0; + default: + ZEND_UNREACHABLE(); + } + return 0; +} + +static int zend_jit_cmp_long_long(dasm_State **Dst, + const zend_op *opline, + zend_ssa_range *op1_range, + zend_jit_addr op1_addr, + zend_ssa_range *op2_range, + zend_jit_addr op2_addr, + zend_jit_addr res_addr, + zend_uchar smart_branch_opcode, + uint32_t target_label, + uint32_t target_label2, + const void *exit_addr, + bool skip_comparison) +{ + bool swap = 0; + bool result; + + if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) { + if (!smart_branch_opcode || + smart_branch_opcode == ZEND_JMPZ_EX || + smart_branch_opcode == ZEND_JMPNZ_EX) { + | brk #0 // TODO + } + if (smart_branch_opcode && !exit_addr) { + | brk #0 // TODO + } + return 1; + } + + if (skip_comparison) { + if (Z_MODE(op1_addr) != IS_REG && + (Z_MODE(op2_addr) == IS_REG || + (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL))) { + swap = 1; + } + } else if (Z_MODE(op1_addr) == IS_REG) { + if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { + | brk #0 // TODO + } else { + | LONG_CMP Z_REG(op1_addr), op2_addr, TMP1, TMP2 + } + } else if (Z_MODE(op2_addr) == IS_REG) { + if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) { + | brk #0 // TODO + } else { + | brk #0 // TODO + } + swap = 1; + } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { + | brk #0 // TODO + swap = 1; + } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { + | brk #0 // TODO + } else { + | GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP2 + if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { + | brk #0 // TODO + } else { + | LONG_CMP ZREG_TMP1, op2_addr, TMP2, TMP3 + } + } + + if (smart_branch_opcode) { + if (smart_branch_opcode == ZEND_JMPZ_EX || + smart_branch_opcode == ZEND_JMPNZ_EX) { + + | brk #0 // TODO + } + if (smart_branch_opcode == ZEND_JMPZ || + smart_branch_opcode == ZEND_JMPZ_EX) { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + | brk #0 // TODO + break; + case ZEND_IS_NOT_EQUAL: + | brk #0 // TODO + break; + case ZEND_IS_NOT_IDENTICAL: + | brk #0 // TODO + break; + case ZEND_IS_SMALLER: + if (swap) { + | brk #0 // TODO + } else { + if (exit_addr) { + | bge >1 + |.cold_code + |1: + | EXT_JMP exit_addr, TMP1 + |.code + } else { + | brk #0 // TODO + } + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | brk #0 // TODO + break; + default: + ZEND_UNREACHABLE(); + } + } else if (smart_branch_opcode == ZEND_JMPNZ || + smart_branch_opcode == ZEND_JMPNZ_EX) { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + | brk #0 // TODO + break; + case ZEND_IS_NOT_EQUAL: + | brk #0 // TODO + break; + case ZEND_IS_NOT_IDENTICAL: + | brk #0 // TODO + break; + case ZEND_IS_SMALLER: + if (swap) { + | brk #0 // TODO + } else { + if (exit_addr) { + | brk #0 // TODO + } else { + | blt => target_label + } + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | brk #0 // TODO + break; + default: + ZEND_UNREACHABLE(); + } + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + | brk #0 // TODO + } else { + ZEND_UNREACHABLE(); + } + } else { + | brk #0 // TODO + } + + return 1; +} + +static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, bool swap, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) +{ + | brk #0 // TODO + + return 1; +} + +static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) +{ + zend_reg tmp_reg = ZREG_V0; + + | brk #0 // TODO + + return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2, exit_addr); +} + +static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) +{ + zend_reg tmp_reg = ZREG_V0; + + | brk #0 // TODO + + return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2, exit_addr); +} + +static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) +{ + bool swap = 0; + + | brk #0 // TODO + + return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2, exit_addr); +} + +static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) +{ + | brk #0 // TODO + + return 1; +} + +static int zend_jit_cmp(dasm_State **Dst, + const zend_op *opline, + uint32_t op1_info, + zend_ssa_range *op1_range, + zend_jit_addr op1_addr, + uint32_t op2_info, + zend_ssa_range *op2_range, + zend_jit_addr op2_addr, + zend_jit_addr res_addr, + int may_throw, + zend_uchar smart_branch_opcode, + uint32_t target_label, + uint32_t target_label2, + const void *exit_addr, + bool skip_comparison) +{ + bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var); + bool has_slow; + + has_slow = + (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && + (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && + ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || + (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))); + + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { + if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { + if (op1_info & MAY_BE_DOUBLE) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4, TMP1w, TMP2 + } else { + | brk #0 // TODO + } + } + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { + if (op2_info & MAY_BE_DOUBLE) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, TMP1w, TMP2 + |.cold_code + |3: + | brk #0 // TODO + |.code + } else { + | brk #0 // TODO + } + } + if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { + return 0; + } + if (op1_info & MAY_BE_DOUBLE) { + |.cold_code + |4: + | brk #0 // TODO + |.code + } + } else if ((op1_info & MAY_BE_DOUBLE) && + !(op1_info & MAY_BE_LONG) && + (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + | brk #0 // TODO + } else if ((op2_info & MAY_BE_DOUBLE) && + !(op2_info & MAY_BE_LONG) && + (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { + | brk #0 // TODO + } + + if (has_slow || + (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || + (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { + if (has_slow) { + |.cold_code + |9: + } + | brk #0 // TODO + if (has_slow) { + | b >6 + |.code + } + } + + |6: + + return 1; +} + +static int zend_jit_identical(dasm_State **Dst, + const zend_op *opline, + uint32_t op1_info, + zend_ssa_range *op1_range, + zend_jit_addr op1_addr, + uint32_t op2_info, + zend_ssa_range *op2_range, + zend_jit_addr op2_addr, + zend_jit_addr res_addr, + int may_throw, + zend_uchar smart_branch_opcode, + uint32_t target_label, + uint32_t target_label2, + const void *exit_addr, + bool skip_comparison) +{ + uint32_t identical_label = (uint32_t)-1; + uint32_t not_identical_label = (uint32_t)-1; + + | brk #0 // TODO + return 1; +} + +static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, zend_uchar branch_opcode, const void *exit_addr) +{ + uint32_t true_label = -1; + uint32_t false_label = -1; + bool set_bool = 0; + bool set_bool_not = 0; + bool set_delayed = 0; + bool jmp_done = 0; + + | brk #0 // TODO + return 1; +} + +static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr) +{ + if (op1_addr != op1_def_addr) { + if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { + return 0; + } + if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { + op1_addr = op1_def_addr; + } + } + + if (!zend_jit_simple_assign(Dst, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 0, 0)) { + return 0; + } + if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { + return 0; + } + return 1; +} + +static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_use_addr, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr op2_def_addr, uint32_t res_info, zend_jit_addr res_addr, int may_throw) +{ + ZEND_ASSERT(opline->op1_type == IS_CV); + + if (op2_addr != op2_def_addr) { + if (!zend_jit_update_regs(Dst, opline->op2.var, op2_addr, op2_def_addr, op2_info)) { + return 0; + } + if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) { + op2_addr = op2_def_addr; + } + } + + if (Z_MODE(op1_addr) != IS_REG + && Z_MODE(op1_use_addr) == IS_REG + && !Z_LOAD(op1_use_addr) + && !Z_STORE(op1_use_addr)) { + /* Force type update */ + op1_info |= MAY_BE_UNDEF; + } + if (!zend_jit_assign_to_variable(Dst, opline, op1_use_addr, op1_addr, op1_info, op1_def_info, opline->op2_type, op2_addr, op2_info, res_addr, + may_throw)) { + return 0; + } + if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) { + return 0; + } + if (opline->result_type != IS_UNUSED) { + if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { + return 0; + } + } + + return 1; +} + +/* copy of hidden zend_closure */ +typedef struct _zend_closure { + zend_object std; + zend_function func; + zval this_ptr; + zend_class_entry *called_scope; + zif_handler orig_internal_handler; +} zend_closure; + +static int zend_jit_stack_check(dasm_State **Dst, const zend_op *opline, uint32_t used_stack) +{ + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + + | brk #0 // TODO + + return 1; +} + +static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func, bool is_closure, bool use_this, bool stack_check) +{ + uint32_t used_stack; + + // TMP1 -> zend_function + // FCARG1x -> used_stack + // It's safe to use FCARG1x directly as x86 does only for the case where 'func' is NULL. + // FCARG1x would be further passed to external helper functions, zend_jit_int_extend_stack_helper + // and zend_jit_extend_stack_helper, if needed. + + if (func) { + used_stack = zend_vm_calc_used_stack(opline->extended_value, func); + } else { + used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value) * sizeof(zval); + + | // if (EXPECTED(ZEND_USER_CODE(func->type))) { + if (!is_closure) { + | LOAD_32BIT_VAL FCARG1w, used_stack + | // Check whether TMP1 is an internal function. + | ldrb TMP2w, [TMP1, #offsetof(zend_function, type)] + | tst TMP2w, #1 + | bne >1 + } else { + | brk #0 // TODO: test + | LOAD_32BIT_VAL FCARG1w, used_stack + } + | // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval); + | LOAD_32BIT_VAL TMP2w, opline->extended_value + if (!is_closure) { + | ldr TMP3w, [TMP1, #offsetof(zend_function, op_array.num_args)] + | cmp TMP2w, TMP3w + | csel TMP2w, TMP2w, TMP3w, le + | ldr TMP3w, [TMP1, #offsetof(zend_function, op_array.last_var)] + | sub TMP2w, TMP2w, TMP3w + | ldr TMP3w, [TMP1, #offsetof(zend_function, op_array.T)] + | sub TMP2w, TMP2w, TMP3w + } else { + | brk #0 // TODO + } + | lsl TMP2w, TMP2w, #5 + | sxtw TMP2, TMP2w + | sub FCARG1x, FCARG1x, TMP2 + |1: + } + + zend_jit_start_reuse_ip(); + + | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { + | MEM_LOAD_ZTS ldr, RX, executor_globals, vm_stack_top, TMP2 + + if (stack_check) { + | // Check Stack Overflow + | MEM_LOAD_ZTS ldr, TMP2, executor_globals, vm_stack_end, TMP3 + | sub TMP2, TMP2, RX + if (func) { + || if (used_stack <= MAX_IMM12) { + | cmp TMP2, #used_stack + || } else { + | LOAD_32BIT_VAL TMP3, used_stack + | cmp TMP2, TMP3 + || } + } else { + | cmp TMP2, FCARG1x + } + + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + + | blt >1 + |.cold_code + |1: + | brk #0 // TODO: test. Cold. + | EXT_JMP exit_addr, TMP3 + |.code + } else { + | blt >1 + | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); + |.cold_code + |1: + if (func) { + | brk #0 // TODO + } + if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { + | brk #0 // TODO + } else { + if (!is_closure) { + | brk #0 // TODO + } else { + | brk #0 // TODO + } + | brk #0 // TODO + } + | brk #0 // TODO + |.code + } + } + + if (func) { + || if (used_stack <= MAX_IMM12) { + | MEM_LOAD_OP_STORE_ZTS add, ldr, str, #used_stack, executor_globals, vm_stack_top, TMP2, TMP3 + || } else { + | LOAD_32BIT_VAL TMP4, used_stack + | MEM_LOAD_OP_STORE_ZTS add, ldr, str, TMP4, executor_globals, vm_stack_top, TMP2, TMP3 + || } + } else { + | MEM_LOAD_OP_STORE_ZTS add, ldr, str, FCARG1x, executor_globals, vm_stack_top, TMP2, TMP3 + } + | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); + if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) { + | // ZEND_SET_CALL_INFO(call, 0, call_info); + | LOAD_32BIT_VAL TMP2w, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION) + | str TMP2w, EX:RX->This.u1.type_info + } + if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { + | // call->func = func; + |1: + | ADDR_STORE EX:RX->func, func, TMP2 + } else { + if (!is_closure) { + | // call->func = func; + if (func + && op_array == &func->op_array + && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) + && (sizeof(void*) != 8 || IS_SIGNED_32BIT(func))) { + | brk #0 // TODO + } else { + | str TMP1, EX:RX->func + } + } else { + | // call->func = &closure->func; + | brk #0 // TODO + } + |1: + } + if (opline->opcode == ZEND_INIT_METHOD_CALL) { + | // Z_PTR(call->This) = obj; + | brk #0 // TODO + } else if (!is_closure) { + | // Z_CE(call->This) = called_scope; + | str xzr, EX:RX->This.value.ptr + } else { + | brk #0 // TODO + } + | // ZEND_CALL_NUM_ARGS(call) = num_args; + | LOAD_32BIT_VAL TMP2w, opline->extended_value + | str TMP2w, EX:RX->This.u2.num_args + + return 1; +} + +static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, zend_jit_trace_rec *trace) +{ + int skip; + + if (trace) { + zend_jit_trace_rec *p = trace; + + ssa_op++; + while (1) { + if (p->op == ZEND_JIT_TRACE_VM) { + switch (p->opline->opcode) { + case ZEND_SEND_ARRAY: + case ZEND_SEND_USER: + case ZEND_SEND_UNPACK: + case ZEND_INIT_FCALL: + case ZEND_INIT_METHOD_CALL: + case ZEND_INIT_STATIC_METHOD_CALL: + case ZEND_INIT_FCALL_BY_NAME: + case ZEND_INIT_NS_FCALL_BY_NAME: + case ZEND_INIT_DYNAMIC_CALL: + case ZEND_NEW: + case ZEND_INIT_USER_CALL: + case ZEND_FAST_CALL: + case ZEND_JMP: + case ZEND_JMPZNZ: + case ZEND_JMPZ: + case ZEND_JMPNZ: + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: + case ZEND_FE_RESET_R: + case ZEND_FE_RESET_RW: + case ZEND_JMP_SET: + case ZEND_COALESCE: + case ZEND_JMP_NULL: + case ZEND_ASSERT_CHECK: + case ZEND_CATCH: + case ZEND_DECLARE_ANON_CLASS: + case ZEND_FE_FETCH_R: + case ZEND_FE_FETCH_RW: + return 1; + case ZEND_DO_ICALL: + case ZEND_DO_UCALL: + case ZEND_DO_FCALL_BY_NAME: + case ZEND_DO_FCALL: + return 0; + case ZEND_SEND_VAL: + case ZEND_SEND_VAR: + case ZEND_SEND_VAL_EX: + case ZEND_SEND_VAR_EX: + case ZEND_SEND_FUNC_ARG: + case ZEND_SEND_REF: + case ZEND_SEND_VAR_NO_REF: + case ZEND_SEND_VAR_NO_REF_EX: + /* skip */ + break; + default: + if (zend_may_throw(opline, ssa_op, op_array, ssa)) { + return 1; + } + } + ssa_op += zend_jit_trace_op_len(opline); + } else if (p->op == ZEND_JIT_TRACE_ENTER || + p->op == ZEND_JIT_TRACE_BACK || + p->op == ZEND_JIT_TRACE_END) { + return 1; + } + p++; + } + } + + if (!call_info) { + const zend_op *end = op_array->opcodes + op_array->last; + + opline++; + ssa_op++; + skip = 1; + while (opline != end) { + if (!skip) { + if (zend_may_throw(opline, ssa_op, op_array, ssa)) { + return 1; + } + } + switch (opline->opcode) { + case ZEND_SEND_VAL: + case ZEND_SEND_VAR: + case ZEND_SEND_VAL_EX: + case ZEND_SEND_VAR_EX: + case ZEND_SEND_FUNC_ARG: + case ZEND_SEND_REF: + case ZEND_SEND_VAR_NO_REF: + case ZEND_SEND_VAR_NO_REF_EX: + skip = 0; + break; + case ZEND_SEND_ARRAY: + case ZEND_SEND_USER: + case ZEND_SEND_UNPACK: + case ZEND_INIT_FCALL: + case ZEND_INIT_METHOD_CALL: + case ZEND_INIT_STATIC_METHOD_CALL: + case ZEND_INIT_FCALL_BY_NAME: + case ZEND_INIT_NS_FCALL_BY_NAME: + case ZEND_INIT_DYNAMIC_CALL: + case ZEND_NEW: + case ZEND_INIT_USER_CALL: + case ZEND_FAST_CALL: + case ZEND_JMP: + case ZEND_JMPZNZ: + case ZEND_JMPZ: + case ZEND_JMPNZ: + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: + case ZEND_FE_RESET_R: + case ZEND_FE_RESET_RW: + case ZEND_JMP_SET: + case ZEND_COALESCE: + case ZEND_JMP_NULL: + case ZEND_ASSERT_CHECK: + case ZEND_CATCH: + case ZEND_DECLARE_ANON_CLASS: + case ZEND_FE_FETCH_R: + case ZEND_FE_FETCH_RW: + return 1; + case ZEND_DO_ICALL: + case ZEND_DO_UCALL: + case ZEND_DO_FCALL_BY_NAME: + case ZEND_DO_FCALL: + end = opline; + if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) { + /* INIT_FCALL and DO_FCALL in different BasicBlocks */ + return 1; + } + return 0; + } + opline++; + ssa_op++; + } + + return 1; + } else { + const zend_op *end = call_info->caller_call_opline; + + if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) { + /* INIT_FCALL and DO_FCALL in different BasicBlocks */ + return 1; + } + + opline++; + ssa_op++; + skip = 1; + while (opline != end) { + if (skip) { + switch (opline->opcode) { + case ZEND_SEND_VAL: + case ZEND_SEND_VAR: + case ZEND_SEND_VAL_EX: + case ZEND_SEND_VAR_EX: + case ZEND_SEND_FUNC_ARG: + case ZEND_SEND_REF: + case ZEND_SEND_VAR_NO_REF: + case ZEND_SEND_VAR_NO_REF_EX: + skip = 0; + break; + case ZEND_SEND_ARRAY: + case ZEND_SEND_USER: + case ZEND_SEND_UNPACK: + return 1; + } + } else { + if (zend_may_throw(opline, ssa_op, op_array, ssa)) { + return 1; + } + } + opline++; + ssa_op++; + } + + return 0; + } +} + +static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zend_function *func, const zend_op *to_opline) +{ + int32_t exit_point; + const void *exit_addr; + + | brk #0 // TODO + + return 1; +} + +static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, bool stack_check) +{ + zend_func_info *info = ZEND_FUNC_INFO(op_array); + zend_call_info *call_info = NULL; + zend_function *func = NULL; + + if (delayed_call_chain) { + | brk #0 // TODO + } + + if (info) { + call_info = info->callee_info; + while (call_info && call_info->caller_init_opline != opline) { + call_info = call_info->next_callee; + } + if (call_info && call_info->callee_func && !call_info->is_prototype) { + func = call_info->callee_func; + } + } + + if (!func + && trace + && trace->op == ZEND_JIT_TRACE_INIT_CALL) { + | brk #0 // TODO: Tracing mode. ASLR? + } + + if (opline->opcode == ZEND_INIT_FCALL + && func + && func->type == ZEND_INTERNAL_FUNCTION) { + /* load constant address later */ + } else if (func && op_array == &func->op_array) { + /* recursive call */ + | brk #0 // TODO + } else { + | // if (CACHED_PTR(opline->result.num)) + | ldr TMP1, EX->run_time_cache + | ldr TMP1, [TMP1, #opline->result.num] + | cbz TMP1, >1 + |.cold_code + |1: + if (opline->opcode == ZEND_INIT_FCALL + && func + && func->type == ZEND_USER_FUNCTION + && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) { + | brk #0 // TODO + } else { + zval *zv = RT_CONSTANT(opline, opline->op2); + + if (opline->opcode == ZEND_INIT_FCALL) { + | LOAD_ADDR FCARG1x, Z_STR_P(zv); + | EXT_CALL zend_jit_find_func_helper, TMP1 + } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) { + | brk #0 // TODO + } else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) { + | brk #0 // TODO + } else { + ZEND_UNREACHABLE(); + } + | // CACHE_PTR(opline->result.num, fbc); + | ldr TMP2, EX->run_time_cache + | mov TMP1, RETVALx + | str TMP1, [TMP2, #opline->result.num] + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + | brk #0 // TODO. tracing mode. + } else { + | cbnz TMP1, >3 + | // SAVE_OPLINE(); + | brk #0 // TODO: invalid func address. + } + } + |.code + |3: + } + + if (!zend_jit_push_call_frame(Dst, opline, op_array, func, 0, 0, stack_check)) { + return 0; + } + + if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, trace)) { + if (!zend_jit_save_call_chain(Dst, call_level)) { + return 0; + } + } else { + delayed_call_chain = 1; + delayed_call_level = call_level; + } + + return 1; +} + +static int zend_jit_init_method_call(dasm_State **Dst, + const zend_op *opline, + uint32_t b, + const zend_op_array *op_array, + zend_ssa *ssa, + const zend_ssa_op *ssa_op, + int call_level, + uint32_t op1_info, + zend_jit_addr op1_addr, + zend_class_entry *ce, + bool ce_is_instanceof, + bool use_this, + zend_class_entry *trace_ce, + zend_jit_trace_rec *trace, + bool stack_check, + bool polymorphic_side_trace) +{ + zend_func_info *info = ZEND_FUNC_INFO(op_array); + zend_call_info *call_info = NULL; + zend_function *func = NULL; + zval *function_name; + + ZEND_ASSERT(opline->op2_type == IS_CONST); + ZEND_ASSERT(op1_info & MAY_BE_OBJECT); + + | brk #0 // TODO + return 1; +} + +static int zend_jit_init_closure_call(dasm_State **Dst, + const zend_op *opline, + uint32_t b, + const zend_op_array *op_array, + zend_ssa *ssa, + const zend_ssa_op *ssa_op, + int call_level, + zend_jit_trace_rec *trace, + bool stack_check) +{ + zend_function *func = NULL; + zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); + + | brk #0 // TODO + return 1; +} + +static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, const zend_call_info *call_info) +{ + uint32_t num_args = 0; + zend_function *func = call_info->callee_func; + + /* It's okay to handle prototypes here, because they can only increase the accepted arguments. + * Anything legal for the parent method is also legal for the parent method. */ + while (num_args < call_info->num_args) { + zend_arg_info *arg_info = func->op_array.arg_info + num_args; + + if (ZEND_TYPE_IS_SET(arg_info->type)) { + if (ZEND_TYPE_IS_ONLY_MASK(arg_info->type)) { + zend_op *opline = call_info->arg_info[num_args].opline; + zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; + uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type); + if ((OP1_INFO() & (MAY_BE_ANY|MAY_BE_UNDEF)) & ~type_mask) { + break; + } + } else { + break; + } + } + num_args++; + } + return num_args; +} + +static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace) +{ + zend_func_info *info = ZEND_FUNC_INFO(op_array); + zend_call_info *call_info = NULL; + const zend_function *func = NULL; + uint32_t i; + zend_jit_addr res_addr; + uint32_t call_num_args = 0; + bool unknown_num_args = 0; + const void *exit_addr = NULL; + const zend_op *prev_opline; + + if (RETURN_VALUE_USED(opline)) { + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + } else { + /* CPU stack allocated temporary zval */ + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RSP, TMP_ZVAL_OFFSET); + } + + prev_opline = opline - 1; + while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) { + | brk #0 // TODO + prev_opline--; + } + if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY || + prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { + | brk #0 // TODO + unknown_num_args = 1; + } + + if (info) { + call_info = info->callee_info; + while (call_info && call_info->caller_call_opline != opline) { + call_info = call_info->next_callee; + } + if (call_info && call_info->callee_func && !call_info->is_prototype) { + func = call_info->callee_func; + } + } + if (!func) { + /* resolve function at run time */ + } else if (func->type == ZEND_USER_FUNCTION) { + | brk #0 // TODO + ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL); + call_num_args = call_info->num_args; + } else if (func->type == ZEND_INTERNAL_FUNCTION) { + ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL); + call_num_args = call_info->num_args; + } else { + ZEND_UNREACHABLE(); + } + + if (trace && !func) { + | brk #0 // TODO + } + + bool may_have_extra_named_params = + opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS && + (!func || func->common.fn_flags & ZEND_ACC_VARIADIC); + + if (!reuse_ip) { + zend_jit_start_reuse_ip(); + | // call = EX(call); + | ldr RX, EX->call + } + zend_jit_stop_reuse_ip(); + + | // fbc = call->func; + | // mov r2, EX:RX->func ??? + | // SAVE_OPLINE(); + | SET_EX_OPLINE opline, TMP1 + + if (opline->opcode == ZEND_DO_FCALL) { + | brk #0 // TODO + } + + if (!delayed_call_chain) { + if (call_level == 1) { + | str xzr, EX->call + } else { + | //EX(call) = call->prev_execute_data; + | brk #0 // TODO: test + } + } + delayed_call_chain = 0; + + | //call->prev_execute_data = execute_data; + | str EX, EX:RX->prev_execute_data + + if (!func) { + | ldr TMP1, EX:RX->func + } + + if (opline->opcode == ZEND_DO_FCALL) { + | brk #0 // TODO + } + + if (!func + && opline->opcode != ZEND_DO_UCALL + && opline->opcode != ZEND_DO_ICALL) { + | brk #0 // TODO + } + + if ((!func || func->type == ZEND_USER_FUNCTION) + && opline->opcode != ZEND_DO_ICALL) { + | // EX(call) = NULL; + | str xzr, EX:RX->call + + if (RETURN_VALUE_USED(opline)) { + | // EX(return_value) = EX_VAR(opline->result.var); + | LOAD_ZVAL_ADDR TMP3, res_addr + | str TMP3, EX:RX->return_value + } else { + | // EX(return_value) = 0; + | str xzr, EX:RX->return_value + } + + //EX_LOAD_RUN_TIME_CACHE(op_array); + if (!func || func->op_array.cache_size) { + if (func && op_array == &func->op_array) { + /* recursive call */ + if (trace || func->op_array.cache_size > sizeof(void*)) { + | brk #0 // TODO + } + } else { + if (func) { + | brk #0 // TODO + } + | ldr TMP2, [TMP1, #offsetof(zend_op_array, run_time_cache__ptr)] +// Always defined as ZEND_MAP_PTR_KIND_PTR_OR_OFFSET. See Zend/zend_map_ptr.h. +#if ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR + | ldr TMP2, [TMP2] +#elif ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR_OR_OFFSET + if (func && !(func->op_array.fn_flags & ZEND_ACC_CLOSURE)) { + | brk #0 // TODO + } else { + | tst TMP2, #1 + | beq >1 + | MEM_LOAD_OP_ZTS add, ldr, TMP2, compiler_globals, map_ptr_base, TMP3, TMP4 + |1: + | ldr TMP2, [TMP2] + } +#else +# error "Unknown ZEND_MAP_PTR_KIND" +#endif + | str TMP2, EX:RX->run_time_cache + } + } + + | // EG(current_execute_data) = execute_data; + | MEM_STORE_ZTS str, RX, executor_globals, current_execute_data, TMP2 + | mov FP, RX + + | // opline = op_array->opcodes; + if (func && !unknown_num_args) { + | brk #0 // TODO + } else { + | // opline = op_array->opcodes + if (func && zend_accel_in_shm(func->op_array.opcodes)) { + | brk #0 // TODO + } else if (GCC_GLOBAL_REGS) { + | ldr IP, [TMP1, #offsetof(zend_op_array, opcodes)] + } else { + | ldr FCARG1x, [TMP1, #offsetof(zend_op_array, opcodes)] + | str FCARG1x, EX->opline + } + if (func) { + | brk #0 // TODO + } else { + | // first_extra_arg = op_array->num_args; + | ldr TMP3w, [TMP1, #offsetof(zend_op_array, num_args)] + | // num_args = EX_NUM_ARGS(); + | ldr TMP2w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] + | // if (UNEXPECTED(num_args > first_extra_arg)) + | cmp TMP2w, TMP3w + } + | bgt >1 + |.cold_code + |1: + | brk #0 // TDOO: test + |.code + if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) { + if (!func) { + | // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) + | ldr TMP4w, [TMP1, #offsetof(zend_op_array, fn_flags)] + | tst TMP4w, #ZEND_ACC_HAS_TYPE_HINTS + | bne >1 + } + | // opline += num_args; + || ZEND_ASSERT(sizeof(zend_op) == 32); + | mov TMP3w, TMP2w + | lsl TMP3, TMP3, #5 + | ADD_IP TMP3, TMP4 + } + |1: + | // if (EXPECTED((int)num_args < op_array->last_var)) { + if (func) { + | brk #0 // TODO + } else { + | ldr TMP3w, [TMP1, #offsetof(zend_op_array, last_var)] + } + | sub TMP3w, TMP3w, TMP2w + | ble >3 + | brk #0 // TODO: test + |3: + } + + if (ZEND_OBSERVER_ENABLED) { + | brk #0 // TODO: test + | SAVE_IP + | mov FCARG1x, FP + | EXT_CALL zend_observer_fcall_begin, TMP1 + } + + if (trace) { + if (!func && (opline->opcode != ZEND_DO_UCALL)) { + | brk #0 // TODO + } + } else { +#ifdef CONTEXT_THREADED_JIT + | brk #0 // TODO: CONTEXT_THREADED_JIT is always undefined. +#else + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | ADD_HYBRID_SPAD + | JMP_IP TMP1 + } else if (GCC_GLOBAL_REGS) { + | add sp, sp, SPAD // stack alignment + | JMP_IP TMP1 + } else { + | ldp FP, RX, T2 // retore FP and IP + | ldr LR, T4 // retore LR + | add sp, sp, NR_SPAD // stack alignment + | mov RETVALx, #1 // ZEND_VM_ENTER + | ret + } + } +#endif + } + + if ((!func || func->type == ZEND_INTERNAL_FUNCTION) + && (opline->opcode != ZEND_DO_UCALL)) { + if (!func && (opline->opcode != ZEND_DO_ICALL)) { + |8: + } + if (opline->opcode == ZEND_DO_FCALL_BY_NAME) { + | brk #0 // TODO + } + + | // ZVAL_NULL(EX_VAR(opline->result.var)); + | LOAD_ZVAL_ADDR FCARG2x, res_addr + | SET_Z_TYPE_INFO FCARG2x, IS_NULL, TMP2w + + | // EG(current_execute_data) = execute_data; + | MEM_STORE_ZTS str, RX, executor_globals, current_execute_data, TMP2 + + zend_jit_reset_last_valid_opline(); + + | // fbc->internal_function.handler(call, ret); + | mov FCARG1x, RX + if (func) { + | EXT_CALL func->internal_function.handler, TMP1 + } else { + | ldr TMP2, [TMP1, #offsetof(zend_internal_function, handler)] + | blr TMP2 + } + + | // EG(current_execute_data) = execute_data; + | MEM_STORE_ZTS str, FP, executor_globals, current_execute_data, TMP1 + + | // zend_vm_stack_free_args(call); + if (func && !unknown_num_args) { + for (i = 0; i < call_num_args; i++ ) { + uint32_t offset = EX_NUM_TO_VAR(i); + zend_jit_addr arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset); + | ZVAL_PTR_DTOR arg_addr, (MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN), 0, 1, opline, ZREG_TMP1, ZREG_TMP2 + } + } else { + | mov FCARG1x, RX + | EXT_CALL zend_jit_vm_stack_free_args_helper, TMP1 + } + if (may_have_extra_named_params) { + | brk #0 // TODO + } + + |8: + if (opline->opcode == ZEND_DO_FCALL) { + // TODO: optimize ??? + | brk #0 // TODO + } + + if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || + !JIT_G(current_frame) || + !JIT_G(current_frame)->call || + !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) || + prev_opline->opcode == ZEND_SEND_UNPACK || + prev_opline->opcode == ZEND_SEND_ARRAY || + prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { + + | // zend_vm_stack_free_call_frame(call); + | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 2)] + | tst TMP1w, #((ZEND_CALL_ALLOCATED >> 16) & 0xff) + | bne >1 // TODO: test. In current case, don't jump to cold-code. + |.cold_code + |1: + | brk #0 // TODO + | mov FCARG1x, RX + | EXT_CALL zend_jit_free_call_frame, TMP1 + | b >1 + |.code + } + | MEM_STORE_ZTS str, RX, executor_globals, vm_stack_top, TMP1 + |1: + + if (!RETURN_VALUE_USED(opline)) { + zend_class_entry *ce; + bool ce_is_instanceof; + uint32_t func_info = call_info ? + zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) : + (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN); + + /* If an exception is thrown, the return_value may stay at the + * original value of null. */ + func_info |= MAY_BE_NULL; + + if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + | ZVAL_PTR_DTOR res_addr, func_info, 1, 1, opline, ZREG_TMP1, ZREG_TMP2 + } + } + + | // if (UNEXPECTED(EG(exception) != NULL)) { + | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, TMP1, TMP2 + | bne ->icall_throw_handler + + // TODO: Can we avoid checking for interrupts after each call ??? + if (trace && last_valid_opline != opline) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM); + + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + } else { + exit_addr = NULL; + } + if (!zend_jit_check_timeout(Dst, opline + 1, exit_addr)) { + return 0; + } + + if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) { + | brk #0 // TODO + } else if (trace + && trace->op == ZEND_JIT_TRACE_END + && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { + | brk #0 // TODO + } + } + + if (!func) { + |9: + } + + return 1; +} + +static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr) +{ + uint32_t arg_num = opline->op2.num; + zend_jit_addr arg_addr; + + ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM); + + if (!zend_jit_reuse_ip(Dst)) { + return 0; + } + + if (opline->opcode == ZEND_SEND_VAL_EX) { + uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); + + ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM); + + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE + && JIT_G(current_frame) + && JIT_G(current_frame)->call + && JIT_G(current_frame)->call->func) { + if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { + /* Don't generate code that always throws exception */ + return 0; + } + } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + | brk #0 // TODO + } else { + | brk #0 // TODO + } + } + + arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); + + if (opline->op1_type == IS_CONST) { + zval *zv = RT_CONSTANT(opline, opline->op1); + + | ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_TMP1, ZREG_TMP2, ZREG_FPTMP1 + if (Z_REFCOUNTED_P(zv)) { + | brk #0 // TODO: test + } + } else { + | brk #0 // TODO: test + } + + return 1; +} + +static int zend_jit_check_undef_args(dasm_State **Dst, const zend_op *opline) +{ + | brk #0 // TODO + + return 1; +} + +static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold) +{ + zend_jit_addr op1_addr, arg_addr, ref_addr; + + op1_addr = OP1_ADDR(); + arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); + + | brk #0 // TODO + + return 1; +} + +static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr) +{ + uint32_t arg_num = opline->op2.num; + zend_jit_addr arg_addr; + + ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX && + opline->opcode != ZEND_SEND_VAR_NO_REF_EX) || + arg_num <= MAX_ARG_FLAG_NUM); + + arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); + + if (!zend_jit_reuse_ip(Dst)) { + return 0; + } + + if (opline->opcode == ZEND_SEND_VAR_EX) { + | brk #0 // TODO + } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { + | brk #0 // TODO + } else if (opline->opcode == ZEND_SEND_FUNC_ARG) { + | brk #0 // TODO + } + + if (op1_info & MAY_BE_UNDEF) { + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, TMP1w, TMP2 + |.cold_code + |1: + } + + | brk #0 // TODO: test + | SET_EX_OPLINE opline, TMP1 + | LOAD_32BIT_VAL FCARG1w, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, TMP1 + | SET_ZVAL_TYPE_INFO arg_addr, IS_NULL, TMP1w, TMP2 + | cbz RETVALx, ->exception_handler + + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + | brk #0 // TODO: test + | b >7 + |.code + } else { + | brk #0 // TODO: test + } + } + + if (opline->opcode == ZEND_SEND_VAR_NO_REF) { + | brk #0 // TODO: test + } else { + if (op1_info & MAY_BE_REF) { + if (opline->op1_type == IS_CV) { + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + + | brk #0 // TODO: test + } else { + zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 8); + + | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, TMP1w, TMP2 + |.cold_code + |1: + | brk #0 // TODO: test. cold-code. not covered currently + |.code + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_TMP1, ZREG_TMP2, ZREG_TMP3, ZREG_TMP4, ZREG_FPTMP1 + |2: + } + } else { + if (op1_addr != op1_def_addr) { + | brk #0 // TODO: test + } + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_TMP1, ZREG_TMP2, ZREG_TMP3, ZREG_TMP4, ZREG_FPTMP1 + if (opline->op1_type == IS_CV) { + | // In x86 implementation, type flags and value pointer would be stored into eax and r2 respectively, + | // and then ah (bits 8 to 15) and r2 are used inside TRY_ADDREF. + | // In AArch64, we use TMP1w and TMP2 accordingly. + | // Note that, bits 8 to 15 should be extacted, i.e., (TMP1w >> 8) & 0xff. + | lsr TMP1w, TMP1w, #8 + | and TMP1w, TMP1w, #0xff + | TRY_ADDREF op1_info, TMP1w, TMP2, TMP3 + } + } + } + |7: + + return 1; +} + +static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) +{ + uint32_t arg_num = opline->op2.num; + + | brk #0 // TODO + + return 1; +} + +static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) +{ + | brk #0 // TODO + + return 1; +} + +static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label) +{ + | brk #0 // TODO + + return 1; +} + +static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) +{ + uint32_t defined_label = (uint32_t)-1; + uint32_t undefined_label = (uint32_t)-1; + zval *zv = RT_CONSTANT(opline, opline->op1); + zend_jit_addr res_addr = 0; + + | brk #0 // TODO + + return 1; +} + +static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) +{ + uint32_t mask; + zend_jit_addr op1_addr = OP1_ADDR(); + + // TODO: support for is_resource() ??? + ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE); + + | brk #0 // TODO + + return 1; +} + +static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, zend_ssa *ssa, uint32_t var) +{ + uint32_t j, info; + + if (ssa->vars && ssa->var_info) { + info = ssa->var_info[var].type; + for (j = op_array->last_var; j < ssa->vars_count; j++) { + if (ssa->vars[j].var == var) { + info |= ssa->var_info[j].type; + } + } + } else { + info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF | + MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; + } + +#ifdef ZEND_JIT_USE_RC_INFERENCE + /* Refcount may be increased by RETURN opcode */ + if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) { + for (j = 0; j < ssa->cfg.blocks_count; j++) { + if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) && + ssa->cfg.blocks[j].len > 0) { + const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1; + + if (opline->opcode == ZEND_RETURN) { + if (opline->op1_type == IS_CV && opline->op1.var == EX_NUM_TO_VAR(var)) { + info |= MAY_BE_RCN; + break; + } + } + } + } + } +#endif + + return info; +} + +static int zend_jit_leave_frame(dasm_State **Dst) +{ + | // EG(current_execute_data) = EX(prev_execute_data); + | ldr TMP1, EX->prev_execute_data + | MEM_STORE_ZTS str, TMP1, executor_globals, current_execute_data, TMP3 + return 1; +} + +static int zend_jit_free_cv(dasm_State **Dst, uint32_t info, uint32_t var) +{ + if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + uint32_t offset = EX_NUM_TO_VAR(var); + zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset); + | ZVAL_PTR_DTOR addr, info, 1, 1, NULL, ZREG_TMP1, ZREG_TMP2 + } + return 1; +} + +static int zend_jit_free_op(dasm_State **Dst, const zend_op *opline, uint32_t info, uint32_t var_offset) +{ + if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + | brk #0 // TODO + } + return 1; +} + +static int zend_jit_leave_func(dasm_State **Dst, + const zend_op_array *op_array, + const zend_op *opline, + uint32_t op1_info, + bool left_frame, + zend_jit_trace_rec *trace, + zend_jit_trace_info *trace_info, + int indirect_var_access, + int may_throw) +{ + bool may_be_top_frame = + JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || + !JIT_G(current_frame) || + !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)); + bool may_need_call_helper = + indirect_var_access || /* may have symbol table */ + !op_array->function_name || /* may have symbol table */ + may_be_top_frame || + (op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */ + JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || + !JIT_G(current_frame) || + TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */ + (uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */ + bool may_need_release_this = + !(op_array->fn_flags & ZEND_ACC_CLOSURE) && + op_array->scope && + !(op_array->fn_flags & ZEND_ACC_STATIC) && + (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || + !JIT_G(current_frame) || + !TRACE_FRAME_NO_NEED_REKEASE_THIS(JIT_G(current_frame))); + + if (may_need_call_helper || may_need_release_this) { + | ldr FCARG1w, [FP, #offsetof(zend_execute_data, This.u1.type_info)] + } + if (may_need_call_helper) { + if (!left_frame) { + left_frame = 1; + if (!zend_jit_leave_frame(Dst)) { + return 0; + } + } + /* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */ + + | LOAD_32BIT_VAL TMP1w, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE) + | tst FCARG1w, TMP1w + if (trace && trace->op != ZEND_JIT_TRACE_END) { + | brk #0 // TODO: test + } else { + | bne ->leave_function_handler + } + } + + if (op_array->fn_flags & ZEND_ACC_CLOSURE) { + if (!left_frame) { + left_frame = 1; + if (!zend_jit_leave_frame(Dst)) { + return 0; + } + } + | brk #0 // TODO: test + } else if (may_need_release_this) { + if (!left_frame) { + left_frame = 1; + if (!zend_jit_leave_frame(Dst)) { + return 0; + } + } + | brk #0 // TODO: test + // TODO: avoid EG(excption) check for $this->foo() calls + may_throw = 1; + } + + | // EG(vm_stack_top) = (zval*)execute_data; + | MEM_STORE_ZTS str, FP, executor_globals, vm_stack_top, TMP1 + | // execute_data = EX(prev_execute_data); + | ldr FP, EX->prev_execute_data + + if (!left_frame) { + | brk #0 // TODO: teset + | // EG(current_execute_data) = execute_data; + | MEM_STORE_ZTS str, FP, executor_globals, current_execute_data, TMP1 + } + + |9: + if (trace) { + if (trace->op != ZEND_JIT_TRACE_END + && (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { + zend_jit_reset_last_valid_opline(); + } else { + | LOAD_IP + | ADD_IP_FROM_CST sizeof(zend_op), TMP1 + } + + |8: + + if (trace->op == ZEND_JIT_TRACE_BACK + && (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { + const zend_op *next_opline = trace->opline; + + | brk #0 // TODO: test + + return 1; + } else if (may_throw || + (((opline->op1_type & (IS_VAR|IS_TMP_VAR)) + && (op1_info & MAY_BE_RC1) + && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) + && (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) { + | brk #0 // TODO: test + } + + return 1; + } else { + | // if (EG(exception)) + | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, TMP1, TMP2 + | LOAD_IP + | bne ->leave_throw_handler + | // opline = EX(opline) + 1 + | ADD_IP_FROM_CST sizeof(zend_op), TMP1 + } + + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | ADD_HYBRID_SPAD +#ifdef CONTEXT_THREADED_JIT + | brk #0 // TODO: CONTEXT_THREADED_JIT is always undefined +#else + | JMP_IP TMP1 +#endif + } else if (GCC_GLOBAL_REGS) { + | add sp, sp, SPAD // stack alignment +#ifdef CONTEXT_THREADED_JIT + | brk #0 // TODO +#else + | JMP_IP TMP1 +#endif + } else { +#ifdef CONTEXT_THREADED_JIT + ZEND_UNREACHABLE(); + // TODO: context threading can't work without GLOBAL REGS because we have to change + // the value of execute_data in execute_ex() + | brk #0 // TODO +#else + | ldp FP, RX, T2 // restore FP and IP + | ldr LR, T4 // restore LR + | add sp, sp, NR_SPAD // stack alignment + | mov RETVALx, #2 // ZEND_VM_LEAVE ???? + | ret +#endif + } + + return 1; +} + +static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr) +{ + zend_jit_addr ret_addr; + int8_t return_value_used; + + ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name); + ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF)); + + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) { + if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) { + return_value_used = 1; + } else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) { + return_value_used = 0; + } else { + return_value_used = -1; + } + } else { + return_value_used = -1; + } + + // TODO: This macro is only used in four sites. We should design a test variant to cover it. + if (ZEND_OBSERVER_ENABLED) { + | brk #0 // TODO: test + } + + // TMP1 -> ret_addr + // x86 has to select one temp register to store the return address, i.e. 'ret_addr', if the return value would be used. + // In AArch64, we simply use our reserved register, i.e. TMP1. + // if (!EX(return_value)) + if (return_value_used != 0) { + | ldr TMP1, EX->return_value + } + if (return_value_used == -1) { + | tst TMP1, TMP1 + } + ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_TMP1, 0); + + if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | brk #0 // TODO: test + } else if (return_value_used == -1) { + if (jit_return_label >= 0) { + | brk #0 // TODO: test + } else { + | beq >9 + } + } + + if (return_value_used == 0) { + |9: + | brk #0 // TODO: test + return 1; + } + + if (opline->op1_type == IS_CONST) { + zval *zv = RT_CONSTANT(opline, opline->op1); + | ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_TMP2, ZREG_TMP3, ZREG_FPTMP1 + if (Z_REFCOUNTED_P(zv)) { + | brk #0 // TODO: test + } + } else if (opline->op1_type == IS_TMP_VAR) { + | brk #0 // TODO + } else if (opline->op1_type == IS_CV) { + if (op1_info & MAY_BE_REF) { + | brk #0 // TODO + // TMP2 -> op1_addr + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_TMP2, 0); + } + // Note: tmp_reg2 is not used in current case, hence we pass a random one. + | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_TMP2, ZREG_TMP3, ZREG_TMP4, ZREG_TMP4, ZREG_FPTMP1 + if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + | brk #0 // TODO + } + } else { + | brk #0 // TODO + } + + |9: + return 1; +} + +static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, zend_jit_addr val_addr, zend_reg type_reg) +{ + ZEND_ASSERT(type_reg == ZREG_X2); + + | brk #0 // TODO + return 1; +} + +static bool zend_jit_may_avoid_refcounting(const zend_op *opline) +{ + switch (opline->opcode) { + case ZEND_FETCH_OBJ_FUNC_ARG: + if (!JIT_G(current_frame) || + !JIT_G(current_frame)->call->func || + !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { + return 0; + } + /* break missing intentionally */ + case ZEND_FETCH_OBJ_R: + case ZEND_FETCH_OBJ_IS: + if (opline->op2_type == IS_CONST + && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING + && Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] != '\0') { + return 1; + } + break; + case ZEND_FETCH_DIM_FUNC_ARG: + if (!JIT_G(current_frame) || + !JIT_G(current_frame)->call->func || + !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { + return 0; + } + /* break missing intentionally */ + case ZEND_FETCH_DIM_R: + case ZEND_FETCH_DIM_IS: + return 1; + case ZEND_ISSET_ISEMPTY_DIM_OBJ: + if (!(opline->extended_value & ZEND_ISEMPTY)) { + return 1; + } + break; + } + return 0; +} + +static int zend_jit_fetch_dim_read(dasm_State **Dst, + const zend_op *opline, + zend_ssa *ssa, + const zend_ssa_op *ssa_op, + uint32_t op1_info, + zend_jit_addr op1_addr, + bool op1_avoid_refcounting, + uint32_t op2_info, + uint32_t res_info, + zend_jit_addr res_addr, + int may_throw) +{ + zend_jit_addr orig_op1_addr, op2_addr; + const void *exit_addr = NULL; + const void *not_found_exit_addr = NULL; + const void *res_exit_addr = NULL; + bool result_avoid_refcounting = 0; + uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0; + + orig_op1_addr = OP1_ADDR(); + op2_addr = OP2_ADDR(); + + | brk #0 // TODO + return 1; +} + +static int zend_jit_fetch_dim(dasm_State **Dst, + const zend_op *opline, + uint32_t op1_info, + zend_jit_addr op1_addr, + uint32_t op2_info, + zend_jit_addr res_addr, + int may_throw) +{ + zend_jit_addr op2_addr; + + op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; + + | brk #0 // TODO + return 1; +} + +static int zend_jit_isset_isempty_dim(dasm_State **Dst, + const zend_op *opline, + uint32_t op1_info, + zend_jit_addr op1_addr, + bool op1_avoid_refcounting, + uint32_t op2_info, + int may_throw, + zend_uchar smart_branch_opcode, + uint32_t target_label, + uint32_t target_label2, + const void *exit_addr) +{ + zend_jit_addr op2_addr, res_addr; + + // TODO: support for empty() ??? + ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); + + | brk #0 // TODO + return 1; +} + +static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) +{ + zend_jit_addr op1_addr = OP1_ADDR(); + zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2)); + + | brk #0 // TODO + return 1; +} + +static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zend_arg_info *arg_info, bool check_exception) +{ + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + bool in_cold = 0; + uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; + zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1x : ZREG_TMP1; // TODO: use TMP1 + + | brk #0 // TODO + return 1; +} + +static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array) +{ + uint32_t arg_num = opline->op1.num; + zend_arg_info *arg_info = NULL; + + | brk #0 // TODO + return 1; +} + +static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool is_last, int may_throw) +{ + uint32_t arg_num = opline->op1.num; + zval *zv = RT_CONSTANT(opline, opline->op2); + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + + | brk #0 // TODO + return 1; +} + +static zend_property_info* zend_get_known_property_info(const zend_op_array *op_array, zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename) +{ + zend_property_info *info = NULL; + + if ((on_this && (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)) || + !ce || + !(ce->ce_flags & ZEND_ACC_LINKED) || + (ce->ce_flags & ZEND_ACC_TRAIT) || + ce->create_object) { + return NULL; + } + + if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) { + if (ce->info.user.filename != filename) { + /* class declaration might be changed independently */ + return NULL; + } + + if (ce->parent) { + zend_class_entry *parent = ce->parent; + + do { + if (parent->type == ZEND_INTERNAL_CLASS) { + break; + } else if (parent->info.user.filename != filename) { + /* some of parents class declarations might be changed independently */ + /* TODO: this check may be not enough, because even + * in the same it's possible to conditionally define + * few classes with the same name, and "parent" may + * change from request to request. + */ + return NULL; + } + parent = parent->parent; + } while (parent); + } + } + + info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member); + if (info == NULL || + !IS_VALID_PROPERTY_OFFSET(info->offset) || + (info->flags & ZEND_ACC_STATIC)) { + return NULL; + } + + if (!(info->flags & ZEND_ACC_PUBLIC) && + (!on_this || info->ce != ce)) { + return NULL; + } + + return info; +} + +static bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename) +{ + zend_property_info *info; + + if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT)) { + return 1; + } + + if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) { + if (ce->info.user.filename != filename) { + /* class declaration might be changed independently */ + return 1; + } + } + + info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member); + if (info == NULL || + !IS_VALID_PROPERTY_OFFSET(info->offset) || + (info->flags & ZEND_ACC_STATIC)) { + return 1; + } + + if (!(info->flags & ZEND_ACC_PUBLIC) && + (!on_this || info->ce != ce)) { + return 1; + } + + return 0; +} + +static int zend_jit_class_guard(dasm_State **Dst, const zend_op *opline, zend_class_entry *ce) +{ + int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + + | brk #0 // TODO + return 1; +} + +static int zend_jit_fetch_obj(dasm_State **Dst, + const zend_op *opline, + const zend_op_array *op_array, + zend_ssa *ssa, + const zend_ssa_op *ssa_op, + uint32_t op1_info, + zend_jit_addr op1_addr, + bool op1_indirect, + zend_class_entry *ce, + bool ce_is_instanceof, + bool use_this, + bool op1_avoid_refcounting, + zend_class_entry *trace_ce, + int may_throw) +{ + zval *member; + zend_property_info *prop_info; + bool may_be_dynamic = 1; + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); + zend_jit_addr prop_addr; + uint32_t res_info = RES_INFO(); + + ZEND_ASSERT(opline->op2_type == IS_CONST); + ZEND_ASSERT(op1_info & MAY_BE_OBJECT); + + | brk #0 // TODO + return 1; +} + +static int zend_jit_incdec_obj(dasm_State **Dst, + const zend_op *opline, + const zend_op_array *op_array, + zend_ssa *ssa, + const zend_ssa_op *ssa_op, + uint32_t op1_info, + zend_jit_addr op1_addr, + bool op1_indirect, + zend_class_entry *ce, + bool ce_is_instanceof, + bool use_this, + zend_class_entry *trace_ce, + int may_throw) +{ + zval *member; + zend_string *name; + zend_property_info *prop_info; + zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); + zend_jit_addr res_addr = 0; + zend_jit_addr prop_addr; + bool needs_slow_path = 0; + + ZEND_ASSERT(opline->op2_type == IS_CONST); + ZEND_ASSERT(op1_info & MAY_BE_OBJECT); + + | brk #0 // TODO + return 1; +} + +static int zend_jit_assign_obj_op(dasm_State **Dst, + const zend_op *opline, + const zend_op_array *op_array, + zend_ssa *ssa, + const zend_ssa_op *ssa_op, + uint32_t op1_info, + zend_jit_addr op1_addr, + uint32_t val_info, + zend_ssa_range *val_range, + bool op1_indirect, + zend_class_entry *ce, + bool ce_is_instanceof, + bool use_this, + zend_class_entry *trace_ce, + int may_throw) +{ + zval *member; + zend_string *name; + zend_property_info *prop_info; + zend_jit_addr val_addr = OP1_DATA_ADDR(); + zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); + zend_jit_addr prop_addr; + bool needs_slow_path = 0; + binary_op_type binary_op = get_binary_op(opline->extended_value); + + ZEND_ASSERT(opline->op2_type == IS_CONST); + ZEND_ASSERT(op1_info & MAY_BE_OBJECT); + ZEND_ASSERT(opline->result_type == IS_UNUSED); + + | brk #0 // TODO + return 1; +} + +static int zend_jit_assign_obj(dasm_State **Dst, + const zend_op *opline, + const zend_op_array *op_array, + zend_ssa *ssa, + const zend_ssa_op *ssa_op, + uint32_t op1_info, + zend_jit_addr op1_addr, + uint32_t val_info, + bool op1_indirect, + zend_class_entry *ce, + bool ce_is_instanceof, + bool use_this, + zend_class_entry *trace_ce, + int may_throw) +{ + zval *member; + zend_string *name; + zend_property_info *prop_info; + zend_jit_addr val_addr = OP1_DATA_ADDR(); + zend_jit_addr res_addr = 0; + zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); + zend_jit_addr prop_addr; + bool needs_slow_path = 0; + + | brk #0 // TODO + return 1; +} + +static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, int may_throw) +{ + zend_jit_addr op1_addr = OP1_ADDR(); + + | brk #0 // TODO + return 1; +} + +static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) +{ + if (opline->op1_type == IS_CONST) { + zval *zv; + size_t len; + + zv = RT_CONSTANT(opline, opline->op1); + ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); + len = Z_STRLEN_P(zv); + + if (len > 0) { + const char *str = Z_STRVAL_P(zv); + + | SET_EX_OPLINE opline, TMP1 + | LOAD_ADDR CARG1, str + || if (len <= MAX_IMM12) { + | mov CARG2, #len + || } else { + | LOAD_64BIT_VAL CARG2, len + || } + | EXT_CALL zend_write, TMP1 + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + } else { + zend_jit_addr op1_addr = OP1_ADDR(); + + ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); + + | brk #0 // TODO: test + } + return 1; +} + +static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr) +{ + zend_jit_addr res_addr = RES_ADDR(); + + | brk #0 // TODO + return 1; +} + +static int zend_jit_count(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, int may_throw) +{ + zend_jit_addr res_addr = RES_ADDR(); + + | brk #0 // TODO + + if (may_throw) { + return zend_jit_check_exception(Dst); + } + return 1; +} + +static int zend_jit_load_this(dasm_State **Dst, uint32_t var) +{ + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); + + | brk #0 // TODO + return 1; +} + +static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool check_only) +{ + | brk #0 // TODO + return 1; +} + +static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, HashTable *jumptable, int default_b, const void *default_label, const zend_op *next_opline, zend_jit_trace_info *trace_info) +{ + uint32_t count; + Bucket *p; + const zend_op *target; + int b; + int32_t exit_point; + const void *exit_addr; + + | brk #0 // TODO + return 1; +} + +static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info) +{ + HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); + const zend_op *next_opline = NULL; + + if (trace) { + ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); + ZEND_ASSERT(trace->opline != NULL); + next_opline = trace->opline; + } + + | brk #0 // TODO + return 1; +} + +static bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info) +{ + zend_arg_info *arg_info = &op_array->arg_info[-1]; + ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type)); + zend_jit_addr op1_addr = OP1_ADDR(); + bool needs_slow_check = 1; + bool slow_check_in_cold = 1; + uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; + + | brk #0 // TODO + return 1; +} + +static int zend_jit_isset_isempty_cv(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) +{ + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + + | brk #0 // TODO + return 1; +} + +static int zend_jit_fe_reset(dasm_State **Dst, const zend_op *opline, uint32_t op1_info) +{ + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + + | brk #0 // TODO + return 1; +} + +static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, zend_uchar exit_opcode, const void *exit_addr) +{ + zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); + + | brk #0 // TODO + return 1; +} + +static int zend_jit_fetch_constant(dasm_State **Dst, + const zend_op *opline, + const zend_op_array *op_array, + zend_ssa *ssa, + const zend_ssa_op *ssa_op) +{ + zval *zv = RT_CONSTANT(opline, opline->op2) + 1; + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + zend_jit_addr const_addr = ZEND_ADDR_MEM_ZVAL(ZREG_X0, 0); // COPY_GPR0, use X0 here + uint32_t res_info = RES_INFO(); + + | brk #0 // TODO + return 1; +} + +static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) +{ + HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + + ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR); + ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING); + + | brk #0 // TODO + return 1; +} + +static bool zend_jit_noref_guard(dasm_State **Dst, const zend_op *opline, zend_jit_addr var_addr) +{ + int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | brk #0 // IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr + + return 1; +} + +static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_ref_guard, bool add_type_guard) +{ + zend_jit_addr var_addr = *var_addr_ptr; + uint32_t var_info = *var_info_ptr; + const void *exit_addr = NULL; + + if (add_ref_guard || add_type_guard) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + } + + if (add_ref_guard) { + | brk #0 // TODO + } + if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) { + /* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */ + if (Z_REG(var_addr) != ZREG_FCARG1x || Z_OFFSET(var_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, var_addr + } + | EXT_CALL zend_jit_unref_helper, TMP1 + } else { + | brk #0 // GET_ZVAL_PTR FCARG1x, var_addr + var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, offsetof(zend_reference, val)); + *var_addr_ptr = var_addr; + } + + if (var_type != IS_UNKNOWN) { + var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED); + } + if (add_type_guard + && var_type != IS_UNKNOWN + && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { + | brk #0 // TODO + + ZEND_ASSERT(var_info & (1 << var_type)); + if (var_type < IS_STRING) { + var_info = (1 << var_type); + } else if (var_type != IS_ARRAY) { + var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); + } else { + var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); + } + + *var_info_ptr = var_info; + } else { + var_info &= ~MAY_BE_REF; + *var_info_ptr = var_info; + } + + return 1; +} + +static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_indirect_guard) +{ + zend_jit_addr var_addr = *var_addr_ptr; + uint32_t var_info = *var_info_ptr; + int32_t exit_point; + const void *exit_addr; + + if (add_indirect_guard) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | brk #0 // TODO + } else { + /* May be already loaded into FCARG1a or RAX by previus FETCH_OBJ_W/DIM_W */ + if (opline->op1_type != IS_VAR || + (opline-1)->result_type != IS_VAR || + (opline-1)->result.var != opline->op1.var || + (opline-1)->op2_type == IS_VAR || + (opline-1)->op2_type == IS_TMP_VAR) { + | brk #0 // GET_ZVAL_PTR FCARG1x, var_addr + } else if ((opline-1)->opcode == ZEND_FETCH_DIM_W || (opline-1)->opcode == ZEND_FETCH_DIM_RW) { + | brk #0 // TODO + } + } + *var_info_ptr &= ~MAY_BE_INDIRECT; + var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + *var_addr_ptr = var_addr; + + if (var_type != IS_UNKNOWN) { + var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED); + } + if (!(var_type & IS_TRACE_REFERENCE) + && var_type != IS_UNKNOWN + && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { + exit_point = zend_jit_trace_get_exit_point(opline, 0); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + + | brk #0 // TODO + + //var_info = zend_jit_trace_type_to_info_ex(var_type, var_info); + ZEND_ASSERT(var_info & (1 << var_type)); + if (var_type < IS_STRING) { + var_info = (1 << var_type); + } else if (var_type != IS_ARRAY) { + var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); + } else { + var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); + } + + *var_info_ptr = var_info; + } + + return 1; +} + +static bool zend_jit_may_reuse_reg(const zend_op *opline, const zend_ssa_op *ssa_op, zend_ssa *ssa, int def_var, int use_var) +{ + if ((ssa->var_info[def_var].type & ~MAY_BE_GUARD) != (ssa->var_info[use_var].type & ~MAY_BE_GUARD)) { + return 0; + } + + switch (opline->opcode) { + case ZEND_QM_ASSIGN: + case ZEND_SEND_VAR: + case ZEND_ASSIGN: + case ZEND_PRE_INC: + case ZEND_PRE_DEC: + case ZEND_POST_INC: + case ZEND_POST_DEC: + return 1; + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: + case ZEND_BW_OR: + case ZEND_BW_AND: + case ZEND_BW_XOR: + if (def_var == ssa_op->result_def && + use_var == ssa_op->op1_use) { + return 1; + } + break; + default: + break; + } + return 0; +} + +static bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace) +{ + uint32_t op1_info, op2_info; + + switch (opline->opcode) { + case ZEND_QM_ASSIGN: + case ZEND_SEND_VAR: + case ZEND_SEND_VAL: + case ZEND_SEND_VAL_EX: + case ZEND_IS_SMALLER: + case ZEND_IS_SMALLER_OR_EQUAL: + case ZEND_IS_EQUAL: + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_IS_NOT_IDENTICAL: + case ZEND_CASE: + case ZEND_RETURN: + return 1; + case ZEND_ASSIGN: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + return + opline->op1_type == IS_CV && + !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_REF)) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))); + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_DOUBLE))); + case ZEND_BW_OR: + case ZEND_BW_AND: + case ZEND_BW_XOR: + case ZEND_SL: + case ZEND_SR: + case ZEND_MOD: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)); + case ZEND_PRE_INC: + case ZEND_PRE_DEC: + case ZEND_POST_INC: + case ZEND_POST_DEC: + op1_info = OP1_INFO(); + return opline->op1_type == IS_CV && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)); + case ZEND_BOOL: + case ZEND_BOOL_NOT: + case ZEND_JMPZ: + case ZEND_JMPNZ: + case ZEND_JMPZNZ: + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: + return 1; + case ZEND_FETCH_DIM_R: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + if (trace + && trace->op1_type != IS_UNKNOWN + && (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) { + op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY); + } + return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) && + (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || !(op1_info & MAY_BE_RC1)) && + (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) || + (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING) && + (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & MAY_BE_RC1)))); + } + return 0; +} + +static bool zend_jit_var_supports_reg(zend_ssa *ssa, int var) +{ + if (ssa->vars[var].no_val) { + /* we don't need the value */ + return 0; + } + + if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) { + /* Disable global register allocation, + * register allocation for SSA variables connected through Phi functions + */ + if (ssa->vars[var].definition_phi) { + return 0; + } + if (ssa->vars[var].phi_use_chain) { + zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; + do { + if (!ssa->vars[phi->ssa_var].no_val) { + return 0; + } + phi = zend_ssa_next_use_phi(ssa, var, phi); + } while (phi); + } + } + + if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) && + ((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) { + /* bad type */ + return 0; + } + + return 1; +} + +static bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var) +{ + if (!zend_jit_var_supports_reg(ssa, var)) { + return 0; + } + + if (ssa->vars[var].definition >= 0) { + uint32_t def = ssa->vars[var].definition; + if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) { + return 0; + } + } + + if (ssa->vars[var].use_chain >= 0) { + int use = ssa->vars[var].use_chain; + + do { + if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) && + !zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) { + return 0; + } + use = zend_ssa_next_use(ssa->ops, var, use); + } while (use >= 0); + } + + return 1; +} + +static zend_regset zend_jit_get_def_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) +{ + uint32_t op1_info, op2_info; + + switch (opline->opcode) { + case ZEND_FETCH_DIM_R: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + if (((opline->op1_type & (IS_TMP_VAR|IS_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) || + ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && + (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)))) { + return ZEND_REGSET(ZREG_FCARG1x); + } + break; + default: + break; + } + + return ZEND_REGSET_EMPTY; +} + +static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) +{ + uint32_t op1_info, op2_info, res_info; + zend_regset regset = ZEND_REGSET_SCRATCH; + + switch (opline->opcode) { + case ZEND_NOP: + case ZEND_OP_DATA: + case ZEND_JMP: + case ZEND_RETURN: + regset = ZEND_REGSET_EMPTY; + break; + case ZEND_QM_ASSIGN: + if (ssa_op->op1_def == current_var || + ssa_op->result_def == current_var) { + regset = ZEND_REGSET_EMPTY; + break; + } + /* break missing intentionally */ + case ZEND_SEND_VAL: + case ZEND_SEND_VAL_EX: + if (ssa_op->op1_use == current_var) { + regset = ZEND_REGSET_EMPTY; + break; + } + op1_info = OP1_INFO(); + if (!(op1_info & MAY_BE_UNDEF)) { + regset = ZEND_REGSET_EMPTY; + } + break; + case ZEND_SEND_VAR: + if (ssa_op->op1_use == current_var || + ssa_op->op1_def == current_var) { + regset = ZEND_REGSET_EMPTY; + break; + } + op1_info = OP1_INFO(); + if (!(op1_info & MAY_BE_UNDEF)) { + regset = ZEND_REGSET_EMPTY; + } + break; + case ZEND_ASSIGN: + if (ssa_op->op2_use == current_var || + ssa_op->op2_def == current_var || + ssa_op->op1_def == current_var || + ssa_op->result_def == current_var) { + regset = ZEND_REGSET_EMPTY; + break; + } + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + if (opline->op1_type == IS_CV + && !(op2_info & MAY_BE_UNDEF) + && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { + regset = ZEND_REGSET_EMPTY; + } + break; + case ZEND_PRE_INC: + case ZEND_PRE_DEC: + case ZEND_POST_INC: + case ZEND_POST_DEC: + if (ssa_op->op1_use == current_var || + ssa_op->op1_def == current_var || + ssa_op->result_def == current_var) { + regset = ZEND_REGSET_EMPTY; + break; + } + op1_info = OP1_INFO(); + if (opline->op1_type == IS_CV + && (op1_info & MAY_BE_LONG) + && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { + regset = ZEND_REGSET_EMPTY; + } + break; + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { + + regset = ZEND_REGSET_EMPTY; + } + break; + case ZEND_BW_OR: + case ZEND_BW_AND: + case ZEND_BW_XOR: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { + regset = ZEND_REGSET_EMPTY; + } + break; + case ZEND_SL: + case ZEND_SR: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { + regset = ZEND_REGSET_EMPTY; + } + break; + case ZEND_MOD: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { + regset = ZEND_REGSET_EMPTY; + } + break; + case ZEND_IS_SMALLER: + case ZEND_IS_SMALLER_OR_EQUAL: + case ZEND_IS_EQUAL: + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_IS_NOT_IDENTICAL: + case ZEND_CASE: + op1_info = OP1_INFO(); + op2_info = OP2_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && + !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { + regset = ZEND_REGSET_EMPTY; + } + break; + case ZEND_BOOL: + case ZEND_BOOL_NOT: + case ZEND_JMPZ: + case ZEND_JMPNZ: + case ZEND_JMPZNZ: + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: + op1_info = OP1_INFO(); + if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) { + regset = ZEND_REGSET_EMPTY; + } + break; + case ZEND_DO_UCALL: + case ZEND_DO_FCALL: + case ZEND_DO_FCALL_BY_NAME: + case ZEND_INCLUDE_OR_EVAL: + case ZEND_GENERATOR_CREATE: + case ZEND_YIELD: + case ZEND_YIELD_FROM: + regset = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP); + break; + default: + break; + } + + return regset; +} + +#if defined(__clang__) +# pragma clang diagnostic pop +#endif + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * indent-tabs-mode: t + * End: + */ diff --git a/ext/opcache/jit/zend_jit_arm64.h b/ext/opcache/jit/zend_jit_arm64.h new file mode 100644 index 0000000000000..b6e063de2bd81 --- /dev/null +++ b/ext/opcache/jit/zend_jit_arm64.h @@ -0,0 +1,142 @@ +/* + +----------------------------------------------------------------------+ + | Zend JIT | + +----------------------------------------------------------------------+ + | Copyright (c) The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Authors: Dmitry Stogov | + | Hao Sun | + +----------------------------------------------------------------------+ +*/ + +#ifndef HAVE_JIT_ARM64_H +#define HAVE_JIT_ARM64_H + +typedef enum _zend_reg { + ZREG_NONE = -1, + + ZREG_X0, + ZREG_X1, + ZREG_X2, + ZREG_X3, + ZREG_X4, + ZREG_X5, + ZREG_X6, + ZREG_X7, + ZREG_X8, + ZREG_X9, + ZREG_X10, + ZREG_X11, + ZREG_X12, + ZREG_X13, + ZREG_X14, + ZREG_X15, + ZREG_X16, + ZREG_X17, + ZREG_X18, + ZREG_X19, + ZREG_X20, + ZREG_X21, + ZREG_X22, + ZREG_X23, + ZREG_X24, + ZREG_X25, + ZREG_X26, + ZREG_X27, + ZREG_X28, + ZREG_X29, + ZREG_X30, + ZREG_X31, + + ZREG_V0, + ZREG_V1, + ZREG_V2, + ZREG_V3, + ZREG_V4, + ZREG_V5, + ZREG_V6, + ZREG_V7, + ZREG_V8, + ZREG_V9, + ZREG_V10, + ZREG_V11, + ZREG_V12, + ZREG_V13, + ZREG_V14, + ZREG_V15, + ZREG_V16, + ZREG_V17, + ZREG_V18, + ZREG_V19, + ZREG_V20, + ZREG_V21, + ZREG_V22, + ZREG_V23, + ZREG_V24, + ZREG_V25, + ZREG_V26, + ZREG_V27, + ZREG_V28, + ZREG_V29, + ZREG_V30, + ZREG_V31, + + ZREG_NUM, + + ZREG_THIS, /* used for delayed FETCH_THIS deoptimization */ + + /* pseudo constants used by deoptimizer */ + ZREG_LONG_MIN_MINUS_1, + ZREG_LONG_MIN, + ZREG_LONG_MAX, + ZREG_LONG_MAX_PLUS_1, + ZREG_NULL, + + ZREG_ZVAL_TRY_ADDREF, + ZREG_ZVAL_COPY_GPR0, +} zend_reg; + +typedef struct _zend_jit_registers_buf { + uint64_t gpr[32]; /* general purpose integer register */ + double fpr[32]; /* floating point registers */ +} zend_jit_registers_buf; + +#define ZREG_FIRST_FPR ZREG_V0 + +#define ZREG_RSP ZREG_X31 +#define ZREG_RLR ZREG_X30 +#define ZREG_RFP ZREG_X29 +#define ZREG_RPR ZREG_X18 + +# define ZREG_FP ZREG_X27 +# define ZREG_IP ZREG_X28 +# define ZREG_RX ZREG_IP + +typedef uint64_t zend_regset; + +# define ZEND_REGSET_FIXED \ + (ZEND_REGSET(ZREG_RSP) | ZEND_REGSET(ZREG_RLR) | ZEND_REGSET(ZREG_RFP) | \ + ZEND_REGSET(ZREG_RPR) | ZEND_REGSET(ZREG_FP) | ZEND_REGSET(ZREG_IP) | \ + ZEND_REGSET_INTERVAL(ZREG_X8, ZREG_X11) | \ + ZEND_REGSET_INTERVAL(ZREG_V16, ZREG_V17)) +# define ZEND_REGSET_GP \ + ZEND_REGSET_DIFFERENCE(ZEND_REGSET_INTERVAL(ZREG_X0, ZREG_X30), ZEND_REGSET_FIXED) +# define ZEND_REGSET_FP \ + ZEND_REGSET_DIFFERENCE(ZEND_REGSET_INTERVAL(ZREG_V0, ZREG_V31), ZEND_REGSET_FIXED) +# define ZEND_REGSET_SCRATCH \ + (ZEND_REGSET_INTERVAL(ZREG_X0, ZREG_X17) | ZEND_REGSET_INTERVAL(ZREG_V0, ZREG_V7) | \ + ZEND_REGSET_INTERVAL(ZREG_V16, ZREG_V31)) +# define ZEND_REGSET_PRESERVED \ + (ZEND_REGSET_INTERVAL(ZREG_X19, ZREG_X28) | ZEND_REGSET_INTERVAL(ZREG_V8, ZREG_V15)) + +#define ZEND_REGSET_LOW_PRIORITY ZEND_REGSET_EMPTY + +#endif /* ZEND_JIT_ARM64_H */ diff --git a/ext/opcache/jit/zend_jit_disasm.c b/ext/opcache/jit/zend_jit_disasm.c index 451b511793926..371956173a223 100644 --- a/ext/opcache/jit/zend_jit_disasm.c +++ b/ext/opcache/jit/zend_jit_disasm.c @@ -225,6 +225,7 @@ static uint64_t zend_jit_disasm_branch_target(csh cs, const cs_insn *insn) { unsigned int i; +#if defined(__x86_64__) || defined(i386) if (cs_insn_group(cs, insn, X86_GRP_JUMP)) { for (i = 0; i < insn->detail->x86.op_count; i++) { if (insn->detail->x86.operands[i].type == X86_OP_IMM) { @@ -232,6 +233,14 @@ static uint64_t zend_jit_disasm_branch_target(csh cs, const cs_insn *insn) } } } +#elif defined(__aarch64__) + if (cs_insn_group(cs, insn, ARM64_GRP_JUMP)) { + for (i = 0; i < insn->detail->arm64.op_count; i++) { + if (insn->detail->arm64.operands[i].type == ARM64_OP_IMM) + return insn->detail->arm64.operands[i].imm; + } + } +#endif return 0; } @@ -319,6 +328,11 @@ static int zend_jit_disasm(const char *name, # else cs_option(cs, CS_OPT_SYNTAX, CS_OPT_SYNTAX_ATT); # endif +# elif defined(__aarch64__) + if (cs_open(CS_ARCH_ARM64, CS_MODE_ARM, &cs) != CS_ERR_OK) + return 0; + cs_option(cs, CS_OPT_DETAIL, CS_OPT_ON); + cs_option(cs, CS_OPT_SYNTAX, CS_OPT_SYNTAX_ATT); # else if (cs_open(CS_ARCH_X86, CS_MODE_32, &cs) != CS_ERR_OK) return 0; @@ -431,7 +445,11 @@ static int zend_jit_disasm(const char *name, } # ifdef HAVE_CAPSTONE_ITER +# if defined(__aarch64__) + fprintf(stderr, " "ZEND_XLONG_FMT":\t%s ", insn->address, insn->mnemonic); +# else fprintf(stderr, "\t%s ", insn->mnemonic); +# endif p = insn->op_str; # else fprintf(stderr, "\t%s ", insn[i].mnemonic); diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index 0717ee32364bb..0a5e5cfa3badb 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -20,7 +20,12 @@ +----------------------------------------------------------------------+ */ + +#if defined(__x86_64__) || defined(i386) #define HAVE_GDB +#else +#warning Missing GDB JIT support on this platform +#endif #ifdef HAVE_GDB diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 83bf939e5eed6..d16dd39591973 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -14,6 +14,7 @@ +----------------------------------------------------------------------+ | Authors: Dmitry Stogov | | Xinchen Hui | + | Hao Sun | +----------------------------------------------------------------------+ */ @@ -26,11 +27,24 @@ #define ZEND_REGSET_IS_EMPTY(regset) \ (regset == ZEND_REGSET_EMPTY) +#define ZEND_REGSET_IS_SINGLETON(regset) \ + (regset && !(regset & (regset - 1))) + +#if (ZREG_NUM <= 32) #define ZEND_REGSET(reg) \ (1u << (reg)) +#else +#define ZEND_REGSET(reg) \ + (1ull << (reg)) +#endif +#if (ZREG_NUM <= 32) #define ZEND_REGSET_INTERVAL(reg1, reg2) \ (((1u << ((reg2) - (reg1) + 1)) - 1) << (reg1)) +#else +#define ZEND_REGSET_INTERVAL(reg1, reg2) \ + (((1ull << ((reg2) - (reg1) + 1)) - 1) << (reg1)) +#endif #define ZEND_REGSET_IN(regset, reg) \ (((regset) & ZEND_REGSET(reg)) != 0) @@ -50,7 +64,11 @@ #define ZEND_REGSET_DIFFERENCE(set1, set2) \ ((set1) & ~(set2)) -#ifndef _WIN32 +#if defined (__aarch64__) +# define ZEND_REGSET_FIRST(set) ((zend_reg)__builtin_ctzll(set)) +# define ZEND_REGSET_SECOND(set) ((zend_reg)__builtin_ctzll(set ^ (1ull << ZEND_REGSET_FIRST(set)))) +# define ZEND_REGSET_LAST(set) ((zend_reg)(__builtin_clzll(set)^63)) +#elif !defined(_WIN32) # if (ZREG_NUM <= 32) # define ZEND_REGSET_FIRST(set) ((zend_reg)__builtin_ctz(set)) # define ZEND_REGSET_LAST(set) ((zend_reg)(__builtin_clz(set)^31)) @@ -711,4 +729,17 @@ static zend_always_inline bool zend_jit_may_be_polymorphic_call(const zend_op *o } } +/* Instruction cache flush */ +#ifndef JIT_CACHE_FLUSH +# if defined (__aarch64__) +# if ((defined(__GNUC__) && ZEND_GCC_VERSION >= 4003) || __has_builtin(__builtin___clear_cache)) +# define JIT_CACHE_FLUSH(from, to) __builtin___clear_cache((char*)(from), (char*)(to)) +# else +# error "Missing builtin to flush instruction cache for AArch64" +# endif +# else /* Not required to implement on archs with unified caches */ +# define JIT_CACHE_FLUSH(from, to) +# endif +#endif /* !JIT_CACHE_FLUSH */ + #endif /* ZEND_JIT_INTERNAL_H */ diff --git a/ext/opcache/jit/zend_jit_perf_dump.c b/ext/opcache/jit/zend_jit_perf_dump.c index d8f2d6130e8cc..ace998fe9d9ad 100644 --- a/ext/opcache/jit/zend_jit_perf_dump.c +++ b/ext/opcache/jit/zend_jit_perf_dump.c @@ -22,6 +22,9 @@ #include #include #include +#include +#include +#include #if defined(__linux__) #include diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index f27cf39b4cf29..c808f7fcd55e4 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -28,7 +28,12 @@ #include "Optimizer/zend_func_info.h" #include "Optimizer/zend_call_graph.h" #include "zend_jit.h" +#if defined(__x86_64__) || defined(i386) #include "zend_jit_x86.h" +#elif defined(__aarch64__) +#include "zend_jit_arm64.h" +#endif + #include "zend_jit_internal.h" #ifdef HAVE_GCC_GLOBAL_REGS @@ -36,9 +41,12 @@ # if defined(__x86_64__) register zend_execute_data* volatile execute_data __asm__("%r14"); register const zend_op* volatile opline __asm__("%r15"); -# else +# elif defined(i386) register zend_execute_data* volatile execute_data __asm__("%esi"); register const zend_op* volatile opline __asm__("%edi"); +# elif defined(__aarch64__) +register zend_execute_data* volatile execute_data __asm__("x27"); +register const zend_op* volatile opline __asm__("x28"); # endif # pragma GCC diagnostic warning "-Wvolatile-register-var" #endif diff --git a/ext/opcache/tests/jit/arm64/add_001.phpt b/ext/opcache/tests/jit/arm64/add_001.phpt new file mode 100644 index 0000000000000..dc89e6d0216e2 --- /dev/null +++ b/ext/opcache/tests/jit/arm64/add_001.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT ADD: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--EXPECT-- +int(2) diff --git a/ext/opcache/tests/jit/arm64/add_002.phpt b/ext/opcache/tests/jit/arm64/add_002.phpt new file mode 100644 index 0000000000000..b575de26e895a --- /dev/null +++ b/ext/opcache/tests/jit/arm64/add_002.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT ADD: 002 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--EXPECT-- +int(4097) diff --git a/ext/opcache/tests/jit/arm64/add_003.phpt b/ext/opcache/tests/jit/arm64/add_003.phpt new file mode 100644 index 0000000000000..102d07186992c --- /dev/null +++ b/ext/opcache/tests/jit/arm64/add_003.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT ADD: 003 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--EXPECT-- +float(9.223372036854776E+18) diff --git a/ext/opcache/tests/jit/arm64/add_004.phpt b/ext/opcache/tests/jit/arm64/add_004.phpt new file mode 100644 index 0000000000000..97daf4af7593f --- /dev/null +++ b/ext/opcache/tests/jit/arm64/add_004.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT ADD: 004 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--EXPECT-- +float(9.223372036854776E+18) diff --git a/ext/opcache/tests/jit/arm64/add_005.phpt b/ext/opcache/tests/jit/arm64/add_005.phpt new file mode 100644 index 0000000000000..14bc03e7a5e52 --- /dev/null +++ b/ext/opcache/tests/jit/arm64/add_005.phpt @@ -0,0 +1,24 @@ +--TEST-- +JIT ADD: 005 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--EXPECTF-- +Fatal error: Uncaught TypeError: Unsupported operand types: string + int in %s:%d +Stack trace: +#0 %s(%d): foo('hello') +#1 {main} + thrown in %s on line %d diff --git a/ext/opcache/tests/jit/arm64/hot_func_001.phpt b/ext/opcache/tests/jit/arm64/hot_func_001.phpt new file mode 100644 index 0000000000000..c9d17d2fdcd26 --- /dev/null +++ b/ext/opcache/tests/jit/arm64/hot_func_001.phpt @@ -0,0 +1,24 @@ +--TEST-- +JIT HOT_FUNC: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +opcache.jit=tracing +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--EXPECT-- +string(5) "hello" +string(5) "hello" +string(5) "hello" diff --git a/ext/opcache/tests/jit/arm64/hot_func_002.phpt b/ext/opcache/tests/jit/arm64/hot_func_002.phpt new file mode 100644 index 0000000000000..3006c4551f62e --- /dev/null +++ b/ext/opcache/tests/jit/arm64/hot_func_002.phpt @@ -0,0 +1,25 @@ +--TEST-- +JIT HOT_FUNC: 002 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +opcache.jit=tracing +opcache.jit_hot_func=2 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--EXPECT-- +string(5) "hello" +string(5) "hello" +string(5) "hello" diff --git a/ext/opcache/tests/jit/arm64/icall_001.phpt b/ext/opcache/tests/jit/arm64/icall_001.phpt new file mode 100644 index 0000000000000..c0ea68a51f588 --- /dev/null +++ b/ext/opcache/tests/jit/arm64/icall_001.phpt @@ -0,0 +1,24 @@ +--TEST-- +JIT ICALL: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--EXPECT-- +bool(true) +int(0) +int(42) +int(-42) +float(0) +float(2) +string(5) "hello" +array(0) { +} diff --git a/ext/opcache/tests/jit/arm64/loop_001.phpt b/ext/opcache/tests/jit/arm64/loop_001.phpt new file mode 100644 index 0000000000000..3becd5bbcf7bc --- /dev/null +++ b/ext/opcache/tests/jit/arm64/loop_001.phpt @@ -0,0 +1,22 @@ +--TEST-- +JIT LOOP: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--EXPECT-- +10 diff --git a/ext/opcache/tests/jit/arm64/loop_002.phpt b/ext/opcache/tests/jit/arm64/loop_002.phpt new file mode 100644 index 0000000000000..38ffac591ba78 --- /dev/null +++ b/ext/opcache/tests/jit/arm64/loop_002.phpt @@ -0,0 +1,26 @@ +--TEST-- +JIT HOT LOOP: 002 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +opcache.jit=tracing +opcache.jit_hot_func=2 +opcache.jit_hot_loop=2 +opcache.jit_hot_side_exit=0 +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--EXPECT-- +10 diff --git a/ext/opcache/tests/jit/arm64/recv_001.phpt b/ext/opcache/tests/jit/arm64/recv_001.phpt new file mode 100644 index 0000000000000..7852101cf1223 --- /dev/null +++ b/ext/opcache/tests/jit/arm64/recv_001.phpt @@ -0,0 +1,26 @@ +--TEST-- +JIT RECV: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--EXPECT-- +int(1) +float(1) +string(5) "hello" +array(0) { +} diff --git a/ext/opcache/tests/jit/arm64/ret_001.phpt b/ext/opcache/tests/jit/arm64/ret_001.phpt new file mode 100644 index 0000000000000..a750d5b2e98e3 --- /dev/null +++ b/ext/opcache/tests/jit/arm64/ret_001.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT RET: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--EXPECT-- +int(1) diff --git a/ext/opcache/tests/jit/arm64/ret_002.phpt b/ext/opcache/tests/jit/arm64/ret_002.phpt new file mode 100644 index 0000000000000..4ae3efd7332b9 --- /dev/null +++ b/ext/opcache/tests/jit/arm64/ret_002.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT RET: 002 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--EXPECT-- +float(1) diff --git a/ext/opcache/tests/jit/arm64/ret_003.phpt b/ext/opcache/tests/jit/arm64/ret_003.phpt new file mode 100644 index 0000000000000..12bcaa9b76f59 --- /dev/null +++ b/ext/opcache/tests/jit/arm64/ret_003.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT RET: 003 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--EXPECT-- +string(5) "hello" diff --git a/ext/opcache/tests/jit/arm64/skipif.inc b/ext/opcache/tests/jit/arm64/skipif.inc new file mode 100644 index 0000000000000..c5a81810391b7 --- /dev/null +++ b/ext/opcache/tests/jit/arm64/skipif.inc @@ -0,0 +1,3 @@ + diff --git a/ext/opcache/tests/jit/arm64/ucall_001.phpt b/ext/opcache/tests/jit/arm64/ucall_001.phpt new file mode 100644 index 0000000000000..c3fea8c9dafce --- /dev/null +++ b/ext/opcache/tests/jit/arm64/ucall_001.phpt @@ -0,0 +1,19 @@ +--TEST-- +JIT UCALL: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--EXPECT-- +string(5) "hello" diff --git a/ext/opcache/tests/jit/arm64/ucall_002.phpt b/ext/opcache/tests/jit/arm64/ucall_002.phpt new file mode 100644 index 0000000000000..519454afe1299 --- /dev/null +++ b/ext/opcache/tests/jit/arm64/ucall_002.phpt @@ -0,0 +1,21 @@ +--TEST-- +JIT UCALL: 002 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--EXPECT-- +string(5) "hello" +string(6) "world!" diff --git a/ext/opcache/tests/jit/arm64/ucall_003.phpt b/ext/opcache/tests/jit/arm64/ucall_003.phpt new file mode 100644 index 0000000000000..f02b91f72275b --- /dev/null +++ b/ext/opcache/tests/jit/arm64/ucall_003.phpt @@ -0,0 +1,22 @@ +--TEST-- +JIT UCALL: 003 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--EXPECT-- +string(5) "hello" diff --git a/ext/opcache/tests/jit/arm64/ucall_004.phpt b/ext/opcache/tests/jit/arm64/ucall_004.phpt new file mode 100644 index 0000000000000..6cbb17ed26434 --- /dev/null +++ b/ext/opcache/tests/jit/arm64/ucall_004.phpt @@ -0,0 +1,23 @@ +--TEST-- +JIT UCALL: 004 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--SKIPIF-- + +--FILE-- + +--EXPECT-- +string(5) "hello" +string(5) "hello" +string(5) "hello" From 657b00d7c069e81ae4e19e366fad21f7096e6aa9 Mon Sep 17 00:00:00 2001 From: haosun01 Date: Fri, 9 Apr 2021 03:53:27 +0000 Subject: [PATCH 037/229] Hybrid use of registers 1. one **hybrid** solution of register usage After the discussion with Dmitry, we may want to propose one hybrid solution of register usage. 1) Following the x86 implementation, we define REG0/1/2 to be the scratch registers. Clever tricks are utilized in x86 implementation for better register allocation. Note that we define REG0/1/2 as x8/9/10. One reason is that R0 and FCARG1 should be distinguished. 2) Temporary registers are also reserved(i.e. they are excluded from the candidates of register allocator), and they would be used due to the different addressing modes in AArch64. 2. update the 'make clean' target. 3. remove the unnecessary AArch64 related macros in zend_jit_internal.h. [ci skip] Change-Id: I627157b88b2344530d705751eb7f73a223ed83e5 CustomizedGitHooks: yes --- build/Makefile.global | 3 +- ext/opcache/jit/zend_jit_arm64.dasc | 692 ++++++++++++++++++---------- ext/opcache/jit/zend_jit_arm64.h | 39 +- ext/opcache/jit/zend_jit_internal.h | 10 +- ext/opcache/jit/zend_jit_trace.c | 4 +- ext/opcache/jit/zend_jit_x86.h | 1 + 6 files changed, 484 insertions(+), 265 deletions(-) diff --git a/build/Makefile.global b/build/Makefile.global index 41151163fb72a..6941bab6ad412 100644 --- a/build/Makefile.global +++ b/build/Makefile.global @@ -117,6 +117,7 @@ clean: find . -name .libs -a -type d|xargs rm -rf rm -f libphp.la $(SAPI_CLI_PATH) $(SAPI_CGI_PATH) $(SAPI_LITESPEED_PATH) $(SAPI_FPM_PATH) $(OVERALL_TARGET) modules/* libs/* rm -f ext/opcache/jit/zend_jit_x86.c + rm -f ext/opcache/jit/zend_jit_arm64.c distclean: clean rm -f Makefile config.cache config.log config.status Makefile.objects Makefile.fragments libtool main/php_config.h main/internal_functions_cli.c main/internal_functions.c Zend/zend_dtrace_gen.h Zend/zend_dtrace_gen.h.bak Zend/zend_config.h @@ -125,8 +126,6 @@ distclean: clean rm -f scripts/man1/phpize.1 scripts/php-config scripts/man1/php-config.1 sapi/cli/php.1 sapi/cgi/php-cgi.1 sapi/phpdbg/phpdbg.1 ext/phar/phar.1 ext/phar/phar.phar.1 rm -f sapi/fpm/php-fpm.conf sapi/fpm/init.d.php-fpm sapi/fpm/php-fpm.service sapi/fpm/php-fpm.8 sapi/fpm/status.html rm -f ext/phar/phar.phar ext/phar/phar.php - rm -f ext/opcache/jit/zend_jit_x86.c - rm -f ext/opcache/jit/zend_jit_arm64.c if test "$(srcdir)" != "$(builddir)"; then \ rm -f ext/phar/phar/phar.inc; \ fi diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index b4a83e4b8930b..3054efe7b8389 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -47,32 +47,37 @@ |.define A2, [r4+0x4] |.define A1, [r4] +// We use REG0/1/2 and FPR0/1 to replace r0/1/2 and xmm0/1 in the x86 implementation. +// Scratch registers +|.define REG0, x8 +|.define REG0w, w8 +|.define REG1, x9 +|.define REG1w, w9 +|.define REG2, x10 +|.define REG2w, w10 +|.define FPR0, v0 +|.define FPR1, v1 + +|.define ZREG_REG0, ZREG_X8 +|.define ZREG_REG1, ZREG_X9 +|.define ZREG_REG2, ZREG_X10 +|.define ZREG_FPR0, ZREG_V0 +|.define ZREG_FPR1, ZREG_V1 + // Temporaries, not preserved across calls -|.define TMP1, x8 -|.define TMP1w, w8 -|.define TMP2, x9 -|.define TMP2w, w9 -|.define TMP3, x10 -|.define TMP3w, w10 -|.define TMP4, x11 -|.define TMP4w, w11 -|.define FPTMP1, v16 -|.define FPTMP2, v17 - -// Temporary register index in _zend_reg -|.define ZREG_TMP1, ZREG_X8 -|.define ZREG_TMP2, ZREG_X9 -|.define ZREG_TMP3, ZREG_X10 -|.define ZREG_TMP4, ZREG_X11 -|.define ZREG_FPTMP1, ZREG_V16 -|.define ZREG_FPTMP2, ZREG_V17 - -#define ZREG_TMP1 ZREG_X8 -#define ZREG_TMP2 ZREG_X9 -#define ZREG_TMP3 ZREG_X10 -#define ZREG_TMP4 ZREG_X11 -#define ZREG_FPTMP1 ZREG_V16 -#define ZREG_FPTMP2 ZREG_V17 +|.define TMP1, x11 +|.define TMP1w, w11 +|.define TMP2, x12 +|.define TMP2w, w12 +|.define TMP3, x13 +|.define TMP3w, w13 +|.define TMP4, x14 +|.define TMP4w, w14 + +|.define ZREG_TMP1, ZREG_X11 +|.define ZREG_TMP2, ZREG_X12 +|.define ZREG_TMP3, ZREG_X13 +|.define ZREG_TMP4, ZREG_X14 |.define HYBRID_SPAD, #16 // padding for stack alignment @@ -1001,7 +1006,7 @@ static void* dasm_labels[zend_lb_MAX]; || } | // if (!Z_DELREF_P(cv)) { | GET_ZVAL_PTR FCARG1x, addr, Rx(tmp_reg2) -| GC_DELREF FCARG1x, Rw(tmp_reg1) +| GC_DELREF FCARG1x, Rw(tmp_reg1) || if (RC_MAY_BE_1(op_info)) { || if (RC_MAY_BE_N(op_info)) { || if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { @@ -1218,14 +1223,14 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) const void *handler = zend_get_opcode_handler_func(EG(exception_op)); | ADD_HYBRID_SPAD - | EXT_CALL handler, TMP1 + | EXT_CALL handler, REG0 | JMP_IP TMP1 } else { const void *handler = EG(exception_op)->handler; if (GCC_GLOBAL_REGS) { | add sp, sp, SPAD // stack alignment - | EXT_JMP handler, TMP1 + | EXT_JMP handler, REG0 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { | brk #0 // TODO: test } else { @@ -1233,7 +1238,7 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) | ldp FP, RX, T2 // retore FP and IP | ldr LR, T4 // retore LR | add sp, sp, NR_SPAD // stack alignment - | EXT_JMP handler, TMP1 + | EXT_JMP handler, REG0 } } @@ -1371,7 +1376,7 @@ static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst) } |->hybrid_runtime_jit: - | EXT_CALL zend_runtime_jit, TMP1 + | EXT_CALL zend_runtime_jit, REG0 | JMP_IP TMP1 return 1; } @@ -1459,40 +1464,40 @@ static int zend_jit_hybrid_hot_trace_stub(dasm_State **Dst) } // On entry from counter stub: - // TMP4 -> zend_op_trace_info.counter + // REG2 -> zend_op_trace_info.counter |->hybrid_hot_trace: | mov TMP1w, #ZEND_JIT_COUNTER_INIT - | strh TMP1w, [TMP4] + | strh TMP1w, [REG2] | mov FCARG1x, FP | GET_IP FCARG2x - | EXT_CALL zend_jit_trace_hot_root, TMP1 + | EXT_CALL zend_jit_trace_hot_root, REG0 | cmp RETVALw, #0 // Result is < 0 on failure. | blt >1 - | MEM_LOAD_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 + | MEM_LOAD_ZTS ldr, FP, executor_globals, current_execute_data, REG0 | LOAD_IP | JMP_IP TMP1 |1: - | EXT_JMP zend_jit_halt_op->handler, TMP1 + | EXT_JMP zend_jit_halt_op->handler, REG0 return 1; } static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost) { - | ldr TMP1, EX->func - | ldr TMP2, [TMP1, #offsetof(zend_op_array, reserved[zend_func_info_rid])] - | ldr TMP2, [TMP2, #offsetof(zend_jit_op_array_trace_extension, offset)] - | add TMP3, TMP2, IP - | ldr TMP4, [TMP3, #offsetof(zend_op_trace_info, counter)] - | ldrh TMP1w, [TMP4] - | LOAD_32BIT_VAL TMP2w, cost - | sub TMP1w, TMP1w, TMP2w - | strh TMP1w, [TMP4] - | cmp TMP1w, #0 + | ldr REG0, EX->func + | ldr REG1, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | ldr REG1, [REG1, #offsetof(zend_jit_op_array_trace_extension, offset)] + | add TMP1, REG1, IP + | ldr REG2, [TMP1, #offsetof(zend_op_trace_info, counter)] + | ldrh TMP2w, [REG2] + | LOAD_32BIT_VAL TMP3w, cost + | sub TMP2w, TMP2w, TMP3w + | strh TMP2w, [REG2] + | cmp TMP2w, #0 | ble ->hybrid_hot_trace - | ldr TMP1, [TMP3, #offsetof(zend_op_trace_info, orig_handler)] - | br TMP1 + | ldr TMP2, [TMP1, #offsetof(zend_op_trace_info, orig_handler)] + | br TMP2 return 1; } @@ -1586,7 +1591,7 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst) | // EX(opline) = opline | SAVE_IP | // zend_jit_trace_exit(trace_num, exit_num) - | EXT_CALL zend_jit_trace_exit, TMP1 + | EXT_CALL zend_jit_trace_exit, REG0 | | add sp, sp, #(33 * 16) // including the pre-allocated 16 bytes | @@ -1595,7 +1600,7 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst) | bne >1 // not zero | // execute_data = EG(current_execute_data) - | MEM_LOAD_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 + | MEM_LOAD_ZTS ldr, FP, executor_globals, current_execute_data, REG0 | // opline = EX(opline) | LOAD_IP @@ -1618,30 +1623,39 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst) | blt ->trace_halt | // execute_data = EG(current_execute_data) - | MEM_LOAD_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 + | MEM_LOAD_ZTS ldr, FP, executor_globals, current_execute_data, REG0 | // opline = EX(opline) | LOAD_IP | // check for interrupt (try to avoid this ???) - | MEM_LOAD_CMP_ZTS ldrb, wzr, executor_globals, vm_interrupt, TMP1w, TMP2 + | MEM_LOAD_CMP_ZTS ldrb, wzr, executor_globals, vm_interrupt, REG0w, TMP1 | bne ->interrupt_handler if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | ADD_HYBRID_SPAD - | ldr TMP1, EX->func - | ldr TMP1, [TMP1, #offsetof(zend_op_array, reserved[zend_func_info_rid])] - | ldr TMP1, [TMP1, #offsetof(zend_jit_op_array_trace_extension, offset)] - | add TMP1, IP, TMP1 - | ldr TMP1, [TMP1] - | br TMP1 + | ldr REG0, EX->func + | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] + | add REG0, IP, REG0 + | ldr REG0, [REG0] + | br REG0 } else if (GCC_GLOBAL_REGS) { | add sp, sp, SPAD // stack alignment - | ldr TMP1, EX->func - | ldr TMP1, [TMP1, #offsetof(zend_op_array, reserved[zend_func_info_rid])] - | ldr TMP1, [TMP1, #offsetof(zend_jit_op_array_trace_extension, offset)] - | add TMP1, IP, TMP1 - | ldr TMP1, [TMP1] - | br TMP1 + | ldr REG0, EX->func + | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] + | add REG0, IP, REG0 + | ldr REG0, [REG0] + | br REG0 + } else { + | ldr IP, EX->opline + | mov FCARG1x, FP + | ldr REG0, EX->func + | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] + | add REG0, IP, REG0 + | ldr REG0, [REG0] + | blr REG0 | | tst RETVALw, RETVALw | blt ->trace_halt @@ -1993,8 +2007,8 @@ static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level) | str xzr, EX:RX->prev_execute_data } else { | brk #0 // TODO: test - | ldr TMP1, EX->call - | str TMP1, EX:RX->prev_execute_data + | ldr REG0, EX->call + | str REG0, EX:RX->prev_execute_data } | // EX(call) = call; | str RX, EX->call @@ -2044,7 +2058,7 @@ static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void *timeout_exit_addr) { if (timeout_exit_addr) { - | MEM_LOAD_CMP_ZTS ldrb, wzr, executor_globals, vm_interrupt, TMP1w, TMP2 + | MEM_LOAD_CMP_ZTS ldrb, wzr, executor_globals, vm_interrupt, REG0w, TMP1 | beq =>loop_label | EXT_JMP timeout_exit_addr, TMP1 } else { @@ -2056,7 +2070,7 @@ static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void static int zend_jit_check_exception(dasm_State **Dst) { - | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, TMP1, TMP2 + | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, REG0, TMP1 | bne ->exception_handler return 1; } @@ -2089,7 +2103,7 @@ static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_t if (STACK_REG(parent_stack, i) < ZREG_NUM) { ZEND_REGSET_EXCL(regset, STACK_REG(parent_stack, i)); } else if (STACK_REG(parent_stack, i) == ZREG_ZVAL_COPY_GPR0) { - ZEND_REGSET_EXCL(regset, ZREG_X0); + ZEND_REGSET_EXCL(regset, ZREG_REG0); } } } @@ -2097,7 +2111,7 @@ static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_t } if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) { - ZEND_REGSET_EXCL(regset, ZREG_X0); + ZEND_REGSET_EXCL(regset, ZREG_REG0); } current_trace_num = trace_num; @@ -2112,7 +2126,7 @@ static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_t | add sp, sp, #16 } else { zend_reg tmp1 = ZEND_REGSET_FIRST(regset); - zend_reg tmp2 = ZEND_REGSET_SECOND(regset); + zend_reg tmp2 = ZEND_REGSET_FIRST(ZEND_REGSET_EXCL(regset, tmp1)); | LOAD_32BIT_VAL Rw(tmp1), trace_num | MEM_STORE_ZTS str, Rw(tmp1), executor_globals, jit_trace_num, Rx(tmp2) @@ -2157,12 +2171,12 @@ static int zend_jit_trace_return(dasm_State **Dst, bool original_handler) | JMP_IP TMP1 } else { | brk #0 // TODO: test - | ldr TMP1, EX->func - | ldr TMP1, [TMP1, #offsetof(zend_op_array, reserved[zend_func_info_rid])] - | ldr TMP1, [TMP1, #offsetof(zend_jit_op_array_trace_extension, offset)] - | add TMP1, IP, TMP1 - | ldr TMP1, [TMP1] - | br TMP1 + | ldr REG0, EX->func + | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] + | add REG0, IP, REG0 + | ldr REG0, [REG0] + | br REG0 } } else if (GCC_GLOBAL_REGS) { | add sp, sp, SPAD // stack alignment @@ -2170,23 +2184,23 @@ static int zend_jit_trace_return(dasm_State **Dst, bool original_handler) | JMP_IP TMP1 } else { | brk #0 // TODO: test - | ldr TMP1, EX->func - | ldr TMP1, [TMP1, #offsetof(zend_op_array, reserved[zend_func_info_rid])] - | ldr TMP1, [TMP1, #offsetof(zend_jit_op_array_trace_extension, offset)] - | add TMP1, IP, TMP1 - | ldr TMP1, [TMP1] - | br TMP1 + | ldr REG0, EX->func + | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] + | add REG0, IP, REG0 + | ldr REG0, [REG0] + | br REG0 } } else { if (original_handler) { | brk #0 // TODO: test | mov FCARG1x, FP - | ldr TMP1, EX->func - | ldr TMP1, [TMP1, #offsetof(zend_op_array, reserved[zend_func_info_rid])] - | ldr TMP1, [TMP1, #offsetof(zend_jit_op_array_trace_extension, offset)] - | add TMP1, IP, TMP1 - | ldr TMP1, [TMP1] - | blr TMP1 + | ldr REG0, EX->func + | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] + | add REG0, IP, REG0 + | ldr REG0, [REG0] + | blr REG0 } | ldp FP, RX, T2 // retore FP and IP | ldr LR, T4 // retore LR @@ -2268,7 +2282,7 @@ static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_thr if (!GCC_GLOBAL_REGS) { | mov FCARG1x, FP } - | EXT_CALL handler, TMP1 + | EXT_CALL handler, REG0 if (may_throw) { zend_jit_check_exception(Dst); } @@ -2308,7 +2322,7 @@ static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) const void *handler = opline->handler; | ADD_HYBRID_SPAD - | EXT_JMP handler, TMP1 + | EXT_JMP handler, REG0 } else { const void *handler = zend_get_opcode_handler_func(opline); @@ -2325,7 +2339,7 @@ static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) | ldr LR, T4 // retore LR | add sp, sp, NR_SPAD // stack alignment } - | EXT_JMP handler, TMP1 + | EXT_JMP handler, REG0 } zend_jit_reset_last_valid_opline(); return 1; @@ -2455,7 +2469,7 @@ static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr sr static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags, const zend_op *opline) { - zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_X0, 0); // COPY_GPR0, use X0 here + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); | brk #0 // TODO @@ -2483,7 +2497,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op } if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { | brk #0 // TODO: test - | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_TMP1, ZREG_TMP2, ZREG_TMP3, ZREG_TMP4, ZREG_FPTMP1 + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, MAY_BE_LONG)) { return 0; @@ -2509,7 +2523,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { | brk #0 // TODO: test - | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_TMP1, ZREG_TMP2, ZREG_TMP3, ZREG_TMP4, ZREG_FPTMP1 + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } |.cold_code |1: @@ -2519,7 +2533,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op } else { if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { - | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_TMP1, ZREG_TMP2, ZREG_TMP3, ZREG_TMP4, ZREG_FPTMP1 + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } } if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { @@ -2569,28 +2583,24 @@ static int zend_jit_math_long_long(dasm_State **Dst, { bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); zend_reg result_reg; - - // x86 defines a 'tmp_reg' to handle integer overflow case. - // In AArch64, we directly use our reserved TMP1. - // zend_reg tmp_reg = ZREG_X0; + zend_reg tmp_reg = ZREG_REG0; if (Z_MODE(res_addr) == IS_REG && (res_info & MAY_BE_LONG)) { if (may_overflow && (res_info & MAY_BE_GUARD) && JIT_G(current_frame) && zend_jit_opline_uses_reg(opline, Z_REG(res_addr))) { - result_reg = ZREG_TMP3; // to store the result temporarily. Use TMP3 + result_reg = ZREG_REG0; } else { result_reg = Z_REG(res_addr); } } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { - | brk #0 // TODO: test result_reg = Z_REG(op1_addr); - } else if (Z_REG(res_addr) != ZREG_X0) { - result_reg = ZREG_TMP3; // Use TMP3 + } else if (Z_REG(res_addr) != ZREG_REG0) { + result_reg = ZREG_REG0; } else { - | brk #0 // TODO: test /* ASSIGN_DIM_OP */ result_reg = ZREG_FCARG1x; + tmp_reg = ZREG_FCARG1x; } if (opcode == ZEND_MUL && @@ -2670,6 +2680,9 @@ static int zend_jit_math_long_long(dasm_State **Dst, } if (may_overflow && (!(res_info & MAY_BE_GUARD) || (res_info & MAY_BE_ANY) == MAY_BE_DOUBLE)) { + zend_reg tmp_reg1 = ZREG_FPR0; + zend_reg tmp_reg2 = ZREG_FPR1; + if (res_info & MAY_BE_LONG) { |.cold_code |1: @@ -2682,7 +2695,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, if (Z_MODE(res_addr) == IS_REG) { | brk #0 // TODO: test } else { - | SET_ZVAL_LVAL res_addr, 0x43e0000000000000, TMP1, TMP2 + | SET_ZVAL_LVAL res_addr, 0x43e0000000000000, REG0, TMP1 } break; } else if (opcode == ZEND_SUB) { @@ -2691,10 +2704,10 @@ static int zend_jit_math_long_long(dasm_State **Dst, } } - | DOUBLE_GET_ZVAL_LVAL ZREG_FPTMP1, op1_addr, ZREG_TMP1, ZREG_TMP2 - | DOUBLE_GET_ZVAL_LVAL ZREG_FPTMP2, op2_addr, ZREG_TMP1, ZREG_TMP2 - | DOUBLE_MATH_REG opcode, ZREG_FPTMP1, ZREG_FPTMP1, ZREG_FPTMP2 - | SET_ZVAL_DVAL res_addr, ZREG_FPTMP1, ZREG_TMP1 + | DOUBLE_GET_ZVAL_LVAL tmp_reg1, op1_addr, tmp_reg, ZREG_TMP1 + | DOUBLE_GET_ZVAL_LVAL tmp_reg2, op2_addr, tmp_reg, ZREG_TMP1 + | DOUBLE_MATH_REG opcode, tmp_reg1, tmp_reg1, tmp_reg2 + | SET_ZVAL_DVAL res_addr, tmp_reg1, ZREG_TMP1 } while (0); if (Z_MODE(res_addr) == IS_MEM_ZVAL @@ -2719,7 +2732,7 @@ static int zend_jit_math_long_double(dasm_State **Dst, uint32_t res_use_info) { zend_reg result_reg = - (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_V0; + (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_FPR0; zend_reg tmp_reg; | brk #0 // TODO @@ -2934,18 +2947,18 @@ static int zend_jit_math_helper(dasm_State **Dst, | brk #0 // TODO: test } | LOAD_ZVAL_ADDR CARG3, op2_addr - | SET_EX_OPLINE opline, TMP1 + | SET_EX_OPLINE opline, REG0 if (opcode == ZEND_ADD) { - | EXT_CALL add_function, TMP1 + | EXT_CALL add_function, REG0 } else if (opcode == ZEND_SUB) { | brk #0 // TODO: test - | EXT_CALL sub_function, TMP1 + | EXT_CALL sub_function, REG0 } else if (opcode == ZEND_MUL) { | brk #0 // TODO: test - | EXT_CALL mul_function, TMP1 + | EXT_CALL mul_function, REG0 } else if (opcode == ZEND_DIV) { | brk #0 // TODO: test - | EXT_CALL div_function, TMP1 + | EXT_CALL div_function, REG0 } else { ZEND_UNREACHABLE(); } @@ -3094,8 +3107,8 @@ static int zend_jit_simple_assign(dasm_State **Dst, { zend_reg tmp_reg; - if (Z_MODE(var_addr) == IS_REG || Z_REG(var_addr) != ZREG_X0) { - tmp_reg = ZREG_TMP1; // TODO: same issue with zend_jit_math_long_long + if (Z_MODE(var_addr) == IS_REG || Z_REG(var_addr) != ZREG_REG0) { + tmp_reg = ZREG_REG0; } else { /* ASSIGN_DIM */ tmp_reg = ZREG_FCARG1x; @@ -3105,7 +3118,7 @@ static int zend_jit_simple_assign(dasm_State **Dst, zval *zv = Z_ZV(val_addr); if (!res_addr) { - | ZVAL_COPY_CONST var_addr, var_info, var_def_info, zv, tmp_reg, ZREG_TMP2, ZREG_FPTMP1 + | ZVAL_COPY_CONST var_addr, var_info, var_def_info, zv, tmp_reg, ZREG_TMP1, ZREG_FPR0 } else { | brk #0 // TODO } @@ -3327,11 +3340,11 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { | brk #0 // TODO } else { - | GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP2 + | GET_ZVAL_LVAL ZREG_REG0, op1_addr, TMP1 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { | brk #0 // TODO } else { - | LONG_CMP ZREG_TMP1, op2_addr, TMP2, TMP3 + | LONG_CMP ZREG_REG0, op2_addr, TMP1, TMP2 } } @@ -3430,7 +3443,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { - zend_reg tmp_reg = ZREG_V0; + zend_reg tmp_reg = ZREG_FPR0; | brk #0 // TODO @@ -3439,7 +3452,7 @@ static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zen static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { - zend_reg tmp_reg = ZREG_V0; + zend_reg tmp_reg = ZREG_FPR0; | brk #0 // TODO @@ -3663,11 +3676,8 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con { uint32_t used_stack; - // TMP1 -> zend_function - // FCARG1x -> used_stack - // It's safe to use FCARG1x directly as x86 does only for the case where 'func' is NULL. - // FCARG1x would be further passed to external helper functions, zend_jit_int_extend_stack_helper - // and zend_jit_extend_stack_helper, if needed. + // REG0 -> zend_function + // FCARG1 -> used_stack if (func) { used_stack = zend_vm_calc_used_stack(opline->extended_value, func); @@ -3677,51 +3687,51 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con | // if (EXPECTED(ZEND_USER_CODE(func->type))) { if (!is_closure) { | LOAD_32BIT_VAL FCARG1w, used_stack - | // Check whether TMP1 is an internal function. - | ldrb TMP2w, [TMP1, #offsetof(zend_function, type)] - | tst TMP2w, #1 + | // Check whether REG0 is an internal function. + | ldrb TMP1w, [REG0, #offsetof(zend_function, type)] + | tst TMP1w, #1 | bne >1 } else { | brk #0 // TODO: test | LOAD_32BIT_VAL FCARG1w, used_stack } | // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval); - | LOAD_32BIT_VAL TMP2w, opline->extended_value + | LOAD_32BIT_VAL REG2w, opline->extended_value if (!is_closure) { - | ldr TMP3w, [TMP1, #offsetof(zend_function, op_array.num_args)] - | cmp TMP2w, TMP3w - | csel TMP2w, TMP2w, TMP3w, le - | ldr TMP3w, [TMP1, #offsetof(zend_function, op_array.last_var)] - | sub TMP2w, TMP2w, TMP3w - | ldr TMP3w, [TMP1, #offsetof(zend_function, op_array.T)] - | sub TMP2w, TMP2w, TMP3w + | ldr TMP1w, [REG0, #offsetof(zend_function, op_array.num_args)] + | cmp REG2w, TMP1w + | csel REG2w, REG2w, TMP1w, le + | ldr TMP1w, [REG0, #offsetof(zend_function, op_array.last_var)] + | sub REG2w, REG2w, TMP1w + | ldr TMP1w, [REG0, #offsetof(zend_function, op_array.T)] + | sub REG2w, REG2w, TMP1w } else { | brk #0 // TODO } - | lsl TMP2w, TMP2w, #5 - | sxtw TMP2, TMP2w - | sub FCARG1x, FCARG1x, TMP2 + | lsl REG2w, REG2w, #5 + | sxtw REG2, REG2w + | sub FCARG1x, FCARG1x, REG2 |1: } zend_jit_start_reuse_ip(); | // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { - | MEM_LOAD_ZTS ldr, RX, executor_globals, vm_stack_top, TMP2 + | MEM_LOAD_ZTS ldr, RX, executor_globals, vm_stack_top, TMP1 if (stack_check) { | // Check Stack Overflow - | MEM_LOAD_ZTS ldr, TMP2, executor_globals, vm_stack_end, TMP3 - | sub TMP2, TMP2, RX + | MEM_LOAD_ZTS ldr, REG2, executor_globals, vm_stack_end, TMP1 + | sub REG2, REG2, RX if (func) { || if (used_stack <= MAX_IMM12) { - | cmp TMP2, #used_stack + | cmp REG2, #used_stack || } else { - | LOAD_32BIT_VAL TMP3, used_stack - | cmp TMP2, TMP3 + | LOAD_32BIT_VAL TMP1, used_stack + | cmp REG2, TMP1 || } } else { - | cmp TMP2, FCARG1x + | cmp REG2, FCARG1x } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { @@ -3736,7 +3746,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con |.cold_code |1: | brk #0 // TODO: test. Cold. - | EXT_JMP exit_addr, TMP3 + | EXT_JMP exit_addr, TMP1 |.code } else { | blt >1 @@ -3763,24 +3773,24 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con if (func) { || if (used_stack <= MAX_IMM12) { - | MEM_LOAD_OP_STORE_ZTS add, ldr, str, #used_stack, executor_globals, vm_stack_top, TMP2, TMP3 + | MEM_LOAD_OP_STORE_ZTS add, ldr, str, #used_stack, executor_globals, vm_stack_top, REG2, TMP1 || } else { - | LOAD_32BIT_VAL TMP4, used_stack - | MEM_LOAD_OP_STORE_ZTS add, ldr, str, TMP4, executor_globals, vm_stack_top, TMP2, TMP3 + | LOAD_32BIT_VAL TMP1, used_stack + | MEM_LOAD_OP_STORE_ZTS add, ldr, str, TMP1, executor_globals, vm_stack_top, REG2, TMP2 || } } else { - | MEM_LOAD_OP_STORE_ZTS add, ldr, str, FCARG1x, executor_globals, vm_stack_top, TMP2, TMP3 + | MEM_LOAD_OP_STORE_ZTS add, ldr, str, FCARG1x, executor_globals, vm_stack_top, REG2, TMP1 } | // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) { | // ZEND_SET_CALL_INFO(call, 0, call_info); - | LOAD_32BIT_VAL TMP2w, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION) - | str TMP2w, EX:RX->This.u1.type_info + | LOAD_32BIT_VAL TMP1w, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION) + | str TMP1w, EX:RX->This.u1.type_info } if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { | // call->func = func; |1: - | ADDR_STORE EX:RX->func, func, TMP2 + | ADDR_STORE EX:RX->func, func, REG1 } else { if (!is_closure) { | // call->func = func; @@ -3790,7 +3800,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con && (sizeof(void*) != 8 || IS_SIGNED_32BIT(func))) { | brk #0 // TODO } else { - | str TMP1, EX:RX->func + | str REG0, EX:RX->func } } else { | // call->func = &closure->func; @@ -3808,8 +3818,8 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con | brk #0 // TODO } | // ZEND_CALL_NUM_ARGS(call) = num_args; - | LOAD_32BIT_VAL TMP2w, opline->extended_value - | str TMP2w, EX:RX->This.u2.num_args + | LOAD_32BIT_VAL TMP1w, opline->extended_value + | str TMP1w, EX:RX->This.u2.num_args return 1; } @@ -4039,9 +4049,9 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t | brk #0 // TODO } else { | // if (CACHED_PTR(opline->result.num)) - | ldr TMP1, EX->run_time_cache - | ldr TMP1, [TMP1, #opline->result.num] - | cbz TMP1, >1 + | ldr REG0, EX->run_time_cache + | ldr REG0, [REG0, #opline->result.num] + | cbz REG0, >1 |.cold_code |1: if (opline->opcode == ZEND_INIT_FCALL @@ -4054,7 +4064,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t if (opline->opcode == ZEND_INIT_FCALL) { | LOAD_ADDR FCARG1x, Z_STR_P(zv); - | EXT_CALL zend_jit_find_func_helper, TMP1 + | EXT_CALL zend_jit_find_func_helper, REG0 } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) { | brk #0 // TODO } else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) { @@ -4063,13 +4073,14 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t ZEND_UNREACHABLE(); } | // CACHE_PTR(opline->result.num, fbc); - | ldr TMP2, EX->run_time_cache - | mov TMP1, RETVALx - | str TMP1, [TMP2, #opline->result.num] + | ldr REG1, EX->run_time_cache + | // Get the return value of function zend_jit_find_func_helper + | mov REG0, RETVALx + | str REG0, [REG1, #opline->result.num] if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { | brk #0 // TODO. tracing mode. } else { - | cbnz TMP1, >3 + | cbnz REG0, >3 | // SAVE_OPLINE(); | brk #0 // TODO: invalid func address. } @@ -4237,7 +4248,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | // fbc = call->func; | // mov r2, EX:RX->func ??? | // SAVE_OPLINE(); - | SET_EX_OPLINE opline, TMP1 + | SET_EX_OPLINE opline, REG0 if (opline->opcode == ZEND_DO_FCALL) { | brk #0 // TODO @@ -4257,7 +4268,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | str EX, EX:RX->prev_execute_data if (!func) { - | ldr TMP1, EX:RX->func + | ldr REG0, EX:RX->func } if (opline->opcode == ZEND_DO_FCALL) { @@ -4277,8 +4288,8 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend if (RETURN_VALUE_USED(opline)) { | // EX(return_value) = EX_VAR(opline->result.var); - | LOAD_ZVAL_ADDR TMP3, res_addr - | str TMP3, EX:RX->return_value + | LOAD_ZVAL_ADDR REG2, res_addr + | str REG2, EX:RX->return_value } else { | // EX(return_value) = 0; | str xzr, EX:RX->return_value @@ -4295,29 +4306,29 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend if (func) { | brk #0 // TODO } - | ldr TMP2, [TMP1, #offsetof(zend_op_array, run_time_cache__ptr)] + | ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)] // Always defined as ZEND_MAP_PTR_KIND_PTR_OR_OFFSET. See Zend/zend_map_ptr.h. #if ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR - | ldr TMP2, [TMP2] + | ldr REG2, [REG2] #elif ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR_OR_OFFSET if (func && !(func->op_array.fn_flags & ZEND_ACC_CLOSURE)) { | brk #0 // TODO } else { - | tst TMP2, #1 + | tst REG2, #1 | beq >1 - | MEM_LOAD_OP_ZTS add, ldr, TMP2, compiler_globals, map_ptr_base, TMP3, TMP4 + | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 |1: - | ldr TMP2, [TMP2] + | ldr REG2, [REG2] } #else # error "Unknown ZEND_MAP_PTR_KIND" #endif - | str TMP2, EX:RX->run_time_cache + | str REG2, EX:RX->run_time_cache } } | // EG(current_execute_data) = execute_data; - | MEM_STORE_ZTS str, RX, executor_globals, current_execute_data, TMP2 + | MEM_STORE_ZTS str, RX, executor_globals, current_execute_data, REG1 | mov FP, RX | // opline = op_array->opcodes; @@ -4328,20 +4339,20 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend if (func && zend_accel_in_shm(func->op_array.opcodes)) { | brk #0 // TODO } else if (GCC_GLOBAL_REGS) { - | ldr IP, [TMP1, #offsetof(zend_op_array, opcodes)] + | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] } else { - | ldr FCARG1x, [TMP1, #offsetof(zend_op_array, opcodes)] + | ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)] | str FCARG1x, EX->opline } if (func) { | brk #0 // TODO } else { | // first_extra_arg = op_array->num_args; - | ldr TMP3w, [TMP1, #offsetof(zend_op_array, num_args)] + | ldr REG2w, [REG0, #offsetof(zend_op_array, num_args)] | // num_args = EX_NUM_ARGS(); - | ldr TMP2w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] + | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] | // if (UNEXPECTED(num_args > first_extra_arg)) - | cmp TMP2w, TMP3w + | cmp REG1w, REG2w } | bgt >1 |.cold_code @@ -4351,24 +4362,24 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) { if (!func) { | // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) - | ldr TMP4w, [TMP1, #offsetof(zend_op_array, fn_flags)] - | tst TMP4w, #ZEND_ACC_HAS_TYPE_HINTS + | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] + | tst TMP1w, #ZEND_ACC_HAS_TYPE_HINTS | bne >1 } | // opline += num_args; || ZEND_ASSERT(sizeof(zend_op) == 32); - | mov TMP3w, TMP2w - | lsl TMP3, TMP3, #5 - | ADD_IP TMP3, TMP4 + | mov REG2w, REG1w + | lsl REG2, REG2, #5 + | ADD_IP REG2, TMP1 } |1: | // if (EXPECTED((int)num_args < op_array->last_var)) { if (func) { | brk #0 // TODO } else { - | ldr TMP3w, [TMP1, #offsetof(zend_op_array, last_var)] + | ldr REG2w, [REG0, #offsetof(zend_op_array, last_var)] } - | sub TMP3w, TMP3w, TMP2w + | sub REG2w, REG2w, REG1w | ble >3 | brk #0 // TODO: test |3: @@ -4378,7 +4389,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | brk #0 // TODO: test | SAVE_IP | mov FCARG1x, FP - | EXT_CALL zend_observer_fcall_begin, TMP1 + | EXT_CALL zend_observer_fcall_begin, REG0 } if (trace) { @@ -4417,24 +4428,24 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | // ZVAL_NULL(EX_VAR(opline->result.var)); | LOAD_ZVAL_ADDR FCARG2x, res_addr - | SET_Z_TYPE_INFO FCARG2x, IS_NULL, TMP2w + | SET_Z_TYPE_INFO FCARG2x, IS_NULL, TMP1w | // EG(current_execute_data) = execute_data; - | MEM_STORE_ZTS str, RX, executor_globals, current_execute_data, TMP2 + | MEM_STORE_ZTS str, RX, executor_globals, current_execute_data, REG1 zend_jit_reset_last_valid_opline(); | // fbc->internal_function.handler(call, ret); | mov FCARG1x, RX if (func) { - | EXT_CALL func->internal_function.handler, TMP1 + | EXT_CALL func->internal_function.handler, REG0 } else { - | ldr TMP2, [TMP1, #offsetof(zend_internal_function, handler)] - | blr TMP2 + | ldr TMP1, [REG0, #offsetof(zend_internal_function, handler)] + | blr TMP1 } | // EG(current_execute_data) = execute_data; - | MEM_STORE_ZTS str, FP, executor_globals, current_execute_data, TMP1 + | MEM_STORE_ZTS str, FP, executor_globals, current_execute_data, REG0 | // zend_vm_stack_free_args(call); if (func && !unknown_num_args) { @@ -4445,7 +4456,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } } else { | mov FCARG1x, RX - | EXT_CALL zend_jit_vm_stack_free_args_helper, TMP1 + | EXT_CALL zend_jit_vm_stack_free_args_helper, REG0 } if (may_have_extra_named_params) { | brk #0 // TODO @@ -4473,11 +4484,11 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend |1: | brk #0 // TODO | mov FCARG1x, RX - | EXT_CALL zend_jit_free_call_frame, TMP1 + | EXT_CALL zend_jit_free_call_frame, REG0 | b >1 |.code } - | MEM_STORE_ZTS str, RX, executor_globals, vm_stack_top, TMP1 + | MEM_STORE_ZTS str, RX, executor_globals, vm_stack_top, REG0 |1: if (!RETURN_VALUE_USED(opline)) { @@ -4497,7 +4508,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } | // if (UNEXPECTED(EG(exception) != NULL)) { - | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, TMP1, TMP2 + | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, REG0, TMP1 | bne ->icall_throw_handler // TODO: Can we avoid checking for interrupts after each call ??? @@ -4572,7 +4583,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t o if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); - | ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_TMP1, ZREG_TMP2, ZREG_FPTMP1 + | ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 if (Z_REFCOUNTED_P(zv)) { | brk #0 // TODO: test } @@ -4633,9 +4644,9 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend } | brk #0 // TODO: test - | SET_EX_OPLINE opline, TMP1 + | SET_EX_OPLINE opline, REG0 | LOAD_32BIT_VAL FCARG1w, opline->op1.var - | EXT_CALL zend_jit_undefined_op_helper, TMP1 + | EXT_CALL zend_jit_undefined_op_helper, REG0 | SET_ZVAL_TYPE_INFO arg_addr, IS_NULL, TMP1w, TMP2 | cbz RETVALx, ->exception_handler @@ -4664,22 +4675,22 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend |1: | brk #0 // TODO: test. cold-code. not covered currently |.code - | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_TMP1, ZREG_TMP2, ZREG_TMP3, ZREG_TMP4, ZREG_FPTMP1 + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 |2: } } else { if (op1_addr != op1_def_addr) { | brk #0 // TODO: test } - | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_TMP1, ZREG_TMP2, ZREG_TMP3, ZREG_TMP4, ZREG_FPTMP1 + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 if (opline->op1_type == IS_CV) { | // In x86 implementation, type flags and value pointer would be stored into eax and r2 respectively, | // and then ah (bits 8 to 15) and r2 are used inside TRY_ADDREF. - | // In AArch64, we use TMP1w and TMP2 accordingly. - | // Note that, bits 8 to 15 should be extacted, i.e., (TMP1w >> 8) & 0xff. - | lsr TMP1w, TMP1w, #8 - | and TMP1w, TMP1w, #0xff - | TRY_ADDREF op1_info, TMP1w, TMP2, TMP3 + | // In AArch64, we use REG0w and REG2 accordingly. + | // Note that, bits 8 to 15 should be extacted, i.e., (REG0w >> 8) & 0xff. + | lsr REG0w, REG0w, #8 + | and REG0w, REG0w, #0xff + | TRY_ADDREF op1_info, REG0w, REG2, TMP1 } } } @@ -4777,8 +4788,8 @@ static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, zend_ssa *ssa, u static int zend_jit_leave_frame(dasm_State **Dst) { | // EG(current_execute_data) = EX(prev_execute_data); - | ldr TMP1, EX->prev_execute_data - | MEM_STORE_ZTS str, TMP1, executor_globals, current_execute_data, TMP3 + | ldr REG0, EX->prev_execute_data + | MEM_STORE_ZTS str, REG0, executor_globals, current_execute_data, REG2 return 1; } @@ -4873,14 +4884,14 @@ static int zend_jit_leave_func(dasm_State **Dst, } | // EG(vm_stack_top) = (zval*)execute_data; - | MEM_STORE_ZTS str, FP, executor_globals, vm_stack_top, TMP1 + | MEM_STORE_ZTS str, FP, executor_globals, vm_stack_top, REG0 | // execute_data = EX(prev_execute_data); | ldr FP, EX->prev_execute_data if (!left_frame) { | brk #0 // TODO: teset | // EG(current_execute_data) = execute_data; - | MEM_STORE_ZTS str, FP, executor_globals, current_execute_data, TMP1 + | MEM_STORE_ZTS str, FP, executor_globals, current_execute_data, REG0 } |9: @@ -4913,7 +4924,7 @@ static int zend_jit_leave_func(dasm_State **Dst, return 1; } else { | // if (EG(exception)) - | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, TMP1, TMP2 + | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, REG0, TMP1 | LOAD_IP | bne ->leave_throw_handler | // opline = EX(opline) + 1 @@ -4977,18 +4988,24 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o | brk #0 // TODO: test } - // TMP1 -> ret_addr - // x86 has to select one temp register to store the return address, i.e. 'ret_addr', if the return value would be used. - // In AArch64, we simply use our reserved register, i.e. TMP1. // if (!EX(return_value)) - if (return_value_used != 0) { - | ldr TMP1, EX->return_value - } - if (return_value_used == -1) { - | tst TMP1, TMP1 + if (Z_MODE(op1_addr) == IS_REG && Z_REG(op1_addr) == ZREG_REG1) { + if (return_value_used != 0) { + | ldr REG2, EX->return_value + } + if (return_value_used == -1) { + | tst REG2, REG2 + } + ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, 0); + } else { + if (return_value_used != 0) { + | ldr REG1, EX->return_value + } + if (return_value_used == -1) { + | tst REG1, REG1 + } + ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG1, 0); } - ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_TMP1, 0); - if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | brk #0 // TODO: test @@ -5008,7 +5025,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); - | ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_TMP2, ZREG_TMP3, ZREG_FPTMP1 + | ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 if (Z_REFCOUNTED_P(zv)) { | brk #0 // TODO: test } @@ -5017,11 +5034,9 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o } else if (opline->op1_type == IS_CV) { if (op1_info & MAY_BE_REF) { | brk #0 // TODO - // TMP2 -> op1_addr - op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_TMP2, 0); + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); } - // Note: tmp_reg2 is not used in current case, hence we pass a random one. - | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_TMP2, ZREG_TMP3, ZREG_TMP4, ZREG_TMP4, ZREG_FPTMP1 + | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { | brk #0 // TODO } @@ -5035,7 +5050,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, zend_jit_addr val_addr, zend_reg type_reg) { - ZEND_ASSERT(type_reg == ZREG_X2); + ZEND_ASSERT(type_reg == ZREG_REG2); | brk #0 // TODO return 1; @@ -5155,7 +5170,7 @@ static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zen zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); bool in_cold = 0; uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; - zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1x : ZREG_TMP1; // TODO: use TMP1 + zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1x : ZREG_REG0; | brk #0 // TODO return 1; @@ -5417,14 +5432,14 @@ static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_i if (len > 0) { const char *str = Z_STRVAL_P(zv); - | SET_EX_OPLINE opline, TMP1 + | SET_EX_OPLINE opline, REG0 | LOAD_ADDR CARG1, str || if (len <= MAX_IMM12) { | mov CARG2, #len || } else { | LOAD_64BIT_VAL CARG2, len || } - | EXT_CALL zend_write, TMP1 + | EXT_CALL zend_write, REG0 if (!zend_jit_check_exception(Dst)) { return 0; } @@ -5546,7 +5561,7 @@ static int zend_jit_fetch_constant(dasm_State **Dst, { zval *zv = RT_CONSTANT(opline, opline->op2) + 1; zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); - zend_jit_addr const_addr = ZEND_ADDR_MEM_ZVAL(ZREG_X0, 0); // COPY_GPR0, use X0 here + zend_jit_addr const_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); uint32_t res_info = RES_INFO(); | brk #0 // TODO @@ -5601,7 +5616,7 @@ static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, ui if (Z_REG(var_addr) != ZREG_FCARG1x || Z_OFFSET(var_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, var_addr } - | EXT_CALL zend_jit_unref_helper, TMP1 + | EXT_CALL zend_jit_unref_helper, REG0 } else { | brk #0 // GET_ZVAL_PTR FCARG1x, var_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, offsetof(zend_reference, val)); @@ -5861,6 +5876,19 @@ static bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, return 1; } +static bool zend_needs_extra_reg_for_const(const zend_op *opline, zend_uchar op_type, znode_op op) +{ +|| if (op_type == IS_CONST) { +|| zval *zv = RT_CONSTANT(opline, op); +|| if (Z_TYPE_P(zv) == IS_DOUBLE && Z_DVAL_P(zv) != 0) { +|| return 1; +|| } else if (Z_TYPE_P(zv) == IS_LONG) { +|| return 1; +|| } +|| } + return 0; +} + static zend_regset zend_jit_get_def_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use) { uint32_t op1_info, op2_info; @@ -5905,12 +5933,18 @@ static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend case ZEND_SEND_VAL: case ZEND_SEND_VAL_EX: if (ssa_op->op1_use == current_var) { - regset = ZEND_REGSET_EMPTY; + regset = ZEND_REGSET(ZREG_REG0); break; } op1_info = OP1_INFO(); if (!(op1_info & MAY_BE_UNDEF)) { - regset = ZEND_REGSET_EMPTY; + if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { + regset = ZEND_REGSET(ZREG_FPR0); + } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { + regset = ZEND_REGSET(ZREG_REG0); + } else { + regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2)); + } } break; case ZEND_SEND_VAR: @@ -5921,7 +5955,15 @@ static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend } op1_info = OP1_INFO(); if (!(op1_info & MAY_BE_UNDEF)) { - regset = ZEND_REGSET_EMPTY; + if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { + regset = ZEND_REGSET(ZREG_FPR0); + } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { + } else { + regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2)); + if (op1_info & MAY_BE_REF) { + ZEND_REGSET_INCL(regset, ZREG_REG1); + } + } } break; case ZEND_ASSIGN: @@ -5937,7 +5979,13 @@ static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend if (opline->op1_type == IS_CV && !(op2_info & MAY_BE_UNDEF) && !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { - regset = ZEND_REGSET_EMPTY; + if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) { + regset = ZEND_REGSET(ZREG_FPR0); + } else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) { + regset = ZEND_REGSET(ZREG_REG0); + } else { + regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2)); + } } break; case ZEND_PRE_INC: @@ -5955,6 +6003,9 @@ static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend && (op1_info & MAY_BE_LONG) && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { regset = ZEND_REGSET_EMPTY; + if (op1_info & MAY_BE_DOUBLE) { + regset = ZEND_REGSET(ZREG_FPR0); + } } break; case ZEND_ADD: @@ -5966,6 +6017,50 @@ static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { regset = ZEND_REGSET_EMPTY; + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { + if (ssa_op->result_def != current_var && + (ssa_op->op1_use != current_var || !last_use)) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } + res_info = OP1_INFO(); + if (res_info & MAY_BE_DOUBLE) { + ZEND_REGSET_INCL(regset, ZREG_FPR0); + ZEND_REGSET_INCL(regset, ZREG_FPR1); + } + } + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { + if (ssa_op->result_def != current_var) { + ZEND_REGSET_INCL(regset, ZREG_FPR0); + } + } + if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { + if (zend_is_commutative(opline->opcode)) { + if (ssa_op->result_def != current_var) { + ZEND_REGSET_INCL(regset, ZREG_FPR0); + } + } else { + ZEND_REGSET_INCL(regset, ZREG_FPR0); + if (ssa_op->result_def != current_var && + (ssa_op->op1_use != current_var || !last_use)) { + ZEND_REGSET_INCL(regset, ZREG_FPR1); + } + } + } + if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { + if (ssa_op->result_def != current_var && + (ssa_op->op1_use != current_var || !last_use) && + (!zend_is_commutative(opline->opcode) || ssa_op->op2_use != current_var || !last_use)) { + ZEND_REGSET_INCL(regset, ZREG_FPR0); + } + } + if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || + zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { + if (!ZEND_REGSET_IN(regset, ZREG_REG0)) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } else { + ZEND_REGSET_INCL(regset, ZREG_REG1); + } + } } break; case ZEND_BW_OR: @@ -5976,6 +6071,18 @@ static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { regset = ZEND_REGSET_EMPTY; + if (ssa_op->result_def != current_var && + (ssa_op->op1_use != current_var || !last_use)) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } + if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || + zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { + if (!ZEND_REGSET_IN(regset, ZREG_REG0)) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } else { + ZEND_REGSET_INCL(regset, ZREG_REG1); + } + } } break; case ZEND_SL: @@ -5985,6 +6092,13 @@ static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { regset = ZEND_REGSET_EMPTY; + if (ssa_op->result_def != current_var && + (ssa_op->op1_use != current_var || !last_use)) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } + if (opline->op2_type != IS_CONST && ssa_op->op2_use != current_var) { + ZEND_REGSET_INCL(regset, ZREG_REG1); + } } break; case ZEND_MOD: @@ -5993,6 +6107,30 @@ static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) { regset = ZEND_REGSET_EMPTY; + if (opline->op2_type == IS_CONST && + Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG && + zend_long_is_power_of_two(Z_LVAL_P(RT_CONSTANT(opline, opline->op2))) && + OP1_HAS_RANGE() && + OP1_MIN_RANGE() >= 0) { + if (ssa_op->result_def != current_var && + (ssa_op->op1_use != current_var || !last_use)) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } + if (sizeof(void*) == 8 + && !IS_SIGNED_32BIT(Z_LVAL_P(RT_CONSTANT(opline, opline->op2)) - 1)) { + if (!ZEND_REGSET_IN(regset, ZREG_REG0)) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } else { + ZEND_REGSET_INCL(regset, ZREG_REG1); + } + } + } else { + ZEND_REGSET_INCL(regset, ZREG_REG0); + ZEND_REGSET_INCL(regset, ZREG_REG2); + if (opline->op2_type == IS_CONST) { + ZEND_REGSET_INCL(regset, ZREG_REG1); + } + } } break; case ZEND_IS_SMALLER: @@ -6007,6 +6145,32 @@ static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) && !(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { regset = ZEND_REGSET_EMPTY; + if (!(opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ))) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && + opline->op1_type != IS_CONST && opline->op2_type != IS_CONST) { + if (ssa_op->op1_use != current_var && + ssa_op->op2_use != current_var) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } + } + if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) { + ZEND_REGSET_INCL(regset, ZREG_FPR0); + } + if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) { + ZEND_REGSET_INCL(regset, ZREG_FPR0); + } + if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) { + if (ssa_op->op1_use != current_var && + ssa_op->op2_use != current_var) { + ZEND_REGSET_INCL(regset, ZREG_FPR0); + } + } + if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) || + zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } } break; case ZEND_BOOL: @@ -6019,6 +6183,15 @@ static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend op1_info = OP1_INFO(); if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) { regset = ZEND_REGSET_EMPTY; + if (op1_info & MAY_BE_DOUBLE) { + ZEND_REGSET_INCL(regset, ZREG_FPR0); + } + if (opline->opcode == ZEND_BOOL || + opline->opcode == ZEND_BOOL_NOT || + opline->opcode == ZEND_JMPZ_EX || + opline->opcode == ZEND_JMPNZ_EX) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } } break; case ZEND_DO_UCALL: @@ -6034,6 +6207,43 @@ static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend break; } + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + if (ssa_op == ssa->ops + && JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].op == ZEND_JIT_TRACE_INIT_CALL + && (JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + ZEND_REGSET_INCL(regset, ZREG_REG1); + } + } + + /* %r0 is used to check EG(vm_interrupt) */ + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + if (ssa_op == ssa->ops + && (JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_LOOP || + JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL)) { +#if ZTS + ZEND_REGSET_INCL(regset, ZREG_REG0); +#else + if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(vm_interrupt)))) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } +#endif + } + } else { + uint32_t b = ssa->cfg.map[ssa_op - ssa->ops]; + + if ((ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) != 0 + && ssa->cfg.blocks[b].start == ssa_op - ssa->ops) { +#if ZTS + ZEND_REGSET_INCL(regset, ZREG_REG0); +#else + if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(vm_interrupt)))) { + ZEND_REGSET_INCL(regset, ZREG_REG0); + } +#endif + } + } + return regset; } diff --git a/ext/opcache/jit/zend_jit_arm64.h b/ext/opcache/jit/zend_jit_arm64.h index b6e063de2bd81..40cb4ba5a8d97 100644 --- a/ext/opcache/jit/zend_jit_arm64.h +++ b/ext/opcache/jit/zend_jit_arm64.h @@ -105,28 +105,40 @@ typedef enum _zend_reg { } zend_reg; typedef struct _zend_jit_registers_buf { - uint64_t gpr[32]; /* general purpose integer register */ - double fpr[32]; /* floating point registers */ + uint64_t gpr[32]; /* general purpose integer register */ + double fpr[32]; /* floating point registers */ } zend_jit_registers_buf; -#define ZREG_FIRST_FPR ZREG_V0 +#define ZREG_RSP ZREG_X31 +#define ZREG_RLR ZREG_X30 +#define ZREG_RFP ZREG_X29 +#define ZREG_RPR ZREG_X18 + +#define ZREG_FP ZREG_X27 +#define ZREG_IP ZREG_X28 +#define ZREG_RX ZREG_IP + +#define ZREG_REG0 ZREG_X8 +#define ZREG_REG1 ZREG_X9 +#define ZREG_REG2 ZREG_X10 -#define ZREG_RSP ZREG_X31 -#define ZREG_RLR ZREG_X30 -#define ZREG_RFP ZREG_X29 -#define ZREG_RPR ZREG_X18 +#define ZREG_FPR0 ZREG_V0 +#define ZREG_FPR1 ZREG_V1 -# define ZREG_FP ZREG_X27 -# define ZREG_IP ZREG_X28 -# define ZREG_RX ZREG_IP +#define ZREG_TMP1 ZREG_X11 +#define ZREG_TMP2 ZREG_X12 +#define ZREG_TMP3 ZREG_X13 +#define ZREG_TMP4 ZREG_X14 + +#define ZREG_COPY ZREG_REG0 +#define ZREG_FIRST_FPR ZREG_V0 typedef uint64_t zend_regset; # define ZEND_REGSET_FIXED \ (ZEND_REGSET(ZREG_RSP) | ZEND_REGSET(ZREG_RLR) | ZEND_REGSET(ZREG_RFP) | \ ZEND_REGSET(ZREG_RPR) | ZEND_REGSET(ZREG_FP) | ZEND_REGSET(ZREG_IP) | \ - ZEND_REGSET_INTERVAL(ZREG_X8, ZREG_X11) | \ - ZEND_REGSET_INTERVAL(ZREG_V16, ZREG_V17)) + ZEND_REGSET_INTERVAL(ZREG_TMP1, ZREG_TMP4)) # define ZEND_REGSET_GP \ ZEND_REGSET_DIFFERENCE(ZEND_REGSET_INTERVAL(ZREG_X0, ZREG_X30), ZEND_REGSET_FIXED) # define ZEND_REGSET_FP \ @@ -137,6 +149,7 @@ typedef uint64_t zend_regset; # define ZEND_REGSET_PRESERVED \ (ZEND_REGSET_INTERVAL(ZREG_X19, ZREG_X28) | ZEND_REGSET_INTERVAL(ZREG_V8, ZREG_V15)) -#define ZEND_REGSET_LOW_PRIORITY ZEND_REGSET_EMPTY +#define ZEND_REGSET_LOW_PRIORITY \ + (ZEND_REGSET(ZREG_REG0) | ZEND_REGSET(ZREG_REG1) | ZEND_REGSET(ZREG_FPR0) | ZEND_REGSET(ZREG_FPR1)) #endif /* ZEND_JIT_ARM64_H */ diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index d16dd39591973..638ed243c4573 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -64,11 +64,7 @@ #define ZEND_REGSET_DIFFERENCE(set1, set2) \ ((set1) & ~(set2)) -#if defined (__aarch64__) -# define ZEND_REGSET_FIRST(set) ((zend_reg)__builtin_ctzll(set)) -# define ZEND_REGSET_SECOND(set) ((zend_reg)__builtin_ctzll(set ^ (1ull << ZEND_REGSET_FIRST(set)))) -# define ZEND_REGSET_LAST(set) ((zend_reg)(__builtin_clzll(set)^63)) -#elif !defined(_WIN32) +#if !defined(_WIN32) # if (ZREG_NUM <= 32) # define ZEND_REGSET_FIRST(set) ((zend_reg)__builtin_ctz(set)) # define ZEND_REGSET_LAST(set) ((zend_reg)(__builtin_clz(set)^31)) @@ -76,7 +72,7 @@ # define ZEND_REGSET_FIRST(set) ((zend_reg)__builtin_ctzll(set)) # define ZEND_REGSET_LAST(set) ((zend_reg)(__builtin_clzll(set)^63)) # else -# errir "Too many registers" +# error "Too many registers" # endif #else # include @@ -95,7 +91,7 @@ uint32_t __inline __zend_jit_clz(uint32_t value) { return 32; } # define ZEND_REGSET_FIRST(set) ((zend_reg)__zend_jit_ctz(set)) -# define ZEND_REGSET_LAST(set) ((zend_reg)(__zend_jit_clz(set)^31))) +# define ZEND_REGSET_LAST(set) ((zend_reg)(__zend_jit_clz(set)^31)) #endif #define ZEND_REGSET_FOREACH(set, reg) \ diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 0803605769896..d52479056ffc2 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -6947,7 +6947,7 @@ static void zend_jit_dump_exit_info(zend_jit_trace_info *t) } else if (STACK_REG(stack, j) == ZREG_ZVAL_COPY_GPR0) { fprintf(stderr, " "); zend_dump_var(op_array, (j < op_array->last_var) ? IS_CV : 0, j); - fprintf(stderr, ":unknown(zval_copy(%s))", zend_reg_name[0]); + fprintf(stderr, ":unknown(zval_copy(%s))", zend_reg_name[ZREG_COPY]); } } fprintf(stderr, "\n"); @@ -7478,7 +7478,7 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf } else if (STACK_REG(stack, i) == ZREG_ZVAL_TRY_ADDREF) { Z_TRY_ADDREF_P(EX_VAR_NUM(i)); } else if (STACK_REG(stack, i) == ZREG_ZVAL_COPY_GPR0) { - zval *val = (zval*)regs->gpr[0]; + zval *val = (zval*)regs->gpr[ZREG_COPY]; if (UNEXPECTED(Z_TYPE_P(val) == IS_UNDEF)) { /* Undefined array index or property */ diff --git a/ext/opcache/jit/zend_jit_x86.h b/ext/opcache/jit/zend_jit_x86.h index 795569517d2b8..8867dc04672d8 100644 --- a/ext/opcache/jit/zend_jit_x86.h +++ b/ext/opcache/jit/zend_jit_x86.h @@ -88,6 +88,7 @@ typedef struct _zend_jit_registers_buf { } zend_jit_registers_buf; #define ZREG_FIRST_FPR ZREG_XMM0 +#define ZREG_COPY ZREG_R0 #define ZREG_RAX ZREG_R0 #define ZREG_RCX ZREG_R1 From 797612786ddc0262f8d7a2e6682abaec435d0072 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Thu, 8 Apr 2021 03:17:34 +0000 Subject: [PATCH 038/229] Support failed JIT test case: assign_002.phpt Reference is involved in this test case, i.e. "$ref2 = & $ref1;". 1. Fix one bug in zend_do_fcall(). For each stack slot, the type information gets initialized during the call frame allocation phase. Opcode ZEND_ASSIGN_REF is associated to this statement. It's worth noting that PHP JIT doesn't apply to this opcode actually. That means the original handler(i.e. interpreter version) will be invoked at runtime. Note that this mode works for a number of opcodes, not only ZEND_ASSIGN_REF. In the execution of original handler, the runtime type information of $ref2 is accessed and this bug is triggered. 2. Support macros GET_Z_PTR and ZVAL_DEREF. 3. Cover new paths in function zend_jit_simple_assign() and macro ZVAL_COPY_CONST. --- ext/opcache/jit/zend_jit_arm64.dasc | 64 +++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 3054efe7b8389..63570b4523bff 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -461,7 +461,11 @@ static void* dasm_labels[zend_lb_MAX]; |.macro SET_ZVAL_TYPE_INFO, addr, type, tmp_reg1, tmp_reg2 || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -| LOAD_32BIT_VAL tmp_reg1, type +|| if (type <= MAX_IMM12) { +| mov tmp_reg1, #type +|| } else { +| LOAD_32BIT_VAL tmp_reg1, type +|| } | SAFE_MEM_ACC_WITH_UOFFSET str, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg2 |.endmacro @@ -471,7 +475,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro GET_Z_PTR, reg, zv -| mov reg, aword [zv] +| ldr reg, [zv] |.endmacro |.macro SET_Z_PTR, zv, val @@ -737,7 +741,6 @@ static void* dasm_labels[zend_lb_MAX]; || } || if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { || if (dst_def_info == MAY_BE_DOUBLE) { -| brk #0 // TODO: test || if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { | SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2) || } @@ -957,11 +960,10 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro ZVAL_DEREF, reg, info, tmp_reg -| brk #0 // TODO || if (info & MAY_BE_REF) { | IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1, tmp_reg | GET_Z_PTR reg, reg -| add reg, offsetof(zend_reference, val) +| add reg, reg, #offsetof(zend_reference, val) |1: || } |.endmacro @@ -3126,7 +3128,44 @@ static int zend_jit_simple_assign(dasm_State **Dst, | brk #0 // TODO } } else { - | brk #0 // TODO + if (val_info & MAY_BE_UNDEF) { + | brk #0 + } + if (val_info & MAY_BE_REF) { + if (val_type == IS_CV) { + ZEND_ASSERT(Z_REG(var_addr) != ZREG_REG2); + if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_REG2 || Z_OFFSET(val_addr) != 0) { + | LOAD_ZVAL_ADDR REG2, val_addr + } + | ZVAL_DEREF REG2, val_info, TMP1w + val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, 0); + } else { + zend_jit_addr ref_addr; + + | brk #0 + } + } + + if (!res_addr) { + | ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + } else { + | brk #0 // TODO + } + + if (val_type == IS_CV) { + if (!res_addr) { + | lsr REG2w, REG2w, #8 + | and REG2w, REG2w, #0xff + | TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1 + } else { + | brk #0 // TODO + } + } else { + if (res_addr) { + | brk #0 // TODO + } + } + |3: } return 1; } @@ -4379,9 +4418,18 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } else { | ldr REG2w, [REG0, #offsetof(zend_op_array, last_var)] } - | sub REG2w, REG2w, REG1w + | subs REG2w, REG2w, REG1w | ble >3 - | brk #0 // TODO: test + | // zval *var = EX_VAR_NUM(num_args); + | lsl REG1, REG1, #4 + | add REG1, REG1, FP + || ZEND_ASSERT(ZEND_CALL_FRAME_SLOT * sizeof(zval) <= MAX_IMM12); + | add REG1, REG1, #(ZEND_CALL_FRAME_SLOT * sizeof(zval)) + |2: + | SET_Z_TYPE_INFO REG1, IS_UNDEF, TMP1w + | add REG1, REG1, #16 + | subs REG2w, REG2w, #1 + | bne <2 |3: } From 1684d277b5f39cf1ce367e9204d36b22b65abab9 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Thu, 8 Apr 2021 05:46:26 +0000 Subject: [PATCH 039/229] Support failed JIT test case: assign_010.phpt Following the previous patch, we continue to support failed JIT test cases involving reference. In assign_010.phpt, major changes are done to support the assignment "$a = $b" where "$b" is a reference. Honestly speaking, I didn't fully understand the syntax here but rather to translate the x86 implementation into AArch64. Besides, test case assign_011.phpt would pass as well with this patch. --- ext/opcache/jit/zend_jit_arm64.dasc | 116 +++++++++++++++++++++++++++- 1 file changed, 115 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 63570b4523bff..f5ca680c539f3 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -3176,7 +3176,39 @@ static int zend_jit_assign_to_typed_ref(dasm_State **Dst, zend_jit_addr val_addr, bool check_exception) { + | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { + | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] + | cmp TMP1, #0 + | bne >2 + |.cold_code + |2: | brk #0 // TODO + if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2x || Z_OFFSET(val_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG2x, val_addr + } + if (opline) { + | SET_EX_OPLINE opline, REG0 + } + if (val_type == IS_CONST) { + | EXT_CALL zend_jit_assign_const_to_typed_ref, REG0 + } else if (val_type == IS_TMP_VAR) { + | EXT_CALL zend_jit_assign_tmp_to_typed_ref, REG0 + } else if (val_type == IS_VAR) { + | EXT_CALL zend_jit_assign_var_to_typed_ref, REG0 + } else if (val_type == IS_CV) { + | EXT_CALL zend_jit_assign_cv_to_typed_ref, REG0 + } else { + ZEND_UNREACHABLE(); + } + if (check_exception) { + | // if (UNEXPECTED(EG(exception) != NULL)) { + | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, REG0, TMP1 + | beq >8 // END OF zend_jit_assign_to_variable() + | b ->exception_handler + } else { + | b >8 + } + |.code return 1; } @@ -3214,7 +3246,89 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, int done = 0; zend_reg ref_reg, tmp_reg; - | brk #0 // TODO + if (Z_MODE(var_addr) == IS_REG || Z_REG(var_use_addr) != ZREG_REG0) { + ref_reg = ZREG_FCARG1x; + tmp_reg = ZREG_REG0; + } else { + /* ASSIGN_DIM */ + ref_reg = ZREG_REG0; + tmp_reg = ZREG_FCARG1x; + } + + if (var_info & MAY_BE_REF) { + if (Z_MODE(var_use_addr) != IS_MEM_ZVAL || Z_REG(var_use_addr) != ref_reg || Z_OFFSET(var_use_addr) != 0) { + | LOAD_ZVAL_ADDR Rx(ref_reg), var_use_addr + var_addr = var_use_addr = ZEND_ADDR_MEM_ZVAL(ref_reg, 0); + } + | // if (Z_ISREF_P(variable_ptr)) { + | IF_NOT_Z_TYPE Rx(ref_reg), IS_REFERENCE, >1, TMP1w + | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { + | GET_Z_PTR FCARG1x, Rx(ref_reg) + if (!zend_jit_assign_to_typed_ref(Dst, opline, val_type, val_addr, check_exception)) { + return 0; + } + | add Rx(ref_reg), FCARG1x, #offsetof(zend_reference, val) + |1: + } + if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + if (RC_MAY_BE_1(var_info)) { + int in_cold = 0; + + if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | IF_ZVAL_REFCOUNTED var_use_addr, >1, TMP1w, TMP2 + |.cold_code + |1: + | brk #0 // TODO + in_cold = 1; + } + if (Z_REG(var_use_addr) == ZREG_FCARG1x || Z_REG(var_use_addr) == ZREG_REG0) { + bool keep_gc = 0; + + | brk #0 // TODO + } else { + | brk #0 // TODO + | GET_ZVAL_PTR FCARG1x, var_use_addr, TMP1 + if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 1)) { + return 0; + } + } + | GC_DELREF FCARG1x, TMP1w + if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { + | brk #0 // TODO + } else { + | brk #0 // TODO + } + | ZVAL_DTOR_FUNC var_info, opline, TMP1 + if (in_cold || (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0)) { + if (check_exception) { + | brk #0 // TODO + } else { + | brk #0 // TODO + } + } + if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { + |4: + | brk #0 // TODO + if (in_cold) { + | brk #0 // TODO + } + } + if (in_cold) { + |.code + } else { + done = 1; + } + } else /* if (RC_MAY_BE_N(var_info)) */ { + | brk #0 + } + } + + if (!done && !zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0, 0)) { + return 0; + } + + |8: + return 1; } From 7537d21d998c123ede90c1213a739ffccaaae97a Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Thu, 8 Apr 2021 06:33:09 +0000 Subject: [PATCH 040/229] Support failed JIT test case: assign_012.phpt Support the case where arguments might be reference. Besides, another two test cases, assign_019.phpt and assign_032.phpt, would pass as well with this patch. --- ext/opcache/jit/zend_jit_arm64.dasc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index f5ca680c539f3..2a65efed73d73 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -4828,7 +4828,12 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend if (opline->op1_type == IS_CV) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); - | brk #0 // TODO: test + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | ZVAL_DEREF FCARG1x, op1_info, TMP1w + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | lsr REG0w, REG0w, #8 + | and REG0w, REG0w, #0xff + | TRY_ADDREF op1_info, REG0w, REG2, TMP1 } else { zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 8); From 31b0f9f5e03e22c3b70b2d591b43eb804dbf6648 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Fri, 9 Apr 2021 00:28:21 +0000 Subject: [PATCH 041/229] Support failed JIT test case: assign_027.phpt This patch is trivial, supporting the comparion with constant values, i.e. "$i < 2" in this test case. --- ext/opcache/jit/zend_jit_arm64.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 2a65efed73d73..7f20f86cf1d57 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -580,10 +580,10 @@ static void* dasm_labels[zend_lb_MAX]; |.macro LONG_CMP, reg, addr, tmp_reg1, tmp_reg2 || if (Z_MODE(addr) == IS_CONST_ZVAL) { -| brk #0 // TODO: test || if(Z_LVAL_P(Z_ZV(addr)) >= 0 && Z_LVAL_P(Z_ZV(addr)) <= MAX_IMM12) { | cmp Rx(reg), #(Z_LVAL_P(Z_ZV(addr))) || } else { +| brk #0 // TODO | LOAD_64BIT_VAL tmp_reg1, Z_LVAL_P(Z_ZV(addr)) | cmp Rx(reg), tmp_reg1 || } From d7a142786396c4d015401638e953959975fb13e5 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Fri, 9 Apr 2021 00:49:59 +0000 Subject: [PATCH 042/229] Support failed JIT test case: assign_024.phpt Support assginment with undefined variable, and a warning would be emitted. Besides, test case assign_023.phpt would pass as well with this patch. --- ext/opcache/jit/zend_jit_arm64.dasc | 35 ++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 7f20f86cf1d57..87c03c6da3ef6 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -3129,7 +3129,40 @@ static int zend_jit_simple_assign(dasm_State **Dst, } } else { if (val_info & MAY_BE_UNDEF) { - | brk #0 + if (in_cold) { + | brk #0 // TODO + | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2, TMP1w, TMP2 + } else { + | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1, TMP1w, TMP2 + |.cold_code + |1: + } + | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + if (save_r1) { + | brk #0 // TODO + | str FCARG1x, T1 // save + } + | SET_ZVAL_TYPE_INFO var_addr, IS_NULL, TMP1w, TMP2 + if (res_addr) { + | brk #0 // TODO + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 + } + if (opline) { + | SET_EX_OPLINE opline, Rx(tmp_reg) + } + ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP); + | LOAD_32BIT_VAL FCARG1w, Z_OFFSET(val_addr) + | EXT_CALL zend_jit_undefined_op_helper, REG0 + if (save_r1) { + | brk #0 // TODO + | ldr FCARG1x, T1 // restore + } + | b >3 + if (in_cold) { + |2: + } else { + |.code + } } if (val_info & MAY_BE_REF) { if (val_type == IS_CV) { From d7ae4d04ba378575a58e18d731da0ed32c87a3d8 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Fri, 9 Apr 2021 06:13:35 +0000 Subject: [PATCH 043/229] Support failed JIT test case: assign_022.phpt Major changes are made to support statement "$a[0] = $unref", where opcode ASSIGN_DIM is involved. Besides, one bug in macro GC_DELREF is fixed. The reference count would be further checked after decreasing in macro ZVAL_PTR_DTOR, hence, instruction "subs" should be used to set the flags. After fixing this bug, external function zend_jit_array_free() is used as the dtor for the array "$a". --- ext/opcache/jit/zend_jit_arm64.dasc | 342 +++++++++++++++++++++++++++- 1 file changed, 330 insertions(+), 12 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 87c03c6da3ef6..5abbbe6859743 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -197,7 +197,13 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro LOAD_ADDR_ZTS, reg, struct, field -| brk #0 // TODO +| .if ZTS +| brk #0 // TODO +| LOAD_TSRM_CACHE reg +| add reg, reg, #(struct.._offset + offsetof(zend_..struct, field)) +| .else +| LOAD_ADDR reg, &struct.field +| .endif |.endmacro |.macro ADDR_OP1, addr_ins, addr, tmp_reg @@ -621,7 +627,6 @@ static void* dasm_labels[zend_lb_MAX]; |.macro GET_ZVAL_LVAL, reg, addr, tmp_reg || if (Z_MODE(addr) == IS_CONST_ZVAL) { || if (Z_LVAL_P(Z_ZV(addr)) == 0) { -| brk #0 // TODO: test | mov Rx(reg), xzr || } else { | brk #0 // TODO: test @@ -911,13 +916,14 @@ static void* dasm_labels[zend_lb_MAX]; |.macro GC_DELREF, zv, tmp_reg | ldr tmp_reg, [zv] -| sub tmp_reg, tmp_reg, #1 +| subs tmp_reg, tmp_reg, #1 | str tmp_reg, [zv] |.endmacro -|.macro IF_GC_MAY_NOT_LEAK, ptr, label, tmp_reg -| ldrh tmp_reg, [ptr, #4] -| tst tmp_reg, #(GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT)) +|.macro IF_GC_MAY_NOT_LEAK, ptr, label, tmp_reg1, tmp_reg2 +| ldrh tmp_reg1, [ptr, #4] +| LOAD_32BIT_VAL tmp_reg2, (GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT)) +| tst tmp_reg1, tmp_reg2 | bne label |.endmacro @@ -984,7 +990,27 @@ static void* dasm_labels[zend_lb_MAX]; |.macro ZVAL_DTOR_FUNC, var_info, opline, tmp_reg || do { || if (has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_INDIRECT))) { -| brk #0 // TODO: test +|| zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); +|| if (type == IS_STRING && !ZEND_DEBUG) { +| brk #0 // TODO +|| break; +|| } else if (type == IS_ARRAY) { +|| if ((var_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) { +|| if (opline && ((var_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) { +| brk #0 // TODO +|| } +| brk #0 // TODO +|| } else { +| EXT_CALL zend_jit_array_free, tmp_reg +|| } +|| break; +|| } else if (type == IS_OBJECT) { +|| if (opline) { +| brk #0 // TODO +|| } +| brk #0 // TODO +|| break; +|| } || } || if (opline) { | SET_EX_OPLINE opline, tmp_reg @@ -1033,7 +1059,7 @@ static void* dasm_labels[zend_lb_MAX]; | GET_ZVAL_PTR FCARG1x, ref_addr, Rx(tmp_reg2) |1: || } -| IF_GC_MAY_NOT_LEAK FCARG1x, >4, Rw(tmp_reg1) +| IF_GC_MAY_NOT_LEAK FCARG1x, >4, Rw(tmp_reg1), Rw(tmp_reg2) | // gc_possible_root(Z_COUNTED_P(z)) | EXT_CALL gc_possible_root, Rx(tmp_reg1) || } @@ -1052,8 +1078,45 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro -|.macro SEPARATE_ARRAY, addr, op_info, cold -| brk #0 // TODO +|.macro SEPARATE_ARRAY, addr, op_info, cold, tmp_reg1, tmp_reg2 +|| if (RC_MAY_BE_N(op_info)) { +|| if (Z_REG(addr) != ZREG_FP) { +| brk #0 // TODO +|| } else { +| GET_ZVAL_LVAL ZREG_FCARG1x, addr, Rx(tmp_reg1) +|| if (RC_MAY_BE_1(op_info)) { +| // if (GC_REFCOUNT() > 1) +| ldr Rw(tmp_reg1), [FCARG1x] +| cmp Rw(tmp_reg1), #1 +|| if (cold) { +| bhi >1 +|.cold_code +|1: +|| } else { +| brk #0 // TODO +| bls >2 +|| } +|| } +| IF_NOT_ZVAL_REFCOUNTED addr, >1, Rw(tmp_reg1), Rx(tmp_reg2) +| GC_DELREF FCARG1x, Rw(tmp_reg1) +|1: +| EXT_CALL zend_array_dup, REG0 +| mov REG0, RETVALx +| SET_ZVAL_PTR addr, REG0, Rx(tmp_reg1) +| SET_ZVAL_TYPE_INFO addr, IS_ARRAY_EX, Rw(tmp_reg1), Rx(tmp_reg2) +| mov FCARG1x, REG0 +|| if (RC_MAY_BE_1(op_info)) { +|| if (cold) { +| b >2 +|.code +|| } +|| } +|2: +|| } +|| } else { +| brk #0 // TODO +| GET_ZVAL_LVAL ZREG_FCARG1x, addr, Rx(tmp_reg1) +|| } |.endmacro |.macro EFREE_REG_REFERENCE @@ -3089,7 +3152,146 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o zend_jit_addr op2_addr = OP2_ADDR(); zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); - | brk #0 // TODO + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE + && type == BP_VAR_R + && !exit_addr) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + } + + if (op2_info & MAY_BE_LONG) { + bool op2_loaded = 0; + bool packed_loaded = 0; + + if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { + | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) + | brk #0 // TODO + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, TMP1w, TMP2 + } + if (op1_info & MAY_BE_PACKED_GUARD) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | brk #0 // TODO + } + if (type == BP_VAR_W) { + | // hval = Z_LVAL_P(dim); + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + op2_loaded = 1; + } + if (op1_info & MAY_BE_ARRAY_PACKED) { + zend_long val = -1; + + if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + val = Z_LVAL_P(Z_ZV(op2_addr)); + if (val >= 0 && val < HT_MAX_SIZE) { + packed_loaded = 1; + } + } else { + if (!op2_loaded) { + | // hval = Z_LVAL_P(dim); + | brk #0 // TODO + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + op2_loaded = 1; + } + packed_loaded = 1; + } + if (packed_loaded) { + | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); + if (op1_info & MAY_BE_ARRAY_HASH) { + | brk #0 // TODO + } + | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) + + | ldr REG0w, [FCARG1x, #offsetof(zend_array, nNumUsed)] + if (val == 0) { + | cmp REG0, #0 + } else if (val > 0 && !op2_loaded) { + | brk #0 // TODO + | LOAD_64BIT_VAL TMP1, val + | cmp REG0, TMP1 + } else { + | brk #0 // TODO + | cmp REG0, FCARG2x + } + + if (type == BP_JIT_IS) { + | brk #0 // TODO + } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { + | brk #0 // TODO + } else if (type == BP_VAR_IS && not_found_exit_addr) { + | brk #0 // TODO + } else if (type == BP_VAR_IS && found_exit_addr) { + | brk #0 // TODO + } else { + | bls >2 // NOT_FOUND + } + | // _ret = &_ht->arData[_h].val; + if (val >= 0) { + | ldr REG0, [FCARG1x, #offsetof(zend_array, arData)] + if (val != 0) { + | brk #0 // TODO + | LOAD_64BIT_VAL TMP1, val * sizeof(Bucket) + | add REG0, REG0, TMP1 + } + } else { + | brk #0 // TODO + | mov REG0, FCARG2x + | lsl REG0, REG0, #5 + | ldr TMP1, [FCARG1x, #offsetof(zend_array, arData)] + | add REG0, REG0, TMP1 + } + } + } + switch (type) { + case BP_JIT_IS: + | brk #0 // TODO + break; + case BP_VAR_R: + case BP_VAR_IS: + case BP_VAR_UNSET: + | brk #0 // TODO + break; + case BP_VAR_RW: + | brk #0 // TODO + break; + case BP_VAR_W: + if (packed_loaded) { + | IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w + } + | brk #0 // TODO + break; + default: + ZEND_UNREACHABLE(); + } + + if (type != BP_JIT_IS && (op2_info & MAY_BE_STRING)) { + | brk #0 // TODO + | b >8 + } + } + + if (op2_info & MAY_BE_STRING) { + | brk #0 // TODO + } + + if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) { + | brk #0 // TODO + } + + if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { + if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { + |.cold_code + |3: + } + | brk #0 // TODO + } return 1; } @@ -3369,7 +3571,123 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t { zend_jit_addr op2_addr, op3_addr, res_addr; - | brk #0 // TODO + op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; + op3_addr = OP1_DATA_ADDR(); + if (opline->result_type == IS_UNUSED) { + res_addr = 0; + } else { + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + } + + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + + | brk #0 // TODO + + val_info &= ~MAY_BE_UNDEF; + } + + if (op1_info & MAY_BE_REF) { + | brk #0 // TODO + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + + if (op1_info & MAY_BE_ARRAY) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { + | brk #0 // TODO + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, TMP1w, TMP2 + } + |3: + | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 + } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { + | brk #0 // TODO + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + |6: + if (opline->op2_type == IS_UNUSED) { + uint32_t var_info = MAY_BE_NULL; + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + + | brk #0 // TODO + + if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0)) { + return 0; + } + } else { + uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_W, op1_info, op2_info, NULL, NULL, NULL)) { + return 0; + } + + if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { + var_info |= MAY_BE_REF; + } + if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + var_info |= MAY_BE_RC1; + } + + |8: + | // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE); + if (opline->op1_type == IS_VAR) { + ZEND_ASSERT(opline->result_type == IS_UNUSED); + if (!zend_jit_assign_to_variable_call(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { + return 0; + } + } else { + if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { + return 0; + } + } + } + } + + if (((op1_info & MAY_BE_ARRAY) && + (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE))) || + (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)))) { + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + |.cold_code + |7: + } + + if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) && + (op1_info & MAY_BE_ARRAY)) { + | brk #0 // TODO + } + + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + | brk #0 // TODO + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + | brk #0 // TODO + } + |.code + } + } + +#ifdef ZEND_JIT_USE_RC_INFERENCE + if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { + /* ASSIGN_DIM may increase refcount of the key */ + op2_info |= MAY_BE_RCN; + } +#endif + + |9: + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline + + if (may_throw) { + zend_jit_check_exception(Dst); + } + return 1; } From 5538a4cee808aa1454bfc69349b566e75316522e Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Mon, 12 Apr 2021 03:22:21 +0000 Subject: [PATCH 044/229] Support failed JIT test case: assign_025.phpt Major changes are: 1. Support opcode FETCH_DIM_W for "$arr[0][0] = $ref;" in the loop. See the updates in function zend_jit_fetch_dim(). 2. Spill the registers and store the values into memory. See the updates in function zend_jit_spill_store(). This is done for Phi function. 3. Invoke function zend_array_destory() as dtor for arrays. This is done by zend_jit_free_cv() when leaving the function foo(). --- ext/opcache/jit/zend_jit_arm64.dasc | 160 ++++++++++++++++++++++++++-- 1 file changed, 153 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 5abbbe6859743..6dc23c7f3c4f4 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -999,7 +999,7 @@ static void* dasm_labels[zend_lb_MAX]; || if (opline && ((var_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) { | brk #0 // TODO || } -| brk #0 // TODO +| EXT_CALL zend_array_destroy, tmp_reg || } else { | EXT_CALL zend_jit_array_free, tmp_reg || } @@ -1114,7 +1114,6 @@ static void* dasm_labels[zend_lb_MAX]; |2: || } || } else { -| brk #0 // TODO | GET_ZVAL_LVAL ZREG_FCARG1x, addr, Rx(tmp_reg1) || } |.endmacro @@ -2462,7 +2461,16 @@ static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_ad ZEND_ASSERT(Z_MODE(src) == IS_REG); ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL); - | brk #0 // TODO + if ((info & MAY_BE_ANY) == MAY_BE_LONG) { + | SET_ZVAL_LVAL_FROM_REG dst, Rx(Z_REG(src)), TMP1 + if (set_type) { + | SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2 + } + } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { + | brk #0 // TODO + } else { + ZEND_UNREACHABLE(); + } return 1; } @@ -2527,7 +2535,25 @@ static int zend_jit_load_var(dasm_State **Dst, uint32_t info, int var, zend_reg static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info) { if (!zend_jit_same_addr(src, dst)) { - | brk #0 // TODO: test + if (Z_MODE(src) == IS_REG) { + if (Z_MODE(dst) == IS_REG) { + | brk #0 // TODO + } else if (Z_MODE(dst) == IS_MEM_ZVAL) { + | brk #0 // TODO + } else { + ZEND_UNREACHABLE(); + } + } else if (Z_MODE(src) == IS_MEM_ZVAL) { + if (Z_MODE(dst) == IS_REG) { + if (!zend_jit_load_reg(Dst, src, dst, info)) { + return 0; + } + } else { + ZEND_UNREACHABLE(); + } + } else { + ZEND_UNREACHABLE(); + } } return 1; } @@ -3205,7 +3231,10 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o if (packed_loaded) { | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); if (op1_info & MAY_BE_ARRAY_HASH) { - | brk #0 // TODO + || ZEND_ASSERT(HASH_FLAG_PACKED <= MAX_IMM12); + | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] + | tst TMP1w, #HASH_FLAG_PACKED + | beq >4 // HASH_FIND } | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) @@ -3265,7 +3294,24 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o if (packed_loaded) { | IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w } - | brk #0 // TODO + if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || packed_loaded) { + |2: + | //retval = zend_hash_index_add_new(ht, hval, &EG(uninitialized_zval)); + if (!op2_loaded) { + | // hval = Z_LVAL_P(dim); + | brk #0 // TODO + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + } + | LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval + | EXT_CALL zend_hash_index_add_new, REG0 + | mov REG0, RETVALx + if (op1_info & MAY_BE_ARRAY_HASH) { + | brk #0 // TODO + } + } + if (op1_info & MAY_BE_ARRAY_HASH) { + | brk #0 // TODO + } break; default: ZEND_UNREACHABLE(); @@ -5649,7 +5695,107 @@ static int zend_jit_fetch_dim(dasm_State **Dst, op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; - | brk #0 // TODO + if (op1_info & MAY_BE_REF) { + | brk #0 // TODO + } + + if (op1_info & MAY_BE_ARRAY) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, TMP1w, TMP2 + } + |3: + | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 + } + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { + if (op1_info & MAY_BE_ARRAY) { + |.cold_code + |7: + } + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + | brk #0 // TODO + } + if ((op1_info & MAY_BE_UNDEF) + && opline->opcode == ZEND_FETCH_DIM_RW) { + | brk #0 // TODO + } + | // ZVAL_ARR(container, zend_new_array(8)); + if (Z_REG(op1_addr) != ZREG_FP) { + | brk #0 // TODO + } + | EXT_CALL _zend_new_array_0, REG0 + | mov REG0, RETVALx + if (Z_REG(op1_addr) != ZREG_FP) { + | brk #0 // TODO + } + | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 + | mov FCARG1x, REG0 + if (op1_info & MAY_BE_ARRAY) { + | brk #0 // TODO + | b >1 + |.code + |1: + } + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + |6: + if (opline->op2_type == IS_UNUSED) { + | brk #0 // TODO + } else { + uint32_t type; + + switch (opline->opcode) { + case ZEND_FETCH_DIM_W: + case ZEND_FETCH_LIST_W: + type = BP_VAR_W; + break; + case ZEND_FETCH_DIM_RW: + type = BP_VAR_RW; + break; + case ZEND_FETCH_DIM_UNSET: + type = BP_VAR_UNSET; + break; + default: + ZEND_UNREACHABLE(); + } + + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, type, op1_info, op2_info, NULL, NULL, NULL)) { + return 0; + } + + |8: + | SET_ZVAL_PTR res_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2 + + if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { + |.cold_code + |9: + | brk #0 // TODO + |.code + } + } + } + + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + | brk #7 + } + +#ifdef ZEND_JIT_USE_RC_INFERENCE + if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { + /* ASSIGN_DIM may increase refcount of the key */ + op2_info |= MAY_BE_RCN; + } +#endif + + |8: + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline + + if (may_throw) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } return 1; } From 70a0b183c090f46d4bf42c3b7cad4527e6ee331c Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Mon, 12 Apr 2021 04:46:05 +0000 Subject: [PATCH 045/229] Support failed JIT test case: assign_026.phpt For statement "$a = new stdClass;", opcode NEW is used and JIT would invoke the original handler at runtime. Our major changes are made to support statements "$a->a=1;" and "$a->b=2;" where opcode ASSIGN_OBJ are used. --- ext/opcache/jit/zend_jit_arm64.dasc | 152 +++++++++++++++++++++++++++- 1 file changed, 151 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 6dc23c7f3c4f4..fb34c999b52be 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -6071,7 +6071,157 @@ static int zend_jit_assign_obj(dasm_State **Dst, zend_jit_addr prop_addr; bool needs_slow_path = 0; - | brk #0 // TODO + if (RETURN_VALUE_USED(opline)) { + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + } + + ZEND_ASSERT(opline->op2_type == IS_CONST); + ZEND_ASSERT(op1_info & MAY_BE_OBJECT); + + member = RT_CONSTANT(opline, opline->op2); + ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); + name = Z_STR_P(member); + prop_info = zend_get_known_property_info(op_array, ce, name, opline->op1_type == IS_UNUSED, op_array->filename); + + if (opline->op1_type == IS_UNUSED || use_this) { + | brk #0 // TODO + } else { + if (opline->op1_type == IS_VAR + && (op1_info & MAY_BE_INDIRECT) + && Z_REG(op1_addr) == ZREG_FP) { + | brk #0 // TODO + |1: + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + if (op1_info & MAY_BE_REF) { + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + | ZVAL_DEREF FCARG1x, op1_info, TMP1w + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | brk #0 // TODO + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, TMP1w, TMP2 + |.cold_code + |1: + | brk #0 // TODO + |.code + } + } + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + } + + if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { + prop_info = zend_get_known_property_info(op_array, trace_ce, name, opline->op1_type == IS_UNUSED, op_array->filename); + prop_info = zend_get_known_property_info(op_array, trace_ce, name, opline->op1_type == IS_UNUSED, op_array->filename); + if (prop_info) { + ce = trace_ce; + ce_is_instanceof = 0; + if (!(op1_info & MAY_BE_CLASS_GUARD)) { + if (!zend_jit_class_guard(Dst, opline, trace_ce)) { + return 0; + } + if (ssa->var_info && ssa_op->op1_use >= 0) { + ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; + ssa->var_info[ssa_op->op1_use].ce = ce; + ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; + } + if (ssa->var_info && ssa_op->op1_def >= 0) { + ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; + ssa->var_info[ssa_op->op1_def].ce = ce; + ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; + } + } + } + } + + if (!prop_info) { + needs_slow_path = 1; + + | ldr REG0, EX->run_time_cache + | LOAD_32BIT_VAL TMP1w, opline->extended_value + | add TMP1, REG0, TMP1 + | ldr REG2, [TMP1] + | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] + | cmp REG2, TMP1 + | bne >5 + if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { + | brk #0 // TODO + } + | brk #0 // TODO + prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { + | brk #0 // TODO + } + } else { + prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, prop_info->offset); + | brk #0 // TODO + } + + if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { + // value = zend_assign_to_variable(property_val, value, OP_DATA_TYPE, EX_USES_STRICT_TYPES()); + if (opline->result_type == IS_UNUSED) { + if (!zend_jit_assign_to_variable_call(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { + return 0; + } + } else { + if (!zend_jit_assign_to_variable(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { + return 0; + } + } + } + + if (needs_slow_path) { + |.cold_code + |5: + | SET_EX_OPLINE opline, REG0 + | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); + | LOAD_ADDR FCARG2x, name + + | LOAD_ZVAL_ADDR CARG3, val_addr + | ldr CARG4, EX->run_time_cache + | LOAD_32BIT_VAL TMP1w, opline->extended_value + | add CARG4, CARG4, TMP1 + if (RETURN_VALUE_USED(opline)) { + | brk #0 // TODO + | LOAD_ZVAL_ADDR CARG5, res_addr + } else { + | mov CARG5, xzr + } + + | EXT_CALL zend_jit_assign_obj_helper, REG0 + + if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + val_info |= MAY_BE_RC1|MAY_BE_RCN; + } + + |8: + | // FREE_OP_DATA(); + | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline + | b >9 + |.code + } + + |9: + if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline + } + + if (may_throw) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + return 1; } From 3c21fac1f30f84d242f3a2161f483d17ad957b76 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Mon, 12 Apr 2021 07:05:34 +0000 Subject: [PATCH 046/229] Support failed JIT test case: assign_dim_op_001.phpt This test case covers one new path in macro TRY_ADDREF, touching macro GC_ADDREF for the first time. --- ext/opcache/jit/zend_jit_arm64.dasc | 2 -- 1 file changed, 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index fb34c999b52be..0896aaede31f3 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -908,7 +908,6 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro GC_ADDREF, zv, tmp_reg -| brk #0 // TODO: test | ldr tmp_reg, [zv] | add tmp_reg, tmp_reg, #1 | str tmp_reg, [zv] @@ -949,7 +948,6 @@ static void* dasm_labels[zend_lb_MAX]; || if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | IF_NOT_REFCOUNTED type_flags_reg, >1 || } -| // brk #0 // TODO: test | GC_ADDREF value_ptr_reg, tmp_reg |1: || } From 54a40ecd2868e93307bb9798aca3fed670b45671 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Mon, 12 Apr 2021 13:04:26 +0000 Subject: [PATCH 047/229] Support failed JIT test case: assign_dim_002.phpt There are 6 user function calls in this test cases. The first 3 functions, i.e. foo(), foo1() and foo2(), can be supported already. In this patch, we mainly focus on foo3(). Note that based on my test, once foo3() gets supported, the remaining functions foo4() and foo5() can pass as well. Regarding function foo3(), we mainly focus on statement "$array = new ArrayObject();", and the following two opcodes are involved. 0009 V2 = NEW 0 string("ArrayObject") 0010 DO_FCALL Accordingly, functions zend_jit_handler(), zend_jit_cond_jmp() and zend_jit_do_fcall() are invoked to generate the machine code. See the handling process for case ZEND_NEW at file zend_jit.c. Hence, major changes in this patch are made to support this statement. Note that the updates at line 4840 in function zend_jit_do_fcall() are made to support the later internal function call, i.e. var_dump(). Note that another test "noval_001.phpt" would pass with this patch as well. --- ext/opcache/jit/zend_jit_arm64.dasc | 92 +++++++++++++++++++++++++---- 1 file changed, 81 insertions(+), 11 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 0896aaede31f3..44bbf61282de0 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -422,8 +422,14 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro -|.macro CMP_IP, addr -| brk #0 // TODO +|.macro CMP_IP, addr, tmp_reg1, tmp_reg2 +| LOAD_ADDR tmp_reg1, addr +|| if (GCC_GLOBAL_REGS) { +| cmp IP, tmp_reg1 +|| } else { +| ldr tmp_reg2, EX->opline +| cmp tmp_reg2, tmp_reg1 +|| } |.endmacro |.macro LOAD_ZVAL_ADDR, reg, addr @@ -1128,8 +1134,24 @@ static void* dasm_labels[zend_lb_MAX]; | brk #0 // TODO |.endmacro -|.macro OBJ_RELEASE, reg, exit_label +|.macro OBJ_RELEASE, reg, exit_label, tmp_reg1, tmp_reg2 +| GC_DELREF Rx(reg), Rw(tmp_reg1) +| bne >1 | brk #0 // TODO +| // zend_objects_store_del(obj); +|| if (reg != ZREG_FCARG1x) { +| mov FCARG1x, Rx(reg) +|| } +| EXT_CALL zend_objects_store_del, Rx(tmp_reg1) +| b exit_label +|1: +| IF_GC_MAY_NOT_LEAK Rx(reg), >1, Rw(tmp_reg1), Rw(tmp_reg2) +| // gc_possible_root(obj) +|| if (reg != ZREG_FCARG1x) { +| mov FCARG1x, Rx(reg) +|| } +| EXT_CALL gc_possible_root, Rx(tmp_reg1) +|1: |.endmacro |.macro UNDEFINED_OFFSET, opline @@ -2068,7 +2090,6 @@ static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level) if (call_level == 1) { | str xzr, EX:RX->prev_execute_data } else { - | brk #0 // TODO: test | ldr REG0, EX->call | str REG0, EX:RX->prev_execute_data } @@ -2430,7 +2451,8 @@ static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label) static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label) { - | brk #0 // TODO + | CMP_IP next_opline, TMP1, TMP2 + | bne =>target_label zend_jit_set_last_valid_opline(next_opline); @@ -4799,7 +4821,17 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | SET_EX_OPLINE opline, REG0 if (opline->opcode == ZEND_DO_FCALL) { - | brk #0 // TODO + if (!func) { + if (trace) { + uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + | brk #0 // TODO + } + } } if (!delayed_call_chain) { @@ -4807,7 +4839,8 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | str xzr, EX->call } else { | //EX(call) = call->prev_execute_data; - | brk #0 // TODO: test + | ldr REG0, EX:RX->prev_execute_data + | str REG0, EX->call } } delayed_call_chain = 0; @@ -4820,13 +4853,38 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } if (opline->opcode == ZEND_DO_FCALL) { - | brk #0 // TODO + if (!func) { + if (!trace) { + | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] + || ZEND_ASSERT(ZEND_ACC_DEPRECATED <= MAX_IMM12); + | tst TMP1w, #ZEND_ACC_DEPRECATED + | bne >1 + |.cold_code + |1: + | brk #0 // TODO + if (!GCC_GLOBAL_REGS) { + | mov FCARG1x, RX + } + | EXT_CALL zend_jit_deprecated_helper, REG0 + | and RETVALw, RETVALw, #0xff + | cmp RETVALw, #0 // Result is 0 on exception + | ldr REG0, EX:RX->func // reload + | bne >1 + | b ->exception_handler + |.code + |1: + } + } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { + | brk #0 + } } if (!func && opline->opcode != ZEND_DO_UCALL && opline->opcode != ZEND_DO_ICALL) { - | brk #0 // TODO + | ldrb TMP1w, [REG0, #offsetof(zend_function, type)] + | cmp TMP1w, #ZEND_USER_FUNCTION + | bne >8 } if ((!func || func->type == ZEND_USER_FUNCTION) @@ -5022,7 +5080,19 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend |8: if (opline->opcode == ZEND_DO_FCALL) { // TODO: optimize ??? - | brk #0 // TODO + | // if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) + | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 2)] + | tst TMP1w, #(ZEND_CALL_RELEASE_THIS >> 16) + | bne >1 + |.cold_code + |1: + | add TMP1, RX, #offsetof(zend_execute_data, This) + | GET_Z_PTR FCARG1x, TMP1 + | // OBJ_RELEASE(object); + | OBJ_RELEASE ZREG_FCARG1x, >2, ZREG_TMP1, ZREG_TMP2 + | b >2 + |.code + |2: } if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || @@ -5084,7 +5154,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) { - | brk #0 // TODO + | LOAD_IP_ADDR (opline + 1) } else if (trace && trace->op == ZEND_JIT_TRACE_END && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { From cb0dc59333c6ef2e9ab1a31d2ac8a92891f4a1c6 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Mon, 12 Apr 2021 14:31:20 +0000 Subject: [PATCH 048/229] Support failed JIT test case: assign_static_prop_001.phpt For function Foo(), the original handlers would be invoked for the first two statements. And the third statement "$a = 42", where ASSIGN opcode is involved, covers the cold code in function zend_jit_assign_to_variable(). For function $main(), statement "var_dump(Foo::$prop);" covers a new path in function zend_ jit_send_val() for SEND_VAL opcode. Besides, another 2 test cases, i.e. fetch_dim_r_003.phpt and fetch_dim_r_004.phpt, would pass as well with this patch. --- ext/opcache/jit/zend_jit_arm64.dasc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 44bbf61282de0..117ccc990f2b8 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -3579,7 +3579,6 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, | IF_ZVAL_REFCOUNTED var_use_addr, >1, TMP1w, TMP2 |.cold_code |1: - | brk #0 // TODO in_cold = 1; } if (Z_REG(var_use_addr) == ZREG_FCARG1x || Z_REG(var_use_addr) == ZREG_REG0) { @@ -3587,7 +3586,6 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, | brk #0 // TODO } else { - | brk #0 // TODO | GET_ZVAL_PTR FCARG1x, var_use_addr, TMP1 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 1)) { return 0; @@ -3595,10 +3593,11 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, } | GC_DELREF FCARG1x, TMP1w if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { - | brk #0 // TODO + | bne >4 } else { | brk #0 // TODO } + | brk #0 | ZVAL_DTOR_FUNC var_info, opline, TMP1 if (in_cold || (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0)) { if (check_exception) { @@ -3609,9 +3608,10 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, } if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { |4: - | brk #0 // TODO + | IF_GC_MAY_NOT_LEAK FCARG1x, >8, TMP1w, TMP2w + | EXT_CALL gc_possible_root, REG0 if (in_cold) { - | brk #0 // TODO + | b >8 } } if (in_cold) { @@ -5210,12 +5210,12 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t o if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); - | ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 + | ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 if (Z_REFCOUNTED_P(zv)) { | brk #0 // TODO: test } } else { - | brk #0 // TODO: test + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } return 1; From 77b3d71a1645ab3da48ca2d8075a336730f298a8 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Tue, 13 Apr 2021 01:32:37 +0000 Subject: [PATCH 049/229] Support failed JIT test case: assign_036.phpt This patch mainly supports the opcode FETCH_OBJ_R for statement "$a->result = "okey";". --- ext/opcache/jit/zend_jit_arm64.dasc | 197 +++++++++++++++++++++++++++- 1 file changed, 196 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 117ccc990f2b8..7a8d06aa9f426 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -6049,7 +6049,202 @@ static int zend_jit_fetch_obj(dasm_State **Dst, ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); - | brk #0 // TODO + member = RT_CONSTANT(opline, opline->op2); + ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); + prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); + + if (opline->op1_type == IS_UNUSED || use_this) { + | brk #0 // TODO + | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 + } else { + if (opline->op1_type == IS_VAR + && opline->opcode == ZEND_FETCH_OBJ_W + && (op1_info & MAY_BE_INDIRECT) + && Z_REG(op1_addr) == ZREG_FP) { + | brk #0 // TODO + |1: + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + if (op1_info & MAY_BE_REF) { + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + | ZVAL_DEREF FCARG1x, op1_info, TMP1w + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | brk #0 // TODO + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7, TMP1w, TMP2 + } + } + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + } + + if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { + prop_info = zend_get_known_property_info(op_array, trace_ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); + if (prop_info) { + ce = trace_ce; + ce_is_instanceof = 0; + if (!(op1_info & MAY_BE_CLASS_GUARD)) { + if (!zend_jit_class_guard(Dst, opline, trace_ce)) { + return 0; + } + if (ssa->var_info && ssa_op->op1_use >= 0) { + ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; + ssa->var_info[ssa_op->op1_use].ce = ce; + ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; + } + } + } + } + + if (!prop_info) { + | ldr REG0, EX->run_time_cache + | LOAD_32BIT_VAL TMP1, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + | add TMP1, TMP1, REG0 + | ldr REG2, [TMP1] + | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] + | cmp REG2, TMP1 + | bne >5 + | brk #0 // TODO: currently jump to Label 5. + | LOAD_32BIT_VAL TMP1, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) + | add TMP1, TMP1, REG0 + | ldr REG0, [TMP1] + may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); + if (may_be_dynamic) { + | brk #0 // TODO + | tst REG0, REG0 + if (opline->opcode == ZEND_FETCH_OBJ_W) { + | brk #0 // TODO + | blt >5 + } else { + | brk #0 // TODO + | blt >8 // dynamic property + } + } + | brk #0 // TODO + prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + if (opline->opcode == ZEND_FETCH_OBJ_W + && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS) + && (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS))) { + uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; + + | brk #0 // TODO + |.cold_code + |1: + | brk #0 // TODO + |.code + } + } else { + prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, prop_info->offset); + | brk #0 // TODO + } + if (op1_avoid_refcounting) { + SET_STACK_REG(JIT_G(current_frame)->stack, + EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); + } + + if (opline->opcode == ZEND_FETCH_OBJ_W) { + if (Z_REG(prop_addr) != ZREG_FCARG1x || Z_OFFSET(prop_addr) != 0) { + | brk #0 // TODO + | LOAD_ZVAL_ADDR FCARG1x, prop_addr + } + | brk #0 // TODO + | SET_ZVAL_PTR res_addr, FCARG1x, TMP1 + | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2 + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) { + ssa->var_info[ssa_op->result_def].indirect_reference = 1; + } + } else { + bool result_avoid_refcounting = 0; + + if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info) { + uint32_t flags = 0; + uint32_t old_info; + zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; + int32_t exit_point; + const void *exit_addr; + zend_uchar type; + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + + if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) + && !use_this + && !op1_avoid_refcounting) { + flags = ZEND_JIT_EXIT_FREE_OP1; + } + + | brk #0 // TODO + } else { + if (!zend_jit_zval_copy_deref(Dst, res_addr, prop_addr, ZREG_REG2)) { + return 0; + } + } + } + + |.cold_code + + if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) { + |5: + | SET_EX_OPLINE opline, REG0 + if (opline->opcode == ZEND_FETCH_OBJ_W) { + | brk #0 // TODO + | EXT_CALL zend_jit_fetch_obj_w_slow, REG0 + } else if (opline->opcode != ZEND_FETCH_OBJ_IS) { + | EXT_CALL zend_jit_fetch_obj_r_slow, REG0 + } else { + | brk #0 // TODO + | EXT_CALL zend_jit_fetch_obj_is_slow, REG0 + } + | b >9 + } + + if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { + |7: + | brk #0 // TODO + } + + if (!prop_info + && may_be_dynamic + && opline->opcode != ZEND_FETCH_OBJ_W) { + |8: + | brk #0 // TODO + } + + |.code; + |9: // END + if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) { + if (opline->op1_type == IS_VAR + && opline->opcode == ZEND_FETCH_OBJ_W + && (op1_info & MAY_BE_RC1)) { + zend_jit_addr orig_op1_addr = OP1_ADDR(); + + | brk #0 // TODO + } else if (!op1_avoid_refcounting) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline + } + } + + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE + && prop_info + && opline->op1_type != IS_VAR + && opline->op1_type != IS_TMP_VAR) { + may_throw = 0; + } + + if (may_throw) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + return 1; } From 3748319f773fb592e3fd488be235cdbd022bfbf7 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Tue, 13 Apr 2021 07:19:51 +0000 Subject: [PATCH 050/229] Support failed JIT test case: assign_035.phpt 1. For statement "echo $a->test()", opcode INIT_METHOD_CALL is involved. The updates in function zend_jit_init_method_call() and zend_jit_push_call_frame() are made to support it. 2. The updates in function zend_jit_leave_func() are made to support the RETURN opcode used in functions $closure and $test. 3. The updates in function zend_jit_assign_to_variable() are used to support statement "$x = $arr". 4. The updates in function zend_jit_fetch_dimension_address_inner() and zend_jit_simple_assign() are made to support statement "$x['a'] = $closure()", where opcode ASSIGN_DIM is involved. --- ext/opcache/jit/zend_jit_arm64.dasc | 280 ++++++++++++++++++++++++++-- 1 file changed, 268 insertions(+), 12 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 7a8d06aa9f426..fe49efc87052f 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -635,7 +635,6 @@ static void* dasm_labels[zend_lb_MAX]; || if (Z_LVAL_P(Z_ZV(addr)) == 0) { | mov Rx(reg), xzr || } else { -| brk #0 // TODO: test | LOAD_64BIT_VAL Rx(reg), Z_LVAL_P(Z_ZV(addr)) || } || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { @@ -1001,7 +1000,7 @@ static void* dasm_labels[zend_lb_MAX]; || } else if (type == IS_ARRAY) { || if ((var_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) { || if (opline && ((var_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) { -| brk #0 // TODO +| SET_EX_OPLINE opline, tmp_reg || } | EXT_CALL zend_array_destroy, tmp_reg || } else { @@ -3344,7 +3343,39 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } if (op2_info & MAY_BE_STRING) { - | brk #0 // TODO + |3: + if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { + | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) + | brk #0 // TODO + | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3, TMP1w, TMP2 + } + | // offset_key = Z_STR_P(dim); + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + | // retval = zend_hash_find(ht, offset_key); + switch (type) { + case BP_JIT_IS: + | brk #0 // TODO + break; + case BP_VAR_R: + case BP_VAR_IS: + case BP_VAR_UNSET: + | brk #0 // TODO + break; + case BP_VAR_RW: + | brk #0 // TODO + break; + case BP_VAR_W: + if (opline->op2_type != IS_CONST) { + | brk #0 // TODO + | EXT_CALL zend_jit_symtable_lookup_w, REG0 + } else { + | EXT_CALL zend_hash_lookup, REG0 + } + | mov REG0, RETVALx + break; + default: + ZEND_UNREACHABLE(); + } } if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) { @@ -3443,7 +3474,20 @@ static int zend_jit_simple_assign(dasm_State **Dst, } else { zend_jit_addr ref_addr; - | brk #0 + if (in_cold) { + | brk #0 // TODO + | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1, TMP1w, TMP2 + } else { + | IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1, TMP1w, TMP2 + |.cold_code + |1: + } + | brk #0 // TODO + if (in_cold) { + |1: + } else { + |.code + } } } @@ -3595,13 +3639,14 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { | bne >4 } else { - | brk #0 // TODO + | bne >8 } - | brk #0 | ZVAL_DTOR_FUNC var_info, opline, TMP1 if (in_cold || (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0)) { if (check_exception) { - | brk #0 // TODO + | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, REG0, TMP1 + | beq >8 + | b ->exception_handler } else { | brk #0 // TODO } @@ -4380,7 +4425,30 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con } if (opline->opcode == ZEND_INIT_METHOD_CALL) { | // Z_PTR(call->This) = obj; - | brk #0 // TODO + | ldr REG1, T1 + | str REG1, EX:RX->This.value.ptr + if (opline->op1_type == IS_UNUSED || use_this) { + | // call->call_info |= ZEND_CALL_HAS_THIS; + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + | brk #0 // TODO + } else { + | brk #0 // TODO + } + } else { + if (opline->op1_type == IS_CV) { + | // GC_ADDREF(obj); + | GC_ADDREF REG1, TMP1w + } + | // call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS; + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + | brk #0 // TODO + } else { + | ldr TMP1w, EX:RX->This.u1.type_info + | LOAD_32BIT_VAL TMP2w, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) + | orr TMP1w, TMP1w, TMP2w + | str TMP1w, EX:RX->This.u1.type_info + } + } } else if (!is_closure) { | // Z_CE(call->This) = called_scope; | str xzr, EX:RX->This.value.ptr @@ -4700,7 +4768,183 @@ static int zend_jit_init_method_call(dasm_State **Dst, ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); - | brk #0 // TODO + function_name = RT_CONSTANT(opline, opline->op2); + + if (info) { + call_info = info->callee_info; + while (call_info && call_info->caller_init_opline != opline) { + call_info = call_info->next_callee; + } + if (call_info && call_info->callee_func && !call_info->is_prototype) { + func = call_info->callee_func; + } + } + + if (polymorphic_side_trace) { + /* function is passed in r0 from parent_trace */ + } else { + if (opline->op1_type == IS_UNUSED || use_this) { + zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); + + | brk #0 // TODO + | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 + } else { + if (op1_info & MAY_BE_REF) { + if (opline->op1_type == IS_CV) { + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + | ZVAL_DEREF FCARG1x, op1_info, TMP1w + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } else { + /* Hack: Convert reference to regular value to simplify JIT code */ + ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP); + | brk #0 // TODO + } + } + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | brk #0 // TODO + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, TMP1w, TMP2 + |.cold_code + |1: + | brk #0 // TODO: currently not jump to cold code. + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + | SET_EX_OPLINE opline, REG0 + if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) { + | EXT_CALL zend_jit_invalid_method_call_tmp, REG0 + } else { + | EXT_CALL zend_jit_invalid_method_call, REG0 + } + | b ->exception_handler + |.code + } + } + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + } + + if (delayed_call_chain) { + if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { + return 0; + } + } + + | str FCARG1x, T1 // save + + if (func) { + | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); + | brk #0 // TODO + } else { + | // if (CACHED_PTR(opline->result.num) == obj->ce)) { + | ldr REG0, EX->run_time_cache + | LOAD_32BIT_VAL TMP1w, opline->result.num + | add TMP1, REG0, TMP1 + | ldr REG2, [TMP1] + | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] + | cmp REG2, TMP1 + | bne >1 + | brk #0 // TODO: currently jump to label 1. + | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); + | LOAD_32BIT_VAL TMP1w, (opline->result.num + sizeof(void*)) + | add TMP1, TMP1, REG0 + | ldr REG0, [TMP1] + } + + |.cold_code + |1: + | LOAD_ADDR FCARG2x, function_name + | mov CARG3, sp + | SET_EX_OPLINE opline, REG0 + if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) { + | brk #0 // TODO + | EXT_CALL zend_jit_find_method_tmp_helper, REG0 + } else { + | EXT_CALL zend_jit_find_method_helper, REG0 + } + | mov REG0, RETVALx + | cbnz REG0, >2 + | b ->exception_handler + |.code + |2: + } + + if (!func + && trace + && trace->op == ZEND_JIT_TRACE_INIT_CALL + && trace->func + ) { + int32_t exit_point; + const void *exit_addr; + + exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_METHOD_CALL); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + + func = (zend_function*)trace->func; + + if (func->type == ZEND_USER_FUNCTION && + (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || + (func->common.fn_flags & ZEND_ACC_CLOSURE) || + !func->common.function_name)) { + const zend_op *opcodes = func->op_array.opcodes; + + | brk #0 // TODO + } else { + | brk #0 // TODO + } + } + + if (!func) { + | // if (fbc->common.fn_flags & ZEND_ACC_STATIC) { + | ldr TMP1w, [REG0, #offsetof(zend_function, common.fn_flags)] + || ZEND_ASSERT(ZEND_ACC_STATIC <= MAX_IMM12); + | tst TMP1w, #ZEND_ACC_STATIC + | bne >1 + |.cold_code + |1: + } + + if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) { + | brk #0 // TODO + } + + if (!func) { + | brk #0 // TODO + | b >9 + |.code + } + + if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) { + if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 0, use_this, stack_check)) { + return 0; + } + } + + if (!func) { + |9: + } + zend_jit_start_reuse_ip(); + + if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, trace)) { + if (!zend_jit_save_call_chain(Dst, call_level)) { + return 0; + } + } else { + delayed_call_chain = 1; + delayed_call_level = call_level; + } + return 1; } @@ -5502,7 +5746,11 @@ static int zend_jit_leave_func(dasm_State **Dst, return 0; } } - | brk #0 // TODO: test + | // OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func))); + | ldr FCARG1x, EX->func + | sub FCARG1x, FCARG1x, #sizeof(zend_object) + | OBJ_RELEASE ZREG_FCARG1x, >4, ZREG_TMP1, ZREG_TMP2 + |4: } else if (may_need_release_this) { if (!left_frame) { left_frame = 1; @@ -5510,7 +5758,15 @@ static int zend_jit_leave_func(dasm_State **Dst, return 0; } } - | brk #0 // TODO: test + | // if (call_info & ZEND_CALL_RELEASE_THIS) + | LOAD_32BIT_VAL TMP1w, ZEND_CALL_RELEASE_THIS + | tst FCARG1w, TMP1w + | beq >4 + | // zend_object *object = Z_OBJ(execute_data->This); + | ldr FCARG1x, EX->This.value.obj + | // OBJ_RELEASE(object); + | OBJ_RELEASE ZREG_FCARG1x, >4, ZREG_TMP1, ZREG_TMP2 + |4: // TODO: avoid EG(excption) check for $this->foo() calls may_throw = 1; } @@ -5846,7 +6102,7 @@ static int zend_jit_fetch_dim(dasm_State **Dst, } if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { - | brk #7 + | brk #0 // TODO } #ifdef ZEND_JIT_USE_RC_INFERENCE From 99118a6931ddb2a83d735e646dfe4173fe811568 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Tue, 13 Apr 2021 10:29:55 +0000 Subject: [PATCH 051/229] Support failed JIT test case: fetch_dim_func_args_001.phpt 1. For statement "$a->change($a = array("a" => range(1, 5)));", the following opcodes will be generated: 0002 ASSIGN CV0($a) V1 0003 INIT_METHOD_CALL 1 CV0($a) string("change") 0004 INIT_NS_FCALL_BY_NAME 2 string("A\range") 0005 SEND_VAL_EX int(1) 1 0006 SEND_VAL_EX int(5) 2 0007 V1 = DO_FCALL_BY_NAME The updates in function zend_jit_init_fcall(), zend_jit_send_val() and zend_jit_do_fcall() are made to support INIT_NS_FCALL_BY_NAME, SEND_VAL_EX and DO_FCALL_BY_NAME respectively. 2. For method $change(), opcode RECV is used to obtain the argument: 0000 #1.CV0($config) [rc1, rcn, array of [any, ref]] = RECV 1 Accordingly the updates in functions zend_jit_recv() and zend_jit_verify_arg_type() are made. 3. For statement "array_keys($config["a"])", the following opcodes will be generated: 0001 INIT_NS_FCALL_BY_NAME 1 string("A\array_keys") 0002 CHECK_FUNC_ARG 1 0003 #3.V1 [ref, rc1, rcn, any] = FETCH_DIM_FUNC_ARG #1.CV0($config) ... -> #2.CV0($config) [rc1, rcn, ... 0004 SEND_FUNC_ARG #3.V1 [ref, rc1, rcn, any] 1 0005 #4.V1 [ref, rc1, rcn, any] = DO_FCALL_BY_NAME CHECK_FUNC_ARG and SEND_FUNC_ARG are not supported before. See the updates in functions zend_jit_check_func_arg() and zend_jit_send_var(). Besides, a new path is covered in macro OBJ_RELEASE when leaving. --- ext/opcache/jit/zend_jit_arm64.dasc | 213 ++++++++++++++++++++++++++-- 1 file changed, 204 insertions(+), 9 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index fe49efc87052f..588b6962f2f52 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -1136,7 +1136,6 @@ static void* dasm_labels[zend_lb_MAX]; |.macro OBJ_RELEASE, reg, exit_label, tmp_reg1, tmp_reg2 | GC_DELREF Rx(reg), Rw(tmp_reg1) | bne >1 -| brk #0 // TODO | // zend_objects_store_del(obj); || if (reg != ZREG_FCARG1x) { | mov FCARG1x, Rx(reg) @@ -4706,13 +4705,14 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) { | brk #0 // TODO } else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) { - | brk #0 // TODO + | LOAD_ADDR FCARG1x, zv; + | EXT_CALL zend_jit_find_ns_func_helper, REG0 } else { ZEND_UNREACHABLE(); } | // CACHE_PTR(opline->result.num, fbc); | ldr REG1, EX->run_time_cache - | // Get the return value of function zend_jit_find_func_helper + | // Get the return value of function zend_jit_find_func_helper/zend_jit_find_ns_func_helper | mov REG0, RETVALx | str REG0, [REG1, #opline->result.num] if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { @@ -5282,7 +5282,38 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend |8: } if (opline->opcode == ZEND_DO_FCALL_BY_NAME) { - | brk #0 // TODO + if (!func) { + if (trace) { + uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + | brk #0 // TODO + } else { + || ZEND_ASSERT(ZEND_ACC_DEPRECATED <= MAX_IMM12); + | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] + | tst TMP1w, #ZEND_ACC_DEPRECATED + | bne >1 + |.cold_code + |1: + | brk #0 // TODO + if (!GCC_GLOBAL_REGS) { + | mov FCARG1x, RX + } + | EXT_CALL zend_jit_deprecated_helper, REG0 + | and RETVALw, RETVALw, #0xff + | cmp RETVALw, #0 // Result is 0 on exception + | ldr REG0, EX:RX->func // reload + | bne >1 + | b ->exception_handler + |.code + |1: + } + } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { + | brk #0 // TODO + } } | // ZVAL_NULL(EX_VAR(opline->result.var)); @@ -5445,7 +5476,17 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t o } | brk #0 // TODO } else { - | brk #0 // TODO + | ldr REG0, EX:RX->func + | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] + | LOAD_32BIT_VAL TMP2w, mask + | tst TMP1w, TMP2w + | bne >1 + |.cold_code + |1: + | brk #0 // TODO + | SET_EX_OPLINE opline, REG0 + | b ->throw_cannot_pass_by_ref + |.code } } @@ -5504,7 +5545,30 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { | brk #0 // TODO } else if (opline->opcode == ZEND_SEND_FUNC_ARG) { - | brk #0 // TODO + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE + && JIT_G(current_frame) + && JIT_G(current_frame)->call + && JIT_G(current_frame)->call->func) { + if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { + if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { + return 0; + } + return 1; + } + } else { + | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] + | LOAD_32BIT_VAL TMP2w, ZEND_CALL_SEND_ARG_BY_REF + | tst TMP1w, TMP2w + | bne >1 + |.cold_code + |1: + | brk #0 // TODO + if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { + return 0; + } + | b >7 + |.code + } } if (op1_info & MAY_BE_UNDEF) { @@ -5579,7 +5643,42 @@ static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) { uint32_t arg_num = opline->op2.num; - | brk #0 // TODO + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE + && JIT_G(current_frame) + && JIT_G(current_frame)->call + && JIT_G(current_frame)->call->func) { + | brk #0 // TODO + } else { + // if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) { + uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); + + if (!zend_jit_reuse_ip(Dst)) { + return 0; + } + + | ldr REG0, EX:RX->func + | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] + | LOAD_32BIT_VAL TMP2w, mask + | tst TMP1w, TMP2w + | bne >1 + |.cold_code + |1: + | brk #0 // TODO + | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); + | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] + | LOAD_32BIT_VAL TMP2w, ZEND_CALL_SEND_ARG_BY_REF + | orr TMP1w, TMP1w, TMP2w + | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] + | b >1 + |.code + | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); + | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] + | LOAD_32BIT_VAL TMP2w, ZEND_CALL_SEND_ARG_BY_REF + | mvn TMP2w, TMP2w + | and TMP1w, TMP1w, TMP2w + | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] + |1: + } return 1; } @@ -6160,7 +6259,59 @@ static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zen uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1x : ZREG_REG0; - | brk #0 // TODO + if (ZEND_ARG_SEND_MODE(arg_info)) { + | brk #0 // TODO + } + + if (type_mask != 0) { + if (is_power_of_two(type_mask)) { + uint32_t type_code = concrete_type(type_mask); + || ZEND_ASSERT(type_code <= MAX_IMM12); + | IF_NOT_ZVAL_TYPE res_addr, type_code, >1, TMP1w, TMP2 + } else { + | brk #0 // TODO + } + + |.cold_code + |1: + + in_cold = 1; + } + + | brk #0 // TODO: currently in cold code + if (Z_REG(res_addr) != ZREG_FCARG1x || Z_OFFSET(res_addr) != 0) { + | brk #0 // TODO + | LOAD_ZVAL_ADDR FCARG1x, res_addr + } + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + | brk #0 // TODO + | SET_EX_OPLINE opline, REG0 + } else { + | ADDR_STORE EX->opline, opline, REG0 + } + | LOAD_ADDR FCARG2x, (ptrdiff_t)arg_info + | EXT_CALL zend_jit_verify_arg_slow, REG0 + | mov REG0w, RETVALw + + if (check_exception) { + | brk #0 // TODO + | and REG0w, REG0w, #0xff + | tst REG0w, REG0w + if (in_cold) { + | bne >1 + | b ->exception_handler + |.code + |1: + } else { + | beq ->exception_handler + } + } else if (in_cold) { + | brk #0 // TODO + | b >1 + |.code + |1: + } + return 1; } @@ -6169,7 +6320,51 @@ static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_ uint32_t arg_num = opline->op1.num; zend_arg_info *arg_info = NULL; - | brk #0 // TODO + if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { + if (EXPECTED(arg_num <= op_array->num_args)) { + arg_info = &op_array->arg_info[arg_num-1]; + } else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) { + arg_info = &op_array->arg_info[op_array->num_args]; + } + if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) { + arg_info = NULL; + } + } + + if (arg_info || (opline+1)->opcode != ZEND_RECV) { + | ldr TMP1w, EX->This.u2.num_args + | LOAD_32BIT_VAL TMP2w, arg_num + | cmp TMP1w, TMP2w + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | brk #0 // TODO + } else { + | blt >1 + |.cold_code + |1: + | brk #0 // TODO + |.code + } + } + + if (arg_info) { + if (!zend_jit_verify_arg_type(Dst, opline, arg_info, 1)) { + return 0; + } + } + + if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { + if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) { + | LOAD_IP_ADDR (opline + 1) + zend_jit_set_last_valid_opline(opline + 1); + } + } + return 1; } From c5d6fc036313820d3f909fd061e04683ea91f8e4 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Wed, 14 Apr 2021 01:26:42 +0000 Subject: [PATCH 052/229] Support failed JIT test case: fetch_dim_r_002.phpt The opcodes for function $foo are: 0001 INIT_FCALL 1 96 string("var_dump") 0002 #2.T1 [null, long] = FETCH_DIM_R array(...) #1.CV0($n) [...] 0003 SEND_VAL #2.T1 [null, long] 1 0004 DO_ICALL 0005 RETURN null Opcode FETCH_DIM_R is not touched before, and the updates in function zend_jit_fetch_dim_read() are made to support it. As different types of arguments are used for $foo, several cases in function zend_jit_fetch_dimension_address_inner() are covered as well. Besides, opcode DO_ICALL can reach one site of cold code in function zend_jit_do_fcall(). --- ext/opcache/jit/zend_jit_arm64.dasc | 238 ++++++++++++++++++++++++++-- 1 file changed, 229 insertions(+), 9 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 588b6962f2f52..a98e83eb88446 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -3212,7 +3212,6 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) - | brk #0 // TODO | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, TMP1w, TMP2 } if (op1_info & MAY_BE_PACKED_GUARD) { @@ -3240,7 +3239,6 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } else { if (!op2_loaded) { | // hval = Z_LVAL_P(dim); - | brk #0 // TODO | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 op2_loaded = 1; } @@ -3303,7 +3301,34 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o case BP_VAR_R: case BP_VAR_IS: case BP_VAR_UNSET: + if (packed_loaded) { + | brk #0 // TODO + } + if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (packed_loaded && (op1_info & MAY_BE_ARRAY_HASH))) { + | brk #0 // TODO + } + if (op1_info & MAY_BE_ARRAY_HASH) { + |4: + if (!op2_loaded) { + | brk #0 // TODO + } + | EXT_CALL _zend_hash_index_find, REG0 + | mov REG0, RETVALx + | tst REG0, REG0 + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { + | brk #0 // TODO + } else if (type == BP_VAR_IS && not_found_exit_addr) { + | brk #0 // TODO + } else if (type == BP_VAR_IS && found_exit_addr) { + | brk #0 // TODO + } else { + | beq >2 // NOT_FOUND + } + } + |.cold_code + |2: | brk #0 // TODO + |.code break; case BP_VAR_RW: | brk #0 // TODO @@ -3336,7 +3361,6 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } if (type != BP_JIT_IS && (op2_info & MAY_BE_STRING)) { - | brk #0 // TODO | b >8 } } @@ -3345,7 +3369,6 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |3: if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) - | brk #0 // TODO | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3, TMP1w, TMP2 } | // offset_key = Z_STR_P(dim); @@ -3358,7 +3381,36 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o case BP_VAR_R: case BP_VAR_IS: case BP_VAR_UNSET: - | brk #0 // TODO + if (opline->op2_type != IS_CONST) { + | ldrb TMP1w, [FCARG2x, #offsetof(zend_string, val)] + | cmp TMP1w, #((uint8_t) ('9')) + | ble >1 + |.cold_code + |1: + | EXT_CALL zend_jit_symtable_find, REG0 + | b >1 + |.code + | EXT_CALL zend_hash_find, REG0 + |1: + } else { + | brk #0 // TODO + | EXT_CALL _zend_hash_find_known_hash, REG0 + } + | mov REG0, RETVALx + | tst REG0, REG0 + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { + | brk #0 // TODO + } else if (type == BP_VAR_IS && not_found_exit_addr) { + | brk #0 // TODO + } else if (type == BP_VAR_IS && found_exit_addr) { + | brk #0 // TODO + } else { + | beq >2 // NOT_FOUND + |.cold_code + |2: + | brk #0 // TODO + |.code + } break; case BP_VAR_RW: | brk #0 // TODO @@ -3386,7 +3438,34 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |.cold_code |3: } - | brk #0 // TODO + | SET_EX_OPLINE opline, REG0 + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + switch (type) { + case BP_VAR_R: + | LOAD_ZVAL_ADDR CARG3, res_addr + | EXT_CALL zend_jit_fetch_dim_r_helper, REG0 + | mov REG0, RETVALx + | b >9 + break; + case BP_JIT_IS: + | brk #0 // TODO + break; + case BP_VAR_IS: + case BP_VAR_UNSET: + | brk #0 // TODO + break; + case BP_VAR_RW: + | brk #0 // TODO + break; + case BP_VAR_W: + | brk #0 // TODO + break; + default: + ZEND_UNREACHABLE(); + } + if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { + |.code + } } return 1; @@ -5381,10 +5460,9 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | // zend_vm_stack_free_call_frame(call); | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 2)] | tst TMP1w, #((ZEND_CALL_ALLOCATED >> 16) & 0xff) - | bne >1 // TODO: test. In current case, don't jump to cold-code. + | bne >1 |.cold_code |1: - | brk #0 // TODO | mov FCARG1x, RX | EXT_CALL zend_jit_free_call_frame, REG0 | b >1 @@ -6102,7 +6180,149 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, orig_op1_addr = OP1_ADDR(); op2_addr = OP2_ADDR(); - | brk #0 // TODO + if (opline->opcode != ZEND_FETCH_DIM_IS + && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + } + + if ((res_info & MAY_BE_GUARD) + && JIT_G(current_frame) + && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) { + uint32_t flags = 0; + uint32_t old_op1_info = 0; + uint32_t old_info; + zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; + int32_t exit_point; + + if (opline->opcode != ZEND_FETCH_LIST_R + && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) + && !op1_avoid_refcounting) { + flags |= ZEND_JIT_EXIT_FREE_OP1; + } + if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) + && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + flags |= ZEND_JIT_EXIT_FREE_OP2; + } + if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) + && !(flags & ZEND_JIT_EXIT_FREE_OP1) + && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) + && (ssa_op+1)->op1_use == ssa_op->result_def + && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG))) + && zend_jit_may_avoid_refcounting(opline+1)) { + result_avoid_refcounting = 1; + ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; + } + + if (op1_avoid_refcounting) { + old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); + SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); + } + + if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) { + old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); + SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); + SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); + exit_point = zend_jit_trace_get_exit_point(opline+1, flags); + SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); + res_exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!res_exit_addr) { + return 0; + } + res_info &= ~MAY_BE_GUARD; + ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; + } + + if (opline->opcode == ZEND_FETCH_DIM_IS + && !(res_info & MAY_BE_NULL)) { + old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); + SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0); + SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NULL); + exit_point = zend_jit_trace_get_exit_point(opline+1, flags); + SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); + not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!not_found_exit_addr) { + return 0; + } + } + + if (op1_avoid_refcounting) { + SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); + } + } + + if (op1_info & MAY_BE_REF) { + | brk #0 // TODO + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + + if (op1_info & MAY_BE_ARRAY) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { + | brk #0 // TODO + } + | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr, TMP1 + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, res_exit_addr, not_found_exit_addr, exit_addr)) { + return 0; + } + } + + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { + if (op1_info & MAY_BE_ARRAY) { + |.cold_code + |7: + } + + | brk #0 // TODO + + if (op1_info & MAY_BE_ARRAY) { + |.code + } + } + + if (op1_info & MAY_BE_ARRAY) { + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + + |8: + if (res_exit_addr) { + zend_uchar type = concrete_type(res_info); + + | brk #0 // TODO + } else if (op1_info & MAY_BE_ARRAY_OF_REF) { + | brk #0 // TODO + if (!zend_jit_zval_copy_deref(Dst, res_addr, val_addr, ZREG_REG2)) { + return 0; + } + } else { + | // ZVAL_COPY + | ZVAL_COPY_VALUE res_addr, -1, val_addr, res_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | lsr REG1w, REG1w, #8 + | and REG1w, REG1w, #0xff + | TRY_ADDREF res_info, REG1w, REG2, TMP1 + } + } + |9: // END + +#ifdef ZEND_JIT_USE_RC_INFERENCE + if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { + /* Magic offsetGet() may increase refcount of the key */ + op2_info |= MAY_BE_RCN; + } +#endif + + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline + if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline + } + + if (may_throw) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + return 1; } From ee4a429609bd3ebda835c78015fe1ae0a94594d7 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Wed, 14 Apr 2021 02:57:56 +0000 Subject: [PATCH 053/229] Support failed JIT test case: fetch_dim_rw_001.phpt Opcode FETCH_DIM_RW is not touched before and the udpates in function zend_jit_fetch_dim() and zend_jit_fetch_dimension_address_inner() are made to support it. Besides, one new path is covered in function zend_jit_return() when leaving. --- ext/opcache/jit/zend_jit_arm64.dasc | 63 ++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index a98e83eb88446..0fe74ed7af461 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -3302,7 +3302,20 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o case BP_VAR_IS: case BP_VAR_UNSET: if (packed_loaded) { - | brk #0 // TODO + if (op1_info & MAY_BE_ARRAY_HASH) { + | IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w + } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { + /* perform IS_UNDEF check only after result type guard (during deoptimization) */ + if (!found_exit_addr || (op1_info & MAY_BE_ARRAY_HASH)) { + | brk #0 // TODO + } + } else if (type == BP_VAR_IS && not_found_exit_addr) { + | brk #0 // TODO + } else if (type == BP_VAR_IS && found_exit_addr) { + | brk #0 // TODO + } else { + | brk #0 // TODO + } } if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (packed_loaded && (op1_info & MAY_BE_ARRAY_HASH))) { | brk #0 // TODO @@ -3331,7 +3344,19 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |.code break; case BP_VAR_RW: - | brk #0 // TODO + if (packed_loaded) { + | brk #0 // TODO + } + |2: + |4: + if (!op2_loaded) { + | // hval = Z_LVAL_P(dim); + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + } + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_hash_index_lookup_rw, REG0 + | mov REG0, RETVALx + | cbz REG0, >9 break; case BP_VAR_W: if (packed_loaded) { @@ -6073,7 +6098,28 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o } if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - | brk #0 // TODO: test + if (return_value_used == -1) { + | beq >1 + |.cold_code + |1: + } + if (return_value_used != 1) { + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | brk #0 // TODO + } + | brk #0 // TODO + if (RC_MAY_BE_1(op1_info)) { + | brk #0 // TODO + } + if (return_value_used == -1) { + if (jit_return_label >= 0) { + | brk #0 // TODO + } else { + | brk #0 // TODO + } + |.code + } + } } else if (return_value_used == -1) { if (jit_return_label >= 0) { | brk #0 // TODO: test @@ -6095,7 +6141,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o | brk #0 // TODO: test } } else if (opline->op1_type == IS_TMP_VAR) { - | brk #0 // TODO + | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } else if (opline->op1_type == IS_CV) { if (op1_info & MAY_BE_REF) { | brk #0 // TODO @@ -6359,7 +6405,14 @@ static int zend_jit_fetch_dim(dasm_State **Dst, } if ((op1_info & MAY_BE_UNDEF) && opline->opcode == ZEND_FETCH_DIM_RW) { - | brk #0 // TODO + if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { + | brk #0 // TODO + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, TMP1w, TMP2 + } + | SET_EX_OPLINE opline, REG0 + | LOAD_32BIT_VAL FCARG1w, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + |1: } | // ZVAL_ARR(container, zend_new_array(8)); if (Z_REG(op1_addr) != ZREG_FP) { From ad5328b9fef5c2a74870482b306bfdf65111359f Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Wed, 14 Apr 2021 04:10:25 +0000 Subject: [PATCH 054/229] Support failed JIT test case: fetch_obj_004.phpt Opcode ASSIGN_OBJ is generated for statement "$x->a = 1;" and one new path in function zend_jit_assign_obj() is covered. Note that function zend_jit_assign_to_variable_call() is invoked along this new path. Besides, helper function zend_objects_store_del() is used as the dtor for objects. --- ext/opcache/jit/zend_jit_arm64.dasc | 58 +++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 0fe74ed7af461..ad44f6e297a39 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -1011,7 +1011,7 @@ static void* dasm_labels[zend_lb_MAX]; || if (opline) { | brk #0 // TODO || } -| brk #0 // TODO +| EXT_CALL zend_objects_store_del, REG0 || break; || } || } @@ -1833,7 +1833,7 @@ static int zend_jit_assign_tmp_stub(dasm_State **Dst) uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; |->assign_tmp: - | brk #0 // TODO + | sub sp, sp, #16 if (!zend_jit_assign_to_variable( Dst, NULL, var_addr, var_addr, -1, -1, @@ -1841,6 +1841,7 @@ static int zend_jit_assign_tmp_stub(dasm_State **Dst) 0, 0)) { return 0; } + | add sp, sp, #16 | ret return 1; } @@ -3673,7 +3674,36 @@ static int zend_jit_assign_to_variable_call(dasm_State **Dst, zend_jit_addr __res_addr, bool __check_exception) { - | brk #0 // TODO + if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1x || Z_OFFSET(var_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, var_addr + } + if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2x || Z_OFFSET(val_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG2x, val_addr + } + if (opline) { + | SET_EX_OPLINE opline, REG0 + } + if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { + | bl ->assign_tmp + } else if (val_type == IS_CONST) { + | brk #0 // TODO + } else if (val_type == IS_TMP_VAR) { + | brk #0 // TODO + } else if (val_type == IS_VAR) { + if (!(val_info & MAY_BE_REF)) { + | brk #0 // TODO + } else { + | brk #0 // TODO + } + } else if (val_type == IS_CV) { + if (!(val_info & MAY_BE_REF)) { + | brk #0 // TODO + } else { + | brk #0 // TODO + } + } else { + ZEND_UNREACHABLE(); + } return 1; } @@ -7151,7 +7181,27 @@ static int zend_jit_assign_obj(dasm_State **Dst, } } else { prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, prop_info->offset); - | brk #0 // TODO + if (!ce || ce_is_instanceof || !(ce->ce_flags & ZEND_ACC_IMMUTABLE) || ce->__get || ce->__set) { + // Undefined property with magic __get()/__set() + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | brk #0 // TODO + } else { + | brk #0 // TODO + needs_slow_path = 1; + } + } + if (ZEND_TYPE_IS_SET(prop_info->type)) { + uint32_t info = val_info; + + | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); + | brk #0 // TODO + } } if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { From 27ddf3f12b5cb1d90d52d4c78fda61019a17f5e5 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Wed, 14 Apr 2021 07:02:45 +0000 Subject: [PATCH 055/229] Support failed JIT test case: fetch_obj_002.phpt One new path is covered inside function zend_jit_fetch_obj() due to the use of FETCH_OBJ_R opcode. Note that function zend_jit_zval_copy_deref() is invoked along this new path. Updates in function zend_jit_free() are made to support FREE opcode. Stub function zend_jit_leave_function_stub() is touched for the first time. --- ext/opcache/jit/zend_jit_arm64.dasc | 106 ++++++++++++++++++++++++++-- 1 file changed, 99 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index ad44f6e297a39..12a7d1acd37a9 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -453,7 +453,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro GET_Z_TYPE_INFO, reg, zv -| mov reg, dword [zv+offsetof(zval,u1.type_info)] +| ldr reg, [zv, #offsetof(zval,u1.type_info)] |.endmacro |.macro SET_Z_TYPE_INFO, zv, type, tmp_reg @@ -826,7 +826,8 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro IF_UNDEF, type_reg, label -| brk #0 // TODO +| tst type_reg, type_reg +| beq label |.endmacro |.macro IF_TYPE, type, val, label @@ -1031,7 +1032,6 @@ static void* dasm_labels[zend_lb_MAX]; |.cold_code |1: || } else { -| brk #0 // TODO: test. | IF_NOT_ZVAL_REFCOUNTED addr, >4, Rw(tmp_reg1), Rx(tmp_reg2) || } || } @@ -1338,7 +1338,35 @@ static int zend_jit_exception_handler_undef_stub(dasm_State **Dst) static int zend_jit_leave_function_stub(dasm_State **Dst) { |->leave_function_handler: - | brk #0 // TODO: test + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | LOAD_32BIT_VAL TMP1w, ZEND_CALL_TOP + | tst FCARG1w, TMP1w + | bne >1 + | brk #0 // TODO: currently jump to label 1. + | EXT_CALL zend_jit_leave_nested_func_helper, REG0 + | ADD_HYBRID_SPAD + | JMP_IP TMP1 + |1: + | EXT_CALL zend_jit_leave_top_func_helper, REG0 + | ADD_HYBRID_SPAD + | JMP_IP TMP1 + } else { + if (GCC_GLOBAL_REGS) { + | add sp, sp, SPAD + } else { + | mov FCARG2x, FP + | ldp FP, RX, T2 // retore FP and IP + | ldr LR, T4 // retore LR + | add sp, sp, NR_SPAD + } + | LOAD_32BIT_VAL TMP1w, ZEND_CALL_TOP + | tst FCARG1w, TMP1w + | bne >1 + | brk #0 // TODO: currently jump to label 1. + | EXT_JMP zend_jit_leave_nested_func_helper, REG0 + |1: + | EXT_JMP zend_jit_leave_top_func_helper, REG0 + } return 1; } @@ -6193,7 +6221,23 @@ static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, ze { ZEND_ASSERT(type_reg == ZREG_REG2); - | brk #0 // TODO + | GET_ZVAL_PTR REG1, val_addr, TMP1 + | lsr TMP1w, REG2w, #8 + | and TMP1w, TMP1w, #0xff // TMP1w -> 8-15 bits of REG2w + | IF_NOT_REFCOUNTED TMP1w, >2 + | brk #0 // TODO: currently jump to label 2 directly. + | and TMP2w, REG2w, #0xff // TMP2w -> low 8 bits of REG2w + | IF_NOT_TYPE TMP2w, IS_REFERENCE, >1 + | add TMP3, REG1, #offsetof(zend_reference, val) + | GET_Z_TYPE_INFO REG2w, TMP3 + | GET_Z_PTR REG1, TMP3 + | IF_NOT_REFCOUNTED TMP1w, >2 + |1: + | GC_ADDREF REG1, TMP1w + |2: + | SET_ZVAL_PTR res_addr, REG1, TMP1 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2, TMP1 + return 1; } @@ -6899,7 +6943,30 @@ static int zend_jit_fetch_obj(dasm_State **Dst, } } else { prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, prop_info->offset); - | brk #0 // TODO + | LOAD_32BIT_VAL TMP1w, (prop_info->offset + 8) + | ldr REG2w, [FCARG1x, TMP1] + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) { + /* perform IS_UNDEF check only after result type guard (during deoptimization) */ + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | brk #0 // TODO + } + } else { + | and TMP1w, REG2w, #0xff // low 8 bits. 8-15 bits are used later in zend_jit_zval_copy_deref(). + | IF_UNDEF TMP1w, >5 + } + if (opline->opcode == ZEND_FETCH_OBJ_W + && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS) + && ZEND_TYPE_IS_SET(prop_info->type)) { + uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; + + | brk #0 // TODO + } } if (op1_avoid_refcounting) { SET_STACK_REG(JIT_G(current_frame)->stack, @@ -7266,7 +7333,32 @@ static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_i { zend_jit_addr op1_addr = OP1_ADDR(); - | brk #0 // TODO + if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + if (may_throw) { + | brk #0 // TODO + | SET_EX_OPLINE opline, REG0 + } + if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) { + if (op1_info & MAY_BE_ARRAY) { + | brk #0 // TODO + | IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7, TMP1w, TMP2 + } + | brk #0 // TODO + | LOAD_32BIT_VAL TMP1w, opline->op1.var + offsetof(zval, u2.fe_iter_idx) + | ldr FCARG1w, [FP, TMP1] + | LOAD_32BIT_VAL TMP1w, -1 + | cmp FCARG1w, TMP1w + | beq >7 + | EXT_CALL zend_hash_iterator_del, REG0 + |7: + } + | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline, ZREG_TMP1, ZREG_TMP2 + if (may_throw) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + } return 1; } From 97766b6b281a203eb6e15753f1c9cc1c5f68aaa9 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Wed, 14 Apr 2021 08:25:29 +0000 Subject: [PATCH 056/229] Support failed JIT test case: fetch_obj_003.phpt Opcode ASSIGN_OBJ_OP is used for statement "$x->a += 2;". The updates in function zend_jit_assign_obj_op() are made to support this opcode. --- ext/opcache/jit/zend_jit_arm64.dasc | 131 +++++++++++++++++++++++++++- 1 file changed, 130 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 12a7d1acd37a9..9e92acd38f586 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -7127,7 +7127,136 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, ZEND_ASSERT(op1_info & MAY_BE_OBJECT); ZEND_ASSERT(opline->result_type == IS_UNUSED); - | brk #0 // TODO + member = RT_CONSTANT(opline, opline->op2); + ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); + name = Z_STR_P(member); + prop_info = zend_get_known_property_info(op_array, ce, name, opline->op1_type == IS_UNUSED, op_array->filename); + + if (opline->op1_type == IS_UNUSED || use_this) { + | brk #0 // TODO + | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 + } else { + if (opline->op1_type == IS_VAR + && (op1_info & MAY_BE_INDIRECT) + && Z_REG(op1_addr) == ZREG_FP) { + | brk #0 // TODO + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + if (op1_info & MAY_BE_REF) { + | brk #0 // TODO + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | brk #0 // TODO + } else { + | brk #0 // TODO + } + } + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + } + + if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { + prop_info = zend_get_known_property_info(op_array, trace_ce, name, opline->op1_type == IS_UNUSED, op_array->filename); + if (prop_info) { + ce = trace_ce; + ce_is_instanceof = 0; + if (!(op1_info & MAY_BE_CLASS_GUARD)) { + if (!zend_jit_class_guard(Dst, opline, trace_ce)) { + return 0; + } + if (ssa->var_info && ssa_op->op1_use >= 0) { + ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; + ssa->var_info[ssa_op->op1_use].ce = ce; + ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; + } + if (ssa->var_info && ssa_op->op1_def >= 0) { + ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; + ssa->var_info[ssa_op->op1_def].ce = ce; + ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; + } + } + } + } + + if (!prop_info) { + needs_slow_path = 1; + + | brk #0 // TODO + prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } else { + prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, prop_info->offset); + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | brk #0 // TODO + } else { + | LOAD_32BIT_VAL TMP1w, (prop_info->offset + 8) + | ldrb TMP2w, [FCARG1x, TMP1] + | IF_TYPE TMP2w, IS_UNDEF, >7 + needs_slow_path = 1; + } + if (ZEND_TYPE_IS_SET(prop_info->type)) { + uint32_t info = val_info; + + | brk #0 // TODO + } + } + + if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { + zend_jit_addr var_addr = prop_addr; + uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; + uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; + + var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + | brk #0 + } + + if (needs_slow_path) { + |.cold_code + |7: + | SET_EX_OPLINE opline, REG0 + | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); + | LOAD_ADDR FCARG2x, name + | LOAD_ZVAL_ADDR CARG3, val_addr + | ldr CARG4, EX->run_time_cache + | LOAD_32BIT_VAL TMP1w, (opline+1)->extended_value + | add CARG4, CARG4, TMP1 + | LOAD_ADDR CARG5, binary_op + | EXT_CALL zend_jit_assign_obj_op_helper, REG0 + + if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + val_info |= MAY_BE_RC1|MAY_BE_RCN; + } + + |8: + | // FREE_OP_DATA(); + | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline + | b >9 + |.code + } + + |9: + if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline + } + + if (may_throw) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + return 1; } From 1f9f1b32166c9c3569a270ed9a8a0657de6faf27 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 19 Apr 2021 19:54:54 +0300 Subject: [PATCH 057/229] Replace --SKIPIF-- by --EXTENSIONS-- --- ext/opcache/tests/jit/arm64/add_001.phpt | 4 ++-- ext/opcache/tests/jit/arm64/add_002.phpt | 4 ++-- ext/opcache/tests/jit/arm64/add_003.phpt | 4 ++-- ext/opcache/tests/jit/arm64/add_004.phpt | 4 ++-- ext/opcache/tests/jit/arm64/add_005.phpt | 4 ++-- ext/opcache/tests/jit/arm64/hot_func_001.phpt | 4 ++-- ext/opcache/tests/jit/arm64/hot_func_002.phpt | 4 ++-- ext/opcache/tests/jit/arm64/icall_001.phpt | 4 ++-- ext/opcache/tests/jit/arm64/loop_001.phpt | 4 ++-- ext/opcache/tests/jit/arm64/loop_002.phpt | 4 ++-- ext/opcache/tests/jit/arm64/recv_001.phpt | 4 ++-- ext/opcache/tests/jit/arm64/ret_001.phpt | 4 ++-- ext/opcache/tests/jit/arm64/ret_002.phpt | 4 ++-- ext/opcache/tests/jit/arm64/ret_003.phpt | 4 ++-- ext/opcache/tests/jit/arm64/skipif.inc | 3 --- ext/opcache/tests/jit/arm64/ucall_001.phpt | 4 ++-- ext/opcache/tests/jit/arm64/ucall_002.phpt | 4 ++-- ext/opcache/tests/jit/arm64/ucall_003.phpt | 4 ++-- ext/opcache/tests/jit/arm64/ucall_004.phpt | 4 ++-- 19 files changed, 36 insertions(+), 39 deletions(-) delete mode 100644 ext/opcache/tests/jit/arm64/skipif.inc diff --git a/ext/opcache/tests/jit/arm64/add_001.phpt b/ext/opcache/tests/jit/arm64/add_001.phpt index dc89e6d0216e2..25fb1e7b4c71f 100644 --- a/ext/opcache/tests/jit/arm64/add_001.phpt +++ b/ext/opcache/tests/jit/arm64/add_001.phpt @@ -6,8 +6,8 @@ opcache.enable_cli=1 opcache.file_update_protection=0 opcache.jit_buffer_size=32M ;opcache.jit_debug=257 ---SKIPIF-- - +--EXTENSIONS-- +opcache --FILE-- +--EXTENSIONS-- +opcache --FILE-- +--EXTENSIONS-- +opcache --FILE-- +--EXTENSIONS-- +opcache --FILE-- +--EXTENSIONS-- +opcache --FILE-- +--EXTENSIONS-- +opcache --FILE-- +--EXTENSIONS-- +opcache --FILE-- +--EXTENSIONS-- +opcache --FILE-- +--EXTENSIONS-- +opcache --FILE-- +--EXTENSIONS-- +opcache --FILE-- +--EXTENSIONS-- +opcache --FILE-- +--EXTENSIONS-- +opcache --FILE-- +--EXTENSIONS-- +opcache --FILE-- +--EXTENSIONS-- +opcache --FILE-- diff --git a/ext/opcache/tests/jit/arm64/ucall_001.phpt b/ext/opcache/tests/jit/arm64/ucall_001.phpt index c3fea8c9dafce..df69fc7018d6e 100644 --- a/ext/opcache/tests/jit/arm64/ucall_001.phpt +++ b/ext/opcache/tests/jit/arm64/ucall_001.phpt @@ -6,8 +6,8 @@ opcache.enable_cli=1 opcache.file_update_protection=0 opcache.jit_buffer_size=32M ;opcache.jit_debug=257 ---SKIPIF-- - +--EXTENSIONS-- +opcache --FILE-- +--EXTENSIONS-- +opcache --FILE-- +--EXTENSIONS-- +opcache --FILE-- +--EXTENSIONS-- +opcache --FILE-- Date: Tue, 20 Apr 2021 00:46:54 +0300 Subject: [PATCH 058/229] JIT/AArch64: INIT_FCALL and DO_FCALL support for optimized function code-generation (1204/1205) --- ext/opcache/jit/zend_jit_arm64.dasc | 137 +++++++++++++++++++++++++--- 1 file changed, 122 insertions(+), 15 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 9e92acd38f586..75125aacf3fef 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -31,6 +31,12 @@ |.define CARG4, x3 |.define CARG5, x4 |.define CARG6, x5 +|.define CARG1w, w0 +|.define CARG2w, w1 +|.define CARG3w, w2 +|.define CARG4w, w3 +|.define CARG5w, w4 +|.define CARG6w, w5 |.define RETVALx, x0 |.define RETVALw, w0 |.define FCARG1x, x0 @@ -1447,7 +1453,16 @@ static int zend_jit_cannot_add_element_stub(dasm_State **Dst) static int zend_jit_undefined_function_stub(dasm_State **Dst) { |->undefined_function: - | brk #0 // TODO + | ldr REG0, EX->opline + | movz CARG1, #0 + | LOAD_ADDR CARG2, "Call to undefined function %s()" + | ldr CARG3w, [REG0, #offsetof(zend_op, op2.constant)] + | sxtw CARG3, CARG3w + | add REG0, REG0, CARG3 + | ldr CARG3, [REG0] + | add CARG3, CARG3, #offsetof(zend_string, val) + | EXT_CALL zend_throw_error, REG0 + | b ->exception_handler return 1; } @@ -4836,7 +4851,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t if (!func && trace && trace->op == ZEND_JIT_TRACE_INIT_CALL) { - | brk #0 // TODO: Tracing mode. ASLR? + func = (zend_function*)trace->func; } if (opline->opcode == ZEND_INIT_FCALL @@ -4845,7 +4860,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t /* load constant address later */ } else if (func && op_array == &func->op_array) { /* recursive call */ - | brk #0 // TODO + | ldr REG0, EX->func } else { | // if (CACHED_PTR(opline->result.num)) | ldr REG0, EX->run_time_cache @@ -4857,7 +4872,12 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t && func && func->type == ZEND_USER_FUNCTION && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) { - | brk #0 // TODO + | LOAD_ADDR FCARG1x, func + | EXT_CALL zend_jit_init_func_run_time_cache_helper, REG0 + | ldr REG1, EX->run_time_cache + | mov REG0, RETVALx + | str REG0, [REG1, #opline->result.num] + | b >3 } else { zval *zv = RT_CONSTANT(opline, opline->op2); @@ -4865,7 +4885,8 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t | LOAD_ADDR FCARG1x, Z_STR_P(zv); | EXT_CALL zend_jit_find_func_helper, REG0 } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) { - | brk #0 // TODO + | LOAD_ADDR FCARG1x, Z_STR_P(zv + 1); + | EXT_CALL zend_jit_find_func_helper, REG0 } else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) { | LOAD_ADDR FCARG1x, zv; | EXT_CALL zend_jit_find_ns_func_helper, REG0 @@ -4882,7 +4903,8 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t } else { | cbnz REG0, >3 | // SAVE_OPLINE(); - | brk #0 // TODO: invalid func address. + | SET_EX_OPLINE opline, REG0 + | b ->undefined_function } } |.code @@ -5175,12 +5197,10 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend prev_opline = opline - 1; while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) { - | brk #0 // TODO prev_opline--; } if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY || prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { - | brk #0 // TODO unknown_num_args = 1; } @@ -5196,7 +5216,6 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend if (!func) { /* resolve function at run time */ } else if (func->type == ZEND_USER_FUNCTION) { - | brk #0 // TODO ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL); call_num_args = call_info->num_args; } else if (func->type == ZEND_INTERNAL_FUNCTION) { @@ -5312,11 +5331,12 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend if (func && op_array == &func->op_array) { /* recursive call */ if (trace || func->op_array.cache_size > sizeof(void*)) { - | brk #0 // TODO + | ldr REG2, EX->run_time_cache + | str REG2, EX:RX->run_time_cache } } else { if (func) { - | brk #0 // TODO + | ldr REG0, EX:RX->func } | ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)] // Always defined as ZEND_MAP_PTR_KIND_PTR_OR_OFFSET. See Zend/zend_map_ptr.h. @@ -5324,7 +5344,19 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | ldr REG2, [REG2] #elif ZEND_MAP_PTR_KIND == ZEND_MAP_PTR_KIND_PTR_OR_OFFSET if (func && !(func->op_array.fn_flags & ZEND_ACC_CLOSURE)) { - | brk #0 // TODO + if (ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) { + | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 + } else if ((func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) + && (!func->op_array.scope || (func->op_array.scope->ce_flags & ZEND_ACC_LINKED))) { + | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 + } else { + /* the called op_array may be not persisted yet */ + | tst REG2, #1 + | beq >1 + | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 + |1: + | ldr REG2, [REG2] + } } else { | tst REG2, #1 | beq >1 @@ -5345,11 +5377,82 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | // opline = op_array->opcodes; if (func && !unknown_num_args) { - | brk #0 // TODO + for (i = call_num_args; i < func->op_array.last_var; i++) { + uint32_t n = EX_NUM_TO_VAR(i); + | // ZVAL_UNDEF(EX_VAR(n)) + | str wzr, [RX, #(n + offsetof(zval,u1.type_info))] + } + + if (call_num_args <= func->op_array.num_args) { + if (!trace || (trace->op == ZEND_JIT_TRACE_END + && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { + uint32_t num_args; + + if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) { + if (trace) { + num_args = 0; + } else if (call_info) { + num_args = skip_valid_arguments(op_array, ssa, call_info); + } else { + num_args = call_num_args; + } + } else { + num_args = call_num_args; + } + if (zend_accel_in_shm(func->op_array.opcodes)) { + | LOAD_IP_ADDR (func->op_array.opcodes + num_args) + } else { + | ldr REG0, EX->func + if (GCC_GLOBAL_REGS) { + | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] + if (num_args) { + | add IP, IP, #(num_args * sizeof(zend_op)) + } + } else { + | ldr REG1, [REG0, #offsetof(zend_op_array, opcodes)] + if (num_args) { + | add REG1, REG1, #(num_args * sizeof(zend_op)) + } + | str REG1, EX->opline + } + } + + if (!trace && op_array == &func->op_array) { + /* recursive call */ + if (ZEND_OBSERVER_ENABLED) { + | SAVE_IP + | mov CARG1, FP + | EXT_CALL zend_observer_fcall_begin, REG0 + } +#ifdef CONTEXT_THREADED_JIT + | brk #0 // TODO +#else + | b =>num_args +#endif + return 1; + } + } + } else { + if (!trace || (trace->op == ZEND_JIT_TRACE_END + && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { + if (func && zend_accel_in_shm(func->op_array.opcodes)) { + | LOAD_IP_ADDR (func->op_array.opcodes) + } else if (GCC_GLOBAL_REGS) { + | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] + } else { + | ldr CARG1, [REG0, #offsetof(zend_op_array, opcodes)] + | str CARG1, EX->opline + } + } + if (!GCC_GLOBAL_REGS) { + | mov CARG1, FP + } + | EXT_CALL zend_jit_copy_extra_args_helper, REG0 + } } else { | // opline = op_array->opcodes if (func && zend_accel_in_shm(func->op_array.opcodes)) { - | brk #0 // TODO + | LOAD_IP_ADDR (func->op_array.opcodes) } else if (GCC_GLOBAL_REGS) { | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] } else { @@ -5358,6 +5461,10 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } if (func) { | brk #0 // TODO + | // num_args = EX_NUM_ARGS(); + | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] + | // if (UNEXPECTED(num_args > first_extra_arg)) + | cmp REG1w, #(func->op_array.num_args) } else { | // first_extra_arg = op_array->num_args; | ldr REG2w, [REG0, #offsetof(zend_op_array, num_args)] @@ -5387,7 +5494,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend |1: | // if (EXPECTED((int)num_args < op_array->last_var)) { if (func) { - | brk #0 // TODO + | movz REG2w, #(func->op_array.last_var) } else { | ldr REG2w, [REG0, #offsetof(zend_op_array, last_var)] } From 9512fa4c060b0bf2d239600cb9606402d2297989 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Tue, 20 Apr 2021 02:49:06 +0000 Subject: [PATCH 059/229] Add necessary assertions on range for INIT_FCALL and DO_FCALL Range checks are needed before encoding them into AArch64 instructions as immediates. --- ext/opcache/jit/zend_jit_arm64.dasc | 30 ++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 75125aacf3fef..9535efd9945fb 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -89,7 +89,8 @@ #define TMP_ZVAL_OFFSET 0 #define DASM_ALIGNMENT 16 -#define MAX_IMM12 0xfff // maximum value for imm12 +#define MAX_IMM12 0xfff // maximum value for imm12 +#define LDR_STR_IMM (MAX_IMM12 * 8) // maximum value for imm12 * 8 #include "Zend/zend_cpuinfo.h" @@ -1458,8 +1459,7 @@ static int zend_jit_undefined_function_stub(dasm_State **Dst) | LOAD_ADDR CARG2, "Call to undefined function %s()" | ldr CARG3w, [REG0, #offsetof(zend_op, op2.constant)] | sxtw CARG3, CARG3w - | add REG0, REG0, CARG3 - | ldr CARG3, [REG0] + | ldr CARG3, [REG0, CARG3] | add CARG3, CARG3, #offsetof(zend_string, val) | EXT_CALL zend_throw_error, REG0 | b ->exception_handler @@ -4876,6 +4876,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t | EXT_CALL zend_jit_init_func_run_time_cache_helper, REG0 | ldr REG1, EX->run_time_cache | mov REG0, RETVALx + || ZEND_ASSERT(opline->result.num <= LDR_STR_IMM); | str REG0, [REG1, #opline->result.num] | b >3 } else { @@ -5380,7 +5381,9 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend for (i = call_num_args; i < func->op_array.last_var; i++) { uint32_t n = EX_NUM_TO_VAR(i); | // ZVAL_UNDEF(EX_VAR(n)) - | str wzr, [RX, #(n + offsetof(zval,u1.type_info))] + || ZEND_ASSERT(n <= MAX_IMM12); + | add TMP1, RX, #n + | SET_Z_TYPE_INFO TMP1, IS_UNDEF, TMP2w } if (call_num_args <= func->op_array.num_args) { @@ -5403,17 +5406,18 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | LOAD_IP_ADDR (func->op_array.opcodes + num_args) } else { | ldr REG0, EX->func + || ZEND_ASSERT((num_args * sizeof(zend_op)) <= MAX_IMM12); if (GCC_GLOBAL_REGS) { | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] if (num_args) { | add IP, IP, #(num_args * sizeof(zend_op)) } } else { - | ldr REG1, [REG0, #offsetof(zend_op_array, opcodes)] + | ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)] if (num_args) { - | add REG1, REG1, #(num_args * sizeof(zend_op)) + | add FCARG1x, FCARG1x, #(num_args * sizeof(zend_op)) } - | str REG1, EX->opline + | str FCARG1x, EX->opline } } @@ -5421,7 +5425,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend /* recursive call */ if (ZEND_OBSERVER_ENABLED) { | SAVE_IP - | mov CARG1, FP + | mov FCARG1x, FP | EXT_CALL zend_observer_fcall_begin, REG0 } #ifdef CONTEXT_THREADED_JIT @@ -5440,12 +5444,12 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } else if (GCC_GLOBAL_REGS) { | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] } else { - | ldr CARG1, [REG0, #offsetof(zend_op_array, opcodes)] - | str CARG1, EX->opline + | ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)] + | str FCARG1x, EX->opline } } if (!GCC_GLOBAL_REGS) { - | mov CARG1, FP + | mov FCARG1x, FP } | EXT_CALL zend_jit_copy_extra_args_helper, REG0 } @@ -5460,10 +5464,10 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | str FCARG1x, EX->opline } if (func) { - | brk #0 // TODO | // num_args = EX_NUM_ARGS(); | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] | // if (UNEXPECTED(num_args > first_extra_arg)) + || ZEND_ASSERT(func->op_array.num_args <= MAX_IMM12); | cmp REG1w, #(func->op_array.num_args) } else { | // first_extra_arg = op_array->num_args; @@ -5494,7 +5498,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend |1: | // if (EXPECTED((int)num_args < op_array->last_var)) { if (func) { - | movz REG2w, #(func->op_array.last_var) + | LOAD_32BIT_VAL REG2w, func->op_array.last_var } else { | ldr REG2w, [REG0, #offsetof(zend_op_array, last_var)] } From b7f7df6c06ee8b43c12fd6ee06633d7b9df309a6 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Mon, 19 Apr 2021 13:48:05 +0000 Subject: [PATCH 060/229] Fix one bug in macro IF_GC_MAY_NOT_LEAK Instruction is misused. 'dword', i.e. 32 bits, are loaded from memory. Hence, 'ldr' should be used rather than 'ldrh'. --- ext/opcache/jit/zend_jit_arm64.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 9535efd9945fb..ffc0b0a2d611d 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -933,7 +933,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro IF_GC_MAY_NOT_LEAK, ptr, label, tmp_reg1, tmp_reg2 -| ldrh tmp_reg1, [ptr, #4] +| ldr tmp_reg1, [ptr, #4] | LOAD_32BIT_VAL tmp_reg2, (GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT)) | tst tmp_reg1, tmp_reg2 | bne label From e78e9a6e10f8ada516eaa0c153c73d54d8f0903e Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Thu, 15 Apr 2021 03:45:07 +0000 Subject: [PATCH 061/229] Support failed JIT test case: fetch_obj_001.phpt This test case is a big one. Major changes are: 1. statement "foo($obj->a)" One new path is covered in function zend_jit_fetch_obj() for the involved FETCH_OBJ_W opcode. See the update around label 5. Opcode SEND_REF is used. The updates in function zend_jit_send_ref() are made to support it. Note that macro FREE_OP is executed for the first time. Temproray registers are passed since they are used inside. As a result, its use sites are updated accordingly. 2. statement "$a = array()" in $foo2 One new path in function zend_jit_assign_to_variable() is covered. 3. statements involving variable $d in $bar One new path in function zend_jit_fetch_obj() is covered. See the updates around label 7. Note that in macro EMALLOC, condition ZEND_DEBUG can be covered by DEBUG build, i.e. "./configure --enable-debug". --- ext/opcache/jit/zend_jit_arm64.dasc | 180 +++++++++++++++++++++++++--- 1 file changed, 161 insertions(+), 19 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index ffc0b0a2d611d..73f2116461300 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -1081,10 +1081,10 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro -|.macro FREE_OP, op_type, op, op_info, cold, opline +|.macro FREE_OP, op_type, op, op_info, cold, opline, tmp_reg1, tmp_reg2 || if (op_type & (IS_VAR|IS_TMP_VAR)) { -| brk #0 // TODO: test -| // ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var), op_info, 0, cold, opline +|| zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var); +| ZVAL_PTR_DTOR addr, op_info, 0, cold, opline, tmp_reg1, tmp_reg2 || } |.endmacro @@ -1137,7 +1137,32 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro EMALLOC, size, op_array, opline -| brk #0 // TODO +||#if ZEND_DEBUG +|| const char *filename = op_array->filename ? op_array->filename->val : NULL; +| mov FCARG1x, #size +| LOAD_ADDR FCARG2x, filename +| LOAD_32BIT_VAL CARG3w, opline->lineno +| mov CARG4, xzr +| mov CARG5, xzr +| EXT_CALL _emalloc, REG0 +| mov REG0, RETVALx +||#else +||#ifdef HAVE_BUILTIN_CONSTANT_P +|| if (size > 24 && size <= 32) { +| EXT_CALL _emalloc_32, REG0 +| mov REG0, RETVALx +|| } else { +| mov FCARG1x, #size +| EXT_CALL _emalloc, REG0 +| mov REG0, RETVALx +|| } +||#else +| brk #0 // TODO +| mov FCARG1x, #size +| EXT_CALL _emalloc, REG0 +| mov REG0, RETVALx +||#endif +||#endif |.endmacro |.macro OBJ_RELEASE, reg, exit_label, tmp_reg1, tmp_reg2 @@ -3115,8 +3140,8 @@ static int zend_jit_math_helper(dasm_State **Dst, } else { ZEND_UNREACHABLE(); } - | FREE_OP op1_type, op1, op1_info, 0, opline - | FREE_OP op2_type, op2, op2_info, 0, opline + | FREE_OP op1_type, op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + | FREE_OP op2_type, op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 if (may_throw) { zend_jit_check_exception(Dst); } @@ -3804,7 +3829,36 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, if (Z_REG(var_use_addr) == ZREG_FCARG1x || Z_REG(var_use_addr) == ZREG_REG0) { bool keep_gc = 0; - | brk #0 // TODO + | GET_ZVAL_PTR Rx(tmp_reg), var_use_addr, TMP1 + if (tmp_reg == ZREG_FCARG1x) { + if (Z_MODE(val_addr) == IS_REG) { + keep_gc = 1; + } else if ((val_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) == 0) { + keep_gc = 1; + } else if (Z_MODE(val_addr) == IS_CONST_ZVAL) { + zval *zv = Z_ZV(val_addr); + if (Z_TYPE_P(zv) == IS_DOUBLE) { + if (Z_DVAL_P(zv) == 0) { + keep_gc = 1; + } + } else if (IS_SIGNED_32BIT(Z_LVAL_P(zv))) { + keep_gc = 1; + } + } else if (Z_MODE(val_addr) == IS_MEM_ZVAL) { + if ((val_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { + keep_gc = 1; + } + } + } + if (!keep_gc) { + | str Rx(tmp_reg), T1 // save + } + if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 0)) { + return 0; + } + if (!keep_gc) { + | ldr FCARG1x, T1 // restore + } } else { | GET_ZVAL_PTR FCARG1x, var_use_addr, TMP1 if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 1)) { @@ -3969,7 +4023,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t #endif |9: - | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 if (may_throw) { zend_jit_check_exception(Dst); @@ -5792,7 +5846,75 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend op1_addr = OP1_ADDR(); arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); - | brk #0 // TODO + if (!zend_jit_reuse_ip(Dst)) { + return 0; + } + + if (opline->op1_type == IS_VAR) { + if (op1_info & MAY_BE_INDIRECT) { + | LOAD_ZVAL_ADDR REG0, op1_addr + | // if (EXPECTED(Z_TYPE_P(ret) == IS_INDIRECT)) { + | IF_NOT_Z_TYPE REG0, IS_INDIRECT, >1, TMP1w + | // ret = Z_INDIRECT_P(ret); + | GET_Z_PTR REG0, REG0 + |1: + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + } + } else if (opline->op1_type == IS_CV) { + if (op1_info & MAY_BE_UNDEF) { + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + | brk #0 // TODO + } + op1_info &= ~MAY_BE_UNDEF; + op1_info |= MAY_BE_NULL; + } + } else { + ZEND_UNREACHABLE(); + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) { + if (op1_info & MAY_BE_REF) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >2, TMP1w, TMP2 + | GET_ZVAL_PTR REG1, op1_addr, TMP1 + | GC_ADDREF REG1, TMP1w + | SET_ZVAL_PTR arg_addr, REG1, TMP1 + | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX, TMP1w, TMP2 + | b >6 + } + |2: + | // ZVAL_NEW_REF(arg, varptr); + if (opline->op1_type == IS_VAR) { + if (Z_REG(op1_addr) != ZREG_REG0 || Z_OFFSET(op1_addr) != 0) { + | brk #0 // TODO + | LOAD_ZVAL_ADDR REG0, op1_addr + } + | str REG0, T1 // save + } + | EMALLOC sizeof(zend_reference), op_array, opline // Allocate space in REG0 + | mov TMP1w, #2 + | str TMP1w, [REG0] + || ZEND_ASSERT(GC_REFERENCE <= MAX_IMM12); + | mov TMP1w, #GC_REFERENCE + | str TMP1w, [REG0, #offsetof(zend_reference, gc.u.type_info)] + | str xzr, [REG0, #offsetof(zend_reference, sources.ptr)] + ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val)); + if (opline->op1_type == IS_VAR) { + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG1, 0); + + | ldr REG1, T1 // restore + | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_REG2, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | SET_ZVAL_PTR val_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX, TMP1w, TMP2 + } else { + | brk #0 // TODO + } + | SET_ZVAL_PTR arg_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX, TMP1w, TMP2 + } + + |6: + | FREE_OP opline->op1_type, opline->op1, op1_info, !cold, opline, ZREG_TMP1, ZREG_TMP2 + |7: return 1; } @@ -6543,9 +6665,9 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, } #endif - | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) { - | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 } if (may_throw) { @@ -6670,7 +6792,7 @@ static int zend_jit_fetch_dim(dasm_State **Dst, #endif |8: - | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 if (may_throw) { if (!zend_jit_check_exception(Dst)) { @@ -7127,7 +7249,6 @@ static int zend_jit_fetch_obj(dasm_State **Dst, |5: | SET_EX_OPLINE opline, REG0 if (opline->opcode == ZEND_FETCH_OBJ_W) { - | brk #0 // TODO | EXT_CALL zend_jit_fetch_obj_w_slow, REG0 } else if (opline->opcode != ZEND_FETCH_OBJ_IS) { | EXT_CALL zend_jit_fetch_obj_r_slow, REG0 @@ -7140,7 +7261,28 @@ static int zend_jit_fetch_obj(dasm_State **Dst, if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { |7: - | brk #0 // TODO + if (opline->opcode != ZEND_FETCH_OBJ_IS) { + | SET_EX_OPLINE opline, REG0 + if (opline->opcode != ZEND_FETCH_OBJ_W + && (op1_info & MAY_BE_UNDEF)) { + zend_jit_addr orig_op1_addr = OP1_ADDR(); + + | brk #0 // TODO + } else if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | brk #0 // TODO + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + | LOAD_ADDR FCARG2x, Z_STRVAL_P(member) + if (opline->opcode == ZEND_FETCH_OBJ_W) { + | EXT_CALL zend_jit_invalid_property_write, REG0 + | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2 + } else { + | brk #0 // TODO + } + | b >9 + } else { + | brk #0 // TODO + } } if (!prop_info @@ -7160,7 +7302,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, | brk #0 // TODO } else if (!op1_avoid_refcounting) { - | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 } } @@ -7352,14 +7494,14 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, |8: | // FREE_OP_DATA(); - | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline + | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2 | b >9 |.code } |9: if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) { - | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 } if (may_throw) { @@ -7550,14 +7692,14 @@ static int zend_jit_assign_obj(dasm_State **Dst, |8: | // FREE_OP_DATA(); - | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline + | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2 | b >9 |.code } |9: if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) { - | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 } if (may_throw) { From 91f417157767ba9cb73915f112c3b048b655e677 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Thu, 15 Apr 2021 09:16:29 +0000 Subject: [PATCH 062/229] Support failed JIT test case: cmp_001.phpt Comparison between LONG and DOUBLE is (partially) supported in a similar way to comparison between two LONG values. See the updates in function zend_jit_cmp(). Key difference lies in handling NaN. 1. Instruction 'fcmp' is used to substitue 'ucomisd' in x86 implementation. Both of them raise invalid operation exception only when either source operand is an SNaN.[1][2] 2. Parity flag is used in x86 to check whether either operand is NaN.[3] I think this is QNaN case. As for AArch64, we use instruction 'bvs'.[4] It's worthing noting that condition codes have different meanings for floating-point comparions(e.g. 'fcmp')[4] compared to the general-purpose comparisons(e.g. 'cmp').[5] For instance, 'b.hs' after 'fcmp' can check not only the cases "greater than, equal to" but also the case "unordered"(that is NaN). We may simply treat it as a combination of 'jae' and 'jp' in x86. 3. Instruction 'SETcc' is used in x86 for the case of ">=" or ">". Note that flag "swap" is set in implementation, and it falls into cases ZEND_IS_SMALLER or ZEND_IS_SMALLER_OR_EQUAL. We can use 'cset' in AArch64. However, it's weird that the NaN check is missing in x86. I suppose it might be a bug. Take the case ">=" as an example. The two operands can be either DOUBLE + LONG or DOUBLE + DOUBLE. See the relevant code where flag "swap" is set(i.e. function zend_jit_cmp_double_long() and function zend_jit_cmp_double_double()). For the case "NaN >= 1.0", the expected result should be FALSE, however, JIT/x86 would produce TRUE due to the following "setae al". Unfortunately I haven't constructed one test case to trigger that. In our implementation, we choose to follow the case of "<" or "<=", and I believe our implementation is safe anyway.. 4. Temporary FP register is also needed and we reserve v16. See the updates in file zend_jit_arm64.h. 5. Macro SET_ZVAL_TYPE_INFO_FROM_REG is misused in function zend_jit_zval_copy_deref(). The second argument should be 32-bit long and we fix it. Note that simple test cases involving NaN are tested locally. I believe it would get deeper testing by cmp_003.phpt(we will support it later). [1] https://developer.arm.com/documentation/dui0204/f/vector-floating-point-programming/vfp-instructions/fcmp?lang=en [2] https://www.felixcloutier.com/x86/ucomisd [3] https://en.wikipedia.org/wiki/Parity_flag [4] https://community.arm.com/developer/ip-products/processors/b/processors-ip-blog/posts/condition-codes-4-floating-point-comparisons-using-vfp [5] https://community.arm.com/developer/ip-products/processors/b/processors-ip-blog/posts/condition-codes-1-condition-flags-and-codes --- ext/opcache/jit/zend_jit_arm64.dasc | 127 ++++++++++++++++++++++++++-- ext/opcache/jit/zend_jit_arm64.h | 3 +- 2 files changed, 123 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 73f2116461300..c6e440b8a7ac0 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -79,11 +79,13 @@ |.define TMP3w, w13 |.define TMP4, x14 |.define TMP4w, w14 +|.define FPTMP, v16 |.define ZREG_TMP1, ZREG_X11 |.define ZREG_TMP2, ZREG_X12 |.define ZREG_TMP3, ZREG_X13 |.define ZREG_TMP4, ZREG_X14 +|.define ZREG_FPTMP, ZREG_V16 |.define HYBRID_SPAD, #16 // padding for stack alignment @@ -533,6 +535,42 @@ static void* dasm_labels[zend_lb_MAX]; | brk #0 // TODO |.endmacro +// Define DOUBLE_CMP and DOUBLE_CMP to replace SSE_AVX_OP and SSE_OP in x86 implementation. +// Conduct floating point operation 'ins'. Operand1 is from 'reg', and operands2 is from 'addr'. +// Use DOUBLE_CMP for comparisons and use DOUBLE_OP for arithmetic operations. +|.macro DOUBLE_CMP, ins, reg, addr, tmp_reg, fp_tmp_reg +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| brk #0 // TODO +| LOAD_ADDR Rx(tmp_reg), Z_ZV(addr) +| ldr Rd(fp_tmp_reg-ZREG_V0), [Rx(tmp_reg)] +| ins Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| SAFE_MEM_ACC_WITH_UOFFSET ldr, Rd(fp_tmp_reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) +| ins Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) +|| } else if (Z_MODE(addr) == IS_REG) { +| brk #0 // TODO +| ins Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) +|| } else { +|| ZEND_UNREACHABLE(); +|| } +|.endmacro + +|.macro DOUBLE_OP, ins, reg, addr, tmp_reg, fp_tmp_reg +| brk #0 // TODO +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| LOAD_ADDR Rx(tmp_reg), Z_ZV(addr) +| ldr Rd(fp_tmp_reg-ZREG_V0), [Rx(tmp_reg)] +| ins Rd(reg-ZREG_V0), Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| SAFE_MEM_ACC_WITH_UOFFSET ldr, Rd(fp_tmp_reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) +| ins Rd(reg-ZREG_V0), Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) +|| } else if (Z_MODE(addr) == IS_REG) { +| ins Rd(reg-ZREG_V0), Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) +|| } else { +|| ZEND_UNREACHABLE(); +|| } +|.endmacro + // Define DOUBLE_GET_LONG to replace SSE_GET_LONG in x86 implementation. // Convert the LONG value 'lval' into DOUBLE type, and move it into 'reg' |.macro DOUBLE_GET_LONG, reg, lval, tmp_reg @@ -4281,7 +4319,49 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, bool swap, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { - | brk #0 // TODO + if (smart_branch_opcode) { + | brk #0 // TODO + } else { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + | brk #0 // TODO + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | brk #0 // TODO + break; + case ZEND_IS_SMALLER: + | bvs >1 + | mov REG0, #IS_TRUE + || if (swap) { + | bhi >2 // TODO: why the NaN check is missing in x86? + || } else { + | blo >2 + || } + |1: + | mov REG0, #IS_FALSE + |2: + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | bvs >1 + | mov REG0, #IS_TRUE + || if (swap) { + | bhs >2 // TODO: why the NaN check is missing in x86? + || } else { + | bls >2 + || } + |1: + | mov REG0, #IS_FALSE + |2: + break; + default: + ZEND_UNREACHABLE(); + } + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + } return 1; } @@ -4290,7 +4370,8 @@ static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zen { zend_reg tmp_reg = ZREG_FPR0; - | brk #0 // TODO + | DOUBLE_GET_ZVAL_LVAL tmp_reg, op1_addr, ZREG_REG0, ZREG_TMP1 + | DOUBLE_CMP fcmp, tmp_reg, op2_addr, ZREG_TMP1, ZREG_FPTMP return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2, exit_addr); } @@ -4299,7 +4380,8 @@ static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zen { zend_reg tmp_reg = ZREG_FPR0; - | brk #0 // TODO + | DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, ZREG_REG0, ZREG_TMP1 + | DOUBLE_CMP fcmp, tmp_reg, op1_addr, ZREG_TMP1, ZREG_FPTMP return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2, exit_addr); } @@ -4358,7 +4440,13 @@ static int zend_jit_cmp(dasm_State **Dst, | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, TMP1w, TMP2 |.cold_code |3: - | brk #0 // TODO + if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, TMP1w, TMP2 + } + if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { + return 0; + } + | b >6 |.code } else { | brk #0 // TODO @@ -4370,7 +4458,34 @@ static int zend_jit_cmp(dasm_State **Dst, if (op1_info & MAY_BE_DOUBLE) { |.cold_code |4: - | brk #0 // TODO + if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, TMP1w, TMP2 + } + if (op2_info & MAY_BE_DOUBLE) { + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + if (!same_ops) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5, TMP1w, TMP2 + } else { + | brk #0 // TODO + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, TMP1w, TMP2 + } + } + | brk #0 // TODO + if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { + return 0; + } + | b >6 + } + if (!same_ops) { + |5: + if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, TMP1w, TMP2 + } + if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { + return 0; + } + | b >6 + } |.code } } else if ((op1_info & MAY_BE_DOUBLE) && @@ -6469,7 +6584,7 @@ static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, ze | GC_ADDREF REG1, TMP1w |2: | SET_ZVAL_PTR res_addr, REG1, TMP1 - | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2, TMP1 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 return 1; } diff --git a/ext/opcache/jit/zend_jit_arm64.h b/ext/opcache/jit/zend_jit_arm64.h index 40cb4ba5a8d97..c0a2cb901ecf4 100644 --- a/ext/opcache/jit/zend_jit_arm64.h +++ b/ext/opcache/jit/zend_jit_arm64.h @@ -129,6 +129,7 @@ typedef struct _zend_jit_registers_buf { #define ZREG_TMP2 ZREG_X12 #define ZREG_TMP3 ZREG_X13 #define ZREG_TMP4 ZREG_X14 +#define ZREG_FPTMP ZREG_V16 #define ZREG_COPY ZREG_REG0 #define ZREG_FIRST_FPR ZREG_V0 @@ -138,7 +139,7 @@ typedef uint64_t zend_regset; # define ZEND_REGSET_FIXED \ (ZEND_REGSET(ZREG_RSP) | ZEND_REGSET(ZREG_RLR) | ZEND_REGSET(ZREG_RFP) | \ ZEND_REGSET(ZREG_RPR) | ZEND_REGSET(ZREG_FP) | ZEND_REGSET(ZREG_IP) | \ - ZEND_REGSET_INTERVAL(ZREG_TMP1, ZREG_TMP4)) + ZEND_REGSET_INTERVAL(ZREG_TMP1, ZREG_TMP4) | ZEND_REGSET(ZREG_FPTMP)) # define ZEND_REGSET_GP \ ZEND_REGSET_DIFFERENCE(ZEND_REGSET_INTERVAL(ZREG_X0, ZREG_X30), ZEND_REGSET_FIXED) # define ZEND_REGSET_FP \ From b3519ad59d3c47749284fdeb8595e649e4f18696 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Thu, 15 Apr 2021 10:04:27 +0000 Subject: [PATCH 063/229] Support failed JIT test case: cmp_002.phpt 'smart_branch_opcode' JMPZ is used in this test case. Similar to the previous patch, I still didn't get why NaN check is missing for the cases ">" and ">=". In our implementation, we add such checks. --- ext/opcache/jit/zend_jit_arm64.dasc | 61 ++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index c6e440b8a7ac0..cbe4d0c7b2bb8 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -4320,7 +4320,66 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, bool swap, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { if (smart_branch_opcode) { - | brk #0 // TODO + if (smart_branch_opcode == ZEND_JMPZ) { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + | brk #0 // TODO + break; + case ZEND_IS_NOT_EQUAL: + | brk #0 // TODO + break; + case ZEND_IS_NOT_IDENTICAL: + | brk #0 // TODO + break; + case ZEND_IS_SMALLER: + if (swap) { + if (exit_addr) { + | brk #0 // TODO + } else { + | bvs => target_label // TODO: why the NaN check is missing in x86? + | bls => target_label + } + } else { + if (exit_addr) { + | brk #0 // TODO + } else { + | bhs => target_label + } + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + if (exit_addr) { + | brk #0 // TODO + } else { + | bvs => target_label // TODO: why the NaN check is missing in x86? + | blo => target_label + } + } else { + if (exit_addr) { + | brk #0 // TODO + } else { + | bhi => target_label + } + } + break; + default: + ZEND_UNREACHABLE(); + } + } else if (smart_branch_opcode == ZEND_JMPNZ) { + | brk #0 // TODO + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + | brk #0 // TODO + } else if (smart_branch_opcode == ZEND_JMPZ_EX) { + | brk #0 // TODO + } else if (smart_branch_opcode == ZEND_JMPNZ_EX) { + | brk #0 // TODO + } else { + ZEND_UNREACHABLE(); + } } else { switch (opline->opcode) { case ZEND_IS_EQUAL: From f8a275d6856a686bd37060f746f066b2be8d49ac Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Thu, 15 Apr 2021 14:35:38 +0000 Subject: [PATCH 064/229] Support failed JIT test case: cmp_004.phpt The following opcodes would be generated for $foo: 0000 #2.CV0($test) [bool] RANGE[0..1] = RECV 1 0001 #3.CV1($x) [long] RANGE[MIN..MAX] = RECV 2 0002 JMPZ #2.CV0($test) [bool] RANGE[0..1] BB4 0003 #4.T2 [bool] ... = IS_SMALLER_OR_EQUAL int(1) #3.CV1($x) ... 0004 JMP BB5 ... The updates in function zend_jit_verify_arg_type() are made to support RECV opcode. The updates in function zend_jit_bool_jmpznz() are made to support JMPZ opcode. New path is covered in functions zend_jit_cmp() and zend_jit_cmp_long_long() for IS_SMALLER_OR_EQUAL opcode. --- ext/opcache/jit/zend_jit_arm64.dasc | 172 ++++++++++++++++++++++++++-- 1 file changed, 164 insertions(+), 8 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index cbe4d0c7b2bb8..e73a7e1364568 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -671,8 +671,28 @@ static void* dasm_labels[zend_lb_MAX]; | brk #0 // TODO |.endmacro -|.macro LONG_OP_WITH_CONST, long_ins, op1_addr, lval -| brk #0 // TODO +// Define LONG_CMP_WITH_CONST to replace LONG_OP_WITH_CONST in the x86 implementation. +// Note that the 'long_ins' in all use sites of LONG_OP_WITH_CONST are always 'cmp'. +// Note that this macro is different from LONG_CMP. +|.macro LONG_CMP_WITH_CONST, cmp_ins, op1_addr, lval, tmp_reg1, tmp_reg2 +|| if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { +| SAFE_MEM_ACC_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(op1_addr)), Z_OFFSET(op1_addr), tmp_reg2 +|| if (lval >=0 && lval <= MAX_IMM12) { +| cmp_ins tmp_reg1, #lval +|| } else { +| LOAD_64BIT_VAL tmp_reg2, lval +| cmp_ins tmp_reg1, tmp_reg2 +|| } +|| } else if (Z_MODE(op1_addr) == IS_REG) { +|| if (lval >=0 && lval <= MAX_IMM12) { +| cmp_ins Rx(Z_REG(op1_addr)), #lval +|| } else { +| LOAD_64BIT_VAL tmp_reg1, lval +| cmp_ins Rx(Z_REG(op1_addr)), tmp_reg1 +|| } +|| } else { +|| ZEND_UNREACHABLE(); +|| } |.endmacro |.macro GET_ZVAL_LVAL, reg, addr, tmp_reg @@ -895,9 +915,11 @@ static void* dasm_labels[zend_lb_MAX]; | IF_NOT_TYPE tmp_reg, val, label |.endmacro -|.macro CMP_ZVAL_TYPE, addr, val +|.macro CMP_ZVAL_TYPE, addr, val, tmp_reg1, tmp_reg2 || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -| cmp byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val +|| ZEND_ASSERT(val <= MAX_IMM12); +| SAFE_MEM_ACC_WITH_UOFFSET ldrb, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), tmp_reg2 +| cmp tmp_reg1, #val |.endmacro |.macro IF_ZVAL_TYPE, addr, val, label, tmp_reg1, tmp_reg2 @@ -4218,7 +4240,7 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, } swap = 1; } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { - | brk #0 // TODO + | LONG_CMP_WITH_CONST cmp, op2_addr, Z_LVAL_P(Z_ZV(op1_addr)), TMP1, TMP2 swap = 1; } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { | brk #0 // TODO @@ -4311,7 +4333,36 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, ZEND_UNREACHABLE(); } } else { - | brk #0 // TODO + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + | brk #0 // TODO + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | brk #0 // TODO + break; + case ZEND_IS_SMALLER: + if (swap) { + | brk #0 // TODO + } else { + | brk #0 // TODO + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | cset REG0w, ge + } else { + | brk #0 // TODO + } + break; + default: + ZEND_UNREACHABLE(); + } + | add REG0w, REG0w, #2 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 } return 1; @@ -4608,7 +4659,107 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ bool set_delayed = 0; bool jmp_done = 0; - | brk #0 // TODO + if (branch_opcode == ZEND_BOOL) { + set_bool = 1; + } else if (branch_opcode == ZEND_BOOL_NOT) { + set_bool = 1; + set_bool_not = 1; + } else if (branch_opcode == ZEND_JMPZ) { + false_label = target_label; + } else if (branch_opcode == ZEND_JMPNZ) { + true_label = target_label; + } else if (branch_opcode == ZEND_JMPZNZ) { + true_label = target_label2; + false_label = target_label; + } else if (branch_opcode == ZEND_JMPZ_EX) { + set_bool = 1; + false_label = target_label; + } else if (branch_opcode == ZEND_JMPNZ_EX) { + set_bool = 1; + true_label = target_label; + } else { + ZEND_UNREACHABLE(); + } + + if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { + if (zend_is_true(Z_ZV(op1_addr))) { + /* Always TRUE */ + | brk #0 // TODO + } else { + /* Always FALSE */ + | brk #0 // TODO + } + return 1; + } + + if (op1_info & MAY_BE_REF) { + | brk #0 // TODO + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { + if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) { + /* Always TRUE */ + | brk #0 // TODO + } else { + if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) { + /* Always FALSE */ + if (set_bool) { + | brk #0 // TODO + } + } else { + | CMP_ZVAL_TYPE op1_addr, IS_TRUE, TMP1w, TMP2 + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { + | brk #0 // TODO + } + if (!(op1_info & MAY_BE_TRUE)) { + /* It's FALSE */ + | brk #0 // TODO + } else { + if (exit_addr) { + | brk #0 // TODO + } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { + if (set_bool) { + | brk #0 // TODO + } else { + if (true_label != (uint32_t)-1) { + | brk #0 // TODO + } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { + | bne =>false_label + jmp_done = 1; + } else { + | brk #0 // TODO + } + } + } else if (set_bool) { + | brk #0 // TODO + } + } + } + + /* It's FALSE, but may be UNDEF */ + if (op1_info & MAY_BE_UNDEF) { + | brk #0 // TODO + } + + if (!jmp_done) { + | brk #0 // TODO + } + } + } + + if (op1_info & MAY_BE_LONG) { + | brk #0 // TODO + } + + if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { + | brk #0 // TODO + } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { + | brk #0 // TODO + } + + |9: + return 1; } @@ -7023,7 +7174,12 @@ static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zen || ZEND_ASSERT(type_code <= MAX_IMM12); | IF_NOT_ZVAL_TYPE res_addr, type_code, >1, TMP1w, TMP2 } else { - | brk #0 // TODO + | mov REG2w, #1 + | SAFE_MEM_ACC_WITH_UOFFSET ldrb, REG1w, Rx(Z_REG(res_addr)), Z_OFFSET(res_addr)+offsetof(zval, u1.v.type), TMP1 + | lsl REG2w, REG2w, REG1w + | LOAD_32BIT_VAL TMP1w, type_mask + | tst REG2w, TMP1w + | beq >1 } |.cold_code From a5f7af2b2aa212795523bff5f612736759ca053d Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Fri, 16 Apr 2021 03:27:31 +0000 Subject: [PATCH 065/229] Support failed JIT test case: cmp_003.phpt This test case is a big one. This patch mainly handles smart_branch_opcode cases in function zend_jit_cmp_double_common(). Note that I failed to construct test cases to verify whether the missing NaN check in x86 is buggy or not. One TODO is left to remind us when the relevant code is touched. --- ext/opcache/jit/zend_jit_arm64.dasc | 291 ++++++++++++++++++++++++++-- 1 file changed, 276 insertions(+), 15 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index e73a7e1364568..0453eb0b9edd8 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -778,14 +778,12 @@ static void* dasm_labels[zend_lb_MAX]; // Define GET_ZVAL_DVAL to replace SSE_GET_ZVAL_DVAL in x86 implementation. |.macro GET_ZVAL_DVAL, reg, addr, tmp_reg -| brk #0 // TODO: test || if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) { || if (Z_MODE(addr) == IS_CONST_ZVAL) { | brk #0 // TODO: test | LOAD_ADDR Rx(tmp_reg), Z_ZV(addr) | ldr Rd(reg-ZREG_V0), [Rx(tmp_reg)] || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { -| brk #0 // TODO: test | SAFE_MEM_ACC_WITH_UOFFSET ldr, Rd(reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) || } else if (Z_MODE(addr) == IS_REG) { | brk #0 // TODO: test @@ -4377,10 +4375,20 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: - | brk #0 // TODO + if (exit_addr) { + | brk #0 // TODO + } else { + | bne => target_label + } break; case ZEND_IS_NOT_EQUAL: - | brk #0 // TODO + | bvs >1 + if (exit_addr) { + | brk #0 // TODO + } else { + | beq => target_label + } + |1: break; case ZEND_IS_NOT_IDENTICAL: | brk #0 // TODO @@ -4421,13 +4429,171 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z ZEND_UNREACHABLE(); } } else if (smart_branch_opcode == ZEND_JMPNZ) { - | brk #0 // TODO + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + | bvs >1 + if (exit_addr) { + | brk #0 // TODO + } else { + | beq => target_label + } + |1: + break; + case ZEND_IS_NOT_EQUAL: + if (exit_addr) { + | brk #0 // TODO + } else { + | bne => target_label + } + break; + case ZEND_IS_NOT_IDENTICAL: + | brk #0 // TODO + break; + case ZEND_IS_SMALLER: + if (swap) { + if (exit_addr) { + | brk #0 // TODO + } else { + | bvs >1 // Always False if involving NaN + | bhi => target_label + |1: + } + } else { + | bvs >1 + if (exit_addr) { + | brk #0 // TODO + } else { + | blo => target_label + } + |1: + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + if (exit_addr) { + | brk #0 // TODO + } else { + | bvs >1 // Always False if involving NaN + | bhs => target_label + |1: + } + } else { + | bvs >1 + if (exit_addr) { + | brk #0 // TODO + } else { + | bls => target_label + } + |1: + } + break; + default: + ZEND_UNREACHABLE(); + } } else if (smart_branch_opcode == ZEND_JMPZNZ) { | brk #0 // TODO } else if (smart_branch_opcode == ZEND_JMPZ_EX) { - | brk #0 // TODO + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + | bne => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | bvs >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + | beq => target_label + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + break; + case ZEND_IS_SMALLER: + if (swap) { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + | bvs => target_label // TODO: why the NaN check is missing in x86? + | bls => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + | bhs => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + | bvs => target_label // TODO: why the NaN check is missing in x86? + | blo => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + | bhi => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } + break; + default: + ZEND_UNREACHABLE(); + } } else if (smart_branch_opcode == ZEND_JMPNZ_EX) { - | brk #0 // TODO + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + | bvs >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + | beq => target_label + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + | bvs => target_label + | bne => target_label + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + break; + case ZEND_IS_SMALLER: + if (swap) { + | cset REG0w, hi + | add REG0w, REG0w, #2 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + | bvs >1 // Always False if involving NaN + | bhi => target_label + |1: + } else { + | bvs >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + | blo => target_label + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | cset REG0w, hs + | add REG0w, REG0w, #2 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + | bvs >1 // Always False if involving NaN + | bhs => target_label + |1: + } else { + | bvs >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + | bls => target_label + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + } + break; + default: + ZEND_UNREACHABLE(); + } } else { ZEND_UNREACHABLE(); } @@ -4437,11 +4603,22 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: - | brk #0 // TODO + | bvs >1 + | mov REG0, #IS_TRUE + | beq >2 + |1: + | mov REG0, #IS_FALSE + |2: break; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: - | brk #0 // TODO + | bvs >1 + | mov REG0, #IS_FALSE + | beq >2 + |1: + | mov REG0, #IS_TRUE + |2: + break; break; case ZEND_IS_SMALLER: | bvs >1 @@ -4500,7 +4677,19 @@ static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, z { bool swap = 0; - | brk #0 // TODO + if (Z_MODE(op1_addr) == IS_REG) { + | brk #0 // TODO + | DOUBLE_CMP fcmp, Z_REG(op1_addr), op2_addr, ZREG_TMP1, ZREG_FPTMP + } else if (Z_MODE(op2_addr) == IS_REG) { + | brk #0 // TODO: construct test cases to verify whether the missing NaN check in x86 is buggy or not. + | DOUBLE_CMP fcmp, Z_REG(op2_addr), op1_addr, ZREG_TMP1, ZREG_FPTMP + swap = 1; + } else { + zend_reg tmp_reg = ZREG_FPR0; + + | GET_ZVAL_DVAL tmp_reg, op1_addr, ZREG_TMP1 + | DOUBLE_CMP fcmp, tmp_reg, op2_addr, ZREG_TMP1, ZREG_FPTMP + } return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2, exit_addr); } @@ -4580,7 +4769,6 @@ static int zend_jit_cmp(dasm_State **Dst, | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, TMP1w, TMP2 } } - | brk #0 // TODO if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; } @@ -4693,7 +4881,8 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ } if (op1_info & MAY_BE_REF) { - | brk #0 // TODO + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | ZVAL_DEREF FCARG1x, op1_info, TMP1w op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); } @@ -4710,7 +4899,20 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ } else { | CMP_ZVAL_TYPE op1_addr, IS_TRUE, TMP1w, TMP2 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { - | brk #0 // TODO + if ((op1_info & MAY_BE_LONG) && + !(op1_info & MAY_BE_UNDEF) && + !set_bool) { + if (exit_addr) { + | brk #0 // TODO + } else if (false_label != (uint32_t)-1) { + | brk #0 // TODO + } else { + | brk #0 // TODO + } + jmp_done = 1; + } else { + | bgt >2 + } } if (!(op1_info & MAY_BE_TRUE)) { /* It's FALSE */ @@ -4732,23 +4934,82 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ } } } else if (set_bool) { - | brk #0 // TODO + | cset REG0w, eq + if (set_bool_not) { + | brk #0 // TODO + | neg REG0w, REG0w + | add REG0w, REG0w, #3 + } else { + | add REG0w, REG0w, #2 + } + if ((op1_info & MAY_BE_UNDEF) && (op1_info & MAY_BE_ANY)) { + set_delayed = 1; + } else { + | brk #0 // TODO + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + } } } } /* It's FALSE, but may be UNDEF */ if (op1_info & MAY_BE_UNDEF) { + if (op1_info & MAY_BE_ANY) { + if (set_delayed) { + | CMP_ZVAL_TYPE op1_addr, IS_UNDEF, TMP1w, TMP2 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + | beq >1 + } else { + | brk #0 // TODO + | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, TMP1w, TMP2 + } + |.cold_code + |1: + } | brk #0 // TODO + | LOAD_32BIT_VAL FCARG1w, opline->op1.var + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_undefined_op_helper, REG0 + + if (may_throw) { + if (!zend_jit_check_exception_undef_result(Dst, opline)) { + return 0; + } + } + + if (exit_addr) { + if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { + | brk #0 // TODO + } + } else if (false_label != (uint32_t)-1) { + | brk #0 // TODO + } + if (op1_info & MAY_BE_ANY) { + if (exit_addr) { + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | brk #0 // TODO + } + } else if (false_label == (uint32_t)-1) { + | brk #0 // TODO + } + |.code + } } if (!jmp_done) { - | brk #0 // TODO + if (exit_addr) { + | brk #0 // TODO + } else if (false_label != (uint32_t)-1) { + | brk #0 // TODO + } else if (op1_info & MAY_BE_LONG) { + | b >9 + } } } } if (op1_info & MAY_BE_LONG) { + |2: | brk #0 // TODO } From 1d4fdbd4e92c94a5665d3e8f51bad0c179f64fd0 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Fri, 16 Apr 2021 10:14:38 +0000 Subject: [PATCH 066/229] Support failed JIT test case: shift_left_001.phpt This patch supports SL opcode. The range of the second operand is checked against 0 and 64. If the second operand is negative, exception would be raised. If the second operand is >= 64, the result is 0. Besides, new path in macro ZVAL_COPY_VLAUE is covered for RETURN opcode. --- ext/opcache/jit/zend_jit_arm64.dasc | 130 ++++++++++++++++++++++++++-- 1 file changed, 124 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 0453eb0b9edd8..33cf097bbc108 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -531,8 +531,11 @@ static void* dasm_labels[zend_lb_MAX]; | brk #0 // TODO |.endmacro -|.macro UNDEF_OPLINE_RESULT -| brk #0 // TODO +|.macro UNDEF_OPLINE_RESULT, tmp_reg +| ldr REG0, EX->opline +| ldr REG0w, OP:REG0->result.var +| add REG0, FP, REG0 +| SET_Z_TYPE_INFO REG0, IS_UNDEF, tmp_reg |.endmacro // Define DOUBLE_CMP and DOUBLE_CMP to replace SSE_AVX_OP and SSE_OP in x86 implementation. @@ -837,7 +840,6 @@ static void* dasm_labels[zend_lb_MAX]; || if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { || if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) { || zend_uchar type = concrete_type(src_info); -| brk #0 // TODO: test | SET_ZVAL_TYPE_INFO dst_addr, type, Rw(tmp_reg1), Rx(tmp_reg2) || } || } @@ -852,7 +854,6 @@ static void* dasm_labels[zend_lb_MAX]; || if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) { || if (Z_MODE(src_addr) == IS_REG) { || if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) { -| brk #0 // TODO: test | SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg) || } || } else if (Z_MODE(dst_addr) == IS_REG) { @@ -1552,7 +1553,11 @@ static int zend_jit_undefined_function_stub(dasm_State **Dst) static int zend_jit_negative_shift_stub(dasm_State **Dst) { |->negative_shift: - | brk #0 // TODO + | UNDEF_OPLINE_RESULT TMP1w + | LOAD_ADDR CARG1, zend_ce_arithmetic_error + | LOAD_ADDR CARG2, "Bit shift by negative number" + | EXT_CALL zend_throw_error, REG0 + | b ->exception_handler return 1; } @@ -3264,7 +3269,120 @@ static int zend_jit_long_math_helper(dasm_State **Dst, zend_reg result_reg; zval tmp; - | brk #0 // TODO + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { + | brk #0 // TODO + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, TMP1w, TMP2 + } + if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { + | brk #0 // TODO + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, TMP1w, TMP2 + } + + if (opcode == ZEND_MOD && Z_MODE(op2_addr) == IS_CONST_ZVAL && + op1_range && + op1_range->min >= 0) { + zend_long l = Z_LVAL_P(Z_ZV(op2_addr)); + + if (zend_long_is_power_of_two(l)) { + /* Optimisation for mod of power of 2 */ + opcode = ZEND_BW_AND; + ZVAL_LONG(&tmp, l - 1); + op2_addr = ZEND_ADDR_CONST_ZVAL(&tmp); + } + } + + if (opcode == ZEND_MOD) { + | brk #0 // TODO + } else if (Z_MODE(res_addr) == IS_REG) { + if ((opline->opcode == ZEND_SL || opline->opcode == ZEND_SR) + && opline->op2_type != IS_CONST) { + result_reg = ZREG_REG0; + } else { + result_reg = Z_REG(res_addr); + } + } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { + result_reg = Z_REG(op1_addr); + } else if (Z_REG(res_addr) != ZREG_REG0) { + result_reg = ZREG_REG0; + } else { + /* ASSIGN_DIM_OP */ + result_reg = ZREG_FCARG1x; + } + + if (opcode == ZEND_SL) { + if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); + + | brk #0 // TODO + } else { + if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_REG1) { + | GET_ZVAL_LVAL ZREG_REG1, op2_addr, TMP1 + } + if (!op2_range || + op2_range->min < 0 || + op2_range->max >= SIZEOF_ZEND_LONG * 8) { + + | cmp REG1, #(SIZEOF_ZEND_LONG*8) + | bhs >1 + |.cold_code + |1: + | mov Rx(result_reg), xzr + | cmp REG1, xzr + | bgt >1 + | SET_EX_OPLINE opline, REG0 + | b ->negative_shift + |.code + } + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | lsl Rx(result_reg), Rx(result_reg), REG1 + |1: + } + } else if (opcode == ZEND_SR) { + | brk #0 // TODO + } else if (opcode == ZEND_MOD) { + | brk #0 // TODO + } else if (same_ops) { + | brk #0 // TODO + } else { + | brk #0 // TODO + } + + if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != result_reg) { + | SET_ZVAL_LVAL_FROM_REG res_addr, Rx(result_reg), TMP1 + } + if (Z_MODE(res_addr) == IS_MEM_ZVAL) { + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { + | brk #0 // TODO + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 + } + } + } + + if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) || + (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { + if ((op1_info & MAY_BE_LONG) && + (op2_info & MAY_BE_LONG)) { + |.cold_code + } + |6: + | brk #0 // TODO + if (may_throw) { + zend_jit_check_exception(Dst); + } + if (Z_MODE(res_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); + if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { + return 0; + } + } + if ((op1_info & MAY_BE_LONG) && + (op2_info & MAY_BE_LONG)) { + | b >5 + |.code + } + } + |5: return 1; } From 17d0be85ee55d4ccc1f012dbd36cf55988356dea Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Fri, 16 Apr 2021 11:04:40 +0000 Subject: [PATCH 067/229] Support failed JIT test case: shift_left_002.phpt This patch handles the case where the second operand is constant. --- ext/opcache/jit/zend_jit_arm64.dasc | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 33cf097bbc108..d8dfd2e7d4928 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -3313,7 +3313,20 @@ static int zend_jit_long_math_helper(dasm_State **Dst, if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); - | brk #0 // TODO + if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { + if (EXPECTED(op2_lval > 0)) { + | mov Rx(result_reg), xzr + } else { + | SET_EX_OPLINE opline, REG0 + | b ->negative_shift + } + } else if (Z_MODE(op1_addr) == IS_REG && op2_lval == 1) { + | add Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) + } else { + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | mov TMP1w, #op2_lval + | lsl Rx(result_reg), Rx(result_reg), TMP1 + } } else { if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_REG1) { | GET_ZVAL_LVAL ZREG_REG1, op2_addr, TMP1 From 2f7f11d06fea6110984088f5ee2b28042899cdc6 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Fri, 16 Apr 2021 11:32:28 +0000 Subject: [PATCH 068/229] Support failed JIT test case: shift_right_001.phpt Similar to left shift, it's trivial to support right shift. Note that bot shift_right_001.phpt and shift_right_002.phpt can pass with this patch. --- ext/opcache/jit/zend_jit_arm64.dasc | 37 ++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index d8dfd2e7d4928..e651d1cfc80f2 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -3351,7 +3351,42 @@ static int zend_jit_long_math_helper(dasm_State **Dst, |1: } } else if (opcode == ZEND_SR) { - | brk #0 // TODO + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); + + if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { + if (EXPECTED(op2_lval > 0)) { + | asr Rx(result_reg), Rx(result_reg), #((SIZEOF_ZEND_LONG * 8) - 1) + } else { + | SET_EX_OPLINE opline, REG0 + | b ->negative_shift + } + } else { + | mov TMP1w, #op2_lval + | asr Rx(result_reg), Rx(result_reg), TMP1 + } + } else { + if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_REG1) { + | GET_ZVAL_LVAL ZREG_REG1, op2_addr, TMP1 + } + if (!op2_range || + op2_range->min < 0 || + op2_range->max >= SIZEOF_ZEND_LONG * 8) { + | cmp REG1, #(SIZEOF_ZEND_LONG*8) + | bhs >1 + |.cold_code + |1: + | cmp REG1, xzr + | mov REG1w, #((SIZEOF_ZEND_LONG * 8) - 1) + | bgt >1 + | SET_EX_OPLINE opline, REG0 + | b ->negative_shift + |.code + } + |1: + | asr Rx(result_reg), Rx(result_reg), REG1 + } } else if (opcode == ZEND_MOD) { | brk #0 // TODO } else if (same_ops) { From bddad32201c805b754a30b790a6864d79b446e1d Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Fri, 16 Apr 2021 12:54:29 +0000 Subject: [PATCH 069/229] Support failed JIT test case: shift_right_003.phpt The following opcodes would be generated: ... BB1: 0003 JMP BB3 BB2: 0004 INIT_FCALL 1 96 string("chr") 0005 #10.T3 [long] = SR #3.CV0($int) [long] #7.CV2($i) ... 0006 #11.T4 [long] RANGE[0..127] = BW_AND #10.T3 [long] ... 0007 #12.T3 [long] RANGE[128..255] = BW_OR #11.T4 [long] ... 0008 SEND_VAL #12.T3 [long] RANGE[128..255] 1 0009 #13.V3 [ref, rc1, rcn, any] = DO_ICALL 0010 ASSIGN_OP (CONCAT) #6.CV1($out) [rc1, rcn, string] 0011 ADD #7.CV2($i)... int(7) #7.CV2($i) ... -> #15.CV2($i) ... BB3: 0012 #8.T4 [long] = SR #3.CV0($int) #7.CV2($i) [long, double] 0013 #9.T3 [bool] RANGE[0..1] = IS_SMALLER int(128) #8.T4 0014 JMPNZ #9.T3 [bool] RANGE[0..1] BB2 ... Main changes are: 1. SR opcode covers new path in function zend_jit_long_math_helper(). 2. BW_AND and BW_OR opcodes are supported. See macro LONG_OP. 3. Function zend_jit_concat_helper() is added to support ASSIGN_OP opcode. Speficically, CONCAT and FAST_CONCAT is supported for statements "$out .= ...". 4. New path is covered in function zend_jit_cmp_long_long() by IS_SMALLER opcode. 5. New path is covered in macros ZVAL_PTR_DTOR and ZVAL_DTOR_FUNC when leaving. --- ext/opcache/jit/zend_jit_arm64.dasc | 102 +++++++++++++++++++++++++--- 1 file changed, 91 insertions(+), 11 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index e651d1cfc80f2..c7f383d3cec85 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -728,13 +728,14 @@ static void* dasm_labels[zend_lb_MAX]; | brk #0 // LONG_OP imul, reg, addr || break; || case ZEND_BW_OR: -| brk #0 // LONG_OP or, reg, addr +| LONG_OP orr, reg, addr, tmp_reg1, tmp_reg2 || break; || case ZEND_BW_AND: -| brk #0 // LONG_OP and, reg, addr +| LONG_OP and, reg, addr, tmp_reg1, tmp_reg2 || break; || case ZEND_BW_XOR: -| brk #0 // LONG_OP xor, reg, addr +| brk #0 // TODO +| LONG_OP eor, reg, addr, tmp_reg1, tmp_reg2 || break; || default: || ZEND_UNREACHABLE(); @@ -1062,7 +1063,7 @@ static void* dasm_labels[zend_lb_MAX]; || if (has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_INDIRECT))) { || zend_uchar type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)); || if (type == IS_STRING && !ZEND_DEBUG) { -| brk #0 // TODO +| EXT_CALL _efree, tmp_reg || break; || } else if (type == IS_ARRAY) { || if ((var_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) { @@ -1109,7 +1110,6 @@ static void* dasm_labels[zend_lb_MAX]; || if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { | bne >3 || } else { -| brk #0 // TODO: test | bne >4 || } || } @@ -3274,7 +3274,6 @@ static int zend_jit_long_math_helper(dasm_State **Dst, | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, TMP1w, TMP2 } if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { - | brk #0 // TODO | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, TMP1w, TMP2 } @@ -3392,7 +3391,8 @@ static int zend_jit_long_math_helper(dasm_State **Dst, } else if (same_ops) { | brk #0 // TODO } else { - | brk #0 // TODO + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | LONG_MATH opcode, result_reg, op2_addr, TMP1, TMP2 } if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != result_reg) { @@ -3401,7 +3401,6 @@ static int zend_jit_long_math_helper(dasm_State **Dst, if (Z_MODE(res_addr) == IS_MEM_ZVAL) { if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { - | brk #0 // TODO | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 } } @@ -3465,7 +3464,50 @@ static int zend_jit_concat_helper(dasm_State **Dst, zend_jit_addr res_addr, int may_throw) { - | brk #0 // TODO + if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { + | brk #0 // TODO + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, TMP1w, TMP2 + } + if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >6, TMP1w, TMP2 + } + if (Z_MODE(op1_addr) == IS_MEM_ZVAL && Z_REG(op1_addr) == Z_REG(res_addr) && Z_OFFSET(op1_addr) == Z_OFFSET(res_addr)) { + if (Z_REG(res_addr) != ZREG_FCARG1x || Z_OFFSET(res_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, res_addr + } + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + | EXT_CALL zend_jit_fast_assign_concat_helper, REG0 + } else { + if (Z_REG(res_addr) != ZREG_FCARG1x || Z_OFFSET(res_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, res_addr + } + | LOAD_ZVAL_ADDR FCARG2x, op1_addr + | LOAD_ZVAL_ADDR CARG3, op2_addr + | EXT_CALL zend_jit_fast_concat_helper, REG0 + } + /* concatination with empty string may increase refcount */ + op1_info |= MAY_BE_RCN; + op2_info |= MAY_BE_RCN; + | FREE_OP op1_type, op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + | FREE_OP op2_type, op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + |5: + } + if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) || + (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) { + if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { + |.cold_code + |6: + } + | brk #0 // TODO + if (may_throw) { + zend_jit_check_exception(Dst); + } + if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { + | b <5 + |.code + } + } return 1; } @@ -4273,7 +4315,41 @@ static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED); ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); - | brk #0 // TODO + op1_addr = OP1_ADDR(); + op2_addr = OP2_ADDR(); + + if (op1_info & MAY_BE_REF) { + binary_op_type binary_op = get_binary_op(opline->extended_value); + | brk #0 // TODO + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + + int result; + switch (opline->extended_value) { + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: + case ZEND_DIV: + result = zend_jit_math_helper(Dst, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, op1_def_info, op1_info, may_overflow, may_throw); + break; + case ZEND_BW_OR: + case ZEND_BW_AND: + case ZEND_BW_XOR: + case ZEND_SL: + case ZEND_SR: + case ZEND_MOD: + result = zend_jit_long_math_helper(Dst, opline, opline->extended_value, + opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, + opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, + opline->op1.var, op1_addr, op1_def_info, op1_info, may_throw); + break; + case ZEND_CONCAT: + result = zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, may_throw); + break; + default: + ZEND_UNREACHABLE(); + } + |9: return 1; } @@ -4476,7 +4552,11 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, break; case ZEND_IS_SMALLER: if (swap) { - | brk #0 // TODO + if (exit_addr) { + | brk #0 // TODO + } else { + | bgt => target_label + } } else { if (exit_addr) { | brk #0 // TODO From bb677547225babca6c6e6afbce5b9371a17e0b6d Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Fri, 16 Apr 2021 13:49:00 +0000 Subject: [PATCH 070/229] Support failed JIT test case: mod_001.phpt Instructions 'cqo' + 'idiv' are used in x86 to conduct MOD operation. Specific registers RDX:RAX are used. We use instructions 'sdiv' + 'msub' to accomplish the equivalent functionality[1], and there is no such contrain on registers. Similary to left/right shift operations in the previous patches, boundary values, i.e. the second operand is 0 or -1, are handled. [1] https://stackoverflow.com/questions/35351470/obtaining-remainder-using-single-aarch64-instruction --- ext/opcache/jit/zend_jit_arm64.dasc | 73 +++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index c7f383d3cec85..398289448ab34 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -1564,7 +1564,11 @@ static int zend_jit_negative_shift_stub(dasm_State **Dst) static int zend_jit_mod_by_zero_stub(dasm_State **Dst) { |->mod_by_zero: - | brk #0 // TODO + | UNDEF_OPLINE_RESULT TMP1w + | LOAD_ADDR CARG1, zend_ce_division_by_zero_error + | LOAD_ADDR CARG2, "Modulo by zero" + | EXT_CALL zend_throw_error, REG0 + | b ->exception_handler return 1; } @@ -3291,7 +3295,7 @@ static int zend_jit_long_math_helper(dasm_State **Dst, } if (opcode == ZEND_MOD) { - | brk #0 // TODO + result_reg = ZREG_REG0; } else if (Z_MODE(res_addr) == IS_REG) { if ((opline->opcode == ZEND_SL || opline->opcode == ZEND_SR) && opline->op2_type != IS_CONST) { @@ -3387,7 +3391,70 @@ static int zend_jit_long_math_helper(dasm_State **Dst, | asr Rx(result_reg), Rx(result_reg), REG1 } } else if (opcode == ZEND_MOD) { - | brk #0 // TODO + // REG0 -> result_reg + if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); + + if (op2_lval == 0) { + | SET_EX_OPLINE opline, REG0 + | b ->mod_by_zero + } else if (op2_lval == -1) { + | mov REG0, xzr + } else { + | GET_ZVAL_LVAL ZREG_REG1, op1_addr, TMP1 + | GET_ZVAL_LVAL ZREG_REG2, op2_addr, TMP1 + | sdiv REG0, REG1, REG2 + | msub REG0, REG0, REG2, REG1 + } + } else { + if (!op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) { + if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { + | SAFE_MEM_ACC_WITH_UOFFSET ldr, TMP1, Rx(Z_REG(op2_addr)), Z_OFFSET(op2_addr), TMP2 + | cmp TMP1, #0 + } else if (Z_MODE(op2_addr) == IS_REG) { + | tst Rx(Z_REG(op2_addr)), Rx(Z_REG(op2_addr)) + } + | beq >1 + |.cold_code + |1: + | SET_EX_OPLINE opline, REG0 + | b ->mod_by_zero + |.code + } + + /* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */ + if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) { + if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { + | SAFE_MEM_ACC_WITH_UOFFSET ldr, TMP1, Rx(Z_REG(op2_addr)), Z_OFFSET(op2_addr), TMP2 + | cmn TMP1, #1 + } else if (Z_MODE(op2_addr) == IS_REG) { + | cmn Rx(Z_REG(op2_addr)), #1 + } + | beq >1 + |.cold_code + |1: + | SET_ZVAL_LVAL_FROM_REG res_addr, xzr, TMP1 + if (Z_MODE(res_addr) == IS_MEM_ZVAL) { + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 + } + } + } + | b >5 + |.code + } + + | GET_ZVAL_LVAL ZREG_REG1, op1_addr, TMP1 + if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { + | ldr TMP1, [Rx(Z_REG(op2_addr)), #Z_OFFSET(op2_addr)] + | sdiv REG0, REG1, TMP1 + | msub REG0, REG0, TMP1, REG1 + } else if (Z_MODE(op2_addr) == IS_REG) { + | sdiv REG0, REG1, Rx(Z_REG(op2_addr)) + | msub REG0, REG0, Rx(Z_REG(op2_addr)), REG1 + } + } } else if (same_ops) { | brk #0 // TODO } else { From f11b4407fbb5ebf8749942bae66c7b6912d43396 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Sat, 17 Apr 2021 04:20:23 +0000 Subject: [PATCH 071/229] Support failed JIT test case: send_ref_001.phpt Part of generated opcodes for $foo are: ... BB1: 0002 INIT_FCALL 1 96 string("foo") 0003 #5.V1 [rcn, object (instanceof A)] = FETCH_THIS 0004 SEND_REF #5.V1 [rcn, object (instanceof A)] 1 0005 DO_UCALL Updates in functions zend_jit_fetch_this() and zend_jit_load_this() are made to support FETCH_THIS opcode. One new path is covered in function zend_jit_send_ref() by SEND_REF opcode. --- ext/opcache/jit/zend_jit_arm64.dasc | 31 ++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 398289448ab34..e1ba826fa7220 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -6784,7 +6784,6 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend | // ZVAL_NEW_REF(arg, varptr); if (opline->op1_type == IS_VAR) { if (Z_REG(op1_addr) != ZREG_REG0 || Z_OFFSET(op1_addr) != 0) { - | brk #0 // TODO | LOAD_ZVAL_ADDR REG0, op1_addr } | str REG0, T1 // save @@ -8707,13 +8706,39 @@ static int zend_jit_load_this(dasm_State **Dst, uint32_t var) { zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); - | brk #0 // TODO + | ldr FCARG1x, EX->This.value.ptr + | SET_ZVAL_PTR var_addr, FCARG1x, TMP1 + | SET_ZVAL_TYPE_INFO var_addr, IS_OBJECT_EX, TMP1w, TMP2 + | GC_ADDREF FCARG1x, TMP1w return 1; } static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool check_only) { - | brk #0 // TODO + if (!op_array->scope || (op_array->fn_flags & ZEND_ACC_STATIC)) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + if (!JIT_G(current_frame) || + !TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) { + + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + | brk #0 // TODO + + if (JIT_G(current_frame)) { + TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame)); + } + } + } else { + | brk #0 // TODO + } + } + + if (!check_only) { + if (!zend_jit_load_this(Dst, opline->result.var)) { + return 0; + } + } return 1; } From 4c872ba24adbc9b867bfdaaa96fe20e2c8b2365a Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Sat, 17 Apr 2021 05:11:51 +0000 Subject: [PATCH 072/229] Support failed JIT test case: send_val_001.phpt Updates in function zend_jit_type_check() are made to support TYPE_CHECK opcode for statement "is_array($type)". New path is touched in function zend_jit_concat_helper() to support opcode CONCAT for statement "$type ."ops"". Besides, one new path is covered in function zend_jit_return() when leaving. --- ext/opcache/jit/zend_jit_arm64.dasc | 95 ++++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index e1ba826fa7220..073073792a980 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -3533,7 +3533,6 @@ static int zend_jit_concat_helper(dasm_State **Dst, { if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { - | brk #0 // TODO | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, TMP1w, TMP2 } if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { @@ -7009,7 +7008,87 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t // TODO: support for is_resource() ??? ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE); - | brk #0 // TODO + if (op1_info & MAY_BE_UNDEF) { + | brk #0 // TODO + } + + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + mask = opline->extended_value; + if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) { + | brk #0 // TODO + } else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) { + | brk #0 // TODO + } else { + bool invert = 0; + zend_uchar type; + + switch (mask) { + case MAY_BE_NULL: type = IS_NULL; break; + case MAY_BE_FALSE: type = IS_FALSE; break; + case MAY_BE_TRUE: type = IS_TRUE; break; + case MAY_BE_LONG: type = IS_LONG; break; + case MAY_BE_DOUBLE: type = IS_DOUBLE; break; + case MAY_BE_STRING: type = IS_STRING; break; + case MAY_BE_ARRAY: type = IS_ARRAY; break; + case MAY_BE_OBJECT: type = IS_OBJECT; break; + case MAY_BE_ANY - MAY_BE_NULL: type = IS_NULL; invert = 1; break; + case MAY_BE_ANY - MAY_BE_FALSE: type = IS_FALSE; invert = 1; break; + case MAY_BE_ANY - MAY_BE_TRUE: type = IS_TRUE; invert = 1; break; + case MAY_BE_ANY - MAY_BE_LONG: type = IS_LONG; invert = 1; break; + case MAY_BE_ANY - MAY_BE_DOUBLE: type = IS_DOUBLE; invert = 1; break; + case MAY_BE_ANY - MAY_BE_STRING: type = IS_STRING; invert = 1; break; + case MAY_BE_ANY - MAY_BE_ARRAY: type = IS_ARRAY; invert = 1; break; + case MAY_BE_ANY - MAY_BE_OBJECT: type = IS_OBJECT; invert = 1; break; + case MAY_BE_ANY - MAY_BE_RESOURCE: type = IS_OBJECT; invert = 1; break; + default: + type = 0; + } + + if (op1_info & MAY_BE_REF) { + | brk #0 // TODO + } + if (type == 0) { + | brk #0 // TODO + } else { + if (smart_branch_opcode && + (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | brk #0 // TODO + } else { + if (op1_info & MAY_BE_REF) { + | brk #0 // TODO + } else { + | LOAD_32BIT_VAL TMP1w, (opline->op1.var + 8) + | ldrb TMP2w, [FP, TMP1] + | cmp TMP2w, #type + } + } + if (exit_addr) { + | brk #0 // TODO + } else if (smart_branch_opcode) { + if (invert) { + | brk #0 // TODO + } else { + if (smart_branch_opcode == ZEND_JMPZ) { + | bne =>target_label + } else if (smart_branch_opcode == ZEND_JMPNZ) { + | brk #0 // TODO + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + | brk #0 // TODO + } else { + ZEND_UNREACHABLE(); + } + } + } else { + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + + | brk #0 // TODO + } + } + } + } + + |7: return 1; } @@ -7338,7 +7417,17 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o } | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { - | brk #0 // TODO + if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || + (op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) || + !op_array->function_name) { + | lsr REG0w, REG0w, #8 + | and REG0w, REG0w, #0xff + | TRY_ADDREF op1_info, REG0w, REG2, TMP1 + } else if (return_value_used != 1) { + | // if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); + | brk #0 // TODO + | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 + } } } else { | brk #0 // TODO From 7309d5a7053ad2841e831aa0a262ac014fc3eb8a Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Sat, 17 Apr 2021 05:55:23 +0000 Subject: [PATCH 073/229] Support failed JIT test case: send_var_ex_001.phpt Opcodes for function $evaluate are: BB0: 0000 ASSIGN_OBJ THIS string("evalParameters") 0001 OP_DATA array(...) 0002 INIT_NS_FCALL_BY_NAME 2 string("A\extract") 0003 CHECK_FUNC_ARG 1 0004 V1 = FETCH_OBJ_FUNC_ARG (ref) THIS string("evalParameters") 0005 SEND_FUNC_ARG V1 1 0006 T1 = FETCH_CONSTANT (unqualified-in-namespace) ... 0007 SEND_VAL_EX T1 2 0008 DO_FCALL_BY_NAME ... Major changes are made in function zend_jit_fetch_constant() to support FETCH_CONSTANT opcode. Besdies, cold code is touched in functions zend_jit_check_func_arg() and zend_jit_send_var() for opcodes CHECK_FUNC_ARG and SEND_FUNC_ARG respectively. --- ext/opcache/jit/zend_jit_arm64.dasc | 56 +++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 073073792a980..29f78b53a5d34 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -6853,7 +6853,6 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend | bne >1 |.cold_code |1: - | brk #0 // TODO if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { return 0; } @@ -6954,7 +6953,6 @@ static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) | bne >1 |.cold_code |1: - | brk #0 // TODO | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] | LOAD_32BIT_VAL TMP2w, ZEND_CALL_SEND_ARG_BY_REF @@ -8907,7 +8905,59 @@ static int zend_jit_fetch_constant(dasm_State **Dst, zend_jit_addr const_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); uint32_t res_info = RES_INFO(); - | brk #0 // TODO + | // c = CACHED_PTR(opline->extended_value); + | ldr FCARG1x, EX->run_time_cache + | LOAD_32BIT_VAL TMP1w, opline->extended_value + | ldr REG0, [FCARG1x, TMP1] + | // if (c != NULL) + | cbz REG0, >9 + | brk #0 // TODO + | // if (!IS_SPECIAL_CACHE_VAL(c)) + | tst REG0, #CACHE_SPECIAL + | bne >9 + |8: + + if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) { + zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; + uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); + int32_t exit_point; + const void *exit_addr = NULL; + + SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); + SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); + exit_point = zend_jit_trace_get_exit_point(opline+1, 0); + SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + res_info &= ~MAY_BE_GUARD; + ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; + + zend_uchar type = concrete_type(res_info); + + | brk #0 // TODO + } else { + | ZVAL_COPY_VALUE res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | lsr REG0w, REG0w, #8 + | and REG0w, REG0w, #0xff + | TRY_ADDREF MAY_BE_ANY, REG0w, REG1, TMP1 + } + + |.cold_code + |9: + | // SAVE_OPLINE(); + | SET_EX_OPLINE opline, REG0 + | // zend_quick_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num OPLINE_CC EXECUTE_DATA_CC); + | LOAD_ADDR FCARG1x, zv + | LOAD_32BIT_VAL FCARG2x, opline->op1.num + | EXT_CALL zend_jit_get_constant, REG0 + | mov REG0, RETVALx + | // ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); + | cbnz REG0, <8 + | brk #0 // TODO + | b ->exception_handler + |.code return 1; } From 0a975548b05d5875b48e43c5182f5912ac7c8cc7 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Sat, 17 Apr 2021 07:57:34 +0000 Subject: [PATCH 074/229] Support failed JIT test case: jmpz_001.phpt Opcode SEND_VAR_EX used in $test and opcode ZEND_SEND_VAR_NO_REF_EX used in $main cover two new branches in function zend_jit_send_var() respectively. The updates in function zend_jit_bool_jmpznz() are made to support opcode JMPNZ_EX used in $test. --- ext/opcache/jit/zend_jit_arm64.dasc | 63 ++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 29f78b53a5d34..6840d77fdb150 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -5184,7 +5184,17 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { if (zend_is_true(Z_ZV(op1_addr))) { /* Always TRUE */ - | brk #0 // TODO + if (set_bool) { + if (set_bool_not) { + | brk #0 // TODO + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } + } + if (true_label != (uint32_t)-1) { + | b =>true_label; + } } else { /* Always FALSE */ | brk #0 // TODO @@ -5219,7 +5229,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ } else if (false_label != (uint32_t)-1) { | brk #0 // TODO } else { - | brk #0 // TODO + | blt >9 } jmp_done = 1; } else { @@ -5237,7 +5247,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ | brk #0 // TODO } else { if (true_label != (uint32_t)-1) { - | brk #0 // TODO + | beq =>true_label } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { | bne =>false_label jmp_done = 1; @@ -6832,9 +6842,52 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend } if (opline->opcode == ZEND_SEND_VAR_EX) { - | brk #0 // TODO + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE + && JIT_G(current_frame) + && JIT_G(current_frame)->call + && JIT_G(current_frame)->call->func) { + if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { + if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) { + return 0; + } + return 1; + } + } else { + uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); + + | ldr REG0, EX:RX->func + | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] + | LOAD_32BIT_VAL TMP2w, mask + | tst TMP1w, TMP2w + | bne >1 + |.cold_code + |1: + | brk #0 // TODO + if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { + return 0; + } + | b >7 + |.code + } } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { - | brk #0 // TODO + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE + && JIT_G(current_frame) + && JIT_G(current_frame)->call + && JIT_G(current_frame)->call->func) { + | brk #0 // TODO + } else { + uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); + + | ldr REG0, EX:RX->func + | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] + | LOAD_32BIT_VAL TMP2w, mask + | tst TMP1w, TMP2w + | bne >1 + |.cold_code + |1: + | brk #0 // TODO + |.code + } } else if (opline->opcode == ZEND_SEND_FUNC_ARG) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) From f133644796e9c497a24804ac9c6d6abdc394de22 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Sun, 18 Apr 2021 12:37:40 +0000 Subject: [PATCH 075/229] Support failed JIT test case: jmpz_ex_001.phpt Opcodes for $Test::method are: BB0: 0000 #0.T0 [rcn, any] = FETCH_OBJ_R THIS string("prop") 0001 #1.T0 [bool] RANGE[0..1] = JMPZ_EX #0.T0 [rcn, any] BB3 BB1: 0002 #2.T1 [rcn, any] = FETCH_OBJ_R THIS string("prop") 0003 INIT_METHOD_CALL 0 #2.T1 [rcn, any] string("method2") 0004 #3.V1 [ref, rc1, rcn, any] = DO_FCALL ... New path is covered in functions zend_jit_fetch_obj() and zend_jit_zval_copy_deref() for FETCH_OBJ_R THIS opcode. New path is covered in function zend_jit_init_method_call() for opcode INIT_METHOD_CALL. Major chagnes lie in function zend_jit_bool_jmpznz() to support opcode JMPZ_EX. Note that macro ZVAL_DTOR_FUNC is updated to remove the hard-coded use of REG0. --- ext/opcache/jit/zend_jit_arm64.dasc | 81 ++++++++++++++++++++++++++--- 1 file changed, 74 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 6840d77fdb150..ca41c6d374666 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -1079,7 +1079,7 @@ static void* dasm_labels[zend_lb_MAX]; || if (opline) { | brk #0 // TODO || } -| EXT_CALL zend_objects_store_del, REG0 +| EXT_CALL zend_objects_store_del, tmp_reg || break; || } || } @@ -5267,7 +5267,6 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ if ((op1_info & MAY_BE_UNDEF) && (op1_info & MAY_BE_ANY)) { set_delayed = 1; } else { - | brk #0 // TODO | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 } } @@ -5332,13 +5331,83 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ if (op1_info & MAY_BE_LONG) { |2: - | brk #0 // TODO + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, TMP1w, TMP2 + } + | brk #0 } if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { | brk #0 // TODO } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { - | brk #0 // TODO + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + |.cold_code + |2: + } + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_is_true, REG0 + | mov REG0, RETVALx + + if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); + + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | IF_NOT_ZVAL_REFCOUNTED op1_addr, >3, TMP1w, TMP2 + } + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + | GC_DELREF FCARG1x, TMP1w + | bne >3 + | brk #0 // TODO: currently jump to label 3. + // In x86, r0 is used in macro ZVAL_DTOR_FUNC as temporary register, hence, r0 should be saved/restored + // before/after this macro. In AArch64, TMP1 is used. As a result, we needn't save/resotre REG0. + | ZVAL_DTOR_FUNC op1_info, opline, TMP1 + |3: + } + if (may_throw) { + | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, REG1, TMP1 + | bne ->exception_handler_undef + } + + if (set_bool) { + if (set_bool_not) { + | brk #0 // TODO + | neg REG0w, REG0w + | add REG0w, REG0w, #3 + } else { + | add REG0w, REG0w, #2 + } + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + if (exit_addr) { + | brk #0 // TODO + } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { + | CMP_ZVAL_TYPE res_addr, IS_FALSE, TMP1w, TMP2 + if (true_label != (uint32_t)-1) { + | brk #0 // TODO + | bne =>true_label + if (false_label != (uint32_t)-1) { + | b =>false_label + } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + | b >9 + } + } else { + | beq =>false_label + } + } + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + | b >9 + |.code + } + } else { + | brk #0 // TODO + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + |.code + } + } } |9: @@ -6012,7 +6081,6 @@ static int zend_jit_init_method_call(dasm_State **Dst, | mov CARG3, sp | SET_EX_OPLINE opline, REG0 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) { - | brk #0 // TODO | EXT_CALL zend_jit_find_method_tmp_helper, REG0 } else { | EXT_CALL zend_jit_find_method_helper, REG0 @@ -7496,9 +7564,9 @@ static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, ze | lsr TMP1w, REG2w, #8 | and TMP1w, TMP1w, #0xff // TMP1w -> 8-15 bits of REG2w | IF_NOT_REFCOUNTED TMP1w, >2 - | brk #0 // TODO: currently jump to label 2 directly. | and TMP2w, REG2w, #0xff // TMP2w -> low 8 bits of REG2w | IF_NOT_TYPE TMP2w, IS_REFERENCE, >1 + | brk #0 // TODO | add TMP3, REG1, #offsetof(zend_reference, val) | GET_Z_TYPE_INFO REG2w, TMP3 | GET_Z_PTR REG1, TMP3 @@ -8128,7 +8196,6 @@ static int zend_jit_fetch_obj(dasm_State **Dst, prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); if (opline->op1_type == IS_UNUSED || use_this) { - | brk #0 // TODO | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 } else { if (opline->op1_type == IS_VAR From c8c41328115b1986e03d260870aafe1549724869 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Sun, 18 Apr 2021 13:56:30 +0000 Subject: [PATCH 076/229] Support failed JIT test case: reg_alloc_003.phpt Opcodes for $test are: BB0: 0000 #1.CV0($char_code) [rc1, rcn, any] = RECV 1 BB1: 0001 #2.T1 [rc1, ...] = BW_AND #1.CV0($char_code) ... 0002 #3.T2 [bool] RANGE[0..1] = BOOL_NOT #2.T1 [rc1, ...] 0003 #4.T1 [bool] RANGE[0..1] = IS_EQUAL #1.CV0($char_code) ... 0004 JMPZ #4.T1 [bool] RANGE[0..1] BB3 ... New path is covered in function zend_jit_long_math_helper() for opcode BW_AND. New path is covered in function zend_jit_bool_jmpznz() for opcode BOOL_NOT. Major changes lie in functions zend_jit_cmp(), zend_jit_cmp_slow() and zend_jit_check_exception_undef_result() to support opocdes IS_EQUAL and JMPZ. --- ext/opcache/jit/zend_jit_arm64.dasc | 105 ++++++++++++++++++++++++++-- 1 file changed, 100 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index ca41c6d374666..fa2050ca21cee 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -2295,7 +2295,8 @@ static int zend_jit_check_exception(dasm_State **Dst) static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline) { if (opline->result_type & (IS_TMP_VAR|IS_VAR)) { - | brk #0 // TODO + | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, REG0, TMP1 + | bne ->exception_handler_undef return 1; } return zend_jit_check_exception(Dst); @@ -3274,7 +3275,6 @@ static int zend_jit_long_math_helper(dasm_State **Dst, zval tmp; if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { - | brk #0 // TODO | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, TMP1w, TMP2 } if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { @@ -5008,7 +5008,46 @@ static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, z static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { - | brk #0 // TODO + | LONG_CMP_WITH_CONST cmp, res_addr, Z_L(0), TMP1, TMP2 + if (smart_branch_opcode) { + if (smart_branch_opcode == ZEND_JMPZ_EX || + smart_branch_opcode == ZEND_JMPNZ_EX) { + | brk #0 // TODO + } + if (smart_branch_opcode == ZEND_JMPZ || + smart_branch_opcode == ZEND_JMPZ_EX) { + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_CASE: + if (exit_addr) { + | brk #0 // TODO + } else { + | bne => target_label + } + break; + case ZEND_IS_NOT_EQUAL: + | brk #0 // TODO + break; + case ZEND_IS_SMALLER: + | brk #0 // TODO + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | brk #0 // TODO + break; + default: + ZEND_UNREACHABLE(); + } + } else if (smart_branch_opcode == ZEND_JMPNZ || + smart_branch_opcode == ZEND_JMPNZ_EX) { + | brk #0 // TODO + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + | brk #0 // TODO + } else { + ZEND_UNREACHABLE(); + } + } else { + | brk #0 // TODO + } return 1; } @@ -5115,7 +5154,42 @@ static int zend_jit_cmp(dasm_State **Dst, |.cold_code |9: } - | brk #0 // TODO + | SET_EX_OPLINE opline, REG0 + if (Z_MODE(op1_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); + if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { + return 0; + } + op1_addr = real_addr; + } + if (Z_MODE(op2_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); + if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { + return 0; + } + op2_addr = real_addr; + } + | LOAD_ZVAL_ADDR FCARG2x, op1_addr + if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { + | brk #0 // TODO + } + if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { + | brk #0 // TODO + } else { + | LOAD_ZVAL_ADDR CARG3, op2_addr + } + | LOAD_ZVAL_ADDR FCARG1x, res_addr + | EXT_CALL compare_function, REG0 + if (opline->opcode != ZEND_CASE) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + } + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + if (may_throw) { + zend_jit_check_exception_undef_result(Dst, opline); + } + if (!zend_jit_cmp_slow(Dst, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { + return 0; + } if (has_slow) { | b >6 |.code @@ -5334,7 +5408,28 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, TMP1w, TMP2 } - | brk #0 + if (Z_MODE(op1_addr) == IS_REG) { + | brk #0 // TODO + | tst Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) + } else { + | LONG_CMP_WITH_CONST cmp, op1_addr, Z_L(0), TMP1, TMP2 + } + if (set_bool) { + | cset REG0w, ne + if (set_bool_not) { + | neg REG0w, REG0w + | add REG0w, REG0w, #3 + } else { + | brk #0 // TODO + | add REG0w, REG0w, #2 + } + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + } + if (exit_addr) { + | brk #0 // TODO + } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { + | brk #0 // TODO + } } if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { From 3bbc9fdfaf02abefff20cc6d0c994a0d83643301 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Sun, 18 Apr 2021 14:55:49 +0000 Subject: [PATCH 077/229] Support failed JIT test case: reg_alloc_002.phpt Opcodes FE_RESET_R and FE_FETCH_R are met for the first time. Updates in funtions zend_jit_fe_reset() and zend_jit_fe_fetch() are made to support them. Besides, one new path is covered in function zend_jit_inc_dec() for opcode POST_INC. --- ext/opcache/jit/zend_jit_arm64.dasc | 94 +++++++++++++++++++++++++++-- 1 file changed, 90 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index fa2050ca21cee..b26fa9092935e 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -42,6 +42,7 @@ |.define FCARG1x, x0 |.define FCARG1w, w0 |.define FCARG2x, x1 +|.define FCARG2w, w1 |.define SPAD, #0x10 // padding for CPU stack alignment |.define NR_SPAD, #0x30 // padding for CPU stack alignment |.define T4, [sp, #0x20] // Used to store old value of LR (CALL VM only) @@ -861,7 +862,6 @@ static void* dasm_labels[zend_lb_MAX]; | brk #0 // TODO: test | GET_ZVAL_LVAL Z_REG(dst_addr), src_addr, Rx(tmp_reg) || } else { -| brk #0 // TODO: test | GET_ZVAL_LVAL reg2, src_addr, Rx(tmp_reg) | SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(reg2), Rx(tmp_reg) || } @@ -2742,7 +2742,6 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, TMP1w, TMP2 } if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { - | brk #0 // TODO: test | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, MAY_BE_LONG)) { @@ -9097,7 +9096,22 @@ static int zend_jit_fe_reset(dasm_State **Dst, const zend_op *opline, uint32_t o { zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); - | brk #0 // TODO + if (opline->op1_type == IS_CONST) { + zval *zv = RT_CONSTANT(opline, opline->op1); + + | ZVAL_COPY_CONST res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 + if (Z_REFCOUNTED_P(zv)) { + | brk #0 // TODO + } + } else { + zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); + + | brk #0 // TODO + } + | // Z_FE_POS_P(res) = 0; + | LOAD_32BIT_VAL TMP1w, (opline->result.var + offsetof(zval, u2.fe_pos)) + | str wzr, [FP, TMP1] + return 1; } @@ -9105,7 +9119,79 @@ static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t o { zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); - | brk #0 // TODO + | // array = EX_VAR(opline->op1.var); + | // fe_ht = Z_ARRVAL_P(array); + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + | // pos = Z_FE_POS_P(array); + | LOAD_32BIT_VAL TMP1w, (opline->op1.var + offsetof(zval, u2.fe_pos)) + | ldr REG0w, [FP, TMP1] + | // p = fe_ht->arData + pos; + || ZEND_ASSERT(sizeof(Bucket) == 32); + | mov FCARG2w, REG0w + | lsl FCARG2x, FCARG2x, #5 + | ldr TMP1, [FCARG1x, #offsetof(zend_array, arData)] + | add FCARG2x, FCARG2x, TMP1 + |1: + | // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) { + | ldr TMP1w, [FCARG1x, #offsetof(zend_array, nNumUsed)] + | cmp TMP1w, REG0w + | // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); + | // ZEND_VM_CONTINUE(); + if (exit_addr) { + | brk #0 // TODO + } else { + | bls =>target_label + } + | // pos++; + | add REG0w, REG0w, #1 + | // value_type = Z_TYPE_INFO_P(value); + | // if (EXPECTED(value_type != IS_UNDEF)) { + if (!exit_addr || exit_opcode == ZEND_JMP) { + | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, >3, TMP1w + } else { + | brk #0 // TODO + } + | // p++; + | add FCARG2x, FCARG2x, #sizeof(Bucket) + | b <1 + |3: + + if (!exit_addr || exit_opcode == ZEND_JMP) { + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2x, 0); + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); + uint32_t val_info; + + | // Z_FE_POS_P(array) = pos + 1; + | LOAD_32BIT_VAL TMP1w, (opline->op1.var + offsetof(zval, u2.fe_pos)) + | str REG0w, [FP, TMP1] + + if (RETURN_VALUE_USED(opline)) { + zend_jit_addr res_addr = RES_ADDR(); + + | brk #0 // TODO + } + + val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT); + if (val_info & MAY_BE_ARRAY) { + val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; + } + if (op1_info & MAY_BE_ARRAY_OF_REF) { + val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY | + MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; + } else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + val_info |= MAY_BE_RC1 | MAY_BE_RCN; + } + + if (opline->op2_type == IS_CV) { + | // zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES()); + if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 1)) { + return 0; + } + } else { + | brk #0 // TODO + } + } + return 1; } From 72981bf004103d72b5d2792f7f8b30d2df6269aa Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 20 Apr 2021 10:44:51 +0300 Subject: [PATCH 078/229] Replace "brk #0" with "NIY" (Not Implemented Yet) macro --- ext/opcache/jit/zend_jit_arm64.dasc | 954 ++++++++++++++-------------- 1 file changed, 480 insertions(+), 474 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index b26fa9092935e..9e9314174392f 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -158,6 +158,12 @@ static void* dasm_labels[zend_lb_MAX]; #define BP_JIT_IS 6 +/* Not Implemented Yet */ +|.macro NIY +|| //ZEND_ASSERT(0); +| brk #0 +|.endmacro + /* In x86/64, HYBRID_SPAD bytes are reserved on the stack only if flag ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE * is not defined, because the 16-byte redzone, allocated on the stack when the flag is defined, can be * reused. In AArch64, it's safe that these bytes are always reserved because the stack layout might @@ -203,12 +209,12 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro LOAD_TSRM_CACHE, reg -| brk #0 // TODO +| NIY // TODO |.endmacro |.macro LOAD_ADDR_ZTS, reg, struct, field | .if ZTS -| brk #0 // TODO +| NIY // TODO | LOAD_TSRM_CACHE reg | add reg, reg, #(struct.._offset + offsetof(zend_..struct, field)) | .else @@ -217,7 +223,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro ADDR_OP1, addr_ins, addr, tmp_reg -| brk #0 // TODO +| NIY // TODO |.endmacro // Move the 48-bit address 'addr' into 'tmp_reg' and store it into the dest addr 'op1' @@ -238,7 +244,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro PUSH_ADDR_ZTS, struct, field, tmp_reg -| brk #0 // TODO +| NIY // TODO |.endmacro // Store the value from a register 'op' into memory 'addr' @@ -249,7 +255,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro MEM_STORE_ZTS, str_ins, op, struct, field, tmp_reg | .if ZTS -| brk #0 // TODO: test +| NIY // TODO: test | LOAD_TSRM_CACHE tmp_reg | str_ins op, [tmp_reg, #(struct.._offset+offsetof(zend_..struct, field))] | .else @@ -265,7 +271,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro MEM_LOAD_ZTS, ldr_ins, op, struct, field, tmp_reg | .if ZTS -| brk #0 // TODO: test +| NIY // TODO: test | LOAD_TSRM_CACHE tmp_reg | ldr_ins op, [tmp_reg, #(struct.._offset+offsetof(zend_..struct, field))] | .else @@ -283,7 +289,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro MEM_LOAD_OP_ZTS, mem_ins, ldr_ins, op, struct, field, tmp_reg1, tmp_reg2 | .if ZTS -| brk #0 // TODO: test +| NIY // TODO: test | LOAD_TSRM_CACHE tmp_reg1 | ldr_ins tmp_reg2, [tmp_reg1, #(struct.._offset+offsetof(zend_..struct, field))] | mem_ins op, op, tmp_reg2 @@ -301,7 +307,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro MEM_LOAD_CMP_ZTS, ldr_ins, op, struct, field, tmp_reg1, tmp_reg2 | .if ZTS -| brk #0 // TODO: test +| NIY // TODO: test | LOAD_TSRM_CACHE tmp_reg1 | ldr_ins tmp_reg2, [tmp_reg1, #(struct.._offset+offsetof(zend_..struct, field))] | cmp tmp_reg2, op @@ -321,7 +327,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro MEM_LOAD_OP_STORE_ZTS, mem_ins, ldr_ins, str_ins, op, struct, field, tmp_reg1, tmp_reg2 | .if ZTS -| brk #0 // TODO: test +| NIY // TODO: test | LOAD_TSRM_CACHE tmp_reg1 | ldr_ins tmp_reg2, [tmp_reg1, #(struct.._offset+offsetof(zend_..struct, field))] | mem_ins tmp_reg2, tmp_reg2, op @@ -332,7 +338,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro MEM_OP3_3, mem_ins, op1, op2, prefix, addr, tmp_reg -| brk #0 // TODO +| NIY // TODO |.endmacro |.macro LOAD_BASE_ADDR, reg, base, offset @@ -353,7 +359,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro PUSH_BASE_ADDR, base, offset, tmp_reg -| brk #0 // TODO +| NIY // TODO |.endmacro |.macro EXT_CALL, func, tmp_reg @@ -387,7 +393,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro LOAD_IP_ADDR_ZTS, struct, field -| brk #0 // TODO +| NIY // TODO |.endmacro |.macro GET_IP, reg @@ -524,12 +530,12 @@ static void* dasm_labels[zend_lb_MAX]; |.macro GET_ZVAL_W2, reg, addr || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -| brk #0 // TODO +| NIY // TODO |.endmacro |.macro SET_ZVAL_W2, addr, val || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -| brk #0 // TODO +| NIY // TODO |.endmacro |.macro UNDEF_OPLINE_RESULT, tmp_reg @@ -544,7 +550,7 @@ static void* dasm_labels[zend_lb_MAX]; // Use DOUBLE_CMP for comparisons and use DOUBLE_OP for arithmetic operations. |.macro DOUBLE_CMP, ins, reg, addr, tmp_reg, fp_tmp_reg || if (Z_MODE(addr) == IS_CONST_ZVAL) { -| brk #0 // TODO +| NIY // TODO | LOAD_ADDR Rx(tmp_reg), Z_ZV(addr) | ldr Rd(fp_tmp_reg-ZREG_V0), [Rx(tmp_reg)] | ins Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) @@ -552,7 +558,7 @@ static void* dasm_labels[zend_lb_MAX]; | SAFE_MEM_ACC_WITH_UOFFSET ldr, Rd(fp_tmp_reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) | ins Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) || } else if (Z_MODE(addr) == IS_REG) { -| brk #0 // TODO +| NIY // TODO | ins Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) || } else { || ZEND_UNREACHABLE(); @@ -560,7 +566,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro DOUBLE_OP, ins, reg, addr, tmp_reg, fp_tmp_reg -| brk #0 // TODO +| NIY // TODO || if (Z_MODE(addr) == IS_CONST_ZVAL) { | LOAD_ADDR Rx(tmp_reg), Z_ZV(addr) | ldr Rd(fp_tmp_reg-ZREG_V0), [Rx(tmp_reg)] @@ -579,7 +585,7 @@ static void* dasm_labels[zend_lb_MAX]; // Convert the LONG value 'lval' into DOUBLE type, and move it into 'reg' |.macro DOUBLE_GET_LONG, reg, lval, tmp_reg || if (lval == 0) { -| brk #0 // TODO: test +| NIY // TODO: test | // vxorps xmm(reg-ZREG_V0), xmm(reg-ZREG_V0), xmm(reg-ZREG_V0) || } else { | LOAD_64BIT_VAL Rx(tmp_reg), lval @@ -596,7 +602,7 @@ static void* dasm_labels[zend_lb_MAX]; | SAFE_MEM_ACC_WITH_UOFFSET ldr, Rx(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg2) | scvtf Rd(reg-ZREG_V0), Rx(tmp_reg1) || } else if (Z_MODE(addr) == IS_REG) { -| brk #0 // TODO: test +| NIY // TODO: test | scvtf Rd(reg-ZREG_V0), Rx(Z_REG(addr)) || } else { || ZEND_UNREACHABLE(); @@ -610,13 +616,13 @@ static void* dasm_labels[zend_lb_MAX]; | fadd Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(src_reg-ZREG_V0) || break; || case ZEND_SUB: -| brk #0 // vsubsd xmm(dst_reg-ZREG_V0), xmm(op1_reg-ZREG_V0), xmm(src_reg-ZREG_V0) +| NIY // vsubsd xmm(dst_reg-ZREG_V0), xmm(op1_reg-ZREG_V0), xmm(src_reg-ZREG_V0) || break; || case ZEND_MUL: -| brk #0 // vmulsd xmm(dst_reg-ZREG_V0), xmm(op1_reg-ZREG_V0), xmm(src_reg-ZREG_V0) +| NIY // vmulsd xmm(dst_reg-ZREG_V0), xmm(op1_reg-ZREG_V0), xmm(src_reg-ZREG_V0) || break; || case ZEND_DIV: -| brk #0 // vdivsd xmm(dst_reg-ZREG_V0), xmm(op1_reg-ZREG_V0), xmm(src_reg-ZREG_V0) +| NIY // vdivsd xmm(dst_reg-ZREG_V0), xmm(op1_reg-ZREG_V0), xmm(src_reg-ZREG_V0) || break; || } |.endmacro @@ -644,7 +650,7 @@ static void* dasm_labels[zend_lb_MAX]; || if(Z_LVAL_P(Z_ZV(addr)) >= 0 && Z_LVAL_P(Z_ZV(addr)) <= MAX_IMM12) { | cmp Rx(reg), #(Z_LVAL_P(Z_ZV(addr))) || } else { -| brk #0 // TODO +| NIY // TODO | LOAD_64BIT_VAL tmp_reg1, Z_LVAL_P(Z_ZV(addr)) | cmp Rx(reg), tmp_reg1 || } @@ -672,7 +678,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro LONG_OP_WITH_32BIT_CONST, long_ins, op1_addr, lval -| brk #0 // TODO +| NIY // TODO |.endmacro // Define LONG_CMP_WITH_CONST to replace LONG_OP_WITH_CONST in the x86 implementation. @@ -723,10 +729,10 @@ static void* dasm_labels[zend_lb_MAX]; | LONG_OP adds, reg, addr, tmp_reg1, tmp_reg2 || break; || case ZEND_SUB: -| brk #0 // LONG_OP sub, reg, addr +| NIY // LONG_OP sub, reg, addr || break; || case ZEND_MUL: -| brk #0 // LONG_OP imul, reg, addr +| NIY // LONG_OP imul, reg, addr || break; || case ZEND_BW_OR: | LONG_OP orr, reg, addr, tmp_reg1, tmp_reg2 @@ -735,7 +741,7 @@ static void* dasm_labels[zend_lb_MAX]; | LONG_OP and, reg, addr, tmp_reg1, tmp_reg2 || break; || case ZEND_BW_XOR: -| brk #0 // TODO +| NIY // TODO | LONG_OP eor, reg, addr, tmp_reg1, tmp_reg2 || break; || default: @@ -744,7 +750,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro LONG_MATH_REG, opcode, dst_reg, src_reg -| brk #0 // TODO +| NIY // TODO |.endmacro // In x86 implementation, argument 'lval' of SET_ZVAL_LVAL can be either a LONG constant @@ -772,7 +778,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro SET_ZVAL_DVAL, addr, reg, tmp_reg || if (Z_MODE(addr) == IS_REG) { || if (reg != Z_REG(addr)) { -| brk #0 // TODO: test +| NIY // TODO: test | fmov Rd(Z_REG(addr)-ZREG_V0), Rd(reg-ZREG_V0) || } || } else { @@ -785,13 +791,13 @@ static void* dasm_labels[zend_lb_MAX]; |.macro GET_ZVAL_DVAL, reg, addr, tmp_reg || if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) { || if (Z_MODE(addr) == IS_CONST_ZVAL) { -| brk #0 // TODO: test +| NIY // TODO: test | LOAD_ADDR Rx(tmp_reg), Z_ZV(addr) | ldr Rd(reg-ZREG_V0), [Rx(tmp_reg)] || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | SAFE_MEM_ACC_WITH_UOFFSET ldr, Rd(reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) || } else if (Z_MODE(addr) == IS_REG) { -| brk #0 // TODO: test +| NIY // TODO: test | fmov Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) || } else { || ZEND_UNREACHABLE(); @@ -808,7 +814,7 @@ static void* dasm_labels[zend_lb_MAX]; | SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 || } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { || zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg; -| brk #0 // TODO: test +| NIY // TODO: test || } else { | // In x64, if the range of this LONG value can be represented via INT type, only move the low 32 bits into dst_addr. | // Note that imm32 is signed extended to 64 bits during mov. @@ -829,7 +835,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro ZVAL_COPY_CONST_2, dst_addr, res_addr, dst_info, dst_def_info, zv, tmp_reg -| brk #0 // TODO +| NIY // TODO |.endmacro // the same as above, but "src" may overlap with "reg1" @@ -859,7 +865,7 @@ static void* dasm_labels[zend_lb_MAX]; | SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg) || } || } else if (Z_MODE(dst_addr) == IS_REG) { -| brk #0 // TODO: test +| NIY // TODO: test | GET_ZVAL_LVAL Z_REG(dst_addr), src_addr, Rx(tmp_reg) || } else { | GET_ZVAL_LVAL reg2, src_addr, Rx(tmp_reg) @@ -867,13 +873,13 @@ static void* dasm_labels[zend_lb_MAX]; || } || } else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { || if (Z_MODE(src_addr) == IS_REG) { -| brk #0 // TODO: test +| NIY // TODO: test | SET_ZVAL_DVAL dst_addr, Z_REG(src_addr), tmp_reg || } else if (Z_MODE(dst_addr) == IS_REG) { -| brk #0 // TODO: test +| NIY // TODO: test | GET_ZVAL_DVAL Z_REG(dst_addr), src_addr, tmp_reg || } else { -| brk #0 // TODO: test +| NIY // TODO: test | GET_ZVAL_DVAL fp_tmp_reg, src_addr, tmp_reg | SET_ZVAL_DVAL dst_addr, fp_tmp_reg, tmp_reg || } @@ -887,7 +893,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro ZVAL_COPY_VALUE_2, dst_addr, dst_info, res_addr, src_addr, src_info, tmp_reg1, tmp_reg2 -| brk #0 // TODO +| NIY // TODO |.endmacro |.macro IF_UNDEF, type_reg, label @@ -1013,7 +1019,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro ADDREF_CONST_2, zv, tmp_reg -| brk #0 // TODO +| NIY // TODO |.endmacro |.macro TRY_ADDREF, val_info, type_flags_reg, value_ptr_reg, tmp_reg @@ -1077,7 +1083,7 @@ static void* dasm_labels[zend_lb_MAX]; || break; || } else if (type == IS_OBJECT) { || if (opline) { -| brk #0 // TODO +| NIY // TODO || } | EXT_CALL zend_objects_store_del, tmp_reg || break; @@ -1150,7 +1156,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro SEPARATE_ARRAY, addr, op_info, cold, tmp_reg1, tmp_reg2 || if (RC_MAY_BE_N(op_info)) { || if (Z_REG(addr) != ZREG_FP) { -| brk #0 // TODO +| NIY // TODO || } else { | GET_ZVAL_LVAL ZREG_FCARG1x, addr, Rx(tmp_reg1) || if (RC_MAY_BE_1(op_info)) { @@ -1162,7 +1168,7 @@ static void* dasm_labels[zend_lb_MAX]; |.cold_code |1: || } else { -| brk #0 // TODO +| NIY // TODO | bls >2 || } || } @@ -1188,11 +1194,11 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro EFREE_REG_REFERENCE -| brk #0 // TODO +| NIY // TODO |.endmacro |.macro EFREE_REFERENCE, ptr -| brk #0 // TODO +| NIY // TODO |.endmacro |.macro EMALLOC, size, op_array, opline @@ -1216,7 +1222,7 @@ static void* dasm_labels[zend_lb_MAX]; | mov REG0, RETVALx || } ||#else -| brk #0 // TODO +| NIY // TODO | mov FCARG1x, #size | EXT_CALL _emalloc, REG0 | mov REG0, RETVALx @@ -1384,7 +1390,7 @@ static inline bool is_signed(double d) static int zend_jit_interrupt_handler_stub(dasm_State **Dst) { |->interrupt_handler: - | brk #0 // TODO + | NIY // TODO return 1; } @@ -1405,7 +1411,7 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) | add sp, sp, SPAD // stack alignment | EXT_JMP handler, REG0 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - | brk #0 // TODO: test + | NIY // TODO: test } else { | mov FCARG1x, FP | ldp FP, RX, T2 // retore FP and IP @@ -1421,7 +1427,7 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) static int zend_jit_exception_handler_undef_stub(dasm_State **Dst) { |->exception_handler_undef: - | brk #0 // TODO + | NIY // TODO return 1; } @@ -1433,7 +1439,7 @@ static int zend_jit_leave_function_stub(dasm_State **Dst) | LOAD_32BIT_VAL TMP1w, ZEND_CALL_TOP | tst FCARG1w, TMP1w | bne >1 - | brk #0 // TODO: currently jump to label 1. + | NIY // TODO: currently jump to label 1. | EXT_CALL zend_jit_leave_nested_func_helper, REG0 | ADD_HYBRID_SPAD | JMP_IP TMP1 @@ -1453,7 +1459,7 @@ static int zend_jit_leave_function_stub(dasm_State **Dst) | LOAD_32BIT_VAL TMP1w, ZEND_CALL_TOP | tst FCARG1w, TMP1w | bne >1 - | brk #0 // TODO: currently jump to label 1. + | NIY // TODO: currently jump to label 1. | EXT_JMP zend_jit_leave_nested_func_helper, REG0 |1: | EXT_JMP zend_jit_leave_top_func_helper, REG0 @@ -1465,7 +1471,7 @@ static int zend_jit_leave_function_stub(dasm_State **Dst) static int zend_jit_leave_throw_stub(dasm_State **Dst) { |->leave_throw_handler: - | brk #0 // TODO: test + | NIY // TODO: test return 1; } @@ -1473,7 +1479,7 @@ static int zend_jit_leave_throw_stub(dasm_State **Dst) static int zend_jit_icall_throw_stub(dasm_State **Dst) { |->icall_throw_handler: - | brk #0 // TODO + | NIY // TODO return 1; } @@ -1481,7 +1487,7 @@ static int zend_jit_icall_throw_stub(dasm_State **Dst) static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) { |->throw_cannot_pass_by_ref: - | brk #0 // TODO + | NIY // TODO return 1; } @@ -1489,7 +1495,7 @@ static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) { |->undefined_offset_ex: - | brk #0 // TODO + | NIY // TODO return 1; } @@ -1497,7 +1503,7 @@ static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) static int zend_jit_undefined_offset_stub(dasm_State **Dst) { |->undefined_offset: - | brk #0 // TODO + | NIY // TODO return 1; } @@ -1514,7 +1520,7 @@ static int zend_jit_undefined_index_ex_stub(dasm_State **Dst) static int zend_jit_undefined_index_stub(dasm_State **Dst) { |->undefined_index: - | brk #0 // TODO + | NIY // TODO return 1; } @@ -1522,7 +1528,7 @@ static int zend_jit_undefined_index_stub(dasm_State **Dst) static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) { |->cannot_add_element_ex: - | brk #0 // TODO + | NIY // TODO return 1; } @@ -1530,7 +1536,7 @@ static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) static int zend_jit_cannot_add_element_stub(dasm_State **Dst) { |->cannot_add_element: - | brk #0 // TODO + | NIY // TODO return 1; } @@ -1575,14 +1581,14 @@ static int zend_jit_mod_by_zero_stub(dasm_State **Dst) static int zend_jit_invalid_this_stub(dasm_State **Dst) { |->invalid_this: - | brk #0 // TODO + | NIY // TODO return 1; } static int zend_jit_double_one_stub(dasm_State **Dst) { |->one: - | brk #0 // TODO + | NIY // TODO return 1; } @@ -1605,7 +1611,7 @@ static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) } |->hybrid_profile_jit: - | brk #0 // TODO + | NIY // TODO return 1; } @@ -1616,7 +1622,7 @@ static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst) } |->hybrid_hot_code: - | brk #0 // TODO + | NIY // TODO return 1; } @@ -1646,7 +1652,7 @@ static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst) */ static int zend_jit_hybrid_hot_counter_stub(dasm_State **Dst, uint32_t cost) { - | brk #0 // TODO + | NIY // TODO return 1; } @@ -1758,7 +1764,7 @@ static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst) static int zend_jit_trace_halt_stub(dasm_State **Dst) { |->trace_halt: - | brk #0 // TODO + | NIY // TODO return 1; } @@ -1836,7 +1842,7 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst) } |1: - | brk #0 // TODO: test + | NIY // TODO: test | blt ->trace_halt | // execute_data = EG(current_execute_data) @@ -1891,7 +1897,7 @@ static int zend_jit_trace_escape_stub(dasm_State **Dst) { |->trace_escape: | - | brk #0 // TODO + | NIY // TODO return 1; } @@ -1925,7 +1931,7 @@ static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n) static int zend_jit_context_threaded_call_stub(dasm_State **Dst) { |->context_threaded_call: - | brk #0 // TODO + | NIY // TODO return 1; } #endif @@ -1949,7 +1955,7 @@ static int zend_jit_assign_const_stub(dasm_State **Dst) uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; |->assign_const: - | brk #0 // TODO + | NIY // TODO if (!zend_jit_assign_to_variable( Dst, NULL, var_addr, var_addr, -1, -1, @@ -1988,7 +1994,7 @@ static int zend_jit_assign_var_stub(dasm_State **Dst) uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF; |->assign_var: - | brk #0 // TODOa + | NIY // TODOa | ret return 1; } @@ -2000,7 +2006,7 @@ static int zend_jit_assign_cv_noref_stub(dasm_State **Dst) uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/; |->assign_cv_noref: - | brk #0 // TODO + | NIY // TODO | ret return 1; } @@ -2012,7 +2018,7 @@ static int zend_jit_assign_cv_stub(dasm_State **Dst) uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/; |->assign_cv: - | brk #0 // TODO + | NIY // TODO | ret return 1; } @@ -2254,7 +2260,7 @@ static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline) static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) { if (delayed_call_chain) { - | brk #0 // TODO: test + | NIY // TODO: test if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { return 0; } @@ -2279,7 +2285,7 @@ static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void | beq =>loop_label | EXT_JMP timeout_exit_addr, TMP1 } else { - | brk #0 // TODO + | NIY // TODO | b =>loop_label } return 1; @@ -2377,7 +2383,7 @@ static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *link_addr; size_t prologue_size; - | brk #0 // TODO + | NIY // TODO return 1; } @@ -2388,7 +2394,7 @@ static int zend_jit_trace_return(dasm_State **Dst, bool original_handler) if (!original_handler) { | JMP_IP TMP1 } else { - | brk #0 // TODO: test + | NIY // TODO: test | ldr REG0, EX->func | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] @@ -2401,7 +2407,7 @@ static int zend_jit_trace_return(dasm_State **Dst, bool original_handler) if (!original_handler) { | JMP_IP TMP1 } else { - | brk #0 // TODO: test + | NIY // TODO: test | ldr REG0, EX->func | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] @@ -2411,7 +2417,7 @@ static int zend_jit_trace_return(dasm_State **Dst, bool original_handler) } } else { if (original_handler) { - | brk #0 // TODO: test + | NIY // TODO: test | mov FCARG1x, FP | ldr REG0, EX->func | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] @@ -2462,7 +2468,7 @@ static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32 return 0; } - | brk #0 // TODO + | NIY // TODO return 1; } @@ -2479,7 +2485,7 @@ static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_arra return 0; } - | brk #0 // TODO + | NIY // TODO return 1; } @@ -2544,7 +2550,7 @@ static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) } else { const void *handler = zend_get_opcode_handler_func(opline); - | brk #0 // TODO: test + | NIY // TODO: test } } else { const void *handler = opline->handler; @@ -2571,7 +2577,7 @@ static int zend_jit_trace_opline_guard(dasm_State **Dst, const zend_op *opline) if (!exit_addr) { return 0; } - | brk #0 // TODO + | NIY // TODO zend_jit_set_last_valid_opline(opline); @@ -2597,7 +2603,7 @@ static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsig #ifdef CONTEXT_THREADED_JIT static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block) { - | brk #0 // TODO + | NIY // TODO return 1; } #endif @@ -2622,7 +2628,7 @@ static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_ad | SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2 } } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - | brk #0 // TODO + | NIY // TODO } else { ZEND_UNREACHABLE(); } @@ -2637,7 +2643,7 @@ static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr if ((info & MAY_BE_ANY) == MAY_BE_LONG) { | GET_ZVAL_LVAL Z_REG(dst), src, TMP1 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - | brk #0 // TODO + | NIY // TODO } else { ZEND_UNREACHABLE(); } @@ -2655,7 +2661,7 @@ static int zend_jit_store_var(dasm_State **Dst, uint32_t info, int var, zend_reg static int zend_jit_store_var_if_necessary(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info) { if (Z_MODE(src) == IS_REG && Z_STORE(src)) { - | brk #0 // TODO: test + | NIY // TODO: test zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); return zend_jit_spill_store(Dst, src, dst, info, 1); } @@ -2692,9 +2698,9 @@ static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr sr if (!zend_jit_same_addr(src, dst)) { if (Z_MODE(src) == IS_REG) { if (Z_MODE(dst) == IS_REG) { - | brk #0 // TODO + | NIY // TODO } else if (Z_MODE(dst) == IS_MEM_ZVAL) { - | brk #0 // TODO + | NIY // TODO } else { ZEND_UNREACHABLE(); } @@ -2717,7 +2723,7 @@ static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); - | brk #0 // TODO + | NIY // TODO return 1; } @@ -2726,13 +2732,13 @@ static int zend_jit_store_const(dasm_State **Dst, int var, zend_reg reg) { zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); - | brk #0 // TODO + | NIY // TODO return 1; } static int zend_jit_free_trampoline(dasm_State **Dst) { - | brk #0 // TODO + | NIY // TODO return 1; } @@ -2750,7 +2756,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { | LONG_OP_WITH_CONST_IMM12 adds, op1_def_addr, Z_L(1), TMP1, TMP2 } else { - | brk #0 // TODO: test + | NIY // TODO: test | LONG_OP_WITH_CONST_IMM12 subs, op1_def_addr, Z_L(1), TMP1, TMP2 } @@ -2762,17 +2768,17 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op zend_jit_trace_stack *stack; uint32_t old_op1_info, old_res_info = 0; - | brk #0 // TODO: test + | NIY // TODO: test } else if (may_overflow) { | bvs >1 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { - | brk #0 // TODO: test + | NIY // TODO: test | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } |.cold_code |1: - | brk #0 // TODO: test + | NIY // TODO: test | b >3 |.code } else { @@ -2784,7 +2790,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { |.cold_code |2: - | brk #0 // TODO: test + | NIY // TODO: test | b >3 |.code } @@ -2855,26 +2861,26 @@ static int zend_jit_math_long_long(dasm_State **Dst, (Z_MODE(op1_addr) == IS_CONST_ZVAL && IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr))) && is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))))) { - | brk #0 // TODO: test + | NIY // TODO: test } else if (opcode == ZEND_DIV && (Z_MODE(op2_addr) == IS_CONST_ZVAL && is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { - | brk #0 // TODO: test + | NIY // TODO: test } else if (opcode == ZEND_ADD && !may_overflow && Z_MODE(op1_addr) == IS_REG && Z_MODE(op2_addr) == IS_CONST_ZVAL) { - | brk #0 // TODO: test + | NIY // TODO: test } else if (opcode == ZEND_ADD && !may_overflow && Z_MODE(op2_addr) == IS_REG && Z_MODE(op1_addr) == IS_CONST_ZVAL) { - | brk #0 // TODO: test + | NIY // TODO: test } else if (opcode == ZEND_SUB && !may_overflow && Z_MODE(op1_addr) == IS_REG && Z_MODE(op2_addr) == IS_CONST_ZVAL) { - | brk #0 // TODO: test + | NIY // TODO: test } else { | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) @@ -2883,7 +2889,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, /* +/- 0 */ may_overflow = 0; } else if (same_ops && opcode != ZEND_DIV) { - | brk #0 // TODO: test + | NIY // TODO: test } else { | LONG_MATH opcode, result_reg, op2_addr, TMP1, TMP2 } @@ -2902,7 +2908,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, | mov Rx(Z_REG(res_addr)), Rx(result_reg) } } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - | brk #0 // TODO + | NIY // TODO } else { ZEND_UNREACHABLE(); } @@ -2910,7 +2916,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, if (res_info & MAY_BE_LONG) { | bvs >1 } else { - | brk #0 // TODO: test + | NIY // TODO: test } } } @@ -2938,13 +2944,13 @@ static int zend_jit_math_long_long(dasm_State **Dst, (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1)) { if (opcode == ZEND_ADD) { if (Z_MODE(res_addr) == IS_REG) { - | brk #0 // TODO: test + | NIY // TODO: test } else { | SET_ZVAL_LVAL res_addr, 0x43e0000000000000, REG0, TMP1 } break; } else if (opcode == ZEND_SUB) { - | brk #0 // TODO: test + | NIY // TODO: test break; } } @@ -2980,7 +2986,7 @@ static int zend_jit_math_long_double(dasm_State **Dst, (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_FPR0; zend_reg tmp_reg; - | brk #0 // TODO + | NIY // TODO return 1; } @@ -2994,7 +3000,7 @@ static int zend_jit_math_double_long(dasm_State **Dst, { zend_reg result_reg, tmp_reg; - | brk #0 // TODO + | NIY // TODO return 1; } @@ -3008,7 +3014,7 @@ static int zend_jit_math_double_double(dasm_State **Dst, bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); zend_reg result_reg; - | brk #0 // TODO + | NIY // TODO return 1; } @@ -3038,7 +3044,7 @@ static int zend_jit_math_helper(dasm_State **Dst, if (op1_info & MAY_BE_DOUBLE) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, TMP1w, TMP2 } else { - | brk #0 // TODO: test + | NIY // TODO: test | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, TMP1w, TMP2 } } @@ -3047,7 +3053,7 @@ static int zend_jit_math_helper(dasm_State **Dst, | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1, TMP1w, TMP2 |.cold_code |1: - | brk #0 // TODO: test + | NIY // TODO: test if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, TMP1w, TMP2 } @@ -3057,7 +3063,7 @@ static int zend_jit_math_helper(dasm_State **Dst, | b >5 |.code } else { - | brk #0 // TODO: test + | NIY // TODO: test | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, TMP1w, TMP2 } } @@ -3070,7 +3076,7 @@ static int zend_jit_math_helper(dasm_State **Dst, if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, TMP1w, TMP2 } - | brk #0 // TODO: test + | NIY // TODO: test if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops) { @@ -3099,7 +3105,7 @@ static int zend_jit_math_helper(dasm_State **Dst, } else if ((op1_info & MAY_BE_DOUBLE) && !(op1_info & MAY_BE_LONG) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { - | brk #0 // TODO: test + | NIY // TODO: test if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, TMP1w, TMP2 } @@ -3134,7 +3140,7 @@ static int zend_jit_math_helper(dasm_State **Dst, } else if ((op2_info & MAY_BE_DOUBLE) && !(op2_info & MAY_BE_LONG) && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { - | brk #0 // TODO: test + | NIY // TODO: test if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, TMP1w, TMP2 } @@ -3179,30 +3185,30 @@ static int zend_jit_math_helper(dasm_State **Dst, |6: if (Z_MODE(res_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); - | brk #0 // TODO: test + | NIY // TODO: test | LOAD_ZVAL_ADDR FCARG1x, real_addr } else if (Z_REG(res_addr) != ZREG_FCARG1x || Z_OFFSET(res_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, res_addr } if (Z_MODE(op1_addr) == IS_REG) { - | brk #0 // TODO: test + | NIY // TODO: test } | LOAD_ZVAL_ADDR FCARG2x, op1_addr if (Z_MODE(op2_addr) == IS_REG) { - | brk #0 // TODO: test + | NIY // TODO: test } | LOAD_ZVAL_ADDR CARG3, op2_addr | SET_EX_OPLINE opline, REG0 if (opcode == ZEND_ADD) { | EXT_CALL add_function, REG0 } else if (opcode == ZEND_SUB) { - | brk #0 // TODO: test + | NIY // TODO: test | EXT_CALL sub_function, REG0 } else if (opcode == ZEND_MUL) { - | brk #0 // TODO: test + | NIY // TODO: test | EXT_CALL mul_function, REG0 } else if (opcode == ZEND_DIV) { - | brk #0 // TODO: test + | NIY // TODO: test | EXT_CALL div_function, REG0 } else { ZEND_UNREACHABLE(); @@ -3213,7 +3219,7 @@ static int zend_jit_math_helper(dasm_State **Dst, zend_jit_check_exception(Dst); } if (Z_MODE(res_addr) == IS_REG) { - | brk #0 // TODO: test + | NIY // TODO: test } if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { @@ -3245,7 +3251,7 @@ static int zend_jit_add_arrays(dasm_State **Dst, const zend_op *opline, uint32_t zend_jit_addr op1_addr = OP1_ADDR(); zend_jit_addr op2_addr = OP2_ADDR(); - | brk #0 // TODO + | NIY // TODO return 1; } @@ -3455,7 +3461,7 @@ static int zend_jit_long_math_helper(dasm_State **Dst, } } } else if (same_ops) { - | brk #0 // TODO + | NIY // TODO } else { | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 | LONG_MATH opcode, result_reg, op2_addr, TMP1, TMP2 @@ -3479,7 +3485,7 @@ static int zend_jit_long_math_helper(dasm_State **Dst, |.cold_code } |6: - | brk #0 // TODO + | NIY // TODO if (may_throw) { zend_jit_check_exception(Dst); } @@ -3564,7 +3570,7 @@ static int zend_jit_concat_helper(dasm_State **Dst, |.cold_code |6: } - | brk #0 // TODO + | NIY // TODO if (may_throw) { zend_jit_check_exception(Dst); } @@ -3621,7 +3627,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o if (!exit_addr) { return 0; } - | brk #0 // TODO + | NIY // TODO } if (type == BP_VAR_W) { | // hval = Z_LVAL_P(dim); @@ -3658,22 +3664,22 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o if (val == 0) { | cmp REG0, #0 } else if (val > 0 && !op2_loaded) { - | brk #0 // TODO + | NIY // TODO | LOAD_64BIT_VAL TMP1, val | cmp REG0, TMP1 } else { - | brk #0 // TODO + | NIY // TODO | cmp REG0, FCARG2x } if (type == BP_JIT_IS) { - | brk #0 // TODO + | NIY // TODO } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { - | brk #0 // TODO + | NIY // TODO } else if (type == BP_VAR_IS && not_found_exit_addr) { - | brk #0 // TODO + | NIY // TODO } else if (type == BP_VAR_IS && found_exit_addr) { - | brk #0 // TODO + | NIY // TODO } else { | bls >2 // NOT_FOUND } @@ -3681,12 +3687,12 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o if (val >= 0) { | ldr REG0, [FCARG1x, #offsetof(zend_array, arData)] if (val != 0) { - | brk #0 // TODO + | NIY // TODO | LOAD_64BIT_VAL TMP1, val * sizeof(Bucket) | add REG0, REG0, TMP1 } } else { - | brk #0 // TODO + | NIY // TODO | mov REG0, FCARG2x | lsl REG0, REG0, #5 | ldr TMP1, [FCARG1x, #offsetof(zend_array, arData)] @@ -3696,7 +3702,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } switch (type) { case BP_JIT_IS: - | brk #0 // TODO + | NIY // TODO break; case BP_VAR_R: case BP_VAR_IS: @@ -3707,45 +3713,45 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { /* perform IS_UNDEF check only after result type guard (during deoptimization) */ if (!found_exit_addr || (op1_info & MAY_BE_ARRAY_HASH)) { - | brk #0 // TODO + | NIY // TODO } } else if (type == BP_VAR_IS && not_found_exit_addr) { - | brk #0 // TODO + | NIY // TODO } else if (type == BP_VAR_IS && found_exit_addr) { - | brk #0 // TODO + | NIY // TODO } else { - | brk #0 // TODO + | NIY // TODO } } if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (packed_loaded && (op1_info & MAY_BE_ARRAY_HASH))) { - | brk #0 // TODO + | NIY // TODO } if (op1_info & MAY_BE_ARRAY_HASH) { |4: if (!op2_loaded) { - | brk #0 // TODO + | NIY // TODO } | EXT_CALL _zend_hash_index_find, REG0 | mov REG0, RETVALx | tst REG0, REG0 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { - | brk #0 // TODO + | NIY // TODO } else if (type == BP_VAR_IS && not_found_exit_addr) { - | brk #0 // TODO + | NIY // TODO } else if (type == BP_VAR_IS && found_exit_addr) { - | brk #0 // TODO + | NIY // TODO } else { | beq >2 // NOT_FOUND } } |.cold_code |2: - | brk #0 // TODO + | NIY // TODO |.code break; case BP_VAR_RW: if (packed_loaded) { - | brk #0 // TODO + | NIY // TODO } |2: |4: @@ -3767,18 +3773,18 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | //retval = zend_hash_index_add_new(ht, hval, &EG(uninitialized_zval)); if (!op2_loaded) { | // hval = Z_LVAL_P(dim); - | brk #0 // TODO + | NIY // TODO | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 } | LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval | EXT_CALL zend_hash_index_add_new, REG0 | mov REG0, RETVALx if (op1_info & MAY_BE_ARRAY_HASH) { - | brk #0 // TODO + | NIY // TODO } } if (op1_info & MAY_BE_ARRAY_HASH) { - | brk #0 // TODO + | NIY // TODO } break; default: @@ -3801,7 +3807,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | // retval = zend_hash_find(ht, offset_key); switch (type) { case BP_JIT_IS: - | brk #0 // TODO + | NIY // TODO break; case BP_VAR_R: case BP_VAR_IS: @@ -3818,31 +3824,31 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | EXT_CALL zend_hash_find, REG0 |1: } else { - | brk #0 // TODO + | NIY // TODO | EXT_CALL _zend_hash_find_known_hash, REG0 } | mov REG0, RETVALx | tst REG0, REG0 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { - | brk #0 // TODO + | NIY // TODO } else if (type == BP_VAR_IS && not_found_exit_addr) { - | brk #0 // TODO + | NIY // TODO } else if (type == BP_VAR_IS && found_exit_addr) { - | brk #0 // TODO + | NIY // TODO } else { | beq >2 // NOT_FOUND |.cold_code |2: - | brk #0 // TODO + | NIY // TODO |.code } break; case BP_VAR_RW: - | brk #0 // TODO + | NIY // TODO break; case BP_VAR_W: if (opline->op2_type != IS_CONST) { - | brk #0 // TODO + | NIY // TODO | EXT_CALL zend_jit_symtable_lookup_w, REG0 } else { | EXT_CALL zend_hash_lookup, REG0 @@ -3855,7 +3861,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) { - | brk #0 // TODO + | NIY // TODO } if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { @@ -3873,17 +3879,17 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | b >9 break; case BP_JIT_IS: - | brk #0 // TODO + | NIY // TODO break; case BP_VAR_IS: case BP_VAR_UNSET: - | brk #0 // TODO + | NIY // TODO break; case BP_VAR_RW: - | brk #0 // TODO + | NIY // TODO break; case BP_VAR_W: - | brk #0 // TODO + | NIY // TODO break; default: ZEND_UNREACHABLE(); @@ -3924,15 +3930,15 @@ static int zend_jit_simple_assign(dasm_State **Dst, if (!res_addr) { | ZVAL_COPY_CONST var_addr, var_info, var_def_info, zv, tmp_reg, ZREG_TMP1, ZREG_FPR0 } else { - | brk #0 // TODO + | NIY // TODO } if (Z_REFCOUNTED_P(zv)) { - | brk #0 // TODO + | NIY // TODO } } else { if (val_info & MAY_BE_UNDEF) { if (in_cold) { - | brk #0 // TODO + | NIY // TODO | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2, TMP1w, TMP2 } else { | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1, TMP1w, TMP2 @@ -3941,12 +3947,12 @@ static int zend_jit_simple_assign(dasm_State **Dst, } | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); if (save_r1) { - | brk #0 // TODO + | NIY // TODO | str FCARG1x, T1 // save } | SET_ZVAL_TYPE_INFO var_addr, IS_NULL, TMP1w, TMP2 if (res_addr) { - | brk #0 // TODO + | NIY // TODO | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 } if (opline) { @@ -3956,7 +3962,7 @@ static int zend_jit_simple_assign(dasm_State **Dst, | LOAD_32BIT_VAL FCARG1w, Z_OFFSET(val_addr) | EXT_CALL zend_jit_undefined_op_helper, REG0 if (save_r1) { - | brk #0 // TODO + | NIY // TODO | ldr FCARG1x, T1 // restore } | b >3 @@ -3978,14 +3984,14 @@ static int zend_jit_simple_assign(dasm_State **Dst, zend_jit_addr ref_addr; if (in_cold) { - | brk #0 // TODO + | NIY // TODO | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1, TMP1w, TMP2 } else { | IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1, TMP1w, TMP2 |.cold_code |1: } - | brk #0 // TODO + | NIY // TODO if (in_cold) { |1: } else { @@ -3997,7 +4003,7 @@ static int zend_jit_simple_assign(dasm_State **Dst, if (!res_addr) { | ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } else { - | brk #0 // TODO + | NIY // TODO } if (val_type == IS_CV) { @@ -4006,11 +4012,11 @@ static int zend_jit_simple_assign(dasm_State **Dst, | and REG2w, REG2w, #0xff | TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1 } else { - | brk #0 // TODO + | NIY // TODO } } else { if (res_addr) { - | brk #0 // TODO + | NIY // TODO } } |3: @@ -4030,7 +4036,7 @@ static int zend_jit_assign_to_typed_ref(dasm_State **Dst, | bne >2 |.cold_code |2: - | brk #0 // TODO + | NIY // TODO if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2x || Z_OFFSET(val_addr) != 0) { | LOAD_ZVAL_ADDR FCARG2x, val_addr } @@ -4085,20 +4091,20 @@ static int zend_jit_assign_to_variable_call(dasm_State **Dst, if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { | bl ->assign_tmp } else if (val_type == IS_CONST) { - | brk #0 // TODO + | NIY // TODO } else if (val_type == IS_TMP_VAR) { - | brk #0 // TODO + | NIY // TODO } else if (val_type == IS_VAR) { if (!(val_info & MAY_BE_REF)) { - | brk #0 // TODO + | NIY // TODO } else { - | brk #0 // TODO + | NIY // TODO } } else if (val_type == IS_CV) { if (!(val_info & MAY_BE_REF)) { - | brk #0 // TODO + | NIY // TODO } else { - | brk #0 // TODO + | NIY // TODO } } else { ZEND_UNREACHABLE(); @@ -4209,7 +4215,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, | beq >8 | b ->exception_handler } else { - | brk #0 // TODO + | NIY // TODO } } if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { @@ -4226,7 +4232,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, done = 1; } } else /* if (RC_MAY_BE_N(var_info)) */ { - | brk #0 + | NIY // TODO } } @@ -4259,25 +4265,25 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t return 0; } - | brk #0 // TODO + | NIY // TODO val_info &= ~MAY_BE_UNDEF; } if (op1_info & MAY_BE_REF) { - | brk #0 // TODO + | NIY // TODO op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); } if (op1_info & MAY_BE_ARRAY) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { - | brk #0 // TODO + | NIY // TODO | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, TMP1w, TMP2 } |3: | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { - | brk #0 // TODO + | NIY // TODO } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { @@ -4286,7 +4292,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t uint32_t var_info = MAY_BE_NULL; zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); - | brk #0 // TODO + | NIY // TODO if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0)) { return 0; @@ -4331,16 +4337,16 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) && (op1_info & MAY_BE_ARRAY)) { - | brk #0 // TODO + | NIY // TODO } if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { - | brk #0 // TODO + | NIY // TODO } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { - | brk #0 // TODO + | NIY // TODO } |.code } @@ -4369,7 +4375,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint3 ZEND_ASSERT(opline->result_type == IS_UNUSED); - | brk #0 // TODO + | NIY // TODO return 1; } @@ -4385,7 +4391,7 @@ static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t if (op1_info & MAY_BE_REF) { binary_op_type binary_op = get_binary_op(opline->extended_value); - | brk #0 // TODO + | NIY // TODO op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); } @@ -4517,10 +4523,10 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPZ_EX || smart_branch_opcode == ZEND_JMPNZ_EX) { - | brk #0 // TODO + | NIY // TODO } if (smart_branch_opcode && !exit_addr) { - | brk #0 // TODO + | NIY // TODO } return 1; } @@ -4533,26 +4539,26 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, } } else if (Z_MODE(op1_addr) == IS_REG) { if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { - | brk #0 // TODO + | NIY // TODO } else { | LONG_CMP Z_REG(op1_addr), op2_addr, TMP1, TMP2 } } else if (Z_MODE(op2_addr) == IS_REG) { if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) { - | brk #0 // TODO + | NIY // TODO } else { - | brk #0 // TODO + | NIY // TODO } swap = 1; } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { | LONG_CMP_WITH_CONST cmp, op2_addr, Z_LVAL_P(Z_ZV(op1_addr)), TMP1, TMP2 swap = 1; } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { - | brk #0 // TODO + | NIY // TODO } else { | GET_ZVAL_LVAL ZREG_REG0, op1_addr, TMP1 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { - | brk #0 // TODO + | NIY // TODO } else { | LONG_CMP ZREG_REG0, op2_addr, TMP1, TMP2 } @@ -4562,7 +4568,7 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, if (smart_branch_opcode == ZEND_JMPZ_EX || smart_branch_opcode == ZEND_JMPNZ_EX) { - | brk #0 // TODO + | NIY // TODO } if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) { @@ -4571,17 +4577,17 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: - | brk #0 // TODO + | NIY // TODO break; case ZEND_IS_NOT_EQUAL: - | brk #0 // TODO + | NIY // TODO break; case ZEND_IS_NOT_IDENTICAL: - | brk #0 // TODO + | NIY // TODO break; case ZEND_IS_SMALLER: if (swap) { - | brk #0 // TODO + | NIY // TODO } else { if (exit_addr) { | bge >1 @@ -4590,12 +4596,12 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, | EXT_JMP exit_addr, TMP1 |.code } else { - | brk #0 // TODO + | NIY // TODO } } break; case ZEND_IS_SMALLER_OR_EQUAL: - | brk #0 // TODO + | NIY // TODO break; default: ZEND_UNREACHABLE(); @@ -4607,37 +4613,37 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: - | brk #0 // TODO + | NIY // TODO break; case ZEND_IS_NOT_EQUAL: - | brk #0 // TODO + | NIY // TODO break; case ZEND_IS_NOT_IDENTICAL: - | brk #0 // TODO + | NIY // TODO break; case ZEND_IS_SMALLER: if (swap) { if (exit_addr) { - | brk #0 // TODO + | NIY // TODO } else { | bgt => target_label } } else { if (exit_addr) { - | brk #0 // TODO + | NIY // TODO } else { | blt => target_label } } break; case ZEND_IS_SMALLER_OR_EQUAL: - | brk #0 // TODO + | NIY // TODO break; default: ZEND_UNREACHABLE(); } } else if (smart_branch_opcode == ZEND_JMPZNZ) { - | brk #0 // TODO + | NIY // TODO } else { ZEND_UNREACHABLE(); } @@ -4647,24 +4653,24 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: - | brk #0 // TODO + | NIY // TODO break; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: - | brk #0 // TODO + | NIY // TODO break; case ZEND_IS_SMALLER: if (swap) { - | brk #0 // TODO + | NIY // TODO } else { - | brk #0 // TODO + | NIY // TODO } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { | cset REG0w, ge } else { - | brk #0 // TODO + | NIY // TODO } break; default: @@ -4687,7 +4693,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z case ZEND_CASE: case ZEND_CASE_STRICT: if (exit_addr) { - | brk #0 // TODO + | NIY // TODO } else { | bne => target_label } @@ -4695,26 +4701,26 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z case ZEND_IS_NOT_EQUAL: | bvs >1 if (exit_addr) { - | brk #0 // TODO + | NIY // TODO } else { | beq => target_label } |1: break; case ZEND_IS_NOT_IDENTICAL: - | brk #0 // TODO + | NIY // TODO break; case ZEND_IS_SMALLER: if (swap) { if (exit_addr) { - | brk #0 // TODO + | NIY // TODO } else { | bvs => target_label // TODO: why the NaN check is missing in x86? | bls => target_label } } else { if (exit_addr) { - | brk #0 // TODO + | NIY // TODO } else { | bhs => target_label } @@ -4723,14 +4729,14 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { if (exit_addr) { - | brk #0 // TODO + | NIY // TODO } else { | bvs => target_label // TODO: why the NaN check is missing in x86? | blo => target_label } } else { if (exit_addr) { - | brk #0 // TODO + | NIY // TODO } else { | bhi => target_label } @@ -4747,7 +4753,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z case ZEND_CASE_STRICT: | bvs >1 if (exit_addr) { - | brk #0 // TODO + | NIY // TODO } else { | beq => target_label } @@ -4755,18 +4761,18 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z break; case ZEND_IS_NOT_EQUAL: if (exit_addr) { - | brk #0 // TODO + | NIY // TODO } else { | bne => target_label } break; case ZEND_IS_NOT_IDENTICAL: - | brk #0 // TODO + | NIY // TODO break; case ZEND_IS_SMALLER: if (swap) { if (exit_addr) { - | brk #0 // TODO + | NIY // TODO } else { | bvs >1 // Always False if involving NaN | bhi => target_label @@ -4775,7 +4781,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z } else { | bvs >1 if (exit_addr) { - | brk #0 // TODO + | NIY // TODO } else { | blo => target_label } @@ -4785,7 +4791,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { if (exit_addr) { - | brk #0 // TODO + | NIY // TODO } else { | bvs >1 // Always False if involving NaN | bhs => target_label @@ -4794,7 +4800,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z } else { | bvs >1 if (exit_addr) { - | brk #0 // TODO + | NIY // TODO } else { | bls => target_label } @@ -4805,7 +4811,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z ZEND_UNREACHABLE(); } } else if (smart_branch_opcode == ZEND_JMPZNZ) { - | brk #0 // TODO + | NIY // TODO } else if (smart_branch_opcode == ZEND_JMPZ_EX) { switch (opline->opcode) { case ZEND_IS_EQUAL: @@ -4989,10 +4995,10 @@ static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, z bool swap = 0; if (Z_MODE(op1_addr) == IS_REG) { - | brk #0 // TODO + | NIY // TODO | DOUBLE_CMP fcmp, Z_REG(op1_addr), op2_addr, ZREG_TMP1, ZREG_FPTMP } else if (Z_MODE(op2_addr) == IS_REG) { - | brk #0 // TODO: construct test cases to verify whether the missing NaN check in x86 is buggy or not. + | NIY // TODO: construct test cases to verify whether the missing NaN check in x86 is buggy or not. | DOUBLE_CMP fcmp, Z_REG(op2_addr), op1_addr, ZREG_TMP1, ZREG_FPTMP swap = 1; } else { @@ -5011,7 +5017,7 @@ static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_a if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ_EX || smart_branch_opcode == ZEND_JMPNZ_EX) { - | brk #0 // TODO + | NIY // TODO } if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) { @@ -5019,33 +5025,33 @@ static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_a case ZEND_IS_EQUAL: case ZEND_CASE: if (exit_addr) { - | brk #0 // TODO + | NIY // TODO } else { | bne => target_label } break; case ZEND_IS_NOT_EQUAL: - | brk #0 // TODO + | NIY // TODO break; case ZEND_IS_SMALLER: - | brk #0 // TODO + | NIY // TODO break; case ZEND_IS_SMALLER_OR_EQUAL: - | brk #0 // TODO + | NIY // TODO break; default: ZEND_UNREACHABLE(); } } else if (smart_branch_opcode == ZEND_JMPNZ || smart_branch_opcode == ZEND_JMPNZ_EX) { - | brk #0 // TODO + | NIY // TODO } else if (smart_branch_opcode == ZEND_JMPZNZ) { - | brk #0 // TODO + | NIY // TODO } else { ZEND_UNREACHABLE(); } } else { - | brk #0 // TODO + | NIY // TODO } return 1; @@ -5081,7 +5087,7 @@ static int zend_jit_cmp(dasm_State **Dst, if (op1_info & MAY_BE_DOUBLE) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4, TMP1w, TMP2 } else { - | brk #0 // TODO + | NIY // TODO } } if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { @@ -5098,7 +5104,7 @@ static int zend_jit_cmp(dasm_State **Dst, | b >6 |.code } else { - | brk #0 // TODO + | NIY // TODO } } if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { @@ -5115,7 +5121,7 @@ static int zend_jit_cmp(dasm_State **Dst, if (!same_ops) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5, TMP1w, TMP2 } else { - | brk #0 // TODO + | NIY // TODO | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, TMP1w, TMP2 } } @@ -5139,11 +5145,11 @@ static int zend_jit_cmp(dasm_State **Dst, } else if ((op1_info & MAY_BE_DOUBLE) && !(op1_info & MAY_BE_LONG) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { - | brk #0 // TODO + | NIY // TODO } else if ((op2_info & MAY_BE_DOUBLE) && !(op2_info & MAY_BE_LONG) && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { - | brk #0 // TODO + | NIY // TODO } if (has_slow || @@ -5170,10 +5176,10 @@ static int zend_jit_cmp(dasm_State **Dst, } | LOAD_ZVAL_ADDR FCARG2x, op1_addr if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { - | brk #0 // TODO + | NIY // TODO } if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { - | brk #0 // TODO + | NIY // TODO } else { | LOAD_ZVAL_ADDR CARG3, op2_addr } @@ -5219,7 +5225,7 @@ static int zend_jit_identical(dasm_State **Dst, uint32_t identical_label = (uint32_t)-1; uint32_t not_identical_label = (uint32_t)-1; - | brk #0 // TODO + | NIY // TODO return 1; } @@ -5259,7 +5265,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ /* Always TRUE */ if (set_bool) { if (set_bool_not) { - | brk #0 // TODO + | NIY // TODO | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 } else { | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 @@ -5270,7 +5276,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ } } else { /* Always FALSE */ - | brk #0 // TODO + | NIY // TODO } return 1; } @@ -5284,12 +5290,12 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) { /* Always TRUE */ - | brk #0 // TODO + | NIY // TODO } else { if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) { /* Always FALSE */ if (set_bool) { - | brk #0 // TODO + | NIY // TODO } } else { | CMP_ZVAL_TYPE op1_addr, IS_TRUE, TMP1w, TMP2 @@ -5298,9 +5304,9 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ !(op1_info & MAY_BE_UNDEF) && !set_bool) { if (exit_addr) { - | brk #0 // TODO + | NIY // TODO } else if (false_label != (uint32_t)-1) { - | brk #0 // TODO + | NIY // TODO } else { | blt >9 } @@ -5311,13 +5317,13 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ } if (!(op1_info & MAY_BE_TRUE)) { /* It's FALSE */ - | brk #0 // TODO + | NIY // TODO } else { if (exit_addr) { - | brk #0 // TODO + | NIY // TODO } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { if (set_bool) { - | brk #0 // TODO + | NIY // TODO } else { if (true_label != (uint32_t)-1) { | beq =>true_label @@ -5325,13 +5331,13 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ | bne =>false_label jmp_done = 1; } else { - | brk #0 // TODO + | NIY // TODO } } } else if (set_bool) { | cset REG0w, eq if (set_bool_not) { - | brk #0 // TODO + | NIY // TODO | neg REG0w, REG0w | add REG0w, REG0w, #3 } else { @@ -5354,13 +5360,13 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 | beq >1 } else { - | brk #0 // TODO + | NIY // TODO | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, TMP1w, TMP2 } |.cold_code |1: } - | brk #0 // TODO + | NIY // TODO | LOAD_32BIT_VAL FCARG1w, opline->op1.var | SET_EX_OPLINE opline, REG0 | EXT_CALL zend_jit_undefined_op_helper, REG0 @@ -5373,18 +5379,18 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ if (exit_addr) { if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { - | brk #0 // TODO + | NIY // TODO } } else if (false_label != (uint32_t)-1) { - | brk #0 // TODO + | NIY // TODO } if (op1_info & MAY_BE_ANY) { if (exit_addr) { if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { - | brk #0 // TODO + | NIY // TODO } } else if (false_label == (uint32_t)-1) { - | brk #0 // TODO + | NIY // TODO } |.code } @@ -5392,9 +5398,9 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ if (!jmp_done) { if (exit_addr) { - | brk #0 // TODO + | NIY // TODO } else if (false_label != (uint32_t)-1) { - | brk #0 // TODO + | NIY // TODO } else if (op1_info & MAY_BE_LONG) { | b >9 } @@ -5408,7 +5414,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, TMP1w, TMP2 } if (Z_MODE(op1_addr) == IS_REG) { - | brk #0 // TODO + | NIY // TODO | tst Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) } else { | LONG_CMP_WITH_CONST cmp, op1_addr, Z_L(0), TMP1, TMP2 @@ -5419,20 +5425,20 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ | neg REG0w, REG0w | add REG0w, REG0w, #3 } else { - | brk #0 // TODO + | NIY // TODO | add REG0w, REG0w, #2 } | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 } if (exit_addr) { - | brk #0 // TODO + | NIY // TODO } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { - | brk #0 // TODO + | NIY // TODO } } if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - | brk #0 // TODO + | NIY // TODO } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { |.cold_code @@ -5455,7 +5461,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 | GC_DELREF FCARG1x, TMP1w | bne >3 - | brk #0 // TODO: currently jump to label 3. + | NIY // TODO: currently jump to label 3. // In x86, r0 is used in macro ZVAL_DTOR_FUNC as temporary register, hence, r0 should be saved/restored // before/after this macro. In AArch64, TMP1 is used. As a result, we needn't save/resotre REG0. | ZVAL_DTOR_FUNC op1_info, opline, TMP1 @@ -5468,7 +5474,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ if (set_bool) { if (set_bool_not) { - | brk #0 // TODO + | NIY // TODO | neg REG0w, REG0w | add REG0w, REG0w, #3 } else { @@ -5476,11 +5482,11 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ } | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 if (exit_addr) { - | brk #0 // TODO + | NIY // TODO } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { | CMP_ZVAL_TYPE res_addr, IS_FALSE, TMP1w, TMP2 if (true_label != (uint32_t)-1) { - | brk #0 // TODO + | NIY // TODO | bne =>true_label if (false_label != (uint32_t)-1) { | b =>false_label @@ -5496,7 +5502,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ |.code } } else { - | brk #0 // TODO + | NIY // TODO if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { |.code @@ -5583,7 +5589,7 @@ static int zend_jit_stack_check(dasm_State **Dst, const zend_op *opline, uint32_ return 0; } - | brk #0 // TODO + | NIY // TODO return 1; } @@ -5608,7 +5614,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con | tst TMP1w, #1 | bne >1 } else { - | brk #0 // TODO: test + | NIY // TODO: test | LOAD_32BIT_VAL FCARG1w, used_stack } | // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval); @@ -5622,7 +5628,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con | ldr TMP1w, [REG0, #offsetof(zend_function, op_array.T)] | sub REG2w, REG2w, TMP1w } else { - | brk #0 // TODO + | NIY // TODO } | lsl REG2w, REG2w, #5 | sxtw REG2, REG2w @@ -5661,7 +5667,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con | blt >1 |.cold_code |1: - | brk #0 // TODO: test. Cold. + | NIY // TODO: test. Cold. | EXT_JMP exit_addr, TMP1 |.code } else { @@ -5670,19 +5676,19 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con |.cold_code |1: if (func) { - | brk #0 // TODO + | NIY // TODO } if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { - | brk #0 // TODO + | NIY // TODO } else { if (!is_closure) { - | brk #0 // TODO + | NIY // TODO } else { - | brk #0 // TODO + | NIY // TODO } - | brk #0 // TODO + | NIY // TODO } - | brk #0 // TODO + | NIY // TODO |.code } } @@ -5714,13 +5720,13 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con && op_array == &func->op_array && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) && (sizeof(void*) != 8 || IS_SIGNED_32BIT(func))) { - | brk #0 // TODO + | NIY // TODO } else { | str REG0, EX:RX->func } } else { | // call->func = &closure->func; - | brk #0 // TODO + | NIY // TODO } |1: } @@ -5731,9 +5737,9 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con if (opline->op1_type == IS_UNUSED || use_this) { | // call->call_info |= ZEND_CALL_HAS_THIS; if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - | brk #0 // TODO + | NIY // TODO } else { - | brk #0 // TODO + | NIY // TODO } } else { if (opline->op1_type == IS_CV) { @@ -5742,7 +5748,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con } | // call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS; if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - | brk #0 // TODO + | NIY // TODO } else { | ldr TMP1w, EX:RX->This.u1.type_info | LOAD_32BIT_VAL TMP2w, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) @@ -5754,7 +5760,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con | // Z_CE(call->This) = called_scope; | str xzr, EX:RX->This.value.ptr } else { - | brk #0 // TODO + | NIY // TODO } | // ZEND_CALL_NUM_ARGS(call) = num_args; | LOAD_32BIT_VAL TMP1w, opline->extended_value @@ -5948,7 +5954,7 @@ static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zen int32_t exit_point; const void *exit_addr; - | brk #0 // TODO + | NIY // TODO return 1; } @@ -5960,7 +5966,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t zend_function *func = NULL; if (delayed_call_chain) { - | brk #0 // TODO + | NIY // TODO } if (info) { @@ -6025,7 +6031,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t | mov REG0, RETVALx | str REG0, [REG1, #opline->result.num] if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - | brk #0 // TODO. tracing mode. + | NIY // TODO. tracing mode. } else { | cbnz REG0, >3 | // SAVE_OPLINE(); @@ -6096,7 +6102,7 @@ static int zend_jit_init_method_call(dasm_State **Dst, if (opline->op1_type == IS_UNUSED || use_this) { zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); - | brk #0 // TODO + | NIY // TODO | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 } else { if (op1_info & MAY_BE_REF) { @@ -6109,7 +6115,7 @@ static int zend_jit_init_method_call(dasm_State **Dst, } else { /* Hack: Convert reference to regular value to simplify JIT code */ ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP); - | brk #0 // TODO + | NIY // TODO } } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { @@ -6120,12 +6126,12 @@ static int zend_jit_init_method_call(dasm_State **Dst, if (!exit_addr) { return 0; } - | brk #0 // TODO + | NIY // TODO } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, TMP1w, TMP2 |.cold_code |1: - | brk #0 // TODO: currently not jump to cold code. + | NIY // TODO: currently not jump to cold code. if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr } @@ -6152,7 +6158,7 @@ static int zend_jit_init_method_call(dasm_State **Dst, if (func) { | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); - | brk #0 // TODO + | NIY // TODO } else { | // if (CACHED_PTR(opline->result.num) == obj->ce)) { | ldr REG0, EX->run_time_cache @@ -6162,7 +6168,7 @@ static int zend_jit_init_method_call(dasm_State **Dst, | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] | cmp REG2, TMP1 | bne >1 - | brk #0 // TODO: currently jump to label 1. + | NIY // TODO: currently jump to label 1. | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); | LOAD_32BIT_VAL TMP1w, (opline->result.num + sizeof(void*)) | add TMP1, TMP1, REG0 @@ -6208,9 +6214,9 @@ static int zend_jit_init_method_call(dasm_State **Dst, !func->common.function_name)) { const zend_op *opcodes = func->op_array.opcodes; - | brk #0 // TODO + | NIY // TODO } else { - | brk #0 // TODO + | NIY // TODO } } @@ -6225,11 +6231,11 @@ static int zend_jit_init_method_call(dasm_State **Dst, } if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) { - | brk #0 // TODO + | NIY // TODO } if (!func) { - | brk #0 // TODO + | NIY // TODO | b >9 |.code } @@ -6270,7 +6276,7 @@ static int zend_jit_init_closure_call(dasm_State **Dst, zend_function *func = NULL; zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); - | brk #0 // TODO + | NIY // TODO return 1; } @@ -6351,7 +6357,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } if (trace && !func) { - | brk #0 // TODO + | NIY // TODO } bool may_have_extra_named_params = @@ -6379,7 +6385,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend if (!exit_addr) { return 0; } - | brk #0 // TODO + | NIY // TODO } } } @@ -6411,7 +6417,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | bne >1 |.cold_code |1: - | brk #0 // TODO + | NIY // TODO if (!GCC_GLOBAL_REGS) { | mov FCARG1x, RX } @@ -6425,7 +6431,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend |1: } } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { - | brk #0 + | NIY // TODO } } @@ -6553,7 +6559,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | EXT_CALL zend_observer_fcall_begin, REG0 } #ifdef CONTEXT_THREADED_JIT - | brk #0 // TODO + | NIY // TODO #else | b =>num_args #endif @@ -6604,7 +6610,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | bgt >1 |.cold_code |1: - | brk #0 // TDOO: test + | NIY // TODO: test |.code if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) { if (!func) { @@ -6642,7 +6648,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } if (ZEND_OBSERVER_ENABLED) { - | brk #0 // TODO: test + | NIY // TODO: test | SAVE_IP | mov FCARG1x, FP | EXT_CALL zend_observer_fcall_begin, REG0 @@ -6650,11 +6656,11 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend if (trace) { if (!func && (opline->opcode != ZEND_DO_UCALL)) { - | brk #0 // TODO + | NIY // TODO } } else { #ifdef CONTEXT_THREADED_JIT - | brk #0 // TODO: CONTEXT_THREADED_JIT is always undefined. + | NIY // TODO: CONTEXT_THREADED_JIT is always undefined. #else if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | ADD_HYBRID_SPAD @@ -6687,7 +6693,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend if (!exit_addr) { return 0; } - | brk #0 // TODO + | NIY // TODO } else { || ZEND_ASSERT(ZEND_ACC_DEPRECATED <= MAX_IMM12); | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] @@ -6695,7 +6701,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | bne >1 |.cold_code |1: - | brk #0 // TODO + | NIY // TODO if (!GCC_GLOBAL_REGS) { | mov FCARG1x, RX } @@ -6709,7 +6715,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend |1: } } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { - | brk #0 // TODO + | NIY // TODO } } @@ -6746,7 +6752,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | EXT_CALL zend_jit_vm_stack_free_args_helper, REG0 } if (may_have_extra_named_params) { - | brk #0 // TODO + | NIY // TODO } |8: @@ -6829,7 +6835,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } else if (trace && trace->op == ZEND_JIT_TRACE_END && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { - | brk #0 // TODO + | NIY // TODO } } @@ -6870,7 +6876,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t o if (!exit_addr) { return 0; } - | brk #0 // TODO + | NIY // TODO } else { | ldr REG0, EX:RX->func | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] @@ -6879,7 +6885,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t o | bne >1 |.cold_code |1: - | brk #0 // TODO + | NIY // TODO | SET_EX_OPLINE opline, REG0 | b ->throw_cannot_pass_by_ref |.code @@ -6893,7 +6899,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t o | ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 if (Z_REFCOUNTED_P(zv)) { - | brk #0 // TODO: test + | NIY // TODO: test } } else { | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 @@ -6904,7 +6910,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t o static int zend_jit_check_undef_args(dasm_State **Dst, const zend_op *opline) { - | brk #0 // TODO + | NIY // TODO return 1; } @@ -6933,7 +6939,7 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend } else if (opline->op1_type == IS_CV) { if (op1_info & MAY_BE_UNDEF) { if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { - | brk #0 // TODO + | NIY // TODO } op1_info &= ~MAY_BE_UNDEF; op1_info |= MAY_BE_NULL; @@ -6975,7 +6981,7 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend | SET_ZVAL_PTR val_addr, REG0, TMP1 | SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX, TMP1w, TMP2 } else { - | brk #0 // TODO + | NIY // TODO } | SET_ZVAL_PTR arg_addr, REG0, TMP1 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX, TMP1w, TMP2 @@ -7024,7 +7030,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend | bne >1 |.cold_code |1: - | brk #0 // TODO + | NIY // TODO if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { return 0; } @@ -7036,7 +7042,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { - | brk #0 // TODO + | NIY // TODO } else { uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); @@ -7047,7 +7053,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend | bne >1 |.cold_code |1: - | brk #0 // TODO + | NIY // TODO |.code } } else if (opline->opcode == ZEND_SEND_FUNC_ARG) { @@ -7083,7 +7089,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend |1: } - | brk #0 // TODO: test + | NIY // TODO: test | SET_EX_OPLINE opline, REG0 | LOAD_32BIT_VAL FCARG1w, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, REG0 @@ -7091,16 +7097,16 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend | cbz RETVALx, ->exception_handler if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { - | brk #0 // TODO: test + | NIY // TODO: test | b >7 |.code } else { - | brk #0 // TODO: test + | NIY // TODO: test } } if (opline->opcode == ZEND_SEND_VAR_NO_REF) { - | brk #0 // TODO: test + | NIY // TODO: test } else { if (op1_info & MAY_BE_REF) { if (opline->op1_type == IS_CV) { @@ -7118,14 +7124,14 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, TMP1w, TMP2 |.cold_code |1: - | brk #0 // TODO: test. cold-code. not covered currently + | NIY // TODO: test. cold-code. not covered currently |.code | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 |2: } } else { if (op1_addr != op1_def_addr) { - | brk #0 // TODO: test + | NIY // TODO: test } | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 if (opline->op1_type == IS_CV) { @@ -7152,7 +7158,7 @@ static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { - | brk #0 // TODO + | NIY // TODO } else { // if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) { uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); @@ -7189,14 +7195,14 @@ static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) { - | brk #0 // TODO + | NIY // TODO return 1; } static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label) { - | brk #0 // TODO + | NIY // TODO return 1; } @@ -7208,7 +7214,7 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, zend_uchar zval *zv = RT_CONSTANT(opline, opline->op1); zend_jit_addr res_addr = 0; - | brk #0 // TODO + | NIY // TODO return 1; } @@ -7222,15 +7228,15 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE); if (op1_info & MAY_BE_UNDEF) { - | brk #0 // TODO + | NIY // TODO } if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { mask = opline->extended_value; if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) { - | brk #0 // TODO + | NIY // TODO } else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) { - | brk #0 // TODO + | NIY // TODO } else { bool invert = 0; zend_uchar type; @@ -7258,18 +7264,18 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t } if (op1_info & MAY_BE_REF) { - | brk #0 // TODO + | NIY // TODO } if (type == 0) { - | brk #0 // TODO + | NIY // TODO } else { if (smart_branch_opcode && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - | brk #0 // TODO + | NIY // TODO } else { if (op1_info & MAY_BE_REF) { - | brk #0 // TODO + | NIY // TODO } else { | LOAD_32BIT_VAL TMP1w, (opline->op1.var + 8) | ldrb TMP2w, [FP, TMP1] @@ -7277,17 +7283,17 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t } } if (exit_addr) { - | brk #0 // TODO + | NIY // TODO } else if (smart_branch_opcode) { if (invert) { - | brk #0 // TODO + | NIY // TODO } else { if (smart_branch_opcode == ZEND_JMPZ) { | bne =>target_label } else if (smart_branch_opcode == ZEND_JMPNZ) { - | brk #0 // TODO + | NIY // TODO } else if (smart_branch_opcode == ZEND_JMPZNZ) { - | brk #0 // TODO + | NIY // TODO } else { ZEND_UNREACHABLE(); } @@ -7295,7 +7301,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t } else { zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); - | brk #0 // TODO + | NIY // TODO } } } @@ -7365,7 +7371,7 @@ static int zend_jit_free_cv(dasm_State **Dst, uint32_t info, uint32_t var) static int zend_jit_free_op(dasm_State **Dst, const zend_op *opline, uint32_t info, uint32_t var_offset) { if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { - | brk #0 // TODO + | NIY // TODO } return 1; } @@ -7416,7 +7422,7 @@ static int zend_jit_leave_func(dasm_State **Dst, | LOAD_32BIT_VAL TMP1w, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE) | tst FCARG1w, TMP1w if (trace && trace->op != ZEND_JIT_TRACE_END) { - | brk #0 // TODO: test + | NIY // TODO: test } else { | bne ->leave_function_handler } @@ -7460,7 +7466,7 @@ static int zend_jit_leave_func(dasm_State **Dst, | ldr FP, EX->prev_execute_data if (!left_frame) { - | brk #0 // TODO: teset + | NIY // TODO: teset | // EG(current_execute_data) = execute_data; | MEM_STORE_ZTS str, FP, executor_globals, current_execute_data, REG0 } @@ -7481,7 +7487,7 @@ static int zend_jit_leave_func(dasm_State **Dst, && (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { const zend_op *next_opline = trace->opline; - | brk #0 // TODO: test + | NIY // TODO: test return 1; } else if (may_throw || @@ -7489,7 +7495,7 @@ static int zend_jit_leave_func(dasm_State **Dst, && (op1_info & MAY_BE_RC1) && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) && (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) { - | brk #0 // TODO: test + | NIY // TODO: test } return 1; @@ -7505,14 +7511,14 @@ static int zend_jit_leave_func(dasm_State **Dst, if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | ADD_HYBRID_SPAD #ifdef CONTEXT_THREADED_JIT - | brk #0 // TODO: CONTEXT_THREADED_JIT is always undefined + | NIY // TODO: CONTEXT_THREADED_JIT is always undefined #else | JMP_IP TMP1 #endif } else if (GCC_GLOBAL_REGS) { | add sp, sp, SPAD // stack alignment #ifdef CONTEXT_THREADED_JIT - | brk #0 // TODO + | NIY // TODO #else | JMP_IP TMP1 #endif @@ -7521,7 +7527,7 @@ static int zend_jit_leave_func(dasm_State **Dst, ZEND_UNREACHABLE(); // TODO: context threading can't work without GLOBAL REGS because we have to change // the value of execute_data in execute_ex() - | brk #0 // TODO + | NIY // TODO #else | ldp FP, RX, T2 // restore FP and IP | ldr LR, T4 // restore LR @@ -7556,7 +7562,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o // TODO: This macro is only used in four sites. We should design a test variant to cover it. if (ZEND_OBSERVER_ENABLED) { - | brk #0 // TODO: test + | NIY // TODO: test } // if (!EX(return_value)) @@ -7586,24 +7592,24 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o } if (return_value_used != 1) { if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - | brk #0 // TODO + | NIY // TODO } - | brk #0 // TODO + | NIY // TODO if (RC_MAY_BE_1(op1_info)) { - | brk #0 // TODO + | NIY // TODO } if (return_value_used == -1) { if (jit_return_label >= 0) { - | brk #0 // TODO + | NIY // TODO } else { - | brk #0 // TODO + | NIY // TODO } |.code } } } else if (return_value_used == -1) { if (jit_return_label >= 0) { - | brk #0 // TODO: test + | NIY // TODO: test } else { | beq >9 } @@ -7611,7 +7617,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o if (return_value_used == 0) { |9: - | brk #0 // TODO: test + | NIY // TODO: test return 1; } @@ -7619,13 +7625,13 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o zval *zv = RT_CONSTANT(opline, opline->op1); | ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 if (Z_REFCOUNTED_P(zv)) { - | brk #0 // TODO: test + | NIY // TODO: test } } else if (opline->op1_type == IS_TMP_VAR) { | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } else if (opline->op1_type == IS_CV) { if (op1_info & MAY_BE_REF) { - | brk #0 // TODO + | NIY // TODO op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); } | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 @@ -7638,12 +7644,12 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o | TRY_ADDREF op1_info, REG0w, REG2, TMP1 } else if (return_value_used != 1) { | // if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); - | brk #0 // TODO + | NIY // TODO | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 } } } else { - | brk #0 // TODO + | NIY // TODO } |9: @@ -7660,7 +7666,7 @@ static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, ze | IF_NOT_REFCOUNTED TMP1w, >2 | and TMP2w, REG2w, #0xff // TMP2w -> low 8 bits of REG2w | IF_NOT_TYPE TMP2w, IS_REFERENCE, >1 - | brk #0 // TODO + | NIY // TODO | add TMP3, REG1, #offsetof(zend_reference, val) | GET_Z_TYPE_INFO REG2w, TMP3 | GET_Z_PTR REG1, TMP3 @@ -7808,13 +7814,13 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, } if (op1_info & MAY_BE_REF) { - | brk #0 // TODO + | NIY // TODO op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); } if (op1_info & MAY_BE_ARRAY) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { - | brk #0 // TODO + | NIY // TODO } | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr, TMP1 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, res_exit_addr, not_found_exit_addr, exit_addr)) { @@ -7828,7 +7834,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, |7: } - | brk #0 // TODO + | NIY // TODO if (op1_info & MAY_BE_ARRAY) { |.code @@ -7842,9 +7848,9 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, if (res_exit_addr) { zend_uchar type = concrete_type(res_info); - | brk #0 // TODO + | NIY // TODO } else if (op1_info & MAY_BE_ARRAY_OF_REF) { - | brk #0 // TODO + | NIY // TODO if (!zend_jit_zval_copy_deref(Dst, res_addr, val_addr, ZREG_REG2)) { return 0; } @@ -7892,7 +7898,7 @@ static int zend_jit_fetch_dim(dasm_State **Dst, op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; if (op1_info & MAY_BE_REF) { - | brk #0 // TODO + | NIY // TODO } if (op1_info & MAY_BE_ARRAY) { @@ -7908,12 +7914,12 @@ static int zend_jit_fetch_dim(dasm_State **Dst, |7: } if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { - | brk #0 // TODO + | NIY // TODO } if ((op1_info & MAY_BE_UNDEF) && opline->opcode == ZEND_FETCH_DIM_RW) { if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { - | brk #0 // TODO + | NIY // TODO | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, TMP1w, TMP2 } | SET_EX_OPLINE opline, REG0 @@ -7923,18 +7929,18 @@ static int zend_jit_fetch_dim(dasm_State **Dst, } | // ZVAL_ARR(container, zend_new_array(8)); if (Z_REG(op1_addr) != ZREG_FP) { - | brk #0 // TODO + | NIY // TODO } | EXT_CALL _zend_new_array_0, REG0 | mov REG0, RETVALx if (Z_REG(op1_addr) != ZREG_FP) { - | brk #0 // TODO + | NIY // TODO } | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 | mov FCARG1x, REG0 if (op1_info & MAY_BE_ARRAY) { - | brk #0 // TODO + | NIY // TODO | b >1 |.code |1: @@ -7944,7 +7950,7 @@ static int zend_jit_fetch_dim(dasm_State **Dst, if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { |6: if (opline->op2_type == IS_UNUSED) { - | brk #0 // TODO + | NIY // TODO } else { uint32_t type; @@ -7974,14 +7980,14 @@ static int zend_jit_fetch_dim(dasm_State **Dst, if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { |.cold_code |9: - | brk #0 // TODO + | NIY // TODO |.code } } } if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { - | brk #0 // TODO + | NIY // TODO } #ifdef ZEND_JIT_USE_RC_INFERENCE @@ -8019,7 +8025,7 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, // TODO: support for empty() ??? ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); - | brk #0 // TODO + | NIY // TODO return 1; } @@ -8028,7 +8034,7 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_ zend_jit_addr op1_addr = OP1_ADDR(); zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2)); - | brk #0 // TODO + | NIY // TODO return 1; } @@ -8040,7 +8046,7 @@ static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zen zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1x : ZREG_REG0; if (ZEND_ARG_SEND_MODE(arg_info)) { - | brk #0 // TODO + | NIY // TODO } if (type_mask != 0) { @@ -8063,13 +8069,13 @@ static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zen in_cold = 1; } - | brk #0 // TODO: currently in cold code + | NIY // TODO: currently in cold code if (Z_REG(res_addr) != ZREG_FCARG1x || Z_OFFSET(res_addr) != 0) { - | brk #0 // TODO + | NIY // TODO | LOAD_ZVAL_ADDR FCARG1x, res_addr } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - | brk #0 // TODO + | NIY // TODO | SET_EX_OPLINE opline, REG0 } else { | ADDR_STORE EX->opline, opline, REG0 @@ -8079,7 +8085,7 @@ static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zen | mov REG0w, RETVALw if (check_exception) { - | brk #0 // TODO + | NIY // TODO | and REG0w, REG0w, #0xff | tst REG0w, REG0w if (in_cold) { @@ -8091,7 +8097,7 @@ static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zen | beq ->exception_handler } } else if (in_cold) { - | brk #0 // TODO + | NIY // TODO | b >1 |.code |1: @@ -8127,12 +8133,12 @@ static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_ if (!exit_addr) { return 0; } - | brk #0 // TODO + | NIY // TODO } else { | blt >1 |.cold_code |1: - | brk #0 // TODO + | NIY // TODO |.code } } @@ -8159,7 +8165,7 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zen zval *zv = RT_CONSTANT(opline, opline->op2); zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); - | brk #0 // TODO + | NIY // TODO return 1; } @@ -8255,7 +8261,7 @@ static int zend_jit_class_guard(dasm_State **Dst, const zend_op *opline, zend_cl return 0; } - | brk #0 // TODO + | NIY // TODO return 1; } @@ -8296,7 +8302,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, && opline->opcode == ZEND_FETCH_OBJ_W && (op1_info & MAY_BE_INDIRECT) && Z_REG(op1_addr) == ZREG_FP) { - | brk #0 // TODO + | NIY // TODO |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); } @@ -8315,7 +8321,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, if (!exit_addr) { return 0; } - | brk #0 // TODO + | NIY // TODO } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7, TMP1w, TMP2 } @@ -8349,33 +8355,33 @@ static int zend_jit_fetch_obj(dasm_State **Dst, | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] | cmp REG2, TMP1 | bne >5 - | brk #0 // TODO: currently jump to Label 5. + | NIY // TODO: currently jump to Label 5. | LOAD_32BIT_VAL TMP1, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) | add TMP1, TMP1, REG0 | ldr REG0, [TMP1] may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); if (may_be_dynamic) { - | brk #0 // TODO + | NIY // TODO | tst REG0, REG0 if (opline->opcode == ZEND_FETCH_OBJ_W) { - | brk #0 // TODO + | NIY // TODO | blt >5 } else { - | brk #0 // TODO + | NIY // TODO | blt >8 // dynamic property } } - | brk #0 // TODO + | NIY // TODO prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); if (opline->opcode == ZEND_FETCH_OBJ_W && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS) && (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS))) { uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; - | brk #0 // TODO + | NIY // TODO |.cold_code |1: - | brk #0 // TODO + | NIY // TODO |.code } } else { @@ -8391,7 +8397,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, if (!exit_addr) { return 0; } - | brk #0 // TODO + | NIY // TODO } } else { | and TMP1w, REG2w, #0xff // low 8 bits. 8-15 bits are used later in zend_jit_zval_copy_deref(). @@ -8402,7 +8408,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, && ZEND_TYPE_IS_SET(prop_info->type)) { uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; - | brk #0 // TODO + | NIY // TODO } } if (op1_avoid_refcounting) { @@ -8412,10 +8418,10 @@ static int zend_jit_fetch_obj(dasm_State **Dst, if (opline->opcode == ZEND_FETCH_OBJ_W) { if (Z_REG(prop_addr) != ZREG_FCARG1x || Z_OFFSET(prop_addr) != 0) { - | brk #0 // TODO + | NIY // TODO | LOAD_ZVAL_ADDR FCARG1x, prop_addr } - | brk #0 // TODO + | NIY // TODO | SET_ZVAL_PTR res_addr, FCARG1x, TMP1 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) { @@ -8439,7 +8445,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, flags = ZEND_JIT_EXIT_FREE_OP1; } - | brk #0 // TODO + | NIY // TODO } else { if (!zend_jit_zval_copy_deref(Dst, res_addr, prop_addr, ZREG_REG2)) { return 0; @@ -8457,7 +8463,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, } else if (opline->opcode != ZEND_FETCH_OBJ_IS) { | EXT_CALL zend_jit_fetch_obj_r_slow, REG0 } else { - | brk #0 // TODO + | NIY // TODO | EXT_CALL zend_jit_fetch_obj_is_slow, REG0 } | b >9 @@ -8471,9 +8477,9 @@ static int zend_jit_fetch_obj(dasm_State **Dst, && (op1_info & MAY_BE_UNDEF)) { zend_jit_addr orig_op1_addr = OP1_ADDR(); - | brk #0 // TODO + | NIY // TODO } else if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { - | brk #0 // TODO + | NIY // TODO | LOAD_ZVAL_ADDR FCARG1x, op1_addr } | LOAD_ADDR FCARG2x, Z_STRVAL_P(member) @@ -8481,11 +8487,11 @@ static int zend_jit_fetch_obj(dasm_State **Dst, | EXT_CALL zend_jit_invalid_property_write, REG0 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2 } else { - | brk #0 // TODO + | NIY // TODO } | b >9 } else { - | brk #0 // TODO + | NIY // TODO } } @@ -8493,7 +8499,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, && may_be_dynamic && opline->opcode != ZEND_FETCH_OBJ_W) { |8: - | brk #0 // TODO + | NIY // TODO } |.code; @@ -8504,7 +8510,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, && (op1_info & MAY_BE_RC1)) { zend_jit_addr orig_op1_addr = OP1_ADDR(); - | brk #0 // TODO + | NIY // TODO } else if (!op1_avoid_refcounting) { | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 } @@ -8551,7 +8557,7 @@ static int zend_jit_incdec_obj(dasm_State **Dst, ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); - | brk #0 // TODO + | NIY // TODO return 1; } @@ -8590,17 +8596,17 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, prop_info = zend_get_known_property_info(op_array, ce, name, opline->op1_type == IS_UNUSED, op_array->filename); if (opline->op1_type == IS_UNUSED || use_this) { - | brk #0 // TODO + | NIY // TODO | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 } else { if (opline->op1_type == IS_VAR && (op1_info & MAY_BE_INDIRECT) && Z_REG(op1_addr) == ZREG_FP) { - | brk #0 // TODO + | NIY // TODO op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); } if (op1_info & MAY_BE_REF) { - | brk #0 // TODO + | NIY // TODO op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { @@ -8611,9 +8617,9 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, if (!exit_addr) { return 0; } - | brk #0 // TODO + | NIY // TODO } else { - | brk #0 // TODO + | NIY // TODO } } | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 @@ -8645,7 +8651,7 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, if (!prop_info) { needs_slow_path = 1; - | brk #0 // TODO + | NIY // TODO prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); } else { prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, prop_info->offset); @@ -8656,7 +8662,7 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, if (!exit_addr) { return 0; } - | brk #0 // TODO + | NIY // TODO } else { | LOAD_32BIT_VAL TMP1w, (prop_info->offset + 8) | ldrb TMP2w, [FCARG1x, TMP1] @@ -8666,7 +8672,7 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, if (ZEND_TYPE_IS_SET(prop_info->type)) { uint32_t info = val_info; - | brk #0 // TODO + | NIY // TODO } } @@ -8676,7 +8682,7 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); - | brk #0 + | NIY // TODO } if (needs_slow_path) { @@ -8754,12 +8760,12 @@ static int zend_jit_assign_obj(dasm_State **Dst, prop_info = zend_get_known_property_info(op_array, ce, name, opline->op1_type == IS_UNUSED, op_array->filename); if (opline->op1_type == IS_UNUSED || use_this) { - | brk #0 // TODO + | NIY // TODO } else { if (opline->op1_type == IS_VAR && (op1_info & MAY_BE_INDIRECT) && Z_REG(op1_addr) == ZREG_FP) { - | brk #0 // TODO + | NIY // TODO |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); } @@ -8778,12 +8784,12 @@ static int zend_jit_assign_obj(dasm_State **Dst, if (!exit_addr) { return 0; } - | brk #0 // TODO + | NIY // TODO } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, TMP1w, TMP2 |.cold_code |1: - | brk #0 // TODO + | NIY // TODO |.code } } @@ -8825,12 +8831,12 @@ static int zend_jit_assign_obj(dasm_State **Dst, | cmp REG2, TMP1 | bne >5 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { - | brk #0 // TODO + | NIY // TODO } - | brk #0 // TODO + | NIY // TODO prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { - | brk #0 // TODO + | NIY // TODO } } else { prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, prop_info->offset); @@ -8843,9 +8849,9 @@ static int zend_jit_assign_obj(dasm_State **Dst, if (!exit_addr) { return 0; } - | brk #0 // TODO + | NIY // TODO } else { - | brk #0 // TODO + | NIY // TODO needs_slow_path = 1; } } @@ -8853,7 +8859,7 @@ static int zend_jit_assign_obj(dasm_State **Dst, uint32_t info = val_info; | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); - | brk #0 // TODO + | NIY // TODO } } @@ -8882,7 +8888,7 @@ static int zend_jit_assign_obj(dasm_State **Dst, | LOAD_32BIT_VAL TMP1w, opline->extended_value | add CARG4, CARG4, TMP1 if (RETURN_VALUE_USED(opline)) { - | brk #0 // TODO + | NIY // TODO | LOAD_ZVAL_ADDR CARG5, res_addr } else { | mov CARG5, xzr @@ -8921,15 +8927,15 @@ static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_i if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { if (may_throw) { - | brk #0 // TODO + | NIY // TODO | SET_EX_OPLINE opline, REG0 } if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) { if (op1_info & MAY_BE_ARRAY) { - | brk #0 // TODO + | NIY // TODO | IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7, TMP1w, TMP2 } - | brk #0 // TODO + | NIY // TODO | LOAD_32BIT_VAL TMP1w, opline->op1.var + offsetof(zval, u2.fe_iter_idx) | ldr FCARG1w, [FP, TMP1] | LOAD_32BIT_VAL TMP1w, -1 @@ -8978,7 +8984,7 @@ static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_i ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); - | brk #0 // TODO: test + | NIY // TODO: test } return 1; } @@ -8987,7 +8993,7 @@ static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1 { zend_jit_addr res_addr = RES_ADDR(); - | brk #0 // TODO + | NIY // TODO return 1; } @@ -8995,7 +9001,7 @@ static int zend_jit_count(dasm_State **Dst, const zend_op *opline, uint32_t op1_ { zend_jit_addr res_addr = RES_ADDR(); - | brk #0 // TODO + | NIY // TODO if (may_throw) { return zend_jit_check_exception(Dst); @@ -9024,14 +9030,14 @@ static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const ze int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); - | brk #0 // TODO + | NIY // TODO if (JIT_G(current_frame)) { TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame)); } } } else { - | brk #0 // TODO + | NIY // TODO } } @@ -9052,7 +9058,7 @@ static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend int32_t exit_point; const void *exit_addr; - | brk #0 // TODO + | NIY // TODO return 1; } @@ -9067,7 +9073,7 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o next_opline = trace->opline; } - | brk #0 // TODO + | NIY // TODO return 1; } @@ -9080,7 +9086,7 @@ static bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, bool slow_check_in_cold = 1; uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; - | brk #0 // TODO + | NIY // TODO return 1; } @@ -9088,7 +9094,7 @@ static int zend_jit_isset_isempty_cv(dasm_State **Dst, const zend_op *opline, ui { zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); - | brk #0 // TODO + | NIY // TODO return 1; } @@ -9101,12 +9107,12 @@ static int zend_jit_fe_reset(dasm_State **Dst, const zend_op *opline, uint32_t o | ZVAL_COPY_CONST res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 if (Z_REFCOUNTED_P(zv)) { - | brk #0 // TODO + | NIY // TODO } } else { zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); - | brk #0 // TODO + | NIY // TODO } | // Z_FE_POS_P(res) = 0; | LOAD_32BIT_VAL TMP1w, (opline->result.var + offsetof(zval, u2.fe_pos)) @@ -9138,7 +9144,7 @@ static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t o | // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); | // ZEND_VM_CONTINUE(); if (exit_addr) { - | brk #0 // TODO + | NIY // TODO } else { | bls =>target_label } @@ -9149,7 +9155,7 @@ static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t o if (!exit_addr || exit_opcode == ZEND_JMP) { | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, >3, TMP1w } else { - | brk #0 // TODO + | NIY // TODO } | // p++; | add FCARG2x, FCARG2x, #sizeof(Bucket) @@ -9168,7 +9174,7 @@ static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t o if (RETURN_VALUE_USED(opline)) { zend_jit_addr res_addr = RES_ADDR(); - | brk #0 // TODO + | NIY // TODO } val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT); @@ -9188,7 +9194,7 @@ static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t o return 0; } } else { - | brk #0 // TODO + | NIY // TODO } } @@ -9212,7 +9218,7 @@ static int zend_jit_fetch_constant(dasm_State **Dst, | ldr REG0, [FCARG1x, TMP1] | // if (c != NULL) | cbz REG0, >9 - | brk #0 // TODO + | NIY // TODO | // if (!IS_SPECIAL_CACHE_VAL(c)) | tst REG0, #CACHE_SPECIAL | bne >9 @@ -9237,7 +9243,7 @@ static int zend_jit_fetch_constant(dasm_State **Dst, zend_uchar type = concrete_type(res_info); - | brk #0 // TODO + | NIY // TODO } else { | ZVAL_COPY_VALUE res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 | lsr REG0w, REG0w, #8 @@ -9256,7 +9262,7 @@ static int zend_jit_fetch_constant(dasm_State **Dst, | mov REG0, RETVALx | // ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); | cbnz REG0, <8 - | brk #0 // TODO + | NIY // TODO | b ->exception_handler |.code return 1; @@ -9270,7 +9276,7 @@ static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t o ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR); ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING); - | brk #0 // TODO + | NIY // TODO return 1; } @@ -9282,7 +9288,7 @@ static bool zend_jit_noref_guard(dasm_State **Dst, const zend_op *opline, zend_j if (!exit_addr) { return 0; } - | brk #0 // IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr + | NIY // IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr return 1; } @@ -9303,7 +9309,7 @@ static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, ui } if (add_ref_guard) { - | brk #0 // TODO + | NIY // TODO } if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) { /* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */ @@ -9312,7 +9318,7 @@ static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, ui } | EXT_CALL zend_jit_unref_helper, REG0 } else { - | brk #0 // GET_ZVAL_PTR FCARG1x, var_addr + | NIY // GET_ZVAL_PTR FCARG1x, var_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, offsetof(zend_reference, val)); *var_addr_ptr = var_addr; } @@ -9323,7 +9329,7 @@ static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, ui if (add_type_guard && var_type != IS_UNKNOWN && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { - | brk #0 // TODO + | NIY // TODO ZEND_ASSERT(var_info & (1 << var_type)); if (var_type < IS_STRING) { @@ -9357,7 +9363,7 @@ static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, if (!exit_addr) { return 0; } - | brk #0 // TODO + | NIY // TODO } else { /* May be already loaded into FCARG1a or RAX by previus FETCH_OBJ_W/DIM_W */ if (opline->op1_type != IS_VAR || @@ -9365,9 +9371,9 @@ static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, (opline-1)->result.var != opline->op1.var || (opline-1)->op2_type == IS_VAR || (opline-1)->op2_type == IS_TMP_VAR) { - | brk #0 // GET_ZVAL_PTR FCARG1x, var_addr + | NIY // GET_ZVAL_PTR FCARG1x, var_addr } else if ((opline-1)->opcode == ZEND_FETCH_DIM_W || (opline-1)->opcode == ZEND_FETCH_DIM_RW) { - | brk #0 // TODO + | NIY // TODO } } *var_info_ptr &= ~MAY_BE_INDIRECT; @@ -9387,7 +9393,7 @@ static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, return 0; } - | brk #0 // TODO + | NIY // TODO //var_info = zend_jit_trace_type_to_info_ex(var_type, var_info); ZEND_ASSERT(var_info & (1 << var_type)); From ea15f908330c4536f297b1b0e7d5198a8dc88431 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 20 Apr 2021 11:06:06 +0300 Subject: [PATCH 079/229] Use NIY_STUB instead of NIY in stubs --- ext/opcache/jit/zend_jit_arm64.dasc | 57 ++++++++++++++++------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 9e9314174392f..924289b62a723 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -164,6 +164,11 @@ static void* dasm_labels[zend_lb_MAX]; | brk #0 |.endmacro +|.macro NIY_STUB +|| //ZEND_ASSERT(0); +| brk #0 +|.endmacro + /* In x86/64, HYBRID_SPAD bytes are reserved on the stack only if flag ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE * is not defined, because the 16-byte redzone, allocated on the stack when the flag is defined, can be * reused. In AArch64, it's safe that these bytes are always reserved because the stack layout might @@ -1390,7 +1395,7 @@ static inline bool is_signed(double d) static int zend_jit_interrupt_handler_stub(dasm_State **Dst) { |->interrupt_handler: - | NIY // TODO + | NIY_STUB // TODO return 1; } @@ -1411,7 +1416,7 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) | add sp, sp, SPAD // stack alignment | EXT_JMP handler, REG0 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - | NIY // TODO: test + | NIY_STUB // TODO: test } else { | mov FCARG1x, FP | ldp FP, RX, T2 // retore FP and IP @@ -1427,7 +1432,7 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) static int zend_jit_exception_handler_undef_stub(dasm_State **Dst) { |->exception_handler_undef: - | NIY // TODO + | NIY_STUB // TODO return 1; } @@ -1439,7 +1444,7 @@ static int zend_jit_leave_function_stub(dasm_State **Dst) | LOAD_32BIT_VAL TMP1w, ZEND_CALL_TOP | tst FCARG1w, TMP1w | bne >1 - | NIY // TODO: currently jump to label 1. + | NIY_STUB // TODO: currently jump to label 1. | EXT_CALL zend_jit_leave_nested_func_helper, REG0 | ADD_HYBRID_SPAD | JMP_IP TMP1 @@ -1459,7 +1464,7 @@ static int zend_jit_leave_function_stub(dasm_State **Dst) | LOAD_32BIT_VAL TMP1w, ZEND_CALL_TOP | tst FCARG1w, TMP1w | bne >1 - | NIY // TODO: currently jump to label 1. + | NIY_STUB // TODO: currently jump to label 1. | EXT_JMP zend_jit_leave_nested_func_helper, REG0 |1: | EXT_JMP zend_jit_leave_top_func_helper, REG0 @@ -1471,7 +1476,7 @@ static int zend_jit_leave_function_stub(dasm_State **Dst) static int zend_jit_leave_throw_stub(dasm_State **Dst) { |->leave_throw_handler: - | NIY // TODO: test + | NIY_STUB // TODO: test return 1; } @@ -1479,7 +1484,7 @@ static int zend_jit_leave_throw_stub(dasm_State **Dst) static int zend_jit_icall_throw_stub(dasm_State **Dst) { |->icall_throw_handler: - | NIY // TODO + | NIY_STUB // TODO return 1; } @@ -1487,7 +1492,7 @@ static int zend_jit_icall_throw_stub(dasm_State **Dst) static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) { |->throw_cannot_pass_by_ref: - | NIY // TODO + | NIY_STUB // TODO return 1; } @@ -1495,7 +1500,7 @@ static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) { |->undefined_offset_ex: - | NIY // TODO + | NIY_STUB // TODO return 1; } @@ -1503,7 +1508,7 @@ static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) static int zend_jit_undefined_offset_stub(dasm_State **Dst) { |->undefined_offset: - | NIY // TODO + | NIY_STUB // TODO return 1; } @@ -1520,7 +1525,7 @@ static int zend_jit_undefined_index_ex_stub(dasm_State **Dst) static int zend_jit_undefined_index_stub(dasm_State **Dst) { |->undefined_index: - | NIY // TODO + | NIY_STUB // TODO return 1; } @@ -1528,7 +1533,7 @@ static int zend_jit_undefined_index_stub(dasm_State **Dst) static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) { |->cannot_add_element_ex: - | NIY // TODO + | NIY_STUB // TODO return 1; } @@ -1536,7 +1541,7 @@ static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) static int zend_jit_cannot_add_element_stub(dasm_State **Dst) { |->cannot_add_element: - | NIY // TODO + | NIY_STUB // TODO return 1; } @@ -1581,14 +1586,14 @@ static int zend_jit_mod_by_zero_stub(dasm_State **Dst) static int zend_jit_invalid_this_stub(dasm_State **Dst) { |->invalid_this: - | NIY // TODO + | NIY_STUB // TODO return 1; } static int zend_jit_double_one_stub(dasm_State **Dst) { |->one: - | NIY // TODO + | NIY_STUB // TODO return 1; } @@ -1611,7 +1616,7 @@ static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) } |->hybrid_profile_jit: - | NIY // TODO + | NIY_STUB // TODO return 1; } @@ -1622,7 +1627,7 @@ static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst) } |->hybrid_hot_code: - | NIY // TODO + | NIY_STUB // TODO return 1; } @@ -1652,7 +1657,7 @@ static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst) */ static int zend_jit_hybrid_hot_counter_stub(dasm_State **Dst, uint32_t cost) { - | NIY // TODO + | NIY_STUB // TODO return 1; } @@ -1764,7 +1769,7 @@ static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst) static int zend_jit_trace_halt_stub(dasm_State **Dst) { |->trace_halt: - | NIY // TODO + | NIY_STUB // TODO return 1; } @@ -1842,7 +1847,7 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst) } |1: - | NIY // TODO: test + | NIY_STUB // TODO: test | blt ->trace_halt | // execute_data = EG(current_execute_data) @@ -1897,7 +1902,7 @@ static int zend_jit_trace_escape_stub(dasm_State **Dst) { |->trace_escape: | - | NIY // TODO + | NIY_STUB // TODO return 1; } @@ -1931,7 +1936,7 @@ static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n) static int zend_jit_context_threaded_call_stub(dasm_State **Dst) { |->context_threaded_call: - | NIY // TODO + | NIY_STUB // TODO return 1; } #endif @@ -1955,7 +1960,7 @@ static int zend_jit_assign_const_stub(dasm_State **Dst) uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; |->assign_const: - | NIY // TODO + | NIY_STUB // TODO if (!zend_jit_assign_to_variable( Dst, NULL, var_addr, var_addr, -1, -1, @@ -1994,7 +1999,7 @@ static int zend_jit_assign_var_stub(dasm_State **Dst) uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF; |->assign_var: - | NIY // TODOa + | NIY_STUB // TODOa | ret return 1; } @@ -2006,7 +2011,7 @@ static int zend_jit_assign_cv_noref_stub(dasm_State **Dst) uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/; |->assign_cv_noref: - | NIY // TODO + | NIY_STUB // TODO | ret return 1; } @@ -2018,7 +2023,7 @@ static int zend_jit_assign_cv_stub(dasm_State **Dst) uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/; |->assign_cv: - | NIY // TODO + | NIY_STUB // TODO | ret return 1; } From f24a8a2a242a2ae9b36507db0c17ed56ff51db7c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 20 Apr 2021 13:26:55 +0300 Subject: [PATCH 080/229] Avoid useless "flags" byte extraction (it was inspired by unsuitable x86 specific optimization) --- ext/opcache/jit/zend_jit_arm64.dasc | 28 +++++----------------------- 1 file changed, 5 insertions(+), 23 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 924289b62a723..b1bfd32d0746d 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -958,12 +958,12 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro IF_REFCOUNTED, type_flags, label -| IF_FLAGS type_flags, IS_TYPE_REFCOUNTED, label +| tst type_flags, #(IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT) +| bne label |.endmacro |.macro IF_NOT_REFCOUNTED, type_flags, label -| //IF_NOT_FLAGS type_flags, IS_TYPE_REFCOUNTED, label -| tst type_flags, type_flags +| tst type_flags, #(IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT) | beq label |.endmacro @@ -4013,8 +4013,6 @@ static int zend_jit_simple_assign(dasm_State **Dst, if (val_type == IS_CV) { if (!res_addr) { - | lsr REG2w, REG2w, #8 - | and REG2w, REG2w, #0xff | TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1 } else { | NIY // TODO @@ -7120,8 +7118,6 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend | LOAD_ZVAL_ADDR FCARG1x, op1_addr | ZVAL_DEREF FCARG1x, op1_info, TMP1w | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 - | lsr REG0w, REG0w, #8 - | and REG0w, REG0w, #0xff | TRY_ADDREF op1_info, REG0w, REG2, TMP1 } else { zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 8); @@ -7140,12 +7136,6 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend } | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 if (opline->op1_type == IS_CV) { - | // In x86 implementation, type flags and value pointer would be stored into eax and r2 respectively, - | // and then ah (bits 8 to 15) and r2 are used inside TRY_ADDREF. - | // In AArch64, we use REG0w and REG2 accordingly. - | // Note that, bits 8 to 15 should be extacted, i.e., (REG0w >> 8) & 0xff. - | lsr REG0w, REG0w, #8 - | and REG0w, REG0w, #0xff | TRY_ADDREF op1_info, REG0w, REG2, TMP1 } } @@ -7644,8 +7634,6 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || (op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) || !op_array->function_name) { - | lsr REG0w, REG0w, #8 - | and REG0w, REG0w, #0xff | TRY_ADDREF op1_info, REG0w, REG2, TMP1 } else if (return_value_used != 1) { | // if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); @@ -7666,16 +7654,14 @@ static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, ze ZEND_ASSERT(type_reg == ZREG_REG2); | GET_ZVAL_PTR REG1, val_addr, TMP1 - | lsr TMP1w, REG2w, #8 - | and TMP1w, TMP1w, #0xff // TMP1w -> 8-15 bits of REG2w - | IF_NOT_REFCOUNTED TMP1w, >2 + | IF_NOT_REFCOUNTED REG2w, >2 | and TMP2w, REG2w, #0xff // TMP2w -> low 8 bits of REG2w | IF_NOT_TYPE TMP2w, IS_REFERENCE, >1 | NIY // TODO | add TMP3, REG1, #offsetof(zend_reference, val) | GET_Z_TYPE_INFO REG2w, TMP3 | GET_Z_PTR REG1, TMP3 - | IF_NOT_REFCOUNTED TMP1w, >2 + | IF_NOT_REFCOUNTED REG2w, >2 |1: | GC_ADDREF REG1, TMP1w |2: @@ -7862,8 +7848,6 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, } else { | // ZVAL_COPY | ZVAL_COPY_VALUE res_addr, -1, val_addr, res_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 - | lsr REG1w, REG1w, #8 - | and REG1w, REG1w, #0xff | TRY_ADDREF res_info, REG1w, REG2, TMP1 } } @@ -9251,8 +9235,6 @@ static int zend_jit_fetch_constant(dasm_State **Dst, | NIY // TODO } else { | ZVAL_COPY_VALUE res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 - | lsr REG0w, REG0w, #8 - | and REG0w, REG0w, #0xff | TRY_ADDREF MAY_BE_ANY, REG0w, REG1, TMP1 } From c12157700a01c9cb4ab190951c9572db3471830c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 20 Apr 2021 15:36:37 +0300 Subject: [PATCH 081/229] Trmporary disable PROFITABILITY_CHECKS for better test coverage --- ext/opcache/jit/zend_jit_arm64.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index b1bfd32d0746d..837a557ff8e05 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -138,7 +138,7 @@ static size_t tsrm_tls_offset; * type information. Disabling this option allows testing some JIT handlers in the * presence of try/catch blocks, which prevent SSA construction. */ #ifndef PROFITABILITY_CHECKS -# define PROFITABILITY_CHECKS 1 +# define PROFITABILITY_CHECKS 0 // TODO: temporary disabled for better test coverage #endif |.type EX, zend_execute_data, FP From b12340fc469d8bfa5c5ccb97a62d678788d091c9 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 20 Apr 2021 15:49:17 +0300 Subject: [PATCH 082/229] Added tests for ASSIGN --- ext/opcache/tests/jit/assign_037.phpt | 20 ++++++++++++++++++++ ext/opcache/tests/jit/assign_038.phpt | 20 ++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 ext/opcache/tests/jit/assign_037.phpt create mode 100644 ext/opcache/tests/jit/assign_038.phpt diff --git a/ext/opcache/tests/jit/assign_037.phpt b/ext/opcache/tests/jit/assign_037.phpt new file mode 100644 index 0000000000000..292837ebdae16 --- /dev/null +++ b/ext/opcache/tests/jit/assign_037.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT ASSIGN: Assign refcounted string (with result) +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +string(2) "aa" +string(2) "aa" diff --git a/ext/opcache/tests/jit/assign_038.phpt b/ext/opcache/tests/jit/assign_038.phpt new file mode 100644 index 0000000000000..bcbda7b02798b --- /dev/null +++ b/ext/opcache/tests/jit/assign_038.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT ASSIGN: Assign constant (with result) +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +opcache.optimization_level=0 ; disable optimizer to produce ASSIGN with result +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +string(2) "bb" +string(2) "bb" From 5d5c165a7b485e5f3ae0da797219404f4bc01cce Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 20 Apr 2021 16:54:25 +0300 Subject: [PATCH 083/229] Support for most "uncommon" ASSIGN cases --- ext/opcache/jit/zend_jit_arm64.dasc | 282 +++++++++++++++++++++----- ext/opcache/tests/jit/assign_039.phpt | 18 ++ 2 files changed, 250 insertions(+), 50 deletions(-) create mode 100644 ext/opcache/tests/jit/assign_039.phpt diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 837a557ff8e05..ddb54d5fe07e0 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -819,7 +819,7 @@ static void* dasm_labels[zend_lb_MAX]; | SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 || } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { || zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg; -| NIY // TODO: test +| NIY // TODO: || } else { | // In x64, if the range of this LONG value can be represented via INT type, only move the low 32 bits into dst_addr. | // Note that imm32 is signed extended to 64 bits during mov. @@ -839,8 +839,47 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro -|.macro ZVAL_COPY_CONST_2, dst_addr, res_addr, dst_info, dst_def_info, zv, tmp_reg -| NIY // TODO +|.macro ZVAL_COPY_CONST_2, dst_addr, res_addr, dst_info, dst_def_info, zv, tmp_reg1, tmp_reg2, fp_tmp_reg +|| if (Z_TYPE_P(zv) > IS_TRUE) { +|| if (Z_TYPE_P(zv) == IS_DOUBLE) { +|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? +|| Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_MODE(res_addr) : fp_tmp_reg); +| LOAD_ADDR Rx(tmp_reg1), zv +| ldr Rd(dst_reg-ZREG_V0), [Rx(tmp_reg1)] +| SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 +| SET_ZVAL_DVAL res_addr, dst_reg, tmp_reg2 +|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { +|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg; +| NIY // TODO: +|| } else { +|| if (Z_MODE(dst_addr) == IS_REG) { +| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) +| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(dst_addr)), Rx(tmp_reg1) +|| } else if (Z_MODE(res_addr) == IS_REG) { +| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) +| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(res_addr)), Rx(tmp_reg1) +|| } else { +| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) +| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) +|| } +|| } +|| } +|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { +|| if (dst_def_info == MAY_BE_DOUBLE) { +|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { +| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE, Rx(tmp_reg1), Rx(tmp_reg2) +|| } +|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<1 || } -| add dword [value_ptr_reg], 2 +| ldr tmp_reg, [value_ptr_reg] +| add tmp_reg, tmp_reg, #2 +| str tmp_reg, [value_ptr_reg] |1: || } |.endmacro @@ -1198,12 +1300,21 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro -|.macro EFREE_REG_REFERENCE -| NIY // TODO -|.endmacro - -|.macro EFREE_REFERENCE, ptr -| NIY // TODO +/* argument is passed in FCARG1x */ +|.macro EFREE_REFERENCE +||#if ZEND_DEBUG +| mov FCARG2x, xzr // filename +| mov CARG3w, wzr // lineno +| mov CARG4, xzr +| mov CARG5, xzr +| EXT_CALL _efree, REG0 +||#else +||#ifdef HAVE_BUILTIN_CONSTANT_P +| EXT_CALL _efree_32, REG0 +||#else +| EXT_CALL _efree, REG0 +||#endif +||#endif |.endmacro |.macro EMALLOC, size, op_array, opline @@ -1960,7 +2071,6 @@ static int zend_jit_assign_const_stub(dasm_State **Dst) uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; |->assign_const: - | NIY_STUB // TODO if (!zend_jit_assign_to_variable( Dst, NULL, var_addr, var_addr, -1, -1, @@ -1999,7 +2109,15 @@ static int zend_jit_assign_var_stub(dasm_State **Dst) uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF; |->assign_var: - | NIY_STUB // TODOa + | sub sp, sp, #16 + if (!zend_jit_assign_to_variable( + Dst, NULL, + var_addr, var_addr, -1, -1, + IS_VAR, val_addr, val_info, + 0, 0)) { + return 0; + } + | add sp, sp, #16 | ret return 1; } @@ -2011,7 +2129,15 @@ static int zend_jit_assign_cv_noref_stub(dasm_State **Dst) uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/; |->assign_cv_noref: - | NIY_STUB // TODO + | sub sp, sp, #16 + if (!zend_jit_assign_to_variable( + Dst, NULL, + var_addr, var_addr, -1, -1, + IS_CV, val_addr, val_info, + 0, 0)) { + return 0; + } + | add sp, sp, #16 | ret return 1; } @@ -2023,7 +2149,15 @@ static int zend_jit_assign_cv_stub(dasm_State **Dst) uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/; |->assign_cv: - | NIY_STUB // TODO + | sub sp, sp, #16 + if (!zend_jit_assign_to_variable( + Dst, NULL, + var_addr, var_addr, -1, -1, + IS_CV, val_addr, val_info, + 0, 0)) { + return 0; + } + | add sp, sp, #16 | ret return 1; } @@ -3935,15 +4069,18 @@ static int zend_jit_simple_assign(dasm_State **Dst, if (!res_addr) { | ZVAL_COPY_CONST var_addr, var_info, var_def_info, zv, tmp_reg, ZREG_TMP1, ZREG_FPR0 } else { - | NIY // TODO + | ZVAL_COPY_CONST_2 var_addr, res_addr, var_info, var_def_info, zv, tmp_reg, ZREG_TMP1, ZREG_FPR0 } if (Z_REFCOUNTED_P(zv)) { - | NIY // TODO + if (!res_addr) { + | ADDREF_CONST zv, TMP1, TMP2 + } else { + | ADDREF_CONST_2 zv, TMP1, TMP2 + } } } else { if (val_info & MAY_BE_UNDEF) { if (in_cold) { - | NIY // TODO | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2, TMP1w, TMP2 } else { | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1, TMP1w, TMP2 @@ -3952,12 +4089,10 @@ static int zend_jit_simple_assign(dasm_State **Dst, } | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); if (save_r1) { - | NIY // TODO | str FCARG1x, T1 // save } | SET_ZVAL_TYPE_INFO var_addr, IS_NULL, TMP1w, TMP2 if (res_addr) { - | NIY // TODO | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 } if (opline) { @@ -3967,7 +4102,6 @@ static int zend_jit_simple_assign(dasm_State **Dst, | LOAD_32BIT_VAL FCARG1w, Z_OFFSET(val_addr) | EXT_CALL zend_jit_undefined_op_helper, REG0 if (save_r1) { - | NIY // TODO | ldr FCARG1x, T1 // restore } | b >3 @@ -3989,14 +4123,45 @@ static int zend_jit_simple_assign(dasm_State **Dst, zend_jit_addr ref_addr; if (in_cold) { - | NIY // TODO | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1, TMP1w, TMP2 } else { | IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1, TMP1w, TMP2 |.cold_code |1: } - | NIY // TODO + | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); + | GET_ZVAL_PTR REG2, val_addr, TMP1 + | GC_DELREF REG2, TMP1 + | // ZVAL_COPY_VALUE(return_value, &ref->val); + ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, offsetof(zend_reference, val)); + if (!res_addr) { + | ZVAL_COPY_VALUE var_addr, var_info, ref_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + } else { + | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, ref_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + } + | beq >2 // GC_DELREF() reached zero + | IF_NOT_REFCOUNTED REG2w, >3 + if (!res_addr) { + | GC_ADDREF Rx(tmp_reg), TMP1 + } else { + | GC_ADDREF_2 Rx(tmp_reg), TMP1 + } + | b >3 + |2: + if (res_addr) { + | IF_NOT_REFCOUNTED REG2w, >2 + | GC_ADDREF Rx(tmp_reg), TMP1 + |2: + } + if (save_r1) { + | str FCARG1x, T1 // save + } + | LOAD_ZVAL_ADDR FCARG1x, val_addr + | EFREE_REFERENCE + if (save_r1) { + | ldr FCARG1x, T1 // restore + } + | b >3 if (in_cold) { |1: } else { @@ -4008,18 +4173,18 @@ static int zend_jit_simple_assign(dasm_State **Dst, if (!res_addr) { | ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } else { - | NIY // TODO + | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, val_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } if (val_type == IS_CV) { if (!res_addr) { | TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1 } else { - | NIY // TODO + | TRY_ADDREF_2 val_info, REG2w, Rx(tmp_reg), TMP1 } } else { if (res_addr) { - | NIY // TODO + | TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1 } } |3: @@ -4039,7 +4204,6 @@ static int zend_jit_assign_to_typed_ref(dasm_State **Dst, | bne >2 |.cold_code |2: - | NIY // TODO if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2x || Z_OFFSET(val_addr) != 0) { | LOAD_ZVAL_ADDR FCARG2x, val_addr } @@ -4094,20 +4258,20 @@ static int zend_jit_assign_to_variable_call(dasm_State **Dst, if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { | bl ->assign_tmp } else if (val_type == IS_CONST) { - | NIY // TODO + | bl ->assign_const } else if (val_type == IS_TMP_VAR) { - | NIY // TODO + | bl ->assign_tmp } else if (val_type == IS_VAR) { if (!(val_info & MAY_BE_REF)) { - | NIY // TODO + | bl ->assign_tmp } else { - | NIY // TODO + | bl ->assign_var } } else if (val_type == IS_CV) { if (!(val_info & MAY_BE_REF)) { - | NIY // TODO + | bl ->assign_cv_noref } else { - | NIY // TODO + | bl ->assign_cv } } else { ZEND_UNREACHABLE(); @@ -4218,7 +4382,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, | beq >8 | b ->exception_handler } else { - | NIY // TODO + | b >8 } } if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { @@ -4235,7 +4399,25 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, done = 1; } } else /* if (RC_MAY_BE_N(var_info)) */ { - | NIY // TODO + if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | IF_NOT_ZVAL_REFCOUNTED var_use_addr, >5, TMP1w, TMP2 + } + if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { + if (Z_REG(var_use_addr) == ZREG_FP) { + | str Rx(Z_REG(var_use_addr)), T1 // save + } + | GET_ZVAL_PTR FCARG1x, var_use_addr, TMP1 + | GC_DELREF FCARG1x, TMP1 + | IF_GC_MAY_NOT_LEAK FCARG1x, >5, TMP1w, TMP2w + | EXT_CALL gc_possible_root, TMP1 + if (Z_REG(var_use_addr) != ZREG_FP) { + | ldr Rx(Z_REG(var_use_addr)), T1 // restore + } + } else { + | GET_ZVAL_PTR Rx(tmp_reg), var_use_addr, TMP1 + | GC_DELREF Rx(tmp_reg), TMP1 + } + |5: } } diff --git a/ext/opcache/tests/jit/assign_039.phpt b/ext/opcache/tests/jit/assign_039.phpt new file mode 100644 index 0000000000000..737d37a15378c --- /dev/null +++ b/ext/opcache/tests/jit/assign_039.phpt @@ -0,0 +1,18 @@ +--TEST-- +JIT ASSIGN: Assign reference IS_VAR +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +NULL From 63d674525bd30e1c00959181af24b0305f7fe1c6 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 20 Apr 2021 18:39:44 +0300 Subject: [PATCH 084/229] Support for missed case in zend_jit_tail_handler(). Fixed ext/opcache/tests/jit/ignored_opcodes.phpt --- ext/opcache/jit/zend_jit_arm64.dasc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index ddb54d5fe07e0..e1cbec1d03b03 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -2689,7 +2689,9 @@ static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) } else { const void *handler = zend_get_opcode_handler_func(opline); - | NIY // TODO: test + | EXT_CALL handler, REG0 + | ADD_HYBRID_SPAD + | JMP_IP TMP1 } } else { const void *handler = opline->handler; From f1c89c3900306028b962dada49626d14eb7ff9c9 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 20 Apr 2021 19:30:46 +0300 Subject: [PATCH 085/229] Support for IS_LONG SUB and XOR. --- ext/opcache/jit/zend_jit_arm64.dasc | 3 +-- ext/opcache/tests/jit/arm64/sub_001.phpt | 20 ++++++++++++++++++++ ext/opcache/tests/jit/arm64/xor_001.phpt | 20 ++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 ext/opcache/tests/jit/arm64/sub_001.phpt create mode 100644 ext/opcache/tests/jit/arm64/xor_001.phpt diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index e1cbec1d03b03..3211b80a5b0d5 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -734,7 +734,7 @@ static void* dasm_labels[zend_lb_MAX]; | LONG_OP adds, reg, addr, tmp_reg1, tmp_reg2 || break; || case ZEND_SUB: -| NIY // LONG_OP sub, reg, addr +| LONG_OP subs, reg, addr, tmp_reg1, tmp_reg2 || break; || case ZEND_MUL: | NIY // LONG_OP imul, reg, addr @@ -746,7 +746,6 @@ static void* dasm_labels[zend_lb_MAX]; | LONG_OP and, reg, addr, tmp_reg1, tmp_reg2 || break; || case ZEND_BW_XOR: -| NIY // TODO | LONG_OP eor, reg, addr, tmp_reg1, tmp_reg2 || break; || default: diff --git a/ext/opcache/tests/jit/arm64/sub_001.phpt b/ext/opcache/tests/jit/arm64/sub_001.phpt new file mode 100644 index 0000000000000..4c98c14738868 --- /dev/null +++ b/ext/opcache/tests/jit/arm64/sub_001.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT SUB: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +int(41) diff --git a/ext/opcache/tests/jit/arm64/xor_001.phpt b/ext/opcache/tests/jit/arm64/xor_001.phpt new file mode 100644 index 0000000000000..abae4cd1beee4 --- /dev/null +++ b/ext/opcache/tests/jit/arm64/xor_001.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT XOR: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +int(4) From b0c488a237c0bc9ca14969020035096a5e9c1d14 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 20 Apr 2021 19:48:57 +0300 Subject: [PATCH 086/229] Support for bitwise operations with the same operands ($a ^ $a) --- ext/opcache/jit/zend_jit_arm64.dasc | 28 +++++++++++++++++++++--- ext/opcache/tests/jit/arm64/xor_002.phpt | 20 +++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 ext/opcache/tests/jit/arm64/xor_002.phpt diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 3211b80a5b0d5..66f4abfc8ae23 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -753,8 +753,29 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro -|.macro LONG_MATH_REG, opcode, dst_reg, src_reg -| NIY // TODO +|.macro LONG_MATH_REG, opcode, dst_reg, src_reg1, src_reg2 +|| switch (opcode) { +|| case ZEND_ADD: +| adds dst_reg, src_reg1, src_reg2 +|| break; +|| case ZEND_SUB: +| subs dst_reg, src_reg1, src_reg2 +|| break; +|| case ZEND_MUL: +| NIY // LONG_OP imul, reg, addr +|| break; +|| case ZEND_BW_OR: +| orr dst_reg, src_reg1, src_reg2 +|| break; +|| case ZEND_BW_AND: +| and dst_reg, src_reg1, src_reg2 +|| break; +|| case ZEND_BW_XOR: +| eor dst_reg, src_reg1, src_reg2 +|| break; +|| default: +|| ZEND_UNREACHABLE(); +|| } |.endmacro // In x86 implementation, argument 'lval' of SET_ZVAL_LVAL can be either a LONG constant @@ -3601,7 +3622,8 @@ static int zend_jit_long_math_helper(dasm_State **Dst, } } } else if (same_ops) { - | NIY // TODO + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | LONG_MATH_REG opcode, Rx(result_reg), Rx(result_reg), Rx(result_reg) } else { | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 | LONG_MATH opcode, result_reg, op2_addr, TMP1, TMP2 diff --git a/ext/opcache/tests/jit/arm64/xor_002.phpt b/ext/opcache/tests/jit/arm64/xor_002.phpt new file mode 100644 index 0000000000000..2f8ff8461300c --- /dev/null +++ b/ext/opcache/tests/jit/arm64/xor_002.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT XOR: 002 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +int(0) From 67b03e4a00e4d3be499711a21176ded08ad2a059 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 20 Apr 2021 20:19:56 +0300 Subject: [PATCH 087/229] Support for bitwise operations with non-integer arguments --- ext/opcache/jit/zend_jit_arm64.dasc | 41 +++++++++++++++++++++++- ext/opcache/tests/jit/arm64/xor_003.phpt | 20 ++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 ext/opcache/tests/jit/arm64/xor_003.phpt diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 66f4abfc8ae23..47a69364a1900 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -3647,7 +3647,46 @@ static int zend_jit_long_math_helper(dasm_State **Dst, |.cold_code } |6: - | NIY // TODO + if (Z_MODE(res_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); + | LOAD_ZVAL_ADDR FCARG1x, real_addr + } else if (Z_REG(res_addr) != ZREG_FCARG1x || Z_OFFSET(res_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, res_addr + } + if (Z_MODE(op1_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); + if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { + return 0; + } + op1_addr = real_addr; + } + | LOAD_ZVAL_ADDR FCARG2x, op1_addr + if (Z_MODE(op2_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); + if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { + return 0; + } + op2_addr = real_addr; + } + | LOAD_ZVAL_ADDR CARG3, op2_addr + | SET_EX_OPLINE opline, REG0 + if (opcode == ZEND_BW_OR) { + | EXT_CALL bitwise_or_function, REG0 + } else if (opcode == ZEND_BW_AND) { + | EXT_CALL bitwise_and_function, REG0 + } else if (opcode == ZEND_BW_XOR) { + | EXT_CALL bitwise_xor_function, REG0 + } else if (opcode == ZEND_SL) { + | EXT_CALL shift_left_function, REG0 + } else if (opcode == ZEND_SR) { + | EXT_CALL shift_right_function, REG0 + } else if (opcode == ZEND_MOD) { + | EXT_CALL mod_function, REG0 + } else { + ZEND_UNREACHABLE(); + } + | FREE_OP op1_type, op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + | FREE_OP op2_type, op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 if (may_throw) { zend_jit_check_exception(Dst); } diff --git a/ext/opcache/tests/jit/arm64/xor_003.phpt b/ext/opcache/tests/jit/arm64/xor_003.phpt new file mode 100644 index 0000000000000..9d0277b4b3583 --- /dev/null +++ b/ext/opcache/tests/jit/arm64/xor_003.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT XOR: 003 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +string(3) "```" From 73d5f43d8c6367451c2ce82e608627a1b8ace6fa Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 20 Apr 2021 21:37:30 +0300 Subject: [PATCH 088/229] Support for RECV/RECV_INIT --- ext/opcache/jit/zend_jit_arm64.dasc | 84 ++++++++++++++++++++++++++--- ext/opcache/tests/jit/recv_002.phpt | 27 ++++++++++ ext/opcache/tests/jit/recv_003.phpt | 36 +++++++++++++ ext/opcache/tests/jit/recv_004.phpt | 22 ++++++++ 4 files changed, 161 insertions(+), 8 deletions(-) create mode 100644 ext/opcache/tests/jit/recv_002.phpt create mode 100644 ext/opcache/tests/jit/recv_003.phpt create mode 100644 ext/opcache/tests/jit/recv_004.phpt diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 47a69364a1900..911dd0671cf1a 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -8279,7 +8279,14 @@ static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zen zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1x : ZREG_REG0; if (ZEND_ARG_SEND_MODE(arg_info)) { - | NIY // TODO + if (opline->opcode == ZEND_RECV_INIT) { + | LOAD_ZVAL_ADDR Rx(tmp_reg), res_addr + | ZVAL_DEREF Rx(tmp_reg), MAY_BE_REF, TMP1w + res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, 0); + } else { + | GET_ZVAL_PTR Rx(tmp_reg), res_addr, TMP1 + res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, offsetof(zend_reference, val)); + } } if (type_mask != 0) { @@ -8302,13 +8309,10 @@ static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zen in_cold = 1; } - | NIY // TODO: currently in cold code if (Z_REG(res_addr) != ZREG_FCARG1x || Z_OFFSET(res_addr) != 0) { - | NIY // TODO | LOAD_ZVAL_ADDR FCARG1x, res_addr } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - | NIY // TODO | SET_EX_OPLINE opline, REG0 } else { | ADDR_STORE EX->opline, opline, REG0 @@ -8318,7 +8322,6 @@ static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zen | mov REG0w, RETVALw if (check_exception) { - | NIY // TODO | and REG0w, REG0w, #0xff | tst REG0w, REG0w if (in_cold) { @@ -8330,7 +8333,6 @@ static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zen | beq ->exception_handler } } else if (in_cold) { - | NIY // TODO | b >1 |.code |1: @@ -8371,7 +8373,14 @@ static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_ | blt >1 |.cold_code |1: - | NIY // TODO + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + | SET_EX_OPLINE opline, REG0 + } else { + | ADDR_STORE EX->opline, opline, REG0 + } + | mov FCARG1x, FP + | EXT_CALL zend_missing_arg_error, REG0 + | b ->exception_handler |.code } } @@ -8398,7 +8407,66 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zen zval *zv = RT_CONSTANT(opline, opline->op2); zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); - | NIY // TODO + if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || + (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { + | ldr TMP1w, EX->This.u2.num_args + | LOAD_32BIT_VAL TMP2w, arg_num + | cmp TMP1w, TMP2w + | bhs >5 + } + | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 + if (Z_REFCOUNTED_P(zv)) { + | ADDREF_CONST zv, REG0, TMP1 + } + + if (Z_CONSTANT_P(zv)) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + | SET_EX_OPLINE opline, REG0 + } else { + | ADDR_STORE EX->opline, opline, REG0 + } + | LOAD_ZVAL_ADDR FCARG1x, res_addr + | ldr REG0, EX->func + | ldr FCARG2x, [REG0, #offsetof(zend_op_array, scope)] + | EXT_CALL zval_update_constant_ex, REG0 + | tst RETVALw, RETVALw + | bne >1 + |.cold_code + |1: + | ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, opline, ZREG_TMP1, ZREG_TMP2 + | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF, TMP1, TMP2 + | b ->exception_handler + |.code + } + + |5: + + if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { + do { + zend_arg_info *arg_info; + + if (arg_num <= op_array->num_args) { + arg_info = &op_array->arg_info[arg_num-1]; + } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) { + arg_info = &op_array->arg_info[op_array->num_args]; + } else { + break; + } + if (!ZEND_TYPE_IS_SET(arg_info->type)) { + break; + } + if (!zend_jit_verify_arg_type(Dst, opline, arg_info, may_throw)) { + return 0; + } + } while (0); + } + + if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { + if (is_last) { + | LOAD_IP_ADDR (opline + 1) + zend_jit_set_last_valid_opline(opline + 1); + } + } return 1; } diff --git a/ext/opcache/tests/jit/recv_002.phpt b/ext/opcache/tests/jit/recv_002.phpt new file mode 100644 index 0000000000000..5e2951a97e2ba --- /dev/null +++ b/ext/opcache/tests/jit/recv_002.phpt @@ -0,0 +1,27 @@ +--TEST-- +JIT RECV: too few arguments +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECTF-- +Fatal error: Uncaught ArgumentCountError: Too few arguments to function test(), 0 passed in /home/dmitry/php/php-arm64/ext/opcache/tests/jit/recv_002.php on line 7 and exactly 1 expected in %s:3 +Stack trace: +#0 %s(7): test() +#1 {main} + thrown in %s on line 3 \ No newline at end of file diff --git a/ext/opcache/tests/jit/recv_003.phpt b/ext/opcache/tests/jit/recv_003.phpt new file mode 100644 index 0000000000000..0eb214e9cac6e --- /dev/null +++ b/ext/opcache/tests/jit/recv_003.phpt @@ -0,0 +1,36 @@ +--TEST-- +JIT RECV: slow type check +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECTF-- +ok + +Fatal error: Uncaught TypeError: test(): Argument #1 ($x) must be of type A, C given, called in %s:9 +Stack trace: +#0 %s(15): test(Object(C)) +#1 {main} + thrown in %s on line 9 diff --git a/ext/opcache/tests/jit/recv_004.phpt b/ext/opcache/tests/jit/recv_004.phpt new file mode 100644 index 0000000000000..4e540f29cb008 --- /dev/null +++ b/ext/opcache/tests/jit/recv_004.phpt @@ -0,0 +1,22 @@ +--TEST-- +JIT RECV: default arguments with type checks +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +int(3) +int(2) From ecee82c4ff110554babccf1d9de5abeb12f72f62 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 20 Apr 2021 22:02:10 +0300 Subject: [PATCH 089/229] Support for passing extra arguments. Fixed ext/opcache/tests/jit/recv_001.phpt --- ext/opcache/jit/zend_jit_arm64.dasc | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 911dd0671cf1a..2fdb13c1b6cbb 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -5923,19 +5923,22 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con |.cold_code |1: if (func) { - | NIY // TODO + | LOAD_32BIT_VAL FCARG1w, used_stack } if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { - | NIY // TODO + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_int_extend_stack_helper, REG0 } else { if (!is_closure) { - | NIY // TODO + | mov FCARG2x, REG0 } else { - | NIY // TODO + | add FCARG2x, REG0, #offsetof(zend_closure, func) } - | NIY // TODO + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_extend_stack_helper, REG0 } - | NIY // TODO + | mov RX, RETVALx + | b >1 |.code } } @@ -6857,7 +6860,15 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | bgt >1 |.cold_code |1: - | NIY // TODO: test + if (!GCC_GLOBAL_REGS) { + | mov FCARG1x, FP + } + | EXT_CALL zend_jit_copy_extra_args_helper, REG0 + if (!func) { + | ldr REG0, EX->func // reload + } + | ldr REG1, [FP, #offsetof(zend_execute_data, This.u2.num_args)] // reload + | b >1 |.code if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) { if (!func) { From 621830e46e14fe6d8b855b3713e862643fe1a3cc Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 20 Apr 2021 23:18:15 +0300 Subject: [PATCH 090/229] Support for most cases of BOOL/BOOL_NOT/JMPZNZ/JMP[N]Z_EX --- ext/opcache/jit/zend_jit_arm64.dasc | 164 +++++++++++++++++++++++----- 1 file changed, 134 insertions(+), 30 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 2fdb13c1b6cbb..77c57ff442f7e 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -5512,18 +5512,26 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ /* Always TRUE */ if (set_bool) { if (set_bool_not) { - | NIY // TODO | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 } else { | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 } } if (true_label != (uint32_t)-1) { - | b =>true_label; + | b =>true_label } } else { /* Always FALSE */ - | NIY // TODO + if (set_bool) { + if (set_bool_not) { + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + } + } + if (false_label != (uint32_t)-1) { + | b =>false_label + } } return 1; } @@ -5537,12 +5545,25 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) { /* Always TRUE */ - | NIY // TODO + if (set_bool) { + if (set_bool_not) { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } + } + if (true_label != (uint32_t)-1) { + | b =>true_label + } } else { if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) { /* Always FALSE */ if (set_bool) { - | NIY // TODO + if (set_bool_not) { + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + } } } else { | CMP_ZVAL_TYPE op1_addr, IS_TRUE, TMP1w, TMP2 @@ -5551,9 +5572,13 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ !(op1_info & MAY_BE_UNDEF) && !set_bool) { if (exit_addr) { - | NIY // TODO + if (branch_opcode == ZEND_JMPNZ) { + | blt >9 + } else { + | NIY // blt &exit_addr + } } else if (false_label != (uint32_t)-1) { - | NIY // TODO + | blt =>false_label } else { | blt >9 } @@ -5564,13 +5589,50 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ } if (!(op1_info & MAY_BE_TRUE)) { /* It's FALSE */ - | NIY // TODO + if (set_bool) { + if (set_bool_not) { + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + } + } } else { if (exit_addr) { - | NIY // TODO + if (set_bool) { + | bne >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | NIY // b &exit_addr + } else { + | b >9 + } + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { + if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { + | NIY // bne &exit_addr + } + } + } else { + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | NIY // beq &exit_addr + } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { + | NIY // bne &exit_addr + } else { + | beq >9 + } + } } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { if (set_bool) { - | NIY // TODO + | bne >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + if (true_label != (uint32_t)-1) { + | b =>true_label + } else { + | b >9 + } + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 } else { if (true_label != (uint32_t)-1) { | beq =>true_label @@ -5578,13 +5640,12 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ | bne =>false_label jmp_done = 1; } else { - | NIY // TODO + | beq >9 } } } else if (set_bool) { | cset REG0w, eq if (set_bool_not) { - | NIY // TODO | neg REG0w, REG0w | add REG0w, REG0w, #3 } else { @@ -5607,13 +5668,11 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 | beq >1 } else { - | NIY // TODO | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, TMP1w, TMP2 } |.cold_code |1: } - | NIY // TODO | LOAD_32BIT_VAL FCARG1w, opline->op1.var | SET_EX_OPLINE opline, REG0 | EXT_CALL zend_jit_undefined_op_helper, REG0 @@ -5626,18 +5685,18 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ if (exit_addr) { if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { - | NIY // TODO + | NIY // b &exit_addr } } else if (false_label != (uint32_t)-1) { - | NIY // TODO + | b =>false_label } if (op1_info & MAY_BE_ANY) { if (exit_addr) { if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { - | NIY // TODO + | b >9 } } else if (false_label == (uint32_t)-1) { - | NIY // TODO + | b >9 } |.code } @@ -5645,9 +5704,15 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ if (!jmp_done) { if (exit_addr) { - | NIY // TODO + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + if (op1_info & MAY_BE_LONG) { + | b >9 + } + } else if (op1_info & MAY_BE_LONG) { + | NIY // b &exit_addr + } } else if (false_label != (uint32_t)-1) { - | NIY // TODO + | b =>false_label } else if (op1_info & MAY_BE_LONG) { | b >9 } @@ -5661,7 +5726,6 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, TMP1w, TMP2 } if (Z_MODE(op1_addr) == IS_REG) { - | NIY // TODO | tst Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) } else { | LONG_CMP_WITH_CONST cmp, op1_addr, Z_L(0), TMP1, TMP2 @@ -5672,15 +5736,25 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ | neg REG0w, REG0w | add REG0w, REG0w, #3 } else { - | NIY // TODO | add REG0w, REG0w, #2 } | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 } if (exit_addr) { - | NIY // TODO + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | NIY // bne &exit_addr + } else { + | NIY // beq &exit_addr + } } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { - | NIY // TODO + if (true_label != (uint32_t)-1) { + | bne =>true_label + if (false_label != (uint32_t)-1) { + | b =>false_label + } + } else { + | beq =>false_label + } } } @@ -5708,10 +5782,12 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 | GC_DELREF FCARG1x, TMP1w | bne >3 - | NIY // TODO: currently jump to label 3. // In x86, r0 is used in macro ZVAL_DTOR_FUNC as temporary register, hence, r0 should be saved/restored - // before/after this macro. In AArch64, TMP1 is used. As a result, we needn't save/resotre REG0. + // before/after this macro. In AArch64, TMP1 is used, but we still have to store REG0, + // because it's clobbered by function call. + | str REG0, T1 // save | ZVAL_DTOR_FUNC op1_info, opline, TMP1 + | ldr REG0, T1 // restore |3: } if (may_throw) { @@ -5721,7 +5797,6 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ if (set_bool) { if (set_bool_not) { - | NIY // TODO | neg REG0w, REG0w | add REG0w, REG0w, #3 } else { @@ -5729,11 +5804,15 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ } | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 if (exit_addr) { - | NIY // TODO + | CMP_ZVAL_TYPE res_addr, IS_FALSE, TMP1w, TMP2 + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | NIY // bne &exit_addr + } else { + | NIY // beq &exit_addr + } } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { | CMP_ZVAL_TYPE res_addr, IS_FALSE, TMP1w, TMP2 if (true_label != (uint32_t)-1) { - | NIY // TODO | bne =>true_label if (false_label != (uint32_t)-1) { | b =>false_label @@ -5749,7 +5828,32 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ |.code } } else { - | NIY // TODO + | tst REG0w, REG0w + if (exit_addr) { + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | NIY // bne &exit_addr + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + | b >9 + } + } else { + | NIY // beq &exit_addr + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + | b >9 + } + } + } else if (true_label != (uint32_t)-1) { + | bne =>true_label + if (false_label != (uint32_t)-1) { + | b =>false_label + } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + | b >9 + } + } else { + | beq =>false_label + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { + | b >9 + } + } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { |.code From 4e291d8d83f3c09704dce9815379d71e45867021 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 21 Apr 2021 01:23:53 +0300 Subject: [PATCH 091/229] Support for DOUBLE math --- ext/opcache/jit/zend_jit_arm64.dasc | 182 ++++++++++++++++++++--- ext/opcache/tests/jit/arm64/add_006.phpt | 26 ++++ 2 files changed, 184 insertions(+), 24 deletions(-) create mode 100644 ext/opcache/tests/jit/arm64/add_006.phpt diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 77c57ff442f7e..0bd83289e60b1 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -614,20 +614,40 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro +|.macro DOUBLE_GET_ZVAL_DVAL, reg, addr, tmp_reg +|| if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) { +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +| LOAD_ZVAL_ADDR Rx(tmp_reg), addr +| ldr Rd(reg-ZREG_V0), [Rx(tmp_reg)] +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +|| if (Z_OFFSET(addr) <= MAX_IMM12 ) { +| ldr Rd(reg-ZREG_V0), [Rx(Z_REG(addr)), #Z_OFFSET(addr)] +|| } else { +| LOAD_ZVAL_ADDR Rx(tmp_reg), addr +| ldr Rd(reg-ZREG_V0), [Rx(tmp_reg)] +|| } +|| } else if (Z_MODE(addr) == IS_REG) { +| fmov Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) +|| } else { +|| ZEND_UNREACHABLE(); +|| } +|| } +|.endmacro + // Define DOUBLE_MATH_REG to replace AVX_MATH_REG in x86 implementation. -|.macro DOUBLE_MATH_REG, opcode, dst_reg, op1_reg, src_reg +|.macro DOUBLE_MATH_REG, opcode, dst_reg, op1_reg, op2_reg || switch (opcode) { || case ZEND_ADD: -| fadd Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(src_reg-ZREG_V0) +| fadd Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) || break; || case ZEND_SUB: -| NIY // vsubsd xmm(dst_reg-ZREG_V0), xmm(op1_reg-ZREG_V0), xmm(src_reg-ZREG_V0) +| fsub Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) || break; || case ZEND_MUL: -| NIY // vmulsd xmm(dst_reg-ZREG_V0), xmm(op1_reg-ZREG_V0), xmm(src_reg-ZREG_V0) +| fmul Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) || break; || case ZEND_DIV: -| NIY // vdivsd xmm(dst_reg-ZREG_V0), xmm(op1_reg-ZREG_V0), xmm(src_reg-ZREG_V0) +| fdiv Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0) || break; || } |.endmacro @@ -3145,9 +3165,26 @@ static int zend_jit_math_long_double(dasm_State **Dst, { zend_reg result_reg = (Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_FPR0; - zend_reg tmp_reg; + zend_reg op2_reg; - | NIY // TODO + | DOUBLE_GET_ZVAL_LVAL result_reg, op1_addr, ZREG_TMP1, ZREG_TMP2 + + if (Z_MODE(op2_addr) == IS_REG) { + op2_reg = Z_REG(op2_addr); + } else { + op2_reg = ZREG_FPR1; + | DOUBLE_GET_ZVAL_DVAL op2_reg, op2_addr, ZREG_TMP1 + } + + | DOUBLE_MATH_REG opcode, result_reg, result_reg, op2_reg + + | SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1 + + if (Z_MODE(res_addr) == IS_MEM_ZVAL) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { + | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 + } + } return 1; } @@ -3159,9 +3196,59 @@ static int zend_jit_math_double_long(dasm_State **Dst, zend_jit_addr res_addr, uint32_t res_use_info) { - zend_reg result_reg, tmp_reg; + zend_reg result_reg, op1_reg, op2_reg; + + if (zend_is_commutative(opcode) + && (Z_MODE(res_addr) != IS_REG || Z_MODE(op1_addr) != IS_REG || Z_REG(res_addr) != Z_REG(op1_addr))) { + if (Z_MODE(res_addr) == IS_REG) { + result_reg = Z_REG(res_addr); + } else { + result_reg = ZREG_FPR0; + } + | DOUBLE_GET_ZVAL_LVAL result_reg, op2_addr, ZREG_TMP1, ZREG_TMP2 + if (Z_MODE(op1_addr) == IS_REG) { + op1_reg = Z_REG(op1_addr); + } else { + op1_reg = ZREG_FPR1; + | DOUBLE_GET_ZVAL_DVAL op1_reg, op1_addr, ZREG_TMP1 + } + | DOUBLE_MATH_REG opcode, result_reg, result_reg, op1_reg + } else { + if (Z_MODE(res_addr) == IS_REG) { + result_reg = Z_REG(res_addr); + } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { + result_reg = Z_REG(op1_addr); + } else { + result_reg = ZREG_FPR0; + } + + if (Z_MODE(op1_addr) == IS_REG) { + op1_reg = Z_REG(op1_addr); + } else { + | DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr, ZREG_TMP1 + op1_reg = result_reg; + } + if ((opcode == ZEND_ADD || opcode == ZEND_SUB) + && Z_MODE(op2_addr) == IS_CONST_ZVAL + && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { + /* +/- 0 */ + } else { + op2_reg = ZREG_FPR1; + | DOUBLE_GET_ZVAL_LVAL op2_reg, op2_addr, ZREG_TMP1, ZREG_TMP2 + | DOUBLE_MATH_REG opcode, result_reg, op1_reg, op2_reg + } + } + + | SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1 + + if (Z_MODE(res_addr) == IS_MEM_ZVAL) { + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { + | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 + } + } + } - | NIY // TODO return 1; } @@ -3173,9 +3260,55 @@ static int zend_jit_math_double_double(dasm_State **Dst, uint32_t res_use_info) { bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); - zend_reg result_reg; + zend_reg result_reg, op1_reg, op2_reg; + zend_jit_addr val_addr; - | NIY // TODO + if (Z_MODE(res_addr) == IS_REG) { + result_reg = Z_REG(res_addr); + } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { + result_reg = Z_REG(op1_addr); + } else if (zend_is_commutative(opcode) && Z_MODE(op2_addr) == IS_REG && Z_LAST_USE(op2_addr)) { + result_reg = Z_REG(op2_addr); + } else { + result_reg = ZREG_FPR0; + } + + if (Z_MODE(op1_addr) == IS_REG) { + op1_reg = Z_REG(op1_addr); + val_addr = op2_addr; + } else if (Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) { + op1_reg = Z_REG(op2_addr); + val_addr = op1_addr; + } else { + | DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr, ZREG_TMP1 + op1_reg = result_reg; + val_addr = op2_addr; + } + + if ((opcode == ZEND_MUL) && + Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) { + | DOUBLE_MATH_REG ZEND_ADD, result_reg, op1_reg, op1_reg + } else { + if (same_ops) { + op2_reg = op1_reg; + } else if (Z_MODE(val_addr) == IS_REG) { + op2_reg = Z_REG(val_addr); + } else { + op2_reg = ZREG_FPR1; + | DOUBLE_GET_ZVAL_DVAL op2_reg, val_addr, ZREG_TMP1 + } + | DOUBLE_MATH_REG opcode, result_reg, op1_reg, op2_reg + } + + | SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1 + + if (Z_MODE(res_addr) == IS_MEM_ZVAL) { + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) { + if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { + | SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2 + } + } + } return 1; } @@ -3205,7 +3338,6 @@ static int zend_jit_math_helper(dasm_State **Dst, if (op1_info & MAY_BE_DOUBLE) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, TMP1w, TMP2 } else { - | NIY // TODO: test | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, TMP1w, TMP2 } } @@ -3214,7 +3346,6 @@ static int zend_jit_math_helper(dasm_State **Dst, | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1, TMP1w, TMP2 |.cold_code |1: - | NIY // TODO: test if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, TMP1w, TMP2 } @@ -3224,7 +3355,6 @@ static int zend_jit_math_helper(dasm_State **Dst, | b >5 |.code } else { - | NIY // TODO: test | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, TMP1w, TMP2 } } @@ -3237,7 +3367,6 @@ static int zend_jit_math_helper(dasm_State **Dst, if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, TMP1w, TMP2 } - | NIY // TODO: test if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops) { @@ -3266,7 +3395,6 @@ static int zend_jit_math_helper(dasm_State **Dst, } else if ((op1_info & MAY_BE_DOUBLE) && !(op1_info & MAY_BE_LONG) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { - | NIY // TODO: test if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, TMP1w, TMP2 } @@ -3301,7 +3429,6 @@ static int zend_jit_math_helper(dasm_State **Dst, } else if ((op2_info & MAY_BE_DOUBLE) && !(op2_info & MAY_BE_LONG) && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { - | NIY // TODO: test if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, TMP1w, TMP2 } @@ -3346,30 +3473,34 @@ static int zend_jit_math_helper(dasm_State **Dst, |6: if (Z_MODE(res_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); - | NIY // TODO: test | LOAD_ZVAL_ADDR FCARG1x, real_addr } else if (Z_REG(res_addr) != ZREG_FCARG1x || Z_OFFSET(res_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, res_addr } if (Z_MODE(op1_addr) == IS_REG) { - | NIY // TODO: test + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); + if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { + return 0; + } + op1_addr = real_addr; } | LOAD_ZVAL_ADDR FCARG2x, op1_addr if (Z_MODE(op2_addr) == IS_REG) { - | NIY // TODO: test + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); + if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { + return 0; + } + op2_addr = real_addr; } | LOAD_ZVAL_ADDR CARG3, op2_addr | SET_EX_OPLINE opline, REG0 if (opcode == ZEND_ADD) { | EXT_CALL add_function, REG0 } else if (opcode == ZEND_SUB) { - | NIY // TODO: test | EXT_CALL sub_function, REG0 } else if (opcode == ZEND_MUL) { - | NIY // TODO: test | EXT_CALL mul_function, REG0 } else if (opcode == ZEND_DIV) { - | NIY // TODO: test | EXT_CALL div_function, REG0 } else { ZEND_UNREACHABLE(); @@ -3380,7 +3511,10 @@ static int zend_jit_math_helper(dasm_State **Dst, zend_jit_check_exception(Dst); } if (Z_MODE(res_addr) == IS_REG) { - | NIY // TODO: test + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); + if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) { + return 0; + } } if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { diff --git a/ext/opcache/tests/jit/arm64/add_006.phpt b/ext/opcache/tests/jit/arm64/add_006.phpt new file mode 100644 index 0000000000000..f15bdf2c4c761 --- /dev/null +++ b/ext/opcache/tests/jit/arm64/add_006.phpt @@ -0,0 +1,26 @@ +--TEST-- +JIT ADD: 006 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +int(8) +float(8) +float(8) +float(8) From 191fb2a3641ca3bcc53e32f26af8935dc5ebaa67 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Wed, 21 Apr 2021 03:09:03 +0000 Subject: [PATCH 092/229] Updates for commits between 121a0f7 and 12dcf34 1. Pre-allocated bytes are missing in function zend_jit_assign_const_stub(). 2. 'w' register should be used for macro SET_ZVAL_TYPE_INFO. 3. 'w' register should be used to load "num_args" in function zend_jit_do_fcall(). 4. Remove the local path name in test case recv_002.phpt 5. One option is disabled temporarily in [1] and several test cases would fail, e.g. shift_right_003.phpt. I suppose new execution paths are touched. We will support them in the near future. [1] https://github.com/php/php-src/commit/63d673d --- ext/opcache/jit/zend_jit_arm64.dasc | 17 +++++++++-------- ext/opcache/tests/jit/recv_002.phpt | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 0bd83289e60b1..33b0e850b0754 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -907,17 +907,17 @@ static void* dasm_labels[zend_lb_MAX]; || if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { || if (dst_def_info == MAY_BE_DOUBLE) { || if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { -| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE, Rx(tmp_reg1), Rx(tmp_reg2) +| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2) || } || } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<assign_const: + | sub sp, sp, #16 if (!zend_jit_assign_to_variable( Dst, NULL, var_addr, var_addr, -1, -1, @@ -2118,6 +2119,7 @@ static int zend_jit_assign_const_stub(dasm_State **Dst) 0, 0)) { return 0; } + | add sp, sp, #16 | ret return 1; } @@ -7105,7 +7107,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend if (!func) { | ldr REG0, EX->func // reload } - | ldr REG1, [FP, #offsetof(zend_execute_data, This.u2.num_args)] // reload + | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] // reload | b >1 |.code if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) { @@ -8678,12 +8680,11 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zen | ldr REG0, EX->func | ldr FCARG2x, [REG0, #offsetof(zend_op_array, scope)] | EXT_CALL zval_update_constant_ex, REG0 - | tst RETVALw, RETVALw - | bne >1 + | cbnz RETVALw, >1 |.cold_code |1: | ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, opline, ZREG_TMP1, ZREG_TMP2 - | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF, TMP1, TMP2 + | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF, TMP1w, TMP2 | b ->exception_handler |.code } diff --git a/ext/opcache/tests/jit/recv_002.phpt b/ext/opcache/tests/jit/recv_002.phpt index 5e2951a97e2ba..0f38edc4400eb 100644 --- a/ext/opcache/tests/jit/recv_002.phpt +++ b/ext/opcache/tests/jit/recv_002.phpt @@ -20,7 +20,7 @@ test(); ?> --EXPECTF-- -Fatal error: Uncaught ArgumentCountError: Too few arguments to function test(), 0 passed in /home/dmitry/php/php-arm64/ext/opcache/tests/jit/recv_002.php on line 7 and exactly 1 expected in %s:3 +Fatal error: Uncaught ArgumentCountError: Too few arguments to function test(), 0 passed in %srecv_002.php on line 7 and exactly 1 expected in %s:3 Stack trace: #0 %s(7): test() #1 {main} From 3f292dcb47acccd88cadb3b8ee9a405e4ad7e28d Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Wed, 21 Apr 2021 03:24:04 +0000 Subject: [PATCH 093/229] Remove the duplicate macro DOUBLE_GET_ZVAL_DVAL We have defined macro GET_ZVAL_DVAL to replace the SSE_GET_ZVAL_DVAL in x86 implementation. Hence, macro DOUBLE_GET_ZVAL_DVAL is not needed. --- ext/opcache/jit/zend_jit_arm64.dasc | 32 +++++------------------------ 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 33b0e850b0754..99d87cac49f7b 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -614,26 +614,6 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro -|.macro DOUBLE_GET_ZVAL_DVAL, reg, addr, tmp_reg -|| if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) { -|| if (Z_MODE(addr) == IS_CONST_ZVAL) { -| LOAD_ZVAL_ADDR Rx(tmp_reg), addr -| ldr Rd(reg-ZREG_V0), [Rx(tmp_reg)] -|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { -|| if (Z_OFFSET(addr) <= MAX_IMM12 ) { -| ldr Rd(reg-ZREG_V0), [Rx(Z_REG(addr)), #Z_OFFSET(addr)] -|| } else { -| LOAD_ZVAL_ADDR Rx(tmp_reg), addr -| ldr Rd(reg-ZREG_V0), [Rx(tmp_reg)] -|| } -|| } else if (Z_MODE(addr) == IS_REG) { -| fmov Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) -|| } else { -|| ZEND_UNREACHABLE(); -|| } -|| } -|.endmacro - // Define DOUBLE_MATH_REG to replace AVX_MATH_REG in x86 implementation. |.macro DOUBLE_MATH_REG, opcode, dst_reg, op1_reg, op2_reg || switch (opcode) { @@ -836,13 +816,11 @@ static void* dasm_labels[zend_lb_MAX]; |.macro GET_ZVAL_DVAL, reg, addr, tmp_reg || if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) { || if (Z_MODE(addr) == IS_CONST_ZVAL) { -| NIY // TODO: test | LOAD_ADDR Rx(tmp_reg), Z_ZV(addr) | ldr Rd(reg-ZREG_V0), [Rx(tmp_reg)] || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | SAFE_MEM_ACC_WITH_UOFFSET ldr, Rd(reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) || } else if (Z_MODE(addr) == IS_REG) { -| NIY // TODO: test | fmov Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) || } else { || ZEND_UNREACHABLE(); @@ -3175,7 +3153,7 @@ static int zend_jit_math_long_double(dasm_State **Dst, op2_reg = Z_REG(op2_addr); } else { op2_reg = ZREG_FPR1; - | DOUBLE_GET_ZVAL_DVAL op2_reg, op2_addr, ZREG_TMP1 + | GET_ZVAL_DVAL op2_reg, op2_addr, ZREG_TMP1 } | DOUBLE_MATH_REG opcode, result_reg, result_reg, op2_reg @@ -3212,7 +3190,7 @@ static int zend_jit_math_double_long(dasm_State **Dst, op1_reg = Z_REG(op1_addr); } else { op1_reg = ZREG_FPR1; - | DOUBLE_GET_ZVAL_DVAL op1_reg, op1_addr, ZREG_TMP1 + | GET_ZVAL_DVAL op1_reg, op1_addr, ZREG_TMP1 } | DOUBLE_MATH_REG opcode, result_reg, result_reg, op1_reg } else { @@ -3227,7 +3205,7 @@ static int zend_jit_math_double_long(dasm_State **Dst, if (Z_MODE(op1_addr) == IS_REG) { op1_reg = Z_REG(op1_addr); } else { - | DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr, ZREG_TMP1 + | GET_ZVAL_DVAL result_reg, op1_addr, ZREG_TMP1 op1_reg = result_reg; } if ((opcode == ZEND_ADD || opcode == ZEND_SUB) @@ -3282,7 +3260,7 @@ static int zend_jit_math_double_double(dasm_State **Dst, op1_reg = Z_REG(op2_addr); val_addr = op1_addr; } else { - | DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr, ZREG_TMP1 + | GET_ZVAL_DVAL result_reg, op1_addr, ZREG_TMP1 op1_reg = result_reg; val_addr = op2_addr; } @@ -3297,7 +3275,7 @@ static int zend_jit_math_double_double(dasm_State **Dst, op2_reg = Z_REG(val_addr); } else { op2_reg = ZREG_FPR1; - | DOUBLE_GET_ZVAL_DVAL op2_reg, val_addr, ZREG_TMP1 + | GET_ZVAL_DVAL op2_reg, val_addr, ZREG_TMP1 } | DOUBLE_MATH_REG opcode, result_reg, op1_reg, op2_reg } From 36b5caa6b9518b895bf60773086f72b41a2aa197 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Wed, 21 Apr 2021 03:44:36 +0000 Subject: [PATCH 094/229] Revisit the encoding of immediate Immediates in different lengths are used by AArch64 instructions. Previous use of MAX_IMM12 is not accurate. Note that immediate value can be optionally shifted and this mode is not supported currently. We may want to support this in the near future. --- ext/opcache/jit/zend_jit_arm64.dasc | 190 ++++++++++++++++++---------- 1 file changed, 120 insertions(+), 70 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 99d87cac49f7b..1aee9f77e1d97 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -92,8 +92,20 @@ #define TMP_ZVAL_OFFSET 0 #define DASM_ALIGNMENT 16 -#define MAX_IMM12 0xfff // maximum value for imm12 -#define LDR_STR_IMM (MAX_IMM12 * 8) // maximum value for imm12 * 8 + +/* Encoding of immediate. TODO: shift mode may be supported in the near future. */ +#define MAX_IMM12 0xfff // maximum value for imm12 +#define MAX_IMM13 0x1fff // maximum value for imm13 +#define MAX_IMM16 0xffff // maximum value for imm16 +#define CMP_IMM MAX_IMM12 // cmp insn +#define MOVZ_IMM MAX_IMM16 // movz insn +#define ADD_SUB_IMM MAX_IMM12 // add/sub insn +#define BW_W_IMM MAX_IMM12 // bitwise insn for 32-bit variant: and, orr, eor +#define BW_X_IMM MAX_IMM13 // bitwise insn for 64-bit variant: and, orr, eor +#define TST_W_IMM MAX_IMM12 // tst insn for 32-bit variant +#define TST_X_IMM MAX_IMM13 // tst insn for 64-bit variant +#define LDR_STR_PIMM (MAX_IMM12*8) // ldr/str insn: pimm is imm12 * 8 +#define LDRB_STRB_PIMM MAX_IMM12 // ldrb/strb insn #include "Zend/zend_cpuinfo.h" @@ -190,22 +202,30 @@ static void* dasm_labels[zend_lb_MAX]; // Type cast to unsigned is used to avoid undefined behavior. |.macro LOAD_32BIT_VAL, reg, val -| mov reg, #((uint32_t)(val) & 0xffff) -| movk reg, #(((uint32_t)(val) >> 16) & 0xffff), lsl #16 +|| if (val >= 0 && val <= MOVZ_IMM) { +| movz reg, #val +|| } else { +| mov reg, #((uint32_t)(val) & 0xffff) +| movk reg, #(((uint32_t)(val) >> 16) & 0xffff), lsl #16 +|| } |.endmacro |.macro LOAD_64BIT_VAL, reg, val -| mov reg, #((uint64_t)(val) & 0xffff) -| movk reg, #(((uint64_t)(val) >> 16) & 0xffff), lsl #16 -| movk reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 -| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 +|| if (val >= 0 && val <= MOVZ_IMM) { +| movz reg, #val +|| } else { +| mov reg, #((uint64_t)(val) & 0xffff) +| movk reg, #(((uint64_t)(val) >> 16) & 0xffff), lsl #16 +| movk reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 +| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 +|| } |.endmacro // Safe memory load/store with an unsigned immediate offset. // When using Z_OFFSET(addr), which is 24-bit long, as the unsigned offset to compute a memory address, -// we should firstly check whether it's greater than MAX_IMM12. +// we should firstly check the range. |.macro SAFE_MEM_ACC_WITH_UOFFSET, ldr_str_ins, op, base_reg, offset, tmp_reg -|| if (offset > MAX_IMM12) { +|| if (offset > LDR_STR_PIMM) { | LOAD_32BIT_VAL tmp_reg, offset | ldr_str_ins op, [base_reg, tmp_reg] || } else { @@ -213,6 +233,15 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro +|.macro SAFE_MEM_ACC_WITH_UOFFSET_BYTE, ldrb_strb_ins, op, base_reg, offset, tmp_reg +|| if (offset > LDRB_STRB_PIMM) { +| LOAD_32BIT_VAL tmp_reg, offset +| ldrb_strb_ins op, [base_reg, tmp_reg] +|| } else { +| ldrb_strb_ins op, [base_reg, #(offset)] +|| } +|.endmacro + |.macro LOAD_TSRM_CACHE, reg | NIY // TODO |.endmacro @@ -348,7 +377,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro LOAD_BASE_ADDR, reg, base, offset || if (offset) { -|| if (offset > MAX_IMM12) { +|| if (offset > ADD_SUB_IMM) { | LOAD_32BIT_VAL reg, offset | add reg, Rx(base), reg || } else { @@ -411,7 +440,7 @@ static void* dasm_labels[zend_lb_MAX]; // In x86 implementation, 'val' can be either a constant or a register. // In AArch64, use ADD_IP for register case, -// and use ADD_IP_FROM_CST for constant case, where the value can be represented by imm12. +// and use ADD_IP_FROM_CST for constant case, where the value can be represented by ADD_SUB_IMM. |.macro ADD_IP, val, tmp_reg || if (GCC_GLOBAL_REGS) { | add IP, IP, val @@ -423,7 +452,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro ADD_IP_FROM_CST, val, tmp_reg -|| ZEND_ASSERT(val >=0 && val <= MAX_IMM12); +|| ZEND_ASSERT(val >=0 && val <= ADD_SUB_IMM); || if (GCC_GLOBAL_REGS) { | add IP, IP, #val || } else { @@ -494,11 +523,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro SET_ZVAL_TYPE_INFO, addr, type, tmp_reg1, tmp_reg2 || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -|| if (type <= MAX_IMM12) { -| mov tmp_reg1, #type -|| } else { -| LOAD_32BIT_VAL tmp_reg1, type -|| } +| LOAD_32BIT_VAL tmp_reg1, type | SAFE_MEM_ACC_WITH_UOFFSET str, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg2 |.endmacro @@ -632,9 +657,11 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro -|.macro LONG_OP, long_ins, reg, addr, tmp_reg1, tmp_reg2 +// Define LONG_ADD_SUB, LONG_BW_OP, LONG_MUL, LONG_CMP to replace the LONG_OP in x86 implementation. +// 'long_ins' should be addition or subtraction. +|.macro LONG_ADD_SUB, long_ins, reg, addr, tmp_reg1, tmp_reg2 || if (Z_MODE(addr) == IS_CONST_ZVAL) { -|| if(Z_LVAL_P(Z_ZV(addr)) >= 0 && Z_LVAL_P(Z_ZV(addr)) <= MAX_IMM12) { +|| if(Z_LVAL_P(Z_ZV(addr)) >= 0 && Z_LVAL_P(Z_ZV(addr)) <= ADD_SUB_IMM) { | long_ins Rx(reg), Rx(reg), #(Z_LVAL_P(Z_ZV(addr))) || } else { | LOAD_64BIT_VAL tmp_reg1, Z_LVAL_P(Z_ZV(addr)) @@ -650,9 +677,33 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro +// 'long_ins' should be 'and', 'orr' or 'eor' +|.macro LONG_BW_OP, long_ins, reg, addr, tmp_reg1, tmp_reg2 +|| if (Z_MODE(addr) == IS_CONST_ZVAL) { +|| if(Z_LVAL_P(Z_ZV(addr)) >= 0 && Z_LVAL_P(Z_ZV(addr)) <= BW_X_IMM) { +| long_ins Rx(reg), Rx(reg), #(Z_LVAL_P(Z_ZV(addr))) +|| } else { +| LOAD_64BIT_VAL tmp_reg1, Z_LVAL_P(Z_ZV(addr)) +| long_ins Rx(reg), Rx(reg), tmp_reg1 +|| } +|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { +| SAFE_MEM_ACC_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg2 +| long_ins Rx(reg), Rx(reg), tmp_reg1 +|| } else if (Z_MODE(addr) == IS_REG) { +| long_ins Rx(reg), Rx(reg), Rx(Z_REG(addr)) +|| } else { +|| ZEND_UNREACHABLE(); +|| } +|.endmacro + +// TODO: integer overflow should be detected. +|.macro LONG_MUL, long_ins, reg, addr, tmp_reg1, tmp_reg2 +| brk #0 // TODO +|.endmacro + |.macro LONG_CMP, reg, addr, tmp_reg1, tmp_reg2 || if (Z_MODE(addr) == IS_CONST_ZVAL) { -|| if(Z_LVAL_P(Z_ZV(addr)) >= 0 && Z_LVAL_P(Z_ZV(addr)) <= MAX_IMM12) { +|| if(Z_LVAL_P(Z_ZV(addr)) >= 0 && Z_LVAL_P(Z_ZV(addr)) <= CMP_IMM) { | cmp Rx(reg), #(Z_LVAL_P(Z_ZV(addr))) || } else { | NIY // TODO @@ -669,8 +720,9 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro -|.macro LONG_OP_WITH_CONST_IMM12, long_ins, op1_addr, lval, tmp_reg1, tmp_reg2 -|| ZEND_ASSERT(lval >=0 && lval <= MAX_IMM12); +// long_ins should be addition or subtraction. +|.macro LONG_ADD_SUB_WITH_IMM, long_ins, op1_addr, lval, tmp_reg1, tmp_reg2 +|| ZEND_ASSERT(lval >=0 && lval <= ADD_SUB_IMM); || if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { | SAFE_MEM_ACC_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(op1_addr)), Z_OFFSET(op1_addr), tmp_reg2 | long_ins tmp_reg1, tmp_reg1, #lval @@ -692,14 +744,14 @@ static void* dasm_labels[zend_lb_MAX]; |.macro LONG_CMP_WITH_CONST, cmp_ins, op1_addr, lval, tmp_reg1, tmp_reg2 || if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { | SAFE_MEM_ACC_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(op1_addr)), Z_OFFSET(op1_addr), tmp_reg2 -|| if (lval >=0 && lval <= MAX_IMM12) { +|| if (lval >=0 && lval <= CMP_IMM) { | cmp_ins tmp_reg1, #lval || } else { | LOAD_64BIT_VAL tmp_reg2, lval | cmp_ins tmp_reg1, tmp_reg2 || } || } else if (Z_MODE(op1_addr) == IS_REG) { -|| if (lval >=0 && lval <= MAX_IMM12) { +|| if (lval >=0 && lval <= CMP_IMM) { | cmp_ins Rx(Z_REG(op1_addr)), #lval || } else { | LOAD_64BIT_VAL tmp_reg1, lval @@ -731,22 +783,22 @@ static void* dasm_labels[zend_lb_MAX]; |.macro LONG_MATH, opcode, reg, addr, tmp_reg1, tmp_reg2 || switch (opcode) { || case ZEND_ADD: -| LONG_OP adds, reg, addr, tmp_reg1, tmp_reg2 +| LONG_ADD_SUB adds, reg, addr, tmp_reg1, tmp_reg2 || break; || case ZEND_SUB: -| LONG_OP subs, reg, addr, tmp_reg1, tmp_reg2 +| LONG_ADD_SUB subs, reg, addr, tmp_reg1, tmp_reg2 || break; || case ZEND_MUL: -| NIY // LONG_OP imul, reg, addr +| NIY // LONG_MUL imul, reg, addr || break; || case ZEND_BW_OR: -| LONG_OP orr, reg, addr, tmp_reg1, tmp_reg2 +| LONG_BW_OP orr, reg, addr, tmp_reg1, tmp_reg2 || break; || case ZEND_BW_AND: -| LONG_OP and, reg, addr, tmp_reg1, tmp_reg2 +| LONG_BW_OP and, reg, addr, tmp_reg1, tmp_reg2 || break; || case ZEND_BW_XOR: -| LONG_OP eor, reg, addr, tmp_reg1, tmp_reg2 +| LONG_BW_OP eor, reg, addr, tmp_reg1, tmp_reg2 || break; || default: || ZEND_UNREACHABLE(); @@ -1022,11 +1074,13 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro IF_TYPE, type, val, label +|| ZEND_ASSERT(val >=0 && val <= CMP_IMM); | cmp type, #val | beq label |.endmacro |.macro IF_NOT_TYPE, type, val, label +|| ZEND_ASSERT(val >=0 && val <= CMP_IMM); | cmp type, #val | bne label |.endmacro @@ -1036,29 +1090,26 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro IF_NOT_Z_TYPE, zv, val, label, tmp_reg -|| ZEND_ASSERT(val >=0 && val <= MAX_IMM12); | ldrb tmp_reg, [zv, #offsetof(zval, u1.v.type)] | IF_NOT_TYPE tmp_reg, val, label |.endmacro |.macro CMP_ZVAL_TYPE, addr, val, tmp_reg1, tmp_reg2 || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -|| ZEND_ASSERT(val <= MAX_IMM12); -| SAFE_MEM_ACC_WITH_UOFFSET ldrb, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), tmp_reg2 +|| ZEND_ASSERT(val <= CMP_IMM); +| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), tmp_reg2 | cmp tmp_reg1, #val |.endmacro |.macro IF_ZVAL_TYPE, addr, val, label, tmp_reg1, tmp_reg2 || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -|| ZEND_ASSERT(val <= MAX_IMM12); -| SAFE_MEM_ACC_WITH_UOFFSET ldrb, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), tmp_reg2 +| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), tmp_reg2 | IF_TYPE tmp_reg1, val, label |.endmacro |.macro IF_NOT_ZVAL_TYPE, addr, val, label, tmp_reg1, tmp_reg2 || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -|| ZEND_ASSERT(val <= MAX_IMM12); -| SAFE_MEM_ACC_WITH_UOFFSET ldrb, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), tmp_reg2 +| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), tmp_reg2 | IF_NOT_TYPE tmp_reg1, val, label |.endmacro @@ -1073,24 +1124,26 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro IF_REFCOUNTED, type_flags, label +|| ZEND_ASSERT((IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT) <= TST_W_IMM); | tst type_flags, #(IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT) | bne label |.endmacro |.macro IF_NOT_REFCOUNTED, type_flags, label +|| ZEND_ASSERT((IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT) <= TST_W_IMM); | tst type_flags, #(IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT) | beq label |.endmacro |.macro IF_ZVAL_FLAGS, addr, mask, label, tmp_reg1, tmp_reg2 || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -| SAFE_MEM_ACC_WITH_UOFFSET ldrb, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags), tmp_reg2 +| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags), tmp_reg2 | IF_FLAGS tmp_reg1, mask, label |.endmacro |.macro IF_NOT_ZVAL_FLAGS, addr, mask, label, tmp_reg1, tmp_reg2 || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -| SAFE_MEM_ACC_WITH_UOFFSET ldrb, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags), tmp_reg2 +| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags), tmp_reg2 | IF_NOT_FLAGS tmp_reg1, mask, label |.endmacro @@ -2603,7 +2656,7 @@ static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t return 0; } - if (var > MAX_IMM12) { + if (var > ADD_SUB_IMM) { | LOAD_32BIT_VAL TMP1, var | add TMP1, FP, TMP1 } else { @@ -2915,10 +2968,10 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op return 0; } if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { - | LONG_OP_WITH_CONST_IMM12 adds, op1_def_addr, Z_L(1), TMP1, TMP2 + | LONG_ADD_SUB_WITH_IMM adds, op1_def_addr, Z_L(1), TMP1, TMP2 } else { | NIY // TODO: test - | LONG_OP_WITH_CONST_IMM12 subs, op1_def_addr, Z_L(1), TMP1, TMP2 + | LONG_ADD_SUB_WITH_IMM subs, op1_def_addr, Z_L(1), TMP1, TMP2 } if (may_overflow && @@ -3107,7 +3160,8 @@ static int zend_jit_math_long_long(dasm_State **Dst, if (Z_MODE(res_addr) == IS_REG) { | NIY // TODO: test } else { - | SET_ZVAL_LVAL res_addr, 0x43e0000000000000, REG0, TMP1 + uint64_t val = 0x43e0000000000000; + | SET_ZVAL_LVAL res_addr, val, REG0, TMP1 } break; } else if (opcode == ZEND_SUB) { @@ -3968,7 +4022,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o if (packed_loaded) { | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); if (op1_info & MAY_BE_ARRAY_HASH) { - || ZEND_ASSERT(HASH_FLAG_PACKED <= MAX_IMM12); + || ZEND_ASSERT(HASH_FLAG_PACKED <= TST_W_IMM); | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] | tst TMP1w, #HASH_FLAG_PACKED | beq >4 // HASH_FIND @@ -6111,10 +6165,10 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con | MEM_LOAD_ZTS ldr, REG2, executor_globals, vm_stack_end, TMP1 | sub REG2, REG2, RX if (func) { - || if (used_stack <= MAX_IMM12) { + || if (used_stack <= CMP_IMM) { | cmp REG2, #used_stack || } else { - | LOAD_32BIT_VAL TMP1, used_stack + | LOAD_32BIT_VAL TMP1w, used_stack | cmp REG2, TMP1 || } } else { @@ -6162,10 +6216,10 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con } if (func) { - || if (used_stack <= MAX_IMM12) { + || if (used_stack <= ADD_SUB_IMM) { | MEM_LOAD_OP_STORE_ZTS add, ldr, str, #used_stack, executor_globals, vm_stack_top, REG2, TMP1 || } else { - | LOAD_32BIT_VAL TMP1, used_stack + | LOAD_32BIT_VAL TMP1w, used_stack | MEM_LOAD_OP_STORE_ZTS add, ldr, str, TMP1, executor_globals, vm_stack_top, REG2, TMP2 || } } else { @@ -6475,7 +6529,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t | EXT_CALL zend_jit_init_func_run_time_cache_helper, REG0 | ldr REG1, EX->run_time_cache | mov REG0, RETVALx - || ZEND_ASSERT(opline->result.num <= LDR_STR_IMM); + || ZEND_ASSERT(opline->result.num <= LDR_STR_PIMM); | str REG0, [REG1, #opline->result.num] | b >3 } else { @@ -6691,7 +6745,7 @@ static int zend_jit_init_method_call(dasm_State **Dst, if (!func) { | // if (fbc->common.fn_flags & ZEND_ACC_STATIC) { | ldr TMP1w, [REG0, #offsetof(zend_function, common.fn_flags)] - || ZEND_ASSERT(ZEND_ACC_STATIC <= MAX_IMM12); + || ZEND_ASSERT(ZEND_ACC_STATIC <= TST_W_IMM); | tst TMP1w, #ZEND_ACC_STATIC | bne >1 |.cold_code @@ -6880,7 +6934,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend if (!func) { if (!trace) { | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] - || ZEND_ASSERT(ZEND_ACC_DEPRECATED <= MAX_IMM12); + || ZEND_ASSERT(ZEND_ACC_DEPRECATED <= TST_W_IMM); | tst TMP1w, #ZEND_ACC_DEPRECATED | bne >1 |.cold_code @@ -6979,7 +7033,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend for (i = call_num_args; i < func->op_array.last_var; i++) { uint32_t n = EX_NUM_TO_VAR(i); | // ZVAL_UNDEF(EX_VAR(n)) - || ZEND_ASSERT(n <= MAX_IMM12); + || ZEND_ASSERT(n <= ADD_SUB_IMM); | add TMP1, RX, #n | SET_Z_TYPE_INFO TMP1, IS_UNDEF, TMP2w } @@ -7004,7 +7058,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | LOAD_IP_ADDR (func->op_array.opcodes + num_args) } else { | ldr REG0, EX->func - || ZEND_ASSERT((num_args * sizeof(zend_op)) <= MAX_IMM12); + || ZEND_ASSERT((num_args * sizeof(zend_op)) <= ADD_SUB_IMM); if (GCC_GLOBAL_REGS) { | ldr IP, [REG0, #offsetof(zend_op_array, opcodes)] if (num_args) { @@ -7065,7 +7119,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | // num_args = EX_NUM_ARGS(); | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] | // if (UNEXPECTED(num_args > first_extra_arg)) - || ZEND_ASSERT(func->op_array.num_args <= MAX_IMM12); + || ZEND_ASSERT(func->op_array.num_args <= TST_W_IMM); | cmp REG1w, #(func->op_array.num_args) } else { | // first_extra_arg = op_array->num_args; @@ -7113,7 +7167,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | // zval *var = EX_VAR_NUM(num_args); | lsl REG1, REG1, #4 | add REG1, REG1, FP - || ZEND_ASSERT(ZEND_CALL_FRAME_SLOT * sizeof(zval) <= MAX_IMM12); + || ZEND_ASSERT(ZEND_CALL_FRAME_SLOT * sizeof(zval) <= ADD_SUB_IMM); | add REG1, REG1, #(ZEND_CALL_FRAME_SLOT * sizeof(zval)) |2: | SET_Z_TYPE_INFO REG1, IS_UNDEF, TMP1w @@ -7171,7 +7225,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } | NIY // TODO } else { - || ZEND_ASSERT(ZEND_ACC_DEPRECATED <= MAX_IMM12); + || ZEND_ASSERT(ZEND_ACC_DEPRECATED <= TST_W_IMM); | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] | tst TMP1w, #ZEND_ACC_DEPRECATED | bne >1 @@ -7444,8 +7498,8 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend | EMALLOC sizeof(zend_reference), op_array, opline // Allocate space in REG0 | mov TMP1w, #2 | str TMP1w, [REG0] - || ZEND_ASSERT(GC_REFERENCE <= MAX_IMM12); - | mov TMP1w, #GC_REFERENCE + || ZEND_ASSERT(GC_REFERENCE <= MOVZ_IMM); + | movz TMP1w, #GC_REFERENCE | str TMP1w, [REG0, #offsetof(zend_reference, gc.u.type_info)] | str xzr, [REG0, #offsetof(zend_reference, sources.ptr)] ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val)); @@ -8521,11 +8575,10 @@ static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zen if (type_mask != 0) { if (is_power_of_two(type_mask)) { uint32_t type_code = concrete_type(type_mask); - || ZEND_ASSERT(type_code <= MAX_IMM12); | IF_NOT_ZVAL_TYPE res_addr, type_code, >1, TMP1w, TMP2 } else { | mov REG2w, #1 - | SAFE_MEM_ACC_WITH_UOFFSET ldrb, REG1w, Rx(Z_REG(res_addr)), Z_OFFSET(res_addr)+offsetof(zval, u1.v.type), TMP1 + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, REG1w, Rx(Z_REG(res_addr)), Z_OFFSET(res_addr)+offsetof(zval, u1.v.type), TMP1 | lsl REG2w, REG2w, REG1w | LOAD_32BIT_VAL TMP1w, type_mask | tst REG2w, TMP1w @@ -8878,14 +8931,14 @@ static int zend_jit_fetch_obj(dasm_State **Dst, if (!prop_info) { | ldr REG0, EX->run_time_cache - | LOAD_32BIT_VAL TMP1, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + | LOAD_32BIT_VAL TMP1w, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) | add TMP1, TMP1, REG0 | ldr REG2, [TMP1] | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] | cmp REG2, TMP1 | bne >5 | NIY // TODO: currently jump to Label 5. - | LOAD_32BIT_VAL TMP1, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) + | LOAD_32BIT_VAL TMP1w, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) | add TMP1, TMP1, REG0 | ldr REG0, [TMP1] may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); @@ -9465,9 +9518,10 @@ static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_i | IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7, TMP1w, TMP2 } | NIY // TODO + int val = -1; | LOAD_32BIT_VAL TMP1w, opline->op1.var + offsetof(zval, u2.fe_iter_idx) | ldr FCARG1w, [FP, TMP1] - | LOAD_32BIT_VAL TMP1w, -1 + | LOAD_32BIT_VAL TMP1w, val | cmp FCARG1w, TMP1w | beq >7 | EXT_CALL zend_hash_iterator_del, REG0 @@ -9498,11 +9552,7 @@ static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_i | SET_EX_OPLINE opline, REG0 | LOAD_ADDR CARG1, str - || if (len <= MAX_IMM12) { - | mov CARG2, #len - || } else { - | LOAD_64BIT_VAL CARG2, len - || } + | LOAD_64BIT_VAL CARG2, len | EXT_CALL zend_write, REG0 if (!zend_jit_check_exception(Dst)) { return 0; @@ -9784,7 +9834,7 @@ static int zend_jit_fetch_constant(dasm_State **Dst, | SET_EX_OPLINE opline, REG0 | // zend_quick_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num OPLINE_CC EXECUTE_DATA_CC); | LOAD_ADDR FCARG1x, zv - | LOAD_32BIT_VAL FCARG2x, opline->op1.num + | LOAD_32BIT_VAL FCARG2w, opline->op1.num | EXT_CALL zend_jit_get_constant, REG0 | mov REG0, RETVALx | // ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); From 44c35078ea887201df24553b1a3745c1447b2156 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 21 Apr 2021 13:05:36 +0300 Subject: [PATCH 095/229] Support for LONG math (except of MUL overflow detection) --- ext/opcache/jit/zend_jit_arm64.dasc | 70 +++++++++++++++++++----- ext/opcache/tests/jit/arm64/mul_001.phpt | 32 +++++++++++ ext/opcache/tests/jit/arm64/mul_002.phpt | 20 +++++++ 3 files changed, 108 insertions(+), 14 deletions(-) create mode 100644 ext/opcache/tests/jit/arm64/mul_001.phpt create mode 100644 ext/opcache/tests/jit/arm64/mul_002.phpt diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 1aee9f77e1d97..40de06840fa97 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -788,9 +788,6 @@ static void* dasm_labels[zend_lb_MAX]; || case ZEND_SUB: | LONG_ADD_SUB subs, reg, addr, tmp_reg1, tmp_reg2 || break; -|| case ZEND_MUL: -| NIY // LONG_MUL imul, reg, addr -|| break; || case ZEND_BW_OR: | LONG_BW_OP orr, reg, addr, tmp_reg1, tmp_reg2 || break; @@ -813,9 +810,6 @@ static void* dasm_labels[zend_lb_MAX]; || case ZEND_SUB: | subs dst_reg, src_reg1, src_reg2 || break; -|| case ZEND_MUL: -| NIY // LONG_OP imul, reg, addr -|| break; || case ZEND_BW_OR: | orr dst_reg, src_reg1, src_reg2 || break; @@ -3075,11 +3069,42 @@ static int zend_jit_math_long_long(dasm_State **Dst, (Z_MODE(op1_addr) == IS_CONST_ZVAL && IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr))) && is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))))) { - | NIY // TODO: test + if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 2) { + if (Z_MODE(op1_addr) == IS_REG) { + | adds Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) + } else { + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) + } + } else { + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | mov TMP1, #floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) + | lsl Rx(result_reg), Rx(result_reg), TMP1 + | // TODO: overflow may be missed + } + } else { + if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 2) { + if (Z_MODE(op2_addr) == IS_REG) { + | adds Rx(result_reg), Rx(Z_REG(op2_addr)), Rx(Z_REG(op2_addr)) + } else { + | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 + | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) + } + } else { + | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 + | mov TMP1, #floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) + | lsl Rx(result_reg), Rx(result_reg), TMP1 + | // TODO: overflow may be missed + } + } } else if (opcode == ZEND_DIV && (Z_MODE(op2_addr) == IS_CONST_ZVAL && is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { - | NIY // TODO: test + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | asr Rx(result_reg), Rx(result_reg), #floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) +#if 0 + /* x86 specific optimizations through LEA instraction are not supported on ARM */ } else if (opcode == ZEND_ADD && !may_overflow && Z_MODE(op1_addr) == IS_REG && @@ -3095,6 +3120,12 @@ static int zend_jit_math_long_long(dasm_State **Dst, Z_MODE(op1_addr) == IS_REG && Z_MODE(op2_addr) == IS_CONST_ZVAL) { | NIY // TODO: test +#endif + } else if (opcode == ZEND_MUL) { + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 + | mul Rx(result_reg), Rx(result_reg), TMP2 + | // TODO: overflow detection } else { | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) @@ -3103,7 +3134,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, /* +/- 0 */ may_overflow = 0; } else if (same_ops && opcode != ZEND_DIV) { - | NIY // TODO: test + | LONG_MATH_REG opcode, Rx(result_reg), Rx(result_reg), Rx(result_reg) } else { | LONG_MATH opcode, result_reg, op2_addr, TMP1, TMP2 } @@ -3122,7 +3153,11 @@ static int zend_jit_math_long_long(dasm_State **Dst, | mov Rx(Z_REG(res_addr)), Rx(result_reg) } } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - | NIY // TODO + | bvc >3 + |.cold_code + |3: + | EXT_JMP exit_addr, TMP1 + |.code } else { ZEND_UNREACHABLE(); } @@ -3130,7 +3165,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, if (res_info & MAY_BE_LONG) { | bvs >1 } else { - | NIY // TODO: test + | bvc >1 } } } @@ -3157,15 +3192,22 @@ static int zend_jit_math_long_long(dasm_State **Dst, if ((Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 1) || (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1)) { if (opcode == ZEND_ADD) { + uint64_t val = 0x43e0000000000000; if (Z_MODE(res_addr) == IS_REG) { - | NIY // TODO: test + | LOAD_64BIT_VAL TMP1, val + | fmov Rd(Z_REG(res_addr)-ZREG_V0), TMP1 } else { - uint64_t val = 0x43e0000000000000; | SET_ZVAL_LVAL res_addr, val, REG0, TMP1 } break; } else if (opcode == ZEND_SUB) { - | NIY // TODO: test + uint64_t val = 0xc3e0000000000000; + if (Z_MODE(res_addr) == IS_REG) { + | LOAD_64BIT_VAL TMP1, val + | fmov Rd(Z_REG(res_addr)-ZREG_V0), TMP1 + } else { + | SET_ZVAL_LVAL res_addr, val, REG0, TMP1 + } break; } } diff --git a/ext/opcache/tests/jit/arm64/mul_001.phpt b/ext/opcache/tests/jit/arm64/mul_001.phpt new file mode 100644 index 0000000000000..94a07bfce4e71 --- /dev/null +++ b/ext/opcache/tests/jit/arm64/mul_001.phpt @@ -0,0 +1,32 @@ +--TEST-- +JIT MUL: 001 integer multiplay +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +int(6) +int(12) +int(333) diff --git a/ext/opcache/tests/jit/arm64/mul_002.phpt b/ext/opcache/tests/jit/arm64/mul_002.phpt new file mode 100644 index 0000000000000..3c8eddb56c09b --- /dev/null +++ b/ext/opcache/tests/jit/arm64/mul_002.phpt @@ -0,0 +1,20 @@ +--TEST-- +JIT MUL: 002 integer overflow +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +float(1.343250910680478E+23) From 196928a8b3ad81080c2bdd86cb6a02d1c92ff545 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 21 Apr 2021 13:45:40 +0300 Subject: [PATCH 096/229] Remove dead conditions --- ext/opcache/jit/zend_jit_arm64.dasc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 40de06840fa97..3ca2662489b88 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -3070,7 +3070,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr))) && is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))))) { if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { - if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 2) { + if (Z_LVAL_P(Z_ZV(op2_addr)) == 2) { if (Z_MODE(op1_addr) == IS_REG) { | adds Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) } else { @@ -3084,7 +3084,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, | // TODO: overflow may be missed } } else { - if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 2) { + if (Z_LVAL_P(Z_ZV(op1_addr)) == 2) { if (Z_MODE(op2_addr) == IS_REG) { | adds Rx(result_reg), Rx(Z_REG(op2_addr)), Rx(Z_REG(op2_addr)) } else { From bd26aea30b61b98bd5a9b86687daa743d029f61d Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 21 Apr 2021 13:52:03 +0300 Subject: [PATCH 097/229] Fixed possible incorrect assumption if constant is a power of 2. --- ext/opcache/jit/zend_jit_arm64.dasc | 52 ++++++++++++++--------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 3ca2662489b88..c3577fa7b4527 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -3063,40 +3063,40 @@ static int zend_jit_math_long_long(dasm_State **Dst, } if (opcode == ZEND_MUL && - ((Z_MODE(op2_addr) == IS_CONST_ZVAL && + Z_MODE(op2_addr) == IS_CONST_ZVAL && IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op2_addr))) && - is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) || - (Z_MODE(op1_addr) == IS_CONST_ZVAL && - IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr))) && - is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))))) { - if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { - if (Z_LVAL_P(Z_ZV(op2_addr)) == 2) { - if (Z_MODE(op1_addr) == IS_REG) { - | adds Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) - } else { - | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 - | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) - } + is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { + + if (Z_LVAL_P(Z_ZV(op2_addr)) == 2) { + if (Z_MODE(op1_addr) == IS_REG) { + | adds Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) } else { | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 - | mov TMP1, #floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) - | lsl Rx(result_reg), Rx(result_reg), TMP1 - | // TODO: overflow may be missed + | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) } } else { - if (Z_LVAL_P(Z_ZV(op1_addr)) == 2) { - if (Z_MODE(op2_addr) == IS_REG) { - | adds Rx(result_reg), Rx(Z_REG(op2_addr)), Rx(Z_REG(op2_addr)) - } else { - | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 - | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) - } + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | mov TMP1, #floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) + | lsl Rx(result_reg), Rx(result_reg), TMP1 + | // TODO: overflow may be missed + } + } else if (opcode == ZEND_MUL && + Z_MODE(op2_addr) == IS_CONST_ZVAL && + IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op2_addr))) && + is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { + + if (Z_LVAL_P(Z_ZV(op1_addr)) == 2) { + if (Z_MODE(op2_addr) == IS_REG) { + | adds Rx(result_reg), Rx(Z_REG(op2_addr)), Rx(Z_REG(op2_addr)) } else { | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 - | mov TMP1, #floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) - | lsl Rx(result_reg), Rx(result_reg), TMP1 - | // TODO: overflow may be missed + | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) } + } else { + | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 + | mov TMP1, #floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) + | lsl Rx(result_reg), Rx(result_reg), TMP1 + | // TODO: overflow may be missed } } else if (opcode == ZEND_DIV && (Z_MODE(op2_addr) == IS_CONST_ZVAL && From aa01684b010c34c900d75c0f5e54d4eb2202b25b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 21 Apr 2021 15:43:46 +0300 Subject: [PATCH 098/229] Support for INC/DEC --- ext/opcache/jit/zend_jit_arm64.dasc | 139 +++++++++++++++++++++++++--- ext/opcache/tests/jit/inc_021.phpt | 29 ++++++ ext/opcache/tests/jit/inc_022.phpt | 31 +++++++ 3 files changed, 185 insertions(+), 14 deletions(-) create mode 100644 ext/opcache/tests/jit/inc_021.phpt create mode 100644 ext/opcache/tests/jit/inc_022.phpt diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index c3577fa7b4527..0fce4a0fc905b 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -64,6 +64,8 @@ |.define REG2w, w10 |.define FPR0, v0 |.define FPR1, v1 +|.define FPR0d, d0 +|.define FPR1d, d1 |.define ZREG_REG0, ZREG_X8 |.define ZREG_REG1, ZREG_X9 @@ -849,7 +851,6 @@ static void* dasm_labels[zend_lb_MAX]; |.macro SET_ZVAL_DVAL, addr, reg, tmp_reg || if (Z_MODE(addr) == IS_REG) { || if (reg != Z_REG(addr)) { -| NIY // TODO: test | fmov Rd(Z_REG(addr)-ZREG_V0), Rd(reg-ZREG_V0) || } || } else { @@ -1766,13 +1767,6 @@ static int zend_jit_invalid_this_stub(dasm_State **Dst) return 1; } -static int zend_jit_double_one_stub(dasm_State **Dst) -{ - |->one: - | NIY_STUB // TODO - return 1; -} - static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst) { if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) { @@ -2264,7 +2258,6 @@ static const zend_jit_stub zend_jit_stubs[] = { JIT_STUB(assign_var), JIT_STUB(assign_cv_noref), JIT_STUB(assign_cv), - JIT_STUB(double_one), #ifdef CONTEXT_THREADED_JIT JIT_STUB(context_threaded_call), #endif @@ -2964,7 +2957,6 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { | LONG_ADD_SUB_WITH_IMM adds, op1_def_addr, Z_L(1), TMP1, TMP2 } else { - | NIY // TODO: test | LONG_ADD_SUB_WITH_IMM subs, op1_def_addr, Z_L(1), TMP1, TMP2 } @@ -2976,17 +2968,39 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op zend_jit_trace_stack *stack; uint32_t old_op1_info, old_res_info = 0; - | NIY // TODO: test + | NIY // TODO: tracing } else if (may_overflow) { | bvs >1 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { - | NIY // TODO: test | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } |.cold_code |1: - | NIY // TODO: test + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + uint64_t val = 0x43e0000000000000; + if (Z_MODE(op1_def_addr) == IS_REG) { + | LOAD_64BIT_VAL TMP1, val + | fmov Rd(Z_REG(op1_def_addr)-ZREG_V0), TMP1 + } else { + | SET_ZVAL_LVAL op1_def_addr, val, REG0, TMP1 + } + } else { + uint64_t val = 0xc3e0000000000000; + if (Z_MODE(op1_def_addr) == IS_REG) { + | LOAD_64BIT_VAL TMP1, val + | fmov Rd(Z_REG(op1_def_addr)-ZREG_V0), TMP1 + } else { + | SET_ZVAL_LVAL op1_def_addr, val, REG0, TMP1 + } + } + if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL) { + | SET_ZVAL_TYPE_INFO op1_def_addr, IS_DOUBLE, TMP1w, TMP2 + } + if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && + opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_DOUBLE, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + } | b >3 |.code } else { @@ -2998,7 +3012,104 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { |.cold_code |2: - | NIY // TODO: test + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { + | SET_EX_OPLINE opline, REG0 + if (op1_info & MAY_BE_UNDEF) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2, TMP1w, TMP2 + | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | LOAD_32BIT_VAL FCARG1w, opline->op1.var + | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 + | EXT_CALL zend_jit_undefined_op_helper, REG0 + op1_info |= MAY_BE_NULL; + } + |2: + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + + | // ZVAL_DEREF(var_ptr); + if (op1_info & MAY_BE_REF) { + | IF_NOT_Z_TYPE, FCARG1x, IS_REFERENCE, >2, TMP1w + | GET_Z_PTR FCARG1x, FCARG1x + | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] + | cbz TMP1, >1 + if (RETURN_VALUE_USED(opline)) { + | LOAD_ZVAL_ADDR FCARG2x, res_addr + } else { + | mov FCARG2x, xzr + } + if (opline->opcode == ZEND_PRE_INC) { + | EXT_CALL zend_jit_pre_inc_typed_ref, REG0 + } else if (opline->opcode == ZEND_PRE_DEC) { + | EXT_CALL zend_jit_pre_dec_typed_ref, REG0 + } else if (opline->opcode == ZEND_POST_INC) { + | EXT_CALL zend_jit_post_inc_typed_ref, REG0 + } else if (opline->opcode == ZEND_POST_DEC) { + | EXT_CALL zend_jit_post_dec_typed_ref, REG0 + } else { + ZEND_UNREACHABLE(); + } + zend_jit_check_exception(Dst); + | b >3 + |1: + | add FCARG1x, FCARG1x, #offsetof(zend_reference, val) + |2: + } + + if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + + | ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | TRY_ADDREF op1_info, REG0w, REG2, TMP1 + } + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) { + | LOAD_ZVAL_ADDR FCARG2x, res_addr + | EXT_CALL zend_jit_pre_inc, REG0 + } else { + | EXT_CALL increment_function, REG0 + } + } else { + if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) { + | LOAD_ZVAL_ADDR FCARG2x, res_addr + | EXT_CALL zend_jit_pre_dec, REG0 + } else { + | EXT_CALL decrement_function, REG0 + } + } + if (may_throw) { + zend_jit_check_exception(Dst); + } + } else { + zend_reg tmp_reg; + + if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + } + if (Z_MODE(op1_def_addr) == IS_REG) { + tmp_reg = Z_REG(op1_def_addr); + } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { + tmp_reg = Z_REG(op1_addr); + } else { + tmp_reg = ZREG_FPR0; + } + | GET_ZVAL_DVAL tmp_reg, op1_addr, ZREG_TMP1 + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + uint64_t val = 0x3ff0000000000000; // 1.0 + | LOAD_64BIT_VAL TMP1, val + | fmov FPR1d, TMP1 + | fadd Rd(tmp_reg-ZREG_V0), Rd(tmp_reg-ZREG_V0), FPR1d + } else { + uint64_t val = 0x3ff0000000000000; // 1.0 + | LOAD_64BIT_VAL TMP1, val + | fmov FPR1d, TMP1 + | fsub Rd(tmp_reg-ZREG_V0), Rd(tmp_reg-ZREG_V0), FPR1d + } + | SET_ZVAL_DVAL op1_def_addr, tmp_reg, ZREG_TMP1 + if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && + opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | TRY_ADDREF op1_def_info, REG0w, REG1, TMP1 + } + } | b >3 |.code } diff --git a/ext/opcache/tests/jit/inc_021.phpt b/ext/opcache/tests/jit/inc_021.phpt new file mode 100644 index 0000000000000..2966d7b264d3e --- /dev/null +++ b/ext/opcache/tests/jit/inc_021.phpt @@ -0,0 +1,29 @@ +--TEST-- +JIT INC: 021 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +float(9.223372036854776E+18) +float(2.1) +float(-9.223372036854776E+18) +float(0.10000000000000009) diff --git a/ext/opcache/tests/jit/inc_022.phpt b/ext/opcache/tests/jit/inc_022.phpt new file mode 100644 index 0000000000000..75971cff6c2e9 --- /dev/null +++ b/ext/opcache/tests/jit/inc_022.phpt @@ -0,0 +1,31 @@ +--TEST-- +JIT INC: 022 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +string(3) "abd" +int(6) +float(2.1) +int(4) +float(0.10000000000000009) From c55af8d8288f6da6cd7605ae30bb51d16b3e90a4 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 21 Apr 2021 16:23:23 +0300 Subject: [PATCH 099/229] Support for CONCAT --- ext/opcache/jit/zend_jit_arm64.dasc | 13 ++++++++++++- ext/opcache/tests/jit/concat_001.phpt | 22 ++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 ext/opcache/tests/jit/concat_001.phpt diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 0fce4a0fc905b..33df0f49f351a 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -4092,7 +4092,18 @@ static int zend_jit_concat_helper(dasm_State **Dst, |.cold_code |6: } - | NIY // TODO + if (Z_REG(res_addr) != ZREG_FCARG1x || Z_OFFSET(res_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, res_addr + } + | LOAD_ZVAL_ADDR FCARG2x, op1_addr + | LOAD_ZVAL_ADDR CARG3, op2_addr + | SET_EX_OPLINE opline, REG0 + | EXT_CALL concat_function, REG0 + /* concatination with empty string may increase refcount */ + op1_info |= MAY_BE_RCN; + op2_info |= MAY_BE_RCN; + | FREE_OP op1_type, op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + | FREE_OP op2_type, op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 if (may_throw) { zend_jit_check_exception(Dst); } diff --git a/ext/opcache/tests/jit/concat_001.phpt b/ext/opcache/tests/jit/concat_001.phpt new file mode 100644 index 0000000000000..85d4633765d31 --- /dev/null +++ b/ext/opcache/tests/jit/concat_001.phpt @@ -0,0 +1,22 @@ +--TEST-- +JIT CONCAT: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +string(2) "ab" +string(2) "a5" From 17a404747d7d084e59b4bee2468fa30e0756a3f2 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 21 Apr 2021 18:33:16 +0300 Subject: [PATCH 100/229] Support for comparison opcodes --- ext/opcache/jit/zend_jit_arm64.dasc | 443 ++++++++++++++++++++++++---- 1 file changed, 383 insertions(+), 60 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 33df0f49f351a..13a4d89802119 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -582,7 +582,6 @@ static void* dasm_labels[zend_lb_MAX]; // Use DOUBLE_CMP for comparisons and use DOUBLE_OP for arithmetic operations. |.macro DOUBLE_CMP, ins, reg, addr, tmp_reg, fp_tmp_reg || if (Z_MODE(addr) == IS_CONST_ZVAL) { -| NIY // TODO | LOAD_ADDR Rx(tmp_reg), Z_ZV(addr) | ldr Rd(fp_tmp_reg-ZREG_V0), [Rx(tmp_reg)] | ins Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) @@ -590,7 +589,6 @@ static void* dasm_labels[zend_lb_MAX]; | SAFE_MEM_ACC_WITH_UOFFSET ldr, Rd(fp_tmp_reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) | ins Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) || } else if (Z_MODE(addr) == IS_REG) { -| NIY // TODO | ins Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) || } else { || ZEND_UNREACHABLE(); @@ -5102,10 +5100,28 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPZ_EX || smart_branch_opcode == ZEND_JMPNZ_EX) { - | NIY // TODO + | SET_ZVAL_TYPE_INFO res_addr, (result ? IS_TRUE : IS_FALSE), TMP1w, TMP2 } if (smart_branch_opcode && !exit_addr) { - | NIY // TODO + if (smart_branch_opcode == ZEND_JMPZ || + smart_branch_opcode == ZEND_JMPZ_EX) { + if (!result) { + | b => target_label + } + } else if (smart_branch_opcode == ZEND_JMPNZ || + smart_branch_opcode == ZEND_JMPNZ_EX) { + if (result) { + | b => target_label + } + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + if (!result) { + | b => target_label + } else { + | b => target_label2 + } + } else { + ZEND_UNREACHABLE(); + } } return 1; } @@ -5118,26 +5134,26 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, } } else if (Z_MODE(op1_addr) == IS_REG) { if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { - | NIY // TODO + | cmp Rx(Z_REG(op1_addr)), xzr } else { | LONG_CMP Z_REG(op1_addr), op2_addr, TMP1, TMP2 } } else if (Z_MODE(op2_addr) == IS_REG) { if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) { - | NIY // TODO + | cmp Rx(Z_REG(op2_addr)), xzr } else { - | NIY // TODO + | LONG_CMP Z_REG(op2_addr), op1_addr, TMP1, TMP2 } swap = 1; } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { | LONG_CMP_WITH_CONST cmp, op2_addr, Z_LVAL_P(Z_ZV(op1_addr)), TMP1, TMP2 swap = 1; } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { - | NIY // TODO + | LONG_CMP_WITH_CONST cmp, op1_addr, Z_LVAL_P(Z_ZV(op2_addr)), TMP1, TMP2 } else { | GET_ZVAL_LVAL ZREG_REG0, op1_addr, TMP1 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { - | NIY // TODO + | cmp Rx(ZREG_REG0), xzr } else { | LONG_CMP ZREG_REG0, op2_addr, TMP1, TMP2 } @@ -5147,7 +5163,36 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, if (smart_branch_opcode == ZEND_JMPZ_EX || smart_branch_opcode == ZEND_JMPNZ_EX) { - | NIY // TODO + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + | cset REG0w, eq + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | cset REG0w, ne + break; + case ZEND_IS_SMALLER: + if (swap) { + | cset REG0w, gt + } else { + | cset REG0w, lt + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | cset REG0w, ge + } else { + | cset REG0w, le + } + break; + default: + ZEND_UNREACHABLE(); + } + | add REG0w, REG0w, #2 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 } if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) { @@ -5156,17 +5201,33 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: - | NIY // TODO + if (exit_addr) { + | NIY // bne &exit_addr + } else { + | bne => target_label + } break; case ZEND_IS_NOT_EQUAL: - | NIY // TODO + if (exit_addr) { + | NIY // beq &exit_addr + } else { + | beq => target_label + } break; case ZEND_IS_NOT_IDENTICAL: - | NIY // TODO + if (exit_addr) { + | NIY // bne &exit_addr + } else { + | beq => target_label + } break; case ZEND_IS_SMALLER: if (swap) { - | NIY // TODO + if (exit_addr) { + | NIY // ble &exit_addr + } else { + | ble => target_label + } } else { if (exit_addr) { | bge >1 @@ -5175,12 +5236,24 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, | EXT_JMP exit_addr, TMP1 |.code } else { - | NIY // TODO + | bge => target_label } } break; case ZEND_IS_SMALLER_OR_EQUAL: - | NIY // TODO + if (swap) { + if (exit_addr) { + | NIY // blt &exit_addr + } else { + | blt => target_label + } + } else { + if (exit_addr) { + | NIY // bgt &exit_addr + } else { + | bgt => target_label + } + } break; default: ZEND_UNREACHABLE(); @@ -5192,37 +5265,89 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: - | NIY // TODO + if (exit_addr) { + | NIY // beq &exit_addr + } else { + | beq => target_label + } break; case ZEND_IS_NOT_EQUAL: - | NIY // TODO + if (exit_addr) { + | NIY // bne &exit_addr + } else { + | bne => target_label + } break; case ZEND_IS_NOT_IDENTICAL: - | NIY // TODO + if (exit_addr) { + | NIY // be &exit_addr + } else { + | bne => target_label + } break; case ZEND_IS_SMALLER: if (swap) { if (exit_addr) { - | NIY // TODO + | NIY // bgt &exit_addr } else { - | bgt => target_label + | bgt => target_label } } else { if (exit_addr) { - | NIY // TODO + | NIY // blt &exit_addr } else { - | blt => target_label + | blt => target_label } } break; case ZEND_IS_SMALLER_OR_EQUAL: - | NIY // TODO + if (swap) { + if (exit_addr) { + | NIY // bge &exit_addr + } else { + | bge => target_label + } + } else { + if (exit_addr) { + | NIY // ble &exit_addr + } else { + | ble => target_label + } + } break; default: ZEND_UNREACHABLE(); } } else if (smart_branch_opcode == ZEND_JMPZNZ) { - | NIY // TODO + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + | bne => target_label + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | beq => target_label + break; + case ZEND_IS_SMALLER: + if (swap) { + | ble => target_label + } else { + | bge => target_label + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | blt => target_label + } else { + | bgt => target_label + } + break; + default: + ZEND_UNREACHABLE(); + } + | b => target_label2 } else { ZEND_UNREACHABLE(); } @@ -5232,24 +5357,24 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: - | NIY // TODO + | cset REG0w, eq break; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: - | NIY // TODO + | cset REG0w, ne break; case ZEND_IS_SMALLER: if (swap) { - | NIY // TODO + | cset REG0w, gt } else { - | NIY // TODO + | cset REG0w, lt } break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { | cset REG0w, ge } else { - | NIY // TODO + | cset REG0w, le } break; default: @@ -5272,7 +5397,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z case ZEND_CASE: case ZEND_CASE_STRICT: if (exit_addr) { - | NIY // TODO + | NIY // bne &exit_addr } else { | bne => target_label } @@ -5280,7 +5405,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z case ZEND_IS_NOT_EQUAL: | bvs >1 if (exit_addr) { - | NIY // TODO + | NIY // beq &exit_addr } else { | beq => target_label } @@ -5292,14 +5417,14 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z case ZEND_IS_SMALLER: if (swap) { if (exit_addr) { - | NIY // TODO + | NIY // tracing } else { | bvs => target_label // TODO: why the NaN check is missing in x86? | bls => target_label } } else { if (exit_addr) { - | NIY // TODO + | NIY // bhs &exit_addr } else { | bhs => target_label } @@ -5308,14 +5433,14 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { if (exit_addr) { - | NIY // TODO + | NIY // tracing } else { | bvs => target_label // TODO: why the NaN check is missing in x86? | blo => target_label } } else { if (exit_addr) { - | NIY // TODO + | NIY // bhi &exit_addr } else { | bhi => target_label } @@ -5332,7 +5457,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z case ZEND_CASE_STRICT: | bvs >1 if (exit_addr) { - | NIY // TODO + | NIY // beq &exit_adr } else { | beq => target_label } @@ -5340,7 +5465,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z break; case ZEND_IS_NOT_EQUAL: if (exit_addr) { - | NIY // TODO + | NIY // bne &exit_adr } else { | bne => target_label } @@ -5351,7 +5476,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z case ZEND_IS_SMALLER: if (swap) { if (exit_addr) { - | NIY // TODO + | NIY // tracng } else { | bvs >1 // Always False if involving NaN | bhi => target_label @@ -5360,7 +5485,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z } else { | bvs >1 if (exit_addr) { - | NIY // TODO + | NIY // blo &exit_addr } else { | blo => target_label } @@ -5370,7 +5495,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { if (exit_addr) { - | NIY // TODO + | NIY // tracing } else { | bvs >1 // Always False if involving NaN | bhs => target_label @@ -5379,7 +5504,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z } else { | bvs >1 if (exit_addr) { - | NIY // TODO + | NIY // bls &exit_addr } else { | bls => target_label } @@ -5390,7 +5515,38 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z ZEND_UNREACHABLE(); } } else if (smart_branch_opcode == ZEND_JMPZNZ) { - | NIY // TODO + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_IS_IDENTICAL: + case ZEND_CASE: + case ZEND_CASE_STRICT: + | bne => target_label + break; + case ZEND_IS_NOT_EQUAL: + case ZEND_IS_NOT_IDENTICAL: + | bvs => target_label2 + | beq => target_label + break; + case ZEND_IS_SMALLER: + if (swap) { + | bvs => target_label + | bls => target_label + } else { + | bhs => target_label + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (swap) { + | bvs => target_label + | blo => target_label + } else { + | bhi => target_label + } + break; + default: + ZEND_UNREACHABLE(); + } + | b => target_label2 } else if (smart_branch_opcode == ZEND_JMPZ_EX) { switch (opline->opcode) { case ZEND_IS_EQUAL: @@ -5574,10 +5730,8 @@ static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, z bool swap = 0; if (Z_MODE(op1_addr) == IS_REG) { - | NIY // TODO | DOUBLE_CMP fcmp, Z_REG(op1_addr), op2_addr, ZREG_TMP1, ZREG_FPTMP } else if (Z_MODE(op2_addr) == IS_REG) { - | NIY // TODO: construct test cases to verify whether the missing NaN check in x86 is buggy or not. | DOUBLE_CMP fcmp, Z_REG(op2_addr), op1_addr, ZREG_TMP1, ZREG_FPTMP swap = 1; } else { @@ -5596,7 +5750,25 @@ static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_a if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ_EX || smart_branch_opcode == ZEND_JMPNZ_EX) { - | NIY // TODO + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_CASE: + | cset REG0w, eq + break; + case ZEND_IS_NOT_EQUAL: + | cset REG0w, ne + break; + case ZEND_IS_SMALLER: + | cset REG0w, lt + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | cset REG0w, le + break; + default: + ZEND_UNREACHABLE(); + } + | add REG0w, REG0w, #2 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 } if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) { @@ -5604,33 +5776,112 @@ static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_a case ZEND_IS_EQUAL: case ZEND_CASE: if (exit_addr) { - | NIY // TODO + | NIY // bne &exit_addr } else { | bne => target_label } break; case ZEND_IS_NOT_EQUAL: - | NIY // TODO + if (exit_addr) { + | NIY // beq &exit_addr + } else { + | beq => target_label + } break; case ZEND_IS_SMALLER: - | NIY // TODO + if (exit_addr) { + | NIY // bge &exit_addr + } else { + | bge => target_label + } break; case ZEND_IS_SMALLER_OR_EQUAL: - | NIY // TODO + if (exit_addr) { + | NIY // bgt &exit_addr + } else { + | bgt => target_label + } break; default: ZEND_UNREACHABLE(); } } else if (smart_branch_opcode == ZEND_JMPNZ || smart_branch_opcode == ZEND_JMPNZ_EX) { - | NIY // TODO + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_CASE: + if (exit_addr) { + | NIY // beq &exit_addr + } else { + | beq => target_label + } + break; + case ZEND_IS_NOT_EQUAL: + if (exit_addr) { + | NIY // bne &exit_addr + } else { + | bne => target_label + } + break; + case ZEND_IS_SMALLER: + if (exit_addr) { + | NIY // blt &exit_addr + } else { + | blt => target_label + } + break; + case ZEND_IS_SMALLER_OR_EQUAL: + if (exit_addr) { + | NIY // ble &exit_addr + } else { + | ble => target_label + } + break; + default: + ZEND_UNREACHABLE(); + } } else if (smart_branch_opcode == ZEND_JMPZNZ) { - | NIY // TODO + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_CASE: + | bne => target_label + break; + case ZEND_IS_NOT_EQUAL: + | beq => target_label + break; + case ZEND_IS_SMALLER: + | bge => target_label + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | bgt => target_label + break; + default: + ZEND_UNREACHABLE(); + } + | b => target_label2 } else { ZEND_UNREACHABLE(); } } else { - | NIY // TODO + switch (opline->opcode) { + case ZEND_IS_EQUAL: + case ZEND_CASE: + | cset REG0w, eq + break; + case ZEND_IS_NOT_EQUAL: + | cset REG0w, ne + break; + case ZEND_IS_SMALLER: + | cset REG0w, lt + break; + case ZEND_IS_SMALLER_OR_EQUAL: + | cset REG0w, le + break; + default: + ZEND_UNREACHABLE(); + } + | add REG0w, REG0w, #2 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 } return 1; @@ -5666,7 +5917,7 @@ static int zend_jit_cmp(dasm_State **Dst, if (op1_info & MAY_BE_DOUBLE) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4, TMP1w, TMP2 } else { - | NIY // TODO + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9, TMP1w, TMP2 } } if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { @@ -5683,7 +5934,7 @@ static int zend_jit_cmp(dasm_State **Dst, | b >6 |.code } else { - | NIY // TODO + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, TMP1w, TMP2 } } if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { @@ -5700,7 +5951,6 @@ static int zend_jit_cmp(dasm_State **Dst, if (!same_ops) { | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5, TMP1w, TMP2 } else { - | NIY // TODO | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, TMP1w, TMP2 } } @@ -5724,11 +5974,71 @@ static int zend_jit_cmp(dasm_State **Dst, } else if ((op1_info & MAY_BE_DOUBLE) && !(op1_info & MAY_BE_LONG) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { - | NIY // TODO + if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, TMP1w, TMP2 + } + if (op2_info & MAY_BE_DOUBLE) { + if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + if (!same_ops && (op2_info & MAY_BE_LONG)) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >3, TMP1w, TMP2 + } else { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, TMP1w, TMP2 + } + } + if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { + return 0; + } + } + if (!same_ops && (op2_info & MAY_BE_LONG)) { + if (op2_info & MAY_BE_DOUBLE) { + |.cold_code + } + |3: + if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, TMP1w, TMP2 + } + if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { + return 0; + } + if (op2_info & MAY_BE_DOUBLE) { + | b >6 + |.code + } + } } else if ((op2_info & MAY_BE_DOUBLE) && !(op2_info & MAY_BE_LONG) && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { - | NIY // TODO + if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, TMP1w, TMP2 + } + if (op1_info & MAY_BE_DOUBLE) { + if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { + if (!same_ops && (op1_info & MAY_BE_LONG)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >3, TMP1w, TMP2 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, TMP1w, TMP2 + } + } + if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { + return 0; + } + } + if (!same_ops && (op1_info & MAY_BE_LONG)) { + if (op1_info & MAY_BE_DOUBLE) { + |.cold_code + } + |3: + if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9, TMP1w, TMP2 + } + if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { + return 0; + } + if (op1_info & MAY_BE_DOUBLE) { + | b >6 + |.code + } + } } if (has_slow || @@ -5755,10 +6065,23 @@ static int zend_jit_cmp(dasm_State **Dst, } | LOAD_ZVAL_ADDR FCARG2x, op1_addr if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { - | NIY // TODO + | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, >1, TMP1w + | LOAD_32BIT_VAL FCARG1x, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval + |1: } if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { - | NIY // TODO + | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, TMP1w, TMP2 + | str FCARG2x, T1 // save + | LOAD_32BIT_VAL FCARG1x, opline->op2.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + | ldr FCARG2x, T1 // restore + | LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval + | b >2 + |1: + | LOAD_ZVAL_ADDR CARG3, op2_addr + |2: } else { | LOAD_ZVAL_ADDR CARG3, op2_addr } From 49286d92c991e8c4c43b63241deaf8b69ad0032c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 21 Apr 2021 20:12:35 +0300 Subject: [PATCH 101/229] Support for BOOL/BOOL_NOT with DOUBLE operand --- ext/opcache/jit/zend_jit_arm64.dasc | 58 ++++++++++++++++++++++++++++- ext/opcache/tests/jit/not_001.phpt | 26 +++++++++++++ ext/opcache/tests/jit/not_002.phpt | 22 +++++++++++ 3 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 ext/opcache/tests/jit/not_001.phpt create mode 100644 ext/opcache/tests/jit/not_002.phpt diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 13a4d89802119..09b2b077c83d8 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -6414,7 +6414,63 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ } if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - | NIY // TODO + | mov TMP1, #0 + | fmov FPR1d, TMP1 + | DOUBLE_CMP fcmp, ZREG_FPR1, op1_addr, ZREG_TMP1, ZREG_FPTMP + + if (set_bool) { + if (exit_addr) { + | NIY // tracing + } else if (false_label != (uint32_t)-1) { // JMPZ_EX + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + | bvs >1 + | beq => false_label + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + |1: + } else if (true_label != (uint32_t)-1) { // JMPNZ_EX + | bvs >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + | bne => true_label + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + } else if (set_bool_not) { // BOOL_NOT + | bvs >1 + | mov REG0w, #IS_TRUE + | beq >2 + |1: + | mov REG0w, #IS_FALSE + |2: + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + } else { // BOOL + | bvs >1 + | mov REG0w, #IS_TRUE + | bne >2 + |1: + | mov REG0w, #IS_FALSE + |2: + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + } + } else { + if (exit_addr) { + | NIY // tracing + } else { + ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1); + if (false_label != (uint32_t)-1) { + | bvs =>false_label + } else { + | bvs >1 + } + if (true_label != (uint32_t)-1) { + | bne =>true_label + if (false_label != (uint32_t)-1) { + | b =>false_label + } + } else { + | beq =>false_label + } + |1: + } + } } else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { |.cold_code diff --git a/ext/opcache/tests/jit/not_001.phpt b/ext/opcache/tests/jit/not_001.phpt new file mode 100644 index 0000000000000..0820b7e942ba6 --- /dev/null +++ b/ext/opcache/tests/jit/not_001.phpt @@ -0,0 +1,26 @@ +--TEST-- +JIT NOT: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +bool(false) +bool(true) +bool(false) +bool(true) diff --git a/ext/opcache/tests/jit/not_002.phpt b/ext/opcache/tests/jit/not_002.phpt new file mode 100644 index 0000000000000..68df4ab774db2 --- /dev/null +++ b/ext/opcache/tests/jit/not_002.phpt @@ -0,0 +1,22 @@ +--TEST-- +JIT NOT: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +bool(false) +bool(true) From c826f3f1dfa8915dd23c24c72dc9b63764173875 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 21 Apr 2021 20:18:00 +0300 Subject: [PATCH 102/229] Fixed copy/paste error --- ext/opcache/jit/zend_jit_arm64.dasc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 09b2b077c83d8..76a6a0f0af951 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -3190,9 +3190,9 @@ static int zend_jit_math_long_long(dasm_State **Dst, | // TODO: overflow may be missed } } else if (opcode == ZEND_MUL && - Z_MODE(op2_addr) == IS_CONST_ZVAL && - IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op2_addr))) && - is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { + Z_MODE(op1_addr) == IS_CONST_ZVAL && + IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr))) && + is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { if (Z_LVAL_P(Z_ZV(op1_addr)) == 2) { if (Z_MODE(op2_addr) == IS_REG) { From f2c161eda0da0f3f62ce30ed1bad95dcaf3a771c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 21 Apr 2021 21:34:34 +0300 Subject: [PATCH 103/229] Support for IS_IDENTICAL/IS_NOT_IDENTICAL --- ext/opcache/jit/zend_jit_arm64.dasc | 387 ++++++++++++++++++++++- ext/opcache/tests/jit/identical_001.phpt | 31 ++ ext/opcache/tests/jit/identical_002.phpt | 131 ++++++++ 3 files changed, 546 insertions(+), 3 deletions(-) create mode 100644 ext/opcache/tests/jit/identical_001.phpt create mode 100644 ext/opcache/tests/jit/identical_002.phpt diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 76a6a0f0af951..d08b344e05591 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -1078,8 +1078,9 @@ static void* dasm_labels[zend_lb_MAX]; | bne label |.endmacro -|.macro IF_Z_TYPE, zv, val, label -| IF_TYPE byte [zv+offsetof(zval, u1.v.type)], val, label +|.macro IF_Z_TYPE, zv, val, label, tmp_reg +| ldrb tmp_reg, [zv, #offsetof(zval, u1.v.type)] +| IF_TYPE tmp_reg, val, label |.endmacro |.macro IF_NOT_Z_TYPE, zv, val, label, tmp_reg @@ -6127,7 +6128,387 @@ static int zend_jit_identical(dasm_State **Dst, uint32_t identical_label = (uint32_t)-1; uint32_t not_identical_label = (uint32_t)-1; - | NIY // TODO + if (smart_branch_opcode && !exit_addr) { + if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { + if (smart_branch_opcode == ZEND_JMPZ) { + not_identical_label = target_label; + } else if (smart_branch_opcode == ZEND_JMPNZ) { + identical_label = target_label; + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + not_identical_label = target_label; + identical_label = target_label2; + } else { + ZEND_UNREACHABLE(); + } + } else { + if (smart_branch_opcode == ZEND_JMPZ) { + identical_label = target_label; + } else if (smart_branch_opcode == ZEND_JMPNZ) { + not_identical_label = target_label; + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + identical_label = target_label; + not_identical_label = target_label2; + } else { + ZEND_UNREACHABLE(); + } + } + } + + if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG && + (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) { + if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { + return 0; + } + return 1; + } else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE && + (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) { + if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { + return 0; + } + return 1; + } + + if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) { + op1_info |= MAY_BE_NULL; + op2_info |= MAY_BE_NULL; + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | IF_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w + |.cold_code + |1: + | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | SET_EX_OPLINE opline, REG0 + | LOAD_32BIT_VAL FCARG1w, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + if (may_throw) { + zend_jit_check_exception_undef_result(Dst, opline); + } + | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval + | b >1 + |.code + |1: + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + | IF_Z_TYPE FCARG2x, IS_UNDEF, >1, TMP1w + |.cold_code + |1: + | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | SET_EX_OPLINE opline, REG0 + | str FCARG1x, T1 // save + | LOAD_32BIT_VAL FCARG1w, opline->op2.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + if (may_throw) { + zend_jit_check_exception_undef_result(Dst, opline); + } + | ldr FCARG1x, T1 // restore + | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval + | b >1 + |.code + |1: + } else if (op1_info & MAY_BE_UNDEF) { + op1_info |= MAY_BE_NULL; + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | IF_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w + |.cold_code + |1: + | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | SET_EX_OPLINE opline, REG0 + | LOAD_32BIT_VAL FCARG1w, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + if (may_throw) { + zend_jit_check_exception_undef_result(Dst, opline); + } + | LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval + | b >1 + |.code + |1: + if (opline->op2_type != IS_CONST) { + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + } + } else if (op2_info & MAY_BE_UNDEF) { + op2_info |= MAY_BE_NULL; + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + | IF_Z_TYPE FCARG2x, IS_UNDEF, >1, TMP1w + |.cold_code + |1: + | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | SET_EX_OPLINE opline, REG0 + | LOAD_32BIT_VAL FCARG1w, opline->op2.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + if (may_throw) { + zend_jit_check_exception_undef_result(Dst, opline); + } + | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval + | b >1 + |.code + |1: + if (opline->op1_type != IS_CONST) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + } else if ((op1_info & op2_info & MAY_BE_ANY) != 0) { + if (opline->op1_type != IS_CONST) { + if (Z_MODE(op1_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); + if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) { + return 0; + } + op1_addr = real_addr; + } + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + if (opline->op2_type != IS_CONST) { + if (Z_MODE(op2_addr) == IS_REG) { + zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); + if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) { + return 0; + } + op2_addr = real_addr; + } + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + } + } + + if ((op1_info & op2_info & MAY_BE_ANY) == 0) { + if ((opline->opcode != ZEND_CASE_STRICT && + (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || + ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && + (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { + | SET_EX_OPLINE opline, REG0 + if (opline->opcode != ZEND_CASE_STRICT) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + } + | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + } + if (smart_branch_opcode) { + if (may_throw) { + zend_jit_check_exception_undef_result(Dst, opline); + } + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | NIY // b &exit_addr + } + } else if (not_identical_label != (uint32_t)-1) { + | b =>not_identical_label + } + } else { + | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE), TMP1w, TMP2 + if (may_throw) { + zend_jit_check_exception(Dst); + } + } + return 1; + } + + if (opline->op1_type & (IS_CV|IS_VAR)) { + | ZVAL_DEREF FCARG1x, op1_info, TMP1w + } + if (opline->op2_type & (IS_CV|IS_VAR)) { + | ZVAL_DEREF FCARG2x, op2_info, TMP1w + } + + if (has_concrete_type(op1_info) + && has_concrete_type(op2_info) + && concrete_type(op1_info) == concrete_type(op2_info) + && concrete_type(op1_info) <= IS_TRUE) { + if (smart_branch_opcode) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | NIY // b &exit_addr + } + } else if (identical_label != (uint32_t)-1) { + | b =>identical_label + } + } else { + | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE), TMP1w, TMP2 + } + } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) { + if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) { + if (smart_branch_opcode) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | NIY // b &exit_addr + } + } else if (identical_label != (uint32_t)-1) { + | b =>identical_label + } + } else { + | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE), TMP1w, TMP2 + } + } else { + if (smart_branch_opcode) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | NIY // b &exit_addr + } + } else if (not_identical_label != (uint32_t)-1) { + | b =>not_identical_label + } + } else { + | SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE), TMP1w, TMP2 + } + } + } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) { + zval *val = Z_ZV(op1_addr); + + | ldrb TMP1w, [FCARG2x, #offsetof(zval, u1.v.type)] + | cmp TMP1w, #Z_TYPE_P(val) + if (smart_branch_opcode) { + if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) { + | bne >8 + | SET_EX_OPLINE opline, REG0 + | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + if (may_throw) { + zend_jit_check_exception_undef_result(Dst, opline); + } + if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { + | NIY // b &exit_addr + } else if (identical_label != (uint32_t)-1) { + | b =>identical_label + } else { + | b >9 + } + |8: + } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { + | NIY // beq &exit_addr + } else if (identical_label != (uint32_t)-1) { + | beq =>identical_label + } else { + | beq >9 + } + } else { + if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { + | cset REG0w, eq + } else { + | cset REG0w, ne + } + | add REG0w, REG0w, #2 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + } + if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && + (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { + | SET_EX_OPLINE opline, REG0 + | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + if (may_throw) { + zend_jit_check_exception_undef_result(Dst, opline); + } + } + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | NIY // b &exit_addr + } + } else if (smart_branch_opcode && not_identical_label != (uint32_t)-1) { + | b =>not_identical_label + } + } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) { + zval *val = Z_ZV(op2_addr); + + | ldrb TMP1w, [FCARG1x, #offsetof(zval, u1.v.type)] + | cmp TMP1w, #Z_TYPE_P(val) + if (smart_branch_opcode) { + if (opline->opcode != ZEND_CASE_STRICT + && opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) { + | bne >8 + | SET_EX_OPLINE opline, REG0 + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + if (may_throw) { + zend_jit_check_exception_undef_result(Dst, opline); + } + if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { + | NIY // b &exit_addr + } else if (identical_label != (uint32_t)-1) { + | b =>identical_label + } else { + | b >9 + } + |8: + } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { + | NIY // beq &exit_addr + } else if (identical_label != (uint32_t)-1) { + | beq =>identical_label + } else { + | beq >9 + } + } else { + if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { + | cset REG0w, eq + } else { + | cset REG0w, ne + } + | add REG0w, REG0w, #2 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + } + if (opline->opcode != ZEND_CASE_STRICT + && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { + | SET_EX_OPLINE opline, REG0 + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + if (may_throw) { + zend_jit_check_exception_undef_result(Dst, opline); + } + } + if (smart_branch_opcode) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | NIY // b &exit_addr + } + } else if (not_identical_label != (uint32_t)-1) { + | b =>not_identical_label + } + } + } else { + if (opline->op1_type == IS_CONST) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + if (opline->op2_type == IS_CONST) { + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + } + | EXT_CALL zend_is_identical, REG0 + if ((opline->opcode != ZEND_CASE_STRICT && + (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || + ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && + (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { + | str REG0, T1 // save + | SET_EX_OPLINE opline, REG0 + if (opline->opcode != ZEND_CASE_STRICT) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + } + | FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + if (may_throw) { + zend_jit_check_exception_undef_result(Dst, opline); + } + | ldr REG0, T1 // restore + } + if (smart_branch_opcode) { + | cmp RETVALw, #0 + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | NIY // bne &exit_addr + } else { + | NIY // beq &exit_addr + } + } else if (not_identical_label != (uint32_t)-1) { + | beq =>not_identical_label + if (identical_label != (uint32_t)-1) { + | b =>identical_label + } + } else if (identical_label != (uint32_t)-1) { + | bne =>identical_label + } + } else { + if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { + | add RETVALw, RETVALw, #2 + } else { + | neg RETVALw, RETVALw + | add RETVALw, RETVALw, #3 + } + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, RETVALw, TMP1 + } + } + + |9: + if (may_throw) { + zend_jit_check_exception(Dst); + } return 1; } diff --git a/ext/opcache/tests/jit/identical_001.phpt b/ext/opcache/tests/jit/identical_001.phpt new file mode 100644 index 0000000000000..03b1f4ea7f068 --- /dev/null +++ b/ext/opcache/tests/jit/identical_001.phpt @@ -0,0 +1,31 @@ +--TEST-- +JIT IDENTICAL: 001 +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +bool(true) +bool(false) +bool(false) +bool(true) +bool(false) +bool(true) +bool(false) diff --git a/ext/opcache/tests/jit/identical_002.phpt b/ext/opcache/tests/jit/identical_002.phpt new file mode 100644 index 0000000000000..789a3f8d36efb --- /dev/null +++ b/ext/opcache/tests/jit/identical_002.phpt @@ -0,0 +1,131 @@ +--TEST-- +JIT IDENTICAL: 002 Comparison with NaN +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=1M +opcache.protect_memory=1 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +bool(true) +bool(false) +bool(false) +bool(false) +bool(true) +bool(true) +int(1) +int(0) +int(0) +int(0) +int(1) +int(1) +1 +5 +6 +8 +9 +A +!bool(true) +bool(false) +bool(false) +bool(false) +!bool(true) +!bool(true) +bool(true) +!bool(false) +!bool(false) +!bool(false) +bool(true) +bool(true) +bool(false) +bool(true) +int(0) +int(1) From 5800ea9a57625549032f31dcc7050213924d065e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 01:35:34 +0300 Subject: [PATCH 104/229] Support for FETCH_DIM* and ASSIGN_DIM (exception handling is incomplete) --- ext/opcache/jit/zend_jit_arm64.dasc | 580 ++++++++++++++++++++++++---- 1 file changed, 497 insertions(+), 83 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index d08b344e05591..c2122a470267a 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -1328,7 +1328,20 @@ static void* dasm_labels[zend_lb_MAX]; |.macro SEPARATE_ARRAY, addr, op_info, cold, tmp_reg1, tmp_reg2 || if (RC_MAY_BE_N(op_info)) { || if (Z_REG(addr) != ZREG_FP) { -| NIY // TODO +| GET_ZVAL_LVAL ZREG_REG0, addr, Rx(tmp_reg1) +|| if (RC_MAY_BE_1(op_info)) { +| // if (GC_REFCOUNT() > 1) +| ldr Rw(tmp_reg1), [REG0] +| cmp Rw(tmp_reg1), #1 +| bls >2 +|| } +|| if (Z_REG(addr) != ZREG_FCARG1x || Z_OFFSET(addr) != 0) { +| LOAD_ZVAL_ADDR FCARG1x, addr +|| } +| EXT_CALL zend_jit_zval_array_dup, REG0 +| mov REG0, RETVALx +|2: +| mov FCARG1x, REG0 || } else { | GET_ZVAL_LVAL ZREG_FCARG1x, addr, Rx(tmp_reg1) || if (RC_MAY_BE_1(op_info)) { @@ -1340,7 +1353,6 @@ static void* dasm_labels[zend_lb_MAX]; |.cold_code |1: || } else { -| NIY // TODO | bls >2 || } || } @@ -1433,30 +1445,30 @@ static void* dasm_labels[zend_lb_MAX]; |.macro UNDEFINED_OFFSET, opline || if (opline == last_valid_opline) { || zend_jit_use_last_valid_opline(); -| call ->undefined_offset_ex +| bl ->undefined_offset_ex || } else { -| SET_EX_OPLINE opline, r0 -| call ->undefined_offset +| SET_EX_OPLINE opline, REG0 +| bl ->undefined_offset || } |.endmacro |.macro UNDEFINED_INDEX, opline || if (opline == last_valid_opline) { || zend_jit_use_last_valid_opline(); -| call ->undefined_index_ex +| bl ->undefined_index_ex || } else { -| SET_EX_OPLINE opline, r0 -| call ->undefined_index +| SET_EX_OPLINE opline, REG0 +| bl ->undefined_index || } |.endmacro |.macro CANNOT_ADD_ELEMENT, opline || if (opline == last_valid_opline) { || zend_jit_use_last_valid_opline(); -| call ->cannot_add_element_ex +| bl ->cannot_add_element_ex || } else { -| SET_EX_OPLINE opline, r0 -| call ->cannot_add_element +| SET_EX_OPLINE opline, REG0 +| bl ->cannot_add_element || } |.endmacro @@ -1592,7 +1604,7 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) | add sp, sp, SPAD // stack alignment | EXT_JMP handler, REG0 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - | NIY_STUB // TODO: test + | NIY_STUB // TODO: tracing } else { | mov FCARG1x, FP | ldp FP, RX, T2 // retore FP and IP @@ -1709,7 +1721,8 @@ static int zend_jit_undefined_index_stub(dasm_State **Dst) static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) { |->cannot_add_element_ex: - | NIY_STUB // TODO + | SAVE_IP + | b ->cannot_add_element return 1; } @@ -1717,7 +1730,20 @@ static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst) static int zend_jit_cannot_add_element_stub(dasm_State **Dst) { |->cannot_add_element: - | NIY_STUB // TODO + | // sub r4, 8 + | ldr REG0, EX->opline + | ldrb TMP1w, OP:REG0->result_type + | cmp TMP1w, #IS_UNUSED + | beq >1 + | ldr REG0w, OP:REG0->result.var + | add REG0, REG0, FP + | SET_Z_TYPE_INFO REG0, IS_NULL, TMP1w + |1: + | mov CARG1, xzr + | LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied" + | EXT_JMP zend_throw_error, REG0 // tail call + | // add r4, 8 + | //ret return 1; } @@ -2458,7 +2484,6 @@ static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline) static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) { if (delayed_call_chain) { - | NIY // TODO: test if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { return 0; } @@ -4159,7 +4184,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o if (!exit_addr) { return 0; } - | NIY // TODO + | NIY // tracing } if (type == BP_VAR_W) { | // hval = Z_LVAL_P(dim); @@ -4196,22 +4221,24 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o if (val == 0) { | cmp REG0, #0 } else if (val > 0 && !op2_loaded) { - | NIY // TODO | LOAD_64BIT_VAL TMP1, val | cmp REG0, TMP1 } else { - | NIY // TODO | cmp REG0, FCARG2x } if (type == BP_JIT_IS) { - | NIY // TODO + if (not_found_exit_addr) { + | NIY // bls ¬_found_exit_addr + } else { + | bls >9 // NOT_FOUND + } } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { - | NIY // TODO + | NIY // bls &exit_addr } else if (type == BP_VAR_IS && not_found_exit_addr) { - | NIY // TODO + | NIY // bls ¬_found_exit_addr } else if (type == BP_VAR_IS && found_exit_addr) { - | NIY // TODO + | bls >7 // NOT_FOUND } else { | bls >2 // NOT_FOUND } @@ -4219,12 +4246,10 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o if (val >= 0) { | ldr REG0, [FCARG1x, #offsetof(zend_array, arData)] if (val != 0) { - | NIY // TODO | LOAD_64BIT_VAL TMP1, val * sizeof(Bucket) | add REG0, REG0, TMP1 } } else { - | NIY // TODO | mov REG0, FCARG2x | lsl REG0, REG0, #5 | ldr TMP1, [FCARG1x, #offsetof(zend_array, arData)] @@ -4234,7 +4259,34 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } switch (type) { case BP_JIT_IS: - | NIY // TODO + if (op1_info & MAY_BE_ARRAY_HASH) { + if (packed_loaded) { + | b >5 + } + |4: + if (!op2_loaded) { + | // hval = Z_LVAL_P(dim); + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + } + | EXT_CALL _zend_hash_index_find, REG0 + | mov REG0, RETVALx + if (not_found_exit_addr) { + | NIY // cbz REG0, ¬_found_exit_addr + } else { + | cbz REG0, >9 // NOT_FOUND + } + if (op2_info & MAY_BE_STRING) { + | b >5 + } + } else if (packed_loaded) { + if (op2_info & MAY_BE_STRING) { + | b >5 + } + } else if (not_found_exit_addr) { + | NIY // b ¬_found_exit_addr + } else { + | b >9 // NOT_FOUND + } break; case BP_VAR_R: case BP_VAR_IS: @@ -4245,45 +4297,72 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { /* perform IS_UNDEF check only after result type guard (during deoptimization) */ if (!found_exit_addr || (op1_info & MAY_BE_ARRAY_HASH)) { - | NIY // TODO + | NIY // IF_Z_TYPE r0, IS_UNDEF, &exit_addr } } else if (type == BP_VAR_IS && not_found_exit_addr) { - | NIY // TODO + | NIY // IF_Z_TYPE r0, IS_UNDEF, ¬_found_exit_addr } else if (type == BP_VAR_IS && found_exit_addr) { - | NIY // TODO + | IF_Z_TYPE REG0, IS_UNDEF, >7, TMP1w // NOT_FOUND } else { - | NIY // TODO + | IF_Z_TYPE REG0, IS_UNDEF, >2, TMP1w // NOT_FOUND } } if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (packed_loaded && (op1_info & MAY_BE_ARRAY_HASH))) { - | NIY // TODO + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { + | NIY // jmp &exit_addr + } else if (type == BP_VAR_IS && not_found_exit_addr) { + | NIY // jmp ¬_found_exit_addr + } else if (type == BP_VAR_IS && found_exit_addr) { + | b >7 // NOT_FOUND + } else { + | b >2 // NOT_FOUND + } } if (op1_info & MAY_BE_ARRAY_HASH) { |4: if (!op2_loaded) { - | NIY // TODO + | // hval = Z_LVAL_P(dim); + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 } | EXT_CALL _zend_hash_index_find, REG0 | mov REG0, RETVALx - | tst REG0, REG0 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { - | NIY // TODO + | NIY // cbz REG0, &exit_addr } else if (type == BP_VAR_IS && not_found_exit_addr) { - | NIY // TODO + | NIY // cbz REG0, ¬_found_exit_addr } else if (type == BP_VAR_IS && found_exit_addr) { - | NIY // TODO + | cbz REG0, >7 // NOT_FOUND } else { - | beq >2 // NOT_FOUND + | cbz REG0, >2 // NOT_FOUND } } |.cold_code |2: - | NIY // TODO + switch (type) { + case BP_VAR_R: + if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { + | // zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval); + | // retval = &EG(uninitialized_zval); + | UNDEFINED_OFFSET opline + | b >9 + } + break; + case BP_VAR_IS: + case BP_VAR_UNSET: + if (!not_found_exit_addr && !found_exit_addr) { + | // retval = &EG(uninitialized_zval); + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 + | b >9 + } + break; + default: + ZEND_UNREACHABLE(); + } |.code break; case BP_VAR_RW: if (packed_loaded) { - | NIY // TODO + | IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w } |2: |4: @@ -4305,18 +4384,23 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | //retval = zend_hash_index_add_new(ht, hval, &EG(uninitialized_zval)); if (!op2_loaded) { | // hval = Z_LVAL_P(dim); - | NIY // TODO | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 } | LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval | EXT_CALL zend_hash_index_add_new, REG0 | mov REG0, RETVALx if (op1_info & MAY_BE_ARRAY_HASH) { - | NIY // TODO + | b >8 } } if (op1_info & MAY_BE_ARRAY_HASH) { - | NIY // TODO + |4: + if (!op2_loaded) { + | // hval = Z_LVAL_P(dim); + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + } + | EXT_CALL zend_hash_index_lookup, REG0 + | mov REG0, RETVALx } break; default: @@ -4339,7 +4423,26 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | // retval = zend_hash_find(ht, offset_key); switch (type) { case BP_JIT_IS: - | NIY // TODO + if (opline->op2_type != IS_CONST) { + | ldrb TMP1w, [FCARG2x, #offsetof(zend_string, val)] + | cmp TMP1w, #((uint8_t) ('9')) + | ble >1 + |.cold_code + |1: + | EXT_CALL zend_jit_symtable_find, REG0 + | b >1 + |.code + | EXT_CALL zend_hash_find, REG0 + |1: + } else { + | EXT_CALL _zend_hash_find_known_hash, REG0 + } + | mov REG0, RETVALx + if (not_found_exit_addr) { + | NIY // cbz REG0, ¬_found_exit_addr + } else { + | cbz REG0, >9 // NOT_FOUND + } break; case BP_VAR_R: case BP_VAR_IS: @@ -4356,31 +4459,49 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | EXT_CALL zend_hash_find, REG0 |1: } else { - | NIY // TODO | EXT_CALL _zend_hash_find_known_hash, REG0 } | mov REG0, RETVALx - | tst REG0, REG0 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { - | NIY // TODO + | NIY // cbz REG0, &exit_addr } else if (type == BP_VAR_IS && not_found_exit_addr) { - | NIY // TODO + | NIY // cbz REG0, ¬_found_exit_addr } else if (type == BP_VAR_IS && found_exit_addr) { - | NIY // TODO + | cbz REG0, >7 } else { - | beq >2 // NOT_FOUND + | cbz REG0, >2 // NOT_FOUND |.cold_code |2: - | NIY // TODO + switch (type) { + case BP_VAR_R: + // zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key)); + | UNDEFINED_INDEX opline + | b >9 + break; + case BP_VAR_IS: + case BP_VAR_UNSET: + | // retval = &EG(uninitialized_zval); + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 + | b >9 + break; + default: + ZEND_UNREACHABLE(); + } |.code } break; case BP_VAR_RW: - | NIY // TODO + | SET_EX_OPLINE opline, REG0 + if (opline->op2_type != IS_CONST) { + | EXT_CALL zend_jit_symtable_lookup_rw, REG0 + } else { + | EXT_CALL zend_jit_hash_lookup_rw, REG0 + } + | mov REG0, RETVALx + | cbz REG0, >9 break; case BP_VAR_W: if (opline->op2_type != IS_CONST) { - | NIY // TODO | EXT_CALL zend_jit_symtable_lookup_w, REG0 } else { | EXT_CALL zend_hash_lookup, REG0 @@ -4393,7 +4514,19 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) { - | NIY // TODO + |5: + if (op1_info & MAY_BE_ARRAY_OF_REF) { + | ZVAL_DEREF REG0, MAY_BE_REF, TMP1w + } + | ldrb TMP1w, [REG0,#offsetof(zval, u1.v.type)] + | cmp TMP1w, #IS_NULL + if (not_found_exit_addr) { + | NIY // jle ¬_found_exit_addr + } else if (found_exit_addr) { + | NIY // jg &found_exit_addr + } else { + | ble >9 // NOT FOUND + } } if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { @@ -4411,17 +4544,41 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | b >9 break; case BP_JIT_IS: - | NIY // TODO + | EXT_CALL zend_jit_fetch_dim_isset_helper, REG0 + | mov REG0, RETVALx + if (not_found_exit_addr) { + | NIY // cbz REG0, ¬_found_exit_addr + if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { + | b >8 + } + } else if (found_exit_addr) { + | NIY // cbnz REG0, &found_exit_addr + if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { + | b >9 + } + } else { + | cbnz REG0, >8 + | b >9 + } break; case BP_VAR_IS: case BP_VAR_UNSET: - | NIY // TODO + | LOAD_ZVAL_ADDR CARG3, res_addr + | EXT_CALL zend_jit_fetch_dim_is_helper, REG0 + | mov REG0, RETVALx + | b >9 break; case BP_VAR_RW: - | NIY // TODO + | EXT_CALL zend_jit_fetch_dim_rw_helper, REG0 + | mov REG0, RETVALx + | cbnz REG0, >8 + | b >9 break; case BP_VAR_W: - | NIY // TODO + | EXT_CALL zend_jit_fetch_dim_w_helper, RETVALx + | mov REG0, RETVALx + | cbnz REG0, >8 + | b >9 break; default: ZEND_UNREACHABLE(); @@ -4843,25 +5000,54 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t return 0; } - | NIY // TODO + | NIY // IF_ZVAL_TYPE op3_addr, IS_UNDEF, &exit_addr val_info &= ~MAY_BE_UNDEF; } if (op1_info & MAY_BE_REF) { - | NIY // TODO + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w + | GET_Z_PTR FCARG2x, FCARG1x + | ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))] + | cmp TMP1w, #IS_ARRAY + | bne >2 + | add FCARG1x, FCARG2x, #offsetof(zend_reference, val) + | b >3 + |.cold_code + |2: + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_prepare_assign_dim_ref, REG0 + | mov FCARG1x, RETVALx + | cbnz FCARG1x, >1 + | b ->exception_handler_undef + |.code + |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); } if (op1_info & MAY_BE_ARRAY) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { - | NIY // TODO | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, TMP1w, TMP2 } |3: | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { - | NIY // TODO + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + | CMP_ZVAL_TYPE op1_addr, IS_FALSE, TMP1w, TMP2 + | bgt >7 + } + | // ZVAL_ARR(container, zend_new_array(8)); + if (Z_REG(op1_addr) != ZREG_FP) { + | str Rx(Z_REG(op1_addr)), T1 // save + } + | EXT_CALL _zend_new_array_0, REG0 + if (Z_REG(op1_addr) != ZREG_FP) { + | ldr Rx(Z_REG(op1_addr)), T1 // restore + } + | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 + | mov FCARG1x, REG0 } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { @@ -4870,7 +5056,19 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t uint32_t var_info = MAY_BE_NULL; zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); - | NIY // TODO + | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); + | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval + | EXT_CALL zend_hash_next_index_insert, REG0 + | // if (UNEXPECTED(!var_ptr)) { + | mov REG0, RETVALx + | cbz REG0, >1 + |.cold_code + |1: + | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); + | CANNOT_ADD_ELEMENT opline + | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); + | b >9 + |.code if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0)) { return 0; @@ -4915,16 +5113,60 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) && (op1_info & MAY_BE_ARRAY)) { - | NIY // TODO + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + | CMP_ZVAL_TYPE op1_addr, IS_FALSE, TMP1w, TMP2 + | bgt >2 + } + | // ZVAL_ARR(container, zend_new_array(8)); + if (Z_REG(op1_addr) != ZREG_FP) { + | str Rx(Z_REG(op1_addr)), T1 // save + } + | EXT_CALL _zend_new_array_0, REG0 + if (Z_REG(op1_addr) != ZREG_FP) { + | ldr Rx(Z_REG(op1_addr)), T1 // restore + } + | SET_ZVAL_LVAL_FROM_REG op1_addr, RETVALx, TMP1 + | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 + | mov FCARG1x, RETVALx + | // ZEND_VM_C_GOTO(assign_dim_op_new_array); + | b <6 + |2: } if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { - | NIY // TODO + | SET_EX_OPLINE opline, REG0 + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + if (opline->op2_type == IS_UNUSED) { + | mov FCARG2x, xzr + } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { + ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); + | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) + } else { + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + } + if (opline->result_type == IS_UNUSED) { + | mov CARG4, xzr + } else { + | LOAD_ZVAL_ADDR CARG4, res_addr + } + | LOAD_ZVAL_ADDR CARG3, op3_addr + | EXT_CALL zend_jit_assign_dim_helper, REG0 + +#ifdef ZEND_JIT_USE_RC_INFERENCE + if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) { + /* ASSIGN_DIM may increase refcount of the value */ + val_info |= MAY_BE_RCN; + } +#endif + + | FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2 } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { - | NIY // TODO + | b >9 // END } |.code } @@ -9108,13 +9350,12 @@ static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, ze | IF_NOT_REFCOUNTED REG2w, >2 | and TMP2w, REG2w, #0xff // TMP2w -> low 8 bits of REG2w | IF_NOT_TYPE TMP2w, IS_REFERENCE, >1 - | NIY // TODO - | add TMP3, REG1, #offsetof(zend_reference, val) - | GET_Z_TYPE_INFO REG2w, TMP3 - | GET_Z_PTR REG1, TMP3 + | add REG1, REG1, #offsetof(zend_reference, val) + | GET_Z_TYPE_INFO REG2w, REG1 + | GET_Z_PTR REG1, REG1 | IF_NOT_REFCOUNTED REG2w, >2 |1: - | GC_ADDREF REG1, TMP1w + | GC_ADDREF REG1, TMP2w |2: | SET_ZVAL_PTR res_addr, REG1, TMP1 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 @@ -9256,13 +9497,18 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, } if (op1_info & MAY_BE_REF) { - | NIY // TODO + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | ZVAL_DEREF FCARG1x, op1_info, TMP1w op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); } if (op1_info & MAY_BE_ARRAY) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { - | NIY // TODO + if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) { + | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, &exit_addr + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, TMP1w, TMP2 + } } | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr, TMP1 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, res_exit_addr, not_found_exit_addr, exit_addr)) { @@ -9276,7 +9522,107 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, |7: } - | NIY // TODO + if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) { + if (exit_addr && !(op1_info & MAY_BE_OBJECT)) { + | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &exit_addr + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, TMP1w, TMP2 + } + } + | SET_EX_OPLINE opline, REG0 + | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr, TMP1 + if (opline->opcode != ZEND_FETCH_DIM_IS) { + if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) { + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + | EXT_CALL zend_jit_fetch_dim_str_offset_r_helper, REG0 + } else { + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + | EXT_CALL zend_jit_fetch_dim_str_r_helper, REG0 + } + | SET_ZVAL_PTR res_addr, RETVALx, TMP1 + | SET_ZVAL_TYPE_INFO res_addr, IS_STRING, TMP1w, TMP2 + } else { + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + | LOAD_ZVAL_ADDR CARG3, res_addr + | EXT_CALL zend_jit_fetch_dim_str_is_helper, REG0 + } + if ((op1_info & MAY_BE_ARRAY) || + (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING)))) { + | b >9 // END + } + |6: + } + + if (op1_info & MAY_BE_OBJECT) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) { + if (exit_addr) { + | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6, TMP1w, TMP2 + } + } + | SET_EX_OPLINE opline, REG0 + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { + ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); + | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) + } else { + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + } + | LOAD_ZVAL_ADDR CARG3, res_addr + if (opline->opcode != ZEND_FETCH_DIM_IS) { + | EXT_CALL zend_jit_fetch_dim_obj_r_helper, REG0 + } else { + | EXT_CALL zend_jit_fetch_dim_obj_is_helper, REG0 + } + if ((op1_info & MAY_BE_ARRAY) || + (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { + | b >9 // END + } + |6: + } + + if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) { + | SET_EX_OPLINE opline, REG0 + if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, TMP1w, TMP2 + | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | LOAD_32BIT_VAL FCARG1w, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + |1: + } + + if (op2_info & MAY_BE_UNDEF) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, TMP1w, TMP2 + | LOAD_32BIT_VAL FCARG1w, opline->op2.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + |1: + } + } + + if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) + && (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_STRING)))) { + if (opline->opcode != ZEND_FETCH_DIM_IS && opline->opcode != ZEND_FETCH_LIST_R) { + if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { + | LOAD_ZVAL_ADDR FCARG1x, orig_op1_addr + } else { + | SET_EX_OPLINE opline, REG0 + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || + Z_REG(op1_addr) != ZREG_FCARG1x || + Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + } + | EXT_CALL zend_jit_invalid_array_access, REG0 + } + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 + if (op1_info & MAY_BE_ARRAY) { + | b >9 // END + } + } if (op1_info & MAY_BE_ARRAY) { |.code @@ -9290,9 +9636,10 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, if (res_exit_addr) { zend_uchar type = concrete_type(res_info); - | NIY // TODO + | NIY // tracing } else if (op1_info & MAY_BE_ARRAY_OF_REF) { - | NIY // TODO + | // ZVAL_COPY_DEREF + | GET_ZVAL_TYPE_INFO Rw(ZREG_REG2), val_addr, TMP1 if (!zend_jit_zval_copy_deref(Dst, res_addr, val_addr, ZREG_REG2)) { return 0; } @@ -9338,7 +9685,24 @@ static int zend_jit_fetch_dim(dasm_State **Dst, op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; if (op1_info & MAY_BE_REF) { - | NIY // TODO + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w + | GET_Z_PTR FCARG2x, FCARG1x + | ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))] + | cmp TMP1w, #IS_ARRAY + | bne >2 + | add FCARG1x, FCARG2x, #offsetof(zend_reference, val) + | b >3 + |.cold_code + |2: + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_prepare_assign_dim_ref, REG0 + | mov FCARG1x, RETVALx + | cbnz FCARG1x, >1 + | b ->exception_handler_undef + |.code + |1: + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); } if (op1_info & MAY_BE_ARRAY) { @@ -9354,12 +9718,12 @@ static int zend_jit_fetch_dim(dasm_State **Dst, |7: } if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { - | NIY // TODO + | CMP_ZVAL_TYPE op1_addr, IS_FALSE, TMP1w, TMP2 + | bgt >7 } if ((op1_info & MAY_BE_UNDEF) && opline->opcode == ZEND_FETCH_DIM_RW) { if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { - | NIY // TODO | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, TMP1w, TMP2 } | SET_EX_OPLINE opline, REG0 @@ -9369,18 +9733,17 @@ static int zend_jit_fetch_dim(dasm_State **Dst, } | // ZVAL_ARR(container, zend_new_array(8)); if (Z_REG(op1_addr) != ZREG_FP) { - | NIY // TODO + | str Rx(Z_REG(op1_addr)), T1 // save } | EXT_CALL _zend_new_array_0, REG0 | mov REG0, RETVALx if (Z_REG(op1_addr) != ZREG_FP) { - | NIY // TODO + | ldr Rx(Z_REG(op1_addr)), T1 // restore } | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 | mov FCARG1x, REG0 if (op1_info & MAY_BE_ARRAY) { - | NIY // TODO | b >1 |.code |1: @@ -9390,7 +9753,21 @@ static int zend_jit_fetch_dim(dasm_State **Dst, if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { |6: if (opline->op2_type == IS_UNUSED) { - | NIY // TODO + | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); + | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval + | EXT_CALL zend_hash_next_index_insert, REG0 + | // if (UNEXPECTED(!var_ptr)) { + | cbz RETVALx, >1 + |.cold_code + |1: + | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); + | CANNOT_ADD_ELEMENT opline + | SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF, TMP1w, TMP2 + | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); + | b >8 + |.code + | SET_ZVAL_PTR res_addr, RETVALx, TMP1 + | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2 } else { uint32_t type; @@ -9420,14 +9797,51 @@ static int zend_jit_fetch_dim(dasm_State **Dst, if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { |.cold_code |9: - | NIY // TODO + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 + | b >8 |.code } } } if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { - | NIY // TODO + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + |.cold_code + |7: + } + + | SET_EX_OPLINE opline, REG0 + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + if (opline->op2_type == IS_UNUSED) { + | mov FCARG2x, xzr + } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { + ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); + | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) + } else { + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + } + | LOAD_ZVAL_ADDR CARG3, res_addr + switch (opline->opcode) { + case ZEND_FETCH_DIM_W: + case ZEND_FETCH_LIST_W: + | EXT_CALL zend_jit_fetch_dim_obj_w_helper, REG0 + break; + case ZEND_FETCH_DIM_RW: + | EXT_CALL zend_jit_fetch_dim_obj_rw_helper, REG0 + break; +// case ZEND_FETCH_DIM_UNSET: +// | EXT_CALL zend_jit_fetch_dim_obj_unset_helper, REG0 +// break; + default: + ZEND_UNREACHABLE(); + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + | b >8 // END + |.code + } } #ifdef ZEND_JIT_USE_RC_INFERENCE From e5e83e2ff3747144dea57f6c96342555c542162b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 09:05:52 +0300 Subject: [PATCH 105/229] Don't use FPR1 for consistency with x86 --- ext/opcache/jit/zend_jit_arm64.dasc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index c2122a470267a..c0d06cb116540 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -83,6 +83,7 @@ |.define TMP4, x14 |.define TMP4w, w14 |.define FPTMP, v16 +|.define FPTMPd, d16 |.define ZREG_TMP1, ZREG_X11 |.define ZREG_TMP2, ZREG_X12 @@ -3119,13 +3120,13 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { uint64_t val = 0x3ff0000000000000; // 1.0 | LOAD_64BIT_VAL TMP1, val - | fmov FPR1d, TMP1 - | fadd Rd(tmp_reg-ZREG_V0), Rd(tmp_reg-ZREG_V0), FPR1d + | fmov FPTMPd, TMP1 + | fadd Rd(tmp_reg-ZREG_V0), Rd(tmp_reg-ZREG_V0), FPTMPd } else { uint64_t val = 0x3ff0000000000000; // 1.0 | LOAD_64BIT_VAL TMP1, val - | fmov FPR1d, TMP1 - | fsub Rd(tmp_reg-ZREG_V0), Rd(tmp_reg-ZREG_V0), FPR1d + | fmov FPTMPd, TMP1 + | fsub Rd(tmp_reg-ZREG_V0), Rd(tmp_reg-ZREG_V0), FPTMPd } | SET_ZVAL_DVAL op1_def_addr, tmp_reg, ZREG_TMP1 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && @@ -7038,8 +7039,8 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { | mov TMP1, #0 - | fmov FPR1d, TMP1 - | DOUBLE_CMP fcmp, ZREG_FPR1, op1_addr, ZREG_TMP1, ZREG_FPTMP + | fmov FPR0d, TMP1 + | DOUBLE_CMP fcmp, ZREG_FPR0, op1_addr, ZREG_TMP1, ZREG_FPTMP if (set_bool) { if (exit_addr) { From 113e94f29afaf4baff839a46c79e3138e7d0433e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 09:37:28 +0300 Subject: [PATCH 106/229] Fixed incorrect register usage --- ext/opcache/jit/zend_jit_arm64.dasc | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index c0d06cb116540..0d7bd76a6e22b 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -6710,7 +6710,7 @@ static int zend_jit_identical(dasm_State **Dst, (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) || ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) { - | str REG0, T1 // save + | str RETVALw, T1 // save | SET_EX_OPLINE opline, REG0 if (opline->opcode != ZEND_CASE_STRICT) { | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 @@ -6719,23 +6719,22 @@ static int zend_jit_identical(dasm_State **Dst, if (may_throw) { zend_jit_check_exception_undef_result(Dst, opline); } - | ldr REG0, T1 // restore + | ldr RETVALw, T1 // restore } if (smart_branch_opcode) { - | cmp RETVALw, #0 if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { - | NIY // bne &exit_addr + | NIY // cbnz RETVALw, &exit_addr } else { - | NIY // beq &exit_addr + | NIY // cbz RETVALw, &exit_addr } } else if (not_identical_label != (uint32_t)-1) { - | beq =>not_identical_label + | cbz RETVALw, =>not_identical_label if (identical_label != (uint32_t)-1) { | b =>identical_label } } else if (identical_label != (uint32_t)-1) { - | bne =>identical_label + | cbnz RETVALw, =>identical_label } } else { if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { From 065948c19f4c19432c04c026a43df40e7ca8fb37 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 09:42:12 +0300 Subject: [PATCH 107/229] Fixed copy/paste error --- ext/opcache/jit/zend_jit_arm64.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 0d7bd76a6e22b..604a969fee9c2 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -3230,7 +3230,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, } } else { | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 - | mov TMP1, #floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) + | mov TMP1, #floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) | lsl Rx(result_reg), Rx(result_reg), TMP1 | // TODO: overflow may be missed } From be81cad8279736b0b980c464a25cf3478d34c885 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 10:12:57 +0300 Subject: [PATCH 108/229] Accurate RETVAL register usage --- ext/opcache/jit/zend_jit_arm64.dasc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 604a969fee9c2..501783d81a6e2 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -4576,7 +4576,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | b >9 break; case BP_VAR_W: - | EXT_CALL zend_jit_fetch_dim_w_helper, RETVALx + | EXT_CALL zend_jit_fetch_dim_w_helper, REG0 | mov REG0, RETVALx | cbnz REG0, >8 | b >9 @@ -5123,12 +5123,13 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t | str Rx(Z_REG(op1_addr)), T1 // save } | EXT_CALL _zend_new_array_0, REG0 + | mov REG0, RETVALx if (Z_REG(op1_addr) != ZREG_FP) { | ldr Rx(Z_REG(op1_addr)), T1 // restore } - | SET_ZVAL_LVAL_FROM_REG op1_addr, RETVALx, TMP1 + | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 - | mov FCARG1x, RETVALx + | mov FCARG1x, REG0 | // ZEND_VM_C_GOTO(assign_dim_op_new_array); | b <6 |2: From e6d0a220f3d16b9664017cacf70b55723d6aa239 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 10:32:33 +0300 Subject: [PATCH 109/229] Support for ISSET_DIM --- ext/opcache/jit/zend_jit_arm64.dasc | 158 +++++++++++++++++++++++++++- 1 file changed, 157 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 501783d81a6e2..6c31333f60676 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -9880,7 +9880,163 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, // TODO: support for empty() ??? ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); - | NIY // TODO + op2_addr = OP2_ADDR(); + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | ZVAL_DEREF FCARG1x, op1_info, TMP1w + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + + if (op1_info & MAY_BE_ARRAY) { + const void *found_exit_addr = NULL; + const void *not_found_exit_addr = NULL; + + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, TMP1w, TMP2 + } + | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr, TMP1 + if (exit_addr + && !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) + && !may_throw + && (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting) + && (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) { + if (smart_branch_opcode == ZEND_JMPNZ) { + found_exit_addr = exit_addr; + } else { + not_found_exit_addr = exit_addr; + } + } + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_JIT_IS, op1_info, op2_info, found_exit_addr, not_found_exit_addr, NULL)) { + return 0; + } + + if (found_exit_addr) { + |9: + return 1; + } else if (not_found_exit_addr) { + |8: + return 1; + } + } + + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { + if (op1_info & MAY_BE_ARRAY) { + |.cold_code + |7: + } + + if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) { + | SET_EX_OPLINE opline, REG0 + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { + ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); + | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) + } else { + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + } + | EXT_CALL zend_jit_isset_dim_helper, REG0 + | cbz RETVALw, >9 + if (op1_info & MAY_BE_ARRAY) { + | b >8 + |.code + } + } else { + if (op2_info & MAY_BE_UNDEF) { + if (op2_info & MAY_BE_ANY) { + | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, TMP1w, TMP2 + } + | LOAD_32BIT_VAL FCARG1w, opline->op2.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + |1: + } + if (op1_info & MAY_BE_ARRAY) { + | b >9 + |.code + } + } + } + +#ifdef ZEND_JIT_USE_RC_INFERENCE + if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { + /* Magic offsetExists() may increase refcount of the key */ + op2_info |= MAY_BE_RCN; + } +#endif + + if (op1_info & (MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)) { + |8: + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + if (!op1_avoid_refcounting) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + } + if (may_throw) { + if (!zend_jit_check_exception_undef_result(Dst, opline)) { + return 0; + } + } + if (!(opline->extended_value & ZEND_ISEMPTY)) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | NIY // b &exit_addr + } else { + | b >8 + } + } else if (smart_branch_opcode) { + if (smart_branch_opcode == ZEND_JMPZ) { + | b =>target_label2 + } else if (smart_branch_opcode == ZEND_JMPNZ) { + | b =>target_label + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + | b =>target_label2 + } else { + ZEND_UNREACHABLE(); + } + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + | b >8 + } + } else { + | NIY // TODO: support for empty() + } + } + + |9: // not found + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + if (!op1_avoid_refcounting) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + } + if (may_throw) { + if (!zend_jit_check_exception_undef_result(Dst, opline)) { + return 0; + } + } + if (!(opline->extended_value & ZEND_ISEMPTY)) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | NIY // b &exit_addr + } + } else if (smart_branch_opcode) { + if (smart_branch_opcode == ZEND_JMPZ) { + | b =>target_label + } else if (smart_branch_opcode == ZEND_JMPNZ) { + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + | b =>target_label + } else { + ZEND_UNREACHABLE(); + } + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + } + } else { + | NIY // TODO: support for empty() + } + + |8: + return 1; } From 09647a0a90589441c0aeea713297dd2110907ffa Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 10:44:58 +0300 Subject: [PATCH 110/229] Support for ASSIGN_OP --- ext/opcache/jit/zend_jit_arm64.dasc | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 6c31333f60676..bc40aecf59629 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -5213,7 +5213,22 @@ static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t if (op1_info & MAY_BE_REF) { binary_op_type binary_op = get_binary_op(opline->extended_value); - | NIY // TODO + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | IF_NOT_Z_TYPE, FCARG1x, IS_REFERENCE, >1, TMP1w + | GET_Z_PTR FCARG1x, FCARG1x + | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] + | cbnz TMP1, >2 + | add FCARG1x, FCARG1x, #offsetof(zend_reference, val) + |.cold_code + |2: + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + | LOAD_ADDR CARG3, binary_op + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 + zend_jit_check_exception(Dst); + | b >9 + |.code + |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); } @@ -5243,7 +5258,7 @@ static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t ZEND_UNREACHABLE(); } |9: - return 1; + return result; } static int zend_jit_is_constant_cmp_long_long(const zend_op *opline, From 0482102b044576e42f5f87fde44257b73249a5b4 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 11:08:15 +0300 Subject: [PATCH 111/229] Support for ASSIGN_DIM_OP --- ext/opcache/jit/zend_jit_arm64.dasc | 197 +++++++++++++++++++++++++++- 1 file changed, 196 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index bc40aecf59629..b7d01ae224a54 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -5197,7 +5197,202 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint3 ZEND_ASSERT(opline->result_type == IS_UNUSED); - | NIY // TODO + op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0; + op3_addr = OP1_DATA_ADDR(); + + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w + | GET_Z_PTR FCARG2x, FCARG1x + | ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))] + | cmp TMP1w, #IS_ARRAY + | bne >2 + | add FCARG1x, FCARG2x, #offsetof(zend_reference, val) + | b >3 + |.cold_code + |2: + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_prepare_assign_dim_ref, REG0 + | mov FCARG1x, RETVALx + | cbnz RETVALx, >1 + | b ->exception_handler_undef + |.code + |1: + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + + if (op1_info & MAY_BE_ARRAY) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, TMP1w, TMP2 + } + |3: + | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 + } + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { + if (op1_info & MAY_BE_ARRAY) { + |.cold_code + |7: + } + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + | CMP_ZVAL_TYPE op1_addr, IS_FALSE, TMP1w, TMP2 + | bgt >7 + } + if (op1_info & MAY_BE_UNDEF) { + if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, TMP1w, TMP2 + } + | SET_EX_OPLINE opline, REG0 + | LOAD_32BIT_VAL FCARG1x, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + |1: + } + | // ZVAL_ARR(container, zend_new_array(8)); + if (Z_REG(op1_addr) != ZREG_FP) { + | str Rx(Z_REG(op1_addr)), T1 // save + } + | EXT_CALL _zend_new_array_0, REG0 + | mov REG0, RETVALx + if (Z_REG(op1_addr) != ZREG_FP) { + | ldr Rx(Z_REG(op1_addr)), T1 // restore + } + | SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2 + | mov FCARG1x, REG0 + if (op1_info & MAY_BE_ARRAY) { + | b >1 + |.code + |1: + } + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + uint32_t var_info; + uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0); + + |6: + if (opline->op2_type == IS_UNUSED) { + var_info = MAY_BE_NULL; + + | // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); + | LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval + | EXT_CALL zend_hash_next_index_insert, REG0 + | mov REG0, RETVALx + | // if (UNEXPECTED(!var_ptr)) { + | cbz REG0, >1 + |.cold_code + |1: + | // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); + | CANNOT_ADD_ELEMENT opline + | //ZEND_VM_C_GOTO(assign_dim_op_ret_null); + | b >9 + |.code + } else { + var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); + if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { + var_info |= MAY_BE_REF; + } + if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + var_info |= MAY_BE_RC1; + } + + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_RW, op1_info, op2_info, NULL, NULL, NULL)) { + return 0; + } + + |8: + if (op1_info & (MAY_BE_ARRAY_OF_REF)) { + binary_op_type binary_op = get_binary_op(opline->extended_value); + | IF_NOT_Z_TYPE, REG0, IS_REFERENCE, >1, TMP1w + | GET_Z_PTR FCARG1x, REG0 + | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] + | cbnz TMP1, >2 + | add REG0, FCARG1x, #offsetof(zend_reference, val) + |.cold_code + |2: + | LOAD_ZVAL_ADDR FCARG2x, op3_addr + | LOAD_ADDR CARG3, binary_op + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 + zend_jit_check_exception(Dst); + | b >9 + |.code + |1: + } + } + + var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + switch (opline->extended_value) { + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: + case ZEND_DIV: + if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info, + 1 /* may overflow */, may_throw)) { + return 0; + } + break; + case ZEND_BW_OR: + case ZEND_BW_AND: + case ZEND_BW_XOR: + case ZEND_SL: + case ZEND_SR: + case ZEND_MOD: + if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, + IS_CV, opline->op1, var_addr, var_info, NULL, + (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, + op1_data_range, + 0, var_addr, var_def_info, var_info, may_throw)) { + return 0; + } + break; + case ZEND_CONCAT: + if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr, + may_throw)) { + return 0; + } + break; + default: + ZEND_UNREACHABLE(); + } + } + + if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { + binary_op_type binary_op; + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + |.cold_code + |7: + } + + | SET_EX_OPLINE opline, REG0 + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + if (opline->op2_type == IS_UNUSED) { + | mov FCARG2x, xzr + } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { + ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); + | LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1) + } else { + | LOAD_ZVAL_ADDR FCARG2x, op2_addr + } + binary_op = get_binary_op(opline->extended_value); + | LOAD_ZVAL_ADDR CARG3, op3_addr + | LOAD_ADDR CARG4, binary_op + | EXT_CALL zend_jit_assign_dim_op_helper, REG0 + if (!zend_jit_check_exception(Dst)) { + return 0; + } + + if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY)) { + | b >9 // END + |.code + } + } + + |9: + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + return 1; } From cb52d2731c316cbf7bb2f44a522e87a84f334dfa Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Wed, 21 Apr 2021 13:44:35 +0000 Subject: [PATCH 112/229] Support LONG MUL with overflow detection Overflow detection for LONG MUL is added in this patch. Quite different from 'subs' and 'adds' where overflow can be easily checked via the V flags, LONG MUL wouldn't set the flags. We use 'smulh' instruction to get the upper 64 bits of the 128-bit result and check the top 65 bits to tell whether integer overflow occurs. [1] Note that LONG MUL can be substituted by 'adds' or 'lsl' in some cases. Hence, flag 'use_mul' is introduced in order to select the proper overflow check check instruction afterwards. [1] https://community.arm.com/developer/ip-products/processors/b/processors-ip-blog/posts/detecting-overflow-from-mul Change-Id: I67e8287e9044c2a96b188d4bf6674736713abfe9 --- ext/opcache/jit/zend_jit_arm64.dasc | 45 ++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index b7d01ae224a54..e7bce53449fa2 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -3179,6 +3179,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); zend_reg result_reg; zend_reg tmp_reg = ZREG_REG0; + bool use_mul = 0; if (Z_MODE(res_addr) == IS_REG && (res_info & MAY_BE_LONG)) { if (may_overflow && (res_info & MAY_BE_GUARD) @@ -3258,10 +3259,22 @@ static int zend_jit_math_long_long(dasm_State **Dst, | NIY // TODO: test #endif } else if (opcode == ZEND_MUL) { - | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 - | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 - | mul Rx(result_reg), Rx(result_reg), TMP2 - | // TODO: overflow detection + use_mul = 1; + | GET_ZVAL_LVAL ZREG_TMP2, op1_addr, TMP1 + | GET_ZVAL_LVAL ZREG_TMP3, op2_addr, TMP1 + | mul Rx(result_reg), TMP2, TMP3 + if(may_overflow) { + /* Use 'smulh' to get the upper 64 bits fo the 128-bit result. + * For signed multiplication, the top 65 bits of the result will contain + * either all zeros or all ones if no overflow occurred. + * Note that 'cmp, TMP1, Rx(result_reg), asr, #63' is not supported by DynASM/arm64 + * currently, and we put 'asr' and 'cmp' separately. + * Flag: bne -> overflow. beq -> no overflow. + */ + | smulh TMP1, TMP2, TMP3 + | asr TMP2, Rx(result_reg), #63 + | cmp TMP1, TMP2 + } } else { | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) @@ -3280,7 +3293,11 @@ static int zend_jit_math_long_long(dasm_State **Dst, int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) { - | bvs >3 + if (use_mul) { + | bne >3 + } else { + | bvs >3 + } |.cold_code |3: | EXT_JMP exit_addr, TMP1 @@ -3289,7 +3306,11 @@ static int zend_jit_math_long_long(dasm_State **Dst, | mov Rx(Z_REG(res_addr)), Rx(result_reg) } } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - | bvc >3 + if (use_mul) { + | beq >3 + } else { + | bvc >3 + } |.cold_code |3: | EXT_JMP exit_addr, TMP1 @@ -3299,9 +3320,17 @@ static int zend_jit_math_long_long(dasm_State **Dst, } } else { if (res_info & MAY_BE_LONG) { - | bvs >1 + if (use_mul) { + | bne >1 + } else { + | bvs >1 + } } else { - | bvc >1 + if (use_mul) { + | beq >1 + } else { + | bvc >1 + } } } } From 29ff75165d9dc7d95603b47bbe30670a2cf6c0f0 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 12:04:31 +0300 Subject: [PATCH 113/229] Support for SEND_VAL/SEND_VAR/SEND_REF/CHECK_FUNC_ARG/CHECK_UNDEF_ARGS --- ext/opcache/jit/zend_jit_arm64.dasc | 127 ++++++++++++++++++++++++---- 1 file changed, 111 insertions(+), 16 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index e7bce53449fa2..415fc27153d6d 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -8812,7 +8812,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t o if (!exit_addr) { return 0; } - | NIY // TODO + | NIY // tracing } else { | ldr REG0, EX:RX->func | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] @@ -8821,7 +8821,6 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t o | bne >1 |.cold_code |1: - | NIY // TODO | SET_EX_OPLINE opline, REG0 | b ->throw_cannot_pass_by_ref |.code @@ -8835,7 +8834,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t o | ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 if (Z_REFCOUNTED_P(zv)) { - | NIY // TODO: test + | ADDREF_CONST zv, REG0, TMP1 } } else { | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 @@ -8846,7 +8845,18 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t o static int zend_jit_check_undef_args(dasm_State **Dst, const zend_op *opline) { - | NIY // TODO + | ldr FCARG1x, EX->call + | ldrb TMP1w, [FCARG1x, #(offsetof(zend_execute_data, This.u1.type_info) + 3)] + | tst TMP1w, #(ZEND_CALL_MAY_HAVE_UNDEF >> 24) + | bne >1 + |.cold_code + |1: + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_handle_undef_args, REG0 + | cbz RETVALw, >2 + | b ->exception_handler + |.code + |2: return 1; } @@ -8875,7 +8885,10 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend } else if (opline->op1_type == IS_CV) { if (op1_info & MAY_BE_UNDEF) { if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { - | NIY // TODO + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, TMP1w, TMP2 + | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 + | b >2 + |1: } op1_info &= ~MAY_BE_UNDEF; op1_info |= MAY_BE_NULL; @@ -8917,7 +8930,9 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend | SET_ZVAL_PTR val_addr, REG0, TMP1 | SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX, TMP1w, TMP2 } else { - | NIY // TODO + | ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | SET_ZVAL_PTR op1_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2 } | SET_ZVAL_PTR arg_addr, REG0, TMP1 | SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX, TMP1w, TMP2 @@ -8966,7 +8981,6 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend | bne >1 |.cold_code |1: - | NIY // TODO if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) { return 0; } @@ -8978,7 +8992,26 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { - | NIY // TODO + if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { + + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + + if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { + if (!(op1_info & MAY_BE_REF)) { + /* Don't generate code that always throws exception */ + return 0; + } else { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + | cmp REG1w, #IS_REFERENCE + | NIY // bne &exit_addr + } + } + return 1; + } } else { uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); @@ -8989,7 +9022,35 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend | bne >1 |.cold_code |1: - | NIY // TODO + + mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2); + + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + if (op1_info & MAY_BE_REF) { + | cmp REG1w, #IS_REFERENCE + | beq >7 + } + | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] + | LOAD_32BIT_VAL TMP2w, mask + | tst TMP1w, TMP2w + | bne >7 + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + | NIY // jmp &exit_addr + } else { + | SET_EX_OPLINE opline, REG0 + | LOAD_ZVAL_ADDR FCARG1x, arg_addr + | EXT_CALL zend_jit_only_vars_by_reference, REG0 + if (!zend_jit_check_exception(Dst)) { + return 0; + } + | b >7 + } + |.code } } else if (opline->opcode == ZEND_SEND_FUNC_ARG) { @@ -9025,7 +9086,6 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend |1: } - | NIY // TODO: test | SET_EX_OPLINE opline, REG0 | LOAD_32BIT_VAL FCARG1w, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, REG0 @@ -9033,16 +9093,35 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend | cbz RETVALx, ->exception_handler if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { - | NIY // TODO: test | b >7 |.code } else { - | NIY // TODO: test + |7: + return 1; } } if (opline->opcode == ZEND_SEND_VAR_NO_REF) { - | NIY // TODO: test + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + if (op1_info & MAY_BE_REF) { + | cmp REG1w, #IS_REFERENCE + | beq >7 + } + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + | NIY // jmp &exit_addr + } else { + | SET_EX_OPLINE opline, REG0 + | LOAD_ZVAL_ADDR FCARG1x, arg_addr + | EXT_CALL zend_jit_only_vars_by_reference, REG0 + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } } else { if (op1_info & MAY_BE_REF) { if (opline->op1_type == IS_CV) { @@ -9058,14 +9137,30 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, TMP1w, TMP2 |.cold_code |1: - | NIY // TODO: test. cold-code. not covered currently + | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + | // ZVAL_COPY_VALUE(return_value, &ref->value); + | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | GC_DELREF FCARG1x, TMP1 + | beq >1 + | IF_NOT_REFCOUNTED REG0w, >2 + | GC_ADDREF REG2, TMP1 + | b >2 + |1: + | EFREE_REFERENCE + | b >2 |.code | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 |2: } } else { if (op1_addr != op1_def_addr) { - | NIY // TODO: test + if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { + return 0; + } + if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { + op1_addr= op1_def_addr; + } } | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 if (opline->op1_type == IS_CV) { @@ -9086,7 +9181,7 @@ static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { - | NIY // TODO + | NIY // tracing } else { // if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) { uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); From a409eed4bf029e7aa4b0a695ce7bbb5bfa859366 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 13:03:52 +0300 Subject: [PATCH 114/229] Support for more cases in INIT_METHOD_CALL, DO_FCALL* and RETURN --- ext/opcache/jit/zend_jit_arm64.dasc | 188 +++++++++++++++++++++++----- 1 file changed, 154 insertions(+), 34 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 415fc27153d6d..5fbb1a3c62b83 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -8030,7 +8030,6 @@ static int zend_jit_init_method_call(dasm_State **Dst, if (opline->op1_type == IS_UNUSED || use_this) { zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); - | NIY // TODO | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 } else { if (op1_info & MAY_BE_REF) { @@ -8043,7 +8042,10 @@ static int zend_jit_init_method_call(dasm_State **Dst, } else { /* Hack: Convert reference to regular value to simplify JIT code */ ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP); - | NIY // TODO + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, TMP1w, TMP2 + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | EXT_CALL zend_jit_unref_helper, REG0 + |1: } } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { @@ -8054,12 +8056,11 @@ static int zend_jit_init_method_call(dasm_State **Dst, if (!exit_addr) { return 0; } - | NIY // TODO + | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, TMP1w, TMP2 |.cold_code |1: - | NIY // TODO: currently not jump to cold code. if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, op1_addr } @@ -8086,7 +8087,11 @@ static int zend_jit_init_method_call(dasm_State **Dst, if (func) { | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); - | NIY // TODO + | ldr REG0, EX->run_time_cache + | LOAD_32BIT_VAL TMP1w, (opline->result.num + sizeof(void*)) + | add REG0, REG0, TMP1 + | ldr REG0, [REG0] + | cbz REG0, >1 } else { | // if (CACHED_PTR(opline->result.num) == obj->ce)) { | ldr REG0, EX->run_time_cache @@ -8096,7 +8101,6 @@ static int zend_jit_init_method_call(dasm_State **Dst, | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] | cmp REG2, TMP1 | bne >1 - | NIY // TODO: currently jump to label 1. | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); | LOAD_32BIT_VAL TMP1w, (opline->result.num + sizeof(void*)) | add TMP1, TMP1, REG0 @@ -8142,9 +8146,9 @@ static int zend_jit_init_method_call(dasm_State **Dst, !func->common.function_name)) { const zend_op *opcodes = func->op_array.opcodes; - | NIY // TODO + | NIY // tracing } else { - | NIY // TODO + | NIY // tracing } } @@ -8159,11 +8163,21 @@ static int zend_jit_init_method_call(dasm_State **Dst, } if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) { - | NIY // TODO + | ldr FCARG1x, T1 // restore + | mov FCARG2x, REG0 + | LOAD_32BIT_VAL CARG3w, opline->extended_value + if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) { + | EXT_CALL zend_jit_push_static_metod_call_frame_tmp, REG0 + } else { + | EXT_CALL zend_jit_push_static_metod_call_frame, REG0 + } + if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !use_this)) { + | cbz RETVALx, ->exception_handler + } + | mov RX, RETVALx } if (!func) { - | NIY // TODO | b >9 |.code } @@ -8285,7 +8299,32 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } if (trace && !func) { - | NIY // TODO + if (trace->op == ZEND_JIT_TRACE_DO_ICALL) { + ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION); +#ifndef ZEND_WIN32 + // TODO: ASLR may cause different addresses in different workers ??? + func = trace->func; + if (JIT_G(current_frame) && + JIT_G(current_frame)->call && + TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { + call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); + } else { + unknown_num_args = 1; + } +#endif + } else if (trace->op == ZEND_JIT_TRACE_ENTER) { + ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION); + if (zend_accel_in_shm(trace->func->op_array.opcodes)) { + func = trace->func; + if (JIT_G(current_frame) && + JIT_G(current_frame)->call && + TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { + call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); + } else { + unknown_num_args = 1; + } + } + } } bool may_have_extra_named_params = @@ -8313,7 +8352,11 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend if (!exit_addr) { return 0; } - | NIY // TODO + | ldr REG0, EX:RX->func + | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] + || ZEND_ASSERT(ZEND_ACC_DEPRECATED <= TST_W_IMM); + | tst TMP1w, #ZEND_ACC_DEPRECATED + | NIY // bne &exit_addr } } } @@ -8345,7 +8388,6 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | bne >1 |.cold_code |1: - | NIY // TODO if (!GCC_GLOBAL_REGS) { | mov FCARG1x, RX } @@ -8359,7 +8401,11 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend |1: } } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { - | NIY // TODO + if (!GCC_GLOBAL_REGS) { + | mov FCARG1x, RX + } + | EXT_CALL zend_jit_deprecated_helper, REG0 + | cbz RETVALw, ->exception_handler } } @@ -8584,7 +8630,6 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } if (ZEND_OBSERVER_ENABLED) { - | NIY // TODO: test | SAVE_IP | mov FCARG1x, FP | EXT_CALL zend_observer_fcall_begin, REG0 @@ -8592,7 +8637,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend if (trace) { if (!func && (opline->opcode != ZEND_DO_UCALL)) { - | NIY // TODO + | b >9 } } else { #ifdef CONTEXT_THREADED_JIT @@ -8629,7 +8674,10 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend if (!exit_addr) { return 0; } - | NIY // TODO + || ZEND_ASSERT(ZEND_ACC_DEPRECATED <= TST_W_IMM); + | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] + | tst TMP1w, #ZEND_ACC_DEPRECATED + | NIY // bne &exit_addr } else { || ZEND_ASSERT(ZEND_ACC_DEPRECATED <= TST_W_IMM); | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] @@ -8637,7 +8685,6 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | bne >1 |.cold_code |1: - | NIY // TODO if (!GCC_GLOBAL_REGS) { | mov FCARG1x, RX } @@ -8651,7 +8698,12 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend |1: } } else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { - | NIY // TODO + if (!GCC_GLOBAL_REGS) { + | mov FCARG1x, RX + } + | EXT_CALL zend_jit_deprecated_helper, REG0 + | cbz RETVALw, ->exception_handler + | ldr REG0, EX:RX->func // reload } } @@ -8688,7 +8740,16 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | EXT_CALL zend_jit_vm_stack_free_args_helper, REG0 } if (may_have_extra_named_params) { - | NIY // TODO + | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 3)] + | tst TMP1w, #(ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24) + | bne >1 + |.cold_code + |1: + | ldr FCARG1x, [RX, #offsetof(zend_execute_data, extra_named_params)] + | EXT_CALL zend_free_extra_named_params, REG0 + | b >2 + |.code + |2: } |8: @@ -8771,7 +8832,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } else if (trace && trace->op == ZEND_JIT_TRACE_END && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { - | NIY // TODO + | LOAD_IP_ADDR (opline + 1) } } @@ -9583,9 +9644,19 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o return_value_used = -1; } - // TODO: This macro is only used in four sites. We should design a test variant to cover it. if (ZEND_OBSERVER_ENABLED) { - | NIY // TODO: test + if (Z_MODE(op1_addr) == IS_REG) { + zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); + + if (!zend_jit_spill_store(Dst, op1_addr, dst, op1_info, 1)) { + return 0; + } + op1_addr = dst; + } + | LOAD_ZVAL_ADDR FCARG2x, op1_addr + | mov FCARG1x, FP + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_observer_fcall_end, REG0 } // if (!EX(return_value)) @@ -9615,24 +9686,38 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o } if (return_value_used != 1) { if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - | NIY // TODO + if (jit_return_label >= 0) { + | IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label, TMP1w, TMP2 + } else { + | IF_NOT_ZVAL_REFCOUNTED op1_addr, >9, TMP1w, TMP2 + } } - | NIY // TODO + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + | GC_DELREF FCARG1x, TMP1 if (RC_MAY_BE_1(op1_info)) { - | NIY // TODO + if (RC_MAY_BE_N(op1_info)) { + if (jit_return_label >= 0) { + | bne =>jit_return_label + } else { + | bne >9 + } + } + | //SAVE_OPLINE() + | ZVAL_DTOR_FUNC op1_info, opline, TMP1 + | //????ldr REG1, EX->return_value // reload ??? } if (return_value_used == -1) { if (jit_return_label >= 0) { - | NIY // TODO + | b =>jit_return_label } else { - | NIY // TODO + | b >9 } |.code } } } else if (return_value_used == -1) { if (jit_return_label >= 0) { - | NIY // TODO: test + | beq =>jit_return_label } else { | beq >9 } @@ -9640,7 +9725,6 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o if (return_value_used == 0) { |9: - | NIY // TODO: test return 1; } @@ -9648,13 +9732,14 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o zval *zv = RT_CONSTANT(opline, opline->op1); | ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 if (Z_REFCOUNTED_P(zv)) { - | NIY // TODO: test + | ADDREF_CONST zv, REG0, TMP1 } } else if (opline->op1_type == IS_TMP_VAR) { | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } else if (opline->op1_type == IS_CV) { if (op1_info & MAY_BE_REF) { - | NIY // TODO + | LOAD_ZVAL_ADDR REG0, op1_addr + | ZVAL_DEREF REG0, op1_info, TMP1w op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); } | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 @@ -9665,12 +9750,47 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o | TRY_ADDREF op1_info, REG0w, REG2, TMP1 } else if (return_value_used != 1) { | // if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); - | NIY // TODO | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 } } } else { - | NIY // TODO + if (op1_info & MAY_BE_REF) { + zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val)); + + | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, TMP1w, TMP2 + |.cold_code + |1: + | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); + | GET_ZVAL_PTR REG0, op1_addr, TMP1 + | // ZVAL_COPY_VALUE(return_value, &ref->value); + | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_REG2, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | GC_DELREF REG0, TMP1 + | beq >2 + | // if (IS_REFCOUNTED()) + if (jit_return_label >= 0) { + | IF_NOT_REFCOUNTED REG2w, =>jit_return_label + } else { + | IF_NOT_REFCOUNTED REG2w, >9 + } + | // ADDREF + | GET_ZVAL_PTR REG2, ret_addr, TMP1 // reload + | GC_ADDREF REG2, TMP1 + if (jit_return_label >= 0) { + | b =>jit_return_label + } else { + | b >9 + } + |2: + | mov FCARG1x, REG0 + | EFREE_REFERENCE + if (jit_return_label >= 0) { + | b =>jit_return_label + } else { + | b >9 + } + |.code + } + | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } |9: From d9b2b26e991692e2977d8ede8e3aa9cc34c62934 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 13:20:07 +0300 Subject: [PATCH 115/229] Get rid of some NIY traps in DynADM macros --- ext/opcache/jit/zend_jit_arm64.dasc | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 5fbb1a3c62b83..2f3db1fe1384b 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -616,8 +616,8 @@ static void* dasm_labels[zend_lb_MAX]; // Convert the LONG value 'lval' into DOUBLE type, and move it into 'reg' |.macro DOUBLE_GET_LONG, reg, lval, tmp_reg || if (lval == 0) { -| NIY // TODO: test -| // vxorps xmm(reg-ZREG_V0), xmm(reg-ZREG_V0), xmm(reg-ZREG_V0) +| mov Rx(tmp_reg), xzr +| fmov Rd(reg-ZREG_V0), Rx(tmp_reg) || } else { | LOAD_64BIT_VAL Rx(tmp_reg), lval | scvtf Rd(reg-ZREG_V0), Rx(tmp_reg) @@ -633,7 +633,6 @@ static void* dasm_labels[zend_lb_MAX]; | SAFE_MEM_ACC_WITH_UOFFSET ldr, Rx(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg2) | scvtf Rd(reg-ZREG_V0), Rx(tmp_reg1) || } else if (Z_MODE(addr) == IS_REG) { -| NIY // TODO: test | scvtf Rd(reg-ZREG_V0), Rx(Z_REG(addr)) || } else { || ZEND_UNREACHABLE(); @@ -707,7 +706,6 @@ static void* dasm_labels[zend_lb_MAX]; || if(Z_LVAL_P(Z_ZV(addr)) >= 0 && Z_LVAL_P(Z_ZV(addr)) <= CMP_IMM) { | cmp Rx(reg), #(Z_LVAL_P(Z_ZV(addr))) || } else { -| NIY // TODO | LOAD_64BIT_VAL tmp_reg1, Z_LVAL_P(Z_ZV(addr)) | cmp Rx(reg), tmp_reg1 || } @@ -973,7 +971,6 @@ static void* dasm_labels[zend_lb_MAX]; | SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg) || } || } else if (Z_MODE(dst_addr) == IS_REG) { -| NIY // TODO: test | GET_ZVAL_LVAL Z_REG(dst_addr), src_addr, Rx(tmp_reg) || } else { | GET_ZVAL_LVAL reg2, src_addr, Rx(tmp_reg) @@ -981,13 +978,10 @@ static void* dasm_labels[zend_lb_MAX]; || } || } else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { || if (Z_MODE(src_addr) == IS_REG) { -| NIY // TODO: test | SET_ZVAL_DVAL dst_addr, Z_REG(src_addr), tmp_reg || } else if (Z_MODE(dst_addr) == IS_REG) { -| NIY // TODO: test | GET_ZVAL_DVAL Z_REG(dst_addr), src_addr, tmp_reg || } else { -| NIY // TODO: test | GET_ZVAL_DVAL fp_tmp_reg, src_addr, tmp_reg | SET_ZVAL_DVAL dst_addr, fp_tmp_reg, tmp_reg || } @@ -1256,7 +1250,7 @@ static void* dasm_labels[zend_lb_MAX]; || break; || } else if (type == IS_OBJECT) { || if (opline) { -| NIY // TODO +| SET_EX_OPLINE opline, REG0 || } | EXT_CALL zend_objects_store_del, tmp_reg || break; @@ -1416,7 +1410,6 @@ static void* dasm_labels[zend_lb_MAX]; | mov REG0, RETVALx || } ||#else -| NIY // TODO | mov FCARG1x, #size | EXT_CALL _emalloc, REG0 | mov REG0, RETVALx From 7d3e0a4af2de72674ef8498558a2cd4bea42d378 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 13:33:22 +0300 Subject: [PATCH 116/229] Support for ECHO with non-constant operand --- ext/opcache/jit/zend_jit_arm64.dasc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 2f3db1fe1384b..8219dc19650f7 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -11505,7 +11505,17 @@ static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_i ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); - | NIY // TODO: test + | SET_EX_OPLINE opline, REG0 + | GET_ZVAL_PTR REG0, op1_addr, TMP1 + | add CARG1, REG0, #offsetof(zend_string, val) + | ldr CARG2, [REG0, #offsetof(zend_string, len)] + | EXT_CALL zend_write, REG0 + if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) { + | ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline, ZREG_TMP1, ZREG_TMP2 + } + if (!zend_jit_check_exception(Dst)) { + return 0; + } } return 1; } From 7d8ee8a9893e199ea2a5279736c3969bfe2fe667 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 14:16:28 +0300 Subject: [PATCH 117/229] Support for missed IS_NOT_IDENTICAL cases --- ext/opcache/jit/zend_jit_arm64.dasc | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 8219dc19650f7..136322647c526 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -5889,7 +5889,14 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z |1: break; case ZEND_IS_NOT_IDENTICAL: - | NIY // TODO + if (exit_addr) { + | NIY // bvs &exit_addr + | NIY // bne &exit_addr + } else { + | bvs >1 + | beq => target_label + |1: + } break; case ZEND_IS_SMALLER: if (swap) { @@ -5948,7 +5955,14 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z } break; case ZEND_IS_NOT_IDENTICAL: - | NIY // TODO + if (exit_addr) { + | bvs >1 + | NIY // beq &exit_addr + |1: + } else { + | bvs => target_label + | bne => target_label + } break; case ZEND_IS_SMALLER: if (swap) { From c68ec03f2b59e4135ba55ca64f67bb5534dd3eb3 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 16:03:23 +0300 Subject: [PATCH 118/229] Support for FREE and FE_FREE --- ext/opcache/jit/zend_jit_arm64.dasc | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 136322647c526..754bdb33dda91 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -11465,19 +11465,15 @@ static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_i if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { if (may_throw) { - | NIY // TODO | SET_EX_OPLINE opline, REG0 } if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) { if (op1_info & MAY_BE_ARRAY) { - | NIY // TODO | IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7, TMP1w, TMP2 } - | NIY // TODO - int val = -1; - | LOAD_32BIT_VAL TMP1w, opline->op1.var + offsetof(zval, u2.fe_iter_idx) + | LOAD_32BIT_VAL TMP1w, (opline->op1.var + offsetof(zval, u2.fe_iter_idx)) | ldr FCARG1w, [FP, TMP1] - | LOAD_32BIT_VAL TMP1w, val + | mvn TMP1w, wzr // TODO: DynAsm fails loading #-1 | cmp FCARG1w, TMP1w | beq >7 | EXT_CALL zend_hash_iterator_del, REG0 From 2aac47b3fbc1549765c6ab7def05e78d7a0bd605 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 16:06:42 +0300 Subject: [PATCH 119/229] Support for DEFINED --- ext/opcache/jit/zend_jit_arm64.dasc | 127 +++++++++++++++++++++++++++- 1 file changed, 124 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 754bdb33dda91..63275c8a4b73c 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -9286,14 +9286,52 @@ static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2) { - | NIY // TODO + if (smart_branch_opcode) { + if (smart_branch_opcode == ZEND_JMPZ) { + if (jmp) { + | b >7 + } + } else if (smart_branch_opcode == ZEND_JMPNZ) { + | b =>target_label + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + | b =>target_label2 + } else { + ZEND_UNREACHABLE(); + } + } else { + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + if (jmp) { + | b >7 + } + } return 1; } static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int jmp, zend_uchar smart_branch_opcode, uint32_t target_label) { - | NIY // TODO + if (smart_branch_opcode) { + if (smart_branch_opcode == ZEND_JMPZ) { + | b =>target_label + } else if (smart_branch_opcode == ZEND_JMPNZ) { + if (jmp) { + | b >7 + } + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + | b =>target_label + } else { + ZEND_UNREACHABLE(); + } + } else { + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + if (jmp) { + | b >7 + } + } return 1; } @@ -9305,7 +9343,90 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, zend_uchar zval *zv = RT_CONSTANT(opline, opline->op1); zend_jit_addr res_addr = 0; - | NIY // TODO + if (smart_branch_opcode && !exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + undefined_label = target_label; + } else if (smart_branch_opcode == ZEND_JMPNZ) { + defined_label = target_label; + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + undefined_label = target_label; + defined_label = target_label2; + } else { + ZEND_UNREACHABLE(); + } + } + + | // if (CACHED_PTR(opline->extended_value)) { + | ldr REG0, EX->run_time_cache + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG0, REG0, opline->extended_value, TMP1 + | cbz REG0, >1 + | tst REG0, #1 + | bne >4 + |.cold_code + |4: + | MEM_LOAD_ZTS ldr, FCARG1x, executor_globals, zend_constants, FCARG1x + | lsr REG0, REG0, #1 + | ldr TMP1w, [FCARG1x, #offsetof(HashTable, nNumOfElements)] + | cmp TMP1, REG0 + + if (smart_branch_opcode) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | NIY // beq &exit_addr + } else { + | beq >3 + } + } else if (undefined_label != (uint32_t)-1) { + | beq =>undefined_label + } else { + | beq >3 + } + } else { + | beq >2 + } + |1: + | SET_EX_OPLINE opline, REG0 + | LOAD_ADDR FCARG1x, zv + | EXT_CALL zend_jit_check_constant, REG0 + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | cbz RETVALx, >3 + } else { + | cbnz RETVALx, >3 + } + | NIY // b &exit_addr + } else if (smart_branch_opcode) { + if (undefined_label != (uint32_t)-1) { + | cbz RETVALx, =>undefined_label + } else { + | cbz RETVALx, >3 + } + if (defined_label != (uint32_t)-1) { + | b =>defined_label + } else { + | b >3 + } + } else { + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + | cbnz RETVALx, >1 + |2: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + | b >3 + } + |.code + if (smart_branch_opcode) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | NIY // b &exit_addr + } + } else if (defined_label != (uint32_t)-1) { + | b =>defined_label + } + } else { + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } + |3: return 1; } From 695047369f20f0cbe526d27a391c73f70d921999 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 16:19:53 +0300 Subject: [PATCH 120/229] Support for STRLEN and COUNT --- ext/opcache/jit/zend_jit_arm64.dasc | 42 +++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 63275c8a4b73c..0bfbe43a2d959 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -11655,7 +11655,25 @@ static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1 { zend_jit_addr res_addr = RES_ADDR(); - | NIY // TODO + if (opline->op1_type == IS_CONST) { + zval *zv; + size_t len; + + zv = RT_CONSTANT(opline, opline->op1); + ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); + len = Z_STRLEN_P(zv); + + | SET_ZVAL_LVAL res_addr, len, TMP1, TMP2 + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 + } else { + ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); + + | GET_ZVAL_PTR REG0, op1_addr, TMP1 + | ldr REG0, [REG0, #offsetof(zend_string, len)] + | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + } return 1; } @@ -11663,7 +11681,27 @@ static int zend_jit_count(dasm_State **Dst, const zend_op *opline, uint32_t op1_ { zend_jit_addr res_addr = RES_ADDR(); - | NIY // TODO + if (opline->op1_type == IS_CONST) { + zval *zv; + zend_long count; + + zv = RT_CONSTANT(opline, opline->op1); + ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY); + count = zend_hash_num_elements(Z_ARRVAL_P(zv)); + + | SET_ZVAL_LVAL res_addr, count, TMP1, TMP2 + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 + } else { + ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_ARRAY); + // Note: See the implementation of ZEND_COUNT in Zend/zend_vm_def.h - arrays do not contain IS_UNDEF starting in php 8.1+. + + | GET_ZVAL_PTR REG0, op1_addr, TMP1 + // Sign-extend the 32-bit value to a potentially 64-bit zend_long + | ldr REG0w, [REG0, #offsetof(HashTable, nNumOfElements)] + | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + } if (may_throw) { return zend_jit_check_exception(Dst); From edf87fda3c7976a6a15895609584efec32404fd5 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 16:45:55 +0300 Subject: [PATCH 121/229] Support for moving between allocated registers --- ext/opcache/jit/zend_jit_arm64.dasc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 0bfbe43a2d959..274b2f844d39b 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -2917,9 +2917,9 @@ static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr sr if (!zend_jit_same_addr(src, dst)) { if (Z_MODE(src) == IS_REG) { if (Z_MODE(dst) == IS_REG) { - | NIY // TODO + | mov Rx(Z_REG(dst)), Rx(Z_REG(src)) } else if (Z_MODE(dst) == IS_MEM_ZVAL) { - | NIY // TODO + | fmov Rd(Z_REG(dst)-ZREG_V0), Rd(Z_REG(src)-ZREG_V0) } else { ZEND_UNREACHABLE(); } From c1f390b983cc254333e04fcb064cad0653194611 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 16:46:17 +0300 Subject: [PATCH 122/229] Support for FE_RESET and FE_FETCH --- ext/opcache/jit/zend_jit_arm64.dasc | 49 +++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 274b2f844d39b..c89936760d013 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -11807,12 +11807,16 @@ static int zend_jit_fe_reset(dasm_State **Dst, const zend_op *opline, uint32_t o | ZVAL_COPY_CONST res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 if (Z_REFCOUNTED_P(zv)) { - | NIY // TODO + | ADDREF_CONST zv, REG0, TMP1 } } else { zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); - | NIY // TODO + | // ZVAL_COPY(res, value); + | ZVAL_COPY_VALUE res_addr, -1, op1_addr, op1_info, ZREG_REG0, ZREG_FCARG1x, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + if (opline->op1_type == IS_CV) { + | TRY_ADDREF op1_info, REG0w, FCARG1x, TMP1 + } } | // Z_FE_POS_P(res) = 0; | LOAD_32BIT_VAL TMP1w, (opline->result.var + offsetof(zval, u2.fe_pos)) @@ -11844,7 +11848,7 @@ static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t o | // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); | // ZEND_VM_CONTINUE(); if (exit_addr) { - | NIY // TODO + | NIY // bls &exit_addr } else { | bls =>target_label } @@ -11855,7 +11859,7 @@ static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t o if (!exit_addr || exit_opcode == ZEND_JMP) { | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, >3, TMP1w } else { - | NIY // TODO + | NIY // IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, &exit_addr } | // p++; | add FCARG2x, FCARG2x, #sizeof(Bucket) @@ -11874,7 +11878,38 @@ static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t o if (RETURN_VALUE_USED(opline)) { zend_jit_addr res_addr = RES_ADDR(); - | NIY // TODO + if ((op1_info & MAY_BE_ARRAY_KEY_LONG) + && (op1_info & MAY_BE_ARRAY_KEY_STRING)) { + | // if (!p->key) { + | ldr REG0, [FCARG2x, #offsetof(Bucket, key)] + | cbz REG0, >2 + } + if (op1_info & MAY_BE_ARRAY_KEY_STRING) { + | // ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key); + | ldr REG0, [FCARG2x, #offsetof(Bucket, key)] + | SET_ZVAL_PTR res_addr, REG0, TMP1 + | ldr TMP1w, [REG0, #offsetof(zend_refcounted, gc.u.type_info)] + || ZEND_ASSERT(IS_STR_INTERNED <= TST_W_IMM); + | tst TMP1w, #IS_STR_INTERNED + | beq >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_STRING, TMP1w, TMP2 + | b >3 + |1: + | GC_ADDREF REG0, TMP1w + | SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX, TMP1w, TMP2 + + if (op1_info & MAY_BE_ARRAY_KEY_LONG) { + | b >3 + |2: + } + } + if (op1_info & MAY_BE_ARRAY_KEY_LONG) { + | // ZVAL_LONG(EX_VAR(opline->result.var), p->h); + | ldr REG0, [FCARG2x, #offsetof(Bucket, h)] + | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2 + } + |3: } val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT); @@ -11894,7 +11929,9 @@ static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t o return 0; } } else { - | NIY // TODO + | // ZVAL_COPY(res, value); + | ZVAL_COPY_VALUE var_addr, -1, val_addr, val_info, ZREG_REG0, ZREG_FCARG1x, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | TRY_ADDREF val_info, REG0w, FCARG1x, TMP1 } } From c86a4af06dd5aa5bb53feb7540b5c6a5906b8cd1 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 16:53:26 +0300 Subject: [PATCH 123/229] Fixed reference-countoing. Use 32-bit registers. --- ext/opcache/jit/zend_jit_arm64.dasc | 46 ++++++++++++++--------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index c89936760d013..47ed8e5a23ea3 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -3076,7 +3076,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); | ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 - | TRY_ADDREF op1_info, REG0w, REG2, TMP1 + | TRY_ADDREF op1_info, REG0w, REG2, TMP1w } if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) { @@ -3125,7 +3125,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 - | TRY_ADDREF op1_def_info, REG0w, REG1, TMP1 + | TRY_ADDREF op1_def_info, REG0w, REG1, TMP1w } } | b >3 @@ -4704,7 +4704,7 @@ static int zend_jit_simple_assign(dasm_State **Dst, } | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); | GET_ZVAL_PTR REG2, val_addr, TMP1 - | GC_DELREF REG2, TMP1 + | GC_DELREF REG2, TMP1w | // ZVAL_COPY_VALUE(return_value, &ref->val); ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, offsetof(zend_reference, val)); if (!res_addr) { @@ -4715,15 +4715,15 @@ static int zend_jit_simple_assign(dasm_State **Dst, | beq >2 // GC_DELREF() reached zero | IF_NOT_REFCOUNTED REG2w, >3 if (!res_addr) { - | GC_ADDREF Rx(tmp_reg), TMP1 + | GC_ADDREF Rx(tmp_reg), TMP1w } else { - | GC_ADDREF_2 Rx(tmp_reg), TMP1 + | GC_ADDREF_2 Rx(tmp_reg), TMP1w } | b >3 |2: if (res_addr) { | IF_NOT_REFCOUNTED REG2w, >2 - | GC_ADDREF Rx(tmp_reg), TMP1 + | GC_ADDREF Rx(tmp_reg), TMP1w |2: } if (save_r1) { @@ -4751,13 +4751,13 @@ static int zend_jit_simple_assign(dasm_State **Dst, if (val_type == IS_CV) { if (!res_addr) { - | TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1 + | TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1w } else { - | TRY_ADDREF_2 val_info, REG2w, Rx(tmp_reg), TMP1 + | TRY_ADDREF_2 val_info, REG2w, Rx(tmp_reg), TMP1w } } else { if (res_addr) { - | TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1 + | TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1w } } |3: @@ -4980,7 +4980,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, | str Rx(Z_REG(var_use_addr)), T1 // save } | GET_ZVAL_PTR FCARG1x, var_use_addr, TMP1 - | GC_DELREF FCARG1x, TMP1 + | GC_DELREF FCARG1x, TMP1w | IF_GC_MAY_NOT_LEAK FCARG1x, >5, TMP1w, TMP2w | EXT_CALL gc_possible_root, TMP1 if (Z_REG(var_use_addr) != ZREG_FP) { @@ -4988,7 +4988,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, } } else { | GET_ZVAL_PTR Rx(tmp_reg), var_use_addr, TMP1 - | GC_DELREF Rx(tmp_reg), TMP1 + | GC_DELREF Rx(tmp_reg), TMP1w } |5: } @@ -9198,7 +9198,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend | LOAD_ZVAL_ADDR FCARG1x, op1_addr | ZVAL_DEREF FCARG1x, op1_info, TMP1w | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 - | TRY_ADDREF op1_info, REG0w, REG2, TMP1 + | TRY_ADDREF op1_info, REG0w, REG2, TMP1w } else { zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 8); @@ -9209,10 +9209,10 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 | // ZVAL_COPY_VALUE(return_value, &ref->value); | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 - | GC_DELREF FCARG1x, TMP1 + | GC_DELREF FCARG1x, TMP1w | beq >1 | IF_NOT_REFCOUNTED REG0w, >2 - | GC_ADDREF REG2, TMP1 + | GC_ADDREF REG2, TMP1w | b >2 |1: | EFREE_REFERENCE @@ -9232,7 +9232,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend } | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 if (opline->op1_type == IS_CV) { - | TRY_ADDREF op1_info, REG0w, REG2, TMP1 + | TRY_ADDREF op1_info, REG0w, REG2, TMP1w } } } @@ -9821,7 +9821,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o } } | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 - | GC_DELREF FCARG1x, TMP1 + | GC_DELREF FCARG1x, TMP1w if (RC_MAY_BE_1(op1_info)) { if (RC_MAY_BE_N(op1_info)) { if (jit_return_label >= 0) { @@ -9875,7 +9875,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || (op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) || !op_array->function_name) { - | TRY_ADDREF op1_info, REG0w, REG2, TMP1 + | TRY_ADDREF op1_info, REG0w, REG2, TMP1w } else if (return_value_used != 1) { | // if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 @@ -9892,7 +9892,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o | GET_ZVAL_PTR REG0, op1_addr, TMP1 | // ZVAL_COPY_VALUE(return_value, &ref->value); | ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_REG2, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 - | GC_DELREF REG0, TMP1 + | GC_DELREF REG0, TMP1w | beq >2 | // if (IS_REFCOUNTED()) if (jit_return_label >= 0) { @@ -9902,7 +9902,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o } | // ADDREF | GET_ZVAL_PTR REG2, ret_addr, TMP1 // reload - | GC_ADDREF REG2, TMP1 + | GC_ADDREF REG2, TMP1w if (jit_return_label >= 0) { | b =>jit_return_label } else { @@ -10229,7 +10229,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, } else { | // ZVAL_COPY | ZVAL_COPY_VALUE res_addr, -1, val_addr, res_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 - | TRY_ADDREF res_info, REG1w, REG2, TMP1 + | TRY_ADDREF res_info, REG1w, REG2, TMP1w } } |9: // END @@ -11815,7 +11815,7 @@ static int zend_jit_fe_reset(dasm_State **Dst, const zend_op *opline, uint32_t o | // ZVAL_COPY(res, value); | ZVAL_COPY_VALUE res_addr, -1, op1_addr, op1_info, ZREG_REG0, ZREG_FCARG1x, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 if (opline->op1_type == IS_CV) { - | TRY_ADDREF op1_info, REG0w, FCARG1x, TMP1 + | TRY_ADDREF op1_info, REG0w, FCARG1x, TMP1w } } | // Z_FE_POS_P(res) = 0; @@ -11931,7 +11931,7 @@ static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t o } else { | // ZVAL_COPY(res, value); | ZVAL_COPY_VALUE var_addr, -1, val_addr, val_info, ZREG_REG0, ZREG_FCARG1x, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 - | TRY_ADDREF val_info, REG0w, FCARG1x, TMP1 + | TRY_ADDREF val_info, REG0w, FCARG1x, TMP1w } } @@ -11983,7 +11983,7 @@ static int zend_jit_fetch_constant(dasm_State **Dst, | NIY // TODO } else { | ZVAL_COPY_VALUE res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 - | TRY_ADDREF MAY_BE_ANY, REG0w, REG1, TMP1 + | TRY_ADDREF MAY_BE_ANY, REG0w, REG1, TMP1w } |.cold_code From 8c0cb57d9fb831c682d7182bdbb26583ca64ff5c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 17:06:19 +0300 Subject: [PATCH 124/229] Support for FETCH_CONST --- ext/opcache/jit/zend_jit_arm64.dasc | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 47ed8e5a23ea3..3b951d5ec9a49 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -11955,7 +11955,6 @@ static int zend_jit_fetch_constant(dasm_State **Dst, | ldr REG0, [FCARG1x, TMP1] | // if (c != NULL) | cbz REG0, >9 - | NIY // TODO | // if (!IS_SPECIAL_CACHE_VAL(c)) | tst REG0, #CACHE_SPECIAL | bne >9 @@ -11980,7 +11979,19 @@ static int zend_jit_fetch_constant(dasm_State **Dst, zend_uchar type = concrete_type(res_info); - | NIY // TODO + if (type < IS_STRING) { + | NIY // IF_NOT_ZVAL_TYPE const_addr, type, &exit_addr + } else { + | GET_ZVAL_TYPE_INFO REG2w, const_addr, TMP1 + | NIY // IF_NOT_TYPE REF2w, type, &exit_addr + } + | ZVAL_COPY_VALUE_V res_addr, -1, const_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0 + if (type < IS_STRING) { + | SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2 + } else { + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 + | TRY_ADDREF res_info, REG2w, REG1, TMP1w + } } else { | ZVAL_COPY_VALUE res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 | TRY_ADDREF MAY_BE_ANY, REG0w, REG1, TMP1w @@ -11997,7 +12008,6 @@ static int zend_jit_fetch_constant(dasm_State **Dst, | mov REG0, RETVALx | // ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION(); | cbnz REG0, <8 - | NIY // TODO | b ->exception_handler |.code return 1; From 0bfe4da47db0db1361e45c79c2d494c1b8de99c9 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 17:55:44 +0300 Subject: [PATCH 125/229] Support for TYPE_CHECK --- ext/opcache/jit/zend_jit_arm64.dasc | 234 ++++++++++++++++++++++++++-- 1 file changed, 218 insertions(+), 16 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 3b951d5ec9a49..ea05dbe338c1f 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -1614,7 +1614,15 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) static int zend_jit_exception_handler_undef_stub(dasm_State **Dst) { |->exception_handler_undef: - | NIY_STUB // TODO + | MEM_LOAD_ZTS ldr, REG0, executor_globals, opline_before_exception, REG0 + | ldrb TMP1w, OP:REG0->result_type + | tst TMP1w, #(IS_TMP_VAR|IS_VAR) + | bne >1 + | ldr REG0w, OP:REG0->result.var + | add REG0, REG0, FP + | SET_Z_TYPE_INFO REG0, IS_UNDEF, TMP1w + |1: + | b ->exception_handler return 1; } @@ -9440,15 +9448,61 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE); if (op1_info & MAY_BE_UNDEF) { - | NIY // TODO + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, TMP1w, TMP2 + |.cold_code + |1: + } + | SET_EX_OPLINE opline, REG0 + | LOAD_32BIT_VAL FCARG1w, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + zend_jit_check_exception_undef_result(Dst, opline); + if (opline->extended_value & MAY_BE_NULL) { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | NIY // b &exit_addr + } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { + | b >7 + } + } else if (!zend_jit_smart_true(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label, target_label2)) { + return 0; + } + } else { + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | NIY // b &exit_addr + } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { + | b >7 + } + } else if (!zend_jit_smart_false(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label)) { + return 0; + } + } + if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { + |.code + } } if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { mask = opline->extended_value; if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) { - | NIY // TODO + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | NIY // b &exit_addr + } + } else if (!zend_jit_smart_true(Dst, opline, 0, smart_branch_opcode, target_label, target_label2)) { + return 0; + } } else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) { - | NIY // TODO + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | NIY // b &exit_addr + } + } else if (!zend_jit_smart_false(Dst, opline, 0, smart_branch_opcode, target_label)) { + return 0; + } } else { bool invert = 0; zend_uchar type; @@ -9476,36 +9530,177 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t } if (op1_info & MAY_BE_REF) { - | NIY // TODO + | LOAD_ZVAL_ADDR REG0, op1_addr + | ZVAL_DEREF REG0, op1_info, TMP1w } if (type == 0) { - | NIY // TODO + if (smart_branch_opcode && + (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && + (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | // if (Z_REFCOUNTED_P(cv)) { + | IF_ZVAL_REFCOUNTED op1_addr, >1, TMP1w, TMP2 + |.cold_code + |1: + } + | // if (!Z_DELREF_P(cv)) { + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + | GC_DELREF FCARG1x, TMP1w + if (RC_MAY_BE_1(op1_info)) { + if (RC_MAY_BE_N(op1_info)) { + | bne >3 + } + if (op1_info & MAY_BE_REF) { + | ldrb REG0w, [REG0, #offsetof(zval,u1.v.type)] + } else { + | LOAD_32BIT_VAL TMP1, (opline->op1.var + offsetof(zval,u1.v.type)) + | ldrb REG0w, [FP, TMP1] + } + | str REG0w, T1 // save + | // zval_dtor_func(r); + | ZVAL_DTOR_FUNC op1_info, opline, TMP1 + | ldr REG1w, T1 // restore + | b >2 + } + if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + if (!RC_MAY_BE_1(op1_info)) { + | b >3 + } + |.code + } + |3: + if (op1_info & MAY_BE_REF) { + | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] + } else { + | LOAD_32BIT_VAL TMP1, (opline->op1.var + offsetof(zval,u1.v.type)) + | ldrb REG1w, [FP, TMP1] + } + |2: + } else { + if (op1_info & MAY_BE_REF) { + | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] + } else { + | LOAD_32BIT_VAL TMP1, (opline->op1.var + offsetof(zval,u1.v.type)) + | ldrb REG1w, [FP, TMP1] + } + } + | mov REG0w, #1 + | lsl REG0w, REG0w, REG1w + | LOAD_32BIT_VAL TMP1w, mask + | tst REG0w, TMP1w + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | NIY // bne &exit_addr + } else { + | NIY // beq &exit_addr + } + } else if (smart_branch_opcode) { + if (smart_branch_opcode == ZEND_JMPZ) { + | beq =>target_label + } else if (smart_branch_opcode == ZEND_JMPNZ) { + | bne =>target_label + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + | beq =>target_label + | b =>target_label2 + } else { + ZEND_UNREACHABLE(); + } + } else { + zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + + | cset REG0w, ne + | add REG0w, REG0w, #2 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + } } else { if (smart_branch_opcode && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - | NIY // TODO + if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | // if (Z_REFCOUNTED_P(cv)) { + | IF_ZVAL_REFCOUNTED op1_addr, >1, TMP1w, TMP2 + |.cold_code + |1: + } + | // if (!Z_DELREF_P(cv)) { + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + | GC_DELREF FCARG1x, TMP1w + if (RC_MAY_BE_1(op1_info)) { + if (RC_MAY_BE_N(op1_info)) { + | bne >3 + } + if (op1_info & MAY_BE_REF) { + | ldrb REG0w, [REG0, #offsetof(zval,u1.v.type)] + } else { + | LOAD_32BIT_VAL TMP1, (opline->op1.var + offsetof(zval,u1.v.type)) + | ldrb REG0w, [FP, TMP1] + } + | str REG0w, T1 // save + | // zval_dtor_func(r); + | ZVAL_DTOR_FUNC op1_info, opline, TMP1 + | ldr REG1w, T1 // restore + | b >2 + } + if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + if (!RC_MAY_BE_1(op1_info)) { + | b >3 + } + |.code + } + |3: + if (op1_info & MAY_BE_REF) { + | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] + } else { + | LOAD_32BIT_VAL TMP1, (opline->op1.var + offsetof(zval,u1.v.type)) + | ldrb REG1w, [FP, TMP1] + } + |2: + | cmp REG1w, #type } else { if (op1_info & MAY_BE_REF) { - | NIY // TODO + | ldrb TMP1w, [REG0, #offsetof(zval,u1.v.type)] + | cmp TMP1w, #type } else { - | LOAD_32BIT_VAL TMP1w, (opline->op1.var + 8) - | ldrb TMP2w, [FP, TMP1] - | cmp TMP2w, #type + | LOAD_32BIT_VAL TMP1w, (opline->op1.var + offsetof(zval,u1.v.type)) + | ldrb TMP1w, [FP, TMP1] + | cmp TMP1w, #type } } if (exit_addr) { - | NIY // TODO + if (invert) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | NIY // bne &exit_addr + } else { + | NIY // beq &exit_addr + } + } else { + if (smart_branch_opcode == ZEND_JMPNZ) { + | NIY // beq &exit_addr + } else { + | NIY // bne &exit_addr + } + } } else if (smart_branch_opcode) { if (invert) { - | NIY // TODO + if (smart_branch_opcode == ZEND_JMPZ) { + | beq =>target_label + } else if (smart_branch_opcode == ZEND_JMPNZ) { + | bne =>target_label + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + | beq =>target_label + | b =>target_label2 + } else { + ZEND_UNREACHABLE(); + } } else { if (smart_branch_opcode == ZEND_JMPZ) { | bne =>target_label } else if (smart_branch_opcode == ZEND_JMPNZ) { - | NIY // TODO + | beq =>target_label } else if (smart_branch_opcode == ZEND_JMPZNZ) { - | NIY // TODO + | bne =>target_label + | b =>target_label2 } else { ZEND_UNREACHABLE(); } @@ -9513,7 +9708,14 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t } else { zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); - | NIY // TODO + if (invert) { + | cset REG0w, ne + } else { + | cset REG0w, eq + } + | add REG0w, REG0w, #2 + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 } } } From 09070fa8bcf1dd72c961ff918f930935f310551e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 20:56:14 +0300 Subject: [PATCH 126/229] Support for BIND_GLOBAL --- ext/opcache/jit/zend_jit_arm64.dasc | 77 ++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index ea05dbe338c1f..28b2b51b37036 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -10829,7 +10829,82 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_ zend_jit_addr op1_addr = OP1_ADDR(); zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2)); - | NIY // TODO + | // idx = (uint32_t)(uintptr_t)CACHED_PTR(opline->extended_value) - 1; + | ldr REG0, EX->run_time_cache + | LOAD_32BIT_VAL TMP1, opline->extended_value + | ldr REG0, [REG0, TMP1] + | sub REG0, REG0, #1 + | // if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket))) + | MEM_LOAD_ZTS ldr, REG1w, executor_globals, symbol_table.nNumUsed, REG1 + | lsl REG1, REG1, #5 + | cmp REG0, REG1 + | bhs >9 + | // Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx); + | MEM_LOAD_ZTS ldr, TMP1, executor_globals, symbol_table.arData, REG1 + | add REG0, REG0, TMP1 + | IF_NOT_Z_TYPE REG0, IS_REFERENCE, >9, TMP1w + | // (EXPECTED(p->key == varname)) + | ldr TMP1, [REG0, #offsetof(Bucket, key)] + | LOAD_ADDR TMP2, varname + | cmp TMP1, TMP2 + | bne >9 + | GET_Z_PTR REG0, REG0 + | GC_ADDREF REG0, TMP1w + |1: + if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | // if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr))) + | IF_ZVAL_REFCOUNTED op1_addr, >2, TMP1w, TMP2 + |.cold_code + |2: + } + | // zend_refcounted *garbage = Z_COUNTED_P(variable_ptr); + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + | // ZVAL_REF(variable_ptr, ref) + | SET_ZVAL_PTR op1_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2 + | // if (GC_DELREF(garbage) == 0) + | GC_DELREF FCARG1x, TMP1w + if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { + | bne >3 + } else { + | bne >5 + } + | ZVAL_DTOR_FUNC op1_info, opline, TMP1 + | b >5 + if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { + |3: + | // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr) + | IF_GC_MAY_NOT_LEAK FCARG1x, >5, TMP1w, TMP2w + | EXT_CALL gc_possible_root, REG0 + | b >5 + } + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + |.code + } + } + + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | // ZVAL_REF(variable_ptr, ref) + | SET_ZVAL_PTR op1_addr, REG0, TMP1 + | SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2 + } + |5: + //END of handler + + |.cold_code + |9: + | LOAD_ADDR FCARG1x, (ptrdiff_t)varname + | ldr FCARG2x, EX->run_time_cache + if (opline->extended_value) { + | LOAD_32BIT_VAL TMP1, opline->extended_value + | add FCARG2x, FCARG2x, TMP1 + } + | EXT_CALL zend_jit_fetch_global_helper, REG0 + | mov REG0, RETVALx + | b <1 + |.code + return 1; } From 761b56ab0f6fbf717b545450493f0e590c809c1b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 22:07:51 +0300 Subject: [PATCH 127/229] Support for FETCH_THIS, FETCH_OBJ, ASSIGN_OBJ and ASSIGN_OBJ_OP --- ext/opcache/jit/zend_jit_arm64.dasc | 455 ++++++++++++++++++++++++---- 1 file changed, 398 insertions(+), 57 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 28b2b51b37036..0ed52bd234a8e 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -1790,7 +1790,11 @@ static int zend_jit_mod_by_zero_stub(dasm_State **Dst) static int zend_jit_invalid_this_stub(dasm_State **Dst) { |->invalid_this: - | NIY_STUB // TODO + | mov CARG1, xzr + | LOAD_ADDR CARG2, "Using $this when not in object context" + | EXT_CALL zend_throw_error, REG0 + | b ->exception_handler + return 1; } @@ -11197,7 +11201,7 @@ static int zend_jit_class_guard(dasm_State **Dst, const zend_op *opline, zend_cl return 0; } - | NIY // TODO + | NIY // tracing return 1; } @@ -11238,7 +11242,9 @@ static int zend_jit_fetch_obj(dasm_State **Dst, && opline->opcode == ZEND_FETCH_OBJ_W && (op1_info & MAY_BE_INDIRECT) && Z_REG(op1_addr) == ZREG_FP) { - | NIY // TODO + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w + | GET_Z_PTR FCARG1x, FCARG1x |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); } @@ -11257,7 +11263,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, if (!exit_addr) { return 0; } - | NIY // TODO + | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7, TMP1w, TMP2 } @@ -11286,43 +11292,53 @@ static int zend_jit_fetch_obj(dasm_State **Dst, if (!prop_info) { | ldr REG0, EX->run_time_cache | LOAD_32BIT_VAL TMP1w, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) - | add TMP1, TMP1, REG0 - | ldr REG2, [TMP1] + | ldr REG2, [REG0, TMP1] | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] | cmp REG2, TMP1 | bne >5 - | NIY // TODO: currently jump to Label 5. | LOAD_32BIT_VAL TMP1w, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) - | add TMP1, TMP1, REG0 - | ldr REG0, [TMP1] + | ldr REG0, [REG0, TMP1] may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); if (may_be_dynamic) { - | NIY // TODO | tst REG0, REG0 if (opline->opcode == ZEND_FETCH_OBJ_W) { - | NIY // TODO | blt >5 } else { - | NIY // TODO | blt >8 // dynamic property } } - | NIY // TODO + | add TMP1, FCARG1x, REG0 + | ldr REG2w, [TMP1, #offsetof(zval,u1.type_info)] + | IF_UNDEF REG2w, >5 + | mov FCARG1x, TMP1 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); if (opline->opcode == ZEND_FETCH_OBJ_W && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS) && (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS))) { uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; - | NIY // TODO + | ldr REG0, EX->run_time_cache + | LOAD_32BIT_VAL TMP1, ((opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2) + | ldr FCARG2x, [REG0, TMP1] + | cbnz FCARG2x, >1 |.cold_code |1: - | NIY // TODO + if (flags == ZEND_FETCH_DIM_WRITE) { + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_check_array_promotion, REG0 + | b >9 + } else if (flags == ZEND_FETCH_REF) { + | LOAD_ZVAL_ADDR CARG3, res_addr + | EXT_CALL zend_jit_create_typed_ref, REG0 + | b >9 + } else { + ZEND_UNREACHABLE(); + } |.code } } else { prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, prop_info->offset); - | LOAD_32BIT_VAL TMP1w, (prop_info->offset + 8) + | LOAD_32BIT_VAL TMP1w, (prop_info->offset + offsetof(zval,u1.type_info)) | ldr REG2w, [FCARG1x, TMP1] if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) { @@ -11333,31 +11349,65 @@ static int zend_jit_fetch_obj(dasm_State **Dst, if (!exit_addr) { return 0; } - | NIY // TODO + | NIY // IF_UNDEF dl, &exit_addr } } else { - | and TMP1w, REG2w, #0xff // low 8 bits. 8-15 bits are used later in zend_jit_zval_copy_deref(). - | IF_UNDEF TMP1w, >5 + | IF_UNDEF REG2w, >5 } if (opline->opcode == ZEND_FETCH_OBJ_W && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS) && ZEND_TYPE_IS_SET(prop_info->type)) { uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; - | NIY // TODO + if (flags == ZEND_FETCH_DIM_WRITE) { + if ((ZEND_TYPE_FULL_MASK(prop_info->type) & (MAY_BE_ITERABLE|MAY_BE_ARRAY)) == 0) { + | cmp REG2w, #IS_FALSE + | ble >1 + |.cold_code + |1: + if (Z_REG(prop_addr) != ZREG_FCARG1x || Z_OFFSET(prop_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, prop_addr + } + | LOAD_ADDR FCARG2x, prop_info + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_check_array_promotion, REG0 + | b >9 + |.code + } + } else if (flags == ZEND_FETCH_REF) { + | and TMP1w, REG2w, #0xff + | IF_TYPE TMP1w, IS_REFERENCE, >1 + if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { + | LOAD_ADDR FCARG2x, prop_info + } else { + int prop_info_offset = + (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); + + | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] + | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] + | LOAD_32BIT_VAL TMP1, prop_info_offset + | ldr FCARG2x, [REG0, TMP1] + } + if (Z_REG(prop_addr) != ZREG_FCARG1x || Z_OFFSET(prop_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, prop_addr + } + | LOAD_ZVAL_ADDR CARG3, res_addr + | EXT_CALL zend_jit_create_typed_ref, REG0 + | b >9 + |1: + } else { + ZEND_UNREACHABLE(); + } } } if (op1_avoid_refcounting) { SET_STACK_REG(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); } - if (opline->opcode == ZEND_FETCH_OBJ_W) { if (Z_REG(prop_addr) != ZREG_FCARG1x || Z_OFFSET(prop_addr) != 0) { - | NIY // TODO | LOAD_ZVAL_ADDR FCARG1x, prop_addr } - | NIY // TODO | SET_ZVAL_PTR res_addr, FCARG1x, TMP1 | SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) { @@ -11381,7 +11431,59 @@ static int zend_jit_fetch_obj(dasm_State **Dst, flags = ZEND_JIT_EXIT_FREE_OP1; } - | NIY // TODO + | LOAD_ZVAL_ADDR REG0, prop_addr + + if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) + && !(flags & ZEND_JIT_EXIT_FREE_OP1) + && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) + && (ssa_op+1)->op1_use == ssa_op->result_def + && zend_jit_may_avoid_refcounting(opline+1)) { + result_avoid_refcounting = 1; + ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; + } + + old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); + SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); + SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0); + exit_point = zend_jit_trace_get_exit_point(opline+1, flags); + SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + + res_info &= ~MAY_BE_GUARD; + ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; + type = concrete_type(res_info); + + | // ZVAL_DEREF() + | and TMP1w, REG2w, #0xff + | IF_NOT_TYPE TMP1w, IS_REFERENCE, >1 + | GET_Z_PTR REG0, REG0 + | add REG0, REG0, #offsetof(zend_reference, val) + if (type < IS_STRING) { + |1: + | NIY // IF_NOT_ZVAL_TYPE val_addr, type, &exit_addr, TMP1w, TMP2 + } else { + | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 + |1: + | and TMP1w, REG2w, #0xff + | NIY // IF_NOT_TYPE TMP1w, type, &exit_addr + } + | // ZVAL_COPY + | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0 + if (type < IS_STRING) { + if (Z_REG(res_addr) != ZREG_FP || + JIT_G(current_frame) == NULL || + STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { + | SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2 + } + } else { + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 + if (!result_avoid_refcounting) { + | TRY_ADDREF res_info, REG2w, REG1, TMP1w + } + } } else { if (!zend_jit_zval_copy_deref(Dst, res_addr, prop_addr, ZREG_REG2)) { return 0; @@ -11399,7 +11501,6 @@ static int zend_jit_fetch_obj(dasm_State **Dst, } else if (opline->opcode != ZEND_FETCH_OBJ_IS) { | EXT_CALL zend_jit_fetch_obj_r_slow, REG0 } else { - | NIY // TODO | EXT_CALL zend_jit_fetch_obj_is_slow, REG0 } | b >9 @@ -11413,9 +11514,14 @@ static int zend_jit_fetch_obj(dasm_State **Dst, && (op1_info & MAY_BE_UNDEF)) { zend_jit_addr orig_op1_addr = OP1_ADDR(); - | NIY // TODO + if (op1_info & MAY_BE_ANY) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, TMP1w, TMP2 + } + | LOAD_32BIT_VAL FCARG1w, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + |1: + | LOAD_ZVAL_ADDR FCARG1x, orig_op1_addr } else if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { - | NIY // TODO | LOAD_ZVAL_ADDR FCARG1x, op1_addr } | LOAD_ADDR FCARG2x, Z_STRVAL_P(member) @@ -11423,11 +11529,13 @@ static int zend_jit_fetch_obj(dasm_State **Dst, | EXT_CALL zend_jit_invalid_property_write, REG0 | SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2 } else { - | NIY // TODO + | EXT_CALL zend_jit_invalid_property_read, REG0 + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 } | b >9 } else { - | NIY // TODO + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 + | b >9 } } @@ -11435,7 +11543,14 @@ static int zend_jit_fetch_obj(dasm_State **Dst, && may_be_dynamic && opline->opcode != ZEND_FETCH_OBJ_W) { |8: - | NIY // TODO + | mov FCARG2x, REG0 + | SET_EX_OPLINE opline, REG0 + if (opline->opcode != ZEND_FETCH_OBJ_IS) { + | EXT_CALL zend_jit_fetch_obj_r_dynamic, REG0 + } else { + | EXT_CALL zend_jit_fetch_obj_is_dynamic, REG0 + } + | b >9 } |.code; @@ -11446,7 +11561,13 @@ static int zend_jit_fetch_obj(dasm_State **Dst, && (op1_info & MAY_BE_RC1)) { zend_jit_addr orig_op1_addr = OP1_ADDR(); - | NIY // TODO + | IF_NOT_ZVAL_REFCOUNTED orig_op1_addr, >1, TMP1w, TMP2 + | GET_ZVAL_PTR FCARG1x, orig_op1_addr, TMP1 + | GC_DELREF FCARG1x, TMP1w + | bne >1 + | SET_EX_OPLINE opline, REG0 + | EXT_CALL zend_jit_extract_helper, REG0 + |1: } else if (!op1_avoid_refcounting) { | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 } @@ -11532,17 +11653,22 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, prop_info = zend_get_known_property_info(op_array, ce, name, opline->op1_type == IS_UNUSED, op_array->filename); if (opline->op1_type == IS_UNUSED || use_this) { - | NIY // TODO | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 } else { if (opline->op1_type == IS_VAR && (op1_info & MAY_BE_INDIRECT) && Z_REG(op1_addr) == ZREG_FP) { - | NIY // TODO + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w + | GET_Z_PTR FCARG1x, FCARG1x + |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); } if (op1_info & MAY_BE_REF) { - | NIY // TODO + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + | ZVAL_DEREF FCARG1x, op1_info, TMP1w op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { @@ -11553,9 +11679,28 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, if (!exit_addr) { return 0; } - | NIY // TODO + | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr } else { - | NIY // TODO + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, TMP1w, TMP2 + |.cold_code + |1: + | SET_EX_OPLINE opline, REG0 + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + | LOAD_ADDR FCARG2x, ZSTR_VAL(name) + if (op1_info & MAY_BE_UNDEF) { + | EXT_CALL zend_jit_invalid_property_assign_op, REG0 + } else { + | EXT_CALL zend_jit_invalid_property_assign, REG0 + } + if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) + && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | b >8 + } else { + | b ->exception_handler + } + |.code } } | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 @@ -11587,7 +11732,25 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, if (!prop_info) { needs_slow_path = 1; - | NIY // TODO + | ldr REG0, EX->run_time_cache + | LOAD_32BIT_VAL TMP1, (opline+1)->extended_value + | ldr REG2, [REG0, TMP1] + | ldr TMP2, [FCARG1x, #offsetof(zend_object, ce)] + | cmp REG2, TMP2 + | bne >7 + if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { + | LOAD_32BIT_VAL TMP1, ((opline+1)->extended_value + sizeof(void*) * 2) + | ldr TMP1, [REG0, TMP1] + | cbnz TMP1, >7 + } + | LOAD_32BIT_VAL TMP1, ((opline+1)->extended_value + sizeof(void*)) + | ldr REG0, [REG0, TMP1] + | tst REG0, REG0 + | blt >7 + | add TMP1, FCARG1x, REG0 + | ldrb TMP2w, [TMP1, #offsetof(zval,u1.v.type)] + | IF_TYPE TMP2w, IS_UNDEF, >7 + | mov FCARG1x, TMP1 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); } else { prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, prop_info->offset); @@ -11598,9 +11761,9 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, if (!exit_addr) { return 0; } - | NIY // TODO + | NIY // IF_TYPE byte [FCARG1a + prop_info->offset + offsetof(zval,u1.v.type)], IS_UNDEF, &exit_addr } else { - | LOAD_32BIT_VAL TMP1w, (prop_info->offset + 8) + | LOAD_32BIT_VAL TMP1w, (prop_info->offset + offsetof(zval,u1.v.type)) | ldrb TMP2w, [FCARG1x, TMP1] | IF_TYPE TMP2w, IS_UNDEF, >7 needs_slow_path = 1; @@ -11608,7 +11771,46 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, if (ZEND_TYPE_IS_SET(prop_info->type)) { uint32_t info = val_info; - | NIY // TODO + if (opline) { + | SET_EX_OPLINE opline, REG0 + } + + | IF_ZVAL_TYPE prop_addr, IS_REFERENCE, >1, TMP1w, TMP2 + |.cold_code + |1: + | GET_ZVAL_PTR FCARG1x, prop_addr, TMP1 + if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2x || Z_OFFSET(val_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG2x, val_addr + } + | LOAD_ADDR CARG3, binary_op + | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 + | b >9 + |.code + + | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); + + if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { + | LOAD_ADDR FCARG2x, prop_info + } else { + int prop_info_offset = + (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); + + | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] + | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] + | LOAD_32BIT_VAL TMP1, prop_info_offset + | ldr FCARG2x, [REG0, TMP1] + } + | LOAD_ZVAL_ADDR FCARG1x, prop_addr + | LOAD_ZVAL_ADDR CARG3, val_addr + | LOAD_ADDR CARG4, binary_op + + | EXT_CALL zend_jit_assign_op_to_typed_prop, REG0 + + if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + info |= MAY_BE_RC1|MAY_BE_RCN; + } + + | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, opline, ZREG_TMP1, ZREG_TMP2 } } @@ -11618,7 +11820,60 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); - | NIY // TODO + | LOAD_ZVAL_ADDR REG0, prop_addr + + | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, TMP1w, TMP2 + | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 + | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] + | cbnz TMP1, >1 + | add REG0, FCARG1x, #offsetof(zend_reference, val) + |.cold_code + |1: + if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2x || Z_OFFSET(val_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG2x, val_addr + } + if (opline) { + | SET_EX_OPLINE opline, REG0 + } + | LOAD_ADDR CARG3, binary_op + | EXT_CALL zend_jit_assign_op_to_typed_ref, REG0 + | b >9 + |.code + |2: + + switch (opline->extended_value) { + case ZEND_ADD: + case ZEND_SUB: + case ZEND_MUL: + case ZEND_DIV: + if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info, + 1 /* may overflow */, 0)) { + return 0; + } + break; + case ZEND_BW_OR: + case ZEND_BW_AND: + case ZEND_BW_XOR: + case ZEND_SL: + case ZEND_SR: + case ZEND_MOD: + if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value, + IS_CV, opline->op1, var_addr, var_info, NULL, + (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, + val_range, + 0, var_addr, var_def_info, var_info, 0)) { + return 0; + } + break; + case ZEND_CONCAT: + if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, var_addr, + 0)) { + return 0; + } + break; + default: + ZEND_UNREACHABLE(); + } } if (needs_slow_path) { @@ -11696,12 +11951,14 @@ static int zend_jit_assign_obj(dasm_State **Dst, prop_info = zend_get_known_property_info(op_array, ce, name, opline->op1_type == IS_UNUSED, op_array->filename); if (opline->op1_type == IS_UNUSED || use_this) { - | NIY // TODO + | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 } else { if (opline->op1_type == IS_VAR && (op1_info & MAY_BE_INDIRECT) && Z_REG(op1_addr) == ZREG_FP) { - | NIY // TODO + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w + | GET_Z_PTR FCARG1x, FCARG1x |1: op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); } @@ -11720,12 +11977,26 @@ static int zend_jit_assign_obj(dasm_State **Dst, if (!exit_addr) { return 0; } - | NIY // TODO + | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, TMP1w, TMP2 |.cold_code |1: - | NIY // TODO + | SET_EX_OPLINE opline, REG0 + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + | LOAD_ADDR FCARG2x, ZSTR_VAL(name) + | EXT_CALL zend_jit_invalid_property_assign, REG0 + if (RETURN_VALUE_USED(opline)) { + | SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2 + } + if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) + && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | b >8 + } else { + | b ->exception_handler + } |.code } } @@ -11733,7 +12004,6 @@ static int zend_jit_assign_obj(dasm_State **Dst, } if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { - prop_info = zend_get_known_property_info(op_array, trace_ce, name, opline->op1_type == IS_UNUSED, op_array->filename); prop_info = zend_get_known_property_info(op_array, trace_ce, name, opline->op1_type == IS_UNUSED, op_array->filename); if (prop_info) { ce = trace_ce; @@ -11767,12 +12037,46 @@ static int zend_jit_assign_obj(dasm_State **Dst, | cmp REG2, TMP1 | bne >5 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { - | NIY // TODO - } - | NIY // TODO + | LOAD_32BIT_VAL TMP1, (opline->extended_value + sizeof(void*) * 2) + | ldr FCARG2x, [REG0, TMP1] + } + | LOAD_32BIT_VAL TMP1, (opline->extended_value + sizeof(void*)) + | ldr REG0, [REG0, TMP1] + | tst REG0, REG0 + | blt >5 + | add TMP2, FCARG1x, REG0 + | ldrb TMP1w, [TMP2, #offsetof(zval,u1.v.type)] + | IF_TYPE TMP1w, IS_UNDEF, >5 + | mov FCARG1x, TMP2 prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { - | NIY // TODO + | cbnz FCARG2x, >1 + |.cold_code + |1: + | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); + | SET_EX_OPLINE opline, REG0 + | LOAD_ZVAL_ADDR CARG3, val_addr + if (RETURN_VALUE_USED(opline)) { + | LOAD_ZVAL_ADDR CARG4, res_addr + } else { + | mov CARG4, xzr + } + + | EXT_CALL zend_jit_assign_to_typed_prop, REG0 + + if ((opline+1)->op1_type == IS_CONST) { + | // TODO: ??? + | // if (Z_TYPE_P(value) == orig_type) { + | // CACHE_PTR_EX(cache_slot + 2, NULL); + } + + if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) + && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { + | b >8 + } else { + | b >9 + } + |.code } } else { prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, prop_info->offset); @@ -11785,9 +12089,11 @@ static int zend_jit_assign_obj(dasm_State **Dst, if (!exit_addr) { return 0; } - | NIY // TODO + | NIY // IF_TYPE byte [FCARG1a + prop_info->offset + offsetof(zval,u1.v.type)], IS_UNDEF, &exit_addr } else { - | NIY // TODO + | LOAD_32BIT_VAL TMP1, (prop_info->offset + offsetof(zval,u1.v.type)) + | ldrb TMP2w, [FCARG1x, TMP1] + | IF_TYPE TMP2w, IS_UNDEF, >5 needs_slow_path = 1; } } @@ -11795,7 +12101,33 @@ static int zend_jit_assign_obj(dasm_State **Dst, uint32_t info = val_info; | // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); - | NIY // TODO + | SET_EX_OPLINE opline, REG0 + if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { + | LOAD_ADDR FCARG2x, prop_info + } else { + int prop_info_offset = + (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); + + | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] + | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] + | LOAD_32BIT_VAL TMP1, prop_info_offset + | ldr FCARG2x, [REG0, TMP1] + } + | LOAD_ZVAL_ADDR FCARG1x, prop_addr + | LOAD_ZVAL_ADDR CARG3, val_addr + if (RETURN_VALUE_USED(opline)) { + | LOAD_ZVAL_ADDR CARG4, res_addr + } else { + | mov CARG4, xzr + } + + | EXT_CALL zend_jit_assign_to_typed_prop, REG0 + + if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { + info |= MAY_BE_RC1|MAY_BE_RCN; + } + + | FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, opline, ZREG_TMP1, ZREG_TMP2 } } @@ -11824,7 +12156,6 @@ static int zend_jit_assign_obj(dasm_State **Dst, | LOAD_32BIT_VAL TMP1w, opline->extended_value | add CARG4, CARG4, TMP1 if (RETURN_VALUE_USED(opline)) { - | NIY // TODO | LOAD_ZVAL_ADDR CARG5, res_addr } else { | mov CARG5, xzr @@ -12007,14 +12338,24 @@ static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const ze int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); - | NIY // TODO + | ldrb TMP1w, EX->This.u1.v.type + | cmp TMP1w, #IS_OBJECT + | NIY // bne &exit_addr if (JIT_G(current_frame)) { TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame)); } } } else { - | NIY // TODO + + | ldrb TMP1w, EX->This.u1.v.type + | cmp TMP1w, #IS_OBJECT + | bne >1 + |.cold_code + |1: + | SET_EX_OPLINE opline, REG0 + | b ->invalid_this + |.code } } @@ -12136,7 +12477,7 @@ static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t o if (!exit_addr || exit_opcode == ZEND_JMP) { | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, >3, TMP1w } else { - | NIY // IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, &exit_addr + | NIY // IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, &exit_addr } | // p++; | add FCARG2x, FCARG2x, #sizeof(Bucket) From f52ec02ea9613c8e8074bbc90e2afabcacce2563 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 22:27:23 +0300 Subject: [PATCH 128/229] Get rid of testing NIY_STUBs --- ext/opcache/jit/zend_jit_arm64.dasc | 3 --- 1 file changed, 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 0ed52bd234a8e..8a8c0cf71ca97 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -1634,7 +1634,6 @@ static int zend_jit_leave_function_stub(dasm_State **Dst) | LOAD_32BIT_VAL TMP1w, ZEND_CALL_TOP | tst FCARG1w, TMP1w | bne >1 - | NIY_STUB // TODO: currently jump to label 1. | EXT_CALL zend_jit_leave_nested_func_helper, REG0 | ADD_HYBRID_SPAD | JMP_IP TMP1 @@ -1654,7 +1653,6 @@ static int zend_jit_leave_function_stub(dasm_State **Dst) | LOAD_32BIT_VAL TMP1w, ZEND_CALL_TOP | tst FCARG1w, TMP1w | bne >1 - | NIY_STUB // TODO: currently jump to label 1. | EXT_JMP zend_jit_leave_nested_func_helper, REG0 |1: | EXT_JMP zend_jit_leave_top_func_helper, REG0 @@ -2048,7 +2046,6 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst) } |1: - | NIY_STUB // TODO: test | blt ->trace_halt | // execute_data = EG(current_execute_data) From 326c9bf937767dbbab6f63297ea92ee6ea92a414 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 23:23:46 +0300 Subject: [PATCH 129/229] Support for PRE/POST_INC/DEC_OBJ --- ext/opcache/jit/zend_jit_arm64.dasc | 316 +++++++++++++++++++++++++++- 1 file changed, 315 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 8a8c0cf71ca97..04c47eda95610 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -11611,7 +11611,321 @@ static int zend_jit_incdec_obj(dasm_State **Dst, ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); - | NIY // TODO + if (opline->result_type != IS_UNUSED) { + res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); + } + + member = RT_CONSTANT(opline, opline->op2); + ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); + name = Z_STR_P(member); + prop_info = zend_get_known_property_info(op_array, ce, name, opline->op1_type == IS_UNUSED, op_array->filename); + + if (opline->op1_type == IS_UNUSED || use_this) { + | GET_ZVAL_PTR FCARG1x, this_addr, TMP1 + } else { + if (opline->op1_type == IS_VAR + && (op1_info & MAY_BE_INDIRECT) + && Z_REG(op1_addr) == ZREG_FP) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w + | GET_Z_PTR FCARG1x, FCARG1x + |1: + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + if (op1_info & MAY_BE_REF) { + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + | ZVAL_DEREF FCARG1x, op1_info, TMP1w + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, TMP1w, TMP2 + |.cold_code + |1: + | SET_EX_OPLINE opline, REG0 + if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + } + | LOAD_ADDR FCARG2x, ZSTR_VAL(name) + | EXT_CALL zend_jit_invalid_property_incdec, REG0 + | b ->exception_handler + |.code + } + } + | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 + } + + if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { + prop_info = zend_get_known_property_info(op_array, trace_ce, name, opline->op1_type == IS_UNUSED, op_array->filename); + if (prop_info) { + ce = trace_ce; + ce_is_instanceof = 0; + if (!(op1_info & MAY_BE_CLASS_GUARD)) { + if (!zend_jit_class_guard(Dst, opline, trace_ce)) { + return 0; + } + if (ssa->var_info && ssa_op->op1_use >= 0) { + ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; + ssa->var_info[ssa_op->op1_use].ce = ce; + ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; + } + if (ssa->var_info && ssa_op->op1_def >= 0) { + ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; + ssa->var_info[ssa_op->op1_def].ce = ce; + ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; + } + } + } + } + + if (!prop_info) { + needs_slow_path = 1; + + | ldr REG0, EX->run_time_cache + | LOAD_32BIT_VAL TMP1, opline->extended_value + | ldr REG2, [REG0, TMP1] + | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] + | cmp REG2, TMP2 + | bne >7 + if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { + | LOAD_32BIT_VAL TMP1, (opline->extended_value + sizeof(void*) * 2) + | ldr TMP1, [REG0, TMP1] + | cbnz TMP1, >7 + } + | LOAD_32BIT_VAL TMP1, (opline->extended_value + sizeof(void*)) + | ldr REG0, [REG0, TMP1] + | tst REG0, REG0 + | blt >7 + | add TMP1, FCARG1x, REG0 + | ldrb TMP2w, [TMP1, #offsetof(zval,u1.v.type)] + | IF_TYPE TMP2w , IS_UNDEF, >7 + | mov FCARG1x, TMP1 + prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } else { + prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, prop_info->offset); + if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + | NIY // IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr + } else { + | LOAD_32BIT_VAL TMP1, prop_info->offset + offsetof(zval,u1.v.type) + | ldrb TMP2w, [FCARG1x, TMP1] + | IF_TYPE TMP2w, IS_UNDEF, >7 + needs_slow_path = 1; + } + if (ZEND_TYPE_IS_SET(prop_info->type)) { + | SET_EX_OPLINE opline, REG0 + if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { + | LOAD_ADDR FCARG2x, prop_info + } else { + int prop_info_offset = + (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); + + | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] + | ldr REG0, [REG1, #offsetof(zend_class_entry, properties_info_table)] + | LOAD_32BIT_VAL TMP1, prop_info_offset + | ldr FCARG2x, [REG0, TMP1] + } + | LOAD_ZVAL_ADDR FCARG1x, prop_addr + if (opline->result_type == IS_UNUSED) { + switch (opline->opcode) { + case ZEND_PRE_INC_OBJ: + case ZEND_POST_INC_OBJ: + | EXT_CALL zend_jit_inc_typed_prop, REG0 + break; + case ZEND_PRE_DEC_OBJ: + case ZEND_POST_DEC_OBJ: + | EXT_CALL zend_jit_dec_typed_prop, REG0 + break; + default: + ZEND_UNREACHABLE(); + } + } else { + | LOAD_ZVAL_ADDR CARG3, res_addr + switch (opline->opcode) { + case ZEND_PRE_INC_OBJ: + | EXT_CALL zend_jit_pre_inc_typed_prop, REG0 + break; + case ZEND_PRE_DEC_OBJ: + | EXT_CALL zend_jit_pre_dec_typed_prop, REG0 + break; + case ZEND_POST_INC_OBJ: + | EXT_CALL zend_jit_post_inc_typed_prop, REG0 + break; + case ZEND_POST_DEC_OBJ: + | EXT_CALL zend_jit_post_dec_typed_prop, REG0 + break; + default: + ZEND_UNREACHABLE(); + } + } + } + } + + if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { + zend_jit_addr var_addr = prop_addr; + + var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + if (Z_REG(prop_addr) != ZREG_FCARG1x || Z_OFFSET(prop_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, prop_addr + } + + | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, TMP1w, TMP2 + | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 + | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] + | cbnz TMP1, >1 + | add FCARG1x, FCARG1x, #offsetof(zend_reference, val) + |.cold_code + |1: + if (opline) { + | SET_EX_OPLINE opline, REG0 + } + if (opline->result_type == IS_UNUSED) { + | mov FCARG2x, xzr + } else { + | LOAD_ZVAL_ADDR FCARG2x, res_addr + } + switch (opline->opcode) { + case ZEND_PRE_INC_OBJ: + | EXT_CALL zend_jit_pre_inc_typed_ref, REG0 + break; + case ZEND_PRE_DEC_OBJ: + | EXT_CALL zend_jit_pre_dec_typed_ref, REG0 + break; + case ZEND_POST_INC_OBJ: + | EXT_CALL zend_jit_post_inc_typed_ref, REG0 + break; + case ZEND_POST_DEC_OBJ: + | EXT_CALL zend_jit_post_dec_typed_ref, REG0 + break; + default: + ZEND_UNREACHABLE(); + } + | b >9 + |.code + + |2: + | IF_NOT_ZVAL_TYPE var_addr, IS_LONG, >2, TMP1w, TMP2 + if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { + if (opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + } + } + if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { + | LONG_ADD_SUB_WITH_IMM adds, var_addr, Z_L(1), TMP1, TMP2 + } else { + | LONG_ADD_SUB_WITH_IMM subs, var_addr, Z_L(1), TMP1, TMP2 + } + | bvs >3 + if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) { + if (opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + } + } + |.cold_code + |2: + if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { + | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_ANY, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | TRY_ADDREF MAY_BE_ANY, REG0w, REG2, TMP1w + } + if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { + if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { + | LOAD_ZVAL_ADDR FCARG2x, res_addr + | EXT_CALL zend_jit_pre_inc, REG0 + } else { + | EXT_CALL increment_function, REG0 + } + } else { + if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { + | LOAD_ZVAL_ADDR FCARG2x, res_addr + | EXT_CALL zend_jit_pre_dec, REG0 + } else { + | EXT_CALL decrement_function, REG0 + } + } + | b >4 + + |3: + if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { + uint64_t val = 0x43e0000000000000; + | LOAD_64BIT_VAL REG0, val + | SET_ZVAL_LVAL_FROM_REG var_addr, REG0, TMP1 + if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { + | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 + } + } else { + uint64_t val = 0xc3e0000000000000; + | LOAD_64BIT_VAL REG0, val + | SET_ZVAL_LVAL_FROM_REG var_addr, REG0, TMP1 + if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { + | SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1 + } + } + | b >4 + |.code + |4: + } + + if (needs_slow_path) { + |.cold_code + |7: + | SET_EX_OPLINE opline, REG0 + | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); + | LOAD_ADDR FCARG2x, name + | ldr CARG3, EX->run_time_cache + | LOAD_32BIT_VAL CARG3, opline->extended_value + if (opline->result_type == IS_UNUSED) { + | mov CARG4, xzr + } else { + | LOAD_ZVAL_ADDR CARG4, res_addr + } + + switch (opline->opcode) { + case ZEND_PRE_INC_OBJ: + | EXT_CALL zend_jit_pre_inc_obj_helper, REG0 + break; + case ZEND_PRE_DEC_OBJ: + | EXT_CALL zend_jit_pre_dec_obj_helper, REG0 + break; + case ZEND_POST_INC_OBJ: + | EXT_CALL zend_jit_post_inc_obj_helper, REG0 + break; + case ZEND_POST_DEC_OBJ: + | EXT_CALL zend_jit_post_dec_obj_helper, REG0 + break; + default: + ZEND_UNREACHABLE(); + } + + | b >9 + |.code + } + + |9: + if (opline->op1_type != IS_UNUSED && !use_this && !op1_indirect) { + | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 + } + + if (may_throw) { + if (!zend_jit_check_exception(Dst)) { + return 0; + } + } + return 1; } From 36358c5a162e02408475a0c74a8280957088b0b7 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 22 Apr 2021 23:47:36 +0300 Subject: [PATCH 130/229] Get rid of NIY in spill code --- ext/opcache/jit/zend_jit_arm64.dasc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 04c47eda95610..38e6c7859e57b 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -2856,7 +2856,10 @@ static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_ad | SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2 } } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - | NIY // TODO + | SET_ZVAL_DVAL dst, Z_REG(src), ZREG_TMP1 + if (set_type) { + | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2 + } } else { ZEND_UNREACHABLE(); } @@ -2871,7 +2874,7 @@ static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr if ((info & MAY_BE_ANY) == MAY_BE_LONG) { | GET_ZVAL_LVAL Z_REG(dst), src, TMP1 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - | NIY // TODO + | GET_ZVAL_DVAL Z_REG(dst), src, ZREG_TMP1 } else { ZEND_UNREACHABLE(); } @@ -2889,7 +2892,6 @@ static int zend_jit_store_var(dasm_State **Dst, uint32_t info, int var, zend_reg static int zend_jit_store_var_if_necessary(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info) { if (Z_MODE(src) == IS_REG && Z_STORE(src)) { - | NIY // TODO: test zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); return zend_jit_spill_store(Dst, src, dst, info, 1); } From b550dbd68b7a8f8e40f5ff0a2b04a1a5a67a08ea Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 23 Apr 2021 01:10:04 +0300 Subject: [PATCH 131/229] Support for VERIFY_RETURN_TYPE, ISSET_ISEMPTY_CV, IN_ARRAY and ADD with array operands. --- ext/opcache/jit/zend_jit_arm64.dasc | 160 ++++++++++++++++++++++++++-- 1 file changed, 154 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 38e6c7859e57b..6726ba03910e4 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -514,9 +514,9 @@ static void* dasm_labels[zend_lb_MAX]; | str tmp_reg, [zv, #offsetof(zval,u1.type_info)] |.endmacro -|.macro GET_ZVAL_TYPE, reg, addr +|.macro GET_ZVAL_TYPE, reg, addr, tmp_reg || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -| mov reg, byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.v.type)] +| SAFE_MEM_ACC_WITH_UOFFSET ldrb, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.v.type), tmp_reg |.endmacro |.macro GET_ZVAL_TYPE_INFO, reg, addr, tmp_reg @@ -3791,7 +3791,13 @@ static int zend_jit_add_arrays(dasm_State **Dst, const zend_op *opline, uint32_t zend_jit_addr op1_addr = OP1_ADDR(); zend_jit_addr op2_addr = OP2_ADDR(); - | NIY // TODO + | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr, TMP1 + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + | EXT_CALL zend_jit_add_arrays_helper, REG0 + | SET_ZVAL_PTR res_addr, RETVALx, TMP1 + | SET_ZVAL_TYPE_INFO res_addr, IS_ARRAY_EX, TMP1w, TMP2 + | FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2 + | FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2 return 1; } @@ -12717,7 +12723,54 @@ static bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, bool slow_check_in_cold = 1; uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; - | NIY // TODO + if (type_mask == 0) { + slow_check_in_cold = 0; + } else { + if (((op1_info & MAY_BE_ANY) & type_mask) == 0) { + slow_check_in_cold = 0; + } else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) { + needs_slow_check = 0; + } else if (is_power_of_two(type_mask)) { + uint32_t type_code = concrete_type(type_mask); + | IF_NOT_ZVAL_TYPE op1_addr, type_code, >7, TMP1w, TMP2 + } else { + | mov REG2w, #1 + | GET_ZVAL_TYPE REG1w, op1_addr, TMP1 + | lsl REG2w, REG2w, REG1w + | LOAD_32BIT_VAL TMP1w, type_mask + | tst REG2w, TMP1w + | beq >7 + } + } + if (needs_slow_check) { + if (slow_check_in_cold) { + |.cold_code + |7: + } + | SET_EX_OPLINE opline, REG1 + if (op1_info & MAY_BE_UNDEF) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >8, TMP1w, TMP2 + | LOAD_32BIT_VAL FCARG1x, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + | LOAD_ADDR_ZTS REG0, executor_globals, uninitialized_zval + } + |8: + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + | ldr FCARG2x, EX->func + | LOAD_ADDR CARG3, (ptrdiff_t)arg_info + | ldr REG0, EX->run_time_cache + | LOAD_32BIT_VAL TMP1, opline->op2.num + | add CARG4, REG0, TMP1 + | EXT_CALL zend_jit_verify_return_slow, REG0 + if (!zend_jit_check_exception(Dst)) { + return 0; + } + if (slow_check_in_cold) { + | b >9 + |.code + } + } + |9: return 1; } @@ -12725,7 +12778,69 @@ static int zend_jit_isset_isempty_cv(dasm_State **Dst, const zend_op *opline, ui { zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); - | NIY // TODO + // TODO: support for empty() ??? + ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); + + if (op1_info & MAY_BE_REF) { + if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { + | LOAD_ZVAL_ADDR FCARG1x, op1_addr + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 0); + } + | ZVAL_DEREF FCARG1x, op1_info, TMP1w + |1: + } + + if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) { + if (exit_addr) { + ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ); + } else if (smart_branch_opcode) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | b =>target_label + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + | b =>target_label2 + } + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } + } else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) { + if (exit_addr) { + ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ); + } else if (smart_branch_opcode) { + if (smart_branch_opcode != ZEND_JMPNZ) { + | b =>target_label + } + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + } + } else { + ZEND_ASSERT(Z_MODE(op1_addr) == IS_MEM_ZVAL); + | LOAD_32BIT_VAL TMP1, (Z_OFFSET(op1_addr)+offsetof(zval, u1.v.type)) + | ldrb TMP1w, [Rx(Z_REG(op1_addr)), TMP1] + | cmp TMP1w, #IS_NULL + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPNZ) { + | NIY // bgt &exit_addr + } else { + | NIY // ble &exit_addr + } + } else if (smart_branch_opcode) { + if (smart_branch_opcode == ZEND_JMPZ) { + | ble =>target_label + } else if (smart_branch_opcode == ZEND_JMPNZ) { + | bgt =>target_label + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + | ble =>target_label + | b =>target_label2 + } else { + ZEND_UNREACHABLE(); + } + } else { + | cset REG0w, gt + | add REG0w, REG0w, #IS_FALSE + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + } + } + return 1; } @@ -12952,7 +13067,40 @@ static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t o ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR); ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING); - | NIY // TODO + | // result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST); + | LOAD_ADDR FCARG1x, ht + if (opline->op1_type != IS_CONST) { + | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 + | EXT_CALL zend_hash_find, REG0 + } else { + zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1)); + | LOAD_ADDR FCARG2x, str + | EXT_CALL _zend_hash_find_known_hash, REG0 + } + if (exit_addr) { + if (smart_branch_opcode == ZEND_JMPZ) { + | NIY // jcb RETVALx, &exit_addr + } else { + | NIY // cbnz RETVALx, &exit_addr + } + } else if (smart_branch_opcode) { + if (smart_branch_opcode == ZEND_JMPZ) { + | cbz RETVALx, =>target_label + } else if (smart_branch_opcode == ZEND_JMPNZ) { + | cbnz RETVALx, =>target_label + } else if (smart_branch_opcode == ZEND_JMPZNZ) { + | cbz RETVALx, =>target_label + | b =>target_label2 + } else { + ZEND_UNREACHABLE(); + } + } else { + | tst RETVALx, RETVALx + | cset REG0w, ne + | add REG0w, REG0w, #IS_FALSE + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 + } + return 1; } From 1289ebcaabc8b55ec0b9e68fbfb9ff1be92c29fb Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 23 Apr 2021 02:13:32 +0300 Subject: [PATCH 132/229] Implement exceptional stubs --- ext/opcache/jit/zend_jit_arm64.dasc | 115 ++++++++++++++++++++++++++-- 1 file changed, 108 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 6726ba03910e4..948739d84e148 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -430,7 +430,11 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro LOAD_IP_ADDR_ZTS, struct, field -| NIY // TODO +| .if ZTS +| NIY // TODO +| .else +| LOAD_IP_ADDR &struct.field +| .endif |.endmacro |.macro GET_IP, reg @@ -1664,7 +1668,21 @@ static int zend_jit_leave_function_stub(dasm_State **Dst) static int zend_jit_leave_throw_stub(dasm_State **Dst) { |->leave_throw_handler: - | NIY_STUB // TODO: test + | // if (opline->opcode != ZEND_HANDLE_EXCEPTION) { + if (GCC_GLOBAL_REGS) { + | ldrb TMP1w, OP:IP->opcode + | cmp TMP1w, #ZEND_HANDLE_EXCEPTION + | beq >5 + | // EG(opline_before_exception) = opline; + | MEM_STORE_ZTS str, IP, executor_globals, opline_before_exception, TMP2 + |5: + | // opline = EG(exception_op); + | LOAD_IP_ADDR_ZTS executor_globals, exception_op + | // HANDLE_EXCEPTION() + | b ->exception_handler + } else { + | NIY_STUB // TODO + } return 1; } @@ -1672,15 +1690,54 @@ static int zend_jit_leave_throw_stub(dasm_State **Dst) static int zend_jit_icall_throw_stub(dasm_State **Dst) { |->icall_throw_handler: - | NIY_STUB // TODO + | // zend_rethrow_exception(zend_execute_data *execute_data) + | ldr IP, EX->opline + | // if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) { + | ldrb TMP1w, OP:IP->opcode + | cmp TMP1w,# ZEND_HANDLE_EXCEPTION + | beq >1 + | // EG(opline_before_exception) = opline; + | MEM_STORE_ZTS str, IP, executor_globals, opline_before_exception, TMP2 + |1: + | // opline = EG(exception_op); + | LOAD_IP_ADDR_ZTS executor_globals, exception_op + || if (GCC_GLOBAL_REGS) { + | str IP, EX->opline + || } + | // HANDLE_EXCEPTION() + | b ->exception_handler return 1; } static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) { + zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + |->throw_cannot_pass_by_ref: - | NIY_STUB // TODO + | ldr REG0, EX->opline + | ldr REG1w, OP:REG0->result.var + | add REG1, REG1, RX + | SET_Z_TYPE_INFO REG1, IS_UNDEF, TMP1w + | // last EX(call) frame may be delayed + | ldr TMP1, EX->call + | cmp RX, TMP1 + | beq >1 + | ldr REG1, EX->call + | str REG1, EX:RX->prev_execute_data + | str RX, EX->call + |1: + | mov RX, REG0 + | ldr FCARG1w, OP:REG0->op2.num + | EXT_CALL zend_cannot_pass_by_reference, REG0 + | ldrb TMP1w, OP:RX->op1_type + | cmp TMP1w, #IS_TMP_VAR + | bne >9 + | ldr REG0, OP:RX->op1.var + | add REG0, REG0, FP + | ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2 + |9: + | b ->exception_handler return 1; } @@ -1688,7 +1745,8 @@ static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) { |->undefined_offset_ex: - | NIY_STUB // TODO + | SAVE_IP + | b ->undefined_offset return 1; } @@ -1696,7 +1754,28 @@ static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst) static int zend_jit_undefined_offset_stub(dasm_State **Dst) { |->undefined_offset: - | NIY_STUB // TODO + | //sub r4, 8 + | ldr REG0, EX->opline + | ldr REG1w, OP:REG0->result.var + | add REG1, REG1, FP + | SET_Z_TYPE_INFO REG1, IS_NULL, TMP1w + | ldrb REG1w, OP:REG0->op2_type + | cmp REG1w, #IS_CONST + | bne >2 + | ldr REG1w, OP:REG0->op2.constant + | sxtw REG1, REG1w + | add REG0, REG0, REG1 + | b >3 + |2: + | ldr REG0w, OP:REG0->op2.var + | add REG0, REG0, FP + |3: + | mov CARG1, #E_WARNING + | LOAD_ADDR CARG2, "Undefined array key " ZEND_LONG_FMT + | ldr CARG3, [REG0] + | EXT_JMP zend_error, REG0 // tail call + | //add r4, 8 // stack alignment + | //ret return 1; } @@ -1713,7 +1792,29 @@ static int zend_jit_undefined_index_ex_stub(dasm_State **Dst) static int zend_jit_undefined_index_stub(dasm_State **Dst) { |->undefined_index: - | NIY_STUB // TODO + | //sub r4, 8 + | ldr REG0, EX->opline + | ldr REG1w, OP:REG0->result.var + | add REG1, REG1, FP + | SET_Z_TYPE_INFO REG1, IS_NULL, TMP1w + | ldrb REG1w, OP:REG0->op2_type + | cmp REG1w, #IS_CONST + | bne >2 + | ldr REG1w, OP:REG0->op2.constant + | sxtw REG1, REG1w + | add REG0, REG0, REG1 + | b >3 + |2: + | ldr REG0w, OP:REG0->op2.var + | add REG0, REG0, FP + |3: + | mov CARG1, #E_WARNING + | LOAD_ADDR CARG2, "Undefined array key \"%s\"" + | ldr CARG3, [REG0] + | add CARG3, CARG3, #offsetof(zend_string, val) + | EXT_JMP zend_error,REG0 // tail call + | //add r4, 8 + | //ret return 1; } From 97762794159c8bdb462121acf1c92f4f74ada435 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 23 Apr 2021 02:52:01 +0300 Subject: [PATCH 133/229] Get rid of NYI in call/return sequences --- ext/opcache/jit/zend_jit_arm64.dasc | 32 ++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 948739d84e148..b6a9060c9d22d 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -7664,7 +7664,6 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con | tst TMP1w, #1 | bne >1 } else { - | NIY // TODO: test | LOAD_32BIT_VAL FCARG1w, used_stack } | // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval); @@ -7678,7 +7677,13 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con | ldr TMP1w, [REG0, #offsetof(zend_function, op_array.T)] | sub REG2w, REG2w, TMP1w } else { - | NIY // TODO + | ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.num_args)] + | cmp REG2w, TMP1w + | csel REG2w, REG2w, TMP1w, le + | ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.last_var)] + | sub REG2w, REG2w, TMP1w + | ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.T)] + | sub REG2w, REG2w, TMP1w } | lsl REG2w, REG2w, #5 | sxtw REG2, REG2w @@ -7717,7 +7722,6 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con | blt >1 |.cold_code |1: - | NIY // TODO: test. Cold. | EXT_JMP exit_addr, TMP1 |.code } else { @@ -7773,13 +7777,15 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con && op_array == &func->op_array && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) && (sizeof(void*) != 8 || IS_SIGNED_32BIT(func))) { - | NIY // TODO + | LOAD_ADDR TMP1, func + | str TMP1, EX:RX->func } else { | str REG0, EX:RX->func } } else { | // call->func = &closure->func; - | NIY // TODO + | add REG1, REG0, #offsetof(zend_closure, func) + | str REG1, EX:RX->func } |1: } @@ -7790,9 +7796,13 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con if (opline->op1_type == IS_UNUSED || use_this) { | // call->call_info |= ZEND_CALL_HAS_THIS; if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - | NIY // TODO + | LOAD_32BIT_VAL TMP1w, ZEND_CALL_HAS_THIS + | str TMP1w, EX:RX->This.u1.type_info } else { - | NIY // TODO + | LOAD_32BIT_VAL TMP1w, ZEND_CALL_HAS_THIS + | ldr TMP2w, EX:RX->This.u1.type_info + | orr TMP2w, TMP2w, TMP1w + | str TMP2w, EX:RX->This.u1.type_info } } else { if (opline->op1_type == IS_CV) { @@ -7801,7 +7811,8 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con } | // call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS; if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - | NIY // TODO + | LOAD_32BIT_VAL TMP1w, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) + | str TMP1w, EX:RX->This.u1.type_info } else { | ldr TMP1w, EX:RX->This.u1.type_info | LOAD_32BIT_VAL TMP2w, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) @@ -8019,7 +8030,9 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t zend_function *func = NULL; if (delayed_call_chain) { - | NIY // TODO + if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { + return 0; + } } if (info) { @@ -9990,7 +10003,6 @@ static int zend_jit_leave_func(dasm_State **Dst, | ldr FP, EX->prev_execute_data if (!left_frame) { - | NIY // TODO: teset | // EG(current_execute_data) = execute_data; | MEM_STORE_ZTS str, FP, executor_globals, current_execute_data, REG0 } From 03537c460294922c1dba62807972aa1751e69b08 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 23 Apr 2021 02:53:24 +0300 Subject: [PATCH 134/229] Temporary diable JIT for SWITCH and MATCH instructions (SWITCH should work, MATCH is going to fail) --- ext/opcache/jit/zend_jit.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 221902445f8db..643c800efed1e 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -3288,13 +3288,14 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op goto jit_failure; } goto done; - case ZEND_SWITCH_LONG: - case ZEND_SWITCH_STRING: - case ZEND_MATCH: - if (!zend_jit_switch(&dasm_state, opline, op_array, ssa, NULL, NULL)) { - goto jit_failure; - } - goto done; +// TODO: Temorary disable JIT for switch and match. Restore it whem implemented! +// case ZEND_SWITCH_LONG: +// case ZEND_SWITCH_STRING: +// case ZEND_MATCH: +// if (!zend_jit_switch(&dasm_state, opline, op_array, ssa, NULL, NULL)) { +// goto jit_failure; +// } +// goto done; case ZEND_VERIFY_RETURN_TYPE: if (opline->op1_type == IS_UNUSED) { /* Always throws */ From 7c6b451ecce251bfeb18b47c0dcad299c7cdac71 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 23 Apr 2021 08:57:13 +0300 Subject: [PATCH 135/229] Fixed INC/DEC_PROP (tests/classes/incdec_property_*.phpt) --- ext/opcache/jit/zend_jit_arm64.dasc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index b6a9060c9d22d..1d1d418478abe 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -12008,7 +12008,8 @@ static int zend_jit_incdec_obj(dasm_State **Dst, | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); | LOAD_ADDR FCARG2x, name | ldr CARG3, EX->run_time_cache - | LOAD_32BIT_VAL CARG3, opline->extended_value + | LOAD_32BIT_VAL TMP1, opline->extended_value + | add CARG3, CARG3, TMP1 if (opline->result_type == IS_UNUSED) { | mov CARG4, xzr } else { From b4f013a2df5741ba1315828f8e4c6ba9a5b40217 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 23 Apr 2021 09:49:50 +0300 Subject: [PATCH 136/229] Fixed load of incorrect size --- ext/opcache/jit/zend_jit_arm64.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 1d1d418478abe..b8b158880a583 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -1733,7 +1733,7 @@ static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst) | ldrb TMP1w, OP:RX->op1_type | cmp TMP1w, #IS_TMP_VAR | bne >9 - | ldr REG0, OP:RX->op1.var + | ldr REG0w, OP:RX->op1.var | add REG0, REG0, FP | ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2 |9: From 23b07035a93c4b6e66d4867bd467a8155d690726 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 23 Apr 2021 10:46:05 +0300 Subject: [PATCH 137/229] Fixed map_ptr resolution --- ext/opcache/jit/zend_jit_arm64.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index b8b158880a583..3eba2b2bad610 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -8598,8 +8598,8 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | beq >1 | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 |1: - | ldr REG2, [REG2] } + | ldr REG2, [REG2] } else { | tst REG2, #1 | beq >1 From 2854043742ed4af5cacba24b32c4b5b4aefabdd9 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 23 Apr 2021 12:07:20 +0300 Subject: [PATCH 138/229] Fixed compilation warnings --- ext/opcache/jit/zend_jit_arm64.dasc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 3eba2b2bad610..960cad6b2eb1c 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -205,7 +205,7 @@ static void* dasm_labels[zend_lb_MAX]; // Type cast to unsigned is used to avoid undefined behavior. |.macro LOAD_32BIT_VAL, reg, val -|| if (val >= 0 && val <= MOVZ_IMM) { +|| if (((uintptr_t)(val)) <= MOVZ_IMM) { | movz reg, #val || } else { | mov reg, #((uint32_t)(val) & 0xffff) @@ -214,7 +214,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro LOAD_64BIT_VAL, reg, val -|| if (val >= 0 && val <= MOVZ_IMM) { +|| if (((uintptr_t)(val)) <= MOVZ_IMM) { | movz reg, #val || } else { | mov reg, #((uint64_t)(val) & 0xffff) @@ -228,7 +228,7 @@ static void* dasm_labels[zend_lb_MAX]; // When using Z_OFFSET(addr), which is 24-bit long, as the unsigned offset to compute a memory address, // we should firstly check the range. |.macro SAFE_MEM_ACC_WITH_UOFFSET, ldr_str_ins, op, base_reg, offset, tmp_reg -|| if (offset > LDR_STR_PIMM) { +|| if (((uintptr_t)(offset)) > LDR_STR_PIMM) { | LOAD_32BIT_VAL tmp_reg, offset | ldr_str_ins op, [base_reg, tmp_reg] || } else { @@ -237,7 +237,7 @@ static void* dasm_labels[zend_lb_MAX]; |.endmacro |.macro SAFE_MEM_ACC_WITH_UOFFSET_BYTE, ldrb_strb_ins, op, base_reg, offset, tmp_reg -|| if (offset > LDRB_STRB_PIMM) { +|| if (((uintptr_t)(offset)) > LDRB_STRB_PIMM) { | LOAD_32BIT_VAL tmp_reg, offset | ldrb_strb_ins op, [base_reg, tmp_reg] || } else { @@ -380,7 +380,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro LOAD_BASE_ADDR, reg, base, offset || if (offset) { -|| if (offset > ADD_SUB_IMM) { +|| if (((uintptr_t)(offset)) > ADD_SUB_IMM) { | LOAD_32BIT_VAL reg, offset | add reg, Rx(base), reg || } else { From c1020c4ecfbf53c5c4e75172a4033c995fa8b1fc Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 23 Apr 2021 12:34:29 +0300 Subject: [PATCH 139/229] Support for interupts --- ext/opcache/jit/zend_jit_arm64.dasc | 47 +++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 960cad6b2eb1c..97341156d5a9f 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -1581,7 +1581,37 @@ static inline bool is_signed(double d) static int zend_jit_interrupt_handler_stub(dasm_State **Dst) { |->interrupt_handler: - | NIY_STUB // TODO + | SAVE_IP + | //EG(vm_interrupt) = 0; + | MEM_STORE_ZTS strb, wzr, executor_globals, vm_interrupt, TMP1 + | //if (EG(timed_out)) { + | MEM_LOAD_ZTS ldrb, REG0w, executor_globals, timed_out, TMP1 + | cbz REG0w, >1 + | //zend_timeout(); + | EXT_CALL zend_timeout, TMP1 + |1: + | //} else if (zend_interrupt_function) { + if (zend_interrupt_function) { + | //zend_interrupt_function(execute_data); + | mov CARG1, FP + | EXT_CALL zend_interrupt_function, TMP1 + | //ZEND_VM_ENTER(); + | //execute_data = EG(current_execute_data); + | MEM_LOAD_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 + | LOAD_IP + } + | //ZEND_VM_CONTINUE() + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | ADD_HYBRID_SPAD + | JMP_IP TMP1 + } else if (GCC_GLOBAL_REGS) { + | add sp, sp, SPAD // stack alignment + | JMP_IP TMP1 + } else { + | NIY_STUB // TODO + } + + return 1; return 1; } @@ -2601,7 +2631,20 @@ static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const void *exit_addr) { - // TODO: not implemented. + | MEM_LOAD_ZTS ldrb, REG0w, executor_globals, vm_interrupt, TMP1 + if (exit_addr) { + | NIY // cbnz REG0w, &exit_addr + } else if (last_valid_opline == opline) { + || zend_jit_use_last_valid_opline(); + | cbnz REG0w, ->interrupt_handler + } else { + | cbz REG0w, >1 + |.cold_code + |1: + | LOAD_IP_ADDR opline + | b ->interrupt_handler + |.code + } return 1; } From 8f98474b5a54c89a046ff9613465436ffc83a832 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 23 Apr 2021 12:57:49 +0300 Subject: [PATCH 140/229] Fixed type check --- ext/opcache/jit/zend_jit_arm64.dasc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 97341156d5a9f..e661d62d9f29c 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -9346,7 +9346,8 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend if (opline->opcode == ZEND_SEND_VAR_NO_REF) { | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 if (op1_info & MAY_BE_REF) { - | cmp REG1w, #IS_REFERENCE + | and TMP1w, REG1w, #0xff + | cmp TMP1w, #IS_REFERENCE | beq >7 } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { From b9281168aba4db2418566e8d7fd52b927979bfee Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 23 Apr 2021 13:29:56 +0300 Subject: [PATCH 141/229] Wrong register --- ext/opcache/jit/zend_jit_arm64.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index e661d62d9f29c..8c30d95e6dc85 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -11860,7 +11860,7 @@ static int zend_jit_incdec_obj(dasm_State **Dst, | LOAD_32BIT_VAL TMP1, opline->extended_value | ldr REG2, [REG0, TMP1] | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] - | cmp REG2, TMP2 + | cmp REG2, TMP1 | bne >7 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { | LOAD_32BIT_VAL TMP1, (opline->extended_value + sizeof(void*) * 2) From a1bf150b9ebb2629c45b498f5b756ed00a0e1f40 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 23 Apr 2021 14:00:44 +0300 Subject: [PATCH 142/229] Fixed condition and avoid usage of non-temporary registers --- ext/opcache/jit/zend_jit_arm64.dasc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 8c30d95e6dc85..89985b469f035 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -2631,14 +2631,14 @@ static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const void *exit_addr) { - | MEM_LOAD_ZTS ldrb, REG0w, executor_globals, vm_interrupt, TMP1 + | MEM_LOAD_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 if (exit_addr) { - | NIY // cbnz REG0w, &exit_addr + | NIY // cbnz TMP1w, &exit_addr } else if (last_valid_opline == opline) { || zend_jit_use_last_valid_opline(); - | cbnz REG0w, ->interrupt_handler + | cbnz TMP1w, ->interrupt_handler } else { - | cbz REG0w, >1 + | cbnz TMP1w, >1 |.cold_code |1: | LOAD_IP_ADDR opline From cf3c313150113cdbfcbc995090642cbdd0481643 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 23 Apr 2021 16:34:36 +0300 Subject: [PATCH 143/229] Disable unsuitable optimization --- ext/opcache/jit/zend_jit_arm64.dasc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 89985b469f035..c5f45c9706fc9 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -5068,6 +5068,8 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, bool keep_gc = 0; | GET_ZVAL_PTR Rx(tmp_reg), var_use_addr, TMP1 +#if 0 + // TODO: This optiization doesn't work on ARM if (tmp_reg == ZREG_FCARG1x) { if (Z_MODE(val_addr) == IS_REG) { keep_gc = 1; @@ -5088,6 +5090,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, } } } +#endif if (!keep_gc) { | str Rx(tmp_reg), T1 // save } From 1c6bea81533e2a82e30d91be134a6a51eb47e99c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 23 Apr 2021 17:04:27 +0300 Subject: [PATCH 144/229] Fixed incorrect efree() --- ext/opcache/jit/zend_jit_arm64.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index c5f45c9706fc9..4d3f466d00c91 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -4890,7 +4890,7 @@ static int zend_jit_simple_assign(dasm_State **Dst, if (save_r1) { | str FCARG1x, T1 // save } - | LOAD_ZVAL_ADDR FCARG1x, val_addr + | GET_ZVAL_PTR FCARG1x, val_addr, TMP1 | EFREE_REFERENCE if (save_r1) { | ldr FCARG1x, T1 // restore From 350115196c805b27bb427110dca5a4d1fe0e4fda Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 23 Apr 2021 17:42:53 +0300 Subject: [PATCH 145/229] Create C call frames for helper functions that perform nested calls --- ext/opcache/jit/zend_jit_arm64.dasc | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 4d3f466d00c91..8a142b8026741 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -2289,6 +2289,8 @@ static int zend_jit_assign_const_stub(dasm_State **Dst) uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; |->assign_const: + | stp x29, x30, [sp,#-16]! + | mov x29, sp | sub sp, sp, #16 if (!zend_jit_assign_to_variable( Dst, NULL, @@ -2297,7 +2299,8 @@ static int zend_jit_assign_const_stub(dasm_State **Dst) 0, 0)) { return 0; } - | add sp, sp, #16 + | mov sp, x29 + | ldp x29, x30, [sp],#16 | ret return 1; } @@ -2309,6 +2312,8 @@ static int zend_jit_assign_tmp_stub(dasm_State **Dst) uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; |->assign_tmp: + | stp x29, x30, [sp,#-16]! + | mov x29, sp | sub sp, sp, #16 if (!zend_jit_assign_to_variable( Dst, NULL, @@ -2317,7 +2322,8 @@ static int zend_jit_assign_tmp_stub(dasm_State **Dst) 0, 0)) { return 0; } - | add sp, sp, #16 + | mov sp, x29 + | ldp x29, x30, [sp],#16 | ret return 1; } @@ -2329,6 +2335,7 @@ static int zend_jit_assign_var_stub(dasm_State **Dst) uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF; |->assign_var: + | stp x29, x30, [sp,#-16]! | sub sp, sp, #16 if (!zend_jit_assign_to_variable( Dst, NULL, @@ -2337,7 +2344,8 @@ static int zend_jit_assign_var_stub(dasm_State **Dst) 0, 0)) { return 0; } - | add sp, sp, #16 + | mov sp, x29 + | ldp x29, x30, [sp],#16 | ret return 1; } @@ -2349,6 +2357,8 @@ static int zend_jit_assign_cv_noref_stub(dasm_State **Dst) uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/; |->assign_cv_noref: + | stp x29, x30, [sp,#-16]! + | mov x29, sp | sub sp, sp, #16 if (!zend_jit_assign_to_variable( Dst, NULL, @@ -2357,7 +2367,8 @@ static int zend_jit_assign_cv_noref_stub(dasm_State **Dst) 0, 0)) { return 0; } - | add sp, sp, #16 + | mov sp, x29 + | ldp x29, x30, [sp],#16 | ret return 1; } @@ -2369,6 +2380,8 @@ static int zend_jit_assign_cv_stub(dasm_State **Dst) uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/; |->assign_cv: + | stp x29, x30, [sp,#-16]! + | mov x29, sp | sub sp, sp, #16 if (!zend_jit_assign_to_variable( Dst, NULL, @@ -2377,7 +2390,8 @@ static int zend_jit_assign_cv_stub(dasm_State **Dst) 0, 0)) { return 0; } - | add sp, sp, #16 + | mov sp, x29 + | ldp x29, x30, [sp],#16 | ret return 1; } From c84cc926336b8001234cb3eaed53217cba7f75a3 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 23 Apr 2021 18:20:41 +0300 Subject: [PATCH 146/229] Fixed type checks and return value handling --- ext/opcache/jit/zend_jit_arm64.dasc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 8a142b8026741..36be7fc86790c 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -5243,6 +5243,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t | str Rx(Z_REG(op1_addr)), T1 // save } | EXT_CALL _zend_new_array_0, REG0 + | mov REG0, RETVALx if (Z_REG(op1_addr) != ZREG_FP) { | ldr Rx(Z_REG(op1_addr)), T1 // restore } @@ -9265,7 +9266,8 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend if (!exit_addr) { return 0; } - | cmp REG1w, #IS_REFERENCE + | and TMP1w, REG1w, #0xff + | cmp TMP1w, #IS_REFERENCE | NIY // bne &exit_addr } } @@ -9286,7 +9288,8 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 if (op1_info & MAY_BE_REF) { - | cmp REG1w, #IS_REFERENCE + | and TMP1w, REG1w, #0xff + | cmp TMP1w, #IS_REFERENCE | beq >7 } | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] From 42bec86a3c3c63dbf5382ee0f5f3828c1922424c Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Thu, 22 Apr 2021 09:20:36 +0000 Subject: [PATCH 147/229] Optimizing LONG MUL to SHIFT: refine the trigger condition and add overflow detection LONG MUL can be optimzied into left shift if either operand is a power of two. Conditions "IS_SIGNED_32BIT()" and "is_power_of_two()" are used to filter out invalid candidates. However, there exists one exception, i.e. -2147483648(that is 0xffff,ffff,8000,0000). See the stand-alone case[1]. Assume "a = 3; b = -2147483648;". The expected result of "a * b" is one negative value. However, it would be optimized to "a << 31", which is positive. This trigger condition is refined. 1) For x86 implementation, another check for positive numbers is added. Note that LONG type, i.e. zend_long, is defined as int32_t for x86 arch and int64_t for x64 arch. This optimization only accepts values which can be represented by int32_t type as default. See IS_SIGNED_32BIlT(), 2) For AArch64, we employ helper function zend_long_is_power_of_two() since values of int64_t type are used. Overflow detection for left shifting is added in this patch as well. Note 1: bit helper functions are arch-independent and we move them into zend_jit_internals.h. Note 2: two test cases are added. Test case mul_003.phpt is used to check the trigger condition and mul_004.phpt is designed to check overflow detection. Note 3: overflow detection for x86 is not implemented yet as I think anotehr temporay register besides R0 is needed. Hence mul_004.phpt would fail on x86 machine. If we can use R1 as tmp_reg, the code can be updated as below. ``` | GET_ZVAL_LVAL result_reg, op1_addr if (may_overflow) { use_ovf_flag = 0; /* Compare 'op' and '((op << n) >> n)' for overflow. * Flag: jne -> overflow. je -> no overflow. */ tmp_reg = ZREG_R1 | mov Ra(tmp_reg), Ra(result_reg) | shl Ra(tmp_reg), floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) | sar Ra(tmp_reg), floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) | cmp Ra(tmp_reg), Ra(result_reg) } | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) ``` [1]. https://godbolt.org/z/1vKbfv8oG Change-Id: Ie90e1d4e7c8b94a0c8f61386dfe650fa2c6879a1 --- ext/opcache/jit/zend_jit.c | 5 -- ext/opcache/jit/zend_jit_arm64.dasc | 108 ++++++++--------------- ext/opcache/jit/zend_jit_internal.h | 55 ++++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 52 ++--------- ext/opcache/tests/jit/arm64/mul_003.phpt | 29 ++++++ ext/opcache/tests/jit/arm64/mul_004.phpt | 59 +++++++++++++ 6 files changed, 189 insertions(+), 119 deletions(-) create mode 100644 ext/opcache/tests/jit/arm64/mul_003.phpt create mode 100644 ext/opcache/tests/jit/arm64/mul_004.phpt diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 643c800efed1e..efbed71995d5f 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -195,11 +195,6 @@ static bool zend_is_commutative(zend_uchar opcode) opcode == ZEND_BW_XOR; } -static bool zend_long_is_power_of_two(zend_long x) -{ - return (x > 0) && !(x & (x - 1)); -} - #define OP_RANGE(ssa_op, opN) \ (((opline->opN##_type & (IS_TMP_VAR|IS_VAR|IS_CV)) && \ ssa->var_info && \ diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 36be7fc86790c..90d8c6ab06b48 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -1534,50 +1534,6 @@ static void zend_jit_stop_reuse_ip(void) reuse_ip = 0; } -/* bit helpers */ - -/* from http://aggregate.org/MAGIC/ */ -static uint32_t ones32(uint32_t x) -{ - x -= ((x >> 1) & 0x55555555); - x = (((x >> 2) & 0x33333333) + (x & 0x33333333)); - x = (((x >> 4) + x) & 0x0f0f0f0f); - x += (x >> 8); - x += (x >> 16); - return x & 0x0000003f; -} - -static uint32_t floor_log2(uint32_t x) -{ - ZEND_ASSERT(x != 0); - x |= (x >> 1); - x |= (x >> 2); - x |= (x >> 4); - x |= (x >> 8); - x |= (x >> 16); - return ones32(x) - 1; -} - -static bool is_power_of_two(uint32_t x) -{ - return !(x & (x - 1)) && x != 0; -} - -static bool has_concrete_type(uint32_t value_type) -{ - return is_power_of_two (value_type & (MAY_BE_ANY|MAY_BE_UNDEF)); -} - -static uint32_t concrete_type(uint32_t value_type) -{ - return floor_log2(value_type & (MAY_BE_ANY|MAY_BE_UNDEF)); -} - -static inline bool is_signed(double d) -{ - return (((unsigned char*)&d)[sizeof(double)-1] & 0x80) != 0; -} - static int zend_jit_interrupt_handler_stub(dasm_State **Dst) { |->interrupt_handler: @@ -3341,7 +3297,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); zend_reg result_reg; zend_reg tmp_reg = ZREG_REG0; - bool use_mul = 0; + bool use_ovf_flag = 1; if (Z_MODE(res_addr) == IS_REG && (res_info & MAY_BE_LONG)) { if (may_overflow && (res_info & MAY_BE_GUARD) @@ -3363,8 +3319,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, if (opcode == ZEND_MUL && Z_MODE(op2_addr) == IS_CONST_ZVAL && - IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op2_addr))) && - is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { + zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { if (Z_LVAL_P(Z_ZV(op2_addr)) == 2) { if (Z_MODE(op1_addr) == IS_REG) { @@ -3374,15 +3329,21 @@ static int zend_jit_math_long_long(dasm_State **Dst, | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) } } else { - | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 - | mov TMP1, #floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) - | lsl Rx(result_reg), Rx(result_reg), TMP1 - | // TODO: overflow may be missed + | GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP2 + | mov TMP2, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) + | lsl Rx(result_reg), TMP1, TMP2 + if (may_overflow) { + /* Compare 'op' and '((op << n) >> n)' for overflow. + * Flag: bne -> overflow. beq -> no overflow. + */ + use_ovf_flag = 0; + | asr TMP3, Rx(result_reg), TMP2 + | cmp TMP1, TMP3 + } } } else if (opcode == ZEND_MUL && Z_MODE(op1_addr) == IS_CONST_ZVAL && - IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr))) && - is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { + zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { if (Z_LVAL_P(Z_ZV(op1_addr)) == 2) { if (Z_MODE(op2_addr) == IS_REG) { @@ -3392,10 +3353,17 @@ static int zend_jit_math_long_long(dasm_State **Dst, | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) } } else { - | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 - | mov TMP1, #floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) - | lsl Rx(result_reg), Rx(result_reg), TMP1 - | // TODO: overflow may be missed + | GET_ZVAL_LVAL ZREG_TMP1, op2_addr, TMP2 + | mov TMP2, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) + | lsl Rx(result_reg), TMP1, TMP2 + if (may_overflow) { + /* Compare 'op' and '((op << n) >> n)' for overflow. + * Flag: bne -> overflow. beq -> no overflow. + */ + use_ovf_flag = 0; + | asr TMP3, Rx(result_reg), TMP2 + | cmp TMP1, TMP3 + } } } else if (opcode == ZEND_DIV && (Z_MODE(op2_addr) == IS_CONST_ZVAL && @@ -3421,7 +3389,6 @@ static int zend_jit_math_long_long(dasm_State **Dst, | NIY // TODO: test #endif } else if (opcode == ZEND_MUL) { - use_mul = 1; | GET_ZVAL_LVAL ZREG_TMP2, op1_addr, TMP1 | GET_ZVAL_LVAL ZREG_TMP3, op2_addr, TMP1 | mul Rx(result_reg), TMP2, TMP3 @@ -3433,6 +3400,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, * currently, and we put 'asr' and 'cmp' separately. * Flag: bne -> overflow. beq -> no overflow. */ + use_ovf_flag = 0; | smulh TMP1, TMP2, TMP3 | asr TMP2, Rx(result_reg), #63 | cmp TMP1, TMP2 @@ -3455,10 +3423,10 @@ static int zend_jit_math_long_long(dasm_State **Dst, int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) { - if (use_mul) { - | bne >3 - } else { + if (use_ovf_flag) { | bvs >3 + } else { + | bne >3 } |.cold_code |3: @@ -3468,10 +3436,10 @@ static int zend_jit_math_long_long(dasm_State **Dst, | mov Rx(Z_REG(res_addr)), Rx(result_reg) } } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - if (use_mul) { - | beq >3 - } else { + if (use_ovf_flag) { | bvc >3 + } else { + | beq >3 } |.cold_code |3: @@ -3482,16 +3450,16 @@ static int zend_jit_math_long_long(dasm_State **Dst, } } else { if (res_info & MAY_BE_LONG) { - if (use_mul) { - | bne >1 - } else { + if (use_ovf_flag) { | bvs >1 + } else { + | bne >1 } } else { - if (use_mul) { - | beq >1 - } else { + if (use_ovf_flag) { | bvc >1 + } else { + | beq >1 } } } diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 638ed243c4573..bba49fb3cbfa6 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -738,4 +738,59 @@ static zend_always_inline bool zend_jit_may_be_polymorphic_call(const zend_op *o # endif #endif /* !JIT_CACHE_FLUSH */ +/* bit helpers */ + +static bool zend_long_is_power_of_two(zend_long x) +{ + return (x > 0) && !(x & (x - 1)); +} + +static uint32_t zend_long_floor_log2(uint64_t x) +{ + ZEND_ASSERT(zend_long_is_power_of_two(x)); + return __builtin_ctzll(x); +} + +/* from http://aggregate.org/MAGIC/ */ +static uint32_t ones32(uint32_t x) +{ + x -= ((x >> 1) & 0x55555555); + x = (((x >> 2) & 0x33333333) + (x & 0x33333333)); + x = (((x >> 4) + x) & 0x0f0f0f0f); + x += (x >> 8); + x += (x >> 16); + return x & 0x0000003f; +} + +static uint32_t floor_log2(uint32_t x) +{ + ZEND_ASSERT(x != 0); + x |= (x >> 1); + x |= (x >> 2); + x |= (x >> 4); + x |= (x >> 8); + x |= (x >> 16); + return ones32(x) - 1; +} + +static bool is_power_of_two(uint32_t x) +{ + return !(x & (x - 1)) && x != 0; +} + +static bool has_concrete_type(uint32_t value_type) +{ + return is_power_of_two (value_type & (MAY_BE_ANY|MAY_BE_UNDEF)); +} + +static uint32_t concrete_type(uint32_t value_type) +{ + return floor_log2(value_type & (MAY_BE_ANY|MAY_BE_UNDEF)); +} + +static inline bool is_signed(double d) +{ + return (((unsigned char*)&d)[sizeof(double)-1] & 0x80) != 0; +} + #endif /* ZEND_JIT_INTERNAL_H */ diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 81c9dd20e9bde..5394f46e08fbd 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -1703,50 +1703,6 @@ static void zend_jit_stop_reuse_ip(void) reuse_ip = 0; } -/* bit helpers */ - -/* from http://aggregate.org/MAGIC/ */ -static uint32_t ones32(uint32_t x) -{ - x -= ((x >> 1) & 0x55555555); - x = (((x >> 2) & 0x33333333) + (x & 0x33333333)); - x = (((x >> 4) + x) & 0x0f0f0f0f); - x += (x >> 8); - x += (x >> 16); - return x & 0x0000003f; -} - -static uint32_t floor_log2(uint32_t x) -{ - ZEND_ASSERT(x != 0); - x |= (x >> 1); - x |= (x >> 2); - x |= (x >> 4); - x |= (x >> 8); - x |= (x >> 16); - return ones32(x) - 1; -} - -static bool is_power_of_two(uint32_t x) -{ - return !(x & (x - 1)) && x != 0; -} - -static bool has_concrete_type(uint32_t value_type) -{ - return is_power_of_two (value_type & (MAY_BE_ANY|MAY_BE_UNDEF)); -} - -static uint32_t concrete_type(uint32_t value_type) -{ - return floor_log2(value_type & (MAY_BE_ANY|MAY_BE_UNDEF)); -} - -static inline bool is_signed(double d) -{ - return (((unsigned char*)&d)[sizeof(double)-1] & 0x80) != 0; -} - static int zend_jit_interrupt_handler_stub(dasm_State **Dst) { |->interrupt_handler: @@ -4245,6 +4201,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, if (opcode == ZEND_MUL && Z_MODE(op2_addr) == IS_CONST_ZVAL && + Z_LVAL_P(Z_ZV(op2_addr)) > 0 && IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op2_addr))) && is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { @@ -4253,9 +4210,13 @@ static int zend_jit_math_long_long(dasm_State **Dst, } else { | GET_ZVAL_LVAL result_reg, op1_addr | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) + if (may_overflow) { + // TODO: check overflow. + } } } else if (opcode == ZEND_MUL && Z_MODE(op1_addr) == IS_CONST_ZVAL && + Z_LVAL_P(Z_ZV(op1_addr)) > 0 && IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr))) && is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { @@ -4264,6 +4225,9 @@ static int zend_jit_math_long_long(dasm_State **Dst, } else { | GET_ZVAL_LVAL result_reg, op2_addr | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) + if (may_overflow) { + // TODO: check overflow. + } } } else if (opcode == ZEND_DIV && (Z_MODE(op2_addr) == IS_CONST_ZVAL && diff --git a/ext/opcache/tests/jit/arm64/mul_003.phpt b/ext/opcache/tests/jit/arm64/mul_003.phpt new file mode 100644 index 0000000000000..7404e86926784 --- /dev/null +++ b/ext/opcache/tests/jit/arm64/mul_003.phpt @@ -0,0 +1,29 @@ +--TEST-- +JIT MUL: 003 boundary value for optmizing MUL to SHIFT +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +int(-6442450944) +int(-6442450944) \ No newline at end of file diff --git a/ext/opcache/tests/jit/arm64/mul_004.phpt b/ext/opcache/tests/jit/arm64/mul_004.phpt new file mode 100644 index 0000000000000..438e4325245a4 --- /dev/null +++ b/ext/opcache/tests/jit/arm64/mul_004.phpt @@ -0,0 +1,59 @@ +--TEST-- +JIT MUL: 004 Overflow check for optmizing MUL to SHIFT +--INI-- +opcache.enable=1 +opcache.enable_cli=1 +opcache.file_update_protection=0 +opcache.jit_buffer_size=32M +;opcache.jit_debug=257 +--EXTENSIONS-- +opcache +--FILE-- + +--EXPECT-- +int(24) +int(-88) +float(7.378697629483821E+19) +int(48) +int(-208) +float(1.4757395258967641E+20) +int(805306368) +int(-805306368) +float(2.9514790517935283E+20) +int(12884901888) +int(-12884901888) +float(1.8446744073709552E+19) \ No newline at end of file From ff72c1575a9b7bad4712dc59fe0fa76a500a7c44 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Sun, 25 Apr 2021 01:40:28 +0000 Subject: [PATCH 148/229] Remove the TODO comments for DOUBLE CMP As explained by Dmitry, 'ucomsd' in x86 sets 'p' flag, and it also always sets 'z' and 'c' flags. [1] Besides, remove one duplicate 'break'. [1]. https://mudongliang.github.io/x86/html/file_module_x86_id_316.html Change-Id: I767214c7ab8db31115801a3ae96b20320757899f --- ext/opcache/jit/zend_jit_arm64.dasc | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 90d8c6ab06b48..3619453c58e2b 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -6050,7 +6050,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z if (exit_addr) { | NIY // tracing } else { - | bvs => target_label // TODO: why the NaN check is missing in x86? + | bvs => target_label | bls => target_label } } else { @@ -6066,7 +6066,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z if (exit_addr) { | NIY // tracing } else { - | bvs => target_label // TODO: why the NaN check is missing in x86? + | bvs => target_label | blo => target_label } } else { @@ -6206,7 +6206,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z case ZEND_IS_SMALLER: if (swap) { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 - | bvs => target_label // TODO: why the NaN check is missing in x86? + | bvs => target_label | bls => target_label | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 } else { @@ -6218,7 +6218,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 - | bvs => target_label // TODO: why the NaN check is missing in x86? + | bvs => target_label | blo => target_label | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 } else { @@ -6309,12 +6309,11 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z | mov REG0, #IS_TRUE |2: break; - break; case ZEND_IS_SMALLER: | bvs >1 | mov REG0, #IS_TRUE || if (swap) { - | bhi >2 // TODO: why the NaN check is missing in x86? + | bhi >2 || } else { | blo >2 || } @@ -6326,7 +6325,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z | bvs >1 | mov REG0, #IS_TRUE || if (swap) { - | bhs >2 // TODO: why the NaN check is missing in x86? + | bhs >2 || } else { | bls >2 || } From 06b9eb500f542373b183e7ca4ea0b85757c55984 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Sun, 25 Apr 2021 06:54:00 +0000 Subject: [PATCH 149/229] Support failed test case: switch_jumptable.phpt Opcodes ZEND_SWITCH_LONG, ZEND_SWITCH_STRING and ZEND_MATCH are supported in this patch. Change-Id: I71ae3d40e65ec1b29d3d9115ac41caef17cf6ae5 --- ext/opcache/jit/zend_jit.c | 15 +- ext/opcache/jit/zend_jit_arm64.dasc | 337 +++++++++++++++++++++++++++- 2 files changed, 342 insertions(+), 10 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index efbed71995d5f..e7374a62d0278 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -3283,14 +3283,13 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op goto jit_failure; } goto done; -// TODO: Temorary disable JIT for switch and match. Restore it whem implemented! -// case ZEND_SWITCH_LONG: -// case ZEND_SWITCH_STRING: -// case ZEND_MATCH: -// if (!zend_jit_switch(&dasm_state, opline, op_array, ssa, NULL, NULL)) { -// goto jit_failure; -// } -// goto done; + case ZEND_SWITCH_LONG: + case ZEND_SWITCH_STRING: + case ZEND_MATCH: + if (!zend_jit_switch(&dasm_state, opline, op_array, ssa, NULL, NULL)) { + goto jit_failure; + } + goto done; case ZEND_VERIFY_RETURN_TYPE: if (opline->op1_type == IS_UNUSED) { /* Always throws */ diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 3619453c58e2b..37285361ce8e4 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -12840,7 +12840,58 @@ static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend int32_t exit_point; const void *exit_addr; - | NIY // TODO + if (default_label) { + | NIY // jz &default_label + } else if (next_opline) { + | cbz REG0, >3 + } else { + | cbz REG0, =>default_b + } + | LOAD_ADDR FCARG1x, jumptable + | ldr TMP1, [FCARG1x, #offsetof(HashTable, arData)] + | sub REG0, REG0, TMP1 + | mov FCARG1x, #(sizeof(Bucket) / sizeof(void*)) + | sdiv REG0, REG0, FCARG1x + | adr FCARG1x, >4 + | ldr TMP1, [FCARG1x, REG0] + | br TMP1 + + |.jmp_table + |.align 8 + |4: + if (trace_info) { + trace_info->jmp_table_size += zend_hash_num_elements(jumptable); + } + + count = jumptable->nNumUsed; + p = jumptable->arData; + do { + if (Z_TYPE(p->val) == IS_UNDEF) { + if (default_label) { + | .addr &default_label + } else if (next_opline) { + | .addr >3 + } else { + | .addr =>default_b + } + } else { + target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); + if (!next_opline) { + b = ssa->cfg.map[target - op_array->opcodes]; + | .addr =>b + } else if (next_opline == target) { + | .addr >3 + } else { + exit_point = zend_jit_trace_get_exit_point(target, 0); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + | .addr &exit_addr + } + } + p++; + count--; + } while (count); + |.code + return 1; } @@ -12855,7 +12906,289 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o next_opline = trace->opline; } - | NIY // TODO + if (opline->op1_type == IS_CONST) { + zval *zv = RT_CONSTANT(opline, opline->op1); + zval *jump_zv = NULL; + int b; + + if (opline->opcode == ZEND_SWITCH_LONG) { + if (Z_TYPE_P(zv) == IS_LONG) { + jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); + } + } else if (opline->opcode == ZEND_SWITCH_STRING) { + if (Z_TYPE_P(zv) == IS_STRING) { + jump_zv = zend_hash_find_ex(jumptable, Z_STR_P(zv), 1); + } + } else if (opline->opcode == ZEND_MATCH) { + if (Z_TYPE_P(zv) == IS_LONG) { + jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); + } else if (Z_TYPE_P(zv) == IS_STRING) { + jump_zv = zend_hash_find_ex(jumptable, Z_STR_P(zv), 1); + } + } else { + ZEND_UNREACHABLE(); + } + if (next_opline) { + const zend_op *target; + + if (jump_zv != NULL) { + target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)); + } else { + target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); + } + ZEND_ASSERT(target == next_opline); + } else { + if (jump_zv != NULL) { + b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes]; + } else { + b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes]; + } + | b =>b + } + } else { + zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes]; + uint32_t op1_info = OP1_INFO(); + zend_jit_addr op1_addr = OP1_ADDR(); + const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); + const zend_op *target; + int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes]; + int b; + int32_t exit_point; + const void *fallback_label = NULL; + const void *default_label = NULL; + const void *exit_addr; + + if (next_opline) { + if (next_opline != opline + 1) { + exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + fallback_label = zend_jit_trace_get_exit_addr(exit_point); + } + if (next_opline != default_opline) { + exit_point = zend_jit_trace_get_exit_point(default_opline, 0); + default_label = zend_jit_trace_get_exit_addr(exit_point); + } + } + + if (opline->opcode == ZEND_SWITCH_LONG) { + if (op1_info & MAY_BE_LONG) { + if (op1_info & MAY_BE_REF) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >1, TMP1w, TMP2 + | GET_ZVAL_LVAL ZREG_FCARG2x, op1_addr, TMP1 + |.cold_code + |1: + | // ZVAL_DEREF(op) + if (fallback_label) { + | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, TMP1w, TMP2 + } + | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 + if (fallback_label) { + | NIY // IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, &fallback_label + } else { + | add TMP1, FCARG2x, #offsetof(zend_reference, val) + | IF_NOT_Z_TYPE TMP1, IS_LONG, >3, TMP2w + } + | ldr FCARG2x, [FCARG2x, #offsetof(zend_reference, val.value.lval)] + | b >2 + |.code + |2: + } else { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { + if (fallback_label) { + | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, TMP1w, TMP2 + } + } + | GET_ZVAL_LVAL ZREG_FCARG2x, op1_addr, TMP1 + } + if (HT_IS_PACKED(jumptable)) { + uint32_t count = jumptable->nNumUsed; + Bucket *p = jumptable->arData; + + | LOAD_32BIT_VAL TMP1w, jumptable->nNumUsed + | cmp FCARG2x, TMP1 + if (default_label) { + | NIY // jae &default_label + } else if (next_opline) { + | bhs >3 + } else { + | bhs =>default_b + } + | adr REG0, >4 + | lsl TMP1, FCARG2x, #3 + | ldr TMP1, [REG0, TMP1] + | br TMP1 + + |.jmp_table + |.align 8 + |4: + if (trace_info) { + trace_info->jmp_table_size += count; + } + p = jumptable->arData; + do { + if (Z_TYPE(p->val) == IS_UNDEF) { + if (default_label) { + | .addr &default_label + } else if (next_opline) { + | .addr >3 + } else { + | .addr =>default_b + } + } else { + target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)); + if (!next_opline) { + b = ssa->cfg.map[target - op_array->opcodes]; + | .addr =>b + } else if (next_opline == target) { + | .addr >3 + } else { + exit_point = zend_jit_trace_get_exit_point(target, 0); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + | .addr &exit_addr + } + } + p++; + count--; + } while (count); + |.code + |3: + } else { + | LOAD_ADDR FCARG1x, jumptable + | EXT_CALL zend_hash_index_find, REG0 + | mov REG0, RETVALx + if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { + return 0; + } + |3: + } + } + } else if (opline->opcode == ZEND_SWITCH_STRING) { + if (op1_info & MAY_BE_STRING) { + if (op1_info & MAY_BE_REF) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >1, TMP1w, TMP2 + | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 + |.cold_code + |1: + | // ZVAL_DEREF(op) + if (fallback_label) { + | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, TMP1w, TMP2 + } + | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 + if (fallback_label) { + | NIY // IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, &fallback_label + } else { + | add TMP1, FCARG2x, #offsetof(zend_reference, val) + | IF_NOT_Z_TYPE TMP1, IS_STRING, >3, TMP2w + } + | ldr FCARG2x, [FCARG2x, #offsetof(zend_reference, val.value.ptr)] + | b >2 + |.code + |2: + } else { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) { + if (fallback_label) { + | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, TMP1w, TMP2 + } + } + | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 + } + | LOAD_ADDR FCARG1x, jumptable + | EXT_CALL zend_hash_find, REG0 + | mov REG0, RETVALx + if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { + return 0; + } + |3: + } + } else if (opline->opcode == ZEND_MATCH) { + if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) { + if (op1_info & MAY_BE_REF) { + | LOAD_ZVAL_ADDR FCARG2x, op1_addr + | ZVAL_DEREF FCARG2x, op1_info, TMP1w + op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2x, 0); + } + | LOAD_ADDR FCARG1x, jumptable + if (op1_info & MAY_BE_LONG) { + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { + if (op1_info & MAY_BE_STRING) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >5, TMP1w, TMP2 + } else if (op1_info & MAY_BE_UNDEF) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, TMP1w, TMP2 + } else if (default_label) { + | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &default_label + } else if (next_opline) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, TMP1w, TMP2 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, =>default_b, TMP1w, TMP2 + } + } + | GET_ZVAL_LVAL ZREG_FCARG2x, op1_addr, TMP1 + | EXT_CALL zend_hash_index_find, REG0 + | mov REG0, RETVALx + if (op1_info & MAY_BE_STRING) { + | b >2 + } + } + if (op1_info & MAY_BE_STRING) { + |5: + if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) { + if (op1_info & MAY_BE_UNDEF) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, TMP1w, TMP2 + } else if (default_label) { + | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &default_label + } else if (next_opline) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, TMP1w, TMP2 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, =>default_b, TMP1w, TMP2 + } + } + | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 + | EXT_CALL zend_hash_find, REG0 + | mov REG0, RETVALx + } + |2: + if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) { + return 0; + } + } + if (op1_info & MAY_BE_UNDEF) { + |6: + if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) { + if (default_label) { + | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, &default_label + } else if (next_opline) { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >3, TMP1w, TMP2 + } else { + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, =>default_b, TMP1w, TMP2 + } + } + | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); + | SET_EX_OPLINE opline, REG0 + | LOAD_32BIT_VAL FCARG1w, opline->op1.var + | EXT_CALL zend_jit_undefined_op_helper, REG0 + if (!zend_jit_check_exception_undef_result(Dst, opline)) { + return 0; + } + } + if (default_label) { + | NIY // jmp &default_label + } else if (next_opline) { + | b >3 + } else { + | b =>default_b + } + |3: + } else { + ZEND_UNREACHABLE(); + } + } return 1; } From 86326b5ddb0a9bcefbe3028cfa98e56571331e3c Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Sun, 25 Apr 2021 10:10:22 +0000 Subject: [PATCH 150/229] Fix the encoding of immediate for logical instructions Previous implementation[1] of the immediate encoding for logical instructions('and/orr/eor/tst') is incorrect. It's more complicated than that of 'add/sub/ldr/str/movz'.[2] Ideally a helper is needed to "determine whether an immediate value can be encoded as the immediate operand of a logical instruction for the given register size"[3]. Macros "BW_OP_x_WITH_CONST" and "TST_x_WITH_CONST" are defined to wrap the logical instrunctions with constants. Currently this helper is not implemented yet. All the uses of bitwise operations and 'tst' are revisited and updated. Note that test case bug80745.phpt will pass with this patch. [1] https://github.com/php/php-src/commit/47d8252 [2] https://dinfuehr.github.io/blog/encoding-of-immediate-values-on-aarch64/ [3] https://github.com/llvm-mirror/llvm/blob/5c95b810cb3a7dee6d49c030363e5bf0bb41427e/lib/Target/AArch64/MCTargetDesc/AArch64AddressingModes.h#L213 Change-Id: I0bfa088cafcffe30e0f18fa1f0638338ef00eb20 --- ext/opcache/jit/zend_jit_arm64.dasc | 188 +++++++++++++--------------- 1 file changed, 87 insertions(+), 101 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 37285361ce8e4..e470ebb6be68a 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -98,15 +98,10 @@ /* Encoding of immediate. TODO: shift mode may be supported in the near future. */ #define MAX_IMM12 0xfff // maximum value for imm12 -#define MAX_IMM13 0x1fff // maximum value for imm13 #define MAX_IMM16 0xffff // maximum value for imm16 #define CMP_IMM MAX_IMM12 // cmp insn #define MOVZ_IMM MAX_IMM16 // movz insn #define ADD_SUB_IMM MAX_IMM12 // add/sub insn -#define BW_W_IMM MAX_IMM12 // bitwise insn for 32-bit variant: and, orr, eor -#define BW_X_IMM MAX_IMM13 // bitwise insn for 64-bit variant: and, orr, eor -#define TST_W_IMM MAX_IMM12 // tst insn for 32-bit variant -#define TST_X_IMM MAX_IMM13 // tst insn for 64-bit variant #define LDR_STR_PIMM (MAX_IMM12*8) // ldr/str insn: pimm is imm12 * 8 #define LDRB_STRB_PIMM MAX_IMM12 // ldrb/strb insn @@ -224,6 +219,25 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro +// TODO: Support logical instruction with an immediate encoded. +// Bitwise operation with constants. 'ins' can be and/orr/eor/ands. Operands are 64-bit. +|.macro BW_OP_64_WITH_CONST, ins, reg, op, val, tmp_reg +| LOAD_64BIT_VAL tmp_reg, val +| ins reg, op, tmp_reg +|.endmacro + +// Test operation 'tst' with constants. Operands are 32-bit. +|.macro TST_32_WITH_CONST, reg, val, tmp_reg +| LOAD_32BIT_VAL tmp_reg, val +| tst reg, tmp_reg +|.endmacro + +// Test operation 'tst' with constants. Operands are 64-bit. +|.macro TST_64_WITH_CONST, reg, val, tmp_reg +| LOAD_64BIT_VAL tmp_reg, val +| tst reg, tmp_reg +|.endmacro + // Safe memory load/store with an unsigned immediate offset. // When using Z_OFFSET(addr), which is 24-bit long, as the unsigned offset to compute a memory address, // we should firstly check the range. @@ -684,12 +698,7 @@ static void* dasm_labels[zend_lb_MAX]; // 'long_ins' should be 'and', 'orr' or 'eor' |.macro LONG_BW_OP, long_ins, reg, addr, tmp_reg1, tmp_reg2 || if (Z_MODE(addr) == IS_CONST_ZVAL) { -|| if(Z_LVAL_P(Z_ZV(addr)) >= 0 && Z_LVAL_P(Z_ZV(addr)) <= BW_X_IMM) { -| long_ins Rx(reg), Rx(reg), #(Z_LVAL_P(Z_ZV(addr))) -|| } else { -| LOAD_64BIT_VAL tmp_reg1, Z_LVAL_P(Z_ZV(addr)) -| long_ins Rx(reg), Rx(reg), tmp_reg1 -|| } +| BW_OP_64_WITH_CONST long_ins, Rx(reg), Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg1 || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | SAFE_MEM_ACC_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg2 | long_ins Rx(reg), Rx(reg), tmp_reg1 @@ -1106,38 +1115,36 @@ static void* dasm_labels[zend_lb_MAX]; | IF_NOT_TYPE tmp_reg1, val, label |.endmacro -|.macro IF_FLAGS, type_flags, mask, label -| tst type_flags, #mask +|.macro IF_FLAGS, type_flags, mask, label, tmp_reg +| TST_32_WITH_CONST type_flags, mask, tmp_reg | bne label |.endmacro -|.macro IF_NOT_FLAGS, type_flags, mask, label -| tst type_flags, #mask +|.macro IF_NOT_FLAGS, type_flags, mask, label, tmp_reg +| TST_32_WITH_CONST type_flags, mask, tmp_reg | beq label |.endmacro -|.macro IF_REFCOUNTED, type_flags, label -|| ZEND_ASSERT((IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT) <= TST_W_IMM); -| tst type_flags, #(IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT) +|.macro IF_REFCOUNTED, type_flags, label, tmp_reg +| TST_32_WITH_CONST type_flags, (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), tmp_reg | bne label |.endmacro -|.macro IF_NOT_REFCOUNTED, type_flags, label -|| ZEND_ASSERT((IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT) <= TST_W_IMM); -| tst type_flags, #(IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT) +|.macro IF_NOT_REFCOUNTED, type_flags, label, tmp_reg +| TST_32_WITH_CONST type_flags, (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), tmp_reg | beq label |.endmacro |.macro IF_ZVAL_FLAGS, addr, mask, label, tmp_reg1, tmp_reg2 || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags), tmp_reg2 -| IF_FLAGS tmp_reg1, mask, label +| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, Rw(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags), Rx(tmp_reg2) +| IF_FLAGS Rw(tmp_reg1), mask, label, Rw(tmp_reg2) |.endmacro |.macro IF_NOT_ZVAL_FLAGS, addr, mask, label, tmp_reg1, tmp_reg2 || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags), tmp_reg2 -| IF_NOT_FLAGS tmp_reg1, mask, label +| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, Rw(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags), Rx(tmp_reg2) +| IF_NOT_FLAGS Rw(tmp_reg1), mask, label, Rw(tmp_reg2) |.endmacro |.macro IF_ZVAL_REFCOUNTED, addr, label, tmp_reg1, tmp_reg2 @@ -1172,8 +1179,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro IF_GC_MAY_NOT_LEAK, ptr, label, tmp_reg1, tmp_reg2 | ldr tmp_reg1, [ptr, #4] -| LOAD_32BIT_VAL tmp_reg2, (GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT)) -| tst tmp_reg1, tmp_reg2 +| TST_32_WITH_CONST tmp_reg1, (GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT)), tmp_reg2 | bne label |.endmacro @@ -1194,7 +1200,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro TRY_ADDREF, val_info, type_flags_reg, value_ptr_reg, tmp_reg || if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { || if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { -| IF_NOT_REFCOUNTED type_flags_reg, >1 +| IF_NOT_REFCOUNTED type_flags_reg, >1, tmp_reg || } | GC_ADDREF value_ptr_reg, tmp_reg |1: @@ -1204,7 +1210,7 @@ static void* dasm_labels[zend_lb_MAX]; |.macro TRY_ADDREF_2, val_info, type_flags_reg, value_ptr_reg, tmp_reg || if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { || if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { -| IF_NOT_REFCOUNTED type_flags_reg, >1 +| IF_NOT_REFCOUNTED type_flags_reg, >1, tmp_reg || } | ldr tmp_reg, [value_ptr_reg] | add tmp_reg, tmp_reg, #2 @@ -1272,11 +1278,11 @@ static void* dasm_labels[zend_lb_MAX]; || if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // if (Z_REFCOUNTED_P(cv)) { || if (cold) { -| IF_ZVAL_REFCOUNTED addr, >1, Rw(tmp_reg1), Rx(tmp_reg2) +| IF_ZVAL_REFCOUNTED addr, >1, tmp_reg1, tmp_reg2 |.cold_code |1: || } else { -| IF_NOT_ZVAL_REFCOUNTED addr, >4, Rw(tmp_reg1), Rx(tmp_reg2) +| IF_NOT_ZVAL_REFCOUNTED addr, >4, tmp_reg1, tmp_reg2 || } || } | // if (!Z_DELREF_P(cv)) { @@ -1301,7 +1307,7 @@ static void* dasm_labels[zend_lb_MAX]; || if ((op_info) & MAY_BE_REF) { || zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, offsetof(zend_reference, val)); | IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1, Rw(tmp_reg1), Rx(tmp_reg2) -| IF_NOT_ZVAL_COLLECTABLE ref_addr, >4, Rw(tmp_reg1), Rx(tmp_reg2) +| IF_NOT_ZVAL_COLLECTABLE ref_addr, >4, tmp_reg1, tmp_reg2 | GET_ZVAL_PTR FCARG1x, ref_addr, Rx(tmp_reg2) |1: || } @@ -1355,7 +1361,7 @@ static void* dasm_labels[zend_lb_MAX]; | bls >2 || } || } -| IF_NOT_ZVAL_REFCOUNTED addr, >1, Rw(tmp_reg1), Rx(tmp_reg2) +| IF_NOT_ZVAL_REFCOUNTED addr, >1, tmp_reg1, tmp_reg2 | GC_DELREF FCARG1x, Rw(tmp_reg1) |1: | EXT_CALL zend_array_dup, REG0 @@ -1606,7 +1612,7 @@ static int zend_jit_exception_handler_undef_stub(dasm_State **Dst) |->exception_handler_undef: | MEM_LOAD_ZTS ldr, REG0, executor_globals, opline_before_exception, REG0 | ldrb TMP1w, OP:REG0->result_type - | tst TMP1w, #(IS_TMP_VAR|IS_VAR) + | TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w | bne >1 | ldr REG0w, OP:REG0->result.var | add REG0, REG0, FP @@ -1621,8 +1627,7 @@ static int zend_jit_leave_function_stub(dasm_State **Dst) { |->leave_function_handler: if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { - | LOAD_32BIT_VAL TMP1w, ZEND_CALL_TOP - | tst FCARG1w, TMP1w + | TST_32_WITH_CONST FCARG1w, ZEND_CALL_TOP, TMP1w | bne >1 | EXT_CALL zend_jit_leave_nested_func_helper, REG0 | ADD_HYBRID_SPAD @@ -1640,8 +1645,7 @@ static int zend_jit_leave_function_stub(dasm_State **Dst) | ldr LR, T4 // retore LR | add sp, sp, NR_SPAD } - | LOAD_32BIT_VAL TMP1w, ZEND_CALL_TOP - | tst FCARG1w, TMP1w + | TST_32_WITH_CONST FCARG1w, ZEND_CALL_TOP, TMP1w | bne >1 | EXT_JMP zend_jit_leave_nested_func_helper, REG0 |1: @@ -4376,9 +4380,8 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o if (packed_loaded) { | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); if (op1_info & MAY_BE_ARRAY_HASH) { - || ZEND_ASSERT(HASH_FLAG_PACKED <= TST_W_IMM); | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] - | tst TMP1w, #HASH_FLAG_PACKED + | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w | beq >4 // HASH_FIND } | // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) @@ -4856,7 +4859,7 @@ static int zend_jit_simple_assign(dasm_State **Dst, | ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, ref_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 } | beq >2 // GC_DELREF() reached zero - | IF_NOT_REFCOUNTED REG2w, >3 + | IF_NOT_REFCOUNTED REG2w, >3, TMP1w if (!res_addr) { | GC_ADDREF Rx(tmp_reg), TMP1w } else { @@ -4865,7 +4868,7 @@ static int zend_jit_simple_assign(dasm_State **Dst, | b >3 |2: if (res_addr) { - | IF_NOT_REFCOUNTED REG2w, >2 + | IF_NOT_REFCOUNTED REG2w, >2, TMP1w | GC_ADDREF Rx(tmp_reg), TMP1w |2: } @@ -5041,7 +5044,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, int in_cold = 0; if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - | IF_ZVAL_REFCOUNTED var_use_addr, >1, TMP1w, TMP2 + | IF_ZVAL_REFCOUNTED var_use_addr, >1, ZREG_TMP1, ZREG_TMP2 |.cold_code |1: in_cold = 1; @@ -5119,7 +5122,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, } } else /* if (RC_MAY_BE_N(var_info)) */ { if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - | IF_NOT_ZVAL_REFCOUNTED var_use_addr, >5, TMP1w, TMP2 + | IF_NOT_ZVAL_REFCOUNTED var_use_addr, >5, ZREG_TMP1, ZREG_TMP2 } if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { if (Z_REG(var_use_addr) == ZREG_FP) { @@ -7504,7 +7507,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { - | IF_NOT_ZVAL_REFCOUNTED op1_addr, >3, TMP1w, TMP2 + | IF_NOT_ZVAL_REFCOUNTED op1_addr, >3, ZREG_TMP1, ZREG_TMP2 } | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 | GC_DELREF FCARG1x, TMP1w @@ -7689,7 +7692,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con | LOAD_32BIT_VAL FCARG1w, used_stack | // Check whether REG0 is an internal function. | ldrb TMP1w, [REG0, #offsetof(zend_function, type)] - | tst TMP1w, #1 + | TST_32_WITH_CONST TMP1w, 1, TMP2w | bne >1 } else { | LOAD_32BIT_VAL FCARG1w, used_stack @@ -8321,8 +8324,7 @@ static int zend_jit_init_method_call(dasm_State **Dst, if (!func) { | // if (fbc->common.fn_flags & ZEND_ACC_STATIC) { | ldr TMP1w, [REG0, #offsetof(zend_function, common.fn_flags)] - || ZEND_ASSERT(ZEND_ACC_STATIC <= TST_W_IMM); - | tst TMP1w, #ZEND_ACC_STATIC + | TST_32_WITH_CONST TMP1w, ZEND_ACC_STATIC, TMP2w | bne >1 |.cold_code |1: @@ -8520,8 +8522,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } | ldr REG0, EX:RX->func | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] - || ZEND_ASSERT(ZEND_ACC_DEPRECATED <= TST_W_IMM); - | tst TMP1w, #ZEND_ACC_DEPRECATED + | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w | NIY // bne &exit_addr } } @@ -8549,8 +8550,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend if (!func) { if (!trace) { | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] - || ZEND_ASSERT(ZEND_ACC_DEPRECATED <= TST_W_IMM); - | tst TMP1w, #ZEND_ACC_DEPRECATED + | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w | bne >1 |.cold_code |1: @@ -8622,14 +8622,14 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 } else { /* the called op_array may be not persisted yet */ - | tst REG2, #1 + | TST_64_WITH_CONST REG2, 1, TMP1 | beq >1 | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 |1: } | ldr REG2, [REG2] } else { - | tst REG2, #1 + | TST_64_WITH_CONST REG2, 1, TMP1 | beq >1 | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 |1: @@ -8737,7 +8737,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | // num_args = EX_NUM_ARGS(); | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] | // if (UNEXPECTED(num_args > first_extra_arg)) - || ZEND_ASSERT(func->op_array.num_args <= TST_W_IMM); + || ZEND_ASSERT(func->op_array.num_args <= CMP_IMM); | cmp REG1w, #(func->op_array.num_args) } else { | // first_extra_arg = op_array->num_args; @@ -8764,7 +8764,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend if (!func) { | // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] - | tst TMP1w, #ZEND_ACC_HAS_TYPE_HINTS + | TST_32_WITH_CONST TMP1w, ZEND_ACC_HAS_TYPE_HINTS, TMP2w | bne >1 } | // opline += num_args; @@ -8840,14 +8840,12 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend if (!exit_addr) { return 0; } - || ZEND_ASSERT(ZEND_ACC_DEPRECATED <= TST_W_IMM); | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] - | tst TMP1w, #ZEND_ACC_DEPRECATED + | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w | NIY // bne &exit_addr } else { - || ZEND_ASSERT(ZEND_ACC_DEPRECATED <= TST_W_IMM); | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] - | tst TMP1w, #ZEND_ACC_DEPRECATED + | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w | bne >1 |.cold_code |1: @@ -8907,7 +8905,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } if (may_have_extra_named_params) { | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 3)] - | tst TMP1w, #(ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24) + | TST_32_WITH_CONST TMP1w, (ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24), TMP2w | bne >1 |.cold_code |1: @@ -8923,7 +8921,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend // TODO: optimize ??? | // if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 2)] - | tst TMP1w, #(ZEND_CALL_RELEASE_THIS >> 16) + | TST_32_WITH_CONST TMP1w, (ZEND_CALL_RELEASE_THIS >> 16), TMP2w | bne >1 |.cold_code |1: @@ -8946,7 +8944,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | // zend_vm_stack_free_call_frame(call); | ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 2)] - | tst TMP1w, #((ZEND_CALL_ALLOCATED >> 16) & 0xff) + | TST_32_WITH_CONST TMP1w, ((ZEND_CALL_ALLOCATED >> 16) & 0xff), TMP2w | bne >1 |.cold_code |1: @@ -9043,8 +9041,7 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t o } else { | ldr REG0, EX:RX->func | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] - | LOAD_32BIT_VAL TMP2w, mask - | tst TMP1w, TMP2w + | TST_32_WITH_CONST TMP1w, mask, TMP2w | bne >1 |.cold_code |1: @@ -9074,7 +9071,7 @@ static int zend_jit_check_undef_args(dasm_State **Dst, const zend_op *opline) { | ldr FCARG1x, EX->call | ldrb TMP1w, [FCARG1x, #(offsetof(zend_execute_data, This.u1.type_info) + 3)] - | tst TMP1w, #(ZEND_CALL_MAY_HAVE_UNDEF >> 24) + | TST_32_WITH_CONST TMP1w, (ZEND_CALL_MAY_HAVE_UNDEF >> 24), TMP2w | bne >1 |.cold_code |1: @@ -9203,8 +9200,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend | ldr REG0, EX:RX->func | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] - | LOAD_32BIT_VAL TMP2w, mask - | tst TMP1w, TMP2w + | TST_32_WITH_CONST TMP1w, mask, TMP2w | bne >1 |.cold_code |1: @@ -9245,8 +9241,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend | ldr REG0, EX:RX->func | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] - | LOAD_32BIT_VAL TMP2w, mask - | tst TMP1w, TMP2w + | TST_32_WITH_CONST TMP1w, mask, TMP2w | bne >1 |.cold_code |1: @@ -9260,8 +9255,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend | beq >7 } | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] - | LOAD_32BIT_VAL TMP2w, mask - | tst TMP1w, TMP2w + | TST_32_WITH_CONST TMP1w, mask, TMP2w | bne >7 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); @@ -9295,8 +9289,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend } } else { | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] - | LOAD_32BIT_VAL TMP2w, ZEND_CALL_SEND_ARG_BY_REF - | tst TMP1w, TMP2w + | TST_32_WITH_CONST TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w | bne >1 |.cold_code |1: @@ -9373,7 +9366,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 | GC_DELREF FCARG1x, TMP1w | beq >1 - | IF_NOT_REFCOUNTED REG0w, >2 + | IF_NOT_REFCOUNTED REG0w, >2, TMP1w | GC_ADDREF REG2, TMP1w | b >2 |1: @@ -9422,8 +9415,7 @@ static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) | ldr REG0, EX:RX->func | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] - | LOAD_32BIT_VAL TMP2w, mask - | tst TMP1w, TMP2w + | TST_32_WITH_CONST TMP1w, mask, TMP2w | bne >1 |.cold_code |1: @@ -9522,7 +9514,7 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, zend_uchar | ldr REG0, EX->run_time_cache | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG0, REG0, opline->extended_value, TMP1 | cbz REG0, >1 - | tst REG0, #1 + | TST_64_WITH_CONST REG0, 1, TMP1 | bne >4 |.cold_code |4: @@ -9693,7 +9685,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // if (Z_REFCOUNTED_P(cv)) { - | IF_ZVAL_REFCOUNTED op1_addr, >1, TMP1w, TMP2 + | IF_ZVAL_REFCOUNTED op1_addr, >1, ZREG_TMP1, ZREG_TMP2 |.cold_code |1: } @@ -9740,8 +9732,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t } | mov REG0w, #1 | lsl REG0w, REG0w, REG1w - | LOAD_32BIT_VAL TMP1w, mask - | tst REG0w, TMP1w + | TST_32_WITH_CONST REG0w, mask, TMP1w if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { | NIY // bne &exit_addr @@ -9773,7 +9764,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // if (Z_REFCOUNTED_P(cv)) { - | IF_ZVAL_REFCOUNTED op1_addr, >1, TMP1w, TMP2 + | IF_ZVAL_REFCOUNTED op1_addr, >1, ZREG_TMP1, ZREG_TMP2 |.cold_code |1: } @@ -9987,8 +9978,7 @@ static int zend_jit_leave_func(dasm_State **Dst, } /* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */ - | LOAD_32BIT_VAL TMP1w, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE) - | tst FCARG1w, TMP1w + | TST_32_WITH_CONST FCARG1w, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE), TMP1w if (trace && trace->op != ZEND_JIT_TRACE_END) { | NIY // TODO: test } else { @@ -10016,8 +10006,7 @@ static int zend_jit_leave_func(dasm_State **Dst, } } | // if (call_info & ZEND_CALL_RELEASE_THIS) - | LOAD_32BIT_VAL TMP1w, ZEND_CALL_RELEASE_THIS - | tst FCARG1w, TMP1w + | TST_32_WITH_CONST FCARG1w, ZEND_CALL_RELEASE_THIS, TMP1w | beq >4 | // zend_object *object = Z_OBJ(execute_data->This); | ldr FCARG1x, EX->This.value.obj @@ -10170,9 +10159,9 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o if (return_value_used != 1) { if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if (jit_return_label >= 0) { - | IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label, TMP1w, TMP2 + | IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label, ZREG_TMP1, ZREG_TMP2 } else { - | IF_NOT_ZVAL_REFCOUNTED op1_addr, >9, TMP1w, TMP2 + | IF_NOT_ZVAL_REFCOUNTED op1_addr, >9, ZREG_TMP1, ZREG_TMP2 } } | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 @@ -10251,9 +10240,9 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o | beq >2 | // if (IS_REFCOUNTED()) if (jit_return_label >= 0) { - | IF_NOT_REFCOUNTED REG2w, =>jit_return_label + | IF_NOT_REFCOUNTED REG2w, =>jit_return_label, TMP1w } else { - | IF_NOT_REFCOUNTED REG2w, >9 + | IF_NOT_REFCOUNTED REG2w, >9, TMP1w } | // ADDREF | GET_ZVAL_PTR REG2, ret_addr, TMP1 // reload @@ -10285,13 +10274,13 @@ static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, ze ZEND_ASSERT(type_reg == ZREG_REG2); | GET_ZVAL_PTR REG1, val_addr, TMP1 - | IF_NOT_REFCOUNTED REG2w, >2 + | IF_NOT_REFCOUNTED REG2w, >2, TMP1w | and TMP2w, REG2w, #0xff // TMP2w -> low 8 bits of REG2w | IF_NOT_TYPE TMP2w, IS_REFERENCE, >1 | add REG1, REG1, #offsetof(zend_reference, val) | GET_Z_TYPE_INFO REG2w, REG1 | GET_Z_PTR REG1, REG1 - | IF_NOT_REFCOUNTED REG2w, >2 + | IF_NOT_REFCOUNTED REG2w, >2, TMP1w |1: | GC_ADDREF REG1, TMP2w |2: @@ -11007,7 +10996,7 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_ if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { | // if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr))) - | IF_ZVAL_REFCOUNTED op1_addr, >2, TMP1w, TMP2 + | IF_ZVAL_REFCOUNTED op1_addr, >2, ZREG_TMP1, ZREG_TMP2 |.cold_code |2: } @@ -11087,8 +11076,7 @@ static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zen | mov REG2w, #1 | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, REG1w, Rx(Z_REG(res_addr)), Z_OFFSET(res_addr)+offsetof(zval, u1.v.type), TMP1 | lsl REG2w, REG2w, REG1w - | LOAD_32BIT_VAL TMP1w, type_mask - | tst REG2w, TMP1w + | TST_32_WITH_CONST REG2w, type_mask, TMP1w | beq >1 } @@ -11710,7 +11698,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, && (op1_info & MAY_BE_RC1)) { zend_jit_addr orig_op1_addr = OP1_ADDR(); - | IF_NOT_ZVAL_REFCOUNTED orig_op1_addr, >1, TMP1w, TMP2 + | IF_NOT_ZVAL_REFCOUNTED orig_op1_addr, >1, ZREG_TMP1, ZREG_TMP2 | GET_ZVAL_PTR FCARG1x, orig_op1_addr, TMP1 | GC_DELREF FCARG1x, TMP1w | bne >1 @@ -13215,8 +13203,7 @@ static bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, | mov REG2w, #1 | GET_ZVAL_TYPE REG1w, op1_addr, TMP1 | lsl REG2w, REG2w, REG1w - | LOAD_32BIT_VAL TMP1w, type_mask - | tst REG2w, TMP1w + | TST_32_WITH_CONST REG2w, type_mask, TMP1w | beq >7 } } @@ -13413,8 +13400,7 @@ static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t o | ldr REG0, [FCARG2x, #offsetof(Bucket, key)] | SET_ZVAL_PTR res_addr, REG0, TMP1 | ldr TMP1w, [REG0, #offsetof(zend_refcounted, gc.u.type_info)] - || ZEND_ASSERT(IS_STR_INTERNED <= TST_W_IMM); - | tst TMP1w, #IS_STR_INTERNED + | TST_32_WITH_CONST TMP1w, IS_STR_INTERNED, TMP2w | beq >1 | SET_ZVAL_TYPE_INFO res_addr, IS_STRING, TMP1w, TMP2 | b >3 @@ -13480,7 +13466,7 @@ static int zend_jit_fetch_constant(dasm_State **Dst, | // if (c != NULL) | cbz REG0, >9 | // if (!IS_SPECIAL_CACHE_VAL(c)) - | tst REG0, #CACHE_SPECIAL + | TST_64_WITH_CONST REG0, CACHE_SPECIAL, TMP1 | bne >9 |8: From 4039c5865a876ff9dd9902d81621c6d9b7a2578e Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Mon, 26 Apr 2021 00:27:40 +0000 Subject: [PATCH 151/229] Add the helper to check whether an immediate is valid for logical instructions We implement a simplified version. Every value with one single bit can be encoded for logical instructions and we suppose this quick check can cover a lot of common masks. Besides, add macro TST_64_WITH_ONE since it's used often to test 64-bit register with constant 1. Change-Id: I850a6ac6acbe2d12f85180e407344580ee6fea61 --- ext/opcache/jit/zend_jit_arm64.dasc | 57 +++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index e470ebb6be68a..99d10ed351928 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -168,6 +168,18 @@ static void* dasm_labels[zend_lb_MAX]; #define BP_JIT_IS 6 +/* helper: determine whether an immediate value can be encoded as the immediate operand of logical instructions */ +static int logical_immediate_p (uint64_t value, uint32_t reg_size) +{ + /* fast path: power of two */ + if (value > 0 && !(value & (value - 1))) { + return 1; + } + + // TODO: slow path + return 0; +} + /* Not Implemented Yet */ |.macro NIY || //ZEND_ASSERT(0); @@ -219,23 +231,45 @@ static void* dasm_labels[zend_lb_MAX]; || } |.endmacro -// TODO: Support logical instruction with an immediate encoded. // Bitwise operation with constants. 'ins' can be and/orr/eor/ands. Operands are 64-bit. |.macro BW_OP_64_WITH_CONST, ins, reg, op, val, tmp_reg -| LOAD_64BIT_VAL tmp_reg, val -| ins reg, op, tmp_reg +|| if (val == 0) { +| ins reg, op, xzr +|| } else if (logical_immediate_p(val, 64)) { +| ins reg, op, #val +|| } else { +| LOAD_64BIT_VAL tmp_reg, val +| ins reg, op, tmp_reg +|| } |.endmacro // Test operation 'tst' with constants. Operands are 32-bit. |.macro TST_32_WITH_CONST, reg, val, tmp_reg -| LOAD_32BIT_VAL tmp_reg, val -| tst reg, tmp_reg +|| if (val == 0) { +| tst reg, wzr +|| } else if (logical_immediate_p((uint32_t)val, 32)) { +| tst reg, #val +|| } else { +| LOAD_32BIT_VAL tmp_reg, val +| tst reg, tmp_reg +|| } |.endmacro // Test operation 'tst' with constants. Operands are 64-bit. |.macro TST_64_WITH_CONST, reg, val, tmp_reg -| LOAD_64BIT_VAL tmp_reg, val -| tst reg, tmp_reg +|| if (val == 0) { +| tst reg, xzr +|| } else if (logical_immediate_p(val, 64)) { +| tst reg, #val +|| } else { +| LOAD_64BIT_VAL tmp_reg, val +| tst reg, tmp_reg +|| } +|.endmacro + +// Note: 1 is a valid immediate for logical instruction. +|.macro TST_64_WITH_ONE, reg +| tst reg, #1 |.endmacro // Safe memory load/store with an unsigned immediate offset. @@ -8622,14 +8656,14 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 } else { /* the called op_array may be not persisted yet */ - | TST_64_WITH_CONST REG2, 1, TMP1 + | TST_64_WITH_ONE REG2 | beq >1 | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 |1: } | ldr REG2, [REG2] } else { - | TST_64_WITH_CONST REG2, 1, TMP1 + | TST_64_WITH_ONE REG2 | beq >1 | MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1 |1: @@ -9514,7 +9548,7 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, zend_uchar | ldr REG0, EX->run_time_cache | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG0, REG0, opline->extended_value, TMP1 | cbz REG0, >1 - | TST_64_WITH_CONST REG0, 1, TMP1 + | TST_64_WITH_ONE REG0 | bne >4 |.cold_code |4: @@ -13466,7 +13500,8 @@ static int zend_jit_fetch_constant(dasm_State **Dst, | // if (c != NULL) | cbz REG0, >9 | // if (!IS_SPECIAL_CACHE_VAL(c)) - | TST_64_WITH_CONST REG0, CACHE_SPECIAL, TMP1 + || ZEND_ASSERT(CACHE_SPECIAL == 1); + | TST_64_WITH_ONE REG0 | bne >9 |8: From 018fca92ad7428d09d6c6f18ef513994bbf6905c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 26 Apr 2021 12:55:21 +0300 Subject: [PATCH 152/229] Added missed UNDEF_OPLINE_RESULT --- ext/opcache/jit/zend_jit_arm64.dasc | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 99d10ed351928..a774f5ae4137c 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -1913,6 +1913,7 @@ static int zend_jit_mod_by_zero_stub(dasm_State **Dst) static int zend_jit_invalid_this_stub(dasm_State **Dst) { |->invalid_this: + | UNDEF_OPLINE_RESULT TMP1w | mov CARG1, xzr | LOAD_ADDR CARG2, "Using $this when not in object context" | EXT_CALL zend_throw_error, REG0 From 91aaea362fdc2d91b37994a799fcc0fc3c84e0e7 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 26 Apr 2021 12:58:50 +0300 Subject: [PATCH 153/229] typo --- ext/opcache/jit/zend_jit_arm64.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index a774f5ae4137c..11504e45f6c19 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -11911,7 +11911,7 @@ static int zend_jit_incdec_obj(dasm_State **Dst, (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] - | ldr REG0, [REG1, #offsetof(zend_class_entry, properties_info_table)] + | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] | LOAD_32BIT_VAL TMP1, prop_info_offset | ldr FCARG2x, [REG0, TMP1] } From 434eb1a3d97678f13d9ec256253b9711d364d1d7 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 26 Apr 2021 13:01:37 +0300 Subject: [PATCH 154/229] Duplicate return --- ext/opcache/jit/zend_jit_arm64.dasc | 2 -- 1 file changed, 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 11504e45f6c19..68e461574915c 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -1608,8 +1608,6 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) } return 1; - - return 1; } static int zend_jit_exception_handler_stub(dasm_State **Dst) From add2544e1cfdc55a288efa9a2d1e501cb7eb7b2d Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 26 Apr 2021 13:10:08 +0300 Subject: [PATCH 155/229] Missed instruction --- ext/opcache/jit/zend_jit_arm64.dasc | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 68e461574915c..188f3ff323b29 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -2329,6 +2329,7 @@ static int zend_jit_assign_var_stub(dasm_State **Dst) |->assign_var: | stp x29, x30, [sp,#-16]! + | mov x29, sp | sub sp, sp, #16 if (!zend_jit_assign_to_variable( Dst, NULL, From 39e80971c771f7d8e4077bd19283efa7f0297e2d Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 26 Apr 2021 13:28:53 +0300 Subject: [PATCH 156/229] Make bit helpers to be inline --- ext/opcache/jit/zend_jit_internal.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index bba49fb3cbfa6..2f9e532e49f4a 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -740,19 +740,19 @@ static zend_always_inline bool zend_jit_may_be_polymorphic_call(const zend_op *o /* bit helpers */ -static bool zend_long_is_power_of_two(zend_long x) +static zend_always_inline bool zend_long_is_power_of_two(zend_long x) { return (x > 0) && !(x & (x - 1)); } -static uint32_t zend_long_floor_log2(uint64_t x) +static zend_always_inline uint32_t zend_long_floor_log2(uint64_t x) { ZEND_ASSERT(zend_long_is_power_of_two(x)); return __builtin_ctzll(x); } /* from http://aggregate.org/MAGIC/ */ -static uint32_t ones32(uint32_t x) +static zend_always_inline uint32_t ones32(uint32_t x) { x -= ((x >> 1) & 0x55555555); x = (((x >> 2) & 0x33333333) + (x & 0x33333333)); @@ -762,7 +762,7 @@ static uint32_t ones32(uint32_t x) return x & 0x0000003f; } -static uint32_t floor_log2(uint32_t x) +static zend_always_inline uint32_t floor_log2(uint32_t x) { ZEND_ASSERT(x != 0); x |= (x >> 1); @@ -773,22 +773,22 @@ static uint32_t floor_log2(uint32_t x) return ones32(x) - 1; } -static bool is_power_of_two(uint32_t x) +static zend_always_inline bool is_power_of_two(uint32_t x) { return !(x & (x - 1)) && x != 0; } -static bool has_concrete_type(uint32_t value_type) +static zend_always_inline bool has_concrete_type(uint32_t value_type) { return is_power_of_two (value_type & (MAY_BE_ANY|MAY_BE_UNDEF)); } -static uint32_t concrete_type(uint32_t value_type) +static zend_always_inline uint32_t concrete_type(uint32_t value_type) { return floor_log2(value_type & (MAY_BE_ANY|MAY_BE_UNDEF)); } -static inline bool is_signed(double d) +static zend_always_inline bool is_signed(double d) { return (((unsigned char*)&d)[sizeof(double)-1] & 0x80) != 0; } From 57f2fe44c62655eb3207efa395982f275f9a45f3 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 26 Apr 2021 17:31:14 +0300 Subject: [PATCH 157/229] Use "red zone" for HYBRID VM. Support for CALL VM and VM without global register variables. --- Zend/zend_vm_gen.php | 2 +- Zend/zend_vm_opcodes.h | 3 +- ext/opcache/jit/zend_jit_arm64.dasc | 148 ++++++++++++++-------------- 3 files changed, 75 insertions(+), 78 deletions(-) diff --git a/Zend/zend_vm_gen.php b/Zend/zend_vm_gen.php index 973ae380ef206..298cf132db21e 100755 --- a/Zend/zend_vm_gen.php +++ b/Zend/zend_vm_gen.php @@ -2361,7 +2361,7 @@ function gen_vm_opcodes_header( } $str .= "\n"; $str .= "#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) && !defined(__SANITIZE_ADDRESS__)\n"; - $str .= "# if ((defined(i386) && !defined(__PIC__)) || defined(__x86_64__) || defined(_M_X64))\n"; + $str .= "# if ((defined(i386) && !defined(__PIC__)) || defined(__x86_64__) || defined(_M_X64) || defined (__aarch64__))\n"; $str .= "# define ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 16\n"; $str .= "# endif\n"; $str .= "#endif\n"; diff --git a/Zend/zend_vm_opcodes.h b/Zend/zend_vm_opcodes.h index d6cc6e28edb14..8defbf7018a8d 100644 --- a/Zend/zend_vm_opcodes.h +++ b/Zend/zend_vm_opcodes.h @@ -35,8 +35,7 @@ #endif #if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) && !defined(__SANITIZE_ADDRESS__) -# if ((defined(i386) && !defined(__PIC__)) || defined(__x86_64__) || \ - defined(_M_X64) || defined(__aarch64__)) +# if ((defined(i386) && !defined(__PIC__)) || defined(__x86_64__) || defined(_M_X64) || defined (__aarch64__)) # define ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 16 # endif #endif diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 188f3ff323b29..a15599c899326 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -43,16 +43,11 @@ |.define FCARG1w, w0 |.define FCARG2x, x1 |.define FCARG2w, w1 -|.define SPAD, #0x10 // padding for CPU stack alignment +|.define SPAD, #0x20 // padding for CPU stack alignment |.define NR_SPAD, #0x30 // padding for CPU stack alignment -|.define T4, [sp, #0x20] // Used to store old value of LR (CALL VM only) -|.define T3, [sp, #0x18] // Used to store old value of IP (CALL VM only) -|.define T2, [sp, #0x10] // Used to store old value of FP (CALL VM only) -|.define T1, [sp] -|.define A4, [r4+0xC] // preallocated slots for arguments of "cdecl" functions (intersect with T1) -|.define A3, [r4+0x8] -|.define A2, [r4+0x4] -|.define A1, [r4] +|.define T3, [sp, #0x28] // Used to store old value of IP (CALL VM only) +|.define T2, [sp, #0x20] // Used to store old value of FP (CALL VM only) +|.define T1, [sp, #0x10] // We use REG0/1/2 and FPR0/1 to replace r0/1/2 and xmm0/1 in the x86 implementation. // Scratch registers @@ -93,7 +88,9 @@ |.define HYBRID_SPAD, #16 // padding for stack alignment -#define TMP_ZVAL_OFFSET 0 +#define SPAD 0x20 +#define NR_SPAD 0x30 +#define TMP_ZVAL_OFFSET 16 #define DASM_ALIGNMENT 16 /* Encoding of immediate. TODO: shift mode may be supported in the near future. */ @@ -191,16 +188,16 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | brk #0 |.endmacro -/* In x86/64, HYBRID_SPAD bytes are reserved on the stack only if flag ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE - * is not defined, because the 16-byte redzone, allocated on the stack when the flag is defined, can be - * reused. In AArch64, it's safe that these bytes are always reserved because the stack layout might - * change along software evolution, making the redzone not reusable any longer. */ |.macro ADD_HYBRID_SPAD +||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE | add sp, sp, HYBRID_SPAD +||#endif |.endmacro |.macro SUB_HYBRID_SPAD +||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE | sub sp, sp, HYBRID_SPAD +||#endif |.endmacro |.macro LOAD_ADDR, reg, addr @@ -1601,10 +1598,13 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) | ADD_HYBRID_SPAD | JMP_IP TMP1 } else if (GCC_GLOBAL_REGS) { - | add sp, sp, SPAD // stack alignment + | ldp x29, x30, [sp], #SPAD // stack alignment | JMP_IP TMP1 } else { - | NIY_STUB // TODO + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], #NR_SPAD // stack alignment + | mov RETVALx, #1 // ZEND_VM_ENTER + | ret } return 1; @@ -1623,15 +1623,14 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) const void *handler = EG(exception_op)->handler; if (GCC_GLOBAL_REGS) { - | add sp, sp, SPAD // stack alignment + | ldp x29, x30, [sp], #SPAD // stack alignment | EXT_JMP handler, REG0 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { | NIY_STUB // TODO: tracing } else { | mov FCARG1x, FP - | ldp FP, RX, T2 // retore FP and IP - | ldr LR, T4 // retore LR - | add sp, sp, NR_SPAD // stack alignment + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], #NR_SPAD // stack alignment | EXT_JMP handler, REG0 } } @@ -1670,12 +1669,11 @@ static int zend_jit_leave_function_stub(dasm_State **Dst) | JMP_IP TMP1 } else { if (GCC_GLOBAL_REGS) { - | add sp, sp, SPAD + | ldp x29, x30, [sp], #SPAD // stack alignment } else { | mov FCARG2x, FP - | ldp FP, RX, T2 // retore FP and IP - | ldr LR, T4 // retore LR - | add sp, sp, NR_SPAD + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], #NR_SPAD // stack alignment } | TST_32_WITH_CONST FCARG1w, ZEND_CALL_TOP, TMP1w | bne >1 @@ -1703,7 +1701,19 @@ static int zend_jit_leave_throw_stub(dasm_State **Dst) | // HANDLE_EXCEPTION() | b ->exception_handler } else { - | NIY_STUB // TODO + | GET_IP TMP1 + | ldrb TMP1w, OP:TMP1->opcode + | cmp TMP1w, #ZEND_HANDLE_EXCEPTION + | beq >5 + | // EG(opline_before_exception) = opline; + | MEM_STORE_ZTS str, IP, executor_globals, opline_before_exception, TMP2 + |5: + | // opline = EG(exception_op); + | LOAD_IP_ADDR_ZTS executor_globals, exception_op + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], #NR_SPAD // stack alignment + | mov RETVALx, #2 // ZEND_VM_LEAVE + | ret } return 1; @@ -2159,12 +2169,11 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst) | ADD_HYBRID_SPAD | JMP_IP TMP1 } else if (GCC_GLOBAL_REGS) { - | add sp, sp, SPAD // stack alignment + | ldp x29, x30, [sp], #SPAD // stack alignment | JMP_IP TMP1 } else { - | ldp FP, RX, T2 // retore FP and IP - | ldr LR, T4 // retore LR - | add sp, sp, NR_SPAD // stack alignment + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], #NR_SPAD // stack alignment | mov RETVALx, #1 // ZEND_VM_ENTER | ret } @@ -2190,7 +2199,7 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst) | ldr REG0, [REG0] | br REG0 } else if (GCC_GLOBAL_REGS) { - | add sp, sp, SPAD // stack alignment + | ldp x29, x30, [sp], #SPAD // stack alignment | ldr REG0, EX->func | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] @@ -2210,10 +2219,9 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst) | tst RETVALw, RETVALw | blt ->trace_halt | - | ldp FP, RX, T2 // retore FP and IP - | ldr LR, T4 // retore LR - | add sp, sp, NR_SPAD // stack alignment - | mov RETVALx, #1 // ZEND_VM_ENTER + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], #NR_SPAD // stack alignment + | mov RETVALx, #1 // ZEND_VM_ENTER | ret } @@ -2282,9 +2290,8 @@ static int zend_jit_assign_const_stub(dasm_State **Dst) uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; |->assign_const: - | stp x29, x30, [sp,#-16]! + | stp x29, x30, [sp,#-32]! | mov x29, sp - | sub sp, sp, #16 if (!zend_jit_assign_to_variable( Dst, NULL, var_addr, var_addr, -1, -1, @@ -2292,8 +2299,7 @@ static int zend_jit_assign_const_stub(dasm_State **Dst) 0, 0)) { return 0; } - | mov sp, x29 - | ldp x29, x30, [sp],#16 + | ldp x29, x30, [sp],#32 | ret return 1; } @@ -2305,9 +2311,8 @@ static int zend_jit_assign_tmp_stub(dasm_State **Dst) uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; |->assign_tmp: - | stp x29, x30, [sp,#-16]! + | stp x29, x30, [sp,#-32]! | mov x29, sp - | sub sp, sp, #16 if (!zend_jit_assign_to_variable( Dst, NULL, var_addr, var_addr, -1, -1, @@ -2315,8 +2320,7 @@ static int zend_jit_assign_tmp_stub(dasm_State **Dst) 0, 0)) { return 0; } - | mov sp, x29 - | ldp x29, x30, [sp],#16 + | ldp x29, x30, [sp],#32 | ret return 1; } @@ -2328,9 +2332,8 @@ static int zend_jit_assign_var_stub(dasm_State **Dst) uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF; |->assign_var: - | stp x29, x30, [sp,#-16]! + | stp x29, x30, [sp,#-32]! | mov x29, sp - | sub sp, sp, #16 if (!zend_jit_assign_to_variable( Dst, NULL, var_addr, var_addr, -1, -1, @@ -2338,8 +2341,7 @@ static int zend_jit_assign_var_stub(dasm_State **Dst) 0, 0)) { return 0; } - | mov sp, x29 - | ldp x29, x30, [sp],#16 + | ldp x29, x30, [sp],#32 | ret return 1; } @@ -2351,9 +2353,8 @@ static int zend_jit_assign_cv_noref_stub(dasm_State **Dst) uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/; |->assign_cv_noref: - | stp x29, x30, [sp,#-16]! + | stp x29, x30, [sp,#-32]! | mov x29, sp - | sub sp, sp, #16 if (!zend_jit_assign_to_variable( Dst, NULL, var_addr, var_addr, -1, -1, @@ -2361,8 +2362,7 @@ static int zend_jit_assign_cv_noref_stub(dasm_State **Dst) 0, 0)) { return 0; } - | mov sp, x29 - | ldp x29, x30, [sp],#16 + | ldp x29, x30, [sp],#32 | ret return 1; } @@ -2374,9 +2374,8 @@ static int zend_jit_assign_cv_stub(dasm_State **Dst) uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/; |->assign_cv: - | stp x29, x30, [sp,#-16]! + | stp x29, x30, [sp,#-32]! | mov x29, sp - | sub sp, sp, #16 if (!zend_jit_assign_to_variable( Dst, NULL, var_addr, var_addr, -1, -1, @@ -2384,8 +2383,7 @@ static int zend_jit_assign_cv_stub(dasm_State **Dst) 0, 0)) { return 0; } - | mov sp, x29 - | ldp x29, x30, [sp],#16 + | ldp x29, x30, [sp],#32 | ret return 1; } @@ -2574,11 +2572,12 @@ static int zend_jit_prologue(dasm_State **Dst) if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | SUB_HYBRID_SPAD } else if (GCC_GLOBAL_REGS) { - | sub sp, sp, SPAD // stack alignment + | sub sp, sp, SPAD // TODO: stp x29, x30, [sp, #-SPAD]! can't be compiled + | stp x29, x30, [sp] // stack alignment } else { - | sub sp, sp, NR_SPAD // stack alignment + | sub sp, sp, NR_SPAD // TODO: stp x29, x30, [sp, #-NR_SPAD]! can't be compiled + | stp x29, x30, [sp] // stack alignment | stp FP, RX, T2 // save FP and IP - | str LR, T4 // save LR | mov FP, FCARG1x } return 1; @@ -2781,7 +2780,7 @@ static int zend_jit_trace_return(dasm_State **Dst, bool original_handler) | br REG0 } } else if (GCC_GLOBAL_REGS) { - | add sp, sp, SPAD // stack alignment + | ldp x29, x30, [sp], #SPAD // stack alignment if (!original_handler) { | JMP_IP TMP1 } else { @@ -2804,10 +2803,9 @@ static int zend_jit_trace_return(dasm_State **Dst, bool original_handler) | ldr REG0, [REG0] | blr REG0 } - | ldp FP, RX, T2 // retore FP and IP - | ldr LR, T4 // retore LR - | add sp, sp, NR_SPAD // stack alignment - | mov RETVALx, #2 // ZEND_VM_LEAVE + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], #NR_SPAD // stack alignment + | mov RETVALx, #2 // ZEND_VM_LEAVE | ret } return 1; @@ -2936,12 +2934,11 @@ static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) const void *handler = opline->handler; if (GCC_GLOBAL_REGS) { - | add sp, sp, SPAD // stack alignment + | ldp x29, x30, [sp], #SPAD // stack alignment } else { | mov FCARG1x, FP - | ldp FP, RX, T2 // retore FP and IP - | ldr LR, T4 // retore LR - | add sp, sp, NR_SPAD // stack alignment + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], #NR_SPAD // stack alignment } | EXT_JMP handler, REG0 } @@ -8314,6 +8311,9 @@ static int zend_jit_init_method_call(dasm_State **Dst, |1: | LOAD_ADDR FCARG2x, function_name | mov CARG3, sp + if (TMP_ZVAL_OFFSET != 0) { + | add CARG3, CARG3, #TMP_ZVAL_OFFSET + } | SET_EX_OPLINE opline, REG0 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) { | EXT_CALL zend_jit_find_method_tmp_helper, REG0 @@ -8847,12 +8847,11 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | ADD_HYBRID_SPAD | JMP_IP TMP1 } else if (GCC_GLOBAL_REGS) { - | add sp, sp, SPAD // stack alignment + | ldp x29, x30, [sp], #SPAD // stack alignment | JMP_IP TMP1 } else { - | ldp FP, RX, T2 // retore FP and IP - | ldr LR, T4 // retore LR - | add sp, sp, NR_SPAD // stack alignment + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], #NR_SPAD // stack alignment | mov RETVALx, #1 // ZEND_VM_ENTER | ret } @@ -10106,7 +10105,7 @@ static int zend_jit_leave_func(dasm_State **Dst, | JMP_IP TMP1 #endif } else if (GCC_GLOBAL_REGS) { - | add sp, sp, SPAD // stack alignment + | ldp x29, x30, [sp], #SPAD // stack alignment #ifdef CONTEXT_THREADED_JIT | NIY // TODO #else @@ -10119,10 +10118,9 @@ static int zend_jit_leave_func(dasm_State **Dst, // the value of execute_data in execute_ex() | NIY // TODO #else - | ldp FP, RX, T2 // restore FP and IP - | ldr LR, T4 // restore LR - | add sp, sp, NR_SPAD // stack alignment - | mov RETVALx, #2 // ZEND_VM_LEAVE ???? + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], #NR_SPAD // stack alignment + | mov RETVALx, #2 // ZEND_VM_LEAVE ???? | ret #endif } From 997374f41cf42db7303d95c55ec79a08182cfc20 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 26 Apr 2021 22:34:02 +0300 Subject: [PATCH 158/229] Disable "red zone" usage (it leads to crashes). --- Zend/zend_vm_gen.php | 2 +- Zend/zend_vm_opcodes.h | 2 +- ext/opcache/jit/zend_jit_arm64.dasc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Zend/zend_vm_gen.php b/Zend/zend_vm_gen.php index 298cf132db21e..973ae380ef206 100755 --- a/Zend/zend_vm_gen.php +++ b/Zend/zend_vm_gen.php @@ -2361,7 +2361,7 @@ function gen_vm_opcodes_header( } $str .= "\n"; $str .= "#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) && !defined(__SANITIZE_ADDRESS__)\n"; - $str .= "# if ((defined(i386) && !defined(__PIC__)) || defined(__x86_64__) || defined(_M_X64) || defined (__aarch64__))\n"; + $str .= "# if ((defined(i386) && !defined(__PIC__)) || defined(__x86_64__) || defined(_M_X64))\n"; $str .= "# define ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 16\n"; $str .= "# endif\n"; $str .= "#endif\n"; diff --git a/Zend/zend_vm_opcodes.h b/Zend/zend_vm_opcodes.h index 8defbf7018a8d..94e74c0a57f12 100644 --- a/Zend/zend_vm_opcodes.h +++ b/Zend/zend_vm_opcodes.h @@ -35,7 +35,7 @@ #endif #if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) && !defined(__SANITIZE_ADDRESS__) -# if ((defined(i386) && !defined(__PIC__)) || defined(__x86_64__) || defined(_M_X64) || defined (__aarch64__)) +# if ((defined(i386) && !defined(__PIC__)) || defined(__x86_64__) || defined(_M_X64)) # define ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 16 # endif #endif diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index a15599c899326..14397939bf3dc 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -86,7 +86,7 @@ |.define ZREG_TMP4, ZREG_X14 |.define ZREG_FPTMP, ZREG_V16 -|.define HYBRID_SPAD, #16 // padding for stack alignment +|.define HYBRID_SPAD, #32 // padding for stack alignment #define SPAD 0x20 #define NR_SPAD 0x30 From 735e4ccf5e9726eb5611eef591ed3bcfaf9148c4 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 26 Apr 2021 22:36:02 +0300 Subject: [PATCH 159/229] Support for ZTS --- TSRM/TSRM.c | 8 ++ ext/opcache/jit/zend_jit_arm64.dasc | 114 +++++++++++++++++++--------- 2 files changed, 88 insertions(+), 34 deletions(-) diff --git a/TSRM/TSRM.c b/TSRM/TSRM.c index 2d60b6a9d6eee..a39564b8930d0 100644 --- a/TSRM/TSRM.c +++ b/TSRM/TSRM.c @@ -741,6 +741,14 @@ TSRM_API size_t tsrm_get_ls_cache_tcb_offset(void) asm ("leal _tsrm_ls_cache@ntpoff,%0" : "=r" (ret)); return ret; +#elif defined(__aarch64__) + size_t ret; + + asm("mov %0, xzr\n\t" + "add %0, %0, #:tprel_hi12:_tsrm_ls_cache, lsl #12\n\t" + "add %0, %0, #:tprel_lo12_nc:_tsrm_ls_cache" + : "=r" (ret)); + return ret; #else return 0; #endif diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 14397939bf3dc..3255017f87cf7 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -137,8 +137,6 @@ const char* zend_reg_name[] = { #if ZTS static size_t tsrm_ls_cache_tcb_offset = 0; -static size_t tsrm_tls_index; -static size_t tsrm_tls_offset; #endif /* By default avoid JITing inline handlers if it does not seem profitable due to lack of @@ -291,14 +289,21 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.endmacro |.macro LOAD_TSRM_CACHE, reg -| NIY // TODO +| //.byte 0x4d, 0xd0, 0x3b, 0xd5 // TODO: hard-coded: mrs TMP3, tpidr_el0 +| .long 0xd53bd04d // TODO: hard-coded: mrs TMP3, tpidr_el0 +|| ZEND_ASSERT(tsrm_ls_cache_tcb_offset <= ADD_SUB_IMM); +| ldr reg, [TMP3, #tsrm_ls_cache_tcb_offset] |.endmacro |.macro LOAD_ADDR_ZTS, reg, struct, field | .if ZTS -| NIY // TODO -| LOAD_TSRM_CACHE reg -| add reg, reg, #(struct.._offset + offsetof(zend_..struct, field)) +| LOAD_TSRM_CACHE TMP3 +|| if (((uintptr_t)(struct.._offset+offsetof(zend_..struct, field))) > ADD_SUB_IMM) { +| LOAD_32BIT_VAL reg, (struct.._offset + offsetof(zend_..struct, field)) +| add reg, reg, TMP3 +|| } else { +| add reg, TMP3, #(struct.._offset + offsetof(zend_..struct, field)) +|| } | .else | LOAD_ADDR reg, &struct.field | .endif @@ -337,9 +342,17 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.macro MEM_STORE_ZTS, str_ins, op, struct, field, tmp_reg | .if ZTS -| NIY // TODO: test -| LOAD_TSRM_CACHE tmp_reg -| str_ins op, [tmp_reg, #(struct.._offset+offsetof(zend_..struct, field))] +| LOAD_TSRM_CACHE TMP3 +| SAFE_MEM_ACC_WITH_UOFFSET str_ins, op, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg +| .else +| MEM_STORE str_ins, op, &struct.field, tmp_reg +| .endif +|.endmacro + +|.macro MEM_STORE_ZTS_BYTE, str_ins, op, struct, field, tmp_reg +| .if ZTS +| LOAD_TSRM_CACHE TMP3 +| SAFE_MEM_ACC_WITH_UOFFSET_BYTE str_ins, op, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg | .else | MEM_STORE str_ins, op, &struct.field, tmp_reg | .endif @@ -353,9 +366,17 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.macro MEM_LOAD_ZTS, ldr_ins, op, struct, field, tmp_reg | .if ZTS -| NIY // TODO: test -| LOAD_TSRM_CACHE tmp_reg -| ldr_ins op, [tmp_reg, #(struct.._offset+offsetof(zend_..struct, field))] +| LOAD_TSRM_CACHE TMP3 +| SAFE_MEM_ACC_WITH_UOFFSET ldr_ins, op, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg +| .else +| MEM_LOAD ldr_ins, op, &struct.field, tmp_reg +| .endif +|.endmacro + +|.macro MEM_LOAD_ZTS_BYTE, ldr_ins, op, struct, field, tmp_reg +| .if ZTS +| LOAD_TSRM_CACHE TMP3 +| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldr_ins, op, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg | .else | MEM_LOAD ldr_ins, op, &struct.field, tmp_reg | .endif @@ -371,9 +392,8 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.macro MEM_LOAD_OP_ZTS, mem_ins, ldr_ins, op, struct, field, tmp_reg1, tmp_reg2 | .if ZTS -| NIY // TODO: test -| LOAD_TSRM_CACHE tmp_reg1 -| ldr_ins tmp_reg2, [tmp_reg1, #(struct.._offset+offsetof(zend_..struct, field))] +| LOAD_TSRM_CACHE TMP3 +| SAFE_MEM_ACC_WITH_UOFFSET ldr_ins, tmp_reg2, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg1 | mem_ins op, op, tmp_reg2 | .else | MEM_LOAD_OP mem_ins, ldr_ins, op, &struct.field, tmp_reg1, tmp_reg2 @@ -389,10 +409,19 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.macro MEM_LOAD_CMP_ZTS, ldr_ins, op, struct, field, tmp_reg1, tmp_reg2 | .if ZTS -| NIY // TODO: test -| LOAD_TSRM_CACHE tmp_reg1 -| ldr_ins tmp_reg2, [tmp_reg1, #(struct.._offset+offsetof(zend_..struct, field))] -| cmp tmp_reg2, op +| LOAD_TSRM_CACHE TMP3 +| SAFE_MEM_ACC_WITH_UOFFSET ldr_ins, tmp_reg1, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg2 +| cmp tmp_reg1, op +| .else +| MEM_LOAD_CMP ldr_ins, op, &struct.field, tmp_reg1, tmp_reg2 +| .endif +|.endmacro + +|.macro MEM_LOAD_CMP_ZTS_BYTE, ldr_ins, op, struct, field, tmp_reg1, tmp_reg2 +| .if ZTS +| LOAD_TSRM_CACHE TMP3 +| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldr_ins, tmp_reg1, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg2 +| cmp tmp_reg1, op | .else | MEM_LOAD_CMP ldr_ins, op, &struct.field, tmp_reg1, tmp_reg2 | .endif @@ -409,11 +438,17 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.macro MEM_LOAD_OP_STORE_ZTS, mem_ins, ldr_ins, str_ins, op, struct, field, tmp_reg1, tmp_reg2 | .if ZTS -| NIY // TODO: test -| LOAD_TSRM_CACHE tmp_reg1 -| ldr_ins tmp_reg2, [tmp_reg1, #(struct.._offset+offsetof(zend_..struct, field))] -| mem_ins tmp_reg2, tmp_reg2, op -| str_ins tmp_reg2, [tmp_reg1, #(struct.._offset+offsetof(zend_..struct, field))] +| LOAD_TSRM_CACHE TMP3 +|| if (((uintptr_t)(struct.._offset+offsetof(zend_..struct, field))) > LDRB_STRB_PIMM) { +| LOAD_32BIT_VAL tmp_reg1, (struct.._offset+offsetof(zend_..struct, field)) +| ldr_ins tmp_reg2, [TMP3, tmp_reg1] +| mem_ins tmp_reg2, tmp_reg2, op +| str_ins tmp_reg2, [TMP3, tmp_reg1] +|| } else { +| ldr_ins tmp_reg2, [TMP3, #(struct.._offset+offsetof(zend_..struct, field))] +| mem_ins tmp_reg2, tmp_reg2, op +| str_ins tmp_reg2, [TMP3, #(struct.._offset+offsetof(zend_..struct, field))] +|| } | .else | MEM_LOAD_OP_STORE mem_ins, ldr_ins, str_ins, op, &struct.field, tmp_reg1, tmp_reg2 | .endif @@ -474,9 +509,16 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) || } |.endmacro -|.macro LOAD_IP_ADDR_ZTS, struct, field +|.macro LOAD_IP_ADDR_ZTS, struct, field, tmp_reg | .if ZTS -| NIY // TODO +|| if (GCC_GLOBAL_REGS) { +| LOAD_TSRM_CACHE IP +| SAFE_MEM_ACC_WITH_UOFFSET ldr, IP, IP, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg +|| } else { +| LOAD_TSRM_CACHE RX +| SAFE_MEM_ACC_WITH_UOFFSET ldr, RX, RX, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg +| str RX, EX->opline +|| } | .else | LOAD_IP_ADDR &struct.field | .endif @@ -1576,9 +1618,9 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) |->interrupt_handler: | SAVE_IP | //EG(vm_interrupt) = 0; - | MEM_STORE_ZTS strb, wzr, executor_globals, vm_interrupt, TMP1 + | MEM_STORE_ZTS_BYTE strb, wzr, executor_globals, vm_interrupt, TMP1 | //if (EG(timed_out)) { - | MEM_LOAD_ZTS ldrb, REG0w, executor_globals, timed_out, TMP1 + | MEM_LOAD_ZTS_BYTE ldrb, REG0w, executor_globals, timed_out, TMP1 | cbz REG0w, >1 | //zend_timeout(); | EXT_CALL zend_timeout, TMP1 @@ -1697,7 +1739,7 @@ static int zend_jit_leave_throw_stub(dasm_State **Dst) | MEM_STORE_ZTS str, IP, executor_globals, opline_before_exception, TMP2 |5: | // opline = EG(exception_op); - | LOAD_IP_ADDR_ZTS executor_globals, exception_op + | LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2 | // HANDLE_EXCEPTION() | b ->exception_handler } else { @@ -1709,7 +1751,7 @@ static int zend_jit_leave_throw_stub(dasm_State **Dst) | MEM_STORE_ZTS str, IP, executor_globals, opline_before_exception, TMP2 |5: | // opline = EG(exception_op); - | LOAD_IP_ADDR_ZTS executor_globals, exception_op + | LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2 | ldp FP, RX, T2 // retore FP and IP | ldp x29, x30, [sp], #NR_SPAD // stack alignment | mov RETVALx, #2 // ZEND_VM_LEAVE @@ -1732,7 +1774,7 @@ static int zend_jit_icall_throw_stub(dasm_State **Dst) | MEM_STORE_ZTS str, IP, executor_globals, opline_before_exception, TMP2 |1: | // opline = EG(exception_op); - | LOAD_IP_ADDR_ZTS executor_globals, exception_op + | LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2 || if (GCC_GLOBAL_REGS) { | str IP, EX->opline || } @@ -2187,7 +2229,7 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst) | LOAD_IP | // check for interrupt (try to avoid this ???) - | MEM_LOAD_CMP_ZTS ldrb, wzr, executor_globals, vm_interrupt, REG0w, TMP1 + | MEM_LOAD_CMP_ZTS_BYTE ldrb, wzr, executor_globals, vm_interrupt, REG0w, TMP1 | bne ->interrupt_handler if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { @@ -2543,6 +2585,10 @@ static int zend_jit_setup(void) tsrm_tls_index = ti[0] * 8; #endif } +# elif defined(__aarch64__) + tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset(); + ZEND_ASSERT(tsrm_ls_cache_tcb_offset != 0); +# elif # endif #endif @@ -2638,7 +2684,7 @@ static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const void *exit_addr) { - | MEM_LOAD_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 + | MEM_LOAD_ZTS_BYTE ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 if (exit_addr) { | NIY // cbnz TMP1w, &exit_addr } else if (last_valid_opline == opline) { @@ -2658,7 +2704,7 @@ static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void *timeout_exit_addr) { if (timeout_exit_addr) { - | MEM_LOAD_CMP_ZTS ldrb, wzr, executor_globals, vm_interrupt, REG0w, TMP1 + | MEM_LOAD_CMP_ZTS_BYTE ldrb, wzr, executor_globals, vm_interrupt, REG0w, TMP1 | beq =>loop_label | EXT_JMP timeout_exit_addr, TMP1 } else { From c1198174415d7c77872dd27bf0affe39c5fbd50a Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 27 Apr 2021 01:25:35 +0300 Subject: [PATCH 160/229] Enable register allocator (it was disabled because ZREG_NUM wasn't available for preprocessor) and fix few related problems. --- ext/opcache/jit/zend_jit_arm64.dasc | 21 ++++++++++++++++++--- ext/opcache/jit/zend_jit_arm64.h | 2 ++ ext/opcache/jit/zend_jit_internal.h | 10 ++++------ ext/opcache/jit/zend_jit_x86.h | 2 ++ 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 3255017f87cf7..166e1481e92a3 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -3123,9 +3123,24 @@ static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr sr if (!zend_jit_same_addr(src, dst)) { if (Z_MODE(src) == IS_REG) { if (Z_MODE(dst) == IS_REG) { - | mov Rx(Z_REG(dst)), Rx(Z_REG(src)) + if ((info & MAY_BE_ANY) == MAY_BE_LONG) { + | mov Rx(Z_REG(dst)), Rx(Z_REG(src)) + } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { + | fmov Rd(Z_REG(dst)-ZREG_V0), Rd(Z_REG(src)-ZREG_V0) + } else { + ZEND_UNREACHABLE(); + } } else if (Z_MODE(dst) == IS_MEM_ZVAL) { - | fmov Rd(Z_REG(dst)-ZREG_V0), Rd(Z_REG(src)-ZREG_V0) + if (!Z_LOAD(src) && !Z_STORE(src)) { + if (!zend_jit_spill_store(Dst, src, dst, info, + JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || + JIT_G(current_frame) == NULL || + STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || + (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) + )) { + return 0; + } + } } else { ZEND_UNREACHABLE(); } @@ -3746,7 +3761,7 @@ static int zend_jit_math_double_double(dasm_State **Dst, } else if (Z_MODE(val_addr) == IS_REG) { op2_reg = Z_REG(val_addr); } else { - op2_reg = ZREG_FPR1; + op2_reg = ZREG_FPTMP; | GET_ZVAL_DVAL op2_reg, val_addr, ZREG_TMP1 } | DOUBLE_MATH_REG opcode, result_reg, op1_reg, op2_reg diff --git a/ext/opcache/jit/zend_jit_arm64.h b/ext/opcache/jit/zend_jit_arm64.h index c0a2cb901ecf4..472598fa57b37 100644 --- a/ext/opcache/jit/zend_jit_arm64.h +++ b/ext/opcache/jit/zend_jit_arm64.h @@ -136,6 +136,8 @@ typedef struct _zend_jit_registers_buf { typedef uint64_t zend_regset; +#define ZEND_REGSET_64BIT 1 + # define ZEND_REGSET_FIXED \ (ZEND_REGSET(ZREG_RSP) | ZEND_REGSET(ZREG_RLR) | ZEND_REGSET(ZREG_RFP) | \ ZEND_REGSET(ZREG_RPR) | ZEND_REGSET(ZREG_FP) | ZEND_REGSET(ZREG_IP) | \ diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 2f9e532e49f4a..f29c4b509e699 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -30,7 +30,7 @@ #define ZEND_REGSET_IS_SINGLETON(regset) \ (regset && !(regset & (regset - 1))) -#if (ZREG_NUM <= 32) +#if (!ZEND_REGSET_64BIT) #define ZEND_REGSET(reg) \ (1u << (reg)) #else @@ -38,7 +38,7 @@ (1ull << (reg)) #endif -#if (ZREG_NUM <= 32) +#if (!ZEND_REGSET_64BIT) #define ZEND_REGSET_INTERVAL(reg1, reg2) \ (((1u << ((reg2) - (reg1) + 1)) - 1) << (reg1)) #else @@ -65,14 +65,12 @@ ((set1) & ~(set2)) #if !defined(_WIN32) -# if (ZREG_NUM <= 32) +# if (!ZEND_REGSET_64BIT) # define ZEND_REGSET_FIRST(set) ((zend_reg)__builtin_ctz(set)) # define ZEND_REGSET_LAST(set) ((zend_reg)(__builtin_clz(set)^31)) -# elif(ZREG_NUM <= 64) +# else # define ZEND_REGSET_FIRST(set) ((zend_reg)__builtin_ctzll(set)) # define ZEND_REGSET_LAST(set) ((zend_reg)(__builtin_clzll(set)^63)) -# else -# error "Too many registers" # endif #else # include diff --git a/ext/opcache/jit/zend_jit_x86.h b/ext/opcache/jit/zend_jit_x86.h index 8867dc04672d8..924db409c3289 100644 --- a/ext/opcache/jit/zend_jit_x86.h +++ b/ext/opcache/jit/zend_jit_x86.h @@ -114,6 +114,8 @@ typedef struct _zend_jit_registers_buf { typedef uint32_t zend_regset; +#define ZEND_REGSET_64BIT 0 + #ifdef _WIN64 # define ZEND_REGSET_FIXED \ (ZEND_REGSET(ZREG_RSP) | ZEND_REGSET(ZREG_R14) | ZEND_REGSET(ZREG_R15)) From 2c90fe1d856472b8f942eb644cdf705fc6dc7849 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 27 Apr 2021 11:30:50 +0300 Subject: [PATCH 161/229] Fixed some compilation warnings --- ext/opcache/jit/zend_jit.c | 6 +++-- ext/opcache/jit/zend_jit_arm64.dasc | 39 +++-------------------------- ext/opcache/jit/zend_jit_gdb.c | 8 ++---- ext/opcache/jit/zend_jit_vtune.c | 4 +++ 4 files changed, 14 insertions(+), 43 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index e7374a62d0278..63f79d85ce65b 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -215,15 +215,17 @@ static bool zend_is_commutative(zend_uchar opcode) #include "jit/zend_jit_helpers.c" #include "jit/zend_jit_disasm.c" #ifndef _WIN32 -#include "jit/zend_jit_gdb.c" +#if defined(__x86_64__) || defined(i386) +# include "jit/zend_jit_gdb.c" +#endif #include "jit/zend_jit_perf_dump.c" #endif #ifdef HAVE_OPROFILE # include "jit/zend_jit_oprofile.c" #endif -#include "jit/zend_jit_vtune.c" #if defined(__x86_64__) || defined(i386) +#include "jit/zend_jit_vtune.c" #include "jit/zend_jit_x86.c" #elif defined(__aarch64__) #include "jit/zend_jit_arm64.c" diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 166e1481e92a3..572972b2ec1f4 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -966,7 +966,6 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | ldr Rd(dst_reg-ZREG_V0), [Rx(tmp_reg1)] | SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 || } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { -|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg; | NIY // TODO: || } else { | // In x64, if the range of this LONG value can be represented via INT type, only move the low 32 bits into dst_addr. @@ -997,7 +996,6 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 | SET_ZVAL_DVAL res_addr, dst_reg, tmp_reg2 || } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { -|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg; | NIY // TODO: || } else { || if (Z_MODE(dst_addr) == IS_REG) { @@ -2790,7 +2788,6 @@ typedef ZEND_SET_ALIGNED(1, int32_t unaligned_int32_t); static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr) { int ret = 0; - uint8_t *p, *end; abort(); // TODO return ret; @@ -2803,9 +2800,6 @@ static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_ static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr) { - const void *link_addr; - size_t prologue_size; - | NIY // TODO return 1; } @@ -2897,16 +2891,6 @@ static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32 static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace) { - zend_jit_op_array_trace_extension *jit_extension = - (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); - size_t offset = jit_extension->offset; - const void *handler = - (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; - - if (!zend_jit_set_valid_ip(Dst, opline)) { - return 0; - } - | NIY // TODO return 1; @@ -3161,8 +3145,6 @@ static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr sr static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags, const zend_op *opline) { - zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); - | NIY // TODO return 1; @@ -3170,8 +3152,6 @@ static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags static int zend_jit_store_const(dasm_State **Dst, int var, zend_reg reg) { - zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); - | NIY // TODO return 1; } @@ -3202,10 +3182,6 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op if (may_overflow && (((op1_def_info & MAY_BE_GUARD) && (op1_def_info & MAY_BE_LONG)) || ((opline->result_type != IS_UNUSED && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG))))) { - int32_t exit_point; - const void *exit_addr; - zend_jit_trace_stack *stack; - uint32_t old_op1_info, old_res_info = 0; | NIY // TODO: tracing } else if (may_overflow) { @@ -8138,9 +8114,6 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, cons static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zend_function *func, const zend_op *to_opline) { - int32_t exit_point; - const void *exit_addr; - | NIY // TODO return 1; @@ -8408,8 +8381,6 @@ static int zend_jit_init_method_call(dasm_State **Dst, (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || (func->common.fn_flags & ZEND_ACC_CLOSURE) || !func->common.function_name)) { - const zend_op *opcodes = func->op_array.opcodes; - | NIY // tracing } else { | NIY // tracing @@ -8478,9 +8449,6 @@ static int zend_jit_init_closure_call(dasm_State **Dst, zend_jit_trace_rec *trace, bool stack_check) { - zend_function *func = NULL; - zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); - | NIY // TODO return 1; } @@ -10135,7 +10103,6 @@ static int zend_jit_leave_func(dasm_State **Dst, if (trace->op == ZEND_JIT_TRACE_BACK && (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { - const zend_op *next_opline = trace->opline; | NIY // TODO: test @@ -10654,8 +10621,6 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, |8: if (res_exit_addr) { - zend_uchar type = concrete_type(res_info); - | NIY // tracing } else if (op1_info & MAY_BE_ARRAY_OF_REF) { | // ZVAL_COPY_DEREF @@ -12883,6 +12848,10 @@ static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const ze int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + | ldrb TMP1w, EX->This.u1.v.type | cmp TMP1w, #IS_OBJECT | NIY // bne &exit_addr diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index 0a5e5cfa3badb..ddb5f123c1913 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -22,12 +22,8 @@ #if defined(__x86_64__) || defined(i386) -#define HAVE_GDB -#else -#warning Missing GDB JIT support on this platform -#endif -#ifdef HAVE_GDB +#define HAVE_GDB #include "zend_elf.h" #include "zend_gdb.h" @@ -497,4 +493,4 @@ static void zend_jit_gdb_init(void) #endif } -#endif +#endif /* defined(__x86_64__) || defined(i386) */ diff --git a/ext/opcache/jit/zend_jit_vtune.c b/ext/opcache/jit/zend_jit_vtune.c index 1f71bd741eb0d..35fd3a031022d 100644 --- a/ext/opcache/jit/zend_jit_vtune.c +++ b/ext/opcache/jit/zend_jit_vtune.c @@ -16,6 +16,8 @@ +----------------------------------------------------------------------+ */ +#if defined(__x86_64__) || defined(i386) + #define HAVE_VTUNE 1 #include "jit/vtune/jitprofiling.h" @@ -40,3 +42,5 @@ static void zend_jit_vtune_register(const char *name, iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, (void*)&jmethod); } + +#endif /* defined(__x86_64__) || defined(i386) */ From 999720bf3ada23d4af2eb9f3467ffd8e87e10a19 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Tue, 27 Apr 2021 01:35:02 +0000 Subject: [PATCH 162/229] Remove the unnecessary 'bvs' check for IS_NOT_IDENTICAL case This 'bvs' can be removed since the follow-up 'bne' also checks NaN.[1] [1] https://community.arm.com/developer/ip-products/processors/b/processors-ip-blog/posts/condition-codes-4-floating-point-comparisons-using-vfp Change-Id: Ie67db499c6be39ae4b339db533f51dacaeab4e4c --- ext/opcache/jit/zend_jit_arm64.dasc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 572972b2ec1f4..deb0fac770948 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -6178,8 +6178,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z | NIY // beq &exit_addr |1: } else { - | bvs => target_label - | bne => target_label + | bne => target_label } break; case ZEND_IS_SMALLER: From 5e05c70ee727815805697a90b39f4d82cd4b4d3d Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Tue, 27 Apr 2021 02:51:19 +0000 Subject: [PATCH 163/229] Optimizing LONG MUL to SHIFT: add overflow detection for x86 Similar to the AArch64 implementation, we compare "op" and "op << n >> n" to detect integer overflow. Note that instructions 'pushf' and 'popf' are used to save/restore the flags set by 'cmp', otherwise, the follow-up 'shl' might change the flags. Note that test case "mul_004.phpt" can pass now for x86. Change-Id: Ieee037f41dce298979b8a0964ca97598939c495c --- ext/opcache/jit/zend_jit_x86.dasc | 61 ++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 5394f46e08fbd..795e840df0cf6 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4180,6 +4180,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); zend_reg result_reg; zend_reg tmp_reg = ZREG_R0; + bool use_ovf_flag = 1; if (Z_MODE(res_addr) == IS_REG && (res_info & MAY_BE_LONG)) { if (may_overflow && (res_info & MAY_BE_GUARD) @@ -4211,7 +4212,23 @@ static int zend_jit_math_long_long(dasm_State **Dst, | GET_ZVAL_LVAL result_reg, op1_addr | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) if (may_overflow) { - // TODO: check overflow. + /* Compare 'op' and '((op << n) >> n)' for overflow. + * Flag: jne -> overflow. je -> no overflow. + */ + use_ovf_flag = 0; + | sar Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) + if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { + | cmp Ra(result_reg), Z_LVAL_P(Z_ZV(op1_addr)) + } else if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { + | cmp Ra(result_reg), [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)] + } else if (Z_MODE(op1_addr) == IS_REG) { + | cmp Ra(result_reg), Ra(Z_REG(op1_addr)) + } else { + ZEND_UNREACHABLE(); + } + | pushf + | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) + | popf } } } else if (opcode == ZEND_MUL && @@ -4226,7 +4243,23 @@ static int zend_jit_math_long_long(dasm_State **Dst, | GET_ZVAL_LVAL result_reg, op2_addr | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) if (may_overflow) { - // TODO: check overflow. + /* Compare 'op' and '((op << n) >> n)' for overflow. + * Flag: jne -> overflow. je -> no overflow. + */ + use_ovf_flag = 0; + | sar Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) + if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { + | cmp Ra(result_reg), Z_LVAL_P(Z_ZV(op2_addr)) + } else if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { + | cmp Ra(result_reg), [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)] + } else if (Z_MODE(op2_addr) == IS_REG) { + | cmp Ra(result_reg), Ra(Z_REG(op2_addr)) + } else { + ZEND_UNREACHABLE(); + } + | pushf + | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) + | popf } } } else if (opcode == ZEND_DIV && @@ -4267,20 +4300,36 @@ static int zend_jit_math_long_long(dasm_State **Dst, int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) { - | jo &exit_addr + if (use_ovf_flag) { + | jo &exit_addr + } else { + | jne &exit_addr + } if (Z_MODE(res_addr) == IS_REG && result_reg != Z_REG(res_addr)) { | mov Ra(Z_REG(res_addr)), Ra(result_reg) } } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - | jno &exit_addr + if (use_ovf_flag) { + | jno &exit_addr + } else { + | je &exit_addr + } } else { ZEND_UNREACHABLE(); } } else { if (res_info & MAY_BE_LONG) { - | jo >1 + if (use_ovf_flag) { + | jo >1 + } else { + | jne >1 + } } else { - | jno >1 + if (use_ovf_flag) { + | jno >1 + } else { + | je >1 + } } } } From ee921d29a94d3c6d3ee23d6a78a0d554dcac1e8b Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Tue, 27 Apr 2021 03:25:57 +0000 Subject: [PATCH 164/229] Support ZEND_JIT_ON_PROF_REQUEST mode This patch implements function zend_jit_hybrid_profile_jit_stub(), which is needed by ZEND_JIT_ON_PROF_REQUEST mode, i.e. 'opcache.jit=122x'. Change-Id: Icb591c81675ffc077180229362882278ac8406c6 --- ext/opcache/jit/zend_jit_arm64.dasc | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index deb0fac770948..a34ae5a8ccf56 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -1989,7 +1989,25 @@ static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) } |->hybrid_profile_jit: - | NIY_STUB // TODO + | // ++zend_jit_profile_counter; + | LOAD_ADDR REG0, &zend_jit_profile_counter + | ldr TMP1, [REG0] + | add TMP1, TMP1, #1 + | str TMP1, [REG0] + | // op_array = (zend_op_array*)EX(func); + | ldr REG0, EX->func + | // run_time_cache = EX(run_time_cache); + | ldr REG2, EX->run_time_cache + | // jit_extension = (const void*)ZEND_FUNC_INFO(op_array); + | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | // ++ZEND_COUNTER_INFO(op_array) + | LOAD_32BIT_VAL TMP1w, (zend_jit_profile_counter_rid * sizeof(void*)) + | ldr TMP2, [REG2, TMP1] + | add TMP2, TMP2, #1 + | str TMP2, [REG2, TMP1] + | // return ((zend_vm_opcode_handler_t)jit_extension->orig_handler)() + | ldr TMP1, [REG0, #offsetof(zend_jit_op_array_extension, orig_handler)] + | br TMP1 return 1; } From 6a94b8b7b39c6fe8b9a6d9a01df045310920cc0e Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Tue, 27 Apr 2021 04:57:26 +0000 Subject: [PATCH 165/229] Support ZEND_JIT_ON_HOT_COUNTERS mode This patch implements stub functions zend_jit_hybrid_hot_code_stub() and zend_jit_hybrid_hot_counter_stub(), which is needed by HOT COUNTER mode, i.e. 'opcache.jit=123x'. Test case "defined_001.phpt" can pass now since HOT COUNTER mode is required by this case. Change-Id: I66efb07c75fc64723dc76eff533d024c868f8022 --- ext/opcache/jit/zend_jit_arm64.dasc | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index a34ae5a8ccf56..7d773636cbbc4 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -2018,7 +2018,13 @@ static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst) } |->hybrid_hot_code: - | NIY_STUB // TODO + || ZEND_ASSERT(ZEND_JIT_COUNTER_INIT <= MOVZ_IMM); + | movz TMP1w, #ZEND_JIT_COUNTER_INIT + | strh TMP1w, [REG2] + | mov FCARG1x, FP + | GET_IP FCARG2x + | EXT_CALL zend_jit_hot_func, REG0 + | JMP_IP TMP1 return 1; } @@ -2048,7 +2054,24 @@ static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst) */ static int zend_jit_hybrid_hot_counter_stub(dasm_State **Dst, uint32_t cost) { - | NIY_STUB // TODO + | ldr REG0, EX->func + | ldr REG1, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] + | ldr REG2, [REG1, #offsetof(zend_jit_op_array_hot_extension, counter)] + | LOAD_32BIT_VAL TMP1w, cost + | ldrh TMP2w, [REG2] + | sub TMP2w, TMP2w, TMP1w + | strh TMP2w, [REG2] + | cmp TMP2w, #0 + | ble ->hybrid_hot_code + | GET_IP REG2 + | ldr TMP1, [REG0, #offsetof(zend_op_array, opcodes)] + | sub REG2, REG2, TMP1 + | // divide by sizeof(zend_op) + || ZEND_ASSERT(sizeof(zend_op) == 32); + | asr REG2, REG2, #2 + | add TMP1, REG1, REG2 + | ldr TMP1, [TMP1, #offsetof(zend_jit_op_array_hot_extension, orig_handlers)] + | br TMP1 return 1; } From 625e9cb2294442af5c99cfaaeae854185eddd376 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 27 Apr 2021 16:57:07 +0300 Subject: [PATCH 166/229] Implement obvious trace side exits --- ext/opcache/jit/zend_jit_arm64.dasc | 350 ++++++++++++++-------------- 1 file changed, 179 insertions(+), 171 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 7d773636cbbc4..a3ec4a303b584 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -2725,7 +2725,7 @@ static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const { | MEM_LOAD_ZTS_BYTE ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 if (exit_addr) { - | NIY // cbnz TMP1w, &exit_addr + | cbnz TMP1w, &exit_addr } else if (last_valid_opline == opline) { || zend_jit_use_last_valid_opline(); | cbnz TMP1w, ->interrupt_handler @@ -2745,9 +2745,8 @@ static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void if (timeout_exit_addr) { | MEM_LOAD_CMP_ZTS_BYTE ldrb, wzr, executor_globals, vm_interrupt, REG0w, TMP1 | beq =>loop_label - | EXT_JMP timeout_exit_addr, TMP1 + | b &timeout_exit_addr } else { - | NIY // TODO | b =>loop_label } return 1; @@ -2907,11 +2906,7 @@ static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t } else { | add TMP1, FP, #var } - | IF_NOT_Z_TYPE TMP1, type, >1, TMP2w - |.cold_code - |1: - | EXT_JMP exit_addr, TMP1 - |.code + | IF_NOT_Z_TYPE TMP1, type, &exit_addr, TMP2w return 1; } @@ -3537,27 +3532,19 @@ static int zend_jit_math_long_long(dasm_State **Dst, const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) { if (use_ovf_flag) { - | bvs >3 + | bvs &exit_addr } else { - | bne >3 + | bne &exit_addr } - |.cold_code - |3: - | EXT_JMP exit_addr, TMP1 - |.code if (Z_MODE(res_addr) == IS_REG && result_reg != Z_REG(res_addr)) { | mov Rx(Z_REG(res_addr)), Rx(result_reg) } } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { if (use_ovf_flag) { - | bvc >3 + | bvc &exit_addr } else { - | beq >3 + | beq &exit_addr } - |.cold_code - |3: - | EXT_JMP exit_addr, TMP1 - |.code } else { ZEND_UNREACHABLE(); } @@ -4507,14 +4494,14 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o if (type == BP_JIT_IS) { if (not_found_exit_addr) { - | NIY // bls ¬_found_exit_addr + | bls ¬_found_exit_addr } else { | bls >9 // NOT_FOUND } } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { - | NIY // bls &exit_addr + | bls &exit_addr } else if (type == BP_VAR_IS && not_found_exit_addr) { - | NIY // bls ¬_found_exit_addr + | bls ¬_found_exit_addr } else if (type == BP_VAR_IS && found_exit_addr) { | bls >7 // NOT_FOUND } else { @@ -4549,7 +4536,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | EXT_CALL _zend_hash_index_find, REG0 | mov REG0, RETVALx if (not_found_exit_addr) { - | NIY // cbz REG0, ¬_found_exit_addr + | cbz REG0, ¬_found_exit_addr } else { | cbz REG0, >9 // NOT_FOUND } @@ -4561,7 +4548,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | b >5 } } else if (not_found_exit_addr) { - | NIY // b ¬_found_exit_addr + | b ¬_found_exit_addr } else { | b >9 // NOT_FOUND } @@ -4575,10 +4562,10 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { /* perform IS_UNDEF check only after result type guard (during deoptimization) */ if (!found_exit_addr || (op1_info & MAY_BE_ARRAY_HASH)) { - | NIY // IF_Z_TYPE r0, IS_UNDEF, &exit_addr + | IF_Z_TYPE REG0, IS_UNDEF, &exit_addr, TMP1w } } else if (type == BP_VAR_IS && not_found_exit_addr) { - | NIY // IF_Z_TYPE r0, IS_UNDEF, ¬_found_exit_addr + | IF_Z_TYPE REG0, IS_UNDEF, ¬_found_exit_addr, TMP1w } else if (type == BP_VAR_IS && found_exit_addr) { | IF_Z_TYPE REG0, IS_UNDEF, >7, TMP1w // NOT_FOUND } else { @@ -4587,9 +4574,9 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (packed_loaded && (op1_info & MAY_BE_ARRAY_HASH))) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { - | NIY // jmp &exit_addr + | b &exit_addr } else if (type == BP_VAR_IS && not_found_exit_addr) { - | NIY // jmp ¬_found_exit_addr + | b ¬_found_exit_addr } else if (type == BP_VAR_IS && found_exit_addr) { | b >7 // NOT_FOUND } else { @@ -4605,9 +4592,9 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | EXT_CALL _zend_hash_index_find, REG0 | mov REG0, RETVALx if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { - | NIY // cbz REG0, &exit_addr + | cbz REG0, &exit_addr } else if (type == BP_VAR_IS && not_found_exit_addr) { - | NIY // cbz REG0, ¬_found_exit_addr + | cbz REG0, ¬_found_exit_addr } else if (type == BP_VAR_IS && found_exit_addr) { | cbz REG0, >7 // NOT_FOUND } else { @@ -4717,7 +4704,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } | mov REG0, RETVALx if (not_found_exit_addr) { - | NIY // cbz REG0, ¬_found_exit_addr + | cbz REG0, ¬_found_exit_addr } else { | cbz REG0, >9 // NOT_FOUND } @@ -4741,9 +4728,9 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } | mov REG0, RETVALx if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { - | NIY // cbz REG0, &exit_addr + | cbz REG0, &exit_addr } else if (type == BP_VAR_IS && not_found_exit_addr) { - | NIY // cbz REG0, ¬_found_exit_addr + | cbz REG0, ¬_found_exit_addr } else if (type == BP_VAR_IS && found_exit_addr) { | cbz REG0, >7 } else { @@ -4799,9 +4786,9 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | ldrb TMP1w, [REG0,#offsetof(zval, u1.v.type)] | cmp TMP1w, #IS_NULL if (not_found_exit_addr) { - | NIY // jle ¬_found_exit_addr + | ble ¬_found_exit_addr } else if (found_exit_addr) { - | NIY // jg &found_exit_addr + | bgt &found_exit_addr } else { | ble >9 // NOT FOUND } @@ -4825,12 +4812,12 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | EXT_CALL zend_jit_fetch_dim_isset_helper, REG0 | mov REG0, RETVALx if (not_found_exit_addr) { - | NIY // cbz REG0, ¬_found_exit_addr + | cbz REG0, ¬_found_exit_addr if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { | b >8 } } else if (found_exit_addr) { - | NIY // cbnz REG0, &found_exit_addr + | cbnz REG0, &found_exit_addr if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) { | b >9 } @@ -5281,7 +5268,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t return 0; } - | NIY // IF_ZVAL_TYPE op3_addr, IS_UNDEF, &exit_addr + | IF_ZVAL_TYPE op3_addr, IS_UNDEF, &exit_addr, TMP1w, TMP2 val_info &= ~MAY_BE_UNDEF; } @@ -5938,21 +5925,21 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, case ZEND_CASE: case ZEND_CASE_STRICT: if (exit_addr) { - | NIY // bne &exit_addr + | bne &exit_addr } else { | bne => target_label } break; case ZEND_IS_NOT_EQUAL: if (exit_addr) { - | NIY // beq &exit_addr + | beq &exit_addr } else { | beq => target_label } break; case ZEND_IS_NOT_IDENTICAL: if (exit_addr) { - | NIY // bne &exit_addr + | bne &exit_addr } else { | beq => target_label } @@ -5960,17 +5947,13 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, case ZEND_IS_SMALLER: if (swap) { if (exit_addr) { - | NIY // ble &exit_addr + | ble &exit_addr } else { | ble => target_label } } else { if (exit_addr) { - | bge >1 - |.cold_code - |1: - | EXT_JMP exit_addr, TMP1 - |.code + | bge &exit_addr } else { | bge => target_label } @@ -5979,13 +5962,13 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { if (exit_addr) { - | NIY // blt &exit_addr + | blt &exit_addr } else { | blt => target_label } } else { if (exit_addr) { - | NIY // bgt &exit_addr + | bgt &exit_addr } else { | bgt => target_label } @@ -6002,21 +5985,21 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, case ZEND_CASE: case ZEND_CASE_STRICT: if (exit_addr) { - | NIY // beq &exit_addr + | beq &exit_addr } else { | beq => target_label } break; case ZEND_IS_NOT_EQUAL: if (exit_addr) { - | NIY // bne &exit_addr + | bne &exit_addr } else { | bne => target_label } break; case ZEND_IS_NOT_IDENTICAL: if (exit_addr) { - | NIY // be &exit_addr + | beq &exit_addr } else { | bne => target_label } @@ -6024,13 +6007,13 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, case ZEND_IS_SMALLER: if (swap) { if (exit_addr) { - | NIY // bgt &exit_addr + | bgt &exit_addr } else { | bgt => target_label } } else { if (exit_addr) { - | NIY // blt &exit_addr + | blt &exit_addr } else { | blt => target_label } @@ -6039,13 +6022,13 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { if (exit_addr) { - | NIY // bge &exit_addr + | bge &exit_addr } else { | bge => target_label } } else { if (exit_addr) { - | NIY // ble &exit_addr + | ble &exit_addr } else { | ble => target_label } @@ -6133,7 +6116,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z case ZEND_CASE: case ZEND_CASE_STRICT: if (exit_addr) { - | NIY // bne &exit_addr + | bne &exit_addr } else { | bne => target_label } @@ -6141,7 +6124,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z case ZEND_IS_NOT_EQUAL: | bvs >1 if (exit_addr) { - | NIY // beq &exit_addr + | beq &exit_addr } else { | beq => target_label } @@ -6149,8 +6132,8 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z break; case ZEND_IS_NOT_IDENTICAL: if (exit_addr) { - | NIY // bvs &exit_addr - | NIY // bne &exit_addr + | bvs &exit_addr + | bne &exit_addr } else { | bvs >1 | beq => target_label @@ -6160,14 +6143,15 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z case ZEND_IS_SMALLER: if (swap) { if (exit_addr) { - | NIY // tracing + | bvs &exit_addr + | bls &exit_addr } else { | bvs => target_label | bls => target_label } } else { if (exit_addr) { - | NIY // bhs &exit_addr + | bhs &exit_addr } else { | bhs => target_label } @@ -6176,14 +6160,15 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { if (exit_addr) { - | NIY // tracing + | bvs &exit_addr + | blo &exit_addr } else { | bvs => target_label | blo => target_label } } else { if (exit_addr) { - | NIY // bhi &exit_addr + | bhi &exit_addr } else { | bhi => target_label } @@ -6200,7 +6185,7 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z case ZEND_CASE_STRICT: | bvs >1 if (exit_addr) { - | NIY // beq &exit_adr + | beq &exit_addr } else { | beq => target_label } @@ -6208,15 +6193,15 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z break; case ZEND_IS_NOT_EQUAL: if (exit_addr) { - | NIY // bne &exit_adr + | bne &exit_addr } else { | bne => target_label } break; case ZEND_IS_NOT_IDENTICAL: if (exit_addr) { - | bvs >1 - | NIY // beq &exit_addr + | bvs >1 + | beq &exit_addr |1: } else { | bne => target_label @@ -6224,17 +6209,17 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z break; case ZEND_IS_SMALLER: if (swap) { + | bvs >1 // Always False if involving NaN if (exit_addr) { - | NIY // tracng + | bhi &exit_addr } else { - | bvs >1 // Always False if involving NaN | bhi => target_label - |1: } + |1: } else { | bvs >1 if (exit_addr) { - | NIY // blo &exit_addr + | blo &exit_addr } else { | blo => target_label } @@ -6243,17 +6228,17 @@ static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, z break; case ZEND_IS_SMALLER_OR_EQUAL: if (swap) { + | bvs >1 // Always False if involving NaN if (exit_addr) { - | NIY // tracing + | bhs &exit_addr } else { - | bvs >1 // Always False if involving NaN | bhs => target_label - |1: } + |1: } else { | bvs >1 if (exit_addr) { - | NIY // bls &exit_addr + | bls &exit_addr } else { | bls => target_label } @@ -6524,28 +6509,28 @@ static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_a case ZEND_IS_EQUAL: case ZEND_CASE: if (exit_addr) { - | NIY // bne &exit_addr + | bne &exit_addr } else { | bne => target_label } break; case ZEND_IS_NOT_EQUAL: if (exit_addr) { - | NIY // beq &exit_addr + | beq &exit_addr } else { | beq => target_label } break; case ZEND_IS_SMALLER: if (exit_addr) { - | NIY // bge &exit_addr + | bge &exit_addr } else { | bge => target_label } break; case ZEND_IS_SMALLER_OR_EQUAL: if (exit_addr) { - | NIY // bgt &exit_addr + | bgt &exit_addr } else { | bgt => target_label } @@ -6559,28 +6544,28 @@ static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_a case ZEND_IS_EQUAL: case ZEND_CASE: if (exit_addr) { - | NIY // beq &exit_addr + | beq &exit_addr } else { | beq => target_label } break; case ZEND_IS_NOT_EQUAL: if (exit_addr) { - | NIY // bne &exit_addr + | bne &exit_addr } else { | bne => target_label } break; case ZEND_IS_SMALLER: if (exit_addr) { - | NIY // blt &exit_addr + | blt &exit_addr } else { | blt => target_label } break; case ZEND_IS_SMALLER_OR_EQUAL: if (exit_addr) { - | NIY // ble &exit_addr + | ble &exit_addr } else { | ble => target_label } @@ -7031,7 +7016,7 @@ static int zend_jit_identical(dasm_State **Dst, } if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { - | NIY // b &exit_addr + | b &exit_addr } } else if (not_identical_label != (uint32_t)-1) { | b =>not_identical_label @@ -7059,7 +7044,7 @@ static int zend_jit_identical(dasm_State **Dst, if (smart_branch_opcode) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { - | NIY // b &exit_addr + | b &exit_addr } } else if (identical_label != (uint32_t)-1) { | b =>identical_label @@ -7072,7 +7057,7 @@ static int zend_jit_identical(dasm_State **Dst, if (smart_branch_opcode) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { - | NIY // b &exit_addr + | b &exit_addr } } else if (identical_label != (uint32_t)-1) { | b =>identical_label @@ -7084,7 +7069,7 @@ static int zend_jit_identical(dasm_State **Dst, if (smart_branch_opcode) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { - | NIY // b &exit_addr + | b &exit_addr } } else if (not_identical_label != (uint32_t)-1) { | b =>not_identical_label @@ -7107,7 +7092,7 @@ static int zend_jit_identical(dasm_State **Dst, zend_jit_check_exception_undef_result(Dst, opline); } if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { - | NIY // b &exit_addr + | b &exit_addr } else if (identical_label != (uint32_t)-1) { | b =>identical_label } else { @@ -7115,7 +7100,7 @@ static int zend_jit_identical(dasm_State **Dst, } |8: } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { - | NIY // beq &exit_addr + | beq &exit_addr } else if (identical_label != (uint32_t)-1) { | beq =>identical_label } else { @@ -7140,7 +7125,7 @@ static int zend_jit_identical(dasm_State **Dst, } if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { - | NIY // b &exit_addr + | b &exit_addr } } else if (smart_branch_opcode && not_identical_label != (uint32_t)-1) { | b =>not_identical_label @@ -7160,7 +7145,7 @@ static int zend_jit_identical(dasm_State **Dst, zend_jit_check_exception_undef_result(Dst, opline); } if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { - | NIY // b &exit_addr + | b &exit_addr } else if (identical_label != (uint32_t)-1) { | b =>identical_label } else { @@ -7168,7 +7153,7 @@ static int zend_jit_identical(dasm_State **Dst, } |8: } else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) { - | NIY // beq &exit_addr + | beq &exit_addr } else if (identical_label != (uint32_t)-1) { | beq =>identical_label } else { @@ -7195,7 +7180,7 @@ static int zend_jit_identical(dasm_State **Dst, if (smart_branch_opcode) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { - | NIY // b &exit_addr + | b &exit_addr } } else if (not_identical_label != (uint32_t)-1) { | b =>not_identical_label @@ -7228,9 +7213,9 @@ static int zend_jit_identical(dasm_State **Dst, if (smart_branch_opcode) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { - | NIY // cbnz RETVALw, &exit_addr + | cbnz RETVALw, &exit_addr } else { - | NIY // cbz RETVALw, &exit_addr + | cbz RETVALw, &exit_addr } } else if (not_identical_label != (uint32_t)-1) { | cbz RETVALw, =>not_identical_label @@ -7357,7 +7342,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ if (branch_opcode == ZEND_JMPNZ) { | blt >9 } else { - | NIY // blt &exit_addr + | blt &exit_addr } } else if (false_label != (uint32_t)-1) { | blt =>false_label @@ -7384,7 +7369,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ | bne >1 | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { - | NIY // b &exit_addr + | b &exit_addr } else { | b >9 } @@ -7392,14 +7377,14 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { - | NIY // bne &exit_addr + | bne &exit_addr } } } else { if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { - | NIY // beq &exit_addr + | beq &exit_addr } else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) { - | NIY // bne &exit_addr + | bne &exit_addr } else { | beq >9 } @@ -7467,7 +7452,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ if (exit_addr) { if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { - | NIY // b &exit_addr + | b &exit_addr } } else if (false_label != (uint32_t)-1) { | b =>false_label @@ -7491,7 +7476,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ | b >9 } } else if (op1_info & MAY_BE_LONG) { - | NIY // b &exit_addr + | b &exit_addr } } else if (false_label != (uint32_t)-1) { | b =>false_label @@ -7524,9 +7509,9 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ } if (exit_addr) { if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { - | NIY // bne &exit_addr + | bne &exit_addr } else { - | NIY // beq &exit_addr + | beq &exit_addr } } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { if (true_label != (uint32_t)-1) { @@ -7547,7 +7532,18 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ if (set_bool) { if (exit_addr) { - | NIY // tracing + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | bvs >1 + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + | bne &exit_addr + |1: + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + } else { + | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 + | bvs &exit_addr + | beq &exit_addr + | SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2 + } } else if (false_label != (uint32_t)-1) { // JMPZ_EX | SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2 | bvs >1 @@ -7579,7 +7575,14 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ } } else { if (exit_addr) { - | NIY // tracing + if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { + | bvs >1 + | bne &exit_addr + |1: + } else { + | bvs &exit_addr + | beq &exit_addr + } } else { ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1); if (false_label != (uint32_t)-1) { @@ -7644,9 +7647,9 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ if (exit_addr) { | CMP_ZVAL_TYPE res_addr, IS_FALSE, TMP1w, TMP2 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { - | NIY // bne &exit_addr + | bne &exit_addr } else { - | NIY // beq &exit_addr + | beq &exit_addr } } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { | CMP_ZVAL_TYPE res_addr, IS_FALSE, TMP1w, TMP2 @@ -7669,12 +7672,12 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ | tst REG0w, REG0w if (exit_addr) { if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { - | NIY // bne &exit_addr + | bne &exit_addr if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { | b >9 } } else { - | NIY // beq &exit_addr + | beq &exit_addr if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { | b >9 } @@ -7858,11 +7861,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con return 0; } - | blt >1 - |.cold_code - |1: - | EXT_JMP exit_addr, TMP1 - |.code + | blt &exit_addr } else { | blt >1 | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); @@ -8330,7 +8329,7 @@ static int zend_jit_init_method_call(dasm_State **Dst, if (!exit_addr) { return 0; } - | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, TMP1w, TMP2 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, TMP1w, TMP2 |.cold_code @@ -8626,7 +8625,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | ldr REG0, EX:RX->func | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w - | NIY // bne &exit_addr + | bne &exit_addr } } } @@ -8944,7 +8943,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w - | NIY // bne &exit_addr + | bne &exit_addr } else { | ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)] | TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w @@ -9139,7 +9138,10 @@ static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t o if (!exit_addr) { return 0; } - | NIY // tracing + | ldr REG0, EX:RX->func + | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] + | TST_32_WITH_CONST TMP1w, mask, TMP2w + | bne &exit_addr } else { | ldr REG0, EX:RX->func | ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)] @@ -9333,7 +9335,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend } | and TMP1w, REG1w, #0xff | cmp TMP1w, #IS_REFERENCE - | NIY // bne &exit_addr + | bne &exit_addr } } return 1; @@ -9365,7 +9367,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend if (!exit_addr) { return 0; } - | NIY // jmp &exit_addr + | b &exit_addr } else { | SET_EX_OPLINE opline, REG0 | LOAD_ZVAL_ADDR FCARG1x, arg_addr @@ -9438,7 +9440,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend if (!exit_addr) { return 0; } - | NIY // jmp &exit_addr + | b &exit_addr } else { | SET_EX_OPLINE opline, REG0 | LOAD_ZVAL_ADDR FCARG1x, arg_addr @@ -9628,7 +9630,7 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, zend_uchar if (smart_branch_opcode) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { - | NIY // beq &exit_addr + | beq &exit_addr } else { | beq >3 } @@ -9650,7 +9652,7 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, zend_uchar } else { | cbnz RETVALx, >3 } - | NIY // b &exit_addr + | b &exit_addr } else if (smart_branch_opcode) { if (undefined_label != (uint32_t)-1) { | cbz RETVALx, =>undefined_label @@ -9673,7 +9675,7 @@ static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, zend_uchar if (smart_branch_opcode) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { - | NIY // b &exit_addr + | b &exit_addr } } else if (defined_label != (uint32_t)-1) { | b =>defined_label @@ -9708,7 +9710,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t if (opline->extended_value & MAY_BE_NULL) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { - | NIY // b &exit_addr + | b &exit_addr } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { | b >7 } @@ -9718,7 +9720,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t } else { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { - | NIY // b &exit_addr + | b &exit_addr } else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) { | b >7 } @@ -9737,7 +9739,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { - | NIY // b &exit_addr + | b &exit_addr } } else if (!zend_jit_smart_true(Dst, opline, 0, smart_branch_opcode, target_label, target_label2)) { return 0; @@ -9746,7 +9748,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t | FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2 if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { - | NIY // b &exit_addr + | b &exit_addr } } else if (!zend_jit_smart_false(Dst, opline, 0, smart_branch_opcode, target_label)) { return 0; @@ -9837,9 +9839,9 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t | TST_32_WITH_CONST REG0w, mask, TMP1w if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { - | NIY // bne &exit_addr + | bne &exit_addr } else { - | NIY // beq &exit_addr + | beq &exit_addr } } else if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { @@ -9917,15 +9919,15 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t if (exit_addr) { if (invert) { if (smart_branch_opcode == ZEND_JMPNZ) { - | NIY // bne &exit_addr + | bne &exit_addr } else { - | NIY // beq &exit_addr + | beq &exit_addr } } else { if (smart_branch_opcode == ZEND_JMPNZ) { - | NIY // beq &exit_addr + | beq &exit_addr } else { - | NIY // bne &exit_addr + | bne &exit_addr } } } else if (smart_branch_opcode) { @@ -10532,7 +10534,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, if (op1_info & MAY_BE_ARRAY) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) { - | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, &exit_addr + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, &exit_addr, TMP1w, TMP2 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, TMP1w, TMP2 } @@ -10552,7 +10554,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) { if (exit_addr && !(op1_info & MAY_BE_OBJECT)) { - | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &exit_addr + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &exit_addr, TMP1w, TMP2 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, TMP1w, TMP2 } @@ -10584,7 +10586,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, if (op1_info & MAY_BE_OBJECT) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) { if (exit_addr) { - | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, TMP1w, TMP2 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6, TMP1w, TMP2 } @@ -11005,7 +11007,7 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, if (!(opline->extended_value & ZEND_ISEMPTY)) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { - | NIY // b &exit_addr + | b &exit_addr } else { | b >8 } @@ -11041,7 +11043,7 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, if (!(opline->extended_value & ZEND_ISEMPTY)) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { - | NIY // b &exit_addr + | b &exit_addr } } else if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { @@ -11498,7 +11500,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, if (!exit_addr) { return 0; } - | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, TMP1w, TMP2 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7, TMP1w, TMP2 } @@ -11584,7 +11586,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, if (!exit_addr) { return 0; } - | NIY // IF_UNDEF dl, &exit_addr + | IF_UNDEF REG2w, &exit_addr } } else { | IF_UNDEF REG2w, >5 @@ -11698,12 +11700,12 @@ static int zend_jit_fetch_obj(dasm_State **Dst, | add REG0, REG0, #offsetof(zend_reference, val) if (type < IS_STRING) { |1: - | NIY // IF_NOT_ZVAL_TYPE val_addr, type, &exit_addr, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE val_addr, type, &exit_addr, TMP1w, TMP2 } else { | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 |1: | and TMP1w, REG2w, #0xff - | NIY // IF_NOT_TYPE TMP1w, type, &exit_addr + | IF_NOT_TYPE TMP1w, type, &exit_addr } | // ZVAL_COPY | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0 @@ -11885,7 +11887,7 @@ static int zend_jit_incdec_obj(dasm_State **Dst, if (!exit_addr) { return 0; } - | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, TMP1w, TMP2 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, TMP1w, TMP2 |.cold_code @@ -11958,7 +11960,9 @@ static int zend_jit_incdec_obj(dasm_State **Dst, if (!exit_addr) { return 0; } - | NIY // IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr + | LOAD_32BIT_VAL TMP1, prop_info->offset + offsetof(zval,u1.v.type) + | ldrb TMP2w, [FCARG1x, TMP1] + | IF_TYPE TMP2w, IS_UNDEF, &exit_addr } else { | LOAD_32BIT_VAL TMP1, prop_info->offset + offsetof(zval,u1.v.type) | ldrb TMP2w, [FCARG1x, TMP1] @@ -12229,7 +12233,7 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, if (!exit_addr) { return 0; } - | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, TMP1w, TMP2 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, TMP1w, TMP2 |.cold_code @@ -12311,7 +12315,9 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, if (!exit_addr) { return 0; } - | NIY // IF_TYPE byte [FCARG1a + prop_info->offset + offsetof(zval,u1.v.type)], IS_UNDEF, &exit_addr + | LOAD_32BIT_VAL TMP1w, (prop_info->offset + offsetof(zval,u1.v.type)) + | ldrb TMP2w, [FCARG1x, TMP1] + | IF_TYPE TMP2w, IS_UNDEF, &exit_addr } else { | LOAD_32BIT_VAL TMP1w, (prop_info->offset + offsetof(zval,u1.v.type)) | ldrb TMP2w, [FCARG1x, TMP1] @@ -12527,7 +12533,7 @@ static int zend_jit_assign_obj(dasm_State **Dst, if (!exit_addr) { return 0; } - | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, TMP1w, TMP2 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, TMP1w, TMP2 |.cold_code @@ -12639,7 +12645,9 @@ static int zend_jit_assign_obj(dasm_State **Dst, if (!exit_addr) { return 0; } - | NIY // IF_TYPE byte [FCARG1a + prop_info->offset + offsetof(zval,u1.v.type)], IS_UNDEF, &exit_addr + | LOAD_32BIT_VAL TMP1, (prop_info->offset + offsetof(zval,u1.v.type)) + | ldrb TMP2w, [FCARG1x, TMP1] + | IF_TYPE TMP2w, IS_UNDEF, &exit_addr } else { | LOAD_32BIT_VAL TMP1, (prop_info->offset + offsetof(zval,u1.v.type)) | ldrb TMP2w, [FCARG1x, TMP1] @@ -12894,7 +12902,7 @@ static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const ze | ldrb TMP1w, EX->This.u1.v.type | cmp TMP1w, #IS_OBJECT - | NIY // bne &exit_addr + | bne &exit_addr if (JIT_G(current_frame)) { TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame)); @@ -12931,7 +12939,7 @@ static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend const void *exit_addr; if (default_label) { - | NIY // jz &default_label + | cbz REG0, &default_label } else if (next_opline) { | cbz REG0, >3 } else { @@ -13068,7 +13076,7 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o |1: | // ZVAL_DEREF(op) if (fallback_label) { - | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label, TMP1w, TMP2 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, TMP1w, TMP2 } @@ -13086,7 +13094,7 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o } else { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { if (fallback_label) { - | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label, TMP1w, TMP2 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, TMP1w, TMP2 } @@ -13100,7 +13108,7 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o | LOAD_32BIT_VAL TMP1w, jumptable->nNumUsed | cmp FCARG2x, TMP1 if (default_label) { - | NIY // jae &default_label + | bhs &default_label } else if (next_opline) { | bhs >3 } else { @@ -13164,7 +13172,7 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o |1: | // ZVAL_DEREF(op) if (fallback_label) { - | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label, TMP1w, TMP2 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, TMP1w, TMP2 } @@ -13182,7 +13190,7 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o } else { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) { if (fallback_label) { - | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label, TMP1w, TMP2 } else { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, TMP1w, TMP2 } @@ -13212,7 +13220,7 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o } else if (op1_info & MAY_BE_UNDEF) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, TMP1w, TMP2 } else if (default_label) { - | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &default_label + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &default_label, TMP1w, TMP2 } else if (next_opline) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, TMP1w, TMP2 } else { @@ -13232,7 +13240,7 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o if (op1_info & MAY_BE_UNDEF) { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, TMP1w, TMP2 } else if (default_label) { - | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &default_label + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &default_label, TMP1w, TMP2 } else if (next_opline) { | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, TMP1w, TMP2 } else { @@ -13252,7 +13260,7 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o |6: if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) { if (default_label) { - | NIY // IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, &default_label + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, &default_label, TMP1w, TMP2 } else if (next_opline) { | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >3, TMP1w, TMP2 } else { @@ -13268,7 +13276,7 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o } } if (default_label) { - | NIY // jmp &default_label + | b &default_label } else if (next_opline) { | b >3 } else { @@ -13386,9 +13394,9 @@ static int zend_jit_isset_isempty_cv(dasm_State **Dst, const zend_op *opline, ui | cmp TMP1w, #IS_NULL if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { - | NIY // bgt &exit_addr + | bgt &exit_addr } else { - | NIY // ble &exit_addr + | ble &exit_addr } } else if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { @@ -13461,7 +13469,7 @@ static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t o | // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); | // ZEND_VM_CONTINUE(); if (exit_addr) { - | NIY // bls &exit_addr + | bls &exit_addr } else { | bls =>target_label } @@ -13472,7 +13480,7 @@ static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t o if (!exit_addr || exit_opcode == ZEND_JMP) { | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, >3, TMP1w } else { - | NIY // IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, &exit_addr + | IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, &exit_addr, TMP1w } | // p++; | add FCARG2x, FCARG2x, #sizeof(Bucket) @@ -13593,10 +13601,10 @@ static int zend_jit_fetch_constant(dasm_State **Dst, zend_uchar type = concrete_type(res_info); if (type < IS_STRING) { - | NIY // IF_NOT_ZVAL_TYPE const_addr, type, &exit_addr + | IF_NOT_ZVAL_TYPE const_addr, type, &exit_addr, TMP1w, TMP2 } else { | GET_ZVAL_TYPE_INFO REG2w, const_addr, TMP1 - | NIY // IF_NOT_TYPE REF2w, type, &exit_addr + | IF_NOT_TYPE REG2w, type, &exit_addr } | ZVAL_COPY_VALUE_V res_addr, -1, const_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0 if (type < IS_STRING) { @@ -13646,9 +13654,9 @@ static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t o } if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { - | NIY // jcb RETVALx, &exit_addr + | cbz RETVALx, &exit_addr } else { - | NIY // cbnz RETVALx, &exit_addr + | cbnz RETVALx, &exit_addr } } else if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { From 0f65a188d1d64d3107a505ce2269ef0c59331c1e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 27 Apr 2021 21:30:39 +0300 Subject: [PATCH 167/229] Fixed incorrect register usage --- ext/opcache/jit/zend_jit_arm64.dasc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index a3ec4a303b584..34e70321fbde5 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -1742,11 +1742,11 @@ static int zend_jit_leave_throw_stub(dasm_State **Dst) | b ->exception_handler } else { | GET_IP TMP1 - | ldrb TMP1w, OP:TMP1->opcode - | cmp TMP1w, #ZEND_HANDLE_EXCEPTION + | ldrb TMP2w, OP:TMP1->opcode + | cmp TMP2w, #ZEND_HANDLE_EXCEPTION | beq >5 | // EG(opline_before_exception) = opline; - | MEM_STORE_ZTS str, IP, executor_globals, opline_before_exception, TMP2 + | MEM_STORE_ZTS str, TMP1, executor_globals, opline_before_exception, TMP2 |5: | // opline = EG(exception_op); | LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2 From 9d809ff3662ce5f752aae1c5591610e5dfd0ff72 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 28 Apr 2021 00:54:22 +0300 Subject: [PATCH 168/229] Implement trace patching --- ext/opcache/jit/zend_jit_arm64.dasc | 69 +++++++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 34e70321fbde5..39a63a21b1453 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -2822,14 +2822,75 @@ static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_t return 1; } -typedef ZEND_SET_ALIGNED(1, uint16_t unaligned_uint16_t); -typedef ZEND_SET_ALIGNED(1, int32_t unaligned_int32_t); - static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr) { int ret = 0; + uint8_t *p, *end; + size_t code_size = size; + ptrdiff_t delta; + + if (jmp_table_size) { + const void **jmp_slot = (const void **)((char*)code + size); + + code_size -= jmp_table_size * sizeof(void*); + do { + jmp_slot--; + if (*jmp_slot == from_addr) { + *jmp_slot = to_addr; + ret++; + } + } while (--jmp_table_size); + } + + p = (uint8_t*)code; + end = p + code_size; + while (p < end) { + uint32_t *ins_ptr = (uint32_t*)p; + uint32_t ins = *ins_ptr; + if ((ins & 0xfc000000u) == 0x14000000u) { + // B (imm26:0..25) + delta = (uint32_t*)from_addr - ins_ptr; + if (((ins ^ (uint32_t)delta) & 0x01ffffffu) == 0) { + delta = (uint32_t*)to_addr - ins_ptr; + if (((delta + 0x02000000) >> 26) == 0) { + abort(); // brnach target out of range + } + *ins_ptr = (ins & 0xfc000000u) | ((uint32_t)delta & 0x03ffffffu); + ret++; + } + } else if ((ins & 0xff000000u) == 0x54000000u || + (ins & 0x7e000000u) == 0x34000000u) { + // B.cond, CBZ, CBNZ (imm19:5..23) + delta = (uint32_t*)from_addr - ins_ptr; + if (((ins ^ ((uint32_t)delta << 5)) & 0x00ffffe0u) == 0) { + delta = (uint32_t*)to_addr - ins_ptr; + if (((delta + 0x80000) >> 19) == 0) { + abort(); // brnach target out of range + } + *ins_ptr = (ins & 0xff00001fu) | (((uint32_t)delta & 0x7ffffu) << 5); + ret++; + } + } else if ((ins & 0x7e000000u) == 0x36000000u) { + // TBZ, TBNZ (imm14:5..18) + delta = (uint32_t*)from_addr - ins_ptr; + if (((ins ^ ((uint32_t)delta << 5)) & 0x0007ffe0u) == 0) { + delta = (uint32_t*)to_addr - ins_ptr; + if (((delta + 0x2000) >> 14) == 0) { + abort(); // brnach target out of range + } + *ins_ptr = (ins & 0xfff8001fu) | (((uint32_t)delta & 0x3fffu) << 5); + ret++; + } + } + p += 4; + } + + JIT_CACHE_FLUSH(code, (char*)code + size); + +#ifdef HAVE_VALGRIND + VALGRIND_DISCARD_TRANSLATIONS(code, size); +#endif - abort(); // TODO return ret; } From cba81db696f2e032eb55023466b173329ff1824c Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Tue, 27 Apr 2021 09:23:06 +0000 Subject: [PATCH 169/229] Fix incorrect macro LDR_STR_PIMM The 'pimm' field of ldr/str instruction has different maximum values for 32-bit register and 64-bit register. Change-Id: I14b4f1eedbcbb303696c22c1403b49d52426d390 --- ext/opcache/jit/zend_jit_arm64.dasc | 79 ++++++++++++++++++++--------- 1 file changed, 56 insertions(+), 23 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 39a63a21b1453..08157b0305200 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -99,7 +99,8 @@ #define CMP_IMM MAX_IMM12 // cmp insn #define MOVZ_IMM MAX_IMM16 // movz insn #define ADD_SUB_IMM MAX_IMM12 // add/sub insn -#define LDR_STR_PIMM (MAX_IMM12*8) // ldr/str insn: pimm is imm12 * 8 +#define LDR_STR_PIMM64 (MAX_IMM12*8) // ldr/str insn for 64-bit register: pimm is imm12 * 8 +#define LDR_STR_PIMM32 (MAX_IMM12*4) // ldr/str insn for 32-bit register: pimm is imm12 * 4 #define LDRB_STRB_PIMM MAX_IMM12 // ldrb/strb insn #include "Zend/zend_cpuinfo.h" @@ -270,8 +271,18 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) // Safe memory load/store with an unsigned immediate offset. // When using Z_OFFSET(addr), which is 24-bit long, as the unsigned offset to compute a memory address, // we should firstly check the range. +// Use SAFE_MEM_ACC_WITH_UOFFSET if 'op' is 64-bit register. Use SAFE_MEM_ACC_WITH_UOFFSET_32 if 'op' is 32-bit register. |.macro SAFE_MEM_ACC_WITH_UOFFSET, ldr_str_ins, op, base_reg, offset, tmp_reg -|| if (((uintptr_t)(offset)) > LDR_STR_PIMM) { +|| if (((uintptr_t)(offset)) > LDR_STR_PIMM64) { +| LOAD_32BIT_VAL tmp_reg, offset +| ldr_str_ins op, [base_reg, tmp_reg] +|| } else { +| ldr_str_ins op, [base_reg, #(offset)] +|| } +|.endmacro + +|.macro SAFE_MEM_ACC_WITH_UOFFSET_32, ldr_str_ins, op, base_reg, offset, tmp_reg +|| if (((uintptr_t)(offset)) > LDR_STR_PIMM32) { | LOAD_32BIT_VAL tmp_reg, offset | ldr_str_ins op, [base_reg, tmp_reg] || } else { @@ -340,6 +351,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | str_ins op, [tmp_reg] |.endmacro +// 'op' is 64-bit register |.macro MEM_STORE_ZTS, str_ins, op, struct, field, tmp_reg | .if ZTS | LOAD_TSRM_CACHE TMP3 @@ -349,15 +361,25 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | .endif |.endmacro -|.macro MEM_STORE_ZTS_BYTE, str_ins, op, struct, field, tmp_reg +// 'op' is 32-bit register +|.macro MEM_STORE_32_ZTS, str_ins, op, struct, field, tmp_reg | .if ZTS | LOAD_TSRM_CACHE TMP3 -| SAFE_MEM_ACC_WITH_UOFFSET_BYTE str_ins, op, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg +| SAFE_MEM_ACC_WITH_UOFFSET_32 str_ins, op, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg | .else | MEM_STORE str_ins, op, &struct.field, tmp_reg | .endif |.endmacro +|.macro MEM_STORE_BYTE_ZTS, strb_ins, op, struct, field, tmp_reg +| .if ZTS +| LOAD_TSRM_CACHE TMP3 +| SAFE_MEM_ACC_WITH_UOFFSET_BYTE strb_ins, op, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg +| .else +| MEM_STORE strb_ins, op, &struct.field, tmp_reg +| .endif +|.endmacro + // Load the value from memory 'addr' into a register 'op' |.macro MEM_LOAD, ldr_ins, op, addr, tmp_reg | LOAD_ADDR tmp_reg, addr @@ -373,15 +395,24 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | .endif |.endmacro -|.macro MEM_LOAD_ZTS_BYTE, ldr_ins, op, struct, field, tmp_reg +|.macro MEM_LOAD_32_ZTS, ldr_ins, op, struct, field, tmp_reg | .if ZTS | LOAD_TSRM_CACHE TMP3 -| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldr_ins, op, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg +| SAFE_MEM_ACC_WITH_UOFFSET_32 ldr_ins, op, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg | .else | MEM_LOAD ldr_ins, op, &struct.field, tmp_reg | .endif |.endmacro +|.macro MEM_LOAD_BYTE_ZTS, ldrb_ins, op, struct, field, tmp_reg +| .if ZTS +| LOAD_TSRM_CACHE TMP3 +| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb_ins, op, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg +| .else +| MEM_LOAD ldrb_ins, op, &struct.field, tmp_reg +| .endif +|.endmacro + // Load the value from memory 'addr' into a tmp register 'tmp_reg1', // and conduct arithmetic operations with 'op'. // Operations can be add/sub/div/mul, and the computation result is stored into 'op'. @@ -417,13 +448,13 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | .endif |.endmacro -|.macro MEM_LOAD_CMP_ZTS_BYTE, ldr_ins, op, struct, field, tmp_reg1, tmp_reg2 +|.macro MEM_LOAD_CMP_BYTE_ZTS, ldrb_ins, op, struct, field, tmp_reg1, tmp_reg2 | .if ZTS | LOAD_TSRM_CACHE TMP3 -| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldr_ins, tmp_reg1, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg2 +| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb_ins, tmp_reg1, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg2 | cmp tmp_reg1, op | .else -| MEM_LOAD_CMP ldr_ins, op, &struct.field, tmp_reg1, tmp_reg2 +| MEM_LOAD_CMP ldrb_ins, op, &struct.field, tmp_reg1, tmp_reg2 | .endif |.endmacro @@ -439,7 +470,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.macro MEM_LOAD_OP_STORE_ZTS, mem_ins, ldr_ins, str_ins, op, struct, field, tmp_reg1, tmp_reg2 | .if ZTS | LOAD_TSRM_CACHE TMP3 -|| if (((uintptr_t)(struct.._offset+offsetof(zend_..struct, field))) > LDRB_STRB_PIMM) { +|| if (((uintptr_t)(struct.._offset+offsetof(zend_..struct, field))) > LDR_STR_PIMM64) { | LOAD_32BIT_VAL tmp_reg1, (struct.._offset+offsetof(zend_..struct, field)) | ldr_ins tmp_reg2, [TMP3, tmp_reg1] | mem_ins tmp_reg2, tmp_reg2, op @@ -607,23 +638,25 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.macro GET_ZVAL_TYPE, reg, addr, tmp_reg || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -| SAFE_MEM_ACC_WITH_UOFFSET ldrb, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.v.type), tmp_reg +| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.v.type), tmp_reg |.endmacro +// 'reg' is 32-bit register |.macro GET_ZVAL_TYPE_INFO, reg, addr, tmp_reg || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -| SAFE_MEM_ACC_WITH_UOFFSET ldr, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg +| SAFE_MEM_ACC_WITH_UOFFSET_32 ldr, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg |.endmacro |.macro SET_ZVAL_TYPE_INFO, addr, type, tmp_reg1, tmp_reg2 || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); | LOAD_32BIT_VAL tmp_reg1, type -| SAFE_MEM_ACC_WITH_UOFFSET str, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg2 +| SAFE_MEM_ACC_WITH_UOFFSET_32 str, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg2 |.endmacro +// 'type' is 32-bit register |.macro SET_ZVAL_TYPE_INFO_FROM_REG, addr, type, tmp_reg || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -| SAFE_MEM_ACC_WITH_UOFFSET str, type, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg +| SAFE_MEM_ACC_WITH_UOFFSET_32 str, type, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg |.endmacro |.macro GET_Z_PTR, reg, zv @@ -1616,9 +1649,9 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) |->interrupt_handler: | SAVE_IP | //EG(vm_interrupt) = 0; - | MEM_STORE_ZTS_BYTE strb, wzr, executor_globals, vm_interrupt, TMP1 + | MEM_STORE_BYTE_ZTS strb, wzr, executor_globals, vm_interrupt, TMP1 | //if (EG(timed_out)) { - | MEM_LOAD_ZTS_BYTE ldrb, REG0w, executor_globals, timed_out, TMP1 + | MEM_LOAD_BYTE_ZTS ldrb, REG0w, executor_globals, timed_out, TMP1 | cbz REG0w, >1 | //zend_timeout(); | EXT_CALL zend_timeout, TMP1 @@ -2268,7 +2301,7 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst) | LOAD_IP | // check for interrupt (try to avoid this ???) - | MEM_LOAD_CMP_ZTS_BYTE ldrb, wzr, executor_globals, vm_interrupt, REG0w, TMP1 + | MEM_LOAD_CMP_BYTE_ZTS ldrb, wzr, executor_globals, vm_interrupt, REG0w, TMP1 | bne ->interrupt_handler if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { @@ -2723,7 +2756,7 @@ static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline) static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const void *exit_addr) { - | MEM_LOAD_ZTS_BYTE ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 + | MEM_LOAD_BYTE_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1 if (exit_addr) { | cbnz TMP1w, &exit_addr } else if (last_valid_opline == opline) { @@ -2743,7 +2776,7 @@ static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void *timeout_exit_addr) { if (timeout_exit_addr) { - | MEM_LOAD_CMP_ZTS_BYTE ldrb, wzr, executor_globals, vm_interrupt, REG0w, TMP1 + | MEM_LOAD_CMP_BYTE_ZTS ldrb, wzr, executor_globals, vm_interrupt, REG0w, TMP1 | beq =>loop_label | b &timeout_exit_addr } else { @@ -2806,7 +2839,7 @@ static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_t | sub sp, sp, #16 | stp TMP1, TMP2, [sp] // save TMP1 and TMP2 | LOAD_32BIT_VAL TMP1w, trace_num - | MEM_STORE_ZTS str, TMP1w, executor_globals, jit_trace_num, TMP2 + | MEM_STORE_32_ZTS str, TMP1w, executor_globals, jit_trace_num, TMP2 | ldp TMP1, TMP2, [sp] // retore TMP1 and TMP2 | add sp, sp, #16 } else { @@ -2814,7 +2847,7 @@ static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_t zend_reg tmp2 = ZEND_REGSET_FIRST(ZEND_REGSET_EXCL(regset, tmp1)); | LOAD_32BIT_VAL Rw(tmp1), trace_num - | MEM_STORE_ZTS str, Rw(tmp1), executor_globals, jit_trace_num, Rx(tmp2) + | MEM_STORE_32_ZTS str, Rw(tmp1), executor_globals, jit_trace_num, Rx(tmp2) (void)tmp1; (void)tmp2; } @@ -8269,7 +8302,7 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t | EXT_CALL zend_jit_init_func_run_time_cache_helper, REG0 | ldr REG1, EX->run_time_cache | mov REG0, RETVALx - || ZEND_ASSERT(opline->result.num <= LDR_STR_PIMM); + || ZEND_ASSERT(opline->result.num <= LDR_STR_PIMM64); | str REG0, [REG1, #opline->result.num] | b >3 } else { @@ -11138,7 +11171,7 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_ | ldr REG0, [REG0, TMP1] | sub REG0, REG0, #1 | // if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket))) - | MEM_LOAD_ZTS ldr, REG1w, executor_globals, symbol_table.nNumUsed, REG1 + | MEM_LOAD_32_ZTS ldr, REG1w, executor_globals, symbol_table.nNumUsed, REG1 | lsl REG1, REG1, #5 | cmp REG0, REG1 | bhs >9 From d96fe3de9b1c64a7fddbc021097dee0b9217f71c Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Tue, 27 Apr 2021 14:57:50 +0000 Subject: [PATCH 170/229] Add the missing parts for macros ZVAL_COPY_CONST and ZVAL_COPY_CONST_2 This patch adds the missing parts. Change-Id: I5ce0128a2ebf7cf73ec109d44dc4aabfd02260f2 --- ext/opcache/jit/zend_jit_arm64.dasc | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 08157b0305200..d85bfcccdb465 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -999,7 +999,9 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | ldr Rd(dst_reg-ZREG_V0), [Rx(tmp_reg1)] | SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 || } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { -| NIY // TODO: +|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg; +| DOUBLE_GET_LONG dst_reg, Z_LVAL_P(zv), tmp_reg1 +| SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 || } else { | // In x64, if the range of this LONG value can be represented via INT type, only move the low 32 bits into dst_addr. | // Note that imm32 is signed extended to 64 bits during mov. @@ -1029,7 +1031,17 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2 | SET_ZVAL_DVAL res_addr, dst_reg, tmp_reg2 || } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { -| NIY // TODO: +|| if (Z_MODE(dst_addr) == IS_REG) { +| DOUBLE_GET_LONG Z_REG(dst_addr), Z_LVAL_P(zv), tmp_reg1 +| SET_ZVAL_DVAL res_addr, Z_REG(dst_addr), tmp_reg2 +|| } else if (Z_MODE(res_addr) == IS_REG) { +| DOUBLE_GET_LONG Z_REG(res_addr), Z_LVAL_P(zv), tmp_reg1 +| SET_ZVAL_DVAL dst_addr, Z_REG(res_addr), tmp_reg2 +|| } else { +| DOUBLE_GET_LONG fp_tmp_reg, Z_LVAL_P(zv), tmp_reg1 +| SET_ZVAL_DVAL dst_addr, fp_tmp_reg, tmp_reg2 +| SET_ZVAL_DVAL res_addr, fp_tmp_reg, tmp_reg2 +|| } || } else { || if (Z_MODE(dst_addr) == IS_REG) { | SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2) From f49a7817be5fa083bbe9068a3540fbc94b5faa26 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 28 Apr 2021 10:06:36 +0300 Subject: [PATCH 171/229] Fixed assertion --- ext/opcache/jit/zend_jit_arm64.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index d85bfcccdb465..258aa6f5a36be 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -302,7 +302,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.macro LOAD_TSRM_CACHE, reg | //.byte 0x4d, 0xd0, 0x3b, 0xd5 // TODO: hard-coded: mrs TMP3, tpidr_el0 | .long 0xd53bd04d // TODO: hard-coded: mrs TMP3, tpidr_el0 -|| ZEND_ASSERT(tsrm_ls_cache_tcb_offset <= ADD_SUB_IMM); +|| ZEND_ASSERT(tsrm_ls_cache_tcb_offset <= LDR_STR_PIMM64); | ldr reg, [TMP3, #tsrm_ls_cache_tcb_offset] |.endmacro From cde0abbbbabf9d96ec2f7d4db46332dd1fb5a30d Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 28 Apr 2021 12:50:26 +0300 Subject: [PATCH 172/229] Support for tracin JIT (incomplete, but abble to run Zend/bench.php) --- ext/opcache/jit/zend_jit_arm64.dasc | 387 ++++++++++++++++++++++++++-- 1 file changed, 367 insertions(+), 20 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 258aa6f5a36be..ad97a6cfed6df 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -2358,7 +2358,18 @@ static int zend_jit_trace_escape_stub(dasm_State **Dst) { |->trace_escape: | - | NIY_STUB // TODO + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | ADD_HYBRID_SPAD + | JMP_IP, TMP1 + } else if (GCC_GLOBAL_REGS) { + | ldp x29, x30, [sp], #SPAD // stack alignment + | JMP_IP, TMP1 + } else { + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], #NR_SPAD // stack alignment + | mov RETVALx, #1 // ZEND_VM_ENTER + | ret + } return 1; } @@ -2946,7 +2957,39 @@ static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_ static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr) { - | NIY // TODO + const void *link_addr; + size_t prologue_size; + + /* Skip prologue. */ + // TODO: don't hardcode this ??? + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { +#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE + prologue_size = 0; +#else + // sub sp, sp, #0x20 + prologue_size = 4; +#endif + } else if (GCC_GLOBAL_REGS) { + // sub sp, sp, #0x20 + // stp x29, x30, [sp] + prologue_size = 8; + } else { + // sub sp, sp, NR_SPAD + // stp x29, x30, [sp] + // stp FP, RX, T2 + // mov FP, FCARG1x + prologue_size = 16; + } + link_addr = (const void*)((const char*)t->code_start + prologue_size); + + if (timeout_exit_addr) { + /* Check timeout for links to LOOP */ + | MEM_LOAD_CMP_BYTE_ZTS ldrb, wzr, executor_globals, vm_interrupt, REG0w, TMP1 + | beq &link_addr + | b &timeout_exit_addr + } else { + | b &link_addr + } return 1; } @@ -3021,19 +3064,165 @@ static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32 { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); if (!exit_addr) { return 0; } - | NIY // TODO + | GET_ZVAL_LVAL ZREG_FCARG1x, var_addr, TMP1 + if (op_info & MAY_BE_ARRAY_PACKED) { + | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] + | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w + | beq &exit_addr + } else { + | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] + | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w + | bne &exit_addr + } return 1; } static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace) { - | NIY // TODO + zend_jit_op_array_trace_extension *jit_extension = + (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); + size_t offset = jit_extension->offset; + const void *handler = + (zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler; + + if (!zend_jit_set_valid_ip(Dst, opline)) { + return 0; + } + if (!GCC_GLOBAL_REGS) { + | mov FCARG1x, FP + } + | EXT_CALL handler, REG0 + if (may_throw + && opline->opcode != ZEND_RETURN + && opline->opcode != ZEND_RETURN_BY_REF) { + | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, REG0, TMP1 + | bne ->exception_handler + } + + while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) { + trace++; + } + + if (!GCC_GLOBAL_REGS + && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) { + if (opline->opcode == ZEND_RETURN || + opline->opcode == ZEND_RETURN_BY_REF || + opline->opcode == ZEND_DO_UCALL || + opline->opcode == ZEND_DO_FCALL_BY_NAME || + opline->opcode == ZEND_DO_FCALL || + opline->opcode == ZEND_GENERATOR_CREATE) { + | MEM_LOAD_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 + } + } + + if (zend_jit_trace_may_exit(op_array, opline)) { + if (opline->opcode == ZEND_RETURN || + opline->opcode == ZEND_RETURN_BY_REF || + opline->opcode == ZEND_GENERATOR_CREATE) { + + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { +#if 0 + /* this check should be handled by the following OPLINE guard or jmp [IP] */ + | LOAD_ADDR TMP1, zend_jit_halt_op + | cmp IP, TMP1 + | beq ->trace_halt +#endif + } else if (GCC_GLOBAL_REGS) { + | cbz IP, ->trace_halt + } else { + | tst RETVALx, RETVALx + | blt ->trace_halt + } + } else if (opline->opcode == ZEND_EXIT || + opline->opcode == ZEND_GENERATOR_RETURN || + opline->opcode == ZEND_YIELD || + opline->opcode == ZEND_YIELD_FROM) { + | b ->trace_halt + } + if (trace->op != ZEND_JIT_TRACE_END || + (trace->stop != ZEND_JIT_TRACE_STOP_RETURN && + trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { + + const zend_op *next_opline = trace->opline; + const zend_op *exit_opline = NULL; + uint32_t exit_point; + const void *exit_addr; + uint32_t old_info = 0; + uint32_t old_res_info = 0; + zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; + + if (zend_is_smart_branch(opline)) { + bool exit_if_true = 0; + exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true); + } else { + switch (opline->opcode) { + case ZEND_JMPZ: + case ZEND_JMPNZ: + case ZEND_JMPZ_EX: + case ZEND_JMPNZ_EX: + case ZEND_JMP_SET: + case ZEND_COALESCE: + case ZEND_JMP_NULL: + case ZEND_FE_RESET_R: + case ZEND_FE_RESET_RW: + exit_opline = (trace->opline == opline + 1) ? + OP_JMP_ADDR(opline, opline->op2) : + opline + 1; + break; + case ZEND_JMPZNZ: + exit_opline = (trace->opline == OP_JMP_ADDR(opline, opline->op2)) ? + ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : + OP_JMP_ADDR(opline, opline->op2); + break; + case ZEND_FE_FETCH_R: + case ZEND_FE_FETCH_RW: + if (opline->op2_type == IS_CV) { + old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var)); + SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1); + } + exit_opline = (trace->opline == opline + 1) ? + ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : + opline + 1; + break; + + } + } + + if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { + old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); + SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); + } + exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { + SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); + } + switch (opline->opcode) { + case ZEND_FE_FETCH_R: + case ZEND_FE_FETCH_RW: + if (opline->op2_type == IS_CV) { + SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info); + } + break; + } + + if (!exit_addr) { + return 0; + } + | CMP_IP next_opline, TMP1, TMP2 + | bne &exit_addr + } + } + + zend_jit_set_last_valid_opline(trace->opline); return 1; } @@ -3324,8 +3513,49 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op if (may_overflow && (((op1_def_info & MAY_BE_GUARD) && (op1_def_info & MAY_BE_LONG)) || ((opline->result_type != IS_UNUSED && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG))))) { + int32_t exit_point; + const void *exit_addr; + zend_jit_trace_stack *stack; + uint32_t old_op1_info, old_res_info = 0; - | NIY // TODO: tracing + stack = JIT_G(current_frame)->stack; + old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); + SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0); + if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { + SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MAX_PLUS_1); + } else { + SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MIN_MINUS_1); + } + if (opline->result_type != IS_UNUSED) { + old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); + if (opline->opcode == ZEND_PRE_INC) { + SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); + SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX_PLUS_1); + } else if (opline->opcode == ZEND_PRE_DEC) { + SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); + SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN_MINUS_1); + } else if (opline->opcode == ZEND_POST_INC) { + SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); + SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX); + } else if (opline->opcode == ZEND_POST_DEC) { + SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); + SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN); + } + } + + exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + | bvs &exit_addr + + if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && + opline->result_type != IS_UNUSED) { + | ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + } + + SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); + if (opline->result_type != IS_UNUSED) { + SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); + } } else if (may_overflow) { | bvs >1 if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && @@ -4556,7 +4786,15 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o if (!exit_addr) { return 0; } - | NIY // tracing + if (op1_info & MAY_BE_ARRAY_PACKED) { + | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] + | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w + | beq &exit_addr + } else { + | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] + | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w + | bne &exit_addr + } } if (type == BP_VAR_W) { | // hval = Z_LVAL_P(dim); @@ -7887,7 +8125,16 @@ static int zend_jit_stack_check(dasm_State **Dst, const zend_op *opline, uint32_ return 0; } - | NIY // TODO + | // Check Stack Overflow + | MEM_LOAD_ZTS ldr, REG1, executor_globals, vm_stack_end, TMP1 + | MEM_LOAD_OP_ZTS sub, ldr, REG1, executor_globals, vm_stack_top, TMP1, TMP2 + || if (used_stack <= CMP_IMM) { + | cmp REG1, #used_stack + || } else { + | LOAD_32BIT_VAL TMP1w, used_stack + | cmp REG1, TMP1 + || } + | blo &exit_addr return 1; } @@ -8526,9 +8773,16 @@ static int zend_jit_init_method_call(dasm_State **Dst, (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || (func->common.fn_flags & ZEND_ACC_CLOSURE) || !func->common.function_name)) { - | NIY // tracing + const zend_op *opcodes = func->op_array.opcodes; + + | LOAD_ADDR TMP1, opcodes + | ldr TMP2, [REG0, #offsetof(zend_op_array, opcodes)] + | cmp TMP2, TMP1 + | bne &exit_addr } else { - | NIY // tracing + | LOAD_ADDR TMP1, func + | cmp REG0, TMP1 + | bne &exit_addr } } @@ -10190,7 +10444,37 @@ static int zend_jit_leave_func(dasm_State **Dst, | TST_32_WITH_CONST FCARG1w, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE), TMP1w if (trace && trace->op != ZEND_JIT_TRACE_END) { - | NIY // TODO: test + | bne >1 + |.cold_code + |1: + if (!GCC_GLOBAL_REGS) { + | mov FCARG2x, FP + } + | EXT_CALL zend_jit_leave_func_helper, REG0 + + if (may_be_top_frame) { + // TODO: try to avoid this check ??? + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { +#if 0 + /* this check should be handled by the following OPLINE guard */ + | LOAD_ADDR TMP1, zend_jit_halt_op + | cmp IP, TMP1 + | beq ->trace_halt +#endif + } else if (GCC_GLOBAL_REGS) { + | cbz IP, ->trace_halt + } else { + | tst RETVALx, RETVALx + | blt ->trace_halt + } + } + + if (!GCC_GLOBAL_REGS) { + | // execute_data = EG(current_execute_data) + | MEM_LOAD_ZTS ldr, FP, executor_globals, current_execute_data, TMP1 + } + | b >8 + |.code } else { | bne ->leave_function_handler } @@ -10251,8 +10535,39 @@ static int zend_jit_leave_func(dasm_State **Dst, if (trace->op == ZEND_JIT_TRACE_BACK && (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { + const zend_op *next_opline = trace->opline; - | NIY // TODO: test + if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) + && (op1_info & MAY_BE_RC1) + && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) { + /* exception might be thrown during destruction of unused return value */ + | // if (EG(exception)) + | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, REG0, TMP1 + | bne ->leave_throw_handler + } + do { + trace++; + } while (trace->op == ZEND_JIT_TRACE_INIT_CALL); + ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); + next_opline = trace->opline; + ZEND_ASSERT(next_opline != NULL); + + if (trace->op == ZEND_JIT_TRACE_END + && trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { + trace_info->flags |= ZEND_JIT_TRACE_LOOP; + | CMP_IP next_opline, TMP1, TMP2 + | beq =>0 // LOOP +#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE + | JMP_IP TMP1 +#else + | b ->trace_escape +#endif + } else { + | CMP_IP next_opline, TMP1, TMP2 + | bne ->trace_escape + } + + zend_jit_set_last_valid_opline(trace->opline); return 1; } else if (may_throw || @@ -10260,7 +10575,9 @@ static int zend_jit_leave_func(dasm_State **Dst, && (op1_info & MAY_BE_RC1) && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) && (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) { - | NIY // TODO: test + | // if (EG(exception)) + | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, REG0, TMP1 + | bne ->leave_throw_handler } return 1; @@ -10769,7 +11086,36 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, |8: if (res_exit_addr) { - | NIY // tracing + zend_uchar type = concrete_type(res_info); + if (op1_info & MAY_BE_ARRAY_OF_REF) { + | ZVAL_DEREF REG0, MAY_BE_REF, TMP1w + } + if (type < IS_STRING) { + | IF_NOT_ZVAL_TYPE val_addr, type, &res_exit_addr, TMP1w, TMP2 + } else { + | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 + | and TMP1w, REG2w, #0xff + | IF_NOT_TYPE TMP1w, type, &res_exit_addr + } + | // ZVAL_COPY + |7: + | ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0 + if (Z_MODE(res_addr) == IS_MEM_ZVAL) { + if (type < IS_STRING) { + if (Z_REG(res_addr) != ZREG_FP || + JIT_G(current_frame) == NULL || + STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) { + | SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2 + } + } else { + | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1 + if (!result_avoid_refcounting) { + | TRY_ADDREF res_info, REG2w, REG1, TMP1w + } + } + } else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) { + return 0; + } } else if (op1_info & MAY_BE_ARRAY_OF_REF) { | // ZVAL_COPY_DEREF | GET_ZVAL_TYPE_INFO Rw(ZREG_REG2), val_addr, TMP1 @@ -13814,7 +14160,7 @@ static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, ui } if (add_ref_guard) { - | NIY // TODO + | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr, TMP1w, TMP2 } if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) { /* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */ @@ -13823,7 +14169,7 @@ static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, ui } | EXT_CALL zend_jit_unref_helper, REG0 } else { - | NIY // GET_ZVAL_PTR FCARG1x, var_addr + | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, offsetof(zend_reference, val)); *var_addr_ptr = var_addr; } @@ -13834,7 +14180,7 @@ static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, ui if (add_type_guard && var_type != IS_UNKNOWN && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { - | NIY // TODO + | IF_NOT_ZVAL_TYPE var_addr, var_type, &exit_addr, TMP1w, TMP2 ZEND_ASSERT(var_info & (1 << var_type)); if (var_type < IS_STRING) { @@ -13868,7 +14214,8 @@ static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, if (!exit_addr) { return 0; } - | NIY // TODO + | IF_NOT_ZVAL_TYPE var_addr, IS_INDIRECT, &exit_addr, TMP1w, TMP2 + | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 } else { /* May be already loaded into FCARG1a or RAX by previus FETCH_OBJ_W/DIM_W */ if (opline->op1_type != IS_VAR || @@ -13876,9 +14223,9 @@ static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, (opline-1)->result.var != opline->op1.var || (opline-1)->op2_type == IS_VAR || (opline-1)->op2_type == IS_TMP_VAR) { - | NIY // GET_ZVAL_PTR FCARG1x, var_addr + | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 } else if ((opline-1)->opcode == ZEND_FETCH_DIM_W || (opline-1)->opcode == ZEND_FETCH_DIM_RW) { - | NIY // TODO + | mov FCARG1x, REG0 } } *var_info_ptr &= ~MAY_BE_INDIRECT; @@ -13898,7 +14245,7 @@ static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, return 0; } - | NIY // TODO + | IF_NOT_Z_TYPE FCARG1x, var_type, &exit_addr, TMP1w //var_info = zend_jit_trace_type_to_info_ex(var_type, var_info); ZEND_ASSERT(var_info & (1 << var_type)); From c114e49100e90d550fe232e3d629361a771b761b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 28 Apr 2021 22:48:27 +0300 Subject: [PATCH 173/229] Support for tracing JIT --- ext/opcache/jit/zend_jit_arm64.dasc | 334 +++++++++++++++++++++++++--- ext/opcache/jit/zend_jit_trace.c | 2 +- 2 files changed, 310 insertions(+), 26 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index ad97a6cfed6df..82573c34dae02 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -1711,7 +1711,15 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) | ldp x29, x30, [sp], #SPAD // stack alignment | EXT_JMP handler, REG0 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - | NIY_STUB // TODO: tracing + | mov FCARG1x, FP + | EXT_CALL handler, REG0 + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], #NR_SPAD // stack alignment + | tst RETVALw, RETVALw + | blt >1 + | mov RETVALw, #1 // ZEND_VM_ENTER + |1: + | ret } else { | mov FCARG1x, FP | ldp FP, RX, T2 // retore FP and IP @@ -2228,7 +2236,18 @@ static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst) static int zend_jit_trace_halt_stub(dasm_State **Dst) { |->trace_halt: - | NIY_STUB // TODO + if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { + | ADD_HYBRID_SPAD + | EXT_JMP zend_jit_halt_op->handler, REG0 + } else if (GCC_GLOBAL_REGS) { + | ldp x29, x30, [sp], #SPAD // stack alignment + | ret // PC must be zero + } else { + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], #NR_SPAD // stack alignment + | sub RETVALx, xzr, #1 // ZEND_VM_RETURN (-1) + | ret + } return 1; } @@ -2908,7 +2927,7 @@ static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size delta = (uint32_t*)from_addr - ins_ptr; if (((ins ^ (uint32_t)delta) & 0x01ffffffu) == 0) { delta = (uint32_t*)to_addr - ins_ptr; - if (((delta + 0x02000000) >> 26) == 0) { + if (((delta + 0x02000000) >> 26) != 0) { abort(); // brnach target out of range } *ins_ptr = (ins & 0xfc000000u) | ((uint32_t)delta & 0x03ffffffu); @@ -2920,7 +2939,7 @@ static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size delta = (uint32_t*)from_addr - ins_ptr; if (((ins ^ ((uint32_t)delta << 5)) & 0x00ffffe0u) == 0) { delta = (uint32_t*)to_addr - ins_ptr; - if (((delta + 0x80000) >> 19) == 0) { + if (((delta + 0x40000) >> 19) != 0) { abort(); // brnach target out of range } *ins_ptr = (ins & 0xff00001fu) | (((uint32_t)delta & 0x7ffffu) << 5); @@ -2931,7 +2950,7 @@ static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size delta = (uint32_t*)from_addr - ins_ptr; if (((ins ^ ((uint32_t)delta << 5)) & 0x0007ffe0u) == 0) { delta = (uint32_t*)to_addr - ins_ptr; - if (((delta + 0x2000) >> 14) == 0) { + if (((delta + 0x2000) >> 14) != 0) { abort(); // brnach target out of range } *ins_ptr = (ins & 0xfff8001fu) | (((uint32_t)delta & 0x3fffu) << 5); @@ -3000,7 +3019,6 @@ static int zend_jit_trace_return(dasm_State **Dst, bool original_handler) if (!original_handler) { | JMP_IP TMP1 } else { - | NIY // TODO: test | ldr REG0, EX->func | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] @@ -3013,7 +3031,6 @@ static int zend_jit_trace_return(dasm_State **Dst, bool original_handler) if (!original_handler) { | JMP_IP TMP1 } else { - | NIY // TODO: test | ldr REG0, EX->func | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] @@ -3023,7 +3040,6 @@ static int zend_jit_trace_return(dasm_State **Dst, bool original_handler) } } else { if (original_handler) { - | NIY // TODO: test | mov FCARG1x, FP | ldr REG0, EX->func | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] @@ -3137,7 +3153,7 @@ static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_arra } else if (GCC_GLOBAL_REGS) { | cbz IP, ->trace_halt } else { - | tst RETVALx, RETVALx + | tst RETVALw, RETVALw | blt ->trace_halt } } else if (opline->opcode == ZEND_EXIT || @@ -3315,7 +3331,8 @@ static int zend_jit_trace_opline_guard(dasm_State **Dst, const zend_op *opline) if (!exit_addr) { return 0; } - | NIY // TODO + | CMP_IP opline, TMP1, TMP2 + | bne &exit_addr zend_jit_set_last_valid_opline(opline); @@ -3476,20 +3493,72 @@ static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr sr static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags, const zend_op *opline) { - | NIY // TODO + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + + | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >1, TMP1w, TMP2 + + if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { + if (!zend_jit_save_call_chain(Dst, -1)) { + return 0; + } + } + + ZEND_ASSERT(opline); + + | LOAD_IP_ADDR (opline - 1) + | b ->trace_escape + |1: return 1; } static int zend_jit_store_const(dasm_State **Dst, int var, zend_reg reg) { - | NIY // TODO + zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); + + if (reg == ZREG_LONG_MIN_MINUS_1) { + uint64_t val = 0xc3e0000000000000; + | SET_ZVAL_LVAL dst, val, TMP1, TMP2 + | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2 + } else if (reg == ZREG_LONG_MIN) { + uint64_t val = 0x8000000000000000; + | SET_ZVAL_LVAL dst, val, TMP1, TMP2 + | SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2 + } else if (reg == ZREG_LONG_MAX) { + uint64_t val = 0x7fffffffffffffff; + | SET_ZVAL_LVAL dst, val, TMP1, TMP2 + | SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2 + } else if (reg == ZREG_LONG_MAX_PLUS_1) { + uint64_t val = 0x43e0000000000000; + | SET_ZVAL_LVAL dst, val, TMP1, TMP2 + | SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2 + } else if (reg == ZREG_NULL) { + | SET_ZVAL_TYPE_INFO dst, IS_NULL, TMP1w, TMP2 + } else if (reg == ZREG_ZVAL_TRY_ADDREF) { + | IF_NOT_ZVAL_REFCOUNTED dst, >1, ZREG_TMP1, ZREG_TMP2 + | GET_ZVAL_PTR TMP1, dst, TMP2 + | GC_ADDREF TMP1, TMP2w + |1: + } else if (reg == ZREG_ZVAL_COPY_GPR0) { + zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); + + | ZVAL_COPY_VALUE dst, -1, val_addr, -1, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 + | TRY_ADDREF -1, REG1w, REG2, TMP1w + } else { + ZEND_UNREACHABLE(); + } return 1; } static int zend_jit_free_trampoline(dasm_State **Dst) { - | NIY // TODO + | // if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) + | ldr TMP1w, [REG0, #offsetof(zend_function, common.fn_flags)] + | tst TMP1w, #ZEND_ACC_CALL_VIA_TRAMPOLINE + | beq >1 + | mov FCARG1x, REG0 + | EXT_CALL zend_jit_free_trampoline_helper, REG0 + |1: return 1; } @@ -8315,7 +8384,43 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con | // Z_CE(call->This) = called_scope; | str xzr, EX:RX->This.value.ptr } else { - | NIY // TODO + if (opline->op2_type == IS_CV) { + | // GC_ADDREF(closure); + | ldr TMP1w, [REG0] + | add TMP1w, TMP1w, #1 + | str TMP1w, [REG0] + } + | // object_or_called_scope = closure->called_scope; + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG1, REG0, offsetof(zend_closure, called_scope), TMP2 + | str REG1, EX:RX->This.value.ptr + | // call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE | + | // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE); + | ldr REG2w, [REG0, #offsetof(zend_closure, func.common.fn_flags)] + | and REG2w, REG2w, #ZEND_ACC_FAKE_CLOSURE + | LOAD_32BIT_VAL TMP1w, (ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE) + | orr REG2w, REG2w, TMP1w + | // if (Z_TYPE(closure->this_ptr) != IS_UNDEF) { + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, TMP1w, REG0, offsetof(zend_closure, this_ptr.u1.v.type), TMP2 + | cmp TMP1w, #IS_UNDEF + | beq >1 + | // call_info |= ZEND_CALL_HAS_THIS; + | LOAD_32BIT_VAL TMP1w, ZEND_CALL_HAS_THIS + | orr REG2w, REG2w, TMP1w + | // object_or_called_scope = Z_OBJ(closure->this_ptr); + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG1, REG0, offsetof(zend_closure, this_ptr.value.ptr), TMP2 + |1: + | // ZEND_SET_CALL_INFO(call, 0, call_info); + | ldr TMP1w, EX:RX->This.u1.type_info + | orr TMP1w, TMP1w, REG2w + | str TMP1w, EX:RX->This.u1.type_info + | // Z_PTR(call->This) = object_or_called_scope; + | str REG1, EX:RX->This.value.ptr + | ldr TMP1, [REG0, #offsetof(zend_closure, func.op_array.run_time_cache__ptr)] + | cmp TMP1, xzr + | bne >1 + | add FCARG1x, REG0, #offsetof(zend_closure, func) + | EXT_CALL zend_jit_init_func_run_time_cache_helper, REG0 + |1: } | // ZEND_CALL_NUM_ARGS(call) = num_args; | LOAD_32BIT_VAL TMP1w, opline->extended_value @@ -8506,7 +8611,54 @@ static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, cons static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zend_function *func, const zend_op *to_opline) { - | NIY // TODO + int32_t exit_point; + const void *exit_addr; + + if (func->type == ZEND_INTERNAL_FUNCTION) { +#ifdef ZEND_WIN32 + // TODO: ASLR may cause different addresses in different workers ??? + return 0; +#endif + } else if (func->type == ZEND_USER_FUNCTION) { + if (!zend_accel_in_shm(func->op_array.opcodes)) { + /* op_array and op_array->opcodes are not persistent. We can't link. */ + return 0; + } + } else { + ZEND_UNREACHABLE(); + return 0; + } + + exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + + | // call = EX(call); + | ldr REG1, EX->call + while (level > 0) { + | ldr REG1, EX:REG1->prev_execute_data + level--; + } + + if (func->type == ZEND_USER_FUNCTION && + (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || + (func->common.fn_flags & ZEND_ACC_CLOSURE) || + !func->common.function_name)) { + const zend_op *opcodes = func->op_array.opcodes; + + | ldr REG1, EX:REG1->func + | LOAD_ADDR REG2, ((ptrdiff_t)opcodes) + | ldr TMP1, [REG1, #offsetof(zend_op_array, opcodes)] + | cmp TMP1, REG2 + | bne &exit_addr + } else { + | LOAD_ADDR REG2, ((ptrdiff_t)func) + | ldr TMP1, EX:REG1->func + | cmp TMP1, REG2 + | bne &exit_addr + } return 1; } @@ -8585,7 +8737,29 @@ static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t | mov REG0, RETVALx | str REG0, [REG1, #opline->result.num] if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - | NIY // TODO. tracing mode. + int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + + if (!func || opline->opcode == ZEND_INIT_FCALL) { + | cbnz REG0, >3 + } else if (func->type == ZEND_USER_FUNCTION + && !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) { + const zend_op *opcodes = func->op_array.opcodes; + + | LOAD_ADDR REG1, ((ptrdiff_t)opcodes) + | ldr TMP1, [REG0, #offsetof(zend_op_array, opcodes)] + | cmp TMP1, REG1 + | beq >3 + } else { + | LOAD_ADDR REG1, ((ptrdiff_t)func) + | cmp REG0, REG1 + | beq >3 + } + | b &exit_addr } else { | cbnz REG0, >3 | // SAVE_OPLINE(); @@ -8848,7 +9022,80 @@ static int zend_jit_init_closure_call(dasm_State **Dst, zend_jit_trace_rec *trace, bool stack_check) { - | NIY // TODO + zend_function *func = NULL; + zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); + + | GET_ZVAL_PTR REG0, op2_addr, TMP1 + + if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure + && !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) { + int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + + if (!exit_addr) { + return 0; + } + + | LOAD_ADDR FCARG1x, ((ptrdiff_t)zend_ce_closure) + | ldr, TMP1, [REG0, #offsetof(zend_object, ce)] + | cmp TMP1, FCARG1x + | bne &exit_addr + if (ssa->var_info && ssa_op->op2_use >= 0) { + ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD; + ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure; + ssa->var_info[ssa_op->op2_use].is_instanceof = 0; + } + } + + if (trace + && trace->op == ZEND_JIT_TRACE_INIT_CALL + && trace->func + && trace->func->type == ZEND_USER_FUNCTION) { + const zend_op *opcodes; + int32_t exit_point; + const void *exit_addr; + + func = (zend_function*)trace->func; + opcodes = func->op_array.opcodes; + exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL); + exit_addr = zend_jit_trace_get_exit_addr(exit_point); + if (!exit_addr) { + return 0; + } + + | LOAD_ADDR FCARG1x, ((ptrdiff_t)opcodes) + | ldr TMP1, [REG0, #offsetof(zend_closure, func.op_array.opcodes)] + | cmp TMP1, FCARG1x + | bne &exit_addr + } + + if (delayed_call_chain) { + if (!zend_jit_save_call_chain(Dst, delayed_call_level)) { + return 0; + } + } + + if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 1, 0, stack_check)) { + return 0; + } + + if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, trace)) { + if (!zend_jit_save_call_chain(Dst, call_level)) { + return 0; + } + } else { + delayed_call_chain = 1; + delayed_call_level = call_level; + } + + if (trace + && trace->op == ZEND_JIT_TRACE_END + && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { + if (!zend_jit_set_valid_ip(Dst, opline + 1)) { + return 0; + } + } + return 1; } @@ -9868,7 +10115,37 @@ static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { - | NIY // tracing + if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { + if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) { + TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call); + | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); + || if (reuse_ip) { + | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] + | orr TMP1w, TMP1w, #ZEND_CALL_SEND_ARG_BY_REF + | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] + || } else { + | ldr REG0, EX->call + | ldr TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] + | orr TMP1w, TMP1w, #ZEND_CALL_SEND_ARG_BY_REF + | str TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] + || } + } + } else { + if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { + TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call); + | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); + || if (reuse_ip) { + | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] + | and TMP1w, TMP1w, #(~ZEND_CALL_SEND_ARG_BY_REF) + | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] + || } else { + | ldr REG0, EX->call + | ldr TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] + | and TMP1w, TMP1w, #(~ZEND_CALL_SEND_ARG_BY_REF) + | str TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] + || } + } + } } else { // if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) { uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); @@ -10394,7 +10671,8 @@ static int zend_jit_free_cv(dasm_State **Dst, uint32_t info, uint32_t var) static int zend_jit_free_op(dasm_State **Dst, const zend_op *opline, uint32_t info, uint32_t var_offset) { if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { - | NIY // TODO + zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset); + | ZVAL_PTR_DTOR addr, info, 0, 1, opline, ZREG_TMP1, ZREG_TMP2 } return 1; } @@ -10464,7 +10742,7 @@ static int zend_jit_leave_func(dasm_State **Dst, } else if (GCC_GLOBAL_REGS) { | cbz IP, ->trace_halt } else { - | tst RETVALx, RETVALx + | tst RETVALw, RETVALw | blt ->trace_halt } } @@ -11697,7 +11975,7 @@ static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_ if (!exit_addr) { return 0; } - | NIY // TODO + | blt &exit_addr } else { | blt >1 |.cold_code @@ -11890,7 +12168,11 @@ static int zend_jit_class_guard(dasm_State **Dst, const zend_op *opline, zend_cl return 0; } - | NIY // tracing + | LOAD_ADDR TMP1, ((ptrdiff_t)ce) + | ldr TMP2, [FCARG1x, #offsetof(zend_object, ce)] + | cmp TMP2, TMP1 + | bne &exit_addr + return 1; } @@ -13534,7 +13816,8 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o } | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 if (fallback_label) { - | NIY // IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, &fallback_label + | add TMP1, FCARG2x, #offsetof(zend_reference, val) + | IF_NOT_Z_TYPE TMP1, IS_LONG, &fallback_label, TMP2w } else { | add TMP1, FCARG2x, #offsetof(zend_reference, val) | IF_NOT_Z_TYPE TMP1, IS_LONG, >3, TMP2w @@ -13630,7 +13913,8 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o } | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 if (fallback_label) { - | NIY // IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, &fallback_label + | add TMP1, FCARG2x, #offsetof(zend_reference, val) + | IF_NOT_Z_TYPE TMP1, IS_STRING, &fallback_label, TMP2w } else { | add TMP1, FCARG2x, #offsetof(zend_reference, val) | IF_NOT_Z_TYPE TMP1, IS_STRING, >3, TMP2w @@ -14139,7 +14423,7 @@ static bool zend_jit_noref_guard(dasm_State **Dst, const zend_op *opline, zend_j if (!exit_addr) { return 0; } - | NIY // IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr + | IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr, TMP1w, TMP2 return 1; } diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index d52479056ffc2..1cc2a6aea6f41 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -7519,7 +7519,7 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf } } if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) { - zend_function *func = (zend_function*)regs->gpr[0]; + zend_function *func = (zend_function*)regs->gpr[ZREG_REG0]; if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) { zend_string_release_ex(func->common.function_name, 0); From f5d844b9cb8fde955308a4485cdffa2c8380afb1 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 28 Apr 2021 23:22:46 +0300 Subject: [PATCH 174/229] Fixed register allocation. This fixes ext/standard/tests/math/acos_basic.phpt with tracing JIT. --- ext/opcache/jit/zend_jit_arm64.dasc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 82573c34dae02..ebda65e479a04 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -4048,7 +4048,7 @@ static int zend_jit_math_long_double(dasm_State **Dst, if (Z_MODE(op2_addr) == IS_REG) { op2_reg = Z_REG(op2_addr); } else { - op2_reg = ZREG_FPR1; + op2_reg = ZREG_FPTMP; | GET_ZVAL_DVAL op2_reg, op2_addr, ZREG_TMP1 } @@ -4085,7 +4085,7 @@ static int zend_jit_math_double_long(dasm_State **Dst, if (Z_MODE(op1_addr) == IS_REG) { op1_reg = Z_REG(op1_addr); } else { - op1_reg = ZREG_FPR1; + op1_reg = ZREG_FPTMP; | GET_ZVAL_DVAL op1_reg, op1_addr, ZREG_TMP1 } | DOUBLE_MATH_REG opcode, result_reg, result_reg, op1_reg @@ -4109,7 +4109,7 @@ static int zend_jit_math_double_long(dasm_State **Dst, && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { /* +/- 0 */ } else { - op2_reg = ZREG_FPR1; + op2_reg = ZREG_FPTMP; | DOUBLE_GET_ZVAL_LVAL op2_reg, op2_addr, ZREG_TMP1, ZREG_TMP2 | DOUBLE_MATH_REG opcode, result_reg, op1_reg, op2_reg } From 412c172ac6a612e5b28e3211c0bf2fe952c06e65 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 29 Apr 2021 00:57:43 +0300 Subject: [PATCH 175/229] Attempt to fix Windows build --- ext/opcache/jit/zend_jit.c | 6 +++--- ext/opcache/jit/zend_jit_disasm.c | 4 ++-- ext/opcache/jit/zend_jit_vm_helpers.c | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 63f79d85ce65b..6caabba0dc2f8 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -39,7 +39,7 @@ #include "Optimizer/zend_call_graph.h" #include "Optimizer/zend_dump.h" -#if defined(__x86_64__) || defined(i386) +#if defined(__x86_64__) || defined(i386) || defined(ZEND_WIN32) #include "jit/zend_jit_x86.h" #elif defined (__aarch64__) #include "jit/zend_jit_arm64.h" @@ -206,7 +206,7 @@ static bool zend_is_commutative(zend_uchar opcode) #define OP2_RANGE() OP_RANGE(ssa_op, op2) #define OP1_DATA_RANGE() OP_RANGE(ssa_op + 1, op1) -#if defined(__x86_64__) || defined(i386) +#if defined(__x86_64__) || defined(i386) || defined(ZEND_WIN32) #include "dynasm/dasm_x86.h" #elif defined(__aarch64__) #include "dynasm/dasm_arm64.h" @@ -224,7 +224,7 @@ static bool zend_is_commutative(zend_uchar opcode) # include "jit/zend_jit_oprofile.c" #endif -#if defined(__x86_64__) || defined(i386) +#if defined(__x86_64__) || defined(i386) || defined(ZEND_WIN32) #include "jit/zend_jit_vtune.c" #include "jit/zend_jit_x86.c" #elif defined(__aarch64__) diff --git a/ext/opcache/jit/zend_jit_disasm.c b/ext/opcache/jit/zend_jit_disasm.c index 371956173a223..eabcc0e7dc05c 100644 --- a/ext/opcache/jit/zend_jit_disasm.c +++ b/ext/opcache/jit/zend_jit_disasm.c @@ -225,7 +225,7 @@ static uint64_t zend_jit_disasm_branch_target(csh cs, const cs_insn *insn) { unsigned int i; -#if defined(__x86_64__) || defined(i386) +#if defined(__x86_64__) || defined(i386) || defined(ZEND_WIN32) if (cs_insn_group(cs, insn, X86_GRP_JUMP)) { for (i = 0; i < insn->detail->x86.op_count; i++) { if (insn->detail->x86.operands[i].type == X86_OP_IMM) { @@ -319,7 +319,7 @@ static int zend_jit_disasm(const char *name, #endif #ifdef HAVE_CAPSTONE -# if defined(__x86_64__) || defined(_WIN64) +# if defined(__x86_64__) || defined(_WIN64) || defined(ZEND_WIN32) if (cs_open(CS_ARCH_X86, CS_MODE_64, &cs) != CS_ERR_OK) return 0; cs_option(cs, CS_OPT_DETAIL, CS_OPT_ON); diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index c808f7fcd55e4..bb92d2b687147 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -28,7 +28,7 @@ #include "Optimizer/zend_func_info.h" #include "Optimizer/zend_call_graph.h" #include "zend_jit.h" -#if defined(__x86_64__) || defined(i386) +#if defined(__x86_64__) || defined(i386) || defined(ZEND_WIN32) #include "zend_jit_x86.h" #elif defined(__aarch64__) #include "zend_jit_arm64.h" From 2b9c2886c5c7864d0dac00f13a42c5e7cf4d7e1c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 29 Apr 2021 01:08:21 +0300 Subject: [PATCH 176/229] ZREG_REG0 is not available in x86 build --- ext/opcache/jit/zend_jit_trace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 1cc2a6aea6f41..2b379e2429ea1 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -7519,7 +7519,7 @@ int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf } } if (t->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) { - zend_function *func = (zend_function*)regs->gpr[ZREG_REG0]; + zend_function *func = (zend_function*)regs->gpr[ZREG_COPY]; if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) { zend_string_release_ex(func->common.function_name, 0); From 20e33e2ab4f881d9c768848e8b6eb451cfcc6585 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 29 Apr 2021 09:40:23 +0300 Subject: [PATCH 177/229] Use TST_32_WITH_CONST macro --- ext/opcache/jit/zend_jit_arm64.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index ebda65e479a04..ea9450be5663f 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -3554,7 +3554,7 @@ static int zend_jit_free_trampoline(dasm_State **Dst) { | // if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) | ldr TMP1w, [REG0, #offsetof(zend_function, common.fn_flags)] - | tst TMP1w, #ZEND_ACC_CALL_VIA_TRAMPOLINE + | TST_32_WITH_CONST TMP1w, ZEND_ACC_CALL_VIA_TRAMPOLINE, TMP2w | beq >1 | mov FCARG1x, REG0 | EXT_CALL zend_jit_free_trampoline_helper, REG0 From 74beb3122b172496fcc8c201030eb00bd4716719 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 29 Apr 2021 09:53:32 +0300 Subject: [PATCH 178/229] Remove useless SAFE_MEM_ACC_WITH_UOFFSET* --- ext/opcache/jit/zend_jit_arm64.dasc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index ea9450be5663f..60f6308c66b6f 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -8391,7 +8391,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con | str TMP1w, [REG0] } | // object_or_called_scope = closure->called_scope; - | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG1, REG0, offsetof(zend_closure, called_scope), TMP2 + | ldr REG1, [REG0, #offsetof(zend_closure, called_scope)] | str REG1, EX:RX->This.value.ptr | // call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE | | // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE); @@ -8400,14 +8400,14 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con | LOAD_32BIT_VAL TMP1w, (ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE) | orr REG2w, REG2w, TMP1w | // if (Z_TYPE(closure->this_ptr) != IS_UNDEF) { - | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, TMP1w, REG0, offsetof(zend_closure, this_ptr.u1.v.type), TMP2 + | ldrb TMP1w, [REG0, #offsetof(zend_closure, this_ptr.u1.v.type)] | cmp TMP1w, #IS_UNDEF | beq >1 | // call_info |= ZEND_CALL_HAS_THIS; | LOAD_32BIT_VAL TMP1w, ZEND_CALL_HAS_THIS | orr REG2w, REG2w, TMP1w | // object_or_called_scope = Z_OBJ(closure->this_ptr); - | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG1, REG0, offsetof(zend_closure, this_ptr.value.ptr), TMP2 + | ldr REG1, [REG0, #offsetof(zend_closure, this_ptr.value.ptr)] |1: | // ZEND_SET_CALL_INFO(call, 0, call_info); | ldr TMP1w, EX:RX->This.u1.type_info From 03466cf55f482233ef9974e3e02d9de8e5786a31 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 29 Apr 2021 10:08:06 +0300 Subject: [PATCH 179/229] Fixed signed/unsigned comparison mess --- ext/opcache/jit/zend_jit_arm64.dasc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 60f6308c66b6f..55fc9a29ae741 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -11975,9 +11975,9 @@ static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_ if (!exit_addr) { return 0; } - | blt &exit_addr + | blo &exit_addr } else { - | blt >1 + | blo >1 |.cold_code |1: if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { From 2bbeb7b0f57fa9d6732740fbb8df757ce97b1bc3 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Thu, 29 Apr 2021 04:32:23 +0000 Subject: [PATCH 180/229] Revisit 32-bit logical operations with constants Add macro "BW_OP_32_WITH_CONST" for safe 32-bit logical operations with constants. Add macro "GET_LOW_8BITS" for 'and' operation with "0xff". Change-Id: I57968069b43ac4c02d876e9290433a74b9d87742 --- ext/opcache/jit/zend_jit_arm64.dasc | 61 +++++++++++++++++------------ 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 55fc9a29ae741..f587bb1d17dec 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -227,6 +227,23 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) || } |.endmacro +// Extract the low 8 bits from 'src_reg' into 'dst_reg'. 0xff can be encoded as imm for 'and' instruction. +|.macro GET_LOW_8BITS, dst_reg, src_reg +| and dst_reg, src_reg, #0xff +|.endmacro + +// Bitwise operation with constants. 'ins' can be and/orr/eor/ands. Operands are 32-bit. +|.macro BW_OP_32_WITH_CONST, ins, reg, op, val, tmp_reg +|| if (val == 0) { +| ins reg, op, wzr +|| } else if (logical_immediate_p((uint32_t)val, 32)) { +| ins reg, op, #val +|| } else { +| LOAD_32BIT_VAL tmp_reg, val +| ins reg, op, tmp_reg +|| } +|.endmacro + // Bitwise operation with constants. 'ins' can be and/orr/eor/ands. Operands are 64-bit. |.macro BW_OP_64_WITH_CONST, ins, reg, op, val, tmp_reg || if (val == 0) { @@ -8375,8 +8392,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con | str TMP1w, EX:RX->This.u1.type_info } else { | ldr TMP1w, EX:RX->This.u1.type_info - | LOAD_32BIT_VAL TMP2w, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS) - | orr TMP1w, TMP1w, TMP2w + | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS), TMP2w | str TMP1w, EX:RX->This.u1.type_info } } @@ -8396,16 +8412,14 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con | // call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE | | // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE); | ldr REG2w, [REG0, #offsetof(zend_closure, func.common.fn_flags)] - | and REG2w, REG2w, #ZEND_ACC_FAKE_CLOSURE - | LOAD_32BIT_VAL TMP1w, (ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE) - | orr REG2w, REG2w, TMP1w + | BW_OP_32_WITH_CONST and, REG2w, REG2w, ZEND_ACC_FAKE_CLOSURE, TMP1w + | BW_OP_32_WITH_CONST orr, REG2w, REG2w, (ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE), TMP1w | // if (Z_TYPE(closure->this_ptr) != IS_UNDEF) { | ldrb TMP1w, [REG0, #offsetof(zend_closure, this_ptr.u1.v.type)] | cmp TMP1w, #IS_UNDEF | beq >1 | // call_info |= ZEND_CALL_HAS_THIS; - | LOAD_32BIT_VAL TMP1w, ZEND_CALL_HAS_THIS - | orr REG2w, REG2w, TMP1w + | BW_OP_32_WITH_CONST orr, REG2w, REG2w, ZEND_CALL_HAS_THIS, TMP1w | // object_or_called_scope = Z_OBJ(closure->this_ptr); | ldr REG1, [REG0, #offsetof(zend_closure, this_ptr.value.ptr)] |1: @@ -9267,7 +9281,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | mov FCARG1x, RX } | EXT_CALL zend_jit_deprecated_helper, REG0 - | and RETVALw, RETVALw, #0xff + | GET_LOW_8BITS RETVALw, RETVALw | cmp RETVALw, #0 // Result is 0 on exception | ldr REG0, EX:RX->func // reload | bne >1 @@ -9561,7 +9575,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | mov FCARG1x, RX } | EXT_CALL zend_jit_deprecated_helper, REG0 - | and RETVALw, RETVALw, #0xff + | GET_LOW_8BITS RETVALw, RETVALw | cmp RETVALw, #0 // Result is 0 on exception | ldr REG0, EX:RX->func // reload | bne >1 @@ -9940,7 +9954,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend if (!exit_addr) { return 0; } - | and TMP1w, REG1w, #0xff + | GET_LOW_8BITS TMP1w, REG1w | cmp TMP1w, #IS_REFERENCE | bne &exit_addr } @@ -9961,7 +9975,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 if (op1_info & MAY_BE_REF) { - | and TMP1w, REG1w, #0xff + | GET_LOW_8BITS TMP1w, REG1w | cmp TMP1w, #IS_REFERENCE | beq >7 } @@ -10037,7 +10051,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend if (opline->opcode == ZEND_SEND_VAR_NO_REF) { | ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 if (op1_info & MAY_BE_REF) { - | and TMP1w, REG1w, #0xff + | GET_LOW_8BITS TMP1w, REG1w | cmp TMP1w, #IS_REFERENCE | beq >7 } @@ -10121,12 +10135,12 @@ static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); || if (reuse_ip) { | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] - | orr TMP1w, TMP1w, #ZEND_CALL_SEND_ARG_BY_REF + | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] || } else { | ldr REG0, EX->call | ldr TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] - | orr TMP1w, TMP1w, #ZEND_CALL_SEND_ARG_BY_REF + | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w | str TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] || } } @@ -10136,12 +10150,12 @@ static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); || if (reuse_ip) { | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] - | and TMP1w, TMP1w, #(~ZEND_CALL_SEND_ARG_BY_REF) + | BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~ZEND_CALL_SEND_ARG_BY_REF), TMP2w | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] || } else { | ldr REG0, EX->call | ldr TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] - | and TMP1w, TMP1w, #(~ZEND_CALL_SEND_ARG_BY_REF) + | BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~ZEND_CALL_SEND_ARG_BY_REF), TMP2w | str TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)] || } } @@ -10162,8 +10176,7 @@ static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) |1: | // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] - | LOAD_32BIT_VAL TMP2w, ZEND_CALL_SEND_ARG_BY_REF - | orr TMP1w, TMP1w, TMP2w + | BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] | b >1 |.code @@ -11078,7 +11091,7 @@ static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, ze | GET_ZVAL_PTR REG1, val_addr, TMP1 | IF_NOT_REFCOUNTED REG2w, >2, TMP1w - | and TMP2w, REG2w, #0xff // TMP2w -> low 8 bits of REG2w + | GET_LOW_8BITS TMP2w, REG2w | IF_NOT_TYPE TMP2w, IS_REFERENCE, >1 | add REG1, REG1, #offsetof(zend_reference, val) | GET_Z_TYPE_INFO REG2w, REG1 @@ -11372,7 +11385,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, | IF_NOT_ZVAL_TYPE val_addr, type, &res_exit_addr, TMP1w, TMP2 } else { | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 - | and TMP1w, REG2w, #0xff + | GET_LOW_8BITS TMP1w, REG2w | IF_NOT_TYPE TMP1w, type, &res_exit_addr } | // ZVAL_COPY @@ -11929,7 +11942,7 @@ static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zen | mov REG0w, RETVALw if (check_exception) { - | and REG0w, REG0w, #0xff + | GET_LOW_8BITS REG0w, REG0w | tst REG0w, REG0w if (in_cold) { | bne >1 @@ -12346,7 +12359,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, |.code } } else if (flags == ZEND_FETCH_REF) { - | and TMP1w, REG2w, #0xff + | GET_LOW_8BITS TMP1w, REG2w | IF_TYPE TMP1w, IS_REFERENCE, >1 if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { | LOAD_ADDR FCARG2x, prop_info @@ -12428,7 +12441,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, type = concrete_type(res_info); | // ZVAL_DEREF() - | and TMP1w, REG2w, #0xff + | GET_LOW_8BITS TMP1w, REG2w | IF_NOT_TYPE TMP1w, IS_REFERENCE, >1 | GET_Z_PTR REG0, REG0 | add REG0, REG0, #offsetof(zend_reference, val) @@ -12438,7 +12451,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, } else { | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 |1: - | and TMP1w, REG2w, #0xff + | GET_LOW_8BITS TMP1w, REG2w | IF_NOT_TYPE TMP1w, type, &exit_addr } | // ZVAL_COPY From 34f1fdc0eef022c212ac781357daaea9d4847da0 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 29 Apr 2021 11:10:58 +0300 Subject: [PATCH 181/229] Moved tests from ext/opcache/tests/jit/arm64 to ext/opcache/tests/jit --- ext/opcache/tests/jit/{arm64 => }/add_001.phpt | 0 ext/opcache/tests/jit/{arm64 => }/add_002.phpt | 0 ext/opcache/tests/jit/{arm64 => }/add_003.phpt | 0 ext/opcache/tests/jit/{arm64 => }/add_004.phpt | 0 ext/opcache/tests/jit/{arm64 => }/add_005.phpt | 0 ext/opcache/tests/jit/{arm64 => }/add_006.phpt | 0 ext/opcache/tests/jit/{arm64 => }/hot_func_001.phpt | 0 ext/opcache/tests/jit/{arm64 => }/hot_func_002.phpt | 0 ext/opcache/tests/jit/{arm64 => }/icall_001.phpt | 0 ext/opcache/tests/jit/{arm64 => }/loop_001.phpt | 0 ext/opcache/tests/jit/{arm64 => }/loop_002.phpt | 0 ext/opcache/tests/jit/{arm64 => }/mul_001.phpt | 0 ext/opcache/tests/jit/{arm64 => }/mul_002.phpt | 0 ext/opcache/tests/jit/{arm64 => }/mul_003.phpt | 0 ext/opcache/tests/jit/{arm64 => }/mul_004.phpt | 0 ext/opcache/tests/jit/{arm64/recv_001.phpt => recv_005.phpt} | 2 +- ext/opcache/tests/jit/{arm64 => }/ret_001.phpt | 0 ext/opcache/tests/jit/{arm64 => }/ret_002.phpt | 0 ext/opcache/tests/jit/{arm64 => }/ret_003.phpt | 0 ext/opcache/tests/jit/{arm64 => }/sub_001.phpt | 0 ext/opcache/tests/jit/{arm64 => }/ucall_001.phpt | 0 ext/opcache/tests/jit/{arm64 => }/ucall_002.phpt | 0 ext/opcache/tests/jit/{arm64 => }/ucall_003.phpt | 0 ext/opcache/tests/jit/{arm64 => }/ucall_004.phpt | 0 ext/opcache/tests/jit/{arm64 => }/xor_001.phpt | 0 ext/opcache/tests/jit/{arm64 => }/xor_002.phpt | 0 ext/opcache/tests/jit/{arm64 => }/xor_003.phpt | 0 27 files changed, 1 insertion(+), 1 deletion(-) rename ext/opcache/tests/jit/{arm64 => }/add_001.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/add_002.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/add_003.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/add_004.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/add_005.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/add_006.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/hot_func_001.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/hot_func_002.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/icall_001.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/loop_001.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/loop_002.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/mul_001.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/mul_002.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/mul_003.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/mul_004.phpt (100%) rename ext/opcache/tests/jit/{arm64/recv_001.phpt => recv_005.phpt} (95%) rename ext/opcache/tests/jit/{arm64 => }/ret_001.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/ret_002.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/ret_003.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/sub_001.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/ucall_001.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/ucall_002.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/ucall_003.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/ucall_004.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/xor_001.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/xor_002.phpt (100%) rename ext/opcache/tests/jit/{arm64 => }/xor_003.phpt (100%) diff --git a/ext/opcache/tests/jit/arm64/add_001.phpt b/ext/opcache/tests/jit/add_001.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/add_001.phpt rename to ext/opcache/tests/jit/add_001.phpt diff --git a/ext/opcache/tests/jit/arm64/add_002.phpt b/ext/opcache/tests/jit/add_002.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/add_002.phpt rename to ext/opcache/tests/jit/add_002.phpt diff --git a/ext/opcache/tests/jit/arm64/add_003.phpt b/ext/opcache/tests/jit/add_003.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/add_003.phpt rename to ext/opcache/tests/jit/add_003.phpt diff --git a/ext/opcache/tests/jit/arm64/add_004.phpt b/ext/opcache/tests/jit/add_004.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/add_004.phpt rename to ext/opcache/tests/jit/add_004.phpt diff --git a/ext/opcache/tests/jit/arm64/add_005.phpt b/ext/opcache/tests/jit/add_005.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/add_005.phpt rename to ext/opcache/tests/jit/add_005.phpt diff --git a/ext/opcache/tests/jit/arm64/add_006.phpt b/ext/opcache/tests/jit/add_006.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/add_006.phpt rename to ext/opcache/tests/jit/add_006.phpt diff --git a/ext/opcache/tests/jit/arm64/hot_func_001.phpt b/ext/opcache/tests/jit/hot_func_001.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/hot_func_001.phpt rename to ext/opcache/tests/jit/hot_func_001.phpt diff --git a/ext/opcache/tests/jit/arm64/hot_func_002.phpt b/ext/opcache/tests/jit/hot_func_002.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/hot_func_002.phpt rename to ext/opcache/tests/jit/hot_func_002.phpt diff --git a/ext/opcache/tests/jit/arm64/icall_001.phpt b/ext/opcache/tests/jit/icall_001.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/icall_001.phpt rename to ext/opcache/tests/jit/icall_001.phpt diff --git a/ext/opcache/tests/jit/arm64/loop_001.phpt b/ext/opcache/tests/jit/loop_001.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/loop_001.phpt rename to ext/opcache/tests/jit/loop_001.phpt diff --git a/ext/opcache/tests/jit/arm64/loop_002.phpt b/ext/opcache/tests/jit/loop_002.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/loop_002.phpt rename to ext/opcache/tests/jit/loop_002.phpt diff --git a/ext/opcache/tests/jit/arm64/mul_001.phpt b/ext/opcache/tests/jit/mul_001.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/mul_001.phpt rename to ext/opcache/tests/jit/mul_001.phpt diff --git a/ext/opcache/tests/jit/arm64/mul_002.phpt b/ext/opcache/tests/jit/mul_002.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/mul_002.phpt rename to ext/opcache/tests/jit/mul_002.phpt diff --git a/ext/opcache/tests/jit/arm64/mul_003.phpt b/ext/opcache/tests/jit/mul_003.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/mul_003.phpt rename to ext/opcache/tests/jit/mul_003.phpt diff --git a/ext/opcache/tests/jit/arm64/mul_004.phpt b/ext/opcache/tests/jit/mul_004.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/mul_004.phpt rename to ext/opcache/tests/jit/mul_004.phpt diff --git a/ext/opcache/tests/jit/arm64/recv_001.phpt b/ext/opcache/tests/jit/recv_005.phpt similarity index 95% rename from ext/opcache/tests/jit/arm64/recv_001.phpt rename to ext/opcache/tests/jit/recv_005.phpt index 8a6c974a64b51..239cbc938cf03 100644 --- a/ext/opcache/tests/jit/arm64/recv_001.phpt +++ b/ext/opcache/tests/jit/recv_005.phpt @@ -1,5 +1,5 @@ --TEST-- -JIT RECV: 001 +JIT RECV: 005 --INI-- opcache.enable=1 opcache.enable_cli=1 diff --git a/ext/opcache/tests/jit/arm64/ret_001.phpt b/ext/opcache/tests/jit/ret_001.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/ret_001.phpt rename to ext/opcache/tests/jit/ret_001.phpt diff --git a/ext/opcache/tests/jit/arm64/ret_002.phpt b/ext/opcache/tests/jit/ret_002.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/ret_002.phpt rename to ext/opcache/tests/jit/ret_002.phpt diff --git a/ext/opcache/tests/jit/arm64/ret_003.phpt b/ext/opcache/tests/jit/ret_003.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/ret_003.phpt rename to ext/opcache/tests/jit/ret_003.phpt diff --git a/ext/opcache/tests/jit/arm64/sub_001.phpt b/ext/opcache/tests/jit/sub_001.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/sub_001.phpt rename to ext/opcache/tests/jit/sub_001.phpt diff --git a/ext/opcache/tests/jit/arm64/ucall_001.phpt b/ext/opcache/tests/jit/ucall_001.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/ucall_001.phpt rename to ext/opcache/tests/jit/ucall_001.phpt diff --git a/ext/opcache/tests/jit/arm64/ucall_002.phpt b/ext/opcache/tests/jit/ucall_002.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/ucall_002.phpt rename to ext/opcache/tests/jit/ucall_002.phpt diff --git a/ext/opcache/tests/jit/arm64/ucall_003.phpt b/ext/opcache/tests/jit/ucall_003.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/ucall_003.phpt rename to ext/opcache/tests/jit/ucall_003.phpt diff --git a/ext/opcache/tests/jit/arm64/ucall_004.phpt b/ext/opcache/tests/jit/ucall_004.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/ucall_004.phpt rename to ext/opcache/tests/jit/ucall_004.phpt diff --git a/ext/opcache/tests/jit/arm64/xor_001.phpt b/ext/opcache/tests/jit/xor_001.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/xor_001.phpt rename to ext/opcache/tests/jit/xor_001.phpt diff --git a/ext/opcache/tests/jit/arm64/xor_002.phpt b/ext/opcache/tests/jit/xor_002.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/xor_002.phpt rename to ext/opcache/tests/jit/xor_002.phpt diff --git a/ext/opcache/tests/jit/arm64/xor_003.phpt b/ext/opcache/tests/jit/xor_003.phpt similarity index 100% rename from ext/opcache/tests/jit/arm64/xor_003.phpt rename to ext/opcache/tests/jit/xor_003.phpt From e5b99bc6915f073fde36c355c057488c0013a41c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 29 Apr 2021 14:42:14 +0300 Subject: [PATCH 182/229] Fixed signed/unsigned comparison mess and add one missing case --- ext/opcache/jit/zend_jit_arm64.dasc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index f587bb1d17dec..1ca972b5a2d37 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -8300,9 +8300,9 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con return 0; } - | blt &exit_addr + | blo &exit_addr } else { - | blt >1 + | blo >1 | // EG(vm_stack_top) = (zval*)((char*)call + used_stack); |.cold_code |1: @@ -14218,7 +14218,11 @@ static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t o | // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); | // ZEND_VM_CONTINUE(); if (exit_addr) { - | bls &exit_addr + if (exit_opcode == ZEND_JMP) { + | bls &exit_addr + } else { + | bls >3 + } } else { | bls =>target_label } From 4a763cf9c110cbbd4e86a31a447239df9444ecb9 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 29 Apr 2021 14:46:42 +0300 Subject: [PATCH 183/229] Temporary disable PROFITABILITY_CHECKS for better test coverage and to be in parity with AArch64 back-end. --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 795e840df0cf6..b0d6bb82b1c1a 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -162,7 +162,7 @@ static size_t tsrm_tls_offset; * type information. Disabling this option allows testing some JIT handlers in the * presence of try/catch blocks, which prevent SSA construction. */ #ifndef PROFITABILITY_CHECKS -# define PROFITABILITY_CHECKS 1 +# define PROFITABILITY_CHECKS 0 // TODO: temporary disabled for better test coverage #endif |.type EX, zend_execute_data, FP From a1389a774cc53e8c9e196b83c01d3621ed46bb9b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 29 Apr 2021 15:31:40 +0300 Subject: [PATCH 184/229] Skip 64-bit related tests on 32-bit platforms --- ext/opcache/tests/jit/add_003.phpt | 4 ++++ ext/opcache/tests/jit/add_004.phpt | 4 ++++ ext/opcache/tests/jit/inc_021.phpt | 4 ++++ ext/opcache/tests/jit/mul_002.phpt | 4 ++++ ext/opcache/tests/jit/mul_003.phpt | 4 ++++ ext/opcache/tests/jit/mul_004.phpt | 4 ++++ 6 files changed, 24 insertions(+) diff --git a/ext/opcache/tests/jit/add_003.phpt b/ext/opcache/tests/jit/add_003.phpt index ca01ce797b8e6..70e814968b124 100644 --- a/ext/opcache/tests/jit/add_003.phpt +++ b/ext/opcache/tests/jit/add_003.phpt @@ -8,6 +8,10 @@ opcache.jit_buffer_size=32M ;opcache.jit_debug=257 --EXTENSIONS-- opcache +--SKIPIF-- + --FILE-- --FILE-- --FILE-- --FILE-- --FILE-- --FILE-- Date: Fri, 30 Apr 2021 01:11:51 +0300 Subject: [PATCH 185/229] Don't optimize MUL into SHIFT if we have to check for overflow --- ext/opcache/jit/zend_jit_arm64.dasc | 68 +++++++------------ ext/opcache/jit/zend_jit_x86.dasc | 101 ++++++++-------------------- ext/opcache/tests/jit/mul_004.phpt | 11 ++- 3 files changed, 64 insertions(+), 116 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 1ca972b5a2d37..45164929f3797 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -3848,53 +3848,37 @@ static int zend_jit_math_long_long(dasm_State **Dst, } if (opcode == ZEND_MUL && - Z_MODE(op2_addr) == IS_CONST_ZVAL && - zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { - - if (Z_LVAL_P(Z_ZV(op2_addr)) == 2) { - if (Z_MODE(op1_addr) == IS_REG) { - | adds Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) - } else { - | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 - | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) - } + Z_MODE(op2_addr) == IS_CONST_ZVAL && + Z_LVAL_P(Z_ZV(op2_addr)) == 2) { + if (Z_MODE(op1_addr) == IS_REG) { + | adds Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) } else { - | GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP2 - | mov TMP2, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) - | lsl Rx(result_reg), TMP1, TMP2 - if (may_overflow) { - /* Compare 'op' and '((op << n) >> n)' for overflow. - * Flag: bne -> overflow. beq -> no overflow. - */ - use_ovf_flag = 0; - | asr TMP3, Rx(result_reg), TMP2 - | cmp TMP1, TMP3 - } + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) } } else if (opcode == ZEND_MUL && - Z_MODE(op1_addr) == IS_CONST_ZVAL && - zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { - - if (Z_LVAL_P(Z_ZV(op1_addr)) == 2) { - if (Z_MODE(op2_addr) == IS_REG) { - | adds Rx(result_reg), Rx(Z_REG(op2_addr)), Rx(Z_REG(op2_addr)) - } else { - | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 - | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) - } + Z_MODE(op2_addr) == IS_CONST_ZVAL && + !may_overflow && + zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | mov TMP1, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) + | lsl Rx(result_reg), Rx(result_reg), TMP1 + } else if (opcode == ZEND_MUL && + Z_MODE(op1_addr) == IS_CONST_ZVAL && + Z_LVAL_P(Z_ZV(op1_addr)) == 2) { + if (Z_MODE(op2_addr) == IS_REG) { + | adds Rx(result_reg), Rx(Z_REG(op2_addr)), Rx(Z_REG(op2_addr)) } else { - | GET_ZVAL_LVAL ZREG_TMP1, op2_addr, TMP2 - | mov TMP2, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) - | lsl Rx(result_reg), TMP1, TMP2 - if (may_overflow) { - /* Compare 'op' and '((op << n) >> n)' for overflow. - * Flag: bne -> overflow. beq -> no overflow. - */ - use_ovf_flag = 0; - | asr TMP3, Rx(result_reg), TMP2 - | cmp TMP1, TMP3 - } + | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 + | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) } + } else if (opcode == ZEND_MUL && + Z_MODE(op1_addr) == IS_CONST_ZVAL && + !may_overflow && + zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { + | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 + | mov TMP1, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) + | lsl Rx(result_reg), Rx(result_reg), TMP1 } else if (opcode == ZEND_DIV && (Z_MODE(op2_addr) == IS_CONST_ZVAL && is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index b0d6bb82b1c1a..fdd6ec9830f08 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4180,7 +4180,6 @@ static int zend_jit_math_long_long(dasm_State **Dst, bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); zend_reg result_reg; zend_reg tmp_reg = ZREG_R0; - bool use_ovf_flag = 1; if (Z_MODE(res_addr) == IS_REG && (res_info & MAY_BE_LONG)) { if (may_overflow && (res_info & MAY_BE_GUARD) @@ -4201,67 +4200,39 @@ static int zend_jit_math_long_long(dasm_State **Dst, } if (opcode == ZEND_MUL && - Z_MODE(op2_addr) == IS_CONST_ZVAL && - Z_LVAL_P(Z_ZV(op2_addr)) > 0 && - IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op2_addr))) && - is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { - - if (Z_MODE(op1_addr) == IS_REG && Z_LVAL_P(Z_ZV(op2_addr)) == 2) { + Z_MODE(op2_addr) == IS_CONST_ZVAL && + Z_LVAL_P(Z_ZV(op2_addr)) == 2) { + if (Z_MODE(op1_addr) == IS_REG && !may_overflow) { | lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Ra(Z_REG(op1_addr))] } else { | GET_ZVAL_LVAL result_reg, op1_addr - | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) - if (may_overflow) { - /* Compare 'op' and '((op << n) >> n)' for overflow. - * Flag: jne -> overflow. je -> no overflow. - */ - use_ovf_flag = 0; - | sar Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) - if (Z_MODE(op1_addr) == IS_CONST_ZVAL) { - | cmp Ra(result_reg), Z_LVAL_P(Z_ZV(op1_addr)) - } else if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { - | cmp Ra(result_reg), [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)] - } else if (Z_MODE(op1_addr) == IS_REG) { - | cmp Ra(result_reg), Ra(Z_REG(op1_addr)) - } else { - ZEND_UNREACHABLE(); - } - | pushf - | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) - | popf - } + | add Ra(result_reg), Ra(result_reg) } } else if (opcode == ZEND_MUL && - Z_MODE(op1_addr) == IS_CONST_ZVAL && - Z_LVAL_P(Z_ZV(op1_addr)) > 0 && - IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr))) && - is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { - - if (Z_MODE(op2_addr) == IS_REG && Z_LVAL_P(Z_ZV(op1_addr)) == 2) { + Z_MODE(op2_addr) == IS_CONST_ZVAL && + !may_overflow && + Z_LVAL_P(Z_ZV(op2_addr)) > 0 && + IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op2_addr))) && + is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { + | GET_ZVAL_LVAL result_reg, op1_addr + | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) + } else if (opcode == ZEND_MUL && + Z_MODE(op1_addr) == IS_CONST_ZVAL && + Z_LVAL_P(Z_ZV(op1_addr)) == 2) { + if (Z_MODE(op2_addr) == IS_REG && !may_overflow) { | lea Ra(result_reg), [Ra(Z_REG(op2_addr))+Ra(Z_REG(op2_addr))] } else { | GET_ZVAL_LVAL result_reg, op2_addr - | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) - if (may_overflow) { - /* Compare 'op' and '((op << n) >> n)' for overflow. - * Flag: jne -> overflow. je -> no overflow. - */ - use_ovf_flag = 0; - | sar Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) - if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { - | cmp Ra(result_reg), Z_LVAL_P(Z_ZV(op2_addr)) - } else if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { - | cmp Ra(result_reg), [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)] - } else if (Z_MODE(op2_addr) == IS_REG) { - | cmp Ra(result_reg), Ra(Z_REG(op2_addr)) - } else { - ZEND_UNREACHABLE(); - } - | pushf - | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) - | popf - } + | lea Ra(result_reg), [Ra(result_reg)+Ra(result_reg)] } + } else if (opcode == ZEND_MUL && + Z_MODE(op1_addr) == IS_CONST_ZVAL && + !may_overflow && + Z_LVAL_P(Z_ZV(op1_addr)) > 0 && + IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr))) && + is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { + | GET_ZVAL_LVAL result_reg, op2_addr + | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) } else if (opcode == ZEND_DIV && (Z_MODE(op2_addr) == IS_CONST_ZVAL && is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { @@ -4300,36 +4271,20 @@ static int zend_jit_math_long_long(dasm_State **Dst, int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) { - if (use_ovf_flag) { - | jo &exit_addr - } else { - | jne &exit_addr - } + | jo &exit_addr if (Z_MODE(res_addr) == IS_REG && result_reg != Z_REG(res_addr)) { | mov Ra(Z_REG(res_addr)), Ra(result_reg) } } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - if (use_ovf_flag) { - | jno &exit_addr - } else { - | je &exit_addr - } + | jno &exit_addr } else { ZEND_UNREACHABLE(); } } else { if (res_info & MAY_BE_LONG) { - if (use_ovf_flag) { - | jo >1 - } else { - | jne >1 - } + | jo >1 } else { - if (use_ovf_flag) { - | jno >1 - } else { - | je >1 - } + | jno >1 } } } diff --git a/ext/opcache/tests/jit/mul_004.phpt b/ext/opcache/tests/jit/mul_004.phpt index 7154ecc53db07..b1855a0ee489e 100644 --- a/ext/opcache/tests/jit/mul_004.phpt +++ b/ext/opcache/tests/jit/mul_004.phpt @@ -35,6 +35,11 @@ function mul2_big_int64(int $a) { var_dump($res); } +function mul2(int $a) { + $res = $a * 2; // $a + $a + var_dump($res); +} + mul2_8(3); mul2_8(-11); mul2_8(0x7fffffffffffffff); @@ -47,6 +52,8 @@ mul2_big_int32(0x10000000000); mul2_big_int64(3); mul2_big_int64(-3); mul2_big_int64(0x100000000); +mul2(10); +mul2(0x7fffffffffffffff); ?> --EXPECT-- int(24) @@ -60,4 +67,6 @@ int(-805306368) float(2.9514790517935283E+20) int(12884901888) int(-12884901888) -float(1.8446744073709552E+19) \ No newline at end of file +float(1.8446744073709552E+19) +int(20) +float(1.8446744073709552E+19) From 6b50c390a61cb3dbfc23e035a750ec99cbd56fef Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 30 Apr 2021 01:25:41 +0300 Subject: [PATCH 186/229] Improve code for MUL overflow detection (less instructions and less temporary registers) --- ext/opcache/jit/zend_jit_arm64.dasc | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 45164929f3797..afb486b3e7e06 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -3903,9 +3903,9 @@ static int zend_jit_math_long_long(dasm_State **Dst, | NIY // TODO: test #endif } else if (opcode == ZEND_MUL) { - | GET_ZVAL_LVAL ZREG_TMP2, op1_addr, TMP1 - | GET_ZVAL_LVAL ZREG_TMP3, op2_addr, TMP1 - | mul Rx(result_reg), TMP2, TMP3 + | GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1 + | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 + | mul Rx(result_reg), TMP1, TMP2 if(may_overflow) { /* Use 'smulh' to get the upper 64 bits fo the 128-bit result. * For signed multiplication, the top 65 bits of the result will contain @@ -3915,9 +3915,8 @@ static int zend_jit_math_long_long(dasm_State **Dst, * Flag: bne -> overflow. beq -> no overflow. */ use_ovf_flag = 0; - | smulh TMP1, TMP2, TMP3 - | asr TMP2, Rx(result_reg), #63 - | cmp TMP1, TMP2 + | smulh TMP1, TMP1, TMP2 + | cmp TMP1, Rx(result_reg), asr #63 } } else { | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 From 2c147fe5ebfe25bad71105f221a93144976b676c Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 30 Apr 2021 05:53:18 +0300 Subject: [PATCH 187/229] Optimize AArch64 code generator - use cbz/cbnz for comparison with zero - use xzr/wzr instead of #0 in mov/cmp - use "subs" imstead of "sub x1,...; cmp x1, xzr" - avoid loading immediate value to register when it may be direclu used as a source immendiate operand in ldr*/str*/add*/sub*/cmp --- ext/opcache/jit/zend_jit_arm64.dasc | 397 +++++++++++++--------------- 1 file changed, 189 insertions(+), 208 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index afb486b3e7e06..5dd65d561220d 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -448,33 +448,6 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | .endif |.endmacro -// Similar to MEM_LOAD_OP/_ZTS, but operations are compare instructions. -// Note that 'op' can be imm12. -|.macro MEM_LOAD_CMP, ldr_ins, op, addr, tmp_reg1, tmp_reg2 -| MEM_LOAD ldr_ins, tmp_reg1, addr, tmp_reg2 -| cmp tmp_reg1, op -|.endmacro - -|.macro MEM_LOAD_CMP_ZTS, ldr_ins, op, struct, field, tmp_reg1, tmp_reg2 -| .if ZTS -| LOAD_TSRM_CACHE TMP3 -| SAFE_MEM_ACC_WITH_UOFFSET ldr_ins, tmp_reg1, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg2 -| cmp tmp_reg1, op -| .else -| MEM_LOAD_CMP ldr_ins, op, &struct.field, tmp_reg1, tmp_reg2 -| .endif -|.endmacro - -|.macro MEM_LOAD_CMP_BYTE_ZTS, ldrb_ins, op, struct, field, tmp_reg1, tmp_reg2 -| .if ZTS -| LOAD_TSRM_CACHE TMP3 -| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb_ins, tmp_reg1, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg2 -| cmp tmp_reg1, op -| .else -| MEM_LOAD_CMP ldrb_ins, op, &struct.field, tmp_reg1, tmp_reg2 -| .endif -|.endmacro - // Load the value from memory 'addr' into a tmp register 'tmp_reg1' and conduct arithmetic operations with 'op'. // The computation result is stored back to memory 'addr'. 'op' can be either imm12 or register. // For constant case, it should be guaranteed that 'op' can be represented by imm12 before using this macro. @@ -1203,20 +1176,27 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.endmacro |.macro IF_UNDEF, type_reg, label -| tst type_reg, type_reg -| beq label +| cbz type_reg, label |.endmacro |.macro IF_TYPE, type, val, label || ZEND_ASSERT(val >=0 && val <= CMP_IMM); -| cmp type, #val -| beq label +|| if (val == 0) { +| cbz type, label +|| } else { +| cmp type, #val +| beq label +|| } |.endmacro |.macro IF_NOT_TYPE, type, val, label || ZEND_ASSERT(val >=0 && val <= CMP_IMM); -| cmp type, #val -| bne label +|| if (val == 0) { +| cbnz type, label +|| } else { +| cmp type, #val +| bne label +|| } |.endmacro |.macro IF_Z_TYPE, zv, val, label, tmp_reg @@ -1995,7 +1975,7 @@ static int zend_jit_undefined_function_stub(dasm_State **Dst) { |->undefined_function: | ldr REG0, EX->opline - | movz CARG1, #0 + | mov CARG1, xzr | LOAD_ADDR CARG2, "Call to undefined function %s()" | ldr CARG3w, [REG0, #offsetof(zend_op, op2.constant)] | sxtw CARG3, CARG3w @@ -2071,8 +2051,7 @@ static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) | // jit_extension = (const void*)ZEND_FUNC_INFO(op_array); | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] | // ++ZEND_COUNTER_INFO(op_array) - | LOAD_32BIT_VAL TMP1w, (zend_jit_profile_counter_rid * sizeof(void*)) - | ldr TMP2, [REG2, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, TMP2, REG2, (zend_jit_profile_counter_rid * sizeof(void*)), TMP1 | add TMP2, TMP2, #1 | str TMP2, [REG2, TMP1] | // return ((zend_vm_opcode_handler_t)jit_extension->orig_handler)() @@ -2127,11 +2106,14 @@ static int zend_jit_hybrid_hot_counter_stub(dasm_State **Dst, uint32_t cost) | ldr REG0, EX->func | ldr REG1, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] | ldr REG2, [REG1, #offsetof(zend_jit_op_array_hot_extension, counter)] - | LOAD_32BIT_VAL TMP1w, cost | ldrh TMP2w, [REG2] - | sub TMP2w, TMP2w, TMP1w + || if (cost > ADD_SUB_IMM) { + | LOAD_32BIT_VAL TMP1w, cost + | subs TMP2w, TMP2w, TMP1w + || } else { + | subs TMP2w, TMP2w, #cost + || } | strh TMP2w, [REG2] - | cmp TMP2w, #0 | ble ->hybrid_hot_code | GET_IP REG2 | ldr TMP1, [REG0, #offsetof(zend_op_array, opcodes)] @@ -2184,7 +2166,7 @@ static int zend_jit_hybrid_hot_trace_stub(dasm_State **Dst) | mov FCARG1x, FP | GET_IP FCARG2x | EXT_CALL zend_jit_trace_hot_root, REG0 - | cmp RETVALw, #0 // Result is < 0 on failure. + | cmp RETVALw, wzr // Result is < 0 on failure. | blt >1 | MEM_LOAD_ZTS ldr, FP, executor_globals, current_execute_data, REG0 | LOAD_IP @@ -2203,10 +2185,13 @@ static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost) | add TMP1, REG1, IP | ldr REG2, [TMP1, #offsetof(zend_op_trace_info, counter)] | ldrh TMP2w, [REG2] - | LOAD_32BIT_VAL TMP3w, cost - | sub TMP2w, TMP2w, TMP3w + || if (cost > ADD_SUB_IMM) { + | LOAD_32BIT_VAL TMP3w, cost + | subs TMP2w, TMP2w, TMP3w + || } else { + | subs TMP2w, TMP2w, #cost + || } | strh TMP2w, [REG2] - | cmp TMP2w, #0 | ble ->hybrid_hot_trace | ldr TMP2, [TMP1, #offsetof(zend_op_trace_info, orig_handler)] | br TMP2 @@ -2349,8 +2334,8 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst) | LOAD_IP | // check for interrupt (try to avoid this ???) - | MEM_LOAD_CMP_BYTE_ZTS ldrb, wzr, executor_globals, vm_interrupt, REG0w, TMP1 - | bne ->interrupt_handler + | MEM_LOAD_BYTE_ZTS ldrb, REG0w, executor_globals, vm_interrupt, TMP1 + | cbnz REG0w, ->interrupt_handler if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | ADD_HYBRID_SPAD @@ -2427,9 +2412,13 @@ static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n) | strb TMP1w, [sp, #-16]! |1: | ldrb TMP1w, [sp] - | LOAD_32BIT_VAL TMP2w, n - | add TMP2w, TMP2w, TMP1w - | str TMP2w, [sp] + || if (n > ADD_SUB_IMM) { + | LOAD_32BIT_VAL TMP2w, n + | add TMP1w, TMP1w, TMP2w + || } else { + | add TMP1w, TMP1w, #n + || } + | str TMP1w, [sp] | b ->trace_exit return 1; @@ -2835,8 +2824,8 @@ static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void *timeout_exit_addr) { if (timeout_exit_addr) { - | MEM_LOAD_CMP_BYTE_ZTS ldrb, wzr, executor_globals, vm_interrupt, REG0w, TMP1 - | beq =>loop_label + | MEM_LOAD_BYTE_ZTS ldrb, REG0w, executor_globals, vm_interrupt, TMP1 + | cbz REG0w, =>loop_label | b &timeout_exit_addr } else { | b =>loop_label @@ -2846,16 +2835,16 @@ static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void static int zend_jit_check_exception(dasm_State **Dst) { - | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, REG0, TMP1 - | bne ->exception_handler + | MEM_LOAD_ZTS ldr, REG0, executor_globals, exception, TMP1 + | cbnz REG0, ->exception_handler return 1; } static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline) { if (opline->result_type & (IS_TMP_VAR|IS_VAR)) { - | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, REG0, TMP1 - | bne ->exception_handler_undef + | MEM_LOAD_ZTS ldr, REG0, executor_globals, exception, TMP1 + | cbnz REG0, ->exception_handler_undef return 1; } return zend_jit_check_exception(Dst); @@ -3020,8 +3009,8 @@ static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, if (timeout_exit_addr) { /* Check timeout for links to LOOP */ - | MEM_LOAD_CMP_BYTE_ZTS ldrb, wzr, executor_globals, vm_interrupt, REG0w, TMP1 - | beq &link_addr + | MEM_LOAD_BYTE_ZTS ldrb, REG0w, executor_globals, vm_interrupt, TMP1 + | cbz REG0w, &link_addr | b &timeout_exit_addr } else { | b &link_addr @@ -3077,18 +3066,13 @@ static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); + zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); if (!exit_addr) { return 0; } - if (var > ADD_SUB_IMM) { - | LOAD_32BIT_VAL TMP1, var - | add TMP1, FP, TMP1 - } else { - | add TMP1, FP, #var - } - | IF_NOT_Z_TYPE TMP1, type, &exit_addr, TMP2w + | IF_NOT_ZVAL_TYPE var_addr, type, &exit_addr, TMP1w, TMP2 return 1; } @@ -3135,8 +3119,8 @@ static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_arra if (may_throw && opline->opcode != ZEND_RETURN && opline->opcode != ZEND_RETURN_BY_REF) { - | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, REG0, TMP1 - | bne ->exception_handler + | MEM_LOAD_ZTS ldr, REG0, executor_globals, exception, TMP1 + | cbnz REG0, ->exception_handler } while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) { @@ -4592,11 +4576,10 @@ static int zend_jit_long_math_helper(dasm_State **Dst, if (!op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) { if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { | SAFE_MEM_ACC_WITH_UOFFSET ldr, TMP1, Rx(Z_REG(op2_addr)), Z_OFFSET(op2_addr), TMP2 - | cmp TMP1, #0 + | cbz TMP1, >1 } else if (Z_MODE(op2_addr) == IS_REG) { - | tst Rx(Z_REG(op2_addr)), Rx(Z_REG(op2_addr)) + | cbz Rx(Z_REG(op2_addr)), >1 } - | beq >1 |.cold_code |1: | SET_EX_OPLINE opline, REG0 @@ -4897,7 +4880,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | ldr REG0w, [FCARG1x, #offsetof(zend_array, nNumUsed)] if (val == 0) { - | cmp REG0, #0 + | cmp REG0, xzr } else if (val > 0 && !op2_loaded) { | LOAD_64BIT_VAL TMP1, val | cmp REG0, TMP1 @@ -5428,8 +5411,7 @@ static int zend_jit_assign_to_typed_ref(dasm_State **Dst, { | // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) { | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] - | cmp TMP1, #0 - | bne >2 + | cbnz TMP1, >2 |.cold_code |2: if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2x || Z_OFFSET(val_addr) != 0) { @@ -5451,8 +5433,8 @@ static int zend_jit_assign_to_typed_ref(dasm_State **Dst, } if (check_exception) { | // if (UNEXPECTED(EG(exception) != NULL)) { - | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, REG0, TMP1 - | beq >8 // END OF zend_jit_assign_to_variable() + | MEM_LOAD_ZTS ldr, REG0, executor_globals, exception, TMP1 + | cbz REG0, >8 // END OF zend_jit_assign_to_variable() | b ->exception_handler } else { | b >8 @@ -5609,8 +5591,8 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, | ZVAL_DTOR_FUNC var_info, opline, TMP1 if (in_cold || (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0)) { if (check_exception) { - | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, REG0, TMP1 - | beq >8 + | MEM_LOAD_ZTS ldr, REG0, executor_globals, exception, TMP1 + | cbz REG0, >8 | b ->exception_handler } else { | b >8 @@ -7939,7 +7921,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ } if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { - | mov TMP1, #0 + | mov TMP1, xzr | fmov FPR0d, TMP1 | DOUBLE_CMP fcmp, ZREG_FPR0, op1_addr, ZREG_TMP1, ZREG_FPTMP @@ -8045,8 +8027,8 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ |3: } if (may_throw) { - | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, REG1, TMP1 - | bne ->exception_handler_undef + | MEM_LOAD_ZTS ldr, REG1, executor_globals, exception, TMP1 + | cbnz REG1, ->exception_handler_undef } if (set_bool) { @@ -8082,28 +8064,27 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ |.code } } else { - | tst REG0w, REG0w if (exit_addr) { if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { - | bne &exit_addr + | cbnz REG0w, &exit_addr if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { | b >9 } } else { - | beq &exit_addr + | cbz REG0w, &exit_addr if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { | b >9 } } } else if (true_label != (uint32_t)-1) { - | bne =>true_label + | cbnz REG0w, =>true_label if (false_label != (uint32_t)-1) { | b =>false_label } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { | b >9 } } else { - | beq =>false_label + | cbz REG0w, =>false_label if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) { | b >9 } @@ -8413,8 +8394,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con | // Z_PTR(call->This) = object_or_called_scope; | str REG1, EX:RX->This.value.ptr | ldr TMP1, [REG0, #offsetof(zend_closure, func.op_array.run_time_cache__ptr)] - | cmp TMP1, xzr - | bne >1 + | cbnz TMP1, >1 | add FCARG1x, REG0, #offsetof(zend_closure, func) | EXT_CALL zend_jit_init_func_run_time_cache_helper, REG0 |1: @@ -8885,22 +8865,34 @@ static int zend_jit_init_method_call(dasm_State **Dst, if (func) { | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); | ldr REG0, EX->run_time_cache - | LOAD_32BIT_VAL TMP1w, (opline->result.num + sizeof(void*)) - | add REG0, REG0, TMP1 + || if (opline->result.num + sizeof(void*) > ADD_SUB_IMM) { + | LOAD_32BIT_VAL TMP1w, (opline->result.num + sizeof(void*)) + | add REG0, REG0, TMP1 + || } else { + | add REG0, REG0, #(opline->result.num + sizeof(void*)) + || } | ldr REG0, [REG0] | cbz REG0, >1 } else { | // if (CACHED_PTR(opline->result.num) == obj->ce)) { | ldr REG0, EX->run_time_cache - | LOAD_32BIT_VAL TMP1w, opline->result.num - | add TMP1, REG0, TMP1 + || if (opline->result.num > ADD_SUB_IMM) { + | LOAD_32BIT_VAL TMP1w, opline->result.num + | add TMP1, REG0, TMP1 + || } else { + | add TMP1, REG0, #opline->result.num + || } | ldr REG2, [TMP1] | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] | cmp REG2, TMP1 | bne >1 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); - | LOAD_32BIT_VAL TMP1w, (opline->result.num + sizeof(void*)) - | add TMP1, TMP1, REG0 + || if (opline->result.num + sizeof(void*) > ADD_SUB_IMM) { + | LOAD_32BIT_VAL TMP1w, (opline->result.num + sizeof(void*)) + | add TMP1, TMP1, REG0 + || } else { + | add TMP1, REG0, #(opline->result.num + sizeof(void*)) + || } | ldr REG0, [TMP1] } @@ -9265,9 +9257,8 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } | EXT_CALL zend_jit_deprecated_helper, REG0 | GET_LOW_8BITS RETVALw, RETVALw - | cmp RETVALw, #0 // Result is 0 on exception | ldr REG0, EX:RX->func // reload - | bne >1 + | cbnz RETVALw, >1 // Result is 0 on exception | b ->exception_handler |.code |1: @@ -9559,9 +9550,8 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } | EXT_CALL zend_jit_deprecated_helper, REG0 | GET_LOW_8BITS RETVALw, RETVALw - | cmp RETVALw, #0 // Result is 0 on exception | ldr REG0, EX:RX->func // reload - | bne >1 + | cbnz RETVALw, >1 // Result is 0 on exception | b ->exception_handler |.code |1: @@ -9678,8 +9668,8 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend } | // if (UNEXPECTED(EG(exception) != NULL)) { - | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, REG0, TMP1 - | bne ->icall_throw_handler + | MEM_LOAD_ZTS ldr, REG0, executor_globals, exception, TMP1 + | cbnz REG0, ->icall_throw_handler // TODO: Can we avoid checking for interrupts after each call ??? if (trace && last_valid_opline != opline) { @@ -10165,9 +10155,7 @@ static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline) |.code | // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); | ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] - | LOAD_32BIT_VAL TMP2w, ZEND_CALL_SEND_ARG_BY_REF - | mvn TMP2w, TMP2w - | and TMP1w, TMP1w, TMP2w + | BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~(ZEND_CALL_SEND_ARG_BY_REF)), TMP2w | str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)] |1: } @@ -10436,8 +10424,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t if (op1_info & MAY_BE_REF) { | ldrb REG0w, [REG0, #offsetof(zval,u1.v.type)] } else { - | LOAD_32BIT_VAL TMP1, (opline->op1.var + offsetof(zval,u1.v.type)) - | ldrb REG0w, [FP, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, REG0w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 } | str REG0w, T1 // save | // zval_dtor_func(r); @@ -10455,16 +10442,14 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t if (op1_info & MAY_BE_REF) { | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] } else { - | LOAD_32BIT_VAL TMP1, (opline->op1.var + offsetof(zval,u1.v.type)) - | ldrb REG1w, [FP, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 } |2: } else { if (op1_info & MAY_BE_REF) { | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] } else { - | LOAD_32BIT_VAL TMP1, (opline->op1.var + offsetof(zval,u1.v.type)) - | ldrb REG1w, [FP, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 } } | mov REG0w, #1 @@ -10515,8 +10500,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t if (op1_info & MAY_BE_REF) { | ldrb REG0w, [REG0, #offsetof(zval,u1.v.type)] } else { - | LOAD_32BIT_VAL TMP1, (opline->op1.var + offsetof(zval,u1.v.type)) - | ldrb REG0w, [FP, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, REG0w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 } | str REG0w, T1 // save | // zval_dtor_func(r); @@ -10534,8 +10518,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t if (op1_info & MAY_BE_REF) { | ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)] } else { - | LOAD_32BIT_VAL TMP1, (opline->op1.var + offsetof(zval,u1.v.type)) - | ldrb REG1w, [FP, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 } |2: | cmp REG1w, #type @@ -10544,8 +10527,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t | ldrb TMP1w, [REG0, #offsetof(zval,u1.v.type)] | cmp TMP1w, #type } else { - | LOAD_32BIT_VAL TMP1w, (opline->op1.var + offsetof(zval,u1.v.type)) - | ldrb TMP1w, [FP, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, TMP1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 | cmp TMP1w, #type } } @@ -10816,8 +10798,8 @@ static int zend_jit_leave_func(dasm_State **Dst, && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) { /* exception might be thrown during destruction of unused return value */ | // if (EG(exception)) - | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, REG0, TMP1 - | bne ->leave_throw_handler + | MEM_LOAD_ZTS ldr, REG0, executor_globals, exception, TMP1 + | cbnz REG0, ->leave_throw_handler } do { trace++; @@ -10850,16 +10832,16 @@ static int zend_jit_leave_func(dasm_State **Dst, && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) && (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) { | // if (EG(exception)) - | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, REG0, TMP1 - | bne ->leave_throw_handler + | MEM_LOAD_ZTS ldr, REG0, executor_globals, exception, TMP1 + | cbnz REG0, ->leave_throw_handler } return 1; } else { | // if (EG(exception)) - | MEM_LOAD_CMP_ZTS ldr, xzr, executor_globals, exception, REG0, TMP1 + | MEM_LOAD_ZTS ldr, REG0, executor_globals, exception, TMP1 | LOAD_IP - | bne ->leave_throw_handler + | cbnz REG0, ->leave_throw_handler | // opline = EX(opline) + 1 | ADD_IP_FROM_CST sizeof(zend_op), TMP1 } @@ -10935,23 +10917,17 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o if (return_value_used != 0) { | ldr REG2, EX->return_value } - if (return_value_used == -1) { - | tst REG2, REG2 - } ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, 0); } else { if (return_value_used != 0) { | ldr REG1, EX->return_value } - if (return_value_used == -1) { - | tst REG1, REG1 - } ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG1, 0); } if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if (return_value_used == -1) { - | beq >1 + | cbz Rx(Z_REG(ret_addr)), >1 |.cold_code |1: } @@ -10988,9 +10964,9 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o } } else if (return_value_used == -1) { if (jit_return_label >= 0) { - | beq =>jit_return_label + | cbz Rx(Z_REG(ret_addr)), =>jit_return_label } else { - | beq >9 + | cbz Rx(Z_REG(ret_addr)), >9 } } @@ -11799,8 +11775,7 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_ | // idx = (uint32_t)(uintptr_t)CACHED_PTR(opline->extended_value) - 1; | ldr REG0, EX->run_time_cache - | LOAD_32BIT_VAL TMP1, opline->extended_value - | ldr REG0, [REG0, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldr, REG0, REG0, opline->extended_value, TMP1 | sub REG0, REG0, #1 | // if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket))) | MEM_LOAD_32_ZTS ldr, REG1w, executor_globals, symbol_table.nNumUsed, REG1 @@ -11865,8 +11840,12 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_ | LOAD_ADDR FCARG1x, (ptrdiff_t)varname | ldr FCARG2x, EX->run_time_cache if (opline->extended_value) { - | LOAD_32BIT_VAL TMP1, opline->extended_value - | add FCARG2x, FCARG2x, TMP1 + if (opline->extended_value > ADD_SUB_IMM) { + | LOAD_32BIT_VAL TMP1, opline->extended_value + | add FCARG2x, FCARG2x, TMP1 + } else { + | add FCARG2x, FCARG2x, #opline->extended_value + } } | EXT_CALL zend_jit_fetch_global_helper, REG0 | mov REG0, RETVALx @@ -11922,18 +11901,16 @@ static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zen } | LOAD_ADDR FCARG2x, (ptrdiff_t)arg_info | EXT_CALL zend_jit_verify_arg_slow, REG0 - | mov REG0w, RETVALw if (check_exception) { - | GET_LOW_8BITS REG0w, REG0w - | tst REG0w, REG0w + | GET_LOW_8BITS REG0w, RETVALw if (in_cold) { - | bne >1 + | cbnz REG0w, >1 | b ->exception_handler |.code |1: } else { - | beq ->exception_handler + | cbz REG0w, ->exception_handler } } else if (in_cold) { | b >1 @@ -11962,8 +11939,12 @@ static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_ if (arg_info || (opline+1)->opcode != ZEND_RECV) { | ldr TMP1w, EX->This.u2.num_args - | LOAD_32BIT_VAL TMP2w, arg_num - | cmp TMP1w, TMP2w + if (arg_num > CMP_IMM) { + | LOAD_32BIT_VAL TMP2w, arg_num + | cmp TMP1w, TMP2w + } else { + | cmp TMP1w, #arg_num + } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); @@ -12013,8 +11994,12 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zen if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { | ldr TMP1w, EX->This.u2.num_args - | LOAD_32BIT_VAL TMP2w, arg_num - | cmp TMP1w, TMP2w + if (arg_num > CMP_IMM) { + | LOAD_32BIT_VAL TMP2w, arg_num + | cmp TMP1w, TMP2w + } else { + | cmp TMP1w, #arg_num + } | bhs >5 } | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 @@ -12258,13 +12243,11 @@ static int zend_jit_fetch_obj(dasm_State **Dst, if (!prop_info) { | ldr REG0, EX->run_time_cache - | LOAD_32BIT_VAL TMP1w, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) - | ldr REG2, [REG0, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG2, REG0, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS), TMP1 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] | cmp REG2, TMP1 | bne >5 - | LOAD_32BIT_VAL TMP1w, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) - | ldr REG0, [REG0, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG0, REG0, ((opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)), TMP1 may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array->filename); if (may_be_dynamic) { | tst REG0, REG0 @@ -12285,8 +12268,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; | ldr REG0, EX->run_time_cache - | LOAD_32BIT_VAL TMP1, ((opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2) - | ldr FCARG2x, [REG0, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, FCARG2x, REG0, ((opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2), TMP1 | cbnz FCARG2x, >1 |.cold_code |1: @@ -12305,8 +12287,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, } } else { prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, prop_info->offset); - | LOAD_32BIT_VAL TMP1w, (prop_info->offset + offsetof(zval,u1.type_info)) - | ldr REG2w, [FCARG1x, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET_32 ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1 if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) { /* perform IS_UNDEF check only after result type guard (during deoptimization) */ @@ -12352,8 +12333,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] - | LOAD_32BIT_VAL TMP1, prop_info_offset - | ldr FCARG2x, [REG0, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 } if (Z_REG(prop_addr) != ZREG_FCARG1x || Z_OFFSET(prop_addr) != 0) { | LOAD_ZVAL_ADDR FCARG1x, prop_addr @@ -12662,18 +12642,15 @@ static int zend_jit_incdec_obj(dasm_State **Dst, needs_slow_path = 1; | ldr REG0, EX->run_time_cache - | LOAD_32BIT_VAL TMP1, opline->extended_value - | ldr REG2, [REG0, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG2, REG0, opline->extended_value, TMP1 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] | cmp REG2, TMP1 | bne >7 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { - | LOAD_32BIT_VAL TMP1, (opline->extended_value + sizeof(void*) * 2) - | ldr TMP1, [REG0, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, TMP1, REG0, (opline->extended_value + sizeof(void*) * 2), TMP1 | cbnz TMP1, >7 } - | LOAD_32BIT_VAL TMP1, (opline->extended_value + sizeof(void*)) - | ldr REG0, [REG0, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG0, REG0, (opline->extended_value + sizeof(void*)), TMP1 | tst REG0, REG0 | blt >7 | add TMP1, FCARG1x, REG0 @@ -12690,12 +12667,10 @@ static int zend_jit_incdec_obj(dasm_State **Dst, if (!exit_addr) { return 0; } - | LOAD_32BIT_VAL TMP1, prop_info->offset + offsetof(zval,u1.v.type) - | ldrb TMP2w, [FCARG1x, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 | IF_TYPE TMP2w, IS_UNDEF, &exit_addr } else { - | LOAD_32BIT_VAL TMP1, prop_info->offset + offsetof(zval,u1.v.type) - | ldrb TMP2w, [FCARG1x, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 | IF_TYPE TMP2w, IS_UNDEF, >7 needs_slow_path = 1; } @@ -12709,8 +12684,7 @@ static int zend_jit_incdec_obj(dasm_State **Dst, | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] - | LOAD_32BIT_VAL TMP1, prop_info_offset - | ldr FCARG2x, [REG0, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 } | LOAD_ZVAL_ADDR FCARG1x, prop_addr if (opline->result_type == IS_UNUSED) { @@ -12859,8 +12833,12 @@ static int zend_jit_incdec_obj(dasm_State **Dst, | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); | LOAD_ADDR FCARG2x, name | ldr CARG3, EX->run_time_cache - | LOAD_32BIT_VAL TMP1, opline->extended_value - | add CARG3, CARG3, TMP1 + if (opline->extended_value > ADD_SUB_IMM) { + | LOAD_32BIT_VAL TMP1, opline->extended_value + | add CARG3, CARG3, TMP1 + } else { + | add CARG3, CARG3, #opline->extended_value + } if (opline->result_type == IS_UNUSED) { | mov CARG4, xzr } else { @@ -13017,18 +12995,15 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, needs_slow_path = 1; | ldr REG0, EX->run_time_cache - | LOAD_32BIT_VAL TMP1, (opline+1)->extended_value - | ldr REG2, [REG0, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG2, REG0, ((opline+1)->extended_value), TMP1 | ldr TMP2, [FCARG1x, #offsetof(zend_object, ce)] | cmp REG2, TMP2 | bne >7 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { - | LOAD_32BIT_VAL TMP1, ((opline+1)->extended_value + sizeof(void*) * 2) - | ldr TMP1, [REG0, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, TMP1, REG0, ((opline+1)->extended_value + sizeof(void*) * 2), TMP1 | cbnz TMP1, >7 } - | LOAD_32BIT_VAL TMP1, ((opline+1)->extended_value + sizeof(void*)) - | ldr REG0, [REG0, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG0, REG0, ((opline+1)->extended_value + sizeof(void*)), TMP1 | tst REG0, REG0 | blt >7 | add TMP1, FCARG1x, REG0 @@ -13045,12 +13020,10 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, if (!exit_addr) { return 0; } - | LOAD_32BIT_VAL TMP1w, (prop_info->offset + offsetof(zval,u1.v.type)) - | ldrb TMP2w, [FCARG1x, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 | IF_TYPE TMP2w, IS_UNDEF, &exit_addr } else { - | LOAD_32BIT_VAL TMP1w, (prop_info->offset + offsetof(zval,u1.v.type)) - | ldrb TMP2w, [FCARG1x, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 | IF_TYPE TMP2w, IS_UNDEF, >7 needs_slow_path = 1; } @@ -13083,8 +13056,7 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] - | LOAD_32BIT_VAL TMP1, prop_info_offset - | ldr FCARG2x, [REG0, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 } | LOAD_ZVAL_ADDR FCARG1x, prop_addr | LOAD_ZVAL_ADDR CARG3, val_addr @@ -13170,8 +13142,12 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, | LOAD_ADDR FCARG2x, name | LOAD_ZVAL_ADDR CARG3, val_addr | ldr CARG4, EX->run_time_cache - | LOAD_32BIT_VAL TMP1w, (opline+1)->extended_value - | add CARG4, CARG4, TMP1 + if ((opline+1)->extended_value > ADD_SUB_IMM) { + | LOAD_32BIT_VAL TMP1w, (opline+1)->extended_value + | add CARG4, CARG4, TMP1 + } else { + | add CARG4, CARG4, #(opline+1)->extended_value + } | LOAD_ADDR CARG5, binary_op | EXT_CALL zend_jit_assign_obj_op_helper, REG0 @@ -13316,18 +13292,20 @@ static int zend_jit_assign_obj(dasm_State **Dst, needs_slow_path = 1; | ldr REG0, EX->run_time_cache - | LOAD_32BIT_VAL TMP1w, opline->extended_value - | add TMP1, REG0, TMP1 + if (opline->extended_value >= ADD_SUB_IMM) { + | LOAD_32BIT_VAL TMP1w, opline->extended_value + | add TMP1, REG0, TMP1 + } else { + | add TMP1, REG0, #opline->extended_value + } | ldr REG2, [TMP1] | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] | cmp REG2, TMP1 | bne >5 if (!ce || ce_is_instanceof || (ce->ce_flags & ZEND_ACC_HAS_TYPE_HINTS)) { - | LOAD_32BIT_VAL TMP1, (opline->extended_value + sizeof(void*) * 2) - | ldr FCARG2x, [REG0, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, FCARG2x, REG0, (opline->extended_value + sizeof(void*) * 2), TMP1 } - | LOAD_32BIT_VAL TMP1, (opline->extended_value + sizeof(void*)) - | ldr REG0, [REG0, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG0, REG0, (opline->extended_value + sizeof(void*)), TMP1 | tst REG0, REG0 | blt >5 | add TMP2, FCARG1x, REG0 @@ -13375,12 +13353,10 @@ static int zend_jit_assign_obj(dasm_State **Dst, if (!exit_addr) { return 0; } - | LOAD_32BIT_VAL TMP1, (prop_info->offset + offsetof(zval,u1.v.type)) - | ldrb TMP2w, [FCARG1x, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 | IF_TYPE TMP2w, IS_UNDEF, &exit_addr } else { - | LOAD_32BIT_VAL TMP1, (prop_info->offset + offsetof(zval,u1.v.type)) - | ldrb TMP2w, [FCARG1x, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1 | IF_TYPE TMP2w, IS_UNDEF, >5 needs_slow_path = 1; } @@ -13398,8 +13374,7 @@ static int zend_jit_assign_obj(dasm_State **Dst, | ldr REG0, [FCARG1x, #offsetof(zend_object, ce)] | ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)] - | LOAD_32BIT_VAL TMP1, prop_info_offset - | ldr FCARG2x, [REG0, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1 } | LOAD_ZVAL_ADDR FCARG1x, prop_addr | LOAD_ZVAL_ADDR CARG3, val_addr @@ -13441,8 +13416,12 @@ static int zend_jit_assign_obj(dasm_State **Dst, | LOAD_ZVAL_ADDR CARG3, val_addr | ldr CARG4, EX->run_time_cache - | LOAD_32BIT_VAL TMP1w, opline->extended_value - | add CARG4, CARG4, TMP1 + if (opline->extended_value > ADD_SUB_IMM) { + | LOAD_32BIT_VAL TMP1w, opline->extended_value + | add CARG4, CARG4, TMP1 + } else { + | add CARG4, CARG4, #opline->extended_value + } if (RETURN_VALUE_USED(opline)) { | LOAD_ZVAL_ADDR CARG5, res_addr } else { @@ -13488,8 +13467,7 @@ static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_i if (op1_info & MAY_BE_ARRAY) { | IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7, TMP1w, TMP2 } - | LOAD_32BIT_VAL TMP1w, (opline->op1.var + offsetof(zval, u2.fe_iter_idx)) - | ldr FCARG1w, [FP, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET_32 ldr, FCARG1w, FP, (opline->op1.var + offsetof(zval, u2.fe_iter_idx)), TMP1 | mvn TMP1w, wzr // TODO: DynAsm fails loading #-1 | cmp FCARG1w, TMP1w | beq >7 @@ -13836,8 +13814,12 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o uint32_t count = jumptable->nNumUsed; Bucket *p = jumptable->arData; - | LOAD_32BIT_VAL TMP1w, jumptable->nNumUsed - | cmp FCARG2x, TMP1 + if (jumptable->nNumUsed > CMP_IMM) { + | LOAD_32BIT_VAL TMP1w, jumptable->nNumUsed + | cmp FCARG2x, TMP1 + } else { + | cmp FCARG2x, #jumptable->nNumUsed + } if (default_label) { | bhs &default_label } else if (next_opline) { @@ -14066,8 +14048,12 @@ static bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, | ldr FCARG2x, EX->func | LOAD_ADDR CARG3, (ptrdiff_t)arg_info | ldr REG0, EX->run_time_cache - | LOAD_32BIT_VAL TMP1, opline->op2.num - | add CARG4, REG0, TMP1 + if (opline->op2.num > ADD_SUB_IMM) { + | LOAD_32BIT_VAL TMP1, opline->op2.num + | add CARG4, REG0, TMP1 + } else { + | add CARG4, REG0, #opline->op2.num + } | EXT_CALL zend_jit_verify_return_slow, REG0 if (!zend_jit_check_exception(Dst)) { return 0; @@ -14121,8 +14107,7 @@ static int zend_jit_isset_isempty_cv(dasm_State **Dst, const zend_op *opline, ui } } else { ZEND_ASSERT(Z_MODE(op1_addr) == IS_MEM_ZVAL); - | LOAD_32BIT_VAL TMP1, (Z_OFFSET(op1_addr)+offsetof(zval, u1.v.type)) - | ldrb TMP1w, [Rx(Z_REG(op1_addr)), TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, TMP1w, Rx(Z_REG(op1_addr)), (Z_OFFSET(op1_addr)+offsetof(zval, u1.v.type)), TMP1 | cmp TMP1w, #IS_NULL if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { @@ -14172,8 +14157,7 @@ static int zend_jit_fe_reset(dasm_State **Dst, const zend_op *opline, uint32_t o } } | // Z_FE_POS_P(res) = 0; - | LOAD_32BIT_VAL TMP1w, (opline->result.var + offsetof(zval, u2.fe_pos)) - | str wzr, [FP, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET_32 str, wzr, FP, (opline->result.var + offsetof(zval, u2.fe_pos)), TMP1 return 1; } @@ -14186,8 +14170,7 @@ static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t o | // fe_ht = Z_ARRVAL_P(array); | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 | // pos = Z_FE_POS_P(array); - | LOAD_32BIT_VAL TMP1w, (opline->op1.var + offsetof(zval, u2.fe_pos)) - | ldr REG0w, [FP, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET_32 ldr, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1 | // p = fe_ht->arData + pos; || ZEND_ASSERT(sizeof(Bucket) == 32); | mov FCARG2w, REG0w @@ -14229,8 +14212,7 @@ static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t o uint32_t val_info; | // Z_FE_POS_P(array) = pos + 1; - | LOAD_32BIT_VAL TMP1w, (opline->op1.var + offsetof(zval, u2.fe_pos)) - | str REG0w, [FP, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET_32 str, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1 if (RETURN_VALUE_USED(opline)) { zend_jit_addr res_addr = RES_ADDR(); @@ -14307,8 +14289,7 @@ static int zend_jit_fetch_constant(dasm_State **Dst, | // c = CACHED_PTR(opline->extended_value); | ldr FCARG1x, EX->run_time_cache - | LOAD_32BIT_VAL TMP1w, opline->extended_value - | ldr REG0, [FCARG1x, TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG0, FCARG1x, opline->extended_value, TMP1 | // if (c != NULL) | cbz REG0, >9 | // if (!IS_SPECIAL_CACHE_VAL(c)) From 1df56b287356590d4e30033d453f51d6014b6b31 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 30 Apr 2021 14:23:16 +0300 Subject: [PATCH 188/229] Introduce CMP_64_WITH_CONST/CMP_64_WITH_CONST_32/CMP_32_WITH_CONST macros --- ext/opcache/jit/zend_jit_arm64.dasc | 105 ++++++++++++++-------------- 1 file changed, 52 insertions(+), 53 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 5dd65d561220d..bfb85904bd56f 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -285,6 +285,45 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | tst reg, #1 |.endmacro +|.macro CMP_32_WITH_CONST, reg, val, tmp_reg +|| if (val == 0) { +| cmp reg, wzr +|| } else if (((int32_t)(val)) > 0 && ((int32_t)(val)) <= CMP_IMM) { +| cmp reg, #val +|| } else if (((int32_t)(val)) < 0 && ((int32_t)(val)) >= -CMP_IMM) { +| cmn reg, #-val +|| } else { +| LOAD_32BIT_VAL tmp_reg, val +| cmp reg, tmp_reg +|| } +|.endmacro + +|.macro CMP_64_WITH_CONST_32, reg, val, tmp_reg +|| if (val == 0) { +| cmp reg, xzr +|| } else if (((int32_t)(val)) > 0 && ((int32_t)(val)) <= CMP_IMM) { +| cmp reg, #val +|| } else if (((int32_t)(val)) < 0 && ((int32_t)(val)) >= -CMP_IMM) { +| cmn reg, #-val +|| } else { +| LOAD_32BIT_VAL tmp_reg, val +| cmp reg, tmp_reg +|| } +|.endmacro + +|.macro CMP_64_WITH_CONST, reg, val, tmp_reg +|| if (val == 0) { +| cmp reg, xzr +|| } else if (((int64_t)(val)) > 0 && ((int64_t)(val)) <= CMP_IMM) { +| cmp reg, #val +|| } else if (((int64_t)(val)) < 0 && ((int64_t)(val)) >= -CMP_IMM) { +| cmn reg, #-val +|| } else { +| LOAD_64BIT_VAL tmp_reg, val +| cmp reg, tmp_reg +|| } +|.endmacro + // Safe memory load/store with an unsigned immediate offset. // When using Z_OFFSET(addr), which is 24-bit long, as the unsigned offset to compute a memory address, // we should firstly check the range. @@ -812,12 +851,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.macro LONG_CMP, reg, addr, tmp_reg1, tmp_reg2 || if (Z_MODE(addr) == IS_CONST_ZVAL) { -|| if(Z_LVAL_P(Z_ZV(addr)) >= 0 && Z_LVAL_P(Z_ZV(addr)) <= CMP_IMM) { -| cmp Rx(reg), #(Z_LVAL_P(Z_ZV(addr))) -|| } else { -| LOAD_64BIT_VAL tmp_reg1, Z_LVAL_P(Z_ZV(addr)) -| cmp Rx(reg), tmp_reg1 -|| } +| CMP_64_WITH_CONST Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg1 || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | SAFE_MEM_ACC_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg2 | cmp Rx(reg), tmp_reg1 @@ -849,22 +883,12 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) // Define LONG_CMP_WITH_CONST to replace LONG_OP_WITH_CONST in the x86 implementation. // Note that the 'long_ins' in all use sites of LONG_OP_WITH_CONST are always 'cmp'. // Note that this macro is different from LONG_CMP. -|.macro LONG_CMP_WITH_CONST, cmp_ins, op1_addr, lval, tmp_reg1, tmp_reg2 +|.macro LONG_CMP_WITH_CONST, op1_addr, lval, tmp_reg1, tmp_reg2 || if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { | SAFE_MEM_ACC_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(op1_addr)), Z_OFFSET(op1_addr), tmp_reg2 -|| if (lval >=0 && lval <= CMP_IMM) { -| cmp_ins tmp_reg1, #lval -|| } else { -| LOAD_64BIT_VAL tmp_reg2, lval -| cmp_ins tmp_reg1, tmp_reg2 -|| } +| CMP_64_WITH_CONST tmp_reg1, lval, tmp_reg2 || } else if (Z_MODE(op1_addr) == IS_REG) { -|| if (lval >=0 && lval <= CMP_IMM) { -| cmp_ins Rx(Z_REG(op1_addr)), #lval -|| } else { -| LOAD_64BIT_VAL tmp_reg1, lval -| cmp_ins Rx(Z_REG(op1_addr)), tmp_reg1 -|| } +| CMP_64_WITH_CONST Rx(Z_REG(op1_addr)), lval, tmp_reg1 || } else { || ZEND_UNREACHABLE(); || } @@ -6264,10 +6288,10 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, } swap = 1; } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { - | LONG_CMP_WITH_CONST cmp, op2_addr, Z_LVAL_P(Z_ZV(op1_addr)), TMP1, TMP2 + | LONG_CMP_WITH_CONST op2_addr, Z_LVAL_P(Z_ZV(op1_addr)), TMP1, TMP2 swap = 1; } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { - | LONG_CMP_WITH_CONST cmp, op1_addr, Z_LVAL_P(Z_ZV(op2_addr)), TMP1, TMP2 + | LONG_CMP_WITH_CONST op1_addr, Z_LVAL_P(Z_ZV(op2_addr)), TMP1, TMP2 } else { | GET_ZVAL_LVAL ZREG_REG0, op1_addr, TMP1 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { @@ -6874,7 +6898,7 @@ static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, z static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, zend_uchar smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { - | LONG_CMP_WITH_CONST cmp, res_addr, Z_L(0), TMP1, TMP2 + | LONG_CMP_WITH_CONST res_addr, Z_L(0), TMP1, TMP2 if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ_EX || smart_branch_opcode == ZEND_JMPNZ_EX) { @@ -7890,7 +7914,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ if (Z_MODE(op1_addr) == IS_REG) { | tst Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) } else { - | LONG_CMP_WITH_CONST cmp, op1_addr, Z_L(0), TMP1, TMP2 + | LONG_CMP_WITH_CONST op1_addr, Z_L(0), TMP1, TMP2 } if (set_bool) { | cset REG0w, ne @@ -8178,12 +8202,7 @@ static int zend_jit_stack_check(dasm_State **Dst, const zend_op *opline, uint32_ | // Check Stack Overflow | MEM_LOAD_ZTS ldr, REG1, executor_globals, vm_stack_end, TMP1 | MEM_LOAD_OP_ZTS sub, ldr, REG1, executor_globals, vm_stack_top, TMP1, TMP2 - || if (used_stack <= CMP_IMM) { - | cmp REG1, #used_stack - || } else { - | LOAD_32BIT_VAL TMP1w, used_stack - | cmp REG1, TMP1 - || } + | CMP_64_WITH_CONST_32 REG1, used_stack, TMP1 | blo &exit_addr return 1; @@ -8246,12 +8265,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con | MEM_LOAD_ZTS ldr, REG2, executor_globals, vm_stack_end, TMP1 | sub REG2, REG2, RX if (func) { - || if (used_stack <= CMP_IMM) { - | cmp REG2, #used_stack - || } else { - | LOAD_32BIT_VAL TMP1w, used_stack - | cmp REG2, TMP1 - || } + | CMP_64_WITH_CONST_32 REG2, used_stack, TMP1 } else { | cmp REG2, FCARG1x } @@ -11939,12 +11953,7 @@ static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_ if (arg_info || (opline+1)->opcode != ZEND_RECV) { | ldr TMP1w, EX->This.u2.num_args - if (arg_num > CMP_IMM) { - | LOAD_32BIT_VAL TMP2w, arg_num - | cmp TMP1w, TMP2w - } else { - | cmp TMP1w, #arg_num - } + | CMP_32_WITH_CONST TMP1w, arg_num, TMP2w if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); @@ -11994,12 +12003,7 @@ static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zen if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { | ldr TMP1w, EX->This.u2.num_args - if (arg_num > CMP_IMM) { - | LOAD_32BIT_VAL TMP2w, arg_num - | cmp TMP1w, TMP2w - } else { - | cmp TMP1w, #arg_num - } + | CMP_32_WITH_CONST TMP1w, arg_num, TMP2w | bhs >5 } | ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0 @@ -13814,12 +13818,7 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o uint32_t count = jumptable->nNumUsed; Bucket *p = jumptable->arData; - if (jumptable->nNumUsed > CMP_IMM) { - | LOAD_32BIT_VAL TMP1w, jumptable->nNumUsed - | cmp FCARG2x, TMP1 - } else { - | cmp FCARG2x, #jumptable->nNumUsed - } + | CMP_64_WITH_CONST_32 FCARG2x, jumptable->nNumUsed, TMP1 if (default_label) { | bhs &default_label } else if (next_opline) { From 731456a68ebac4e533a6d9081f768626f47134b9 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 30 Apr 2021 14:25:40 +0300 Subject: [PATCH 189/229] Introduce ADD_SUB_64_WITH_CONST/ADD_SUB_64_WITH_CONST_32/ADD_SUB_32_WITH_CONST macros --- ext/opcache/jit/zend_jit_arm64.dasc | 142 ++++++++++------------------ 1 file changed, 48 insertions(+), 94 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index bfb85904bd56f..1210f1f18efe4 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -324,6 +324,39 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) || } |.endmacro +|.macro ADD_SUB_32_WITH_CONST, ins, res_reg, op1_reg, val, tmp_reg +|| if (val == 0) { +| ins res_reg, op1_reg, wzr +|| } else if (((int32_t)(val)) > 0 && ((int32_t)(val)) <= CMP_IMM) { +| ins res_reg, op1_reg, #val +|| } else { +| LOAD_32BIT_VAL tmp_reg, val +| ins res_reg, op1_reg, tmp_reg +|| } +|.endmacro + +|.macro ADD_SUB_64_WITH_CONST_32, ins, res_reg, op1_reg, val, tmp_reg +|| if (val == 0) { +| ins res_reg, op1_reg, xzr +|| } else if (((int32_t)(val)) > 0 && ((int32_t)(val)) <= CMP_IMM) { +| ins res_reg, op1_reg, #val +|| } else { +| LOAD_32BIT_VAL tmp_reg, val +| ins res_reg, op1_reg, tmp_reg +|| } +|.endmacro + +|.macro ADD_SUB_64_WITH_CONST, ins, res_reg, op1_reg, val, tmp_reg +|| if (val == 0) { +| ins res_reg, op1_reg, xzr +|| } else if (((int64_t)(val)) > 0 && ((int64_t)(val)) <= CMP_IMM) { +| ins res_reg, op1_reg, #val +|| } else { +| LOAD_64BIT_VAL tmp_reg, val +| ins res_reg, op1_reg, tmp_reg +|| } +|.endmacro + // Safe memory load/store with an unsigned immediate offset. // When using Z_OFFSET(addr), which is 24-bit long, as the unsigned offset to compute a memory address, // we should firstly check the range. @@ -365,12 +398,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.macro LOAD_ADDR_ZTS, reg, struct, field | .if ZTS | LOAD_TSRM_CACHE TMP3 -|| if (((uintptr_t)(struct.._offset+offsetof(zend_..struct, field))) > ADD_SUB_IMM) { -| LOAD_32BIT_VAL reg, (struct.._offset + offsetof(zend_..struct, field)) -| add reg, reg, TMP3 -|| } else { -| add reg, TMP3, #(struct.._offset + offsetof(zend_..struct, field)) -|| } +| ADD_SUB_64_WITH_CONST_32 add, reg, TMP3, (struct.._offset + offsetof(zend_..struct, field)), reg | .else | LOAD_ADDR reg, &struct.field | .endif @@ -520,12 +548,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.macro LOAD_BASE_ADDR, reg, base, offset || if (offset) { -|| if (((uintptr_t)(offset)) > ADD_SUB_IMM) { -| LOAD_32BIT_VAL reg, offset -| add reg, Rx(base), reg -|| } else { -| add reg, Rx(base), #offset -|| } +| ADD_SUB_64_WITH_CONST_32 add, reg, Rx(base), offset, reg || } else { || if (base == ZREG_RSP) { | mov reg, sp @@ -814,12 +837,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) // 'long_ins' should be addition or subtraction. |.macro LONG_ADD_SUB, long_ins, reg, addr, tmp_reg1, tmp_reg2 || if (Z_MODE(addr) == IS_CONST_ZVAL) { -|| if(Z_LVAL_P(Z_ZV(addr)) >= 0 && Z_LVAL_P(Z_ZV(addr)) <= ADD_SUB_IMM) { -| long_ins Rx(reg), Rx(reg), #(Z_LVAL_P(Z_ZV(addr))) -|| } else { -| LOAD_64BIT_VAL tmp_reg1, Z_LVAL_P(Z_ZV(addr)) -| long_ins Rx(reg), Rx(reg), tmp_reg1 -|| } +| ADD_SUB_64_WITH_CONST long_ins, Rx(reg), Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg1 || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | SAFE_MEM_ACC_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg2 | long_ins Rx(reg), Rx(reg), tmp_reg1 @@ -2131,12 +2149,7 @@ static int zend_jit_hybrid_hot_counter_stub(dasm_State **Dst, uint32_t cost) | ldr REG1, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] | ldr REG2, [REG1, #offsetof(zend_jit_op_array_hot_extension, counter)] | ldrh TMP2w, [REG2] - || if (cost > ADD_SUB_IMM) { - | LOAD_32BIT_VAL TMP1w, cost - | subs TMP2w, TMP2w, TMP1w - || } else { - | subs TMP2w, TMP2w, #cost - || } + | ADD_SUB_32_WITH_CONST subs, TMP2w, TMP2w, cost, TMP1w | strh TMP2w, [REG2] | ble ->hybrid_hot_code | GET_IP REG2 @@ -2209,12 +2222,7 @@ static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost) | add TMP1, REG1, IP | ldr REG2, [TMP1, #offsetof(zend_op_trace_info, counter)] | ldrh TMP2w, [REG2] - || if (cost > ADD_SUB_IMM) { - | LOAD_32BIT_VAL TMP3w, cost - | subs TMP2w, TMP2w, TMP3w - || } else { - | subs TMP2w, TMP2w, #cost - || } + | ADD_SUB_32_WITH_CONST subs, TMP2w, TMP2w, cost, TMP3w | strh TMP2w, [REG2] | ble ->hybrid_hot_trace | ldr TMP2, [TMP1, #offsetof(zend_op_trace_info, orig_handler)] @@ -2436,12 +2444,7 @@ static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n) | strb TMP1w, [sp, #-16]! |1: | ldrb TMP1w, [sp] - || if (n > ADD_SUB_IMM) { - | LOAD_32BIT_VAL TMP2w, n - | add TMP1w, TMP1w, TMP2w - || } else { - | add TMP1w, TMP1w, #n - || } + | ADD_SUB_32_WITH_CONST add, TMP1w, TMP1w, n, TMP2w | str TMP1w, [sp] | b ->trace_exit @@ -8879,35 +8882,17 @@ static int zend_jit_init_method_call(dasm_State **Dst, if (func) { | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); | ldr REG0, EX->run_time_cache - || if (opline->result.num + sizeof(void*) > ADD_SUB_IMM) { - | LOAD_32BIT_VAL TMP1w, (opline->result.num + sizeof(void*)) - | add REG0, REG0, TMP1 - || } else { - | add REG0, REG0, #(opline->result.num + sizeof(void*)) - || } - | ldr REG0, [REG0] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG0, REG0, (opline->result.num + sizeof(void*)), TMP1 | cbz REG0, >1 } else { | // if (CACHED_PTR(opline->result.num) == obj->ce)) { | ldr REG0, EX->run_time_cache - || if (opline->result.num > ADD_SUB_IMM) { - | LOAD_32BIT_VAL TMP1w, opline->result.num - | add TMP1, REG0, TMP1 - || } else { - | add TMP1, REG0, #opline->result.num - || } - | ldr REG2, [TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG2, REG0, opline->result.num, TMP1 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] | cmp REG2, TMP1 | bne >1 | // fbc = CACHED_PTR(opline->result.num + sizeof(void*)); - || if (opline->result.num + sizeof(void*) > ADD_SUB_IMM) { - | LOAD_32BIT_VAL TMP1w, (opline->result.num + sizeof(void*)) - | add TMP1, TMP1, REG0 - || } else { - | add TMP1, REG0, #(opline->result.num + sizeof(void*)) - || } - | ldr REG0, [TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG0, REG0, (opline->result.num + sizeof(void*)), TMP1 } |.cold_code @@ -11854,12 +11839,7 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_ | LOAD_ADDR FCARG1x, (ptrdiff_t)varname | ldr FCARG2x, EX->run_time_cache if (opline->extended_value) { - if (opline->extended_value > ADD_SUB_IMM) { - | LOAD_32BIT_VAL TMP1, opline->extended_value - | add FCARG2x, FCARG2x, TMP1 - } else { - | add FCARG2x, FCARG2x, #opline->extended_value - } + | ADD_SUB_64_WITH_CONST_32 add, FCARG2x, FCARG2x, opline->extended_value, TMP1 } | EXT_CALL zend_jit_fetch_global_helper, REG0 | mov REG0, RETVALx @@ -12837,12 +12817,7 @@ static int zend_jit_incdec_obj(dasm_State **Dst, | // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); | LOAD_ADDR FCARG2x, name | ldr CARG3, EX->run_time_cache - if (opline->extended_value > ADD_SUB_IMM) { - | LOAD_32BIT_VAL TMP1, opline->extended_value - | add CARG3, CARG3, TMP1 - } else { - | add CARG3, CARG3, #opline->extended_value - } + | ADD_SUB_64_WITH_CONST_32 add, CARG3, CARG3, opline->extended_value, TMP1 if (opline->result_type == IS_UNUSED) { | mov CARG4, xzr } else { @@ -13146,12 +13121,7 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, | LOAD_ADDR FCARG2x, name | LOAD_ZVAL_ADDR CARG3, val_addr | ldr CARG4, EX->run_time_cache - if ((opline+1)->extended_value > ADD_SUB_IMM) { - | LOAD_32BIT_VAL TMP1w, (opline+1)->extended_value - | add CARG4, CARG4, TMP1 - } else { - | add CARG4, CARG4, #(opline+1)->extended_value - } + | ADD_SUB_64_WITH_CONST_32 add, CARG4, CARG4, (opline+1)->extended_value, TMP1 | LOAD_ADDR CARG5, binary_op | EXT_CALL zend_jit_assign_obj_op_helper, REG0 @@ -13296,13 +13266,7 @@ static int zend_jit_assign_obj(dasm_State **Dst, needs_slow_path = 1; | ldr REG0, EX->run_time_cache - if (opline->extended_value >= ADD_SUB_IMM) { - | LOAD_32BIT_VAL TMP1w, opline->extended_value - | add TMP1, REG0, TMP1 - } else { - | add TMP1, REG0, #opline->extended_value - } - | ldr REG2, [TMP1] + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG2, REG0, opline->extended_value, TMP1 | ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)] | cmp REG2, TMP1 | bne >5 @@ -13420,12 +13384,7 @@ static int zend_jit_assign_obj(dasm_State **Dst, | LOAD_ZVAL_ADDR CARG3, val_addr | ldr CARG4, EX->run_time_cache - if (opline->extended_value > ADD_SUB_IMM) { - | LOAD_32BIT_VAL TMP1w, opline->extended_value - | add CARG4, CARG4, TMP1 - } else { - | add CARG4, CARG4, #opline->extended_value - } + | ADD_SUB_64_WITH_CONST_32 add, CARG4, CARG4, opline->extended_value, TMP1 if (RETURN_VALUE_USED(opline)) { | LOAD_ZVAL_ADDR CARG5, res_addr } else { @@ -14047,12 +14006,7 @@ static bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, | ldr FCARG2x, EX->func | LOAD_ADDR CARG3, (ptrdiff_t)arg_info | ldr REG0, EX->run_time_cache - if (opline->op2.num > ADD_SUB_IMM) { - | LOAD_32BIT_VAL TMP1, opline->op2.num - | add CARG4, REG0, TMP1 - } else { - | add CARG4, REG0, #opline->op2.num - } + | ADD_SUB_64_WITH_CONST_32 add, CARG4, REG0, opline->op2.num, TMP1 | EXT_CALL zend_jit_verify_return_slow, REG0 if (!zend_jit_check_exception(Dst)) { return 0; From 38474a4144871fe7f39297e45049b4d183a07213 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 30 Apr 2021 14:36:41 +0300 Subject: [PATCH 190/229] Use less temporary registers --- ext/opcache/jit/zend_jit_arm64.dasc | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 1210f1f18efe4..9b1cc986e5dc6 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -835,11 +835,11 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) // Define LONG_ADD_SUB, LONG_BW_OP, LONG_MUL, LONG_CMP to replace the LONG_OP in x86 implementation. // 'long_ins' should be addition or subtraction. -|.macro LONG_ADD_SUB, long_ins, reg, addr, tmp_reg1, tmp_reg2 +|.macro LONG_ADD_SUB, long_ins, reg, addr, tmp_reg1 || if (Z_MODE(addr) == IS_CONST_ZVAL) { | ADD_SUB_64_WITH_CONST long_ins, Rx(reg), Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg1 || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { -| SAFE_MEM_ACC_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg2 +| SAFE_MEM_ACC_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg1 | long_ins Rx(reg), Rx(reg), tmp_reg1 || } else if (Z_MODE(addr) == IS_REG) { | long_ins Rx(reg), Rx(reg), Rx(Z_REG(addr)) @@ -849,11 +849,11 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.endmacro // 'long_ins' should be 'and', 'orr' or 'eor' -|.macro LONG_BW_OP, long_ins, reg, addr, tmp_reg1, tmp_reg2 +|.macro LONG_BW_OP, long_ins, reg, addr, tmp_reg1 || if (Z_MODE(addr) == IS_CONST_ZVAL) { | BW_OP_64_WITH_CONST long_ins, Rx(reg), Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg1 || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { -| SAFE_MEM_ACC_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg2 +| SAFE_MEM_ACC_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg1 | long_ins Rx(reg), Rx(reg), tmp_reg1 || } else if (Z_MODE(addr) == IS_REG) { | long_ins Rx(reg), Rx(reg), Rx(Z_REG(addr)) @@ -930,22 +930,22 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) || } |.endmacro -|.macro LONG_MATH, opcode, reg, addr, tmp_reg1, tmp_reg2 +|.macro LONG_MATH, opcode, reg, addr, tmp_reg1 || switch (opcode) { || case ZEND_ADD: -| LONG_ADD_SUB adds, reg, addr, tmp_reg1, tmp_reg2 +| LONG_ADD_SUB adds, reg, addr, tmp_reg1 || break; || case ZEND_SUB: -| LONG_ADD_SUB subs, reg, addr, tmp_reg1, tmp_reg2 +| LONG_ADD_SUB subs, reg, addr, tmp_reg1 || break; || case ZEND_BW_OR: -| LONG_BW_OP orr, reg, addr, tmp_reg1, tmp_reg2 +| LONG_BW_OP orr, reg, addr, tmp_reg1 || break; || case ZEND_BW_AND: -| LONG_BW_OP and, reg, addr, tmp_reg1, tmp_reg2 +| LONG_BW_OP and, reg, addr, tmp_reg1 || break; || case ZEND_BW_XOR: -| LONG_BW_OP eor, reg, addr, tmp_reg1, tmp_reg2 +| LONG_BW_OP eor, reg, addr, tmp_reg1 || break; || default: || ZEND_UNREACHABLE(); @@ -3939,7 +3939,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, } else if (same_ops && opcode != ZEND_DIV) { | LONG_MATH_REG opcode, Rx(result_reg), Rx(result_reg), Rx(result_reg) } else { - | LONG_MATH opcode, result_reg, op2_addr, TMP1, TMP2 + | LONG_MATH opcode, result_reg, op2_addr, TMP1 } } if (may_overflow) { @@ -4652,7 +4652,7 @@ static int zend_jit_long_math_helper(dasm_State **Dst, | LONG_MATH_REG opcode, Rx(result_reg), Rx(result_reg), Rx(result_reg) } else { | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 - | LONG_MATH opcode, result_reg, op2_addr, TMP1, TMP2 + | LONG_MATH opcode, result_reg, op2_addr, TMP1 } if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != result_reg) { From db25a6ed16948981c344e79427e6072f4365caf1 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 30 Apr 2021 14:43:17 +0300 Subject: [PATCH 191/229] Use single instruction --- ext/opcache/jit/zend_jit_arm64.dasc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 9b1cc986e5dc6..123bbeee2427a 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -8898,9 +8898,10 @@ static int zend_jit_init_method_call(dasm_State **Dst, |.cold_code |1: | LOAD_ADDR FCARG2x, function_name - | mov CARG3, sp - if (TMP_ZVAL_OFFSET != 0) { - | add CARG3, CARG3, #TMP_ZVAL_OFFSET + if (TMP_ZVAL_OFFSET == 0) { + | mov CARG3, sp + } else { + | add CARG3, sp, #TMP_ZVAL_OFFSET } | SET_EX_OPLINE opline, REG0 if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) { From 6036d159fb5ab363bef8fff437ef3cee1ab2fedd Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 30 Apr 2021 17:43:51 +0300 Subject: [PATCH 192/229] Imroved code for CV initialization --- ext/opcache/jit/zend_jit_arm64.dasc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 123bbeee2427a..549e61f030d4a 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -9345,12 +9345,10 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | // opline = op_array->opcodes; if (func && !unknown_num_args) { + | ADD_SUB_64_WITH_CONST_32 add, TMP1, RX, (EX_NUM_TO_VAR(call_num_args) + offsetof(zval, u1.type_info)), TMP1 // induction variable for (i = call_num_args; i < func->op_array.last_var; i++) { - uint32_t n = EX_NUM_TO_VAR(i); | // ZVAL_UNDEF(EX_VAR(n)) - || ZEND_ASSERT(n <= ADD_SUB_IMM); - | add TMP1, RX, #n - | SET_Z_TYPE_INFO TMP1, IS_UNDEF, TMP2w + | str wzr, [TMP1], #16 } if (call_num_args <= func->op_array.num_args) { From ce9d59a34795339a821d425aeca8834305a91e3a Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 30 Apr 2021 17:44:23 +0300 Subject: [PATCH 193/229] Imroved code for constants loading --- ext/opcache/jit/zend_jit_arm64.dasc | 72 +++++++++++++++++++++++------ 1 file changed, 59 insertions(+), 13 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 549e61f030d4a..66042b2afe4ad 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -201,29 +201,75 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.macro LOAD_ADDR, reg, addr | // 48-bit virtual address -| mov reg, #((uintptr_t)(addr) & 0xffff) -| movk reg, #(((uintptr_t)(addr) >> 16) & 0xffff), lsl #16 -| movk reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32 +|| if (((uintptr_t)(addr)) == 0) { +| mov reg, xzr +|| } else if (((uintptr_t)(addr)) <= MOVZ_IMM) { +| movz reg, #((uint64_t)(addr)) +|| } else if ((uintptr_t)(addr) & 0xffff) { +| movz reg, #((uintptr_t)(addr) & 0xffff) +|| if (((uintptr_t)(addr) >> 16) & 0xffff) { +| movk reg, #(((uintptr_t)(addr) >> 16) & 0xffff), lsl #16 +|| } +|| if (((uintptr_t)(addr) >> 32) & 0xffff) { +| movk reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32 +|| } +|| } else if (((uintptr_t)(addr) >> 16) & 0xffff) { +| movz reg, #(((uintptr_t)(addr) >> 16) & 0xffff), lsl #16 +|| if (((uintptr_t)(addr) >> 32) & 0xffff) { +| movk reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32 +|| } +|| } else { +| movz reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32 +|| } |.endmacro // Type cast to unsigned is used to avoid undefined behavior. |.macro LOAD_32BIT_VAL, reg, val -|| if (((uintptr_t)(val)) <= MOVZ_IMM) { -| movz reg, #val +|| if (((uint32_t)(val)) <= MOVZ_IMM) { +| movz reg, #((uint32_t)(val)) +|| } else if (((uint32_t)(val) & 0xffff)) { +| movz reg, #((uint32_t)(val) & 0xffff) +|| if ((((uint32_t)(val) >> 16) & 0xffff)) { +| movk reg, #(((uint32_t)(val) >> 16) & 0xffff), lsl #16 +|| } || } else { -| mov reg, #((uint32_t)(val) & 0xffff) -| movk reg, #(((uint32_t)(val) >> 16) & 0xffff), lsl #16 +| movz reg, #(((uint32_t)(val) >> 16) & 0xffff), lsl #16 || } |.endmacro |.macro LOAD_64BIT_VAL, reg, val -|| if (((uintptr_t)(val)) <= MOVZ_IMM) { -| movz reg, #val +|| if (((uint64_t)(val)) == 0) { +| mov reg, xzr +|| } else if (((uint64_t)(val)) <= MOVZ_IMM) { +| movz reg, #((uint64_t)(val)) +|| } else if (~((uint64_t)(val)) <= MOVZ_IMM) { +| movn reg, #(~((uint64_t)(val))) +|| } else if ((uint64_t)(val) & 0xffff) { +| movz reg, #((uint64_t)(val) & 0xffff) +|| if (((uint64_t)(val) >> 16) & 0xffff) { +| movk reg, #(((uint64_t)(val) >> 16) & 0xffff), lsl #16 +|| } +|| if (((uint64_t)(val) >> 32) & 0xffff) { +| movk reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 +|| } +|| if ((((uint64_t)(val) >> 48) & 0xffff)) { +| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 +|| } +|| } else if (((uint64_t)(val) >> 16) & 0xffff) { +| movz reg, #(((uint64_t)(val) >> 16) & 0xffff), lsl #16 +|| if (((uint64_t)(val) >> 32) & 0xffff) { +| movk reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 +|| } +|| if ((((uint64_t)(val) >> 48) & 0xffff)) { +| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 +|| } +|| } else if (((uint64_t)(val) >> 32) & 0xffff) { +| movz reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 +|| if ((((uint64_t)(val) >> 48) & 0xffff)) { +| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 +|| } || } else { -| mov reg, #((uint64_t)(val) & 0xffff) -| movk reg, #(((uint64_t)(val) >> 16) & 0xffff), lsl #16 -| movk reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32 -| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 +| movz reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48 || } |.endmacro From 123df80d40275dc4f89f2575edacc3b976c646b1 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 30 Apr 2021 19:36:21 +0300 Subject: [PATCH 194/229] Use B/BL intead of BR/BLR if possible --- ext/opcache/jit/zend_jit_arm64.dasc | 30 +++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 66042b2afe4ad..69ee2f6cd3019 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -103,6 +103,20 @@ #define LDR_STR_PIMM32 (MAX_IMM12*4) // ldr/str insn for 32-bit register: pimm is imm12 * 4 #define LDRB_STRB_PIMM MAX_IMM12 // ldrb/strb insn +#define B_IMM26 (((1<<26)-1)*4) + +static bool arm64_may_use_b(void *addr) +{ + if (addr >= dasm_buf && addr < dasm_end) { + return (((char*)dasm_end - (char*)dasm_buf) < B_IMM26); + } else if (addr >= dasm_end) { + return (((char*)addr - (char*)dasm_buf) < B_IMM26); + } else if (addr < dasm_buf) { + return (((char*)dasm_end - (char*)addr) < B_IMM26); + } + return 0; +} + #include "Zend/zend_cpuinfo.h" #ifdef HAVE_VALGRIND @@ -609,13 +623,21 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.endmacro |.macro EXT_CALL, func, tmp_reg -| LOAD_ADDR tmp_reg, func -| blr tmp_reg +|| if (arm64_may_use_b(func)) { +| bl &func +|| } else { +| LOAD_ADDR tmp_reg, func +| blr tmp_reg +|| } |.endmacro |.macro EXT_JMP, func, tmp_reg -| LOAD_ADDR tmp_reg, func -| br tmp_reg +|| if (arm64_may_use_b(func)) { +| b &func +|| } else { +| LOAD_ADDR tmp_reg, func +| br tmp_reg +|| } |.endmacro |.macro SAVE_IP From fc377742a673cdc765d2d9ad5317ca7f2616ee34 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 30 Apr 2021 19:44:58 +0300 Subject: [PATCH 195/229] Fix compilation warning --- ext/opcache/jit/zend_jit_arm64.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 69ee2f6cd3019..00c8e2d541bae 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -105,7 +105,7 @@ #define B_IMM26 (((1<<26)-1)*4) -static bool arm64_may_use_b(void *addr) +static bool arm64_may_use_b(const void *addr) { if (addr >= dasm_buf && addr < dasm_end) { return (((char*)dasm_end - (char*)dasm_buf) < B_IMM26); From 7ece98f2f42a2fd98125f2d39416bddf99e5ffcb Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 7 May 2021 09:51:46 +0300 Subject: [PATCH 196/229] Use proper macro --- ext/opcache/jit/zend_jit_arm64.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 00c8e2d541bae..acc63765d52a2 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -11841,7 +11841,7 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_ | // idx = (uint32_t)(uintptr_t)CACHED_PTR(opline->extended_value) - 1; | ldr REG0, EX->run_time_cache - | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldr, REG0, REG0, opline->extended_value, TMP1 + | SAFE_MEM_ACC_WITH_UOFFSET ldr, REG0, REG0, opline->extended_value, TMP1 | sub REG0, REG0, #1 | // if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket))) | MEM_LOAD_32_ZTS ldr, REG1w, executor_globals, symbol_table.nNumUsed, REG1 From 609c3bee2e458887233986400c97303f6df07415 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 7 May 2021 09:57:34 +0300 Subject: [PATCH 197/229] 'lea' -> 'add' --- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index fdd6ec9830f08..d69b92d74dc6a 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4223,7 +4223,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, | lea Ra(result_reg), [Ra(Z_REG(op2_addr))+Ra(Z_REG(op2_addr))] } else { | GET_ZVAL_LVAL result_reg, op2_addr - | lea Ra(result_reg), [Ra(result_reg)+Ra(result_reg)] + | add Ra(result_reg), Ra(result_reg) } } else if (opcode == ZEND_MUL && Z_MODE(op1_addr) == IS_CONST_ZVAL && From b019ffe59cb4f1cdf3bc49876f34b516f9eb9536 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Thu, 6 May 2021 05:25:17 +0000 Subject: [PATCH 198/229] Don't use TMP3 except ZTS Currently temporary register TMP3 is used in particular for ZTS mode. Hence it would be better not to use it in other sites except ZTS. In this patch, the value in TMP1, i.e. "REG1 + IP", might be clobbered and should be calcauted again. Change-Id: I17434d4f4b8d6f1e71f49eadb42e06db975d7bbe --- ext/opcache/jit/zend_jit_arm64.dasc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index acc63765d52a2..3b03c99a6d4c4 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -2290,9 +2290,12 @@ static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost) | add TMP1, REG1, IP | ldr REG2, [TMP1, #offsetof(zend_op_trace_info, counter)] | ldrh TMP2w, [REG2] - | ADD_SUB_32_WITH_CONST subs, TMP2w, TMP2w, cost, TMP3w + | ADD_SUB_32_WITH_CONST subs, TMP2w, TMP2w, cost, TMP1w | strh TMP2w, [REG2] | ble ->hybrid_hot_trace + // Note: "REG1 + IP" is re-calculated as TMP1 is used as temporary register by the prior + // ADD_SUB_32_WITH_CONST. Will optimize in the future if more temporary registers are available. + | add TMP1, REG1, IP | ldr TMP2, [TMP1, #offsetof(zend_op_trace_info, orig_handler)] | br TMP2 From 8c794128be66ffb4f56029d7d9518ce58e471323 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Thu, 6 May 2021 05:55:12 +0000 Subject: [PATCH 199/229] Use ADD_SUB_IMM for macros ADD_SUB_*_WITH_CONST* Macros ADD_SUB_*_WITH_CONST* were introduced in [1], but it would be more accurate to use ADD_SUB_IMM even though ADD_SUB_IMM == CMP_IMM. Regarding the definition of ADD_SUB_IMM, the comment is updated since it is used to guard 'subs' and adds' instructions as well. [1] https://github.com/php/php-src/commit/0609b97 Change-Id: I102ebbf626a438da061f6c01b01029807de1ff14 --- ext/opcache/jit/zend_jit_arm64.dasc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 3b03c99a6d4c4..06f2df3868967 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -98,7 +98,7 @@ #define MAX_IMM16 0xffff // maximum value for imm16 #define CMP_IMM MAX_IMM12 // cmp insn #define MOVZ_IMM MAX_IMM16 // movz insn -#define ADD_SUB_IMM MAX_IMM12 // add/sub insn +#define ADD_SUB_IMM MAX_IMM12 // add/sub/adds/subs insn #define LDR_STR_PIMM64 (MAX_IMM12*8) // ldr/str insn for 64-bit register: pimm is imm12 * 8 #define LDR_STR_PIMM32 (MAX_IMM12*4) // ldr/str insn for 32-bit register: pimm is imm12 * 4 #define LDRB_STRB_PIMM MAX_IMM12 // ldrb/strb insn @@ -387,7 +387,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.macro ADD_SUB_32_WITH_CONST, ins, res_reg, op1_reg, val, tmp_reg || if (val == 0) { | ins res_reg, op1_reg, wzr -|| } else if (((int32_t)(val)) > 0 && ((int32_t)(val)) <= CMP_IMM) { +|| } else if (((int32_t)(val)) > 0 && ((int32_t)(val)) <= ADD_SUB_IMM) { | ins res_reg, op1_reg, #val || } else { | LOAD_32BIT_VAL tmp_reg, val @@ -398,7 +398,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.macro ADD_SUB_64_WITH_CONST_32, ins, res_reg, op1_reg, val, tmp_reg || if (val == 0) { | ins res_reg, op1_reg, xzr -|| } else if (((int32_t)(val)) > 0 && ((int32_t)(val)) <= CMP_IMM) { +|| } else if (((int32_t)(val)) > 0 && ((int32_t)(val)) <= ADD_SUB_IMM) { | ins res_reg, op1_reg, #val || } else { | LOAD_32BIT_VAL tmp_reg, val @@ -409,7 +409,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.macro ADD_SUB_64_WITH_CONST, ins, res_reg, op1_reg, val, tmp_reg || if (val == 0) { | ins res_reg, op1_reg, xzr -|| } else if (((int64_t)(val)) > 0 && ((int64_t)(val)) <= CMP_IMM) { +|| } else if (((int64_t)(val)) > 0 && ((int64_t)(val)) <= ADD_SUB_IMM) { | ins res_reg, op1_reg, #val || } else { | LOAD_64BIT_VAL tmp_reg, val From 4c6d31773093e129578c1775316cef728965a2b2 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Thu, 6 May 2021 08:28:41 +0000 Subject: [PATCH 200/229] Use macros CMP_*_WITH_CONST if possible Macros CMP_*_WITH_CONST were introduced in [1]. We revisit all the uses of 'cmp' instructions with constants, and apply these macros if possible. [1] https://github.com/php/php-src/commit/66ba9af Change-Id: Idd233cb9afba84dd106fcc5ee8bf5417d91989c7 --- ext/opcache/jit/zend_jit_arm64.dasc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 06f2df3868967..37a129ada41bf 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -4980,8 +4980,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o if (val == 0) { | cmp REG0, xzr } else if (val > 0 && !op2_loaded) { - | LOAD_64BIT_VAL TMP1, val - | cmp REG0, TMP1 + | CMP_64_WITH_CONST REG0, val, TMP1 } else { | cmp REG0, FCARG2x } @@ -7575,7 +7574,7 @@ static int zend_jit_identical(dasm_State **Dst, zval *val = Z_ZV(op1_addr); | ldrb TMP1w, [FCARG2x, #offsetof(zval, u1.v.type)] - | cmp TMP1w, #Z_TYPE_P(val) + | CMP_32_WITH_CONST TMP1w, Z_TYPE_P(val), TMP2w if (smart_branch_opcode) { if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) { | bne >8 @@ -7627,7 +7626,7 @@ static int zend_jit_identical(dasm_State **Dst, zval *val = Z_ZV(op2_addr); | ldrb TMP1w, [FCARG1x, #offsetof(zval, u1.v.type)] - | cmp TMP1w, #Z_TYPE_P(val) + | CMP_32_WITH_CONST TMP1w, Z_TYPE_P(val), TMP2w if (smart_branch_opcode) { if (opline->opcode != ZEND_CASE_STRICT && opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) { @@ -9503,8 +9502,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | // num_args = EX_NUM_ARGS(); | ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] | // if (UNEXPECTED(num_args > first_extra_arg)) - || ZEND_ASSERT(func->op_array.num_args <= CMP_IMM); - | cmp REG1w, #(func->op_array.num_args) + | CMP_32_WITH_CONST REG1w, (func->op_array.num_args), TMP1w } else { | // first_extra_arg = op_array->num_args; | ldr REG2w, [REG0, #offsetof(zend_op_array, num_args)] @@ -10590,6 +10588,8 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1 } |2: + // Note: 'type' is of uchar type and holds a positive value, + // hence it's safe to directly encode it as the imm field of 'cmp' instruction. | cmp REG1w, #type } else { if (op1_info & MAY_BE_REF) { From b6e925962036b3d2cdc1eb60c7b427a53b7f92d9 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Thu, 6 May 2021 09:46:45 +0000 Subject: [PATCH 201/229] Remove the deprecated macros Macros PUSH_BASE_ADDR, PUSH_ADDR_ZTS, GET_ZVAL_W2, SET_ZVAL_W2, IS_32BIT, PUSH_ADDR and PUSH_ZVAL_ADDR are 32-bit platform dependent in the x86 implementation, hence they are derepcated in AArch64. Macro ADDR_OP1 is only used by PASH_* macros, and it is deprecated in AArch64. Macros MEM_OP3_3 and AVX_OP are platform dependent and they are not used in AArch64. Macro LONG_OP_WITH_32BIT_CONST is replaced with LONG_ADD_SUB_WITH_IMM. Macro DOUBLE_OP is not used because we use temporary FP register to load the value from address firstly and then conduct FP math operations with the help of DOUBLE_MATH_REG. Macro LONG_MUL is deprecated. MUL between two LONG values is implemented in a different way from ADD or SUB mainly because extra temporary register is needed in order to detect integer overflow. See function zend_jit_math_long_long(). Change-Id: I249f28820aaa86c7638ecf3eee08f8750a232397 --- ext/opcache/jit/zend_jit_arm64.dasc | 92 ++++------------------------- 1 file changed, 12 insertions(+), 80 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 37a129ada41bf..6f301ff63b12d 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -172,8 +172,6 @@ static void* dasm_labels[zend_lb_MAX]; |.section code, cold_code, jmp_table -#define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0x7fffffff) - #define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= (-2147483647 - 1))) #define BP_JIT_IS 6 @@ -464,10 +462,6 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | .endif |.endmacro -|.macro ADDR_OP1, addr_ins, addr, tmp_reg -| NIY // TODO -|.endmacro - // Move the 48-bit address 'addr' into 'tmp_reg' and store it into the dest addr 'op1' |.macro ADDR_STORE, op1, addr, tmp_reg | LOAD_ADDR tmp_reg, addr @@ -481,14 +475,6 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | cmp tmp_reg2, tmp_reg1 |.endmacro -|.macro PUSH_ADDR, addr, tmp_reg -| ADDR_OP1 push, addr, tmp_reg -|.endmacro - -|.macro PUSH_ADDR_ZTS, struct, field, tmp_reg -| NIY // TODO -|.endmacro - // Store the value from a register 'op' into memory 'addr' |.macro MEM_STORE, str_ins, op, addr, tmp_reg | LOAD_ADDR tmp_reg, addr @@ -602,10 +588,6 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | .endif |.endmacro -|.macro MEM_OP3_3, mem_ins, op1, op2, prefix, addr, tmp_reg -| NIY // TODO -|.endmacro - |.macro LOAD_BASE_ADDR, reg, base, offset || if (offset) { | ADD_SUB_64_WITH_CONST_32 add, reg, Rx(base), offset, reg @@ -618,10 +600,6 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) || } |.endmacro -|.macro PUSH_BASE_ADDR, base, offset, tmp_reg -| NIY // TODO -|.endmacro - |.macro EXT_CALL, func, tmp_reg || if (arm64_may_use_b(func)) { | bl &func @@ -737,16 +715,6 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) || } |.endmacro -|.macro PUSH_ZVAL_ADDR, addr, tmp_reg -|| if (Z_MODE(addr) == IS_CONST_ZVAL) { -| PUSH_ADDR Z_ZV(addr), tmp_reg -|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { -| PUSH_BASE_ADDR Z_REG(addr), Z_OFFSET(addr), tmp_reg -|| } else { -|| ZEND_UNREACHABLE(); -|| } -|.endmacro - |.macro GET_Z_TYPE_INFO, reg, zv | ldr reg, [zv, #offsetof(zval,u1.type_info)] |.endmacro @@ -805,16 +773,6 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | SAFE_MEM_ACC_WITH_UOFFSET str, val, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg |.endmacro -|.macro GET_ZVAL_W2, reg, addr -|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -| NIY // TODO -|.endmacro - -|.macro SET_ZVAL_W2, addr, val -|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -| NIY // TODO -|.endmacro - |.macro UNDEF_OPLINE_RESULT, tmp_reg | ldr REG0, EX->opline | ldr REG0w, OP:REG0->result.var @@ -822,35 +780,18 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | SET_Z_TYPE_INFO REG0, IS_UNDEF, tmp_reg |.endmacro -// Define DOUBLE_CMP and DOUBLE_CMP to replace SSE_AVX_OP and SSE_OP in x86 implementation. -// Conduct floating point operation 'ins'. Operand1 is from 'reg', and operands2 is from 'addr'. -// Use DOUBLE_CMP for comparisons and use DOUBLE_OP for arithmetic operations. -|.macro DOUBLE_CMP, ins, reg, addr, tmp_reg, fp_tmp_reg +// Define DOUBLE_CMP to replace SSE_AVX_OP and SSE_OP for comparions in x86 implementation. +// Operand1 is from 'reg', and operand2 is from 'addr'. +|.macro DOUBLE_CMP, reg, addr, tmp_reg, fp_tmp_reg || if (Z_MODE(addr) == IS_CONST_ZVAL) { | LOAD_ADDR Rx(tmp_reg), Z_ZV(addr) | ldr Rd(fp_tmp_reg-ZREG_V0), [Rx(tmp_reg)] -| ins Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) +| fcmp Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { | SAFE_MEM_ACC_WITH_UOFFSET ldr, Rd(fp_tmp_reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) -| ins Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) +| fcmp Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) || } else if (Z_MODE(addr) == IS_REG) { -| ins Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) -|| } else { -|| ZEND_UNREACHABLE(); -|| } -|.endmacro - -|.macro DOUBLE_OP, ins, reg, addr, tmp_reg, fp_tmp_reg -| NIY // TODO -|| if (Z_MODE(addr) == IS_CONST_ZVAL) { -| LOAD_ADDR Rx(tmp_reg), Z_ZV(addr) -| ldr Rd(fp_tmp_reg-ZREG_V0), [Rx(tmp_reg)] -| ins Rd(reg-ZREG_V0), Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) -|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) { -| SAFE_MEM_ACC_WITH_UOFFSET ldr, Rd(fp_tmp_reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg) -| ins Rd(reg-ZREG_V0), Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0) -|| } else if (Z_MODE(addr) == IS_REG) { -| ins Rd(reg-ZREG_V0), Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) +| fcmp Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0) || } else { || ZEND_UNREACHABLE(); || } @@ -930,11 +871,6 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) || } |.endmacro -// TODO: integer overflow should be detected. -|.macro LONG_MUL, long_ins, reg, addr, tmp_reg1, tmp_reg2 -| brk #0 // TODO -|.endmacro - |.macro LONG_CMP, reg, addr, tmp_reg1, tmp_reg2 || if (Z_MODE(addr) == IS_CONST_ZVAL) { | CMP_64_WITH_CONST Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg1 @@ -962,10 +898,6 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) || } |.endmacro -|.macro LONG_OP_WITH_32BIT_CONST, long_ins, op1_addr, lval -| NIY // TODO -|.endmacro - // Define LONG_CMP_WITH_CONST to replace LONG_OP_WITH_CONST in the x86 implementation. // Note that the 'long_ins' in all use sites of LONG_OP_WITH_CONST are always 'cmp'. // Note that this macro is different from LONG_CMP. @@ -6935,7 +6867,7 @@ static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zen zend_reg tmp_reg = ZREG_FPR0; | DOUBLE_GET_ZVAL_LVAL tmp_reg, op1_addr, ZREG_REG0, ZREG_TMP1 - | DOUBLE_CMP fcmp, tmp_reg, op2_addr, ZREG_TMP1, ZREG_FPTMP + | DOUBLE_CMP tmp_reg, op2_addr, ZREG_TMP1, ZREG_FPTMP return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2, exit_addr); } @@ -6945,7 +6877,7 @@ static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zen zend_reg tmp_reg = ZREG_FPR0; | DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, ZREG_REG0, ZREG_TMP1 - | DOUBLE_CMP fcmp, tmp_reg, op1_addr, ZREG_TMP1, ZREG_FPTMP + | DOUBLE_CMP tmp_reg, op1_addr, ZREG_TMP1, ZREG_FPTMP return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2, exit_addr); } @@ -6955,15 +6887,15 @@ static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, z bool swap = 0; if (Z_MODE(op1_addr) == IS_REG) { - | DOUBLE_CMP fcmp, Z_REG(op1_addr), op2_addr, ZREG_TMP1, ZREG_FPTMP + | DOUBLE_CMP Z_REG(op1_addr), op2_addr, ZREG_TMP1, ZREG_FPTMP } else if (Z_MODE(op2_addr) == IS_REG) { - | DOUBLE_CMP fcmp, Z_REG(op2_addr), op1_addr, ZREG_TMP1, ZREG_FPTMP + | DOUBLE_CMP Z_REG(op2_addr), op1_addr, ZREG_TMP1, ZREG_FPTMP swap = 1; } else { zend_reg tmp_reg = ZREG_FPR0; | GET_ZVAL_DVAL tmp_reg, op1_addr, ZREG_TMP1 - | DOUBLE_CMP fcmp, tmp_reg, op2_addr, ZREG_TMP1, ZREG_FPTMP + | DOUBLE_CMP tmp_reg, op2_addr, ZREG_TMP1, ZREG_FPTMP } return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2, exit_addr); @@ -8020,7 +7952,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ if ((op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { | mov TMP1, xzr | fmov FPR0d, TMP1 - | DOUBLE_CMP fcmp, ZREG_FPR0, op1_addr, ZREG_TMP1, ZREG_FPTMP + | DOUBLE_CMP ZREG_FPR0, op1_addr, ZREG_TMP1, ZREG_FPTMP if (set_bool) { if (exit_addr) { From 65b834a686e04387ff4d090e3504d82534d8bb7f Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Thu, 6 May 2021 14:04:53 +0000 Subject: [PATCH 202/229] Improve macro LONG_ADD_SUB_WITH_IMM Offset "Z_OFFSET(op1_addr)" might be loaded into temporary registers for two times, i.e. by these two SAFE_MEM_ACC_WITH_UOFFSET respectively. This patch removes this case. Change-Id: Ib1fde9c9a39f6b5f1ca322dc27b0a9d9d51fbef0 --- ext/opcache/jit/zend_jit_arm64.dasc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 6f301ff63b12d..4678faf7073f0 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -888,9 +888,16 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.macro LONG_ADD_SUB_WITH_IMM, long_ins, op1_addr, lval, tmp_reg1, tmp_reg2 || ZEND_ASSERT(lval >=0 && lval <= ADD_SUB_IMM); || if (Z_MODE(op1_addr) == IS_MEM_ZVAL) { -| SAFE_MEM_ACC_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(op1_addr)), Z_OFFSET(op1_addr), tmp_reg2 -| long_ins tmp_reg1, tmp_reg1, #lval -| SAFE_MEM_ACC_WITH_UOFFSET str, tmp_reg1, Rx(Z_REG(op1_addr)), Z_OFFSET(op1_addr), tmp_reg2 +|| if (((uint32_t)(Z_OFFSET(op1_addr))) > LDR_STR_PIMM64) { +| LOAD_32BIT_VAL tmp_reg2, Z_OFFSET(op1_addr) +| ldr tmp_reg1, [Rx(Z_REG(op1_addr)), tmp_reg2] +| long_ins tmp_reg1, tmp_reg1, #lval +| str tmp_reg1, [Rx(Z_REG(op1_addr)), tmp_reg2] +|| } else { +| ldr tmp_reg1, [Rx(Z_REG(op1_addr)), #Z_OFFSET(op1_addr)] +| long_ins tmp_reg1, tmp_reg1, #lval +| str tmp_reg1, [Rx(Z_REG(op1_addr)), #Z_OFFSET(op1_addr)] +|| } || } else if (Z_MODE(op1_addr) == IS_REG) { | long_ins Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)), #lval || } else { From 77ecce7a6078f00358d848bd95578299a5f628c2 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Thu, 6 May 2021 15:07:45 +0000 Subject: [PATCH 203/229] Use fewer temporary registers if possible For macro "SAFE_MEM_ACC_WITH_UOFFSET* ldr/ldrb", the target register can be used as temporary register to load the offset if needed. Hence one temporary register is saved for macros LONG_CMP, GET_ZVAL_LVAL, CMP_ZVAL_TYPE, IF_ZVAL_TYPE and IF_NOT_ZVAL_TYPE. Change-Id: I43f35186cec9a4157bc73b1cae6643aa59140de2 --- ext/opcache/jit/zend_jit_arm64.dasc | 397 ++++++++++++++-------------- 1 file changed, 198 insertions(+), 199 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 4678faf7073f0..6f8eda9634da8 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -415,10 +415,8 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) || } |.endmacro -// Safe memory load/store with an unsigned immediate offset. -// When using Z_OFFSET(addr), which is 24-bit long, as the unsigned offset to compute a memory address, -// we should firstly check the range. -// Use SAFE_MEM_ACC_WITH_UOFFSET if 'op' is 64-bit register. Use SAFE_MEM_ACC_WITH_UOFFSET_32 if 'op' is 32-bit register. +// Safe memory load/store with an unsigned 32-bit offset. +// 'op' can be used as 'tmp_reg' if 1) 'op' is one GPR, and 2) 'op' != 'base_reg', and 3) ins is 'ldr'. |.macro SAFE_MEM_ACC_WITH_UOFFSET, ldr_str_ins, op, base_reg, offset, tmp_reg || if (((uintptr_t)(offset)) > LDR_STR_PIMM64) { | LOAD_32BIT_VAL tmp_reg, offset @@ -871,12 +869,12 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) || } |.endmacro -|.macro LONG_CMP, reg, addr, tmp_reg1, tmp_reg2 +|.macro LONG_CMP, reg, addr, tmp_reg || if (Z_MODE(addr) == IS_CONST_ZVAL) { -| CMP_64_WITH_CONST Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg1 +| CMP_64_WITH_CONST Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { -| SAFE_MEM_ACC_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg2 -| cmp Rx(reg), tmp_reg1 +| SAFE_MEM_ACC_WITH_UOFFSET ldr, tmp_reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg +| cmp Rx(reg), tmp_reg || } else if (Z_MODE(addr) == IS_REG) { | cmp Rx(reg), Rx(Z_REG(addr)) || } else { @@ -919,7 +917,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) || } |.endmacro -|.macro GET_ZVAL_LVAL, reg, addr, tmp_reg +|.macro GET_ZVAL_LVAL, reg, addr || if (Z_MODE(addr) == IS_CONST_ZVAL) { || if (Z_LVAL_P(Z_ZV(addr)) == 0) { | mov Rx(reg), xzr @@ -927,7 +925,8 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | LOAD_64BIT_VAL Rx(reg), Z_LVAL_P(Z_ZV(addr)) || } || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { -| SAFE_MEM_ACC_WITH_UOFFSET ldr, Rx(reg), Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg +|| ZEND_ASSERT(reg != Z_REG(addr)); +| SAFE_MEM_ACC_WITH_UOFFSET ldr, Rx(reg), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(reg) || } else if (Z_MODE(addr) == IS_REG) { || if (reg != Z_REG(addr)) { | mov Rx(reg), Rx(Z_REG(addr)) @@ -1139,9 +1138,9 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg) || } || } else if (Z_MODE(dst_addr) == IS_REG) { -| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr, Rx(tmp_reg) +| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr || } else { -| GET_ZVAL_LVAL reg2, src_addr, Rx(tmp_reg) +| GET_ZVAL_LVAL reg2, src_addr | SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(reg2), Rx(tmp_reg) || } || } else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { @@ -1173,15 +1172,15 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg) || } || } else if (Z_MODE(dst_addr) == IS_REG) { -| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr, Rx(tmp_reg) +| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr || if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(dst_addr)) { | SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(dst_addr)), Rx(tmp_reg) || } || } else if (Z_MODE(res_addr) == IS_REG) { -| GET_ZVAL_LVAL Z_REG(res_addr), src_addr, Rx(tmp_reg) +| GET_ZVAL_LVAL Z_REG(res_addr), src_addr | SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(res_addr)), Rx(tmp_reg) || } else { -| GET_ZVAL_LVAL reg2, src_addr, Rx(tmp_reg) +| GET_ZVAL_LVAL reg2, src_addr | SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(reg2), Rx(tmp_reg) | SET_ZVAL_LVAL_FROM_REG res_addr, Rx(reg2), Rx(tmp_reg) || } @@ -1258,23 +1257,23 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | IF_NOT_TYPE tmp_reg, val, label |.endmacro -|.macro CMP_ZVAL_TYPE, addr, val, tmp_reg1, tmp_reg2 +|.macro CMP_ZVAL_TYPE, addr, val, tmp_reg || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); || ZEND_ASSERT(val <= CMP_IMM); -| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), tmp_reg2 -| cmp tmp_reg1, #val +| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, Rw(tmp_reg), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), Rx(tmp_reg) +| cmp Rw(tmp_reg), #val |.endmacro -|.macro IF_ZVAL_TYPE, addr, val, label, tmp_reg1, tmp_reg2 +|.macro IF_ZVAL_TYPE, addr, val, label, tmp_reg || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), tmp_reg2 -| IF_TYPE tmp_reg1, val, label +| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, Rw(tmp_reg), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), Rx(tmp_reg) +| IF_TYPE Rw(tmp_reg), val, label |.endmacro -|.macro IF_NOT_ZVAL_TYPE, addr, val, label, tmp_reg1, tmp_reg2 +|.macro IF_NOT_ZVAL_TYPE, addr, val, label, tmp_reg || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), tmp_reg2 -| IF_NOT_TYPE tmp_reg1, val, label +| SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, Rw(tmp_reg), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), Rx(tmp_reg) +| IF_NOT_TYPE Rw(tmp_reg), val, label |.endmacro |.macro IF_FLAGS, type_flags, mask, label, tmp_reg @@ -1468,7 +1467,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) || if (gc && RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { || if ((op_info) & MAY_BE_REF) { || zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, offsetof(zend_reference, val)); -| IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1, Rw(tmp_reg1), Rx(tmp_reg2) +| IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1, tmp_reg1 | IF_NOT_ZVAL_COLLECTABLE ref_addr, >4, tmp_reg1, tmp_reg2 | GET_ZVAL_PTR FCARG1x, ref_addr, Rx(tmp_reg2) |1: @@ -1495,7 +1494,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.macro SEPARATE_ARRAY, addr, op_info, cold, tmp_reg1, tmp_reg2 || if (RC_MAY_BE_N(op_info)) { || if (Z_REG(addr) != ZREG_FP) { -| GET_ZVAL_LVAL ZREG_REG0, addr, Rx(tmp_reg1) +| GET_ZVAL_LVAL ZREG_REG0, addr || if (RC_MAY_BE_1(op_info)) { | // if (GC_REFCOUNT() > 1) | ldr Rw(tmp_reg1), [REG0] @@ -1510,7 +1509,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |2: | mov FCARG1x, REG0 || } else { -| GET_ZVAL_LVAL ZREG_FCARG1x, addr, Rx(tmp_reg1) +| GET_ZVAL_LVAL ZREG_FCARG1x, addr || if (RC_MAY_BE_1(op_info)) { | // if (GC_REFCOUNT() > 1) | ldr Rw(tmp_reg1), [FCARG1x] @@ -1540,7 +1539,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |2: || } || } else { -| GET_ZVAL_LVAL ZREG_FCARG1x, addr, Rx(tmp_reg1) +| GET_ZVAL_LVAL ZREG_FCARG1x, addr || } |.endmacro @@ -3109,7 +3108,7 @@ static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t return 0; } - | IF_NOT_ZVAL_TYPE var_addr, type, &exit_addr, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE var_addr, type, &exit_addr, ZREG_TMP1 return 1; } @@ -3124,7 +3123,7 @@ static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32 return 0; } - | GET_ZVAL_LVAL ZREG_FCARG1x, var_addr, TMP1 + | GET_ZVAL_LVAL ZREG_FCARG1x, var_addr if (op_info & MAY_BE_ARRAY_PACKED) { | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w @@ -3437,7 +3436,7 @@ static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr ZEND_ASSERT(Z_MODE(dst) == IS_REG); if ((info & MAY_BE_ANY) == MAY_BE_LONG) { - | GET_ZVAL_LVAL Z_REG(dst), src, TMP1 + | GET_ZVAL_LVAL Z_REG(dst), src } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { | GET_ZVAL_DVAL Z_REG(dst), src, ZREG_TMP1 } else { @@ -3533,7 +3532,7 @@ static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); - | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >1, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1 if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { if (!zend_jit_save_call_chain(Dst, -1)) { @@ -3603,7 +3602,7 @@ static int zend_jit_free_trampoline(dasm_State **Dst) static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) { if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, ZREG_TMP1 } if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { | ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 @@ -3709,7 +3708,7 @@ static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { | SET_EX_OPLINE opline, REG0 if (op1_info & MAY_BE_UNDEF) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2, ZREG_TMP1 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); | LOAD_32BIT_VAL FCARG1w, opline->op1.var | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 @@ -3874,14 +3873,14 @@ static int zend_jit_math_long_long(dasm_State **Dst, if (Z_MODE(op1_addr) == IS_REG) { | adds Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) } else { - | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | GET_ZVAL_LVAL result_reg, op1_addr | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) } } else if (opcode == ZEND_MUL && Z_MODE(op2_addr) == IS_CONST_ZVAL && !may_overflow && zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { - | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | GET_ZVAL_LVAL result_reg, op1_addr | mov TMP1, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) | lsl Rx(result_reg), Rx(result_reg), TMP1 } else if (opcode == ZEND_MUL && @@ -3890,20 +3889,20 @@ static int zend_jit_math_long_long(dasm_State **Dst, if (Z_MODE(op2_addr) == IS_REG) { | adds Rx(result_reg), Rx(Z_REG(op2_addr)), Rx(Z_REG(op2_addr)) } else { - | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 + | GET_ZVAL_LVAL result_reg, op2_addr | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) } } else if (opcode == ZEND_MUL && Z_MODE(op1_addr) == IS_CONST_ZVAL && !may_overflow && zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { - | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 + | GET_ZVAL_LVAL result_reg, op2_addr | mov TMP1, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) | lsl Rx(result_reg), Rx(result_reg), TMP1 } else if (opcode == ZEND_DIV && (Z_MODE(op2_addr) == IS_CONST_ZVAL && is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { - | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | GET_ZVAL_LVAL result_reg, op1_addr | asr Rx(result_reg), Rx(result_reg), #floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) #if 0 /* x86 specific optimizations through LEA instraction are not supported on ARM */ @@ -3924,8 +3923,8 @@ static int zend_jit_math_long_long(dasm_State **Dst, | NIY // TODO: test #endif } else if (opcode == ZEND_MUL) { - | GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1 - | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 + | GET_ZVAL_LVAL ZREG_TMP1, op1_addr + | GET_ZVAL_LVAL ZREG_TMP2, op2_addr | mul Rx(result_reg), TMP1, TMP2 if(may_overflow) { /* Use 'smulh' to get the upper 64 bits fo the 128-bit result. @@ -3940,7 +3939,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, | cmp TMP1, Rx(result_reg), asr #63 } } else { - | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | GET_ZVAL_LVAL result_reg, op1_addr if ((opcode == ZEND_ADD || opcode == ZEND_SUB) && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { @@ -4233,18 +4232,18 @@ static int zend_jit_math_helper(dasm_State **Dst, if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { if (op1_info & MAY_BE_DOUBLE) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1 } else { - | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 } } if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { if (op2_info & MAY_BE_DOUBLE) { - | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1, ZREG_TMP1 |.cold_code |1: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1 } if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; @@ -4252,7 +4251,7 @@ static int zend_jit_math_helper(dasm_State **Dst, | b >5 |.code } else { - | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 } } if (!zend_jit_math_long_long(Dst, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) { @@ -4262,14 +4261,14 @@ static int zend_jit_math_helper(dasm_State **Dst, |.cold_code |3: if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops) { - | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >1, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >1, ZREG_TMP1 } else { - | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6, ZREG_TMP1 } } if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { @@ -4280,7 +4279,7 @@ static int zend_jit_math_helper(dasm_State **Dst, if (!same_ops) { |1: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 } if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; @@ -4293,14 +4292,14 @@ static int zend_jit_math_helper(dasm_State **Dst, !(op1_info & MAY_BE_LONG) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops && (op2_info & MAY_BE_LONG)) { - | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >1, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >1, ZREG_TMP1 } else { - | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1 } } if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { @@ -4313,7 +4312,7 @@ static int zend_jit_math_helper(dasm_State **Dst, } |1: if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { - | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 } if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; @@ -4327,14 +4326,14 @@ static int zend_jit_math_helper(dasm_State **Dst, !(op2_info & MAY_BE_LONG) && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { - | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1 } if (op1_info & MAY_BE_DOUBLE) { if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops && (op1_info & MAY_BE_LONG)) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >1, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >1, ZREG_TMP1 } else { - | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1 } } if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { @@ -4347,7 +4346,7 @@ static int zend_jit_math_helper(dasm_State **Dst, } |1: if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 } if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; @@ -4443,8 +4442,8 @@ static int zend_jit_add_arrays(dasm_State **Dst, const zend_op *opline, uint32_t zend_jit_addr op1_addr = OP1_ADDR(); zend_jit_addr op2_addr = OP2_ADDR(); - | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr, TMP1 - | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr | EXT_CALL zend_jit_add_arrays_helper, REG0 | SET_ZVAL_PTR res_addr, RETVALx, TMP1 | SET_ZVAL_TYPE_INFO res_addr, IS_ARRAY_EX, TMP1w, TMP2 @@ -4478,10 +4477,10 @@ static int zend_jit_long_math_helper(dasm_State **Dst, zval tmp; if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 } if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { - | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1 } if (opcode == ZEND_MOD && Z_MODE(op2_addr) == IS_CONST_ZVAL && @@ -4529,13 +4528,13 @@ static int zend_jit_long_math_helper(dasm_State **Dst, } else if (Z_MODE(op1_addr) == IS_REG && op2_lval == 1) { | add Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) } else { - | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | GET_ZVAL_LVAL result_reg, op1_addr | mov TMP1w, #op2_lval | lsl Rx(result_reg), Rx(result_reg), TMP1 } } else { if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_REG1) { - | GET_ZVAL_LVAL ZREG_REG1, op2_addr, TMP1 + | GET_ZVAL_LVAL ZREG_REG1, op2_addr } if (!op2_range || op2_range->min < 0 || @@ -4552,12 +4551,12 @@ static int zend_jit_long_math_helper(dasm_State **Dst, | b ->negative_shift |.code } - | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | GET_ZVAL_LVAL result_reg, op1_addr | lsl Rx(result_reg), Rx(result_reg), REG1 |1: } } else if (opcode == ZEND_SR) { - | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | GET_ZVAL_LVAL result_reg, op1_addr if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); @@ -4574,7 +4573,7 @@ static int zend_jit_long_math_helper(dasm_State **Dst, } } else { if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_REG1) { - | GET_ZVAL_LVAL ZREG_REG1, op2_addr, TMP1 + | GET_ZVAL_LVAL ZREG_REG1, op2_addr } if (!op2_range || op2_range->min < 0 || @@ -4604,8 +4603,8 @@ static int zend_jit_long_math_helper(dasm_State **Dst, } else if (op2_lval == -1) { | mov REG0, xzr } else { - | GET_ZVAL_LVAL ZREG_REG1, op1_addr, TMP1 - | GET_ZVAL_LVAL ZREG_REG2, op2_addr, TMP1 + | GET_ZVAL_LVAL ZREG_REG1, op1_addr + | GET_ZVAL_LVAL ZREG_REG2, op2_addr | sdiv REG0, REG1, REG2 | msub REG0, REG0, REG2, REG1 } @@ -4647,7 +4646,7 @@ static int zend_jit_long_math_helper(dasm_State **Dst, |.code } - | GET_ZVAL_LVAL ZREG_REG1, op1_addr, TMP1 + | GET_ZVAL_LVAL ZREG_REG1, op1_addr if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { | ldr TMP1, [Rx(Z_REG(op2_addr)), #Z_OFFSET(op2_addr)] | sdiv REG0, REG1, TMP1 @@ -4658,10 +4657,10 @@ static int zend_jit_long_math_helper(dasm_State **Dst, } } } else if (same_ops) { - | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | GET_ZVAL_LVAL result_reg, op1_addr | LONG_MATH_REG opcode, Rx(result_reg), Rx(result_reg), Rx(result_reg) } else { - | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 + | GET_ZVAL_LVAL result_reg, op1_addr | LONG_MATH opcode, result_reg, op2_addr, TMP1 } @@ -4775,10 +4774,10 @@ static int zend_jit_concat_helper(dasm_State **Dst, { if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1 } if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { - | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >6, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >6, ZREG_TMP1 } if (Z_MODE(op1_addr) == IS_MEM_ZVAL && Z_REG(op1_addr) == Z_REG(res_addr) && Z_OFFSET(op1_addr) == Z_OFFSET(res_addr)) { if (Z_REG(res_addr) != ZREG_FCARG1x || Z_OFFSET(res_addr) != 0) { @@ -4866,7 +4865,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { | // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) - | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, ZREG_TMP1 } if (op1_info & MAY_BE_PACKED_GUARD) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); @@ -4887,7 +4886,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } if (type == BP_VAR_W) { | // hval = Z_LVAL_P(dim); - | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr op2_loaded = 1; } if (op1_info & MAY_BE_ARRAY_PACKED) { @@ -4901,7 +4900,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } else { if (!op2_loaded) { | // hval = Z_LVAL_P(dim); - | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr op2_loaded = 1; } packed_loaded = 1; @@ -4963,7 +4962,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |4: if (!op2_loaded) { | // hval = Z_LVAL_P(dim); - | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr } | EXT_CALL _zend_hash_index_find, REG0 | mov REG0, RETVALx @@ -5019,7 +5018,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |4: if (!op2_loaded) { | // hval = Z_LVAL_P(dim); - | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr } | EXT_CALL _zend_hash_index_find, REG0 | mov REG0, RETVALx @@ -5065,7 +5064,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |4: if (!op2_loaded) { | // hval = Z_LVAL_P(dim); - | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr } | SET_EX_OPLINE opline, REG0 | EXT_CALL zend_jit_hash_index_lookup_rw, REG0 @@ -5081,7 +5080,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | //retval = zend_hash_index_add_new(ht, hval, &EG(uninitialized_zval)); if (!op2_loaded) { | // hval = Z_LVAL_P(dim); - | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr } | LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval | EXT_CALL zend_hash_index_add_new, REG0 @@ -5094,7 +5093,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |4: if (!op2_loaded) { | // hval = Z_LVAL_P(dim); - | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr } | EXT_CALL zend_hash_index_lookup, REG0 | mov REG0, RETVALx @@ -5113,10 +5112,10 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |3: if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { | // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) - | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3, ZREG_TMP1 } | // offset_key = Z_STR_P(dim); - | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr | // retval = zend_hash_find(ht, offset_key); switch (type) { case BP_JIT_IS: @@ -5328,9 +5327,9 @@ static int zend_jit_simple_assign(dasm_State **Dst, } else { if (val_info & MAY_BE_UNDEF) { if (in_cold) { - | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2, ZREG_TMP1 } else { - | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1, TMP1w, TMP2 + | IF_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1 |.cold_code |1: } @@ -5370,9 +5369,9 @@ static int zend_jit_simple_assign(dasm_State **Dst, zend_jit_addr ref_addr; if (in_cold) { - | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1, ZREG_TMP1 } else { - | IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1, TMP1w, TMP2 + | IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1, ZREG_TMP1 |.cold_code |1: } @@ -5699,7 +5698,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t return 0; } - | IF_ZVAL_TYPE op3_addr, IS_UNDEF, &exit_addr, TMP1w, TMP2 + | IF_ZVAL_TYPE op3_addr, IS_UNDEF, &exit_addr, ZREG_TMP1 val_info &= ~MAY_BE_UNDEF; } @@ -5727,13 +5726,13 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t if (op1_info & MAY_BE_ARRAY) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 } |3: | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { - | CMP_ZVAL_TYPE op1_addr, IS_FALSE, TMP1w, TMP2 + | CMP_ZVAL_TYPE op1_addr, IS_FALSE, ZREG_TMP1 | bgt >7 } | // ZVAL_ARR(container, zend_new_array(8)); @@ -5814,7 +5813,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) && (op1_info & MAY_BE_ARRAY)) { if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { - | CMP_ZVAL_TYPE op1_addr, IS_FALSE, TMP1w, TMP2 + | CMP_ZVAL_TYPE op1_addr, IS_FALSE, ZREG_TMP1 | bgt >2 } | // ZVAL_ARR(container, zend_new_array(8)); @@ -5922,7 +5921,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint3 if (op1_info & MAY_BE_ARRAY) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 } |3: | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 @@ -5933,12 +5932,12 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint3 |7: } if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { - | CMP_ZVAL_TYPE op1_addr, IS_FALSE, TMP1w, TMP2 + | CMP_ZVAL_TYPE op1_addr, IS_FALSE, ZREG_TMP1 | bgt >7 } if (op1_info & MAY_BE_UNDEF) { if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 } | SET_EX_OPLINE opline, REG0 | LOAD_32BIT_VAL FCARG1x, opline->op1.var @@ -6290,13 +6289,13 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { | cmp Rx(Z_REG(op1_addr)), xzr } else { - | LONG_CMP Z_REG(op1_addr), op2_addr, TMP1, TMP2 + | LONG_CMP Z_REG(op1_addr), op2_addr, TMP1 } } else if (Z_MODE(op2_addr) == IS_REG) { if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) { | cmp Rx(Z_REG(op2_addr)), xzr } else { - | LONG_CMP Z_REG(op2_addr), op1_addr, TMP1, TMP2 + | LONG_CMP Z_REG(op2_addr), op1_addr, TMP1 } swap = 1; } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) { @@ -6305,11 +6304,11 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { | LONG_CMP_WITH_CONST op1_addr, Z_LVAL_P(Z_ZV(op2_addr)), TMP1, TMP2 } else { - | GET_ZVAL_LVAL ZREG_REG0, op1_addr, TMP1 + | GET_ZVAL_LVAL ZREG_REG0, op1_addr if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { | cmp Rx(ZREG_REG0), xzr } else { - | LONG_CMP ZREG_REG0, op2_addr, TMP1, TMP2 + | LONG_CMP ZREG_REG0, op2_addr, TMP1 } } @@ -7079,18 +7078,18 @@ static int zend_jit_cmp(dasm_State **Dst, if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { if (op1_info & MAY_BE_DOUBLE) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4, ZREG_TMP1 } else { - | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9, ZREG_TMP1 } } if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { if (op2_info & MAY_BE_DOUBLE) { - | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, ZREG_TMP1 |.cold_code |3: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 } if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; @@ -7098,7 +7097,7 @@ static int zend_jit_cmp(dasm_State **Dst, | b >6 |.code } else { - | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1 } } if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) { @@ -7108,14 +7107,14 @@ static int zend_jit_cmp(dasm_State **Dst, |.cold_code |4: if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops) { - | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5, ZREG_TMP1 } else { - | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 } } if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { @@ -7126,7 +7125,7 @@ static int zend_jit_cmp(dasm_State **Dst, if (!same_ops) { |5: if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { - | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1 } if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; @@ -7139,14 +7138,14 @@ static int zend_jit_cmp(dasm_State **Dst, !(op1_info & MAY_BE_LONG) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1 } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops && (op2_info & MAY_BE_LONG)) { - | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >3, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >3, ZREG_TMP1 } else { - | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 } } if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { @@ -7159,7 +7158,7 @@ static int zend_jit_cmp(dasm_State **Dst, } |3: if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { - | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1 } if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; @@ -7173,14 +7172,14 @@ static int zend_jit_cmp(dasm_State **Dst, !(op2_info & MAY_BE_LONG) && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { - | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1 } if (op1_info & MAY_BE_DOUBLE) { if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if (!same_ops && (op1_info & MAY_BE_LONG)) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >3, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >3, ZREG_TMP1 } else { - | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1 } } if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { @@ -7193,7 +7192,7 @@ static int zend_jit_cmp(dasm_State **Dst, } |3: if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9, ZREG_TMP1 } if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) { return 0; @@ -7236,7 +7235,7 @@ static int zend_jit_cmp(dasm_State **Dst, |1: } if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { - | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1 | str FCARG2x, T1 // save | LOAD_32BIT_VAL FCARG1x, opline->op2.var | EXT_CALL zend_jit_undefined_op_helper, REG0 @@ -7764,7 +7763,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ } } } else { - | CMP_ZVAL_TYPE op1_addr, IS_TRUE, TMP1w, TMP2 + | CMP_ZVAL_TYPE op1_addr, IS_TRUE, ZREG_TMP1 if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { if ((op1_info & MAY_BE_LONG) && !(op1_info & MAY_BE_UNDEF) && @@ -7862,11 +7861,11 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ if (op1_info & MAY_BE_UNDEF) { if (op1_info & MAY_BE_ANY) { if (set_delayed) { - | CMP_ZVAL_TYPE op1_addr, IS_UNDEF, TMP1w, TMP2 + | CMP_ZVAL_TYPE op1_addr, IS_UNDEF, ZREG_TMP1 | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 | beq >1 } else { - | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, TMP1w, TMP2 + | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 } |.cold_code |1: @@ -7921,7 +7920,7 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ if (op1_info & MAY_BE_LONG) { |2: if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, ZREG_TMP1 } if (Z_MODE(op1_addr) == IS_REG) { | tst Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) @@ -8076,14 +8075,14 @@ static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_ } | SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1 if (exit_addr) { - | CMP_ZVAL_TYPE res_addr, IS_FALSE, TMP1w, TMP2 + | CMP_ZVAL_TYPE res_addr, IS_FALSE, ZREG_TMP1 if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { | bne &exit_addr } else { | beq &exit_addr } } else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) { - | CMP_ZVAL_TYPE res_addr, IS_FALSE, TMP1w, TMP2 + | CMP_ZVAL_TYPE res_addr, IS_FALSE, ZREG_TMP1 if (true_label != (uint32_t)-1) { | bne =>true_label if (false_label != (uint32_t)-1) { @@ -8845,7 +8844,7 @@ static int zend_jit_init_method_call(dasm_State **Dst, } else { /* Hack: Convert reference to regular value to simplify JIT code */ ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP); - | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1 | LOAD_ZVAL_ADDR FCARG1x, op1_addr | EXT_CALL zend_jit_unref_helper, REG0 |1: @@ -8859,9 +8858,9 @@ static int zend_jit_init_method_call(dasm_State **Dst, if (!exit_addr) { return 0; } - | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 } else { - | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 |.cold_code |1: if (Z_REG(op1_addr) != ZREG_FCARG1x || Z_OFFSET(op1_addr) != 0) { @@ -9813,7 +9812,7 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend } else if (opline->op1_type == IS_CV) { if (op1_info & MAY_BE_UNDEF) { if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 | SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2 | b >2 |1: @@ -9827,7 +9826,7 @@ static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) { if (op1_info & MAY_BE_REF) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >2, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >2, ZREG_TMP1 | GET_ZVAL_PTR REG1, op1_addr, TMP1 | GC_ADDREF REG1, TMP1w | SET_ZVAL_PTR arg_addr, REG1, TMP1 @@ -10007,7 +10006,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend if (op1_info & MAY_BE_UNDEF) { if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { - | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, TMP1w, TMP2 + | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 |.cold_code |1: } @@ -10061,7 +10060,7 @@ static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend } else { zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1x, 8); - | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, TMP1w, TMP2 + | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1 |.cold_code |1: | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); @@ -10326,7 +10325,7 @@ static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t if (op1_info & MAY_BE_UNDEF) { if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { - | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, TMP1w, TMP2 + | IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 |.cold_code |1: } @@ -11012,7 +11011,7 @@ static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_o if (op1_info & MAY_BE_REF) { zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val)); - | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, TMP1w, TMP2 + | IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1 |.cold_code |1: | // zend_refcounted *ref = Z_COUNTED_P(retval_ptr); @@ -11215,12 +11214,12 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, if (op1_info & MAY_BE_ARRAY) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, &exit_addr, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, &exit_addr, ZREG_TMP1 } else { - | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 } } - | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr, TMP1 + | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, res_exit_addr, not_found_exit_addr, exit_addr)) { return 0; } @@ -11235,16 +11234,16 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) { if (exit_addr && !(op1_info & MAY_BE_OBJECT)) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &exit_addr, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &exit_addr, ZREG_TMP1 } else { - | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1 } } | SET_EX_OPLINE opline, REG0 - | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr, TMP1 + | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr if (opline->opcode != ZEND_FETCH_DIM_IS) { if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) { - | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr | EXT_CALL zend_jit_fetch_dim_str_offset_r_helper, REG0 } else { | LOAD_ZVAL_ADDR FCARG2x, op2_addr @@ -11267,9 +11266,9 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, if (op1_info & MAY_BE_OBJECT) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) { if (exit_addr) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 } else { - | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6, ZREG_TMP1 } } | SET_EX_OPLINE opline, REG0 @@ -11298,7 +11297,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) { | SET_EX_OPLINE opline, REG0 if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); | LOAD_32BIT_VAL FCARG1w, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, REG0 @@ -11306,7 +11305,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, } if (op2_info & MAY_BE_UNDEF) { - | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1 | LOAD_32BIT_VAL FCARG1w, opline->op2.var | EXT_CALL zend_jit_undefined_op_helper, REG0 |1: @@ -11349,7 +11348,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, | ZVAL_DEREF REG0, MAY_BE_REF, TMP1w } if (type < IS_STRING) { - | IF_NOT_ZVAL_TYPE val_addr, type, &res_exit_addr, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE val_addr, type, &res_exit_addr, ZREG_TMP1 } else { | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 | GET_LOW_8BITS TMP1w, REG2w @@ -11444,7 +11443,7 @@ static int zend_jit_fetch_dim(dasm_State **Dst, if (op1_info & MAY_BE_ARRAY) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 } |3: | SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2 @@ -11455,13 +11454,13 @@ static int zend_jit_fetch_dim(dasm_State **Dst, |7: } if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY))) { - | CMP_ZVAL_TYPE op1_addr, IS_FALSE, TMP1w, TMP2 + | CMP_ZVAL_TYPE op1_addr, IS_FALSE, ZREG_TMP1 | bgt >7 } if ((op1_info & MAY_BE_UNDEF) && opline->opcode == ZEND_FETCH_DIM_RW) { if (op1_info & (MAY_BE_NULL|MAY_BE_FALSE)) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 } | SET_EX_OPLINE opline, REG0 | LOAD_32BIT_VAL FCARG1w, opline->op1.var @@ -11630,9 +11629,9 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, const void *not_found_exit_addr = NULL; if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 } - | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr, TMP1 + | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr if (exit_addr && !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) && !may_throw @@ -11683,7 +11682,7 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, } else { if (op2_info & MAY_BE_UNDEF) { if (op2_info & MAY_BE_ANY) { - | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1 } | LOAD_32BIT_VAL FCARG1w, opline->op2.var | EXT_CALL zend_jit_undefined_op_helper, REG0 @@ -11879,7 +11878,7 @@ static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zen if (type_mask != 0) { if (is_power_of_two(type_mask)) { uint32_t type_code = concrete_type(type_mask); - | IF_NOT_ZVAL_TYPE res_addr, type_code, >1, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE res_addr, type_code, >1, ZREG_TMP1 } else { | mov REG2w, #1 | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, REG1w, Rx(Z_REG(res_addr)), Z_OFFSET(res_addr)+offsetof(zval, u1.v.type), TMP1 @@ -12208,9 +12207,9 @@ static int zend_jit_fetch_obj(dasm_State **Dst, if (!exit_addr) { return 0; } - | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 } else { - | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7, ZREG_TMP1 } } | GET_ZVAL_PTR FCARG1x, op1_addr, TMP1 @@ -12403,7 +12402,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, | add REG0, REG0, #offsetof(zend_reference, val) if (type < IS_STRING) { |1: - | IF_NOT_ZVAL_TYPE val_addr, type, &exit_addr, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE val_addr, type, &exit_addr, ZREG_TMP1 } else { | GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1 |1: @@ -12455,7 +12454,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, zend_jit_addr orig_op1_addr = OP1_ADDR(); if (op1_info & MAY_BE_ANY) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1 } | LOAD_32BIT_VAL FCARG1w, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, REG0 @@ -12590,9 +12589,9 @@ static int zend_jit_incdec_obj(dasm_State **Dst, if (!exit_addr) { return 0; } - | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 } else { - | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 |.cold_code |1: | SET_EX_OPLINE opline, REG0 @@ -12723,7 +12722,7 @@ static int zend_jit_incdec_obj(dasm_State **Dst, | LOAD_ZVAL_ADDR FCARG1x, prop_addr } - | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, ZREG_TMP1 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] | cbnz TMP1, >1 @@ -12758,7 +12757,7 @@ static int zend_jit_incdec_obj(dasm_State **Dst, |.code |2: - | IF_NOT_ZVAL_TYPE var_addr, IS_LONG, >2, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE var_addr, IS_LONG, >2, ZREG_TMP1 if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { if (opline->result_type != IS_UNUSED) { | ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0 @@ -12929,9 +12928,9 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, if (!exit_addr) { return 0; } - | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 } else { - | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 |.cold_code |1: | SET_EX_OPLINE opline, REG0 @@ -13022,7 +13021,7 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, | SET_EX_OPLINE opline, REG0 } - | IF_ZVAL_TYPE prop_addr, IS_REFERENCE, >1, TMP1w, TMP2 + | IF_ZVAL_TYPE prop_addr, IS_REFERENCE, >1, ZREG_TMP1 |.cold_code |1: | GET_ZVAL_PTR FCARG1x, prop_addr, TMP1 @@ -13068,7 +13067,7 @@ static int zend_jit_assign_obj_op(dasm_State **Dst, var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); | LOAD_ZVAL_ADDR REG0, prop_addr - | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, ZREG_TMP1 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 | ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)] | cbnz TMP1, >1 @@ -13222,9 +13221,9 @@ static int zend_jit_assign_obj(dasm_State **Dst, if (!exit_addr) { return 0; } - | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1 } else { - | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1 |.cold_code |1: | SET_EX_OPLINE opline, REG0 @@ -13437,7 +13436,7 @@ static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_i } if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) { if (op1_info & MAY_BE_ARRAY) { - | IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7, TMP1w, TMP2 + | IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 } | SAFE_MEM_ACC_WITH_UOFFSET_32 ldr, FCARG1w, FP, (opline->op1.var + offsetof(zval, u2.fe_iter_idx)), TMP1 | mvn TMP1w, wzr // TODO: DynAsm fails loading #-1 @@ -13750,15 +13749,15 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o if (opline->opcode == ZEND_SWITCH_LONG) { if (op1_info & MAY_BE_LONG) { if (op1_info & MAY_BE_REF) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >1, TMP1w, TMP2 - | GET_ZVAL_LVAL ZREG_FCARG2x, op1_addr, TMP1 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >1, ZREG_TMP1 + | GET_ZVAL_LVAL ZREG_FCARG2x, op1_addr |.cold_code |1: | // ZVAL_DEREF(op) if (fallback_label) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label, ZREG_TMP1 } else { - | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, ZREG_TMP1 } | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 if (fallback_label) { @@ -13775,12 +13774,12 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o } else { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { if (fallback_label) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label, ZREG_TMP1 } else { - | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1 } } - | GET_ZVAL_LVAL ZREG_FCARG2x, op1_addr, TMP1 + | GET_ZVAL_LVAL ZREG_FCARG2x, op1_addr } if (HT_IS_PACKED(jumptable)) { uint32_t count = jumptable->nNumUsed; @@ -13846,15 +13845,15 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o } else if (opline->opcode == ZEND_SWITCH_STRING) { if (op1_info & MAY_BE_STRING) { if (op1_info & MAY_BE_REF) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >1, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >1, ZREG_TMP1 | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 |.cold_code |1: | // ZVAL_DEREF(op) if (fallback_label) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label, ZREG_TMP1 } else { - | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, ZREG_TMP1 } | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 if (fallback_label) { @@ -13871,9 +13870,9 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o } else { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) { if (fallback_label) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label, ZREG_TMP1 } else { - | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, ZREG_TMP1 } } | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 @@ -13897,18 +13896,18 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o if (op1_info & MAY_BE_LONG) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { if (op1_info & MAY_BE_STRING) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >5, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >5, ZREG_TMP1 } else if (op1_info & MAY_BE_UNDEF) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1 } else if (default_label) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &default_label, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &default_label, ZREG_TMP1 } else if (next_opline) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1 } else { - | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, =>default_b, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, =>default_b, ZREG_TMP1 } } - | GET_ZVAL_LVAL ZREG_FCARG2x, op1_addr, TMP1 + | GET_ZVAL_LVAL ZREG_FCARG2x, op1_addr | EXT_CALL zend_hash_index_find, REG0 | mov REG0, RETVALx if (op1_info & MAY_BE_STRING) { @@ -13919,13 +13918,13 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o |5: if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) { if (op1_info & MAY_BE_UNDEF) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1 } else if (default_label) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &default_label, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &default_label, ZREG_TMP1 } else if (next_opline) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, ZREG_TMP1 } else { - | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, =>default_b, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, =>default_b, ZREG_TMP1 } } | GET_ZVAL_PTR FCARG2x, op1_addr, TMP1 @@ -13941,11 +13940,11 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o |6: if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) { if (default_label) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, &default_label, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, &default_label, ZREG_TMP1 } else if (next_opline) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >3, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >3, ZREG_TMP1 } else { - | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, =>default_b, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, =>default_b, ZREG_TMP1 } } | // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)))); @@ -13989,7 +13988,7 @@ static bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, needs_slow_check = 0; } else if (is_power_of_two(type_mask)) { uint32_t type_code = concrete_type(type_mask); - | IF_NOT_ZVAL_TYPE op1_addr, type_code, >7, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, type_code, >7, ZREG_TMP1 } else { | mov REG2w, #1 | GET_ZVAL_TYPE REG1w, op1_addr, TMP1 @@ -14005,7 +14004,7 @@ static bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, } | SET_EX_OPLINE opline, REG1 if (op1_info & MAY_BE_UNDEF) { - | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >8, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >8, ZREG_TMP1 | LOAD_32BIT_VAL FCARG1x, opline->op1.var | EXT_CALL zend_jit_undefined_op_helper, REG0 | LOAD_ADDR_ZTS REG0, executor_globals, uninitialized_zval @@ -14280,7 +14279,7 @@ static int zend_jit_fetch_constant(dasm_State **Dst, zend_uchar type = concrete_type(res_info); if (type < IS_STRING) { - | IF_NOT_ZVAL_TYPE const_addr, type, &exit_addr, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE const_addr, type, &exit_addr, ZREG_TMP1 } else { | GET_ZVAL_TYPE_INFO REG2w, const_addr, TMP1 | IF_NOT_TYPE REG2w, type, &exit_addr @@ -14366,7 +14365,7 @@ static bool zend_jit_noref_guard(dasm_State **Dst, const zend_op *opline, zend_j if (!exit_addr) { return 0; } - | IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr, TMP1w, TMP2 + | IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr, ZREG_TMP1 return 1; } @@ -14387,7 +14386,7 @@ static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, ui } if (add_ref_guard) { - | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr, ZREG_TMP1 } if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) { /* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */ @@ -14407,7 +14406,7 @@ static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, ui if (add_type_guard && var_type != IS_UNKNOWN && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { - | IF_NOT_ZVAL_TYPE var_addr, var_type, &exit_addr, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE var_addr, var_type, &exit_addr, ZREG_TMP1 ZEND_ASSERT(var_info & (1 << var_type)); if (var_type < IS_STRING) { @@ -14441,7 +14440,7 @@ static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, if (!exit_addr) { return 0; } - | IF_NOT_ZVAL_TYPE var_addr, IS_INDIRECT, &exit_addr, TMP1w, TMP2 + | IF_NOT_ZVAL_TYPE var_addr, IS_INDIRECT, &exit_addr, ZREG_TMP1 | GET_ZVAL_PTR FCARG1x, var_addr, TMP1 } else { /* May be already loaded into FCARG1a or RAX by previus FETCH_OBJ_W/DIM_W */ From 2de4a93d21ca11d162000c9ae71dea539ed24a6c Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Mon, 10 May 2021 02:00:43 +0000 Subject: [PATCH 204/229] Fix commit 6e344ed: temporary register should be kept for GET_ZVAL_LVAL Macro GET_ZVAL_LVAL is optimized in commit 6e3443d, using 'reg' as the temporary register. However, the assertion would fail for "jit=1205". This patch reverts this optimization for GET_ZVAL_LVAL. Change-Id: Iecb134fa9438ef993e6393974caa5710657a462e --- ext/opcache/jit/zend_jit_arm64.dasc | 97 ++++++++++++++--------------- 1 file changed, 48 insertions(+), 49 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 6f8eda9634da8..a12c67cc1987b 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -917,7 +917,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) || } |.endmacro -|.macro GET_ZVAL_LVAL, reg, addr +|.macro GET_ZVAL_LVAL, reg, addr, tmp_reg || if (Z_MODE(addr) == IS_CONST_ZVAL) { || if (Z_LVAL_P(Z_ZV(addr)) == 0) { | mov Rx(reg), xzr @@ -925,8 +925,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | LOAD_64BIT_VAL Rx(reg), Z_LVAL_P(Z_ZV(addr)) || } || } else if (Z_MODE(addr) == IS_MEM_ZVAL) { -|| ZEND_ASSERT(reg != Z_REG(addr)); -| SAFE_MEM_ACC_WITH_UOFFSET ldr, Rx(reg), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(reg) +| SAFE_MEM_ACC_WITH_UOFFSET ldr, Rx(reg), Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg || } else if (Z_MODE(addr) == IS_REG) { || if (reg != Z_REG(addr)) { | mov Rx(reg), Rx(Z_REG(addr)) @@ -1138,9 +1137,9 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg) || } || } else if (Z_MODE(dst_addr) == IS_REG) { -| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr +| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr, Rx(tmp_reg) || } else { -| GET_ZVAL_LVAL reg2, src_addr +| GET_ZVAL_LVAL reg2, src_addr, Rx(tmp_reg) | SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(reg2), Rx(tmp_reg) || } || } else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) { @@ -1172,15 +1171,15 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg) || } || } else if (Z_MODE(dst_addr) == IS_REG) { -| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr +| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr, Rx(tmp_reg) || if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(dst_addr)) { | SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(dst_addr)), Rx(tmp_reg) || } || } else if (Z_MODE(res_addr) == IS_REG) { -| GET_ZVAL_LVAL Z_REG(res_addr), src_addr +| GET_ZVAL_LVAL Z_REG(res_addr), src_addr, Rx(tmp_reg) | SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(res_addr)), Rx(tmp_reg) || } else { -| GET_ZVAL_LVAL reg2, src_addr +| GET_ZVAL_LVAL reg2, src_addr, Rx(tmp_reg) | SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(reg2), Rx(tmp_reg) | SET_ZVAL_LVAL_FROM_REG res_addr, Rx(reg2), Rx(tmp_reg) || } @@ -1494,7 +1493,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.macro SEPARATE_ARRAY, addr, op_info, cold, tmp_reg1, tmp_reg2 || if (RC_MAY_BE_N(op_info)) { || if (Z_REG(addr) != ZREG_FP) { -| GET_ZVAL_LVAL ZREG_REG0, addr +| GET_ZVAL_LVAL ZREG_REG0, addr, Rx(tmp_reg1) || if (RC_MAY_BE_1(op_info)) { | // if (GC_REFCOUNT() > 1) | ldr Rw(tmp_reg1), [REG0] @@ -1509,7 +1508,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |2: | mov FCARG1x, REG0 || } else { -| GET_ZVAL_LVAL ZREG_FCARG1x, addr +| GET_ZVAL_LVAL ZREG_FCARG1x, addr, Rx(tmp_reg1) || if (RC_MAY_BE_1(op_info)) { | // if (GC_REFCOUNT() > 1) | ldr Rw(tmp_reg1), [FCARG1x] @@ -1539,7 +1538,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |2: || } || } else { -| GET_ZVAL_LVAL ZREG_FCARG1x, addr +| GET_ZVAL_LVAL ZREG_FCARG1x, addr, Rx(tmp_reg1) || } |.endmacro @@ -3123,7 +3122,7 @@ static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32 return 0; } - | GET_ZVAL_LVAL ZREG_FCARG1x, var_addr + | GET_ZVAL_LVAL ZREG_FCARG1x, var_addr, TMP1 if (op_info & MAY_BE_ARRAY_PACKED) { | ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)] | TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w @@ -3436,7 +3435,7 @@ static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr ZEND_ASSERT(Z_MODE(dst) == IS_REG); if ((info & MAY_BE_ANY) == MAY_BE_LONG) { - | GET_ZVAL_LVAL Z_REG(dst), src + | GET_ZVAL_LVAL Z_REG(dst), src, TMP1 } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { | GET_ZVAL_DVAL Z_REG(dst), src, ZREG_TMP1 } else { @@ -3873,14 +3872,14 @@ static int zend_jit_math_long_long(dasm_State **Dst, if (Z_MODE(op1_addr) == IS_REG) { | adds Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) } else { - | GET_ZVAL_LVAL result_reg, op1_addr + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) } } else if (opcode == ZEND_MUL && Z_MODE(op2_addr) == IS_CONST_ZVAL && !may_overflow && zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { - | GET_ZVAL_LVAL result_reg, op1_addr + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 | mov TMP1, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) | lsl Rx(result_reg), Rx(result_reg), TMP1 } else if (opcode == ZEND_MUL && @@ -3889,20 +3888,20 @@ static int zend_jit_math_long_long(dasm_State **Dst, if (Z_MODE(op2_addr) == IS_REG) { | adds Rx(result_reg), Rx(Z_REG(op2_addr)), Rx(Z_REG(op2_addr)) } else { - | GET_ZVAL_LVAL result_reg, op2_addr + | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 | adds Rx(result_reg), Rx(result_reg), Rx(result_reg) } } else if (opcode == ZEND_MUL && Z_MODE(op1_addr) == IS_CONST_ZVAL && !may_overflow && zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { - | GET_ZVAL_LVAL result_reg, op2_addr + | GET_ZVAL_LVAL result_reg, op2_addr, TMP1 | mov TMP1, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) | lsl Rx(result_reg), Rx(result_reg), TMP1 } else if (opcode == ZEND_DIV && (Z_MODE(op2_addr) == IS_CONST_ZVAL && is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { - | GET_ZVAL_LVAL result_reg, op1_addr + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 | asr Rx(result_reg), Rx(result_reg), #floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) #if 0 /* x86 specific optimizations through LEA instraction are not supported on ARM */ @@ -3923,8 +3922,8 @@ static int zend_jit_math_long_long(dasm_State **Dst, | NIY // TODO: test #endif } else if (opcode == ZEND_MUL) { - | GET_ZVAL_LVAL ZREG_TMP1, op1_addr - | GET_ZVAL_LVAL ZREG_TMP2, op2_addr + | GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1 + | GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2 | mul Rx(result_reg), TMP1, TMP2 if(may_overflow) { /* Use 'smulh' to get the upper 64 bits fo the 128-bit result. @@ -3939,7 +3938,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, | cmp TMP1, Rx(result_reg), asr #63 } } else { - | GET_ZVAL_LVAL result_reg, op1_addr + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 if ((opcode == ZEND_ADD || opcode == ZEND_SUB) && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { @@ -4442,8 +4441,8 @@ static int zend_jit_add_arrays(dasm_State **Dst, const zend_op *opline, uint32_t zend_jit_addr op1_addr = OP1_ADDR(); zend_jit_addr op2_addr = OP2_ADDR(); - | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr - | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr + | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr, TMP1 + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 | EXT_CALL zend_jit_add_arrays_helper, REG0 | SET_ZVAL_PTR res_addr, RETVALx, TMP1 | SET_ZVAL_TYPE_INFO res_addr, IS_ARRAY_EX, TMP1w, TMP2 @@ -4528,13 +4527,13 @@ static int zend_jit_long_math_helper(dasm_State **Dst, } else if (Z_MODE(op1_addr) == IS_REG && op2_lval == 1) { | add Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr)) } else { - | GET_ZVAL_LVAL result_reg, op1_addr + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 | mov TMP1w, #op2_lval | lsl Rx(result_reg), Rx(result_reg), TMP1 } } else { if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_REG1) { - | GET_ZVAL_LVAL ZREG_REG1, op2_addr + | GET_ZVAL_LVAL ZREG_REG1, op2_addr, TMP1 } if (!op2_range || op2_range->min < 0 || @@ -4551,12 +4550,12 @@ static int zend_jit_long_math_helper(dasm_State **Dst, | b ->negative_shift |.code } - | GET_ZVAL_LVAL result_reg, op1_addr + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 | lsl Rx(result_reg), Rx(result_reg), REG1 |1: } } else if (opcode == ZEND_SR) { - | GET_ZVAL_LVAL result_reg, op1_addr + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); @@ -4573,7 +4572,7 @@ static int zend_jit_long_math_helper(dasm_State **Dst, } } else { if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_REG1) { - | GET_ZVAL_LVAL ZREG_REG1, op2_addr + | GET_ZVAL_LVAL ZREG_REG1, op2_addr, TMP1 } if (!op2_range || op2_range->min < 0 || @@ -4603,8 +4602,8 @@ static int zend_jit_long_math_helper(dasm_State **Dst, } else if (op2_lval == -1) { | mov REG0, xzr } else { - | GET_ZVAL_LVAL ZREG_REG1, op1_addr - | GET_ZVAL_LVAL ZREG_REG2, op2_addr + | GET_ZVAL_LVAL ZREG_REG1, op1_addr, TMP1 + | GET_ZVAL_LVAL ZREG_REG2, op2_addr, TMP1 | sdiv REG0, REG1, REG2 | msub REG0, REG0, REG2, REG1 } @@ -4646,7 +4645,7 @@ static int zend_jit_long_math_helper(dasm_State **Dst, |.code } - | GET_ZVAL_LVAL ZREG_REG1, op1_addr + | GET_ZVAL_LVAL ZREG_REG1, op1_addr, TMP1 if (Z_MODE(op2_addr) == IS_MEM_ZVAL) { | ldr TMP1, [Rx(Z_REG(op2_addr)), #Z_OFFSET(op2_addr)] | sdiv REG0, REG1, TMP1 @@ -4657,10 +4656,10 @@ static int zend_jit_long_math_helper(dasm_State **Dst, } } } else if (same_ops) { - | GET_ZVAL_LVAL result_reg, op1_addr + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 | LONG_MATH_REG opcode, Rx(result_reg), Rx(result_reg), Rx(result_reg) } else { - | GET_ZVAL_LVAL result_reg, op1_addr + | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 | LONG_MATH opcode, result_reg, op2_addr, TMP1 } @@ -4886,7 +4885,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } if (type == BP_VAR_W) { | // hval = Z_LVAL_P(dim); - | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 op2_loaded = 1; } if (op1_info & MAY_BE_ARRAY_PACKED) { @@ -4900,7 +4899,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } else { if (!op2_loaded) { | // hval = Z_LVAL_P(dim); - | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 op2_loaded = 1; } packed_loaded = 1; @@ -4962,7 +4961,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |4: if (!op2_loaded) { | // hval = Z_LVAL_P(dim); - | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 } | EXT_CALL _zend_hash_index_find, REG0 | mov REG0, RETVALx @@ -5018,7 +5017,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |4: if (!op2_loaded) { | // hval = Z_LVAL_P(dim); - | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 } | EXT_CALL _zend_hash_index_find, REG0 | mov REG0, RETVALx @@ -5064,7 +5063,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |4: if (!op2_loaded) { | // hval = Z_LVAL_P(dim); - | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 } | SET_EX_OPLINE opline, REG0 | EXT_CALL zend_jit_hash_index_lookup_rw, REG0 @@ -5080,7 +5079,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | //retval = zend_hash_index_add_new(ht, hval, &EG(uninitialized_zval)); if (!op2_loaded) { | // hval = Z_LVAL_P(dim); - | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 } | LOAD_ADDR_ZTS CARG3, executor_globals, uninitialized_zval | EXT_CALL zend_hash_index_add_new, REG0 @@ -5093,7 +5092,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o |4: if (!op2_loaded) { | // hval = Z_LVAL_P(dim); - | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 } | EXT_CALL zend_hash_index_lookup, REG0 | mov REG0, RETVALx @@ -5115,7 +5114,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3, ZREG_TMP1 } | // offset_key = Z_STR_P(dim); - | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 | // retval = zend_hash_find(ht, offset_key); switch (type) { case BP_JIT_IS: @@ -6304,7 +6303,7 @@ static int zend_jit_cmp_long_long(dasm_State **Dst, } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) { | LONG_CMP_WITH_CONST op1_addr, Z_LVAL_P(Z_ZV(op2_addr)), TMP1, TMP2 } else { - | GET_ZVAL_LVAL ZREG_REG0, op1_addr + | GET_ZVAL_LVAL ZREG_REG0, op1_addr, TMP1 if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) { | cmp Rx(ZREG_REG0), xzr } else { @@ -11219,7 +11218,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 } } - | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr + | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr, TMP1 if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, res_exit_addr, not_found_exit_addr, exit_addr)) { return 0; } @@ -11240,10 +11239,10 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, } } | SET_EX_OPLINE opline, REG0 - | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr + | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr, TMP1 if (opline->opcode != ZEND_FETCH_DIM_IS) { if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) { - | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr + | GET_ZVAL_LVAL ZREG_FCARG2x, op2_addr, TMP1 | EXT_CALL zend_jit_fetch_dim_str_offset_r_helper, REG0 } else { | LOAD_ZVAL_ADDR FCARG2x, op2_addr @@ -11631,7 +11630,7 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { | IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1 } - | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr + | GET_ZVAL_LVAL ZREG_FCARG1x, op1_addr, TMP1 if (exit_addr && !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) && !may_throw @@ -13750,7 +13749,7 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o if (op1_info & MAY_BE_LONG) { if (op1_info & MAY_BE_REF) { | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >1, ZREG_TMP1 - | GET_ZVAL_LVAL ZREG_FCARG2x, op1_addr + | GET_ZVAL_LVAL ZREG_FCARG2x, op1_addr, TMP1 |.cold_code |1: | // ZVAL_DEREF(op) @@ -13779,7 +13778,7 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1 } } - | GET_ZVAL_LVAL ZREG_FCARG2x, op1_addr + | GET_ZVAL_LVAL ZREG_FCARG2x, op1_addr, TMP1 } if (HT_IS_PACKED(jumptable)) { uint32_t count = jumptable->nNumUsed; @@ -13907,7 +13906,7 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o | IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, =>default_b, ZREG_TMP1 } } - | GET_ZVAL_LVAL ZREG_FCARG2x, op1_addr + | GET_ZVAL_LVAL ZREG_FCARG2x, op1_addr, TMP1 | EXT_CALL zend_hash_index_find, REG0 | mov REG0, RETVALx if (op1_info & MAY_BE_STRING) { From 59622d9ff2d9a16cecc835d217e437295ec242ff Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 11 May 2021 11:13:32 +0300 Subject: [PATCH 205/229] Fixed profile based JIT (opcache.jit=1225) --- ext/opcache/jit/zend_jit_arm64.dasc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index a12c67cc1987b..ef5b3d2c89e0f 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -2098,9 +2098,16 @@ static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst) | // jit_extension = (const void*)ZEND_FUNC_INFO(op_array); | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] | // ++ZEND_COUNTER_INFO(op_array) - | SAFE_MEM_ACC_WITH_UOFFSET ldr, TMP2, REG2, (zend_jit_profile_counter_rid * sizeof(void*)), TMP1 - | add TMP2, TMP2, #1 - | str TMP2, [REG2, TMP1] + || if ((zend_jit_profile_counter_rid * sizeof(void*)) > LDR_STR_PIMM64) { + | LOAD_32BIT_VAL TMP1, (zend_jit_profile_counter_rid * sizeof(void*)) + | ldr TMP2, [REG2, TMP1] + | add TMP2, TMP2, #1 + | str TMP2, [REG2, TMP1] + || } else { + | ldr TMP2, [REG2, #(zend_jit_profile_counter_rid * sizeof(void*))] + | add TMP2, TMP2, #1 + | str TMP2, [REG2, #(zend_jit_profile_counter_rid * sizeof(void*))] + || } | // return ((zend_vm_opcode_handler_t)jit_extension->orig_handler)() | ldr TMP1, [REG0, #offsetof(zend_jit_op_array_extension, orig_handler)] | br TMP1 From fd3f247bdab528a3b59c705ff833061a5f2819f3 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Tue, 11 May 2021 07:42:59 +0000 Subject: [PATCH 206/229] Revert the macro uses for Z_TYPE_P(val) in commit 1fff62b As suggested by Dmitry, macro CMP_32_WITH_CONST is not necessary because Z_TYPE_P(val) is of uchar type and it can be encoded as the imm field of 'cmp' instruction directly. Change-Id: Icb8f9ee847b9a08cb1e9127e7faf6c89d1431922 --- ext/opcache/jit/zend_jit_arm64.dasc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index ef5b3d2c89e0f..4ce9c91d0bc47 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -7518,7 +7518,7 @@ static int zend_jit_identical(dasm_State **Dst, zval *val = Z_ZV(op1_addr); | ldrb TMP1w, [FCARG2x, #offsetof(zval, u1.v.type)] - | CMP_32_WITH_CONST TMP1w, Z_TYPE_P(val), TMP2w + | cmp TMP1w, #Z_TYPE_P(val) if (smart_branch_opcode) { if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) { | bne >8 @@ -7570,7 +7570,7 @@ static int zend_jit_identical(dasm_State **Dst, zval *val = Z_ZV(op2_addr); | ldrb TMP1w, [FCARG1x, #offsetof(zval, u1.v.type)] - | CMP_32_WITH_CONST TMP1w, Z_TYPE_P(val), TMP2w + | cmp TMP1w, #Z_TYPE_P(val) if (smart_branch_opcode) { if (opline->opcode != ZEND_CASE_STRICT && opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) { From ee08686f7c96c31bf0be87b6277f1e6fbcca652e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 11 May 2021 18:15:13 +0300 Subject: [PATCH 207/229] Fixed JIT failure on Zend/tests/bug43175.phpt ZTS build, CALL VM. --- ext/opcache/jit/zend_jit_arm64.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 4ce9c91d0bc47..08d2647e4c590 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -643,7 +643,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | SAFE_MEM_ACC_WITH_UOFFSET ldr, IP, IP, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg || } else { | LOAD_TSRM_CACHE RX -| SAFE_MEM_ACC_WITH_UOFFSET ldr, RX, RX, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg +| add RX, RX, #(struct.._offset+offsetof(zend_..struct, field)) | str RX, EX->opline || } | .else From 3014cf4add0f80179ba9d8f54213c44463d00a46 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 11 May 2021 18:48:51 +0300 Subject: [PATCH 208/229] Fix compilation warnings --- ext/opcache/jit/zend_jit_arm64.dasc | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 08d2647e4c590..66b5e31cc7e2b 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -1119,7 +1119,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) || has_concrete_type(src_info & MAY_BE_ANY)) { || if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { || if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) { -|| zend_uchar type = concrete_type(src_info); +|| uint32_t type = concrete_type(src_info); | SET_ZVAL_TYPE_INFO dst_addr, type, Rw(tmp_reg1), Rx(tmp_reg2) || } || } @@ -1206,7 +1206,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) || } || if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) && || has_concrete_type(src_info & MAY_BE_ANY)) { -|| zend_uchar type = concrete_type(src_info); +|| uint32_t type = concrete_type(src_info); || if (Z_MODE(dst_addr) == IS_MEM_ZVAL) { || if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF))) { | SET_ZVAL_TYPE_INFO dst_addr, type, Rw(tmp_reg), Rx(tmp_reg2) @@ -1227,7 +1227,6 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.endmacro |.macro IF_TYPE, type, val, label -|| ZEND_ASSERT(val >=0 && val <= CMP_IMM); || if (val == 0) { | cbz type, label || } else { @@ -1237,7 +1236,6 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.endmacro |.macro IF_NOT_TYPE, type, val, label -|| ZEND_ASSERT(val >=0 && val <= CMP_IMM); || if (val == 0) { | cbnz type, label || } else { @@ -1258,7 +1256,6 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.macro CMP_ZVAL_TYPE, addr, val, tmp_reg || ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); -|| ZEND_ASSERT(val <= CMP_IMM); | SAFE_MEM_ACC_WITH_UOFFSET_BYTE ldrb, Rw(tmp_reg), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), Rx(tmp_reg) | cmp Rw(tmp_reg), #val |.endmacro @@ -11349,7 +11346,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, |8: if (res_exit_addr) { - zend_uchar type = concrete_type(res_info); + uint32_t type = concrete_type(res_info); if (op1_info & MAY_BE_ARRAY_OF_REF) { | ZVAL_DEREF REG0, MAY_BE_REF, TMP1w } @@ -12367,7 +12364,7 @@ static int zend_jit_fetch_obj(dasm_State **Dst, zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; int32_t exit_point; const void *exit_addr; - zend_uchar type; + uint32_t type; zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) @@ -14282,7 +14279,7 @@ static int zend_jit_fetch_constant(dasm_State **Dst, res_info &= ~MAY_BE_GUARD; ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; - zend_uchar type = concrete_type(res_info); + uint32_t type = concrete_type(res_info); if (type < IS_STRING) { | IF_NOT_ZVAL_TYPE const_addr, type, &exit_addr, ZREG_TMP1 From 00270c6c0195c4b1ecb497b69c4fe50b4c5ccc65 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 11 May 2021 23:19:48 +0300 Subject: [PATCH 209/229] Use better code for prologue and fix code generaion for "return -1". --- ext/opcache/jit/zend_jit_arm64.dasc | 106 +++++++++++++--------------- 1 file changed, 50 insertions(+), 56 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 66b5e31cc7e2b..d5d22ba42d001 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -43,8 +43,8 @@ |.define FCARG1w, w0 |.define FCARG2x, x1 |.define FCARG2w, w1 -|.define SPAD, #0x20 // padding for CPU stack alignment -|.define NR_SPAD, #0x30 // padding for CPU stack alignment +|.define SPAD, 0x20 // padding for CPU stack alignment +|.define NR_SPAD, 0x30 // padding for CPU stack alignment |.define T3, [sp, #0x28] // Used to store old value of IP (CALL VM only) |.define T2, [sp, #0x20] // Used to store old value of FP (CALL VM only) |.define T1, [sp, #0x10] @@ -88,8 +88,6 @@ |.define HYBRID_SPAD, #32 // padding for stack alignment -#define SPAD 0x20 -#define NR_SPAD 0x30 #define TMP_ZVAL_OFFSET 16 #define DASM_ALIGNMENT 16 @@ -1724,11 +1722,11 @@ static int zend_jit_interrupt_handler_stub(dasm_State **Dst) | ADD_HYBRID_SPAD | JMP_IP TMP1 } else if (GCC_GLOBAL_REGS) { - | ldp x29, x30, [sp], #SPAD // stack alignment + | ldp x29, x30, [sp], # SPAD // stack alignment | JMP_IP TMP1 } else { - | ldp FP, RX, T2 // retore FP and IP - | ldp x29, x30, [sp], #NR_SPAD // stack alignment + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment | mov RETVALx, #1 // ZEND_VM_ENTER | ret } @@ -1749,13 +1747,13 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) const void *handler = EG(exception_op)->handler; if (GCC_GLOBAL_REGS) { - | ldp x29, x30, [sp], #SPAD // stack alignment + | ldp x29, x30, [sp], # SPAD // stack alignment | EXT_JMP handler, REG0 } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { | mov FCARG1x, FP | EXT_CALL handler, REG0 - | ldp FP, RX, T2 // retore FP and IP - | ldp x29, x30, [sp], #NR_SPAD // stack alignment + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment | tst RETVALw, RETVALw | blt >1 | mov RETVALw, #1 // ZEND_VM_ENTER @@ -1763,8 +1761,8 @@ static int zend_jit_exception_handler_stub(dasm_State **Dst) | ret } else { | mov FCARG1x, FP - | ldp FP, RX, T2 // retore FP and IP - | ldp x29, x30, [sp], #NR_SPAD // stack alignment + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment | EXT_JMP handler, REG0 } } @@ -1803,11 +1801,11 @@ static int zend_jit_leave_function_stub(dasm_State **Dst) | JMP_IP TMP1 } else { if (GCC_GLOBAL_REGS) { - | ldp x29, x30, [sp], #SPAD // stack alignment + | ldp x29, x30, [sp], # SPAD // stack alignment } else { | mov FCARG2x, FP - | ldp FP, RX, T2 // retore FP and IP - | ldp x29, x30, [sp], #NR_SPAD // stack alignment + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment } | TST_32_WITH_CONST FCARG1w, ZEND_CALL_TOP, TMP1w | bne >1 @@ -1844,8 +1842,8 @@ static int zend_jit_leave_throw_stub(dasm_State **Dst) |5: | // opline = EG(exception_op); | LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2 - | ldp FP, RX, T2 // retore FP and IP - | ldp x29, x30, [sp], #NR_SPAD // stack alignment + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment | mov RETVALx, #2 // ZEND_VM_LEAVE | ret } @@ -2286,12 +2284,12 @@ static int zend_jit_trace_halt_stub(dasm_State **Dst) | ADD_HYBRID_SPAD | EXT_JMP zend_jit_halt_op->handler, REG0 } else if (GCC_GLOBAL_REGS) { - | ldp x29, x30, [sp], #SPAD // stack alignment + | ldp x29, x30, [sp], # SPAD // stack alignment | ret // PC must be zero } else { - | ldp FP, RX, T2 // retore FP and IP - | ldp x29, x30, [sp], #NR_SPAD // stack alignment - | sub RETVALx, xzr, #1 // ZEND_VM_RETURN (-1) + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment + | movn RETVALx, #0 // ZEND_VM_RETURN (-1) | ret } return 1; @@ -2360,11 +2358,11 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst) | ADD_HYBRID_SPAD | JMP_IP TMP1 } else if (GCC_GLOBAL_REGS) { - | ldp x29, x30, [sp], #SPAD // stack alignment + | ldp x29, x30, [sp], # SPAD // stack alignment | JMP_IP TMP1 } else { - | ldp FP, RX, T2 // retore FP and IP - | ldp x29, x30, [sp], #NR_SPAD // stack alignment + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment | mov RETVALx, #1 // ZEND_VM_ENTER | ret } @@ -2390,7 +2388,7 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst) | ldr REG0, [REG0] | br REG0 } else if (GCC_GLOBAL_REGS) { - | ldp x29, x30, [sp], #SPAD // stack alignment + | ldp x29, x30, [sp], # SPAD // stack alignment | ldr REG0, EX->func | ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])] | ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)] @@ -2410,9 +2408,9 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst) | tst RETVALw, RETVALw | blt ->trace_halt | - | ldp FP, RX, T2 // retore FP and IP - | ldp x29, x30, [sp], #NR_SPAD // stack alignment - | mov RETVALx, #1 // ZEND_VM_ENTER + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment + | mov RETVALx, #1 // ZEND_VM_ENTER | ret } @@ -2427,11 +2425,11 @@ static int zend_jit_trace_escape_stub(dasm_State **Dst) | ADD_HYBRID_SPAD | JMP_IP, TMP1 } else if (GCC_GLOBAL_REGS) { - | ldp x29, x30, [sp], #SPAD // stack alignment + | ldp x29, x30, [sp], # SPAD // stack alignment | JMP_IP, TMP1 } else { - | ldp FP, RX, T2 // retore FP and IP - | ldp x29, x30, [sp], #NR_SPAD // stack alignment + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment | mov RETVALx, #1 // ZEND_VM_ENTER | ret } @@ -2777,12 +2775,10 @@ static int zend_jit_prologue(dasm_State **Dst) if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { | SUB_HYBRID_SPAD } else if (GCC_GLOBAL_REGS) { - | sub sp, sp, SPAD // TODO: stp x29, x30, [sp, #-SPAD]! can't be compiled - | stp x29, x30, [sp] // stack alignment + | stp x29, x30, [sp, # -SPAD]! // stack alignment } else { - | sub sp, sp, NR_SPAD // TODO: stp x29, x30, [sp, #-NR_SPAD]! can't be compiled - | stp x29, x30, [sp] // stack alignment - | stp FP, RX, T2 // save FP and IP + | stp x29, x30, [sp, # -NR_SPAD]! // stack alignment + | stp FP, RX, T2 // save FP and IP | mov FP, FCARG1x } return 1; @@ -3034,15 +3030,13 @@ static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, prologue_size = 4; #endif } else if (GCC_GLOBAL_REGS) { - // sub sp, sp, #0x20 - // stp x29, x30, [sp] - prologue_size = 8; + // stp x29, x30, [sp, # -SPAD]! + prologue_size = 4; } else { - // sub sp, sp, NR_SPAD - // stp x29, x30, [sp] + // stp x29, x30, [sp, # -NR_SPAD]! // stack alignment // stp FP, RX, T2 // mov FP, FCARG1x - prologue_size = 16; + prologue_size = 12; } link_addr = (const void*)((const char*)t->code_start + prologue_size); @@ -3072,7 +3066,7 @@ static int zend_jit_trace_return(dasm_State **Dst, bool original_handler) | br REG0 } } else if (GCC_GLOBAL_REGS) { - | ldp x29, x30, [sp], #SPAD // stack alignment + | ldp x29, x30, [sp], # SPAD // stack alignment if (!original_handler) { | JMP_IP TMP1 } else { @@ -3093,9 +3087,9 @@ static int zend_jit_trace_return(dasm_State **Dst, bool original_handler) | ldr REG0, [REG0] | blr REG0 } - | ldp FP, RX, T2 // retore FP and IP - | ldp x29, x30, [sp], #NR_SPAD // stack alignment - | mov RETVALx, #2 // ZEND_VM_LEAVE + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment + | mov RETVALx, #2 // ZEND_VM_LEAVE | ret } return 1; @@ -3351,11 +3345,11 @@ static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline) const void *handler = opline->handler; if (GCC_GLOBAL_REGS) { - | ldp x29, x30, [sp], #SPAD // stack alignment + | ldp x29, x30, [sp], # SPAD // stack alignment } else { | mov FCARG1x, FP - | ldp FP, RX, T2 // retore FP and IP - | ldp x29, x30, [sp], #NR_SPAD // stack alignment + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment } | EXT_JMP handler, REG0 } @@ -9518,11 +9512,11 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | ADD_HYBRID_SPAD | JMP_IP TMP1 } else if (GCC_GLOBAL_REGS) { - | ldp x29, x30, [sp], #SPAD // stack alignment + | ldp x29, x30, [sp], # SPAD // stack alignment | JMP_IP TMP1 } else { - | ldp FP, RX, T2 // retore FP and IP - | ldp x29, x30, [sp], #NR_SPAD // stack alignment + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment | mov RETVALx, #1 // ZEND_VM_ENTER | ret } @@ -10864,7 +10858,7 @@ static int zend_jit_leave_func(dasm_State **Dst, | JMP_IP TMP1 #endif } else if (GCC_GLOBAL_REGS) { - | ldp x29, x30, [sp], #SPAD // stack alignment + | ldp x29, x30, [sp], # SPAD // stack alignment #ifdef CONTEXT_THREADED_JIT | NIY // TODO #else @@ -10877,9 +10871,9 @@ static int zend_jit_leave_func(dasm_State **Dst, // the value of execute_data in execute_ex() | NIY // TODO #else - | ldp FP, RX, T2 // retore FP and IP - | ldp x29, x30, [sp], #NR_SPAD // stack alignment - | mov RETVALx, #2 // ZEND_VM_LEAVE ???? + | ldp FP, RX, T2 // retore FP and IP + | ldp x29, x30, [sp], # NR_SPAD // stack alignment + | mov RETVALx, #2 // ZEND_VM_LEAVE ???? | ret #endif } From 0b79199500d8d6e449569f57025766f9b68821d1 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 12 May 2021 01:20:00 +0300 Subject: [PATCH 210/229] Peephole Code Optimization: ldr + sxtw -> ldrsw lsl + add/sub/cmp -> add/sub/cmp (shifted register) LOAD_64BIT_VAL + add -> ADD_SUB_64_WITH_CONST --- ext/opcache/jit/zend_jit_arm64.dasc | 44 ++++++++++++++--------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index d5d22ba42d001..ea88181075f2e 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -670,6 +670,16 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) || } |.endmacro +|.macro ADD_IP_SHIFT, val, shift, tmp_reg +|| if (GCC_GLOBAL_REGS) { +| add IP, IP, val, shift +|| } else { +| ldr tmp_reg, EX->opline +| add tmp_reg, tmp_reg, val, shift +| str tmp_reg, EX->opline +|| } +|.endmacro + |.macro ADD_IP_FROM_CST, val, tmp_reg || ZEND_ASSERT(val >=0 && val <= ADD_SUB_IMM); || if (GCC_GLOBAL_REGS) { @@ -1926,8 +1936,7 @@ static int zend_jit_undefined_offset_stub(dasm_State **Dst) | ldrb REG1w, OP:REG0->op2_type | cmp REG1w, #IS_CONST | bne >2 - | ldr REG1w, OP:REG0->op2.constant - | sxtw REG1, REG1w + | ldrsw REG1, OP:REG0->op2.constant | add REG0, REG0, REG1 | b >3 |2: @@ -1964,8 +1973,7 @@ static int zend_jit_undefined_index_stub(dasm_State **Dst) | ldrb REG1w, OP:REG0->op2_type | cmp REG1w, #IS_CONST | bne >2 - | ldr REG1w, OP:REG0->op2.constant - | sxtw REG1, REG1w + | ldrsw REG1, OP:REG0->op2.constant | add REG0, REG0, REG1 | b >3 |2: @@ -2019,8 +2027,7 @@ static int zend_jit_undefined_function_stub(dasm_State **Dst) | ldr REG0, EX->opline | mov CARG1, xzr | LOAD_ADDR CARG2, "Call to undefined function %s()" - | ldr CARG3w, [REG0, #offsetof(zend_op, op2.constant)] - | sxtw CARG3, CARG3w + | ldrsw CARG3, [REG0, #offsetof(zend_op, op2.constant)] | ldr CARG3, [REG0, CARG3] | add CARG3, CARG3, #offsetof(zend_string, val) | EXT_CALL zend_throw_error, REG0 @@ -4939,14 +4946,11 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o if (val >= 0) { | ldr REG0, [FCARG1x, #offsetof(zend_array, arData)] if (val != 0) { - | LOAD_64BIT_VAL TMP1, val * sizeof(Bucket) - | add REG0, REG0, TMP1 + | ADD_SUB_64_WITH_CONST add, REG0, REG0, (val * sizeof(Bucket)), TMP1 } } else { - | mov REG0, FCARG2x - | lsl REG0, REG0, #5 | ldr TMP1, [FCARG1x, #offsetof(zend_array, arData)] - | add REG0, REG0, TMP1 + | add REG0, TMP1, FCARG2x, lsl #5 } } } @@ -8257,9 +8261,8 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con | ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.T)] | sub REG2w, REG2w, TMP1w } - | lsl REG2w, REG2w, #5 | sxtw REG2, REG2w - | sub FCARG1x, FCARG1x, REG2 + | sub FCARG1x, FCARG1x, REG2, lsl #5 |1: } @@ -9469,8 +9472,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | // opline += num_args; || ZEND_ASSERT(sizeof(zend_op) == 32); | mov REG2w, REG1w - | lsl REG2, REG2, #5 - | ADD_IP REG2, TMP1 + | ADD_IP_SHIFT REG2, lsl #5, TMP1 } |1: | // if (EXPECTED((int)num_args < op_array->last_var)) { @@ -9482,8 +9484,7 @@ static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend | subs REG2w, REG2w, REG1w | ble >3 | // zval *var = EX_VAR_NUM(num_args); - | lsl REG1, REG1, #4 - | add REG1, REG1, FP + | add REG1, FP, REG1, lsl #4 || ZEND_ASSERT(ZEND_CALL_FRAME_SLOT * sizeof(zval) <= ADD_SUB_IMM); | add REG1, REG1, #(ZEND_CALL_FRAME_SLOT * sizeof(zval)) |2: @@ -11783,8 +11784,7 @@ static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_ | sub REG0, REG0, #1 | // if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket))) | MEM_LOAD_32_ZTS ldr, REG1w, executor_globals, symbol_table.nNumUsed, REG1 - | lsl REG1, REG1, #5 - | cmp REG0, REG1 + | cmp REG0, REG1, lsl #5 | bhs >9 | // Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx); | MEM_LOAD_ZTS ldr, TMP1, executor_globals, symbol_table.arData, REG1 @@ -13791,8 +13791,7 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o | bhs =>default_b } | adr REG0, >4 - | lsl TMP1, FCARG2x, #3 - | ldr TMP1, [REG0, TMP1] + | ldr TMP1, [REG0, FCARG2x, lsl #3] | br TMP1 |.jmp_table @@ -14132,9 +14131,8 @@ static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t o | // p = fe_ht->arData + pos; || ZEND_ASSERT(sizeof(Bucket) == 32); | mov FCARG2w, REG0w - | lsl FCARG2x, FCARG2x, #5 | ldr TMP1, [FCARG1x, #offsetof(zend_array, arData)] - | add FCARG2x, FCARG2x, TMP1 + | add FCARG2x, TMP1, FCARG2x, lsl #5 |1: | // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) { | ldr TMP1w, [FCARG1x, #offsetof(zend_array, nNumUsed)] From 44220c53802636ce6870a62d6f11fbf0d957fbca Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 12 May 2021 02:07:24 +0300 Subject: [PATCH 211/229] Fixed incorrect stack size calculation (sizeof(zval) == 16) --- ext/opcache/jit/zend_jit_arm64.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index ea88181075f2e..342010b9df699 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -8262,7 +8262,7 @@ static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, con | sub REG2w, REG2w, TMP1w } | sxtw REG2, REG2w - | sub FCARG1x, FCARG1x, REG2, lsl #5 + | sub FCARG1x, FCARG1x, REG2, lsl #4 |1: } From 1a6bcd0f80e7492f940711d1c07c1c760ebd7550 Mon Sep 17 00:00:00 2001 From: Hao Sun Date: Wed, 12 May 2021 02:54:55 +0000 Subject: [PATCH 212/229] Fix commit 8143a49: macro ADD_SUB_64_WITH_CONST_32 should be used "(struct.._offset+offsetof(zend_..struct, field))" might exceed the range of ADD_SUB_IMM, leading to DynASM error DASM_S_RANGE_I for ZTS+CALL test variant. Using macro ADD_SUB_64_WITH_CONST_32 would fix it. Change-Id: I3233cefbcd1ddea16e7de6725c2dc5ff43373916 --- ext/opcache/jit/zend_jit_arm64.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 342010b9df699..4c03649f5bf2f 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -641,7 +641,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) | SAFE_MEM_ACC_WITH_UOFFSET ldr, IP, IP, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg || } else { | LOAD_TSRM_CACHE RX -| add RX, RX, #(struct.._offset+offsetof(zend_..struct, field)) +| ADD_SUB_64_WITH_CONST_32 add, RX, RX, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg | str RX, EX->opline || } | .else From da2bed357a44ecf495ba2810e96e5bb33e39b7d5 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 12 May 2021 10:46:22 +0300 Subject: [PATCH 213/229] Fixed incorrect range check (missed sign bit) --- ext/opcache/jit/zend_jit_arm64.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 4c03649f5bf2f..057f3317f8d58 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -101,7 +101,7 @@ #define LDR_STR_PIMM32 (MAX_IMM12*4) // ldr/str insn for 32-bit register: pimm is imm12 * 4 #define LDRB_STRB_PIMM MAX_IMM12 // ldrb/strb insn -#define B_IMM26 (((1<<26)-1)*4) +#define B_IMM26 (1<<27) // signed imm26 * 4 static bool arm64_may_use_b(const void *addr) { From b5aa339cfd1a4ac083c566192e93daa6b87d182b Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 12 May 2021 12:56:52 +0300 Subject: [PATCH 214/229] Reenable PROFITABILITY_CHECKS --- ext/opcache/jit/zend_jit_arm64.dasc | 2 +- ext/opcache/jit/zend_jit_x86.dasc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 057f3317f8d58..22c1d86a8638f 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -156,7 +156,7 @@ static size_t tsrm_ls_cache_tcb_offset = 0; * type information. Disabling this option allows testing some JIT handlers in the * presence of try/catch blocks, which prevent SSA construction. */ #ifndef PROFITABILITY_CHECKS -# define PROFITABILITY_CHECKS 0 // TODO: temporary disabled for better test coverage +# define PROFITABILITY_CHECKS 1 #endif |.type EX, zend_execute_data, FP diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index d69b92d74dc6a..31a19dcc1fdd8 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -162,7 +162,7 @@ static size_t tsrm_tls_offset; * type information. Disabling this option allows testing some JIT handlers in the * presence of try/catch blocks, which prevent SSA construction. */ #ifndef PROFITABILITY_CHECKS -# define PROFITABILITY_CHECKS 0 // TODO: temporary disabled for better test coverage +# define PROFITABILITY_CHECKS 1 #endif |.type EX, zend_execute_data, FP From 94ce76a33139f24b97f3377e3dce241aac97cefd Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 12 May 2021 15:30:30 +0300 Subject: [PATCH 215/229] Implemented AArch64 support for GDB/JIT interface. Stack frame description is not accurate, so backtraces that involved JIT-ed code may be brocken. Disassemble and breakpoints on JIT-ed code work fine. --- ext/opcache/jit/zend_jit.c | 4 +--- ext/opcache/jit/zend_jit_gdb.c | 13 ++++++++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 6caabba0dc2f8..8ee47b96fef75 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -215,10 +215,8 @@ static bool zend_is_commutative(zend_uchar opcode) #include "jit/zend_jit_helpers.c" #include "jit/zend_jit_disasm.c" #ifndef _WIN32 -#if defined(__x86_64__) || defined(i386) # include "jit/zend_jit_gdb.c" -#endif -#include "jit/zend_jit_perf_dump.c" +# include "jit/zend_jit_perf_dump.c" #endif #ifdef HAVE_OPROFILE # include "jit/zend_jit_oprofile.c" diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index ddb5f123c1913..38344f7636a1e 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -21,7 +21,7 @@ */ -#if defined(__x86_64__) || defined(i386) +#if defined(__x86_64__) || defined(i386) || defined(__aarch64__) #define HAVE_GDB @@ -93,7 +93,9 @@ enum { DW_REG_8, DW_REG_9, DW_REG_10, DW_REG_11, DW_REG_12, DW_REG_13, DW_REG_14, DW_REG_15, DW_REG_RA, - /* TODO: ARM supports? */ +#elif defined(__aarch64__) + DW_REG_SP = 31, + DW_REG_RA = 30, #else #error "Unsupported target architecture" #endif @@ -161,6 +163,8 @@ static const zend_elf_header zend_elfhdr_template = { .machine = 3, #elif defined(__x86_64__) .machine = 62, +#elif defined(__aarch64__) + .machine = 183, #else # error "Unsupported target architecture" #endif @@ -328,6 +332,9 @@ static void zend_gdbjit_ehframe(zend_gdbjit_ctx *ctx) #elif defined(__x86_64__) DB(DW_CFA_advance_loc|4); /* sub $0x8,%rsp */ DB(DW_CFA_def_cfa_offset); DUV(16); /* Aligned stack frame size. */ +#elif defined(__aarch64__) + DB(DW_CFA_advance_loc|1); /* Only an approximation. */ + DB(DW_CFA_def_cfa_offset); DUV(32); /* Aligned stack frame size. */ #else # error "Unsupported target architecture" #endif @@ -493,4 +500,4 @@ static void zend_jit_gdb_init(void) #endif } -#endif /* defined(__x86_64__) || defined(i386) */ +#endif /* defined(__x86_64__) || defined(i386) || defined(__aarch64__) */ From 5dd539b866c6aadb9ce054bfba0f38d9eb74ec14 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 12 May 2021 15:47:14 +0300 Subject: [PATCH 216/229] Fixed JIT memory usage debug info (opcache.jit_debug=0x200) --- ext/opcache/jit/zend_jit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 8ee47b96fef75..526b7718956e0 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -4407,7 +4407,7 @@ ZEND_EXT_API int zend_jit_startup(void *buf, size_t size, bool reattached) ZEND_EXT_API void zend_jit_shutdown(void) { if (JIT_G(debug) & ZEND_JIT_DEBUG_SIZE) { - fprintf(stderr, "\nJIT memory usage: %td\n", (char*)*dasm_ptr - (char*)dasm_buf); + fprintf(stderr, "\nJIT memory usage: %td\n", (ptrdiff_t)((char*)*dasm_ptr - (char*)dasm_buf)); } #ifdef HAVE_OPROFILE From 78cdb2a7a0112465d318a3c81cbe13db9c8d133d Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 12 May 2021 17:43:06 +0300 Subject: [PATCH 217/229] Allow to print JIT assemble without binary addresses (opcache.jit_debug=0x001) and with (opcache.jit_debug=0x401) for both ARM and x86. --- ext/opcache/jit/zend_jit.h | 1 + ext/opcache/jit/zend_jit_disasm.c | 13 +++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit.h b/ext/opcache/jit/zend_jit.h index 6985ff141ea7c..753dec20ff732 100644 --- a/ext/opcache/jit/zend_jit.h +++ b/ext/opcache/jit/zend_jit.h @@ -53,6 +53,7 @@ #define ZEND_JIT_DEBUG_GDB (1<<8) #define ZEND_JIT_DEBUG_SIZE (1<<9) +#define ZEND_JIT_DEBUG_ASM_ADDR (1<<10) #define ZEND_JIT_DEBUG_TRACE_START (1<<12) #define ZEND_JIT_DEBUG_TRACE_STOP (1<<13) diff --git a/ext/opcache/jit/zend_jit_disasm.c b/ext/opcache/jit/zend_jit_disasm.c index eabcc0e7dc05c..62276f2c98714 100644 --- a/ext/opcache/jit/zend_jit_disasm.c +++ b/ext/opcache/jit/zend_jit_disasm.c @@ -445,13 +445,15 @@ static int zend_jit_disasm(const char *name, } # ifdef HAVE_CAPSTONE_ITER -# if defined(__aarch64__) - fprintf(stderr, " "ZEND_XLONG_FMT":\t%s ", insn->address, insn->mnemonic); -# else + if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_ADDR) { + fprintf(stderr, " "ZEND_XLONG_FMT":", insn->address); + } fprintf(stderr, "\t%s ", insn->mnemonic); -# endif p = insn->op_str; # else + if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_ADDR) { + fprintf(stderr, " "ZEND_XLONG_FMT":", insn[i].address); + } fprintf(stderr, "\t%s ", insn[i].mnemonic); p = insn[i].op_str; # endif @@ -551,6 +553,9 @@ static int zend_jit_disasm(const char *name, } } } + if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_ADDR) { + fprintf(stderr, " "ZEND_XLONG_FMT":", ud_insn_off(&ud)); + } fprintf(stderr, "\t%s\n", ud_insn_asm(&ud)); } #endif From 43b138852ebbb7b40206d4fd0f34a0919ca677e7 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 12 May 2021 20:35:38 +0300 Subject: [PATCH 218/229] Remove unused TMP4. Use intra-precedure call scratch registers for TMP2 and TMP3. --- ext/opcache/jit/zend_jit_arm64.dasc | 21 +++++++++------------ ext/opcache/jit/zend_jit_arm64.h | 9 ++++----- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 22c1d86a8638f..69a775047543c 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -69,21 +69,18 @@ |.define ZREG_FPR1, ZREG_V1 // Temporaries, not preserved across calls -|.define TMP1, x11 -|.define TMP1w, w11 -|.define TMP2, x12 -|.define TMP2w, w12 -|.define TMP3, x13 -|.define TMP3w, w13 -|.define TMP4, x14 -|.define TMP4w, w14 +|.define TMP1, x15 +|.define TMP1w, w15 +|.define TMP2, x16 +|.define TMP2w, w16 +|.define TMP3, x17 +|.define TMP3w, w17 |.define FPTMP, v16 |.define FPTMPd, d16 -|.define ZREG_TMP1, ZREG_X11 -|.define ZREG_TMP2, ZREG_X12 -|.define ZREG_TMP3, ZREG_X13 -|.define ZREG_TMP4, ZREG_X14 +|.define ZREG_TMP1, ZREG_X15 +|.define ZREG_TMP2, ZREG_X16 +|.define ZREG_TMP3, ZREG_X17 |.define ZREG_FPTMP, ZREG_V16 |.define HYBRID_SPAD, #32 // padding for stack alignment diff --git a/ext/opcache/jit/zend_jit_arm64.h b/ext/opcache/jit/zend_jit_arm64.h index 472598fa57b37..173df223e8f18 100644 --- a/ext/opcache/jit/zend_jit_arm64.h +++ b/ext/opcache/jit/zend_jit_arm64.h @@ -125,10 +125,9 @@ typedef struct _zend_jit_registers_buf { #define ZREG_FPR0 ZREG_V0 #define ZREG_FPR1 ZREG_V1 -#define ZREG_TMP1 ZREG_X11 -#define ZREG_TMP2 ZREG_X12 -#define ZREG_TMP3 ZREG_X13 -#define ZREG_TMP4 ZREG_X14 +#define ZREG_TMP1 ZREG_X15 +#define ZREG_TMP2 ZREG_X16 +#define ZREG_TMP3 ZREG_X17 #define ZREG_FPTMP ZREG_V16 #define ZREG_COPY ZREG_REG0 @@ -141,7 +140,7 @@ typedef uint64_t zend_regset; # define ZEND_REGSET_FIXED \ (ZEND_REGSET(ZREG_RSP) | ZEND_REGSET(ZREG_RLR) | ZEND_REGSET(ZREG_RFP) | \ ZEND_REGSET(ZREG_RPR) | ZEND_REGSET(ZREG_FP) | ZEND_REGSET(ZREG_IP) | \ - ZEND_REGSET_INTERVAL(ZREG_TMP1, ZREG_TMP4) | ZEND_REGSET(ZREG_FPTMP)) + ZEND_REGSET_INTERVAL(ZREG_TMP1, ZREG_TMP3) | ZEND_REGSET(ZREG_FPTMP)) # define ZEND_REGSET_GP \ ZEND_REGSET_DIFFERENCE(ZEND_REGSET_INTERVAL(ZREG_X0, ZREG_X30), ZEND_REGSET_FIXED) # define ZEND_REGSET_FP \ From e1e050d75213a2f7b0d1900e8fe96d1a36adaee6 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 12 May 2021 23:12:23 +0300 Subject: [PATCH 219/229] Use better code for trace exits. Don't store CPU registers that can't be used by deoptimizer. --- ext/opcache/jit/zend_jit_arm64.dasc | 57 +++++++++++++++-------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 69a775047543c..fe50a46b3e18e 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -2313,24 +2313,24 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst) | stp d20, d21, [sp, #-16]! | stp d18, d19, [sp, #-16]! | stp d16, d17, [sp, #-16]! - | stp d14, d15, [sp, #-16]! - | stp d12, d13, [sp, #-16]! - | stp d10, d11, [sp, #-16]! - | stp d8, d9, [sp, #-16]! - | stp d6, d7, [sp, #-16]! + | //stp d14, d15, [sp, #-16]! // we don't use preserved registers yet + | //stp d12, d13, [sp, #-16]! + | //stp d10, d11, [sp, #-16]! + | //stp d8, d9, [sp, #-16]! + | stp d6, d7, [sp, #(-16*5)]! | stp d4, d5, [sp, #-16]! | stp d2, d3, [sp, #-16]! | stp d0, d1, [sp, #-16]! | - | str x30, [sp, #-16]! // x31 can be omitted - | stp x28, x29, [sp, #-16]! - | stp x26, x27, [sp, #-16]! - | stp x24, x25, [sp, #-16]! - | stp x22, x23, [sp, #-16]! - | stp x20, x21, [sp, #-16]! - | stp x18, x19, [sp, #-16]! - | stp x16, x17, [sp, #-16]! - | stp x14, x15, [sp, #-16]! + | //str x30, [sp, #-16]! // we don't use callee-saved registers yet (x31 can be omitted) + | stp x28, x29, [sp, #(-16*2)]! // we have to store RX (x28) + | //stp x26, x27, [sp, #-16]! // we don't use callee-saved registers yet + | //stp x24, x25, [sp, #-16]! + | //stp x22, x23, [sp, #-16]! + | //stp x20, x21, [sp, #-16]! + | //stp x18, x19, [sp, #-16]! + | //stp x16, x17, [sp, #-16]! // we don't need temporary registers + | stp x14, x15, [sp, #-(16*7)]! | stp x12, x13, [sp, #-16]! | stp x10, x11, [sp, #-16]! | stp x8, x9, [sp, #-16]! @@ -2339,7 +2339,7 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst) | stp x2, x3, [sp, #-16]! | stp x0, x1, [sp, #-16]! | - | ldr FCARG1w, [sp, #(32 * 16)] // exit_num = POP + | mov FCARG1w, TMP1w // exit_num | mov FCARG2x, sp | | // EX(opline) = opline @@ -2347,7 +2347,7 @@ static int zend_jit_trace_exit_stub(dasm_State **Dst) | // zend_jit_trace_exit(trace_num, exit_num) | EXT_CALL zend_jit_trace_exit, REG0 | - | add sp, sp, #(33 * 16) // including the pre-allocated 16 bytes + | add sp, sp, #(32 * 16) // restore sp | | tst RETVALw, RETVALw @@ -2442,25 +2442,26 @@ static int zend_jit_trace_escape_stub(dasm_State **Dst) } /* Keep 32 exit points in a single code block */ -#define ZEND_JIT_EXIT_POINTS_SPACING 12 // mov + strb + b = bytes +#define ZEND_JIT_EXIT_POINTS_SPACING 4 // bl = bytes #define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n) { uint32_t i; - for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP - 1; i++) { - | mov TMP1w, #i - | strb TMP1w, [sp, #-16]! - | b >1 - } - | mov TMP1w, #i - | strb TMP1w, [sp, #-16]! + | bl >2 |1: - | ldrb TMP1w, [sp] - | ADD_SUB_32_WITH_CONST add, TMP1w, TMP1w, n, TMP2w - | str TMP1w, [sp] - | b ->trace_exit + for (i = 1; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) { + | bl >2 + } + |2: + | adr TMP1, <1 + | sub TMP1, lr, TMP1 + | lsr TMP1, TMP1, #2 + if (n) { + | ADD_SUB_32_WITH_CONST add, TMP1w, TMP1w, n, TMP2w + } + | b ->trace_exit // pass exit_num in TMP1w return 1; } From 8970b684e9a7da9b5c7aeaa92ac4a226d44a10a2 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Wed, 12 May 2021 23:30:12 +0300 Subject: [PATCH 220/229] Use symbolic labels for BL and ADR instructions --- ext/opcache/jit/zend_jit_disasm.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_disasm.c b/ext/opcache/jit/zend_jit_disasm.c index 62276f2c98714..81cd844d9350b 100644 --- a/ext/opcache/jit/zend_jit_disasm.c +++ b/ext/opcache/jit/zend_jit_disasm.c @@ -234,7 +234,9 @@ static uint64_t zend_jit_disasm_branch_target(csh cs, const cs_insn *insn) } } #elif defined(__aarch64__) - if (cs_insn_group(cs, insn, ARM64_GRP_JUMP)) { + if (cs_insn_group(cs, insn, ARM64_GRP_JUMP) + || insn->id == ARM64_INS_BL + || insn->id == ARM64_INS_ADR) { for (i = 0; i < insn->detail->arm64.op_count; i++) { if (insn->detail->arm64.operands[i].type == ARM64_OP_IMM) return insn->detail->arm64.operands[i].imm; From 59ef21999450fa3066582ec9c637e55fbddc587e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 13 May 2021 11:03:14 +0300 Subject: [PATCH 221/229] Fixed ZTS build. --- ext/opcache/jit/zend_jit_arm64.dasc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index fe50a46b3e18e..6f16ba8b98a68 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -73,7 +73,7 @@ |.define TMP1w, w15 |.define TMP2, x16 |.define TMP2w, w16 -|.define TMP3, x17 +|.define TMP3, x17 // TODO: remember about hard-coded: mrs TMP3, tpidr_el0 |.define TMP3w, w17 |.define FPTMP, v16 |.define FPTMPd, d16 @@ -440,8 +440,7 @@ static int logical_immediate_p (uint64_t value, uint32_t reg_size) |.endmacro |.macro LOAD_TSRM_CACHE, reg -| //.byte 0x4d, 0xd0, 0x3b, 0xd5 // TODO: hard-coded: mrs TMP3, tpidr_el0 -| .long 0xd53bd04d // TODO: hard-coded: mrs TMP3, tpidr_el0 +| .long 0xd53bd051 // TODO: hard-coded: mrs TMP3, tpidr_el0 || ZEND_ASSERT(tsrm_ls_cache_tcb_offset <= LDR_STR_PIMM64); | ldr reg, [TMP3, #tsrm_ls_cache_tcb_offset] |.endmacro From 0e8fc156f8617269530d806bd087bfa0da70d9e5 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 13 May 2021 17:39:07 +0300 Subject: [PATCH 222/229] DynASM/ARM64: Add abiulity to plug-in generation of veneers to perform long jumps. --- ext/opcache/jit/dynasm/dasm_arm64.h | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/ext/opcache/jit/dynasm/dasm_arm64.h b/ext/opcache/jit/dynasm/dasm_arm64.h index 8d1d9a9654247..909b51f808a80 100644 --- a/ext/opcache/jit/dynasm/dasm_arm64.h +++ b/ext/opcache/jit/dynasm/dasm_arm64.h @@ -404,6 +404,15 @@ int dasm_link(Dst_DECL, size_t *szp) return DASM_S_OK; } +#ifdef DASM_ADD_VENEER +#define CK_REL(x, o) \ + do { if (!(x) && !(n = DASM_ADD_VENEER(D, buffer, ins, b, cp, o))) \ + return DASM_S_RANGE_REL|(p-D->actionlist-1); \ + } while (0) +#else +#define CK_REL(x, o) CK(x, RANGE_REL) +#endif + #ifdef DASM_CHECKS #define CK(x, st) \ do { if (!(x)) return DASM_S_##st|(p-D->actionlist-1); } while (0) @@ -444,7 +453,7 @@ int dasm_encode(Dst_DECL, void *buffer) if (n < 0) { ptrdiff_t na = (ptrdiff_t)D->globals[-n] - (ptrdiff_t)cp + 4; n = (int)na; - CK((ptrdiff_t)n == na, RANGE_REL); + CK_REL((ptrdiff_t)n == na, na); goto patchrel; } /* fallthrough */ @@ -453,18 +462,18 @@ int dasm_encode(Dst_DECL, void *buffer) n = *DASM_POS2PTR(D, n) - (int)((char *)cp - base) + 4; patchrel: if (!(ins & 0xf800)) { /* B, BL */ - CK((n & 3) == 0 && ((n+0x08000000) >> 28) == 0, RANGE_REL); + CK_REL((n & 3) == 0 && ((n+0x08000000) >> 28) == 0, n); cp[-1] |= ((n >> 2) & 0x03ffffff); } else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */ - CK((n & 3) == 0 && ((n+0x00100000) >> 21) == 0, RANGE_REL); + CK_REL((n & 3) == 0 && ((n+0x00100000) >> 21) == 0, n); cp[-1] |= ((n << 3) & 0x00ffffe0); } else if ((ins & 0x3000) == 0x2000) { /* ADR */ - CK(((n+0x00100000) >> 21) == 0, RANGE_REL); + CK_REL(((n+0x00100000) >> 21) == 0, n); cp[-1] |= ((n << 3) & 0x00ffffe0) | ((n & 3) << 29); } else if ((ins & 0x3000) == 0x3000) { /* ADRP */ cp[-1] |= ((n >> 9) & 0x00ffffe0) | (((n >> 12) & 3) << 29); } else if ((ins & 0x1000)) { /* TBZ, TBNZ */ - CK((n & 3) == 0 && ((n+0x00008000) >> 16) == 0, RANGE_REL); + CK_REL((n & 3) == 0 && ((n+0x00008000) >> 16) == 0, n); cp[-1] |= ((n << 3) & 0x0007ffe0); } else if ((ins & 0x8000)) { /* absolute */ cp[0] = (unsigned int)((ptrdiff_t)cp - 4 + n); @@ -475,7 +484,7 @@ int dasm_encode(Dst_DECL, void *buffer) case DASM_REL_A: { ptrdiff_t na = (((ptrdiff_t)(*b++) << 32) | (unsigned int)n) - (ptrdiff_t)cp + 4; n = (int)na; - CK((ptrdiff_t)n == na, RANGE_REL); + CK_REL((ptrdiff_t)n == na, na); goto patchrel; } case DASM_LABEL_LG: From fc3157ea0f621de5bc4e9a1804dd8dbd13ba92ad Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 13 May 2021 17:41:26 +0300 Subject: [PATCH 223/229] Generate veneers to perform long jumps. This makes symfony_demo app working with -d opcacge.jit=1205. (21 MB of JIT-ed code). --- ext/opcache/jit/zend_jit.c | 18 +++- ext/opcache/jit/zend_jit_arm64.dasc | 127 ++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 526b7718956e0..8cd14fc987bbc 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -209,6 +209,8 @@ static bool zend_is_commutative(zend_uchar opcode) #if defined(__x86_64__) || defined(i386) || defined(ZEND_WIN32) #include "dynasm/dasm_x86.h" #elif defined(__aarch64__) +static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int *b, uint32_t *cp, ptrdiff_t offset); +#define DASM_ADD_VENEER zend_jit_add_veneer #include "dynasm/dasm_arm64.h" #endif @@ -408,6 +410,10 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, return NULL; } +#ifdef __aarch64__ + dasm_venners_size = 0; +#endif + ret = dasm_encode(dasm_state, *dasm_ptr); if (ret != DASM_S_OK) { #if ZEND_DEBUG @@ -416,6 +422,10 @@ static void *dasm_link_and_encode(dasm_State **dasm_state, return NULL; } +#ifdef __aarch64__ + size += dasm_venners_size; +#endif + entry = *dasm_ptr; *dasm_ptr = (void*)((char*)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(size, DASM_ALIGNMENT)); @@ -4396,8 +4406,14 @@ ZEND_EXT_API int zend_jit_startup(void *buf, size_t size, bool reattached) return FAILURE; } - /* save JIT buffer pos */ zend_jit_unprotect(); +#ifdef __aarch64__ + /* reserve space for global labels veneers */ + dasm_labels_veneers = *dasm_ptr; + *dasm_ptr = (void**)*dasm_ptr + zend_lb_MAX; + memset(dasm_labels_veneers, 0, sizeof(void*) * zend_lb_MAX); +#endif + /* save JIT buffer pos */ dasm_ptr[1] = dasm_ptr[0]; zend_jit_protect(); diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 6f16ba8b98a68..5059d738ddbf3 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -15017,6 +15017,133 @@ static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend return regset; } +static size_t dasm_venners_size = 0; +void **dasm_labels_veneers = NULL; + +static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int *b, uint32_t *cp, ptrdiff_t offset) +{ + void *veneer; + ptrdiff_t na; + int n, m; + + /* try to reuse veneers for global labels */ + if ((ins >> 16) == DASM_REL_LG + && *(b-1) < 0 + && dasm_labels_veneers[-*(b-1)]) { + + veneer = dasm_labels_veneers[-*(b-1)]; + na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; + n = (int)na; + + /* check if we can jump to veneer */ + if ((ptrdiff_t)n != na) { + /* pass */ + } else if (!(ins & 0xf800)) { /* B, BL */ + if ((n & 3) == 0 && ((n+0x08000000) >> 28) == 0) { + return n; + } + } else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */ + if ((n & 3) == 0 && ((n+0x00100000) >> 21) == 0) { + return n; + } + } else if ((ins & 0x3000) == 0x2000) { /* ADR */ + /* pass */ + } else if ((ins & 0x3000) == 0x3000) { /* ADRP */ + /* pass */ + } else if ((ins & 0x1000)) { /* TBZ, TBNZ */ + if ((n & 3) == 0 && ((n+0x00008000) >> 16) == 0) { + return n; + } + } + } + + veneer = (char*)buffer + (Dst->codesize + dasm_venners_size); + + if (veneer > dasm_end) { + return 0; /* jit_buffer_size overflow */ + } + + na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4; + n = (int)na; + + /* check if we can jump to veneer */ + if ((ptrdiff_t)n != na) { + ZEND_ASSERT(0); + return 0; + } else if (!(ins & 0xf800)) { /* B, BL */ + if ((n & 3) != 0 || ((n+0x08000000) >> 28) != 0) { + ZEND_ASSERT(0); + return 0; + } + } else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */ + if ((n & 3) != 0 || ((n+0x00100000) >> 21) != 0) { + ZEND_ASSERT(0); + return 0; + } + } else if ((ins & 0x3000) == 0x2000) { /* ADR */ + ZEND_ASSERT(0); + return 0; + } else if ((ins & 0x3000) == 0x3000) { /* ADRP */ + ZEND_ASSERT(0); + return 0; + } else if ((ins & 0x1000)) { /* TBZ, TBNZ */ + if ((n & 3) != 0 || ((n+0x00008000) >> 16) != 0) { + ZEND_ASSERT(0); + return 0; + } + } else if ((ins & 0x8000)) { /* absolute */ + ZEND_ASSERT(0); + return 0; + } else { + ZEND_ASSERT(0); + return 0; + } + + // TODO: support for long veneers (above 128MB) ??? + + /* check if we can use B to jump from veneer */ + na = (ptrdiff_t)cp + offset - (ptrdiff_t)veneer - 4; + m = (int)na; + if ((ptrdiff_t)m != na) { + ZEND_ASSERT(0); + return 0; + } else if ((m & 3) != 0 || ((m+0x08000000) >> 28) != 0) { + ZEND_ASSERT(0); + return 0; + } + + /* generate B instruction */ + *(uint32_t*)veneer = 0x14000000 | ((m >> 2) & 0x03ffffff); + dasm_venners_size += 4; + + if ((ins >> 16) == DASM_REL_LG + && *(b-1) < 0) { + /* reuse this veneer for the future jumps to global label */ + dasm_labels_veneers[-*(b-1)] = veneer; + /* Dst->globals[*(b-1)] = veneer; */ + +#ifdef HAVE_DISASM + if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) { + const char *name = zend_jit_disasm_find_symbol((ptrdiff_t)cp + offset - 4, &offset); + + if (name && !offset) { + if (strstr(name, "@veneer") == NULL) { + char *new_name; + + zend_spprintf(&new_name, 0, "%s@veneer", name); + zend_jit_disasm_add_symbol(new_name, (uint64_t)(uintptr_t)veneer, 4); + efree(new_name); + } else { + zend_jit_disasm_add_symbol(name, (uint64_t)(uintptr_t)veneer, 4); + } + } + } +#endif + } + + return n; +} + #if defined(__clang__) # pragma clang diagnostic pop #endif From f15f01aed7880bdc96800103fa3d6982afaaf7d6 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 14 May 2021 10:59:16 +0300 Subject: [PATCH 224/229] JIT/AArch64: disable register allocation for expected ++/-- overflow case. This fixes ext/opcache/tests/jit/inc_021.phpt with tracing JIT. --- ext/opcache/jit/zend_jit_arm64.dasc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index 5059d738ddbf3..b4aaba2e7a586 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -14558,7 +14558,10 @@ static bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa case ZEND_POST_INC: case ZEND_POST_DEC: op1_info = OP1_INFO(); - return opline->op1_type == IS_CV && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)); + op2_info = OP1_DEF_INFO(); + return opline->op1_type == IS_CV + && !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG)) + && (op2_info & MAY_BE_LONG); case ZEND_BOOL: case ZEND_BOOL_NOT: case ZEND_JMPZ: From d6bf98428de3e26089949c293efd40228a9fc521 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Fri, 14 May 2021 13:04:18 +0300 Subject: [PATCH 225/229] Fixed format specifier --- ext/opcache/jit/zend_jit_disasm.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ext/opcache/jit/zend_jit_disasm.c b/ext/opcache/jit/zend_jit_disasm.c index 81cd844d9350b..8bbbea6e74b0f 100644 --- a/ext/opcache/jit/zend_jit_disasm.c +++ b/ext/opcache/jit/zend_jit_disasm.c @@ -448,13 +448,13 @@ static int zend_jit_disasm(const char *name, # ifdef HAVE_CAPSTONE_ITER if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_ADDR) { - fprintf(stderr, " "ZEND_XLONG_FMT":", insn->address); + fprintf(stderr, " %" PRIx64 ":", insn->address); } fprintf(stderr, "\t%s ", insn->mnemonic); p = insn->op_str; # else if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_ADDR) { - fprintf(stderr, " "ZEND_XLONG_FMT":", insn[i].address); + fprintf(stderr, " %" PRIx64 ":", insn[i].address); } fprintf(stderr, "\t%s ", insn[i].mnemonic); p = insn[i].op_str; @@ -556,7 +556,7 @@ static int zend_jit_disasm(const char *name, } } if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_ADDR) { - fprintf(stderr, " "ZEND_XLONG_FMT":", ud_insn_off(&ud)); + fprintf(stderr, " %" PRIx64 ":", ud_insn_off(&ud)); } fprintf(stderr, "\t%s\n", ud_insn_asm(&ud)); } From a69701de2ec3f2c47a00d854f3e656c81cd387c0 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 17 May 2021 10:52:59 +0300 Subject: [PATCH 226/229] Fix "store to misaligned address" ASAN warnings --- ext/opcache/jit/zend_jit_gdb.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index 38344f7636a1e..be26583277329 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -283,21 +283,26 @@ static void zend_gdbjit_symtab(zend_gdbjit_ctx *ctx) sym->info = ELFSYM_INFO(ELFSYM_BIND_GLOBAL, ELFSYM_TYPE_FUNC); } +typedef ZEND_SET_ALIGNED(1, uint16_t unaligned_uint16_t); +typedef ZEND_SET_ALIGNED(1, uint32_t unaligned_uint32_t); +typedef ZEND_SET_ALIGNED(1, uint16_t unaligned_uint16_t); +typedef ZEND_SET_ALIGNED(1, uintptr_t unaligned_uintptr_t); + #define SECTALIGN(p, a) \ ((p) = (uint8_t *)(((uintptr_t)(p) + ((a)-1)) & ~(uintptr_t)((a)-1))) /* Shortcuts to generate DWARF structures. */ #define DB(x) (*p++ = (x)) #define DI8(x) (*(int8_t *)p = (x), p++) -#define DU16(x) (*(uint16_t *)p = (x), p += 2) -#define DU32(x) (*(uint32_t *)p = (x), p += 4) -#define DADDR(x) (*(uintptr_t *)p = (x), p += sizeof(uintptr_t)) +#define DU16(x) (*(unaligned_uint16_t *)p = (x), p += 2) +#define DU32(x) (*(unaligned_uint32_t *)p = (x), p += 4) +#define DADDR(x) (*(unaligned_uintptr_t *)p = (x), p += sizeof(uintptr_t)) #define DUV(x) (ctx->p = p, zend_gdbjit_uleb128(ctx, (x)), p = ctx->p) #define DSV(x) (ctx->p = p, zend_gdbjit_sleb128(ctx, (x)), p = ctx->p) #define DSTR(str) (ctx->p = p, zend_gdbjit_strz(ctx, (str)), p = ctx->p) #define DALIGNNOP(s) while ((uintptr_t)p & ((s)-1)) *p++ = DW_CFA_nop #define DSECT(name, stmt) \ - { uint32_t *szp_##name = (uint32_t *)p; p += 4; stmt \ + { unaligned_uint32_t *szp_##name = (uint32_t *)p; p += 4; stmt \ *szp_##name = (uint32_t)((p-(uint8_t *)szp_##name)-4); } static void zend_gdbjit_ehframe(zend_gdbjit_ctx *ctx) From f70ac416c311476139b673f729de9d5c64598047 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 17 May 2021 18:06:40 +0300 Subject: [PATCH 227/229] Fixed possible failure when repair after overflow detection --- ext/opcache/jit/zend_jit_arm64.dasc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index b4aaba2e7a586..fea32f9c0a3d9 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -3858,7 +3858,7 @@ static int zend_jit_math_long_long(dasm_State **Dst, } else { result_reg = Z_REG(res_addr); } - } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) { + } else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr) && !may_overflow) { result_reg = Z_REG(op1_addr); } else if (Z_REG(res_addr) != ZREG_REG0) { result_reg = ZREG_REG0; From d6d0b1c3899f912491f6455632715db0b9d67562 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 18 May 2021 12:41:36 +0300 Subject: [PATCH 228/229] Remove unnecessary #ifdef --- ext/opcache/jit/zend_jit_gdb.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ext/opcache/jit/zend_jit_gdb.c b/ext/opcache/jit/zend_jit_gdb.c index be26583277329..eb91dda317e56 100644 --- a/ext/opcache/jit/zend_jit_gdb.c +++ b/ext/opcache/jit/zend_jit_gdb.c @@ -21,8 +21,6 @@ */ -#if defined(__x86_64__) || defined(i386) || defined(__aarch64__) - #define HAVE_GDB #include "zend_elf.h" @@ -504,5 +502,3 @@ static void zend_jit_gdb_init(void) } #endif } - -#endif /* defined(__x86_64__) || defined(i386) || defined(__aarch64__) */ From dae662ffbc175513ec2ae8386f0cb4abc7b3c8d6 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 18 May 2021 13:16:08 +0300 Subject: [PATCH 229/229] Fixed zend_long_is_power_of_two/zend_long_floor_log2 mess --- ext/opcache/jit/zend_jit_arm64.dasc | 4 ++-- ext/opcache/jit/zend_jit_internal.h | 6 ++++-- ext/opcache/jit/zend_jit_x86.dasc | 14 ++++++-------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index fea32f9c0a3d9..d1cd697ec55ee 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -3902,9 +3902,9 @@ static int zend_jit_math_long_long(dasm_State **Dst, | lsl Rx(result_reg), Rx(result_reg), TMP1 } else if (opcode == ZEND_DIV && (Z_MODE(op2_addr) == IS_CONST_ZVAL && - is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { + zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { | GET_ZVAL_LVAL result_reg, op1_addr, TMP1 - | asr Rx(result_reg), Rx(result_reg), #floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) + | asr Rx(result_reg), Rx(result_reg), #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) #if 0 /* x86 specific optimizations through LEA instraction are not supported on ARM */ } else if (opcode == ZEND_ADD && diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index f29c4b509e699..9beb453b309c9 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -21,6 +21,8 @@ #ifndef ZEND_JIT_INTERNAL_H #define ZEND_JIT_INTERNAL_H +#include "zend_bitset.h" + /* Register Set */ #define ZEND_REGSET_EMPTY 0 @@ -743,10 +745,10 @@ static zend_always_inline bool zend_long_is_power_of_two(zend_long x) return (x > 0) && !(x & (x - 1)); } -static zend_always_inline uint32_t zend_long_floor_log2(uint64_t x) +static zend_always_inline uint32_t zend_long_floor_log2(zend_long x) { ZEND_ASSERT(zend_long_is_power_of_two(x)); - return __builtin_ctzll(x); + return zend_ulong_ntz(x); } /* from http://aggregate.org/MAGIC/ */ diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 31a19dcc1fdd8..52efe13cd7ea0 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -4212,10 +4212,9 @@ static int zend_jit_math_long_long(dasm_State **Dst, Z_MODE(op2_addr) == IS_CONST_ZVAL && !may_overflow && Z_LVAL_P(Z_ZV(op2_addr)) > 0 && - IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op2_addr))) && - is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { + zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) { | GET_ZVAL_LVAL result_reg, op1_addr - | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) + | shl Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) } else if (opcode == ZEND_MUL && Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 2) { @@ -4229,15 +4228,14 @@ static int zend_jit_math_long_long(dasm_State **Dst, Z_MODE(op1_addr) == IS_CONST_ZVAL && !may_overflow && Z_LVAL_P(Z_ZV(op1_addr)) > 0 && - IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr))) && - is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { + zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) { | GET_ZVAL_LVAL result_reg, op2_addr - | shl Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) + | shl Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op1_addr))) } else if (opcode == ZEND_DIV && (Z_MODE(op2_addr) == IS_CONST_ZVAL && - is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { + zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) { | GET_ZVAL_LVAL result_reg, op1_addr - | shr Ra(result_reg), floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) + | shr Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr))) } else if (opcode == ZEND_ADD && !may_overflow && Z_MODE(op1_addr) == IS_REG &&