Obsolete:Oprofile
Note: This page was created as a temporary pastebin and probably doesn't make any sense without its proper context
Apache profiling
Full system profiling of an application server, srv43. Dual AMD Opteron 248. 2007-03-07. Truncated after 200 lines. The two columns shown are (loosely speaking) CPU usage and L2 cache misses. There is a strong correlation between the two metrics, which is especially striking in the case of ap_unblock_alarms, a short and simple function to decrement a variable. This demonstrates that L2 cache misses have a real impact on performance.
[root@srv43 ~]# opreport -l | head -n200 warning: /3w_xxxx could not be found. warning: /ext3 could not be found. warning: /ipv6 could not be found. warning: /nfs could not be found. warning: /ohci_hcd could not be found. warning: /oprofile could not be found. warning: /reiserfs could not be found. warning: /scsi_mod could not be found. warning: /sd_mod could not be found. warning: /sunrpc could not be found. warning: /tg3 could not be found. warning: /usr/lib/locale/locale-archive is not in a usable binary format. CPU: AMD64 processors, speed 2205.07 MHz (estimated) Counted CPU_CLK_UNHALTED events (Cycles outside of halt state) with a unit mask of 0x00 (No unit mask) count 1000000 Counted DATA_CACHE_REFILLS_FROM_SYSTEM events (Data cache refills from system) with a unit mask of 0x1f (All cache states) count 2000 samples % samples % app name symbol name 195393 5.7190 190234 7.2153 libc-2.3.6.so memcpy 171730 5.0264 255991 9.7094 httpd ap_unblock_alarms 167041 4.8891 111303 4.2216 libphp5.so zend_hash_find 136979 4.0093 88804 3.3682 libphp5.so _zend_hash_add_or_update 133298 3.9015 189376 7.1828 libc-2.3.6.so _int_free 124500 3.6440 32837 1.2455 libc-2.3.6.so _int_malloc 122811 3.5946 335312 12.7180 libc-2.3.6.so malloc_consolidate 110317 3.2289 122704 4.6540 libphp5.so _efree 107758 3.1540 88188 3.3449 libphp5.so _emalloc 84605 2.4763 4968 0.1884 libphp5.so match 65575 1.9193 72437 2.7474 libphp5.so _zval_ptr_dtor 62234 1.8215 21624 0.8202 libphp5.so zend_do_fcall_common_helper_SPEC 58158 1.7022 52308 1.9840 libphp5.so zend_hash_quick_find 49646 1.4531 60244 2.2850 libc-2.3.6.so free 45518 1.3323 50035 1.8978 libphp5.so _zval_dtor_func 43952 1.2864 28246 1.0713 libc-2.3.6.so malloc 42778 1.2521 25894 0.9821 httpd ap_block_alarms 41493 1.2145 25614 0.9715 libphp5.so _zend_hash_quick_add_or_update 40553 1.1870 23062 0.8747 libphp5.so zend_hash_destroy 37485 1.0972 28241 1.0711 libphp5.so anonymous symbol from section .plt 31693 0.9276 166 0.0063 vmlinux timer_interrupt 30559 0.8944 11006 0.4174 libphp5.so zend_assign_to_variable 29532 0.8644 13120 0.4976 libphp5.so execute 28040 0.8207 13052 0.4950 libphp5.so _zend_hash_index_update_or_next_insert 26241 0.7681 14178 0.5378 libc-2.3.6.so memset 25535 0.7474 11404 0.4325 libphp5.so _erealloc 24855 0.7275 15387 0.5836 vmlinux __d_lookup 24627 0.7208 19846 0.7527 libphp5.so ZEND_SEND_VAR_SPEC_CV_HANDLER 24579 0.7194 12909 0.4896 libphp5.so zend_hash_rehash 24027 0.7032 33415 1.2674 vmlinux unmap_vmas 22997 0.6731 1451 0.0550 libphp5.so php_pcre_exec 20398 0.5970 3869 0.1467 libphp5.so zend_str_tolower_copy 20088 0.5880 10344 0.3923 libphp5.so zend_fetch_dimension_address 18797 0.5502 30399 1.1530 vmlinux copy_page_range 18371 0.5377 9938 0.3769 libphp5.so ZEND_JMPZ_SPEC_TMP_HANDLER 17636 0.5162 7553 0.2865 libphp5.so _estrndup 17604 0.5153 8487 0.3219 libphp5.so ZEND_ASSIGN_REF_SPEC_CV_VAR_HANDLER 16064 0.4702 11209 0.4251 libphp5.so zend_get_hash_value 16053 0.4699 16675 0.6325 libphp5.so ZEND_RECV_SPEC_HANDLER 16013 0.4687 45583 1.7289 apc.so my_prepare_op_array_for_execution 13273 0.3885 1120 0.0425 libphp5.so php_var_unserialize 13267 0.3883 13105 0.4971 apc.so my_copy_hashtable_ex 13158 0.3851 1435 0.0544 vmlinux do_gettimeoffset_pm 12134 0.3552 2076 0.0787 vmlinux __link_path_walk 11505 0.3367 2542 0.0964 libphp5.so compare_function 11472 0.3358 24011 0.9107 libphp5.so shutdown_memory_manager 11468 0.3357 5095 0.1932 libphp5.so ZEND_RECV_INIT_SPEC_CONST_HANDLER 11411 0.3340 1005 0.0381 libphp5.so zend_hash_clean 11174 0.3271 6182 0.2345 libphp5.so _get_zval_ptr_var 11062 0.3238 1772 0.0672 libphp5.so zend_parse_va_args 10846 0.3175 4930 0.1870 libphp5.so ZEND_DO_FCALL_SPEC_CONST_HANDLER 10079 0.2950 6702 0.2542 libphp5.so ZEND_SEND_VAL_SPEC_CONST_HANDLER 10034 0.2937 5778 0.2192 libphp5.so _zval_copy_ctor_func 9917 0.2903 2781 0.1055 libc-2.3.6.so _int_realloc 9781 0.2863 4857 0.1842 libphp5.so zend_assign_to_object 9619 0.2815 6794 0.2577 vmlinux copy_user_generic_c 9431 0.2760 1168 0.0443 libz.so.1.2.1.2 fill_window 9375 0.2744 3225 0.1223 libphp5.so ZEND_ASSIGN_SPEC_CV_VAR_HANDLER 9199 0.2692 3786 0.1436 libphp5.so zend_fetch_var_address_helper_SPEC_CONST 9022 0.2641 2609 0.0990 libphp5.so zend_get_property_info 8820 0.2582 3538 0.1342 libc-2.3.6.so realloc 8775 0.2568 848 0.0322 libphp5.so zend_call_function 8127 0.2379 1774 0.0673 libc-2.3.6.so ____strtol_l_internal 8091 0.2368 345 0.0131 libphp5.so zend_get_parameters_ex 7608 0.2227 3960 0.1502 libc-2.3.6.so strlen 7430 0.2175 14543 0.5516 vmlinux clear_page 7068 0.2069 3992 0.1514 vmlinux page_fault 6879 0.2013 1592 0.0604 libphp5.so zend_hash_copy 6653 0.1947 8629 0.3273 libphp5.so zend_binary_strcmp 6477 0.1896 2985 0.1132 libc-2.3.6.so memchr 6435 0.1883 3604 0.1367 libphp5.so ZEND_INIT_METHOD_CALL_SPEC_CV_CONST_HANDLER 6384 0.1869 7969 0.3023 libphp5.so zend_hash_del_key_or_index 6097 0.1785 2811 0.1066 libphp5.so ZEND_FE_FETCH_SPEC_VAR_HANDLER 6070 0.1777 1216 0.0461 libc-2.3.6.so vfprintf 5990 0.1753 3267 0.1239 libphp5.so ZEND_ADD_ARRAY_ELEMENT_SPEC_CONST_CONST_HANDLER 5966 0.1746 1866 0.0708 vmlinux groups_search 5452 0.1596 409 0.0155 libphp5.so zendi_smart_strcmp 5369 0.1571 1840 0.0698 reiserfs (no symbols) 5288 0.1548 3894 0.1477 libc-2.3.6.so __ctype_tolower_loc 5240 0.1534 3805 0.1443 libphp5.so zend_std_get_method 5029 0.1472 2986 0.1133 libphp5.so ZEND_RETURN_SPEC_CONST_HANDLER 5008 0.1466 2417 0.0917 libmysqlclient.so.10.0.0 (no symbols) 4986 0.1459 1332 0.0505 libphp5.so ZEND_JMPZ_SPEC_VAR_HANDLER 4878 0.1428 526 0.0200 libphp5.so php_escape_html_entities 4815 0.1409 4298 0.1630 libphp5.so ZEND_DO_FCALL_BY_NAME_SPEC_HANDLER 4805 0.1406 1402 0.0532 libphp5.so _zend_hash_init 4781 0.1399 190 0.0072 libphp5.so zend_strtod 4605 0.1348 4012 0.1522 libphp5.so ZEND_INIT_METHOD_CALL_SPEC_UNUSED_CONST_HANDLER 4599 0.1346 3551 0.1347 fss.so kwsincr 4555 0.1333 126 0.0048 libphp5.so zend_assign_to_variable_reference 4537 0.1328 1950 0.0740 libphp5.so ZEND_FETCH_W_SPEC_CONST_HANDLER 4529 0.1326 1403 0.0532 libphp5.so zend_std_read_property 4423 0.1295 2079 0.0789 libz.so.1.2.1.2 deflate_slow 4414 0.1292 422 0.0160 libz.so.1.2.1.2 inflate_fast 4226 0.1237 3736 0.1417 tg3 (no symbols) 4180 0.1223 1905 0.0723 libphp5.so zend_fetch_property_address_read_helper_SPEC_UNUSED_CONST 4123 0.1207 3095 0.1174 libphp5.so ZEND_JMPZ_EX_SPEC_TMP_HANDLER 4059 0.1188 2197 0.0833 libphp5.so ZEND_ADD_STRING_SPEC_TMP_CONST_HANDLER 4040 0.1182 3215 0.1219 libphp5.so ZEND_FETCH_OBJ_R_SPEC_UNUSED_CONST_HANDLER 3890 0.1139 36 0.0014 vmlinux pmtimer_mark_offset 3856 0.1129 3361 0.1275 libc-2.3.6.so strcmp 3839 0.1124 1718 0.0652 libphp5.so ZEND_JMPZ_SPEC_CV_HANDLER 3827 0.1120 736 0.0279 libphp5.so _ecalloc 3823 0.1119 2972 0.1127 vmlinux dput 3751 0.1098 6106 0.2316 apc.so apc_copy_op_array_for_execution 3750 0.1098 2571 0.0975 libphp5.so ZEND_SEND_VAR_SPEC_VAR_HANDLER 3723 0.1090 349 0.0132 vmlinux do_wp_page 3661 0.1072 875 0.0332 libphp5.so add_string_to_string 3626 0.1061 835 0.0317 vmlinux release_pages 3579 0.1048 1549 0.0588 libphp5.so ZEND_RETURN_SPEC_VAR_HANDLER 3564 0.1043 1773 0.0672 libphp5.so zend_make_printable_zval 3554 0.1040 2605 0.0988 libphp5.so ZEND_RETURN_SPEC_CV_HANDLER 3531 0.1033 128 0.0049 libphp5.so php_trim 3489 0.1021 4274 0.1621 vmlinux memcpy 3465 0.1014 1364 0.0517 libphp5.so concat_function 3441 0.1007 2443 0.0927 libz.so.1.2.1.2 deflate_fast 3409 0.0998 355 0.0135 libphp5.so convert_to_boolean 3389 0.0992 100 0.0038 libphp5.so php_pcre_match 3363 0.0984 712 0.0270 vmlinux system_call 3292 0.0964 2353 0.0892 libphp5.so zend_hash_get_current_key_ex 3281 0.0960 3875 0.1470 vmlinux can_share_swap_page 3278 0.0959 5899 0.2237 vmlinux copy_page 3233 0.0946 5200 0.1972 fss.so kwsprep 3211 0.0940 979 0.0371 libphp5.so ZEND_JMP_SPEC_HANDLER 3152 0.0923 2450 0.0929 libphp5.so zend_hash_exists 3107 0.0909 8510 0.3228 apc.so prevent_garbage_collection 3060 0.0896 104 0.0039 libz.so.1.2.1.2 __do_global_ctors_aux 3036 0.0889 16 6.1e-04 libphp5.so zif_strtr 3021 0.0884 313 0.0119 libphp5.so zend_std_write_property 2946 0.0862 106 0.0040 vmlinux strncpy_from_user 2922 0.0855 1834 0.0696 libphp5.so ZEND_SEND_VAL_SPEC_TMP_HANDLER 2910 0.0852 227 0.0086 libphp5.so zend_object_store_get_object 2792 0.0817 1073 0.0407 vmlinux flush_tlb_page 2772 0.0811 1510 0.0573 vmlinux kmem_cache_alloc 2730 0.0799 1861 0.0706 vmlinux find_vma 2695 0.0789 1011 0.0383 libphp5.so ZEND_BOOL_NOT_SPEC_VAR_HANDLER 2674 0.0783 2260 0.0857 libphp5.so zend_llist_del_element 2646 0.0774 1063 0.0403 vmlinux _atomic_dec_and_lock 2629 0.0769 726 0.0275 opreport (no symbols) 2583 0.0756 1284 0.0487 fss.so hasevery 2534 0.0742 1216 0.0461 vmlinux kmem_cache_free 2528 0.0740 1805 0.0685 libphp5.so zend_hash_get_current_data_ex 2442 0.0715 2321 0.0880 libphp5.so virtual_file_ex 2433 0.0712 1451 0.0550 libphp5.so ZEND_SEND_VAR_NO_REF_SPEC_VAR_HANDLER 2416 0.0707 230 0.0087 libphp5.so _safe_emalloc 2397 0.0702 743 0.0282 libphp5.so ZEND_RETURN_SPEC_TMP_HANDLER 2377 0.0696 313 0.0119 vmlinux do_page_fault 2335 0.0683 1 3.8e-05 libtidy-0.99.so.0.0.0 EncodeCharToUTF8Bytes 2319 0.0679 921 0.0349 libphp5.so add_function 2307 0.0675 366 0.0139 vmlinux do_lookup 2300 0.0673 1288 0.0489 libphp5.so ZEND_BOOL_SPEC_TMP_HANDLER 2293 0.0671 556 0.0211 vmlinux path_lookup 2286 0.0669 2802 0.1063 libphp5.so zval_update_constant 2283 0.0668 1257 0.0477 vmlinux schedule 2281 0.0668 132 0.0050 libphp5.so php_pcre_replace 2276 0.0666 346 0.0131 vmlinux permission 2273 0.0665 156 0.0059 apc.so apc_php_malloc 2261 0.0662 1068 0.0405 libphp5.so boolean_not_function 2247 0.0658 224 0.0085 libphp5.so php_pcre_fullinfo 2233 0.0654 461 0.0175 libphp5.so sub_function 2216 0.0649 831 0.0315 libphp5.so zend_hash_index_find 2203 0.0645 1028 0.0390 fss.so treefails 2188 0.0640 827 0.0314 vmlinux __posix_lock_file 2179 0.0638 694 0.0263 libphp5.so format_converter 2178 0.0637 34 0.0013 vmlinux pte_alloc_map 2175 0.0637 950 0.0360 libphp5.so ZEND_INIT_STATIC_METHOD_CALL_SPEC_CONST_HANDLER 2164 0.0633 161 0.0061 libphp5.so zif_array_map 2163 0.0633 688 0.0261 vmlinux __mod_page_state 2111 0.0618 215 0.0082 vmlinux handle_mm_fault 2102 0.0615 2090 0.0793 vmlinux __wake_up_bit 2102 0.0615 316 0.0120 vmlinux do_no_page 2095 0.0613 965 0.0366 libphp5.so zend_isset_isempty_dim_prop_obj_handler_SPEC_VAR_CV 2091 0.0612 1110 0.0421 libphp5.so ZEND_FETCH_DIM_R_SPEC_CV_CONST_HANDLER 2091 0.0612 1513 0.0574 oprofiled (no symbols) 2083 0.0610 214 0.0081 libphp5.so zend_is_callable_ex 2074 0.0607 1284 0.0487 libphp5.so ZEND_ASSIGN_OBJ_SPEC_UNUSED_CONST_HANDLER 2065 0.0604 688 0.0261 libphp5.so _array_init 2046 0.0599 44 0.0017 libphp5.so zval_add_ref 1995 0.0584 2285 0.0867 ld-2.3.6.so do_lookup_x 1990 0.0582 870 0.0330 libphp5.so ZEND_FETCH_DIM_R_SPEC_CV_CV_HANDLER 1973 0.0577 962 0.0365 libphp5.so ZEND_ASSIGN_SPEC_CV_TMP_HANDLER 1966 0.0575 1447 0.0549 libphp5.so zend_fetch_class 1944 0.0569 1298 0.0492 libphp5.so zend_isset_isempty_dim_prop_obj_handler_SPEC_CV_CV 1939 0.0568 695 0.0264 libphp5.so is_identical_function 1886 0.0552 39 0.0015 libphp5.so preg_replace_impl 1858 0.0544 941 0.0357 libphp5.so zend_get_constant 1847 0.0541 249 0.0094 libphp5.so _zend_hash_merge 1838 0.0538 781 0.0296 vmlinux acpi_processor_idle 1829 0.0535 566 0.0215 libphp5.so ZEND_ADD_ARRAY_ELEMENT_SPEC_VAR_UNUSED_HANDLER 1828 0.0535 434 0.0165 libphp5.so zend_objects_store_del_ref 1812 0.0530 1392 0.0528 libphp5.so ZEND_ADD_VAR_SPEC_TMP_CV_HANDLER 1803 0.0528 34 0.0013 libphp5.so php_char_to_str_ex 1795 0.0525 751 0.0285 libphp5.so ZEND_FETCH_DIM_R_SPEC_VAR_CONST_HANDLER 1787 0.0523 175 0.0066 libphp5.so zend_parse_parameters 1773 0.0519 510 0.0193 vmlinux in_group_p 1742 0.0510 1766 0.0670 libc-2.3.6.so strcasecmp