dtrace_scripts

1/28/2016 enyew/dtrace_scripts – Zebi
Last modified 5 months ago
Dtrace references
Intro to Dtrace. https://wikis.oracle.com/display/DTrace/Introduction
Overview of different type of Dtrace script variables.
http://dtrace.org/blogs/brendan/2011/11/25/dtrace­variable­types/
Dtrace cheatsheets.
http://www.solarisinternals.com/wiki/index.php/DTrace_Topics_Cheatsheets_Checklists
Dtrace tunables (e.g. pragma to increase memory allowed for dtrace).
https://wikis.oracle.com/display/DTrace/Options+and+Tunables
Checking return value of a function
dtrace ­n 'fbt:zfs:arc_reclaim_needed:return{ printf("%i\n",arg1); }'
dtrace ­n 'fbt:zfs:arc_evict_needed:return{ printf("%i\n",arg1); }'
Tracing specific arcstats field
dtrace ­n 'BEGIN{ trace(`arc_stats.arcstat_l2_size.value.ui64); }'
dtrace ­n 'BEGIN{ printf("l2_write_bytes=%llu\n", (unsigned long long)`arc_stats.arc
dtrace ­qn 'fbt:zfs:arcstat_l2_size_incr:return
{
printf("\nl2_size=%llu\n", (unsigned long long)`arc_stats.arcstat_l2_size.value.u
printf("l2_write_bytes=%llu\n", (unsigned long long)`arc_stats.arcstat_l2_write_b
stack();
printf("\n");
}
'
Counting and checking where a function is called from
dtrace ­qn '
fbt:zfs:arc_buf_destroy:entry{ @count_stack[stack(), arg1, arg2] = count(); j++; }
profile:::tick­1sec{ i++; }
profile:::tick­1sec/i==10/{ printf("sample collection total time %i sec, total_cnt=%
Checking a function's latency
dtrace ­n '
profile:::tick­1sec{ i++; }
profile:::tick­1sec/i==10/{ exit(0); }
arc_buf_add_ref:entry { self­>s = timestamp; }
arc_buf_add_ref:return /self­>s/ { @["ns"] = quantize(timestamp ­ self­>s); self­>s
Count next level function calls and time taken within function
Note that when calculating time spent in a function, we are excluding the time spent in nextlevel
function calls.
dtrace ­q ­n '
BEGIN
https://nimbus.eng.tegile.com/trac/wiki/enyew/dtrace_scripts 1/17

1/28/2016 enyew/dtrace_scripts – Zebi
{
}
i = 0;
profile:::tick­1sec{ i++; printf("trace time=%i sec\n", i); }
fbt:zfs:spa_unload:entry
{
self­>entered = 1;
}
fbt:zfs::entry
/ self­>entered /
{
self­>ts[stackdepth]=timestamp;
self­>time_in_depth[stackdepth + 1] = 0;
}
fbt:zfs::return
/ self­>entered && self­>ts[stackdepth] /
{
this­>time_delta = timestamp ­ self­>ts[stackdepth];
self­>time_in_depth[stackdepth] += this­>time_delta;
this­>base_time = this­>time_delta ­ self­>time_in_depth[stackdepth + 1];
@call_count[execname, probefunc] = count();
@call_time[execname, probefunc] = sum( this­>base_time );
}
self­>time_in_depth[stackdepth + 1] = 0;
self­>ts[stackdepth] = 0;
fbt:zfs:spa_unload:return
{
self­>entered = 0;
self­>time_in_depth[stackdepth] = 0;
printf("\n");
printf("Function calls as a result of calling %s().\n", probefunc );
printf("Count of function calls.\n");
printa(@call_count);
printf("\n");
printf("Sum of all time spent within the same function, but excluding\n" );
printf("time taken within next level of function calls. (In nanosec).\n" );
printa(@call_time);
}
exit(0);
END
{
}'
Combination of code flow tracking and counting next­level function calls
If max_stack_depth is set too high, dtrace will try to trace many "enter" and "return" probes
and starts dropping.
In the below, it is set to trace "spa_unload".
In this script, we generate our own indent instead of using dtrace ­F option to show indentation
of different stack depth, since the dtrace auto indent seems to get messed up if "return" probe
https://nimbus.eng.tegile.com/trac/wiki/enyew/dtrace_scripts 2/17

1/28/2016 enyew/dtrace_scripts – Zebi
is dropped.
The base_time field is time spent in a function excluding the time spent in next­level function
calls.
We use pragma D here to increase the dtrace dynamic variable usage size to 256MB, else dtrace
will start dropping dynamic variables when there is not enough memory in the pool.
#!/usr/sbin/dtrace ­s
#pragma D option dynvarsize=256m
#pragma D option quiet
BEGIN
{
i = 0;
max_stack_depth = 4;
}
profile:::tick­1sec{ i++; printf("trace time=%i sec\n", i); }
fbt:zfs:spa_unload:entry
{
printf( "\n execname=%s probefunc=%s (cpu=%i pid=%i tid=%i depth=%i)\n",
execname, probefunc,
cpu, pid, tid, stackdepth );
self­>base_depth = stackdepth;
self­>my_indent[stackdepth] = " ";
self­>entered = 1;
}
fbt:zfs::entry
/ self­>entered && (stackdepth ­ self­>base_depth < max_stack_depth) && self­>my_ind
{
self­>my_indent[stackdepth + 1] = strjoin( self­>my_indent[stackdepth], " " );
}
fbt:zfs::entry
/ self­>entered /
{
self­>ts[stackdepth] = timestamp;
self­>time_in_depth[stackdepth + 1] = 0;
}
fbt:zfs::entry
/ self­>entered && (stackdepth ­ self­>base_depth < max_stack_depth) /
{
printf("cpu=%i pid=%i tid=%i depth=%i, %s ­> %­30s\n",
cpu, pid, tid, stackdepth, self­>my_indent[stackdepth], probefunc );
}
fbt:zfs::return
/ self­>entered && self­>ts[stackdepth] /
{
self­>time_delta = timestamp ­ self­>ts[stackdepth];
self­>time_in_depth[stackdepth] += self­>time_delta;
self­>base_time = self­>time_delta ­ self­>time_in_depth[stackdepth + 1];
}
@call_count[execname, probefunc] = count();
@call_time[execname, probefunc] = sum( self­>base_time );
fbt:zfs::return
/ self­>entered && (stackdepth ­ self­>base_depth < max_stack_depth) && self­>ts[sta
{
https://nimbus.eng.tegile.com/trac/wiki/enyew/dtrace_scripts 3/17

1/28/2016 enyew/dtrace_scripts – Zebi
}
printf("cpu=%i pid=%i tid=%i depth=%i, %s my_indent[stackdepth],
probefunc, self­>time_delta, self­>base_time );
fbt:zfs::return
/ self­>entered && self­>ts[stackdepth] /
{
self­>my_indent[stackdepth + 1] = 0;
self­>time_in_depth[stackdepth + 1] = 0;
self­>ts[stackdepth] = 0;
self­>time_delta = 0;
self­>base_time = 0;
}
fbt:zfs:spa_unload:return
{
self­>entered = 0;
self­>base_depth = 0;
self­>my_indent[stackdepth] = 0;
}
END{
printf("\n");
printf("Count of function calls.\n");
printa(@call_count);
printf("\n");
printf("Sum of all time spent within the same function, but excluding\n" );
printf("time taken within next level of function calls. (In nanosec).\n" );
printa(@call_time);
}
printf("\n\n");
Below are example output, when we replace the "spa_unload" in the above script to
"arc_buf_destroy". (There are two places where the function name have to be replaced).
...
...
...
execname=svc.configd probefunc=arc_buf_destroy (cpu=0 pid=11 tid=27 depth=11)
cpu=0 pid=11 tid=27 depth=11, ­> arc_buf_destroy
cpu=0 pid=11 tid=27 depth=12, ­> arc_cksum_verify
cpu=0 pid=11 tid=27 depth=12, arc_buf_data_free
cpu=0 pid=11 tid=27 depth=13,
­> zio_data_buf_free
cpu=0 pid=11 tid=27 depth=13, zio_data_buf_free
cpu=0 pid=11 tid=27 depth=15,

1/28/2016 enyew/dtrace_scripts – Zebi
trace time=2 sec
trace time=3 sec
^C
trace time=4 sec
Function calls as a result of calling ().
Count of function calls.
svc.configd
svc.configd
svc.configd
svc.configd
arc_buf_data_free
arc_buf_destroy
arc_cksum_verify
zio_data_buf_free
Sum of all time spent within the same function, but excluding
time taken within next level of function calls. (In nanosec).
svc.configd
svc.configd
svc.configd
svc.configd
zio_data_buf_free
arc_cksum_verify
arc_buf_data_free
arc_buf_destroy
Check return value of arc_evict_needed()
fbt:zfs:arc_evict_needed:entry
{
self­>data_type = arg0;
}
fbt:zfs:arc_evict_needed:return
{
printf("data_type=%i evict_needed() returns %i\n",self­>data_type, arg1);
self­>data_type = 0;
}
Dtrace async unlinked drain handling
#!/usr/sbin/dtrace ­s
::zfs_unlinked_drain:entry{
self­>myvfs=arg0;
}
::zfs_unlinked_drain:zfs­unlinked­drain­break{
printf("zfsvfs=%p\n", arg0);
}
::zfs_unlinked_drain:zfs­unlinked­drain­qfull{
printf("zfsvfs=%p task_cnt=%d\n", arg0, arg1);
}
::zfs_unlinked_drain:return/arg1/{
printf("zfsvfs=%p work_done_cnt=%d\n", self­>myvfs, arg1);
self­>myvfs=0;
}
::zfs_znode_unlinked_process:zfs­unlinked­process­reason
{
self­>my_znode = (znode_t *)arg2;
self­>my_vnode = (vnode_t *)self­>my_znode­>z_vnode;
printf("zfsvfs=%p reason=%i zp=%p vp=%p obj_num=%d rele=%i rmnode=%i v_count
arg0, arg1, arg2, self­>my_znode­>z_vnode, arg3, arg4, arg5,
https://nimbus.eng.tegile.com/trac/wiki/enyew/dtrace_scripts 5/17

1/28/2016 enyew/dtrace_scripts – Zebi
}
self­>my_vnode­>v_count_dnlc,
self­>my_vnode­>v_count,
self­>my_vnode­>v_path? stringof(self­>my_vnode­>v_path) : "na");
self­>my_znode = 0;
self­>my_vnode = 0;
::zfs_znode_unlinked_process:zfs­unlinked­process­chk­rele
/self­>myvfs/
{
self­>my_znode = (znode_t *)arg2;
printf("zfsvfs=%p zp=%p vp=%p zp_v_count=%d zp_rmnode=%i\n",
self­>myvfs,
arg0, arg1, arg2, arg3);
self­>my_znode = 0;
}
::zfs_rmnode:entry
{
self­>myrmnodezp = (znode_t *)arg0;
self­>myrmnodezptime = timestamp;
self­>myvnode = (vnode_t *)self­>myrmnodezp­>z_vnode;
printf("zfsvfs=%p zp=%p obj_num=%d z_pflags=0x%x vp=%p v_type=%d v_count=%d\
self­>myvfs, arg0,
self­>myrmnodezp­>z_id,
self­>myrmnodezp­>z_pflags,
self­>myvnode,
self­>myvnode­>v_type,
self­>myvnode­>v_count);
}
::zfs_rmnode:return
/self­>myrmnodezptime && ((unsigned long)self­>myvnode­>v_path != 0)/
{
printf("zfsvfs=%p zp=%p obj_num=%d delta=%d nsec vpath=%s\n",
self­>myvfs, self­>myrmnodezp, self­>myrmnodezp­>z_id,
timestamp ­ self­>myrmnodezptime, stringof(self­>myvnode­>v_path));
self­>myrmnodezp = 0;
self­>myrmnodezptime = 0;
self­>myvnode = 0;
}
::zfs_rmnode:return
/self­>myrmnodezptime && ((unsigned long)self­>myvnode­>v_path == 0)/
{
printf("zfsvfs=%p zp=%p obj_num=%d delta=%d nsec vpath=NULL\n",
self­>myvfs, self­>myrmnodezp, self­>myrmnodezp­>z_id,
timestamp ­ self­>myrmnodezptime);
self­>myrmnodezp = 0;
self­>myrmnodezptime = 0;
self­>myvnode = 0;
}
::zfs_zinactive:zfs­zinactive­rmnode{
printf("zfsvfs=%p obj_num=%d\n", arg0, arg1);
}
:::dmu­free­long­range­break
{
printf("dnode=%p\n", arg0);
}
::zfs_unlinked_add:entry{
https://nimbus.eng.tegile.com/trac/wiki/enyew/dtrace_scripts 6/17

1/28/2016 enyew/dtrace_scripts – Zebi
}
self­>myunlinkedaddzp = (znode_t *)arg0;
self­>myunlinkedaddvp = (vnode_t *)self­>myunlinkedaddzp­>z_vnode;
printf("enter zp=%p, obj_num=%d vp=%p v_count=%d\n",
self­>myunlinkedaddzp, self­>myunlinkedaddzp­>z_id,
self­>myunlinkedaddvp, self­>myunlinkedaddvp­>v_count);
::zfs_unlinked_add:return/(unsigned long)(self­>myunlinkedaddvp­>v_path) != 0/
{
printf("enter zp=%p, obj_num=%d vp=%p v_count=%d path=%s\n",
self­>myunlinkedaddzp, self­>myunlinkedaddzp­>z_id,
self­>myunlinkedaddvp,
self­>myunlinkedaddvp­>v_count,
stringof(self­>myunlinkedaddvp­>v_path));
stack();
}
::zfs_unlinked_add:return/(unsigned long)(self­>myunlinkedaddvp­>v_path) == 0/
{
printf("enter zp=%p, obj_num=%d vp=%p v_count=%d path=NULL\n",
self­>myunlinkedaddzp, self­>myunlinkedaddzp­>z_id,
self­>myunlinkedaddvp,
self­>myunlinkedaddvp­>v_count);
stack();
}
::taskq_remove_task:entry{
printf("zfsvfs=%p\n", arg2);
}
::taskq_remove_task:return{
printf("remove_cnt=%d\n", arg1);
}
::zfs_unlinked_task_remove_filter:entry{
self­>myremovezp = (znode_t *)arg1;
printf("zfsvfs=%p obj_num=%d\n", arg0, self­>myremovezp­>z_id);
self­>myremovezp = 0;
}
:::zfs­unlinked­task­remove­chk
{
printf("zfsvfs=%p zp=%p obj_num=%d\n", arg0, arg1, arg2);
}
::zfs_unlinked_task_remove_filter:return{
printf("remove_cnt=%d\n", arg1);
}
spa_sync() side free related work
dtrace ­n '
::spa_sync:entry{
self­>spa = (spa_t *)arg0;
}
::dnode_sync_free_range:entry/self­>spa/{
printf("spa=%s dnode=%p dn_type=%i beginning blkid=%d num_blocks=%d (0x%x)\n",
stringof(self­>spa­>spa_name),
arg0,
https://nimbus.eng.tegile.com/trac/wiki/enyew/dtrace_scripts 7/17

1/28/2016 enyew/dtrace_scripts – Zebi
}
((dnode_t *)arg0)­>dn_type,
arg1, arg2, arg2);
::free_blocks:return{
printf("spa=%s freed_blocks=%d\n",
stringof(self­>spa­>spa_name), arg1);
}
::spa_sync:return{
self­>spa = 0;
}'
Check zfsvfs teardown sequence
dtrace ­n '
::zfs_umount:entry{
self­>umounttime = timestamp;
printf("enter\n");
}
::zfs_umount:return{
printf("return, delta=%d\n", timestamp ­ self­>umounttime);
self­>umounttime = 0;
}
::zfsvfs_teardown:entry
{
self­>myvfs=arg0;
printf("zfsvfs=%p\n", arg0);
}
::zfsvfs_teardown:return
{
printf("zfsvfs=%p\n", self­>myvfs);
self­>myvfs=0;
}
::zfs_unlinked_drain_vfs_remove:entry
/self­>myvfs/
{
printf("zfsvfs=%p\n", self­>myvfs);
}
::zfs_unlinked_drain_vfs_remove:return
/self­>myvfs/
{
printf("zfsvfs=%p\n", self­>myvfs);
}
::zfs_unlinked_drain_task_wait:entry
/self­>myvfs/
{
printf("zfsvfs=%p\n", self­>myvfs);
}
::zfs_unlinked_drain_task_wait:return
/self­>myvfs/
{
https://nimbus.eng.tegile.com/trac/wiki/enyew/dtrace_scripts 8/17

1/28/2016 enyew/dtrace_scripts – Zebi
}
'
printf("zfsvfs=%p\n", self­>myvfs);
Check async delete workers thread dmu_tx_hold_free() throttling
dtrace ­n '
BEGIN
{
MB=1024 * 1024;
KB=1024;
MIL=1000000
}
::dmu_free_long_range_impl:entry
{
self­>spa_name=stringof(((objset_t *)arg0)­>os_spa­>spa_name);
self­>start0 = timestamp;
printf("spa=%s offset=%d length=%d\n",
self­>spa_name, arg2, arg3);
}
::dmu_tx_hold_free:entry/self­>start0/
{
printf("spa=%s start=%d len=%d\n", self­>spa_name, arg2, arg3);
}
::dmu_tx_count_free:dmu­tx­count­free/self­>start0/
{
printf("spa=%s txg=%d dn_nlevels=%d dn_maxblkdid=%d off=%d len=%d\n",
self­>spa_name,
arg2, arg3, arg4, arg5, arg6);
}
::dmu_tx_hold_free:dmu­tx­hold­free/self­>start0/
{
printf("spa=%s txh_type=%i towrite=%d tofree=%d tooverwrite=%d tounref=%d memtoh
self­>spa_name, arg1, arg2, arg3, arg4, arg5, arg6, arg7);
}
::dmu_tx_assign:entry/self­>start0/
{
self­>start1 = timestamp;
printf("spa=%s start\n", self­>spa_name);
}
::dmu_tx_try_assign:entry/self­>start0/
{
self­>start2 = timestamp;
printf("spa=%s start\n", self­>spa_name);
}
::dmu_tx_try_assign:return/self­>start0/
{
printf("spa=%s end return=%i time_taken=%d ns\n",
self­>spa_name, arg1,
(timestamp ­ self­>start2));
}
::dmu_tx_wait:entry/self­>start0/
{
https://nimbus.eng.tegile.com/trac/wiki/enyew/dtrace_scripts 9/17

1/28/2016 enyew/dtrace_scripts – Zebi
}
self­>start3 = timestamp;
printf("spa=%s start\n", self­>spa_name);
::dmu_tx_wait:return/self­>start0/
{
printf("spa=%s end return=%i time_taken=%d ns\n",
self­>spa_name, arg1,
(timestamp ­ self­>start3));
}
::dmu_tx_assign:return/self­>start0/
{
printf("spa=%s end return=%i time_taken=%d ns\n",
self­>spa_name, arg1,
(timestamp ­ self­>start1));
}
::dnode_clear_range:entry/self­>start0/
{
self­>spa_name = stringof(((dmu_tx_t *)arg3)­>tx_pool­>dp_spa­>spa_name);
printf("spa=%s txg=%d dn=%p blkid=%d nblks=%d\n",
self­>spa_name,
((dmu_tx_t *)arg3)­>tx_txg,
arg0, arg1, arg2);
}
::dmu_free_long_range_impl:return/self­>start0/
{
printf("spa=%s return=%i time_taken=%d ms\n",
self­>spa_name,
arg1, (timestamp ­ self­>start0)/MIL);
self­>spa_name = 0;
self­>start0 = 0;
}
'
Trace dsl_scan_sync() bp traverse pause checking code
dtrace ­n '
::dsl_scan_sync:entry{
self­>time0 = timestamp;
}
::dsl_scan_check_pause:entry/self­>time0/{
self­>curr = timestamp;
self­>scn = (dsl_scan_t *)arg0;
printf("scan_start=%9d elapsed=%9d delta=%9d ns\n",
self­>scn­>scn_sync_start_time,
self­>curr ­ self­>scn­>scn_sync_start_time,
self­>curr ­ self­>time0);
self­>time0 = self­>curr;
}
::dsl_scan_check_waiting:entry{
self­>elapsed = arg1;
}
::dsl_scan_check_waiting:return{
https://nimbus.eng.tegile.com/trac/wiki/enyew/dtrace_scripts 10/17

1/28/2016 enyew/dtrace_scripts – Zebi
}
printf("elapsed=%9d ns, return=%i\n",
self­>elapsed, arg1);
::dsl_scan_check_pause:return/self­>time0/{
printf("return=%d\n", arg1);
}
'
Trace txg open, quiesce, sync group dirty data
dtrace ­qn '
::txg_quiesce:txg­quiescing{
MB=1024 *1024;
printf("spa=%s quiese txg=%9d dirty=%4d\n",
stringof(((dsl_pool_t *)arg0)­>dp_spa­>spa_name),
arg1,
arg2/MB);
}
::txg_sync_thread:txg­syncing{
MB=1024 *1024;
printf("spa=%s syncing txg=%9d dirty=%4d\n",
stringof(((dsl_pool_t *)arg0)­>dp_spa­>spa_name),
arg1,
arg2/MB);
}
::txg_sync_thread:txg­opened­dirty{
MB=1024 *1024;
printf("spa=%s opened txg=%9d dirty=%4d\n",
stringof(((spa_t *)arg0)­>spa_name),
arg1,
arg2/MB);
}'
Trace zfs_zget()
dtrace ­n '
::zfs_zget:entry{
self­>obj = arg1;
self­>zpp = (znode_t **)arg2;
}
::zfs_zget:return/self­>obj/{
self­>vnode = (vnode_t *)(*(self­>zpp))­>z_vnode;
}'
printf("obj=%d path=%s\n",
self­>obj,
stringof(self­>vnode­>v_path));
spa_sync(), iterate­to­convergence section of the code, breakdown of time taken
dtrace ­n '
BEGIN{
MIL=1000000;
}
https://nimbus.eng.tegile.com/trac/wiki/enyew/dtrace_scripts 11/17

1/28/2016 enyew/dtrace_scripts – Zebi
::spa_sync:entry{
self­>txg = arg1;
self­>spa = (spa_t *)arg0;
self­>spa_name = stringof(((spa_t *)arg0)­>spa_name);
}
::spa_sync_config_object:entry/self­>txg/{
self­>time0 = timestamp;
}
::spa_sync_config_object:return/self­>txg/{
printf("spa=%s txg=%d pass=%d delta=%d ms\n",
self­>spa_name, self­>txg, self­>spa­>spa_sync_pass,
(timestamp ­ self­>time0)/MIL);
}
::spa_sync_aux_dev:entry/self­>txg/{
self­>time1 = timestamp;
}
::spa_sync_aux_dev:return/self­>txg/{
printf("spa=%s txg=%d pass=%d delta=%d ms\n",
self­>spa_name, self­>txg, self­>spa­>spa_sync_pass,
(timestamp ­ self­>time1)/MIL);
}
::spa_errlog_sync:entry/self­>txg/{
self­>time2 = timestamp;
}
::spa_errlog_sync:return/self­>txg/{
printf("spa=%s txg=%d pass=%d delta=%d ms\n",
self­>spa_name, self­>txg, self­>spa­>spa_sync_pass,
(timestamp ­ self­>time2)/MIL);
}
::dsl_pool_sync:entry/self­>txg/{
self­>time3 = timestamp;
}
::dsl_pool_sync:return/self­>txg/{
printf("spa=%s txg=%d pass=%d delta=%d ms\n",
self­>spa_name, self­>txg, self­>spa­>spa_sync_pass,
(timestamp ­ self­>time3)/MIL);
}
::spa_sync_frees:entry/self­>txg/{
self­>time4 = timestamp;
}
::spa_sync_frees:return/self­>txg/{
printf("spa=%s txg=%d pass=%d delta=%d ms\n",
self­>spa_name, self­>txg, self­>spa­>spa_sync_pass,
(timestamp ­ self­>time4)/MIL);
}
::bplist_iterate:entry/self­>txg/{
self­>time5 = timestamp;
}
https://nimbus.eng.tegile.com/trac/wiki/enyew/dtrace_scripts 12/17

1/28/2016 enyew/dtrace_scripts – Zebi
::bplist_iterate:return/self­>txg/{
printf("spa=%s txg=%d pass=%d delta=%d ms\n",
self­>spa_name, self­>txg, self­>spa­>spa_sync_pass,
(timestamp ­ self­>time5)/MIL);
}
::ddt_sync:entry/self­>txg/{
self­>time6 = timestamp;
}
::ddt_sync:return/self­>txg/{
printf("spa=%s txg=%d pass=%d delta=%d ms\n",
self­>spa_name, self­>txg, self­>spa­>spa_sync_pass,
(timestamp ­ self­>time6)/MIL);
}
::dsl_scan_sync:entry/self­>txg/{
self­>time7 = timestamp;
}
::dsl_scan_sync:return/self­>txg/{
printf("spa=%s txg=%d pass=%d delta=%d ms\n",
self­>spa_name, self­>txg, self­>spa­>spa_sync_pass,
(timestamp ­ self­>time7)/MIL);
}
::vdev_sync:entry/self­>txg/{
self­>time8 = timestamp;
}
::vdev_sync:return/self­>txg/{
printf("spa=%s txg=%d pass=%d delta=%d ms\n",
self­>spa_name, self­>txg, self­>spa­>spa_sync_pass,
(timestamp ­ self­>time8)/MIL);
}
::spa_sync_upgrades:entry/self­>txg/{
self­>time9 = timestamp;
}
::spa_sync_upgrades:return/self­>txg/{
printf("spa=%s txg=%d pass=%d delta=%d ms\n",
self­>spa_name, self­>txg, self­>spa­>spa_sync_pass,
(timestamp ­ self­>time9)/MIL);
}
::dmu_objset_is_dirty:entry/self­>txg/{
self­>time10 = timestamp;
}
::dmu_objset_is_dirty:return/self­>txg/{
printf("spa=%s txg=%d pass=%d delta=%d ms\n",
self­>spa_name, self­>txg, self­>spa­>spa_sync_pass,
(timestamp ­ self­>time10)/MIL);
}
https://nimbus.eng.tegile.com/trac/wiki/enyew/dtrace_scripts 13/17

1/28/2016 enyew/dtrace_scripts – Zebi
::spa_sync:return{
self­>txg = 0;
self­>spa_name = 0;
}'
dsl_pool_sync()
dtrace ­n '
BEGIN{
MIL=1000000;
}
::spa_sync:entry{
self­>txg = arg1;
self­>spa = (spa_t *)arg0;
self­>spa_name = stringof(((spa_t *)arg0)­>spa_name);
}
::dsl_pool_sync:entry/self­>txg/{
self­>time3 = timestamp;
}
::dsl_dataset_sync:entry/self­>txg/{
self­>ds_time1 = timestamp;
self­>ds = (dsl_dataset_t *)arg0;
}
::dsl_dataset_sync:return/self­>txg/{
printf("spa=%s txg=%d pass=%d ds=%p delta=%d ms\n",
self­>spa_name, self­>txg, self­>spa­>spa_sync_pass,
self­>ds,
(timestamp ­ self­>ds_time1)/MIL);
}
::dsl_pool_sync:pool_sync­1setup/self­>txg/
{
self­>ds_sync_zio_delta = timestamp;
}
::dsl_pool_sync:pool_sync­2rootzio/self­>txg/
{
self­>ds_sync_zio_delta = timestamp ­ self­>ds_sync_zio_delta;
}
::dsl_pool_sync_mos:entry/self­>txg/
{
self­>mostime = timestamp;
}
::dsl_pool_sync_mos:return/self­>txg/
{
self­>mostime = timestamp ­ self­>mostime;
}
::dsl_pool_sync:return/self­>txg/{
printf("spa=%s txg=%d pass=%d ds_sync_zio_wait_time=%d ms mos_sync=%d ms delta=%d
self­>spa_name, self­>txg, self­>spa­>spa_sync_pass, self­>mostime,
https://nimbus.eng.tegile.com/trac/wiki/enyew/dtrace_scripts 14/17

1/28/2016 enyew/dtrace_scripts – Zebi
}
self­>ds_sync_zio_delta/MIL,
(timestamp ­ self­>time3)/MIL);
::spa_sync:return{
self­>txg = 0;
self­>spa_name = 0;
}
'
dsl_scan free and scrub traversal time limit
dtrace ­n '
::dsl_scan_sync:entry{
self­>time0 = timestamp;
self­>dp = (dsl_pool_t *)arg0;
self­>spa = (spa_t *)self­>dp­>dp_spa;
self­>spa_name = stringof( self­>spa­>spa_name );
}
::dsl_scan_check_pause:entry/self­>time0/{
self­>curr = timestamp;
self­>scn = (dsl_scan_t *)arg0;
}
::dsl_scan_check_pause:return/self­>time0 && arg1/{
printf("spa=%s scan_start=%9d elapsed=%9d ns return=%d\n",
self­>spa_name,
self­>scn­>scn_sync_start_time,
self­>curr ­ self­>scn­>scn_sync_start_time,
arg1);
}
::dsl_scan_check_waiting:entry/self­>time0/{
self­>scrub_elapsed=arg1;
}
::dsl_scan_check_waiting:return/arg1 == 2/
{
printf("elapsed=%d ns return=%d\n", self­>scrub_elapsed, arg1);
}
::dsl_scan_sync:return{
self­>time0=0;
}
'
dtrace ­n '
::dsl_scan_sync:entry{
self­>time0 = timestamp;
self­>dp = (dsl_pool_t *)arg0;
https://nimbus.eng.tegile.com/trac/wiki/enyew/dtrace_scripts 15/17

1/28/2016 enyew/dtrace_scripts – Zebi
}
self­>spa = (spa_t *)self­>dp­>dp_spa;
self­>spa_name = stringof( self­>spa­>spa_name );
::dsl_scan_free_should_pause:entry/self­>time0/{
self­>curr = timestamp;
self­>scn = (dsl_scan_t *)arg0;
}
::dsl_scan_free_should_pause:return/self­>time0 && arg1/{
printf("spa=%s scan_start=%9d elapsed=%9d ns return=%d\n",
self­>spa_name,
self­>scn­>scn_sync_start_time,
self­>curr ­ self­>scn­>scn_sync_start_time,
arg1);
}
::dsl_scan_check_waiting:entry/self­>time0/{
self­>scrub_elapsed=arg1;
}
::dsl_scan_check_waiting:return/arg1 == 2/
{
printf("elapsed=%d ns return=%d\n", self­>scrub_elapsed, arg1);
}
::dsl_scan_sync:return{
self­>time0=0;
}
'
Tracking NFS op time taken
#!/usr/sbin/dtrace ­qs
#pragma D option dynvarsize=256m
profile:::tick­1sec
{
mytime++;
printf("timestamp=%12d sec elapsed, %Y\n", mytime, walltimestamp);
}
nfssrv:common_dispatch:entry{
}
self­>opstart = timestamp;
self­>req = arg0;
self­>opdelta = 0;
self­>op = ­1;
:::rfs4­compound­argop/self­>req && (arg0 == 0)/{
self­>opstart = timestamp;
self­>idx = arg0;
self­>op = arg1;
https://nimbus.eng.tegile.com/trac/wiki/enyew/dtrace_scripts 16/17

1/28/2016 enyew/dtrace_scripts – Zebi
}
:::rfs4­compound­argop/self­>req && (arg0 > 0)/{
self­>prev_delta = (timestamp ­ self­>opstart)/1000000;
self­>prev_idx = self­>idx;
self­>prev_op = self­>op;
printf("END nfs req=%p i=%d op=%d delta=%d ms\n",
self­>req,
self­>prev_idx,
self­>prev_op,
self­>prev_delta
);
}
self­>opstart = timestamp;
self­>idx = arg0;
self­>op = arg1;
nfssrv:common_dispatch:return/self­>req/{
self­>prev_delta = (timestamp ­ self­>opstart)/1000000;
self­>prev_idx = self­>idx;
self­>prev_op = self­>op;
printf("END nfs req=%p i=%d op=%d delta=%d ms\n",
self­>req,
self­>prev_idx,
self­>prev_op,
self­>prev_delta
);
}
self­>prev_delta = 0;
self­>prev_idx = 0;
self­>prev_op = 0;
self­>op = 0;
self­>idx = 0;
self­>opstart = 0;
self­>req = 0;
https://nimbus.eng.tegile.com/trac/wiki/enyew/dtrace_scripts 17/17