Making DTrace work with node.js on FreeBSD.

a_description.md

Making DTrace work with node.js on FreeBSD.

1. Apply these patches to /usr/src
2. Rebuild and install kernel: sudo make buildkernel && sudo make installkernel
3. Reboot sudo shutdown -r
4. Raise sysctl limits:

• sudo sysctl -w kern.dtrace.helper_actions_max=16000
• sudo sysctl -w kern.dtrace.dof_maxsize=8000000

5. Clone node.js: git clone git://github.com/joyent/node && cd node && git checkout v0.10
6. Configure it: ./configure --prefix=... --with-dtrace
7. Build and install it: gmake -j24 && gmake install
8. Make DTrace device accessible to non-root users: sudo chmod 666 /dev/dtrace/helper
9. Verify that node.js DTrace probes are inserted: DTRACE_DOF_INIT_DEBUG=1 /path/to/node , it should print something like this in case of success:

dtrace DOF node: DTrace ioctl succeeded for DOF at 0x804c00000
dtrace DOF node: found provider 0x8052c3000
dtrace DOF node: fixing _ZN4node28DTRACE_NET_SERVER_CONNECTIONERKN2v89ArgumentsE() symbol
dtrace DOF node: fixing _ZN4node27DTRACE_HTTP_CLIENT_RESPONSEERKN2v89ArgumentsE() symbol
dtrace DOF node: fixing _ZN4node21DTRACE_NET_STREAM_ENDERKN2v89ArgumentsE() symbol
dtrace DOF node: fixing _ZN4node26DTRACE_HTTP_CLIENT_REQUESTERKN2v89ArgumentsE() symbol
dtrace DOF node: fixing _ZN4node26DTRACE_HTTP_SERVER_REQUESTERKN2v89ArgumentsE() symbol
dtrace DOF node: fixing _ZN4node22DTRACE_NET_SOCKET_READERKN2v89ArgumentsE() symbol
dtrace DOF node: fixing _ZN4node23DTRACE_NET_SOCKET_WRITEERKN2v89ArgumentsE() symbol
dtrace DOF node: fixing _ZN4node27DTRACE_HTTP_SERVER_RESPONSEERKN2v89ArgumentsE() symbol
dtrace DOF node: DTrace ioctl succeeded for DOF at 0x8052c2c2c
dtrace DOF node: DTrace ioctl succeeded for DOF at 0x804c00000
dtrace DOF node: found provider 0x8052c3000
dtrace DOF node: fixing _ZN4node28DTRACE_NET_SERVER_CONNECTIONERKN2v89ArgumentsE() symbol
dtrace DOF node: fixing _ZN4node27DTRACE_HTTP_CLIENT_RESPONSEERKN2v89ArgumentsE() symbol
dtrace DOF node: fixing _ZN4node21DTRACE_NET_STREAM_ENDERKN2v89ArgumentsE() symbol
dtrace DOF node: fixing _ZN4node26DTRACE_HTTP_CLIENT_REQUESTERKN2v89ArgumentsE() symbol
dtrace DOF node: fixing _ZN4node26DTRACE_HTTP_SERVER_REQUESTERKN2v89ArgumentsE() symbol
dtrace DOF node: fixing _ZN4node22DTRACE_NET_SOCKET_READERKN2v89ArgumentsE() symbol
dtrace DOF node: fixing _ZN4node23DTRACE_NET_SOCKET_WRITEERKN2v89ArgumentsE() symbol
dtrace DOF node: fixing _ZN4node27DTRACE_HTTP_SERVER_RESPONSEERKN2v89ArgumentsE() symbol
dtrace DOF node: DTrace ioctl succeeded for DOF at 0x8052c2c2c

add-sysctl-variables.patch

commit adf471edf8710d37832a6e54115b1d9b560505af
Author: Fedor Indutny <fedor.indutny@gmail.com>
Date:   Mon Feb 17 01:36:48 2014 +0400

    dtrace: allow raising DOF and actions limits

diff --git a/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c b/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c
index f35cf73..162c43f 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c
@@ -155,11 +155,11 @@
 int		dtrace_destructive_disallow = 0;
 dtrace_optval_t	dtrace_nonroot_maxsize = (16 * 1024 * 1024);
 size_t		dtrace_difo_maxsize = (256 * 1024);
-dtrace_optval_t	dtrace_dof_maxsize = (256 * 1024);
+int	dtrace_dof_maxsize = (256 * 1024);
 size_t		dtrace_global_maxsize = (16 * 1024);
 size_t		dtrace_actions_max = (16 * 1024);
 size_t		dtrace_retain_max = 1024;
-dtrace_optval_t	dtrace_helper_actions_max = 128;
+int	dtrace_helper_actions_max = 128;
 dtrace_optval_t	dtrace_helper_providers_max = 32;
 dtrace_optval_t	dtrace_dstate_defsize = (1 * 1024 * 1024);
 size_t		dtrace_strsize_default = 256;
@@ -11986,7 +11986,7 @@ dtrace_dof_copyin(uintptr_t uarg, int *errp)
 	 * Now we'll allocate the entire DOF and copy it in -- provided
 	 * that the length isn't outrageous.
 	 */
-	if (hdr.dofh_loadsz >= dtrace_dof_maxsize) {
+	if (hdr.dofh_loadsz >= (dtrace_optval_t) dtrace_dof_maxsize) {
 		dtrace_dof_error(&hdr, "load size exceeds maximum");
 		*errp = E2BIG;
 		return (NULL);
@@ -12077,7 +12077,7 @@ dtrace_dof_property(const char *name)
 		return (NULL);
 	}
 
-	if (loadsz >= dtrace_dof_maxsize) {
+	if (loadsz >= (dtrace_optval_t) dtrace_dof_maxsize) {
 		ddi_prop_free(buf);
 		dtrace_dof_error(NULL, "oversized DOF");
 		return (NULL);
@@ -12121,7 +12121,7 @@ dtrace_dof_property(const char *name)
 		return (NULL);
 	}
 
-	if (loadsz >= dtrace_dof_maxsize) {
+	if (loadsz >= (dtrace_optval_t) dtrace_dof_maxsize) {
 		kmem_free(buf, 0);
 		dtrace_dof_error(NULL, "oversized DOF");
 		return (NULL);
diff --git a/sys/cddl/dev/dtrace/dtrace_sysctl.c b/sys/cddl/dev/dtrace/dtrace_sysctl.c
index 1b5b3ab..b99a020 100644
--- a/sys/cddl/dev/dtrace/dtrace_sysctl.c
+++ b/sys/cddl/dev/dtrace/dtrace_sysctl.c
@@ -22,6 +22,9 @@
  *
  */
 
+SYSCTL_DECL(_kern_dtrace);
+SYSCTL_NODE(_kern, OID_AUTO, dtrace, CTLFLAG_RD, 0, "DTrace subsystem");
+
 int	dtrace_debug = 0;
 TUNABLE_INT("debug.dtrace.debug", &dtrace_debug);
 SYSCTL_INT(_debug_dtrace, OID_AUTO, debug, CTLFLAG_RW, &dtrace_debug, 0, "");
@@ -82,3 +85,8 @@ SYSCTL_PROC(_debug_dtrace, OID_AUTO, providers, CTLTYPE_STRING | CTLFLAG_RD,
 
 SYSCTL_INT(_debug_dtrace, OID_AUTO, memstr_max, CTLFLAG_RW, &dtrace_memstr_max,
     0, "largest allowed argument to memstr(), 0 indicates no limit");
+
+SYSCTL_INT(_kern_dtrace, OID_AUTO, dof_maxsize, CTLFLAG_RW | CTLFLAG_TUN,
+    &dtrace_dof_maxsize, 256 * 1024, "largest allowed DOF");
+SYSCTL_INT(_kern_dtrace, OID_AUTO, helper_actions_max, CTLFLAG_RW | CTLFLAG_TUN,
+    &dtrace_helper_actions_max, 128, "maximum number of allowed actions");

fix-drti-bugs.patch

commit c46786f47483e7fc218727aa52cf6b2278a45053
Author: Fedor Indutny <fedor.indutny@gmail.com>
Date:   Mon Feb 17 01:16:13 2014 +0400

    dtrace: proper symbol fixup and import in drti
    
    Application may contain multiple DOFs, merged into one ._SUNW_dof ELF
    section. Process all of them and fix symbols only in those ones that
    actaully define a provider. Use proper strtab for resolving symbols.

diff --git a/cddl/contrib/opensolaris/lib/libdtrace/common/drti.c b/cddl/contrib/opensolaris/lib/libdtrace/common/drti.c
index 3b4a38c..e47cfb4d 100644
--- a/cddl/contrib/opensolaris/lib/libdtrace/common/drti.c
+++ b/cddl/contrib/opensolaris/lib/libdtrace/common/drti.c
@@ -20,6 +20,7 @@
  */
 /*
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2013 Voxer Inc. All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -144,7 +145,8 @@ dtrace_dof_init(void)
 	Lmid_t lmid;
 #else
 	u_long lmid = 0;
-	dof_sec_t *sec;
+	dof_sec_t *sec, *secstart, *dofstrtab, *dofprobes;
+	dof_provider_t *dofprovider;
 	size_t i;
 #endif
 	int fd;
@@ -152,12 +154,13 @@ dtrace_dof_init(void)
 #if !defined(sun)
 	Elf *e;
 	Elf_Scn *scn = NULL;
-	Elf_Data *symtabdata = NULL, *dynsymdata = NULL;
+	Elf_Data *symtabdata = NULL, *dynsymdata = NULL, *dofdata = NULL;
+	dof_hdr_t *dof_next = NULL;
 	GElf_Shdr shdr;
 	int efd, nprobes;
 	char *s;
+	char *dofstrtabraw;
 	size_t shstridx, symtabidx = 0, dynsymidx = 0;
-	unsigned char *dofstrtab = NULL;
 	unsigned char *buf;
 	int fixedprobes = 0;
 #endif
@@ -209,7 +212,9 @@ dtrace_dof_init(void)
 		} else if (shdr.sh_type == SHT_PROGBITS) {
 			s = elf_strptr(e, shstridx, shdr.sh_name);
 			if  (s && strcmp(s, ".SUNW_dof") == 0) {
-				dof = elf_getdata(scn, NULL)->d_buf;
+				dofdata = elf_getdata(scn, NULL);
+				dof = dofdata->d_buf;
+				break;
 			}
 		}
 	}
@@ -219,6 +224,9 @@ dtrace_dof_init(void)
 		close(efd);
 		return;
 	}
+
+	while ((char *) dof < (char *) dofdata->d_buf + dofdata->d_size) {
+		dof_next = (void *) ((char *) dof + dof->dofh_filesz);
 #endif
 
 	if (dof->dofh_ident[DOF_ID_MAG0] != DOF_MAG_MAG0 ||
@@ -290,34 +298,49 @@ dtrace_dof_init(void)
 	 * We are assuming the number of probes is less than the number of
 	 * symbols (libc can have 4k symbols, for example).
 	 */
-	sec = (dof_sec_t *)(dof + 1);
+	secstart = sec = (dof_sec_t *)(dof + 1);
 	buf = (char *)dof;
 	for (i = 0; i < dof->dofh_secnum; i++, sec++) {
-		if (sec->dofs_type == DOF_SECT_STRTAB)
-			dofstrtab = (unsigned char *)(buf + sec->dofs_offset);
-		else if (sec->dofs_type == DOF_SECT_PROBES && dofstrtab)
+		if (sec->dofs_type != DOF_SECT_PROVIDER)
+			continue;
+
+		dofprovider = (void *) (buf + sec->dofs_offset);
+		dofstrtab = secstart + dofprovider->dofpv_strtab;
+		dofprobes = secstart + dofprovider->dofpv_probes;
+
+		if (dofstrtab->dofs_type != DOF_SECT_STRTAB) {
+			fprintf(stderr, "WARNING: expected STRTAB section, but got %d\n",
+					dofstrtab->dofs_type);
 			break;
-	
-	}
-	nprobes = sec->dofs_size / sec->dofs_entsize;
-	fixsymbol(e, symtabdata, symtabidx, nprobes, buf, sec, &fixedprobes,
-	    dofstrtab);
-	if (fixedprobes != nprobes) {
-		/*
-		 * If we haven't fixed all the probes using the
-		 * symtab section, look inside the dynsym
-		 * section.
-		 */
-		fixsymbol(e, dynsymdata, dynsymidx, nprobes, buf, sec,
-		    &fixedprobes, dofstrtab);
-	}
-	if (fixedprobes != nprobes) {
-		fprintf(stderr, "WARNING: number of probes "
-		    "fixed does not match the number of "
-		    "defined probes (%d != %d, "
-		    "respectively)\n", fixedprobes, nprobes);
-		fprintf(stderr, "WARNING: some probes might "
-		    "not fire or your program might crash\n");
+		}
+		if (dofprobes->dofs_type != DOF_SECT_PROBES) {
+			fprintf(stderr, "WARNING: expected PROBES section, but got %d\n",
+			    dofprobes->dofs_type);
+			break;
+		}
+
+		dprintf(1, "found provider %p\n", dofprovider);
+		dofstrtabraw = (char *)(buf + dofstrtab->dofs_offset);
+		nprobes = dofprobes->dofs_size / dofprobes->dofs_entsize;
+		fixsymbol(e, symtabdata, symtabidx, nprobes, buf, dofprobes, &fixedprobes,
+				dofstrtabraw);
+		if (fixedprobes != nprobes) {
+			/*
+			 * If we haven't fixed all the probes using the
+			 * symtab section, look inside the dynsym
+			 * section.
+			 */
+			fixsymbol(e, dynsymdata, dynsymidx, nprobes, buf, dofprobes,
+					&fixedprobes, dofstrtabraw);
+		}
+		if (fixedprobes != nprobes) {
+			fprintf(stderr, "WARNING: number of probes "
+			    "fixed does not match the number of "
+			    "defined probes (%d != %d, "
+			    "respectively)\n", fixedprobes, nprobes);
+			fprintf(stderr, "WARNING: some probes might "
+			    "not fire or your program might crash\n");
+		}
 	}
 #endif
 	if ((gen = ioctl(fd, DTRACEHIOC_ADDDOF, &dh)) == -1)
@@ -330,7 +353,12 @@ dtrace_dof_init(void)
 	}
 
 	(void) close(fd);
+
 #if !defined(sun)
+		/* End of while loop */
+		dof = dof_next;
+	}
+
 	elf_end(e);
 	(void) close(efd);
 #endif