We have run into an issue with number of tomcat active sessions that was causing us a memory bottleneck while doing performance testing.
So to track this we did a heap dump and a set of heap histograms to see what is the size of the Session object and current number of open sessions.
We managed to find the issue with the test scenario but, we found out that doing a heap histo would pause the JVM for a second or 2 which added to our performance issues.
So we need another command line way to report sessions.
Java does provide jconsole and JvisualVM with Mbean plugins but both tools are GUI tools not suitable for reporting from command line and automation.
We needed some way to automate this for doing a report across 9 nodes.
I have found some good command line tools but some of them are out of date and others are too complex for the job or requires an externally exposed JMX which is not allowed by our org. security.
So i needed to look for some java code to do this trick for me, i found out an oracle blog example code that would report JVM arguments, classpath, start time from JMX at the below link:
https://blogs.oracle.com/jmxetc/entry/how_to_retrieve_remote_jvm
The good thing about this code is it uses the Java tools to attache to the JVM locally so i didn't have to expose JVM externally which is forbidden by our security team.
One issue i faced it that i didn't know how to report the active sessions out of tomcat, so I asked my friend Bassem Zohdy who created a demo for me doing the needed functionality:
https://github.com/bassemZohdy/tomcatActiveSessionLocalJMXDemo
What i did is I updated the Oracle blog example code with what Bassem did and i got a cool running commandline tool that can report active sessions.
below is the java code:
//package jvmruntime;
import java.io.File;
import java.lang.management.ManagementFactory;
import java.net.InetAddress;
import java.util.logging.Logger;
import javax.management.remote.JMXServiceURL;
import com.sun.tools.attach.VirtualMachine;
import com.sun.tools.attach.VirtualMachineDescriptor;
import java.lang.management.RuntimeMXBean;
import java.util.ArrayList;
import java.util.List;
import javax.management.MBeanServerConnection;
import javax.management.remote.JMXConnector;
import javax.management.remote.JMXConnectorFactory;
import javax.management.ObjectName;
public class JVMRuntimeClient {
private static final Logger LOG =
Logger.getLogger(JVMRuntimeClient.class.getName());
public JVMRuntimeClient() {
}
public static class ConnectionArgs {
private static final String CONNECTOR_ADDRESS =
"com.sun.management.jmxremote.localConnectorAddress";
public final JMXServiceURL jmxURL;
final public String SYNTAX = "JVMRuntimeClient -url <jmx-url> " +
"| -port <port-number> [-host <host-or-ip] " +
"| -pid <pid> | -help";
public ConnectionArgs(String[] args) {
jmxURL = parseArgs(args);
}
public final JMXServiceURL getJMXServiceURL() {
return jmxURL;
}
private JMXServiceURL parseArgs(String[] args) {
String host = null;
int port = 0;
String pid = null;
JMXServiceURL serviceURL = null;
for (int i=0;i<args.length;i++) {
if (args[i].startsWith("-url")) {
// The '-url' option will let you specify a JMXServiceURL
// on the command line. This is an URL that begins with
// service:jmx:<protocol>
//
if (host != null)
throwSyntaxError(
"-url and -host are mutually exclusive");
if (pid != null)
throwSyntaxError(
"-pid and -url are mutually exclusive");
if (port > 0)
throwSyntaxError(
"-port and -url are mutually exclusive");
if (++i >= args.length)
throwSyntaxError(
"missing JMXServiceURL after -url");
try {
serviceURL = new JMXServiceURL(args[i]);
} catch (Exception x) {
throwSyntaxError("bad JMXServiceURL after -url: " + x);
}
continue;
} else if (args[i].startsWith("-host")) {
// The '-host' and '-port' options will let you specify a host
// and port, and from that will construct the JMXServiceURL of
// the default RMI connector, that is:
// service:jmx:rmi:///jndi/rmi://<host>:<port>/jmxrmi"
//
if (serviceURL != null)
throwSyntaxError("-url and -host are mutually exclusive");
if (pid != null)
throwSyntaxError("-pid and -host are mutually exclusive");
if (++i >= args.length)
throwSyntaxError("missing host after -host");
try {
InetAddress.getByName(args[i]);
host = args[i];
} catch (Exception x) {
throwSyntaxError("bad host after -url: " + x);
}
} else if (args[i].startsWith("-port")) {
// The '-host' and '-port' options will let you specify a host
// and port, and from that will construct the JMXServiceURL of
// the default RMI connector, that is:
// service:jmx:rmi:///jndi/rmi://<host>:<port>/jmxrmi"
//
if (serviceURL != null)
throwSyntaxError("-url and -port are mutually exclusive");
if (pid != null)
throwSyntaxError("-pid and -port are mutually exclusive");
if (++i >= args.length)
throwSyntaxError("missing port number after -port");
try {
port = Integer.parseInt(args[i]);
if (port <= 0)
throwSyntaxError("bad port number after -port: " +
"must be positive");
} catch (Exception x) {
throwSyntaxError("bad port number after -port: " + x);
}
} else if (args[i].startsWith("-pid")) {
// The '-pid' and option will let you specify the PID of the
// target VM you want to connect to. It will then use the
// attach API to dynamically launch the JMX agent in the target
// VM (if needed) and to find out the JMXServiceURL of the
// the default JMX Connector in that VM.
//
if (serviceURL != null)
throwSyntaxError("-url and -pid are mutually exclusive");
if (port > 0)
throwSyntaxError("-port and -pid are mutually exclusive");
if (++i >= args.length)
throwSyntaxError("missing pid after -pid");
try {
pid = args[i];
} catch (Exception x) {
throwSyntaxError("bad pid after -pid: " + x);
}
} else if (args[i].startsWith("-help")) {
final List<String> vmlist = new ArrayList();
for (VirtualMachineDescriptor vd : VirtualMachine.list()) {
vmlist.add(vd.id());
}
System.err.println(SYNTAX);
System.err.println("Running JVMs are: "+vmlist);
throw new IllegalArgumentException(SYNTAX);
} else {
throwSyntaxError("Unknown argument: "+args[i]);
}
}
// A JMXServiceURL was given on the command line, just use this.
//
if (serviceURL != null)
return serviceURL; // A -host -port info was given on the command line.
// Construct the default RMI JMXServiceURL from this.
//
if (port > 0) {
if (host == null)
host = "localhost";
try {
return new JMXServiceURL("service:jmx:rmi:///jndi/rmi://"+
host+":"+port+"/jmxrmi");
} catch (Exception x) {
throwSyntaxError("Bad host or port number: "+x);
}
}
// A PID was given on the command line.
// Use the attach API to find the target's connector address, and
// start it if needed.
//
if (pid != null) {
try {
return getURLForPid(pid);
} catch (Exception x) {
throwSyntaxError("cannot attach to target vm "+pid+": "+x);
}
}
final List<String> vmlist = new ArrayList();
for (VirtualMachineDescriptor vd : VirtualMachine.list()) {
vmlist.add(vd.id());
}
throwSyntaxError("missing argument: "+ "-port | -url | -pid | -list"
+"\\n\\tRunning VMs are: "+vmlist);
// Unreachable.
return null;
}
private void throwSyntaxError(String msg) {
System.err.println(msg);
System.err.println(SYNTAX);
throw new IllegalArgumentException(msg);
}
private JMXServiceURL getURLForPid(String pid) throws Exception {
// attach to the target application
final VirtualMachine vm = VirtualMachine.attach(pid);
// get the connector address
String connectorAddress =
vm.getAgentProperties().getProperty(CONNECTOR_ADDRESS);
// no connector address, so we start the JMX agent
if (connectorAddress == null) {
String agent = vm.getSystemProperties().getProperty("java.home") +
File.separator + "lib" + File.separator + "management-agent.jar";
vm.loadAgent(agent);
// agent is started, get the connector address
connectorAddress =
vm.getAgentProperties().getProperty(CONNECTOR_ADDRESS);
assert connectorAddress != null;
}
return new JMXServiceURL(connectorAddress);
}
}
// @param args the command line arguments:
// must be -url <jmx-url>,
// or -port <port-number> [-host <host-or-ip],
// or -pid <pid>,
// or -help
//
public static void main(String[] args) throws Exception {
// Parse arguments.
final ConnectionArgs cArgs = new ConnectionArgs(args);
// Get target's URL
final JMXServiceURL target = cArgs.getJMXServiceURL();
// Connect to target (assuming no security)
final JMXConnector connector = JMXConnectorFactory.connect(target);
// Get an MBeanServerConnection on the remote VM.
final MBeanServerConnection remote =
connector.getMBeanServerConnection();
final RuntimeMXBean remoteRuntime =
ManagementFactory.newPlatformMXBeanProxy(
remote,
ManagementFactory.RUNTIME_MXBEAN_NAME,
RuntimeMXBean.class);
//Bassem's code
final ObjectName objectName = new ObjectName(
"Catalina:type=Manager,host=localhost,context="+"/");
System.out.println("ActiveSession = "
+ remote.getAttribute(objectName, "activeSessions"));
//END of Bassem's code
//System.out.println("Target VM is: "+remoteRuntime.getName());
//System.out.println("Started since: "+remoteRuntime.getUptime());
//System.out.println("With Classpath: "+remoteRuntime.getClassPath());
//System.out.println("And args: "+remoteRuntime.getInputArguments());
connector.close();
}
}
To compile and run this code you need to do:
javac -classpath /usr/java/jdk1.8.0_25/lib/tools.jar: ./JVMRuntimeClient.java
java -classpath /usr/java/jdk1.8.0_25/lib/tools.jar: JVMRuntimeClient -pid 3689
So to track this we did a heap dump and a set of heap histograms to see what is the size of the Session object and current number of open sessions.
We managed to find the issue with the test scenario but, we found out that doing a heap histo would pause the JVM for a second or 2 which added to our performance issues.
So we need another command line way to report sessions.
Java does provide jconsole and JvisualVM with Mbean plugins but both tools are GUI tools not suitable for reporting from command line and automation.
We needed some way to automate this for doing a report across 9 nodes.
I have found some good command line tools but some of them are out of date and others are too complex for the job or requires an externally exposed JMX which is not allowed by our org. security.
So i needed to look for some java code to do this trick for me, i found out an oracle blog example code that would report JVM arguments, classpath, start time from JMX at the below link:
https://blogs.oracle.com/jmxetc/entry/how_to_retrieve_remote_jvm
The good thing about this code is it uses the Java tools to attache to the JVM locally so i didn't have to expose JVM externally which is forbidden by our security team.
One issue i faced it that i didn't know how to report the active sessions out of tomcat, so I asked my friend Bassem Zohdy who created a demo for me doing the needed functionality:
https://github.com/bassemZohdy/tomcatActiveSessionLocalJMXDemo
What i did is I updated the Oracle blog example code with what Bassem did and i got a cool running commandline tool that can report active sessions.
below is the java code:
//package jvmruntime;
import java.io.File;
import java.lang.management.ManagementFactory;
import java.net.InetAddress;
import java.util.logging.Logger;
import javax.management.remote.JMXServiceURL;
import com.sun.tools.attach.VirtualMachine;
import com.sun.tools.attach.VirtualMachineDescriptor;
import java.lang.management.RuntimeMXBean;
import java.util.ArrayList;
import java.util.List;
import javax.management.MBeanServerConnection;
import javax.management.remote.JMXConnector;
import javax.management.remote.JMXConnectorFactory;
import javax.management.ObjectName;
public class JVMRuntimeClient {
private static final Logger LOG =
Logger.getLogger(JVMRuntimeClient.class.getName());
public JVMRuntimeClient() {
}
public static class ConnectionArgs {
private static final String CONNECTOR_ADDRESS =
"com.sun.management.jmxremote.localConnectorAddress";
public final JMXServiceURL jmxURL;
final public String SYNTAX = "JVMRuntimeClient -url <jmx-url> " +
"| -port <port-number> [-host <host-or-ip] " +
"| -pid <pid> | -help";
public ConnectionArgs(String[] args) {
jmxURL = parseArgs(args);
}
public final JMXServiceURL getJMXServiceURL() {
return jmxURL;
}
private JMXServiceURL parseArgs(String[] args) {
String host = null;
int port = 0;
String pid = null;
JMXServiceURL serviceURL = null;
for (int i=0;i<args.length;i++) {
if (args[i].startsWith("-url")) {
// The '-url' option will let you specify a JMXServiceURL
// on the command line. This is an URL that begins with
// service:jmx:<protocol>
//
if (host != null)
throwSyntaxError(
"-url and -host are mutually exclusive");
if (pid != null)
throwSyntaxError(
"-pid and -url are mutually exclusive");
if (port > 0)
throwSyntaxError(
"-port and -url are mutually exclusive");
if (++i >= args.length)
throwSyntaxError(
"missing JMXServiceURL after -url");
try {
serviceURL = new JMXServiceURL(args[i]);
} catch (Exception x) {
throwSyntaxError("bad JMXServiceURL after -url: " + x);
}
continue;
} else if (args[i].startsWith("-host")) {
// The '-host' and '-port' options will let you specify a host
// and port, and from that will construct the JMXServiceURL of
// the default RMI connector, that is:
// service:jmx:rmi:///jndi/rmi://<host>:<port>/jmxrmi"
//
if (serviceURL != null)
throwSyntaxError("-url and -host are mutually exclusive");
if (pid != null)
throwSyntaxError("-pid and -host are mutually exclusive");
if (++i >= args.length)
throwSyntaxError("missing host after -host");
try {
InetAddress.getByName(args[i]);
host = args[i];
} catch (Exception x) {
throwSyntaxError("bad host after -url: " + x);
}
} else if (args[i].startsWith("-port")) {
// The '-host' and '-port' options will let you specify a host
// and port, and from that will construct the JMXServiceURL of
// the default RMI connector, that is:
// service:jmx:rmi:///jndi/rmi://<host>:<port>/jmxrmi"
//
if (serviceURL != null)
throwSyntaxError("-url and -port are mutually exclusive");
if (pid != null)
throwSyntaxError("-pid and -port are mutually exclusive");
if (++i >= args.length)
throwSyntaxError("missing port number after -port");
try {
port = Integer.parseInt(args[i]);
if (port <= 0)
throwSyntaxError("bad port number after -port: " +
"must be positive");
} catch (Exception x) {
throwSyntaxError("bad port number after -port: " + x);
}
} else if (args[i].startsWith("-pid")) {
// The '-pid' and option will let you specify the PID of the
// target VM you want to connect to. It will then use the
// attach API to dynamically launch the JMX agent in the target
// VM (if needed) and to find out the JMXServiceURL of the
// the default JMX Connector in that VM.
//
if (serviceURL != null)
throwSyntaxError("-url and -pid are mutually exclusive");
if (port > 0)
throwSyntaxError("-port and -pid are mutually exclusive");
if (++i >= args.length)
throwSyntaxError("missing pid after -pid");
try {
pid = args[i];
} catch (Exception x) {
throwSyntaxError("bad pid after -pid: " + x);
}
} else if (args[i].startsWith("-help")) {
final List<String> vmlist = new ArrayList();
for (VirtualMachineDescriptor vd : VirtualMachine.list()) {
vmlist.add(vd.id());
}
System.err.println(SYNTAX);
System.err.println("Running JVMs are: "+vmlist);
throw new IllegalArgumentException(SYNTAX);
} else {
throwSyntaxError("Unknown argument: "+args[i]);
}
}
// A JMXServiceURL was given on the command line, just use this.
//
if (serviceURL != null)
return serviceURL; // A -host -port info was given on the command line.
// Construct the default RMI JMXServiceURL from this.
//
if (port > 0) {
if (host == null)
host = "localhost";
try {
return new JMXServiceURL("service:jmx:rmi:///jndi/rmi://"+
host+":"+port+"/jmxrmi");
} catch (Exception x) {
throwSyntaxError("Bad host or port number: "+x);
}
}
// A PID was given on the command line.
// Use the attach API to find the target's connector address, and
// start it if needed.
//
if (pid != null) {
try {
return getURLForPid(pid);
} catch (Exception x) {
throwSyntaxError("cannot attach to target vm "+pid+": "+x);
}
}
final List<String> vmlist = new ArrayList();
for (VirtualMachineDescriptor vd : VirtualMachine.list()) {
vmlist.add(vd.id());
}
throwSyntaxError("missing argument: "+ "-port | -url | -pid | -list"
+"\\n\\tRunning VMs are: "+vmlist);
// Unreachable.
return null;
}
private void throwSyntaxError(String msg) {
System.err.println(msg);
System.err.println(SYNTAX);
throw new IllegalArgumentException(msg);
}
private JMXServiceURL getURLForPid(String pid) throws Exception {
// attach to the target application
final VirtualMachine vm = VirtualMachine.attach(pid);
// get the connector address
String connectorAddress =
vm.getAgentProperties().getProperty(CONNECTOR_ADDRESS);
// no connector address, so we start the JMX agent
if (connectorAddress == null) {
String agent = vm.getSystemProperties().getProperty("java.home") +
File.separator + "lib" + File.separator + "management-agent.jar";
vm.loadAgent(agent);
// agent is started, get the connector address
connectorAddress =
vm.getAgentProperties().getProperty(CONNECTOR_ADDRESS);
assert connectorAddress != null;
}
return new JMXServiceURL(connectorAddress);
}
}
// @param args the command line arguments:
// must be -url <jmx-url>,
// or -port <port-number> [-host <host-or-ip],
// or -pid <pid>,
// or -help
//
public static void main(String[] args) throws Exception {
// Parse arguments.
final ConnectionArgs cArgs = new ConnectionArgs(args);
// Get target's URL
final JMXServiceURL target = cArgs.getJMXServiceURL();
// Connect to target (assuming no security)
final JMXConnector connector = JMXConnectorFactory.connect(target);
// Get an MBeanServerConnection on the remote VM.
final MBeanServerConnection remote =
connector.getMBeanServerConnection();
final RuntimeMXBean remoteRuntime =
ManagementFactory.newPlatformMXBeanProxy(
remote,
ManagementFactory.RUNTIME_MXBEAN_NAME,
RuntimeMXBean.class);
//Bassem's code
final ObjectName objectName = new ObjectName(
"Catalina:type=Manager,host=localhost,context="+"/");
System.out.println("ActiveSession = "
+ remote.getAttribute(objectName, "activeSessions"));
//END of Bassem's code
//System.out.println("Target VM is: "+remoteRuntime.getName());
//System.out.println("Started since: "+remoteRuntime.getUptime());
//System.out.println("With Classpath: "+remoteRuntime.getClassPath());
//System.out.println("And args: "+remoteRuntime.getInputArguments());
connector.close();
}
}
To compile and run this code you need to do:
javac -classpath /usr/java/jdk1.8.0_25/lib/tools.jar: ./JVMRuntimeClient.java
java -classpath /usr/java/jdk1.8.0_25/lib/tools.jar: JVMRuntimeClient -pid 3689
No comments:
Post a Comment