/* gfs2_hangalyzer - a tool to figure out why gfs2 is hung - by Bob Peterson */
/* Copyright 2008, Red Hat, Inc.                                             */
/*
 * Example of use:
 * ./gfs2_hangalyzer -n trin-11
 *
 * This tool uses an external command like ssh to connect to any node in
 * your cluster to download and discover the other nodes.  Then it uses the
 * same ssh-like command to gather debug data files from all the nodes in
 * your cluster.  The files are then analyzed by the tool to determine the
 * most likely cause of the hang.
 *
 * Warning:
 *
 * I strongly recommend that you use secure rsa keys to identify the system
 * that will be running this tool to all your nodes.  If you don't, when the
 * tool tries to securely connect to each node through ssh, it will ask for
 * your password.  Since this is done many times, it may end up asking you
 * for your password a hundred times, all under forked processes.  This is
 * very annoying.  To set up an rsa key for your cluster, do something like
 * this:  If you have a 7-node cluster and your node names are mynode-01 to 07:
 *
 *     ssh-keygen -t rsa
 *     for i in `seq 1 7` ; do
 *     ssh -l root mynode-0$i "/bin/mkdir /root/.ssh/"
 *     scp ~/.ssh/id_rsa.pub root@mynode-0$i:/root/.ssh/authorized_key
 *     done
 *
 * You need to fully understand the security options, ramifications and issues
 * before you set up rsa keys on your systems.
 *
 * Disclaimers:
 *
 * This tool will only give you meaningful information if GFS2 is, in fact,
 * the cause of your lockup.
 *
 * This a primitive tool and is not meant to be enterprise-ready software.
 * Its sole purpose is to do a quick and dirty analysis of a GFS2 cluster
 * that appears to be locked and try to give information as to why.
 * It was written in a hurry, and all refinement was left for later.
 * Thus there are a number of improvements that should be made, such as
 * moving away from hard-coded sizes for strings and going to malloced data,
 * and the hard-coded number of max mounts and max hung glocks.
 *
 * Glock locking order within GFS2 must be maintained:
 *
 *  1. i_mutex (if required)
 *  2. Rename glock (for rename only)
 *  3. Iopen glock for inode (in Shared mode)
 *  4. Inode glock
 *     (Parents before children, inodes at "same level" with same parent in
 *      lock number order)
 *  5. Rgrp glock(s) (for (de)allocation operations)
 *  6. Transaction glock (via gfs2_trans_begin) for non-read operations
 *  7. Page lock  (always last, very important!)
 */
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <time.h>
#include <limits.h>
#include <sys/wait.h>

#define CONF "/etc/cluster/cluster.conf"
#define MAX_NODES 64
#define MAX_MOUNTS 16
#define MAX_HUNG_GLOCKS 20
#define MAX_HUNG_PIDS 30
#define LOGFILE "gfs2_hangalyzer.log"

struct nodeinfo {
	int node_id;
	char nodename[80];
	int mount_count;
	char debugfs[PATH_MAX];
	char *mounts[MAX_MOUNTS];
};

struct holderinfo {
	char *nodename;
	char *mountpt;
	char line[256];
};

struct glockinfo {
	char glock_id[40];
	int holders;
	struct holderinfo holder[40];
	int lkb_granted_pid;
};

enum locktypes {
	rename_glock = 0,
	iopen_glock = 1,
	inode_glock = 2,
	rgrp_glock = 3,
	trans_glock = 4,
	lock_levels = 5
};

struct pidinfo {
	int pid;
	int locks_held[lock_levels];
};

int verbose = 0;
int use_qarsh = 0;
const char *shell_prg[2] = {"/usr/bin/ssh", "/usr/bin/qarsh"};
const char *shell_prg2[2] = {"ssh", "qarsh"};
char onenode[256];
struct nodeinfo *node[MAX_NODES] = {NULL,};
int node_count;
extern char *optarg;
struct glockinfo hung_glocks[MAX_HUNG_GLOCKS];
int hung_glock_cnt;
int child_ended;
int analyze_only;
char port[10];
struct pidinfo suspect[MAX_HUNG_PIDS];
int suspects = 0;
int has_filters[MAX_NODES] = {0,};
char tmp[256];
FILE *logfile;

/* ------------------------------------------------------------------------ */
/* usage                                                                    */
/* ------------------------------------------------------------------------ */
void usage(char **argv)
{
	printf("Format is: \n\t%s -n anynode [-q][-v][-a][-p]\n", argv[0]);
	printf("-n any node name in the cluster.\n");
	printf("-p port to use for ssh.\n");
	printf("-q use qarsh instead of ssh\n");
	printf("-v increase verbose setting\n");
	printf("-a analyze existing files (skip downloads)\n");
	printf("If -q is specified, qarsh will be used to fetch info, "
	       "otherwise ssh.\n");
}

/* ------------------------------------------------------------------------ */
/* find_suspect - identify a suspect process by its pid, or create one      */
/* ------------------------------------------------------------------------ */
struct pidinfo *find_suspect(int pid)
{
	int i;

	for (i = 0; i < suspects; i++) {
		if (suspect[i].pid == pid)
			return &suspect[i];
	}
	if (suspects >= MAX_HUNG_PIDS) {
		fprintf(stderr, "Error: too many pids to process.  Try "
			"increasing MAX_HUNG_PIDS.\n");
		exit(-1);
	}
	suspects++;
	return &suspect[i];
}

/* ------------------------------------------------------------------------ */
/* init                                                                     */
/* ------------------------------------------------------------------------ */
int init(int argc, char *argv[])
{
	int optchar;

	memset(onenode, 0, sizeof(onenode));
	memset(port, 0, sizeof(port));
	analyze_only = 0;

	unlink(LOGFILE);
	while (1) {
		optchar = getopt(argc, argv, "ap:qvn:");
		if (optchar <= 0)
			break;
		switch (optchar) {
		case 'n':
			strcpy(onenode, optarg);
			break;
		case 'v':
			verbose++;
			break;
		case 'p':
			strcpy(port, optarg);
			break;
		case 'q':
			use_qarsh = 1;
			break;
		case 'a':
			analyze_only = 1;
			break;
		default:
			fprintf(stderr, "Unknown option: %c\n", optchar);
			usage(argv);
			exit(0);
		}
	}
	if (onenode[0] == '\0') {
		fprintf(stderr, "%s error: node not specified.\n", argv[0]);
		usage(argv);
		exit(-1);
	}
	return 0;
}

/* ------------------------------------------------------------------------ */
/* temp_node_file - concoct a temporary filename we can use                 */
/* ------------------------------------------------------------------------ */
const char *temp_node_file(const char *nodename)
{
	static char fname[256];

	sprintf(fname,"/tmp/gfs2_hangalyzer.%s", nodename);
	return fname;
}

/* ------------------------------------------------------------------------ */
/* fn_suffix - concoct a temp filename with a suffix                        */
/* ------------------------------------------------------------------------ */
const char *fn_suffix(int n, const char *mntpt, const char *suffix)
{
	static char fname[256];

	if (mntpt)
		sprintf(fname, "%s.%s.%s", temp_node_file(node[n]->nodename),
			mntpt, suffix);
	else
		sprintf(fname, "%s.%s", temp_node_file(node[n]->nodename),
			suffix);
	return fname;
}

/* ------------------------------------------------------------------------ */
/* glock_to_lkb - convert a gfs2-style glock ID to a dlm lkb id             */
/*                e.g. "2/16" becomes "       2              16"            */
/* ------------------------------------------------------------------------ */
const char *glock_to_lkb(const char *glock_id)
{
	static char lkb_id[256];
	char temp[256], *p;

	memset(lkb_id, 0, sizeof(lkb_id));
	memset(temp, 0, sizeof(temp));
	strncpy(temp, glock_id, sizeof(temp));
	p = strchr(temp, '/');
	if (p) {
		*p = '\0';
		p++;
		sprintf(lkb_id, "%8.8s%16.16s", temp, p);
	}
	return lkb_id;
}

void sighandler(int error)
{
	child_ended = 1;
}

/* ------------------------------------------------------------------------ */
/* do_node_cmd - execute a qarsh/ssh command and save the output in tmp     */
/* Returns: the filename with the prefix appended                           */
/* ------------------------------------------------------------------------ */
const char *do_node_cmd(const char *nodename, const char *cmd,
			const char *mntpt, const char *suffix)
{
	pid_t qarshpid;
	int fd;
	struct stat statbuf;
	static char newname[PATH_MAX];

	logfile = fopen(LOGFILE, "at");
	if (!logfile) {
		perror(LOGFILE);
		exit(-1);
	}
	fprintf(logfile, "%s: %s\n", nodename, cmd);
	fclose(logfile);
	if (!analyze_only) {
		unlink(temp_node_file(nodename));
		child_ended = 0;
		signal(SIGCHLD, sighandler);
		if ((qarshpid = fork()) == 0) {
			close(1);
			fd = creat(temp_node_file(nodename), S_IRUSR |
				   S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH |
				   S_IWOTH);
			if (fd) {
				if (!verbose)
					fprintf(stderr, "%120.120s\r", " ");
				fprintf(stderr, "exec: %s %s %s %s %s \"%s\"\n", 
					shell_prg[use_qarsh],
					shell_prg2[use_qarsh],
					"-l", "root", nodename, cmd);
				if (port[0]) {
					if ((execl(shell_prg[use_qarsh],
						   shell_prg2[use_qarsh],
						   "-p", port,
						   "-l", "root", nodename,
						   cmd, NULL)) == -1) {
						perror(cmd);
						exit(-1);
					}
				} else {
					if ((execl(shell_prg[use_qarsh],
						   shell_prg2[use_qarsh],
						   "-l", "root", nodename,
						   cmd, NULL)) == -1) {
						perror(cmd);
						exit(-1);
					}
				}
				close(fd);
			} else
				perror(temp_node_file(nodename));
			exit(0);
		}
		while (!child_ended) {
			waitpid(qarshpid, NULL, WNOHANG);
			lstat(temp_node_file(nodename), &statbuf);
			/*fprintf(stderr, "Downloading: %ld bytes transferred"
			  "              \r", statbuf.st_size);*/
			if (!child_ended)
				sleep(1);
			/*else if (!verbose)
			  fprintf(stderr, "%120.120s\r", " ");*/
		}
	}
	memset(newname, 0, sizeof(newname));
	strcpy(newname, temp_node_file(nodename));
	if (mntpt) {
		strcat(newname, ".");
		strcat(newname, mntpt);
	}
	strcat(newname, ".");
	strcat(newname, suffix);
	rename(temp_node_file(nodename), newname);
	return newname;
}

/* ------------------------------------------------------------------------ */
/* analyze_cluster_conf                                                     */
/* ------------------------------------------------------------------------ */
int analyze_cluster_conf(void)
{
	FILE *fp;
	char line[1024];
	int nid;
	const char *cluster_conf;

	cluster_conf = do_node_cmd(onenode,
				   "/bin/grep 'clusternode name' "
				   "/etc/cluster/cluster.conf", NULL, "conf");
	fp = fopen(cluster_conf, "rt");
	if (!fp) {
		perror(cluster_conf);
		return -1;
	}
	node_count = 0;
	while (fgets(line, sizeof(line) - 1, fp)) {
		char *p, *p2, *p3, *p4;

		if (verbose >= 2)
			printf("%s", line);
		p = strstr(line,"<clusternode name=");
		if (p) {
			p += 19;
			p2 = strchr(p,'"');
			node[node_count] = malloc(sizeof(struct nodeinfo));
			memset(node[node_count], 0, sizeof(struct nodeinfo));
			strncpy(node[node_count]->nodename, p, p2 - p);

			p3 = strstr(line,"nodeid=");
			if (p3)
				p3 += 8;
			p4 = strchr(p3,'"');
			if (p4)
				*p4 = '\0';
			sscanf(p3, "%d", &nid);
			node[node_count]->node_id = nid;
			if (verbose >= 1)
				printf("%d:nodeid %d: %s\n", node_count + 1,
				       node[node_count]->node_id,
				       node[node_count]->nodename);
			node_count++;
		}
	}
	if (verbose)
		printf("node_count = %d\n", node_count);
	fclose(fp);
	return 0;
}

/* ------------------------------------------------------------------------ */
/* find_debugfs                                                             */
/* ------------------------------------------------------------------------ */
void find_debugfs(int n)
{
	char line[PATH_MAX], tmp[PATH_MAX];
	ssize_t count = 0;
	int fd;
	const char *fn;

	fn = do_node_cmd(node[n]->nodename, "/bin/grep debugfs /proc/mounts",
			 NULL, "mounts");
	fd = open(fn, O_RDONLY);
	count = read(fd, line, PATH_MAX);
	line[count] = '\0';
	if (count)
		sscanf(line, "%s %s", tmp, node[n]->debugfs);
	else {
		fn = do_node_cmd(node[n]->nodename, "/bin/mkdir /sys/kernel"
				 "/debug", NULL, "mkdir");
		fn = do_node_cmd(node[n]->nodename, "/bin/mount -t debugfs "
				 "none /sys/kernel/debug", NULL, "mount");
		strcpy(node[n]->debugfs, "/sys/kernel/debug");
	}
	close(fd);
	return;
}

/* ------------------------------------------------------------------------ */
/* find_mounts                                                              */
/* figure out all the gfs2 mounts on a node, fill in mounts, and return #   */
/* ------------------------------------------------------------------------ */
int find_mounts(int n)
{
	char line[PATH_MAX];
	FILE *file;
	char cmd[256];
	const char *fn;

	sprintf(cmd, "/bin/ls %s/gfs2/", node[n]->debugfs);
	fn = do_node_cmd(node[n]->nodename, cmd, NULL, "debugfs");

	node[n]->mount_count = 0;
	file = fopen(fn, "rt");
	if (!file)
		return -1;
	while (fgets(line, PATH_MAX, file)) {
		char *p;

		while ((p = strchr(line, '\n')))
			*p = '\0';
		while ((p = strchr(line, '\r')))
			*p = '\0';
		node[n]->mounts[node[n]->mount_count] = malloc(PATH_MAX);
		memset(node[n]->mounts[node[n]->mount_count], 0, PATH_MAX);
		strcpy(node[n]->mounts[node[n]->mount_count], line);
		if (node[n]->mount_count < MAX_MOUNTS - 1)
			node[n]->mount_count++;
	}
	return 0;
}

/* ------------------------------------------------------------------------ */
/* glock_number - extract the glock type from a string                      */
/* ------------------------------------------------------------------------ */
const char *glock_type(const char *glock_id)
{
	char *p;

	memset(tmp, 0, sizeof(tmp));
	strncpy(tmp, glock_id, sizeof(tmp));
	p = strchr(tmp, '/');
	if (p)
		*p = '\0';
	return tmp;
}

/* ------------------------------------------------------------------------ */
/* glock_number - extract the glock number from a string                    */
/* ------------------------------------------------------------------------ */
const char *glock_number(const char *glock_id)
{
	char *p;

	memset(tmp, 0, sizeof(tmp));
	strncpy(tmp, glock_id, sizeof(tmp));
	p = strchr(tmp, '/');
	if (p) {
		*p = '\0';
		p++;
	}
	return p;
}

/* ------------------------------------------------------------------------ */
/* fetch_one_glock_dump                                                     */
/* ------------------------------------------------------------------------ */
int fetch_one_glock_dump(int n, const char *mntpt, const char *glock_id)
{
	char cmd[256];
	const char *fn;

	if (!has_filters[n])
		return 0;

	/* Remove the Waiters-only filter */
	sprintf(cmd, "/bin/echo 'n' > %s/gfs2/%s/waiters_only",
		node[n]->debugfs, mntpt);
	fn = do_node_cmd(node[n]->nodename, cmd, mntpt, "waiters_only");
	/* Filter for the specified glock type */
	sprintf(cmd, "/bin/echo '%s' > %s/gfs2/%s/glock_type_filter",
		glock_type(glock_id), node[n]->debugfs, mntpt);
	fn = do_node_cmd(node[n]->nodename, cmd, mntpt, "glock_type_filter");

	/* Filter for the specified glock */
	sprintf(cmd, "/bin/echo '0x%s' > %s/gfs2/%s/glock_number_filter",
		glock_number(glock_id), node[n]->debugfs, mntpt);
	fn = do_node_cmd(node[n]->nodename, cmd, mntpt, "glock_number_filter");

	if (verbose)
		fprintf(stderr, "fetching glock %s on %s: %s\n",
			glock_id, node[n]->nodename, mntpt);
	sprintf(cmd, "/bin/cat %s/gfs2/%s/glocks", node[n]->debugfs, mntpt);
	fn = do_node_cmd(node[n]->nodename, cmd, mntpt, "glocks2");

	/* Remove filter for the specified glock type */
	sprintf(cmd, "/bin/echo '0' > %s/gfs2/%s/glock_type_filter",
		node[n]->debugfs, mntpt);
	fn = do_node_cmd(node[n]->nodename, cmd, mntpt, "glock_type_filter");

	/* Remove filter for the specified glock */
	sprintf(cmd, "/bin/echo '0' > %s/gfs2/%s/glock_number_filter",
		node[n]->debugfs, mntpt);
	fn = do_node_cmd(node[n]->nodename, cmd, mntpt, "glock_number_filter");
	return 0;
}

/* ------------------------------------------------------------------------ */
/* fetch_glock_dumps                                                        */
/* ------------------------------------------------------------------------ */
int fetch_glock_dumps(int n)
{
	int mntpt;
	char cmd[256];
	const char *fn;
	FILE *file;

	for (mntpt = 0; mntpt < node[n]->mount_count; mntpt++) {
		/* Figure out if this node has glock filters */
		sprintf(cmd, "/bin/cat %s/gfs2/%s/glock_number_filter",
			node[n]->debugfs, node[n]->mounts[mntpt]);
		fn = do_node_cmd(node[n]->nodename, cmd,
				 node[n]->mounts[mntpt], "glock_number_filter");
		file = fopen(fn, "rt");
		if (file)
			has_filters[n] = 1;
		fclose(file);

		/* If it has filters, dump only the glocks with waiters */
		if (has_filters[n]) {
			sprintf(cmd, "/bin/echo 'y' > %s/gfs2/%s/waiters_only",
				node[n]->debugfs, node[n]->mounts[mntpt]);
			fn = do_node_cmd(node[n]->nodename, cmd,
					 node[n]->mounts[mntpt],
					 "waiters_only");
		}

		sprintf(cmd, "/bin/cat %s/gfs2/%s/glocks",
			node[n]->debugfs, node[n]->mounts[mntpt]);
		if (verbose)
			fprintf(stderr, "fetching glocks for %s: %s\n",
				node[n]->nodename, node[n]->mounts[mntpt]);
		fn = do_node_cmd(node[n]->nodename, cmd,
				 node[n]->mounts[mntpt], "glocks");
	}
	return 0;
}

/* ------------------------------------------------------------------------ */
/* convert a gfs2 mount id "bobs_roth:test_gfs" to a dlm id "test_gfs"      */
/* ------------------------------------------------------------------------ */
const char *dlm_mntname(const char *gfs2_mntid)
{
	const char *p;

	p = strchr(gfs2_mntid, ':');
	if (p)
		p++;
	return p;
}

/* ------------------------------------------------------------------------ */
/* fetch_dlm_dumps                                                          */
/* ------------------------------------------------------------------------ */
int fetch_dlm_dumps(int n)
{
	int mntpt;
	char cmd[256];
	const char *fn;

	for (mntpt = 0; mntpt < node[n]->mount_count; mntpt++) {
		sprintf(cmd, "/bin/cat %s/dlm/%s",
			node[n]->debugfs, dlm_mntname(node[n]->mounts[mntpt]));
		if (verbose)
			fprintf(stderr, "fetching dlm locks for %s: %s\n",
				node[n]->nodename,
				dlm_mntname(node[n]->mounts[mntpt]));
		fn = do_node_cmd(node[n]->nodename, cmd,
				 node[n]->mounts[mntpt], "dlm");
	}
	return 0;
}

/* ------------------------------------------------------------------------ */
/* fetch_dlm_dumps2 - fetch the _locks dlm debug file                       */
/* ------------------------------------------------------------------------ */
int fetch_dlm_dumps2(int n)
{
	int mntpt;
	char cmd[256];
	const char *fn;

	for (mntpt = 0; mntpt < node[n]->mount_count; mntpt++) {
		sprintf(cmd, "/bin/cat %s/dlm/%s_locks",
			node[n]->debugfs, dlm_mntname(node[n]->mounts[mntpt]));
		if (verbose)
			fprintf(stderr, "fetching dlm locks2 for %s: %s\n",
				node[n]->nodename,
				dlm_mntname(node[n]->mounts[mntpt]));
		fn = do_node_cmd(node[n]->nodename, cmd,
				 node[n]->mounts[mntpt], "dlm2");
	}
	return 0;
}

/* ------------------------------------------------------------------------ */
/* fetch_dlm_waiters - fetch the _waiters dlm file                          */
/* ------------------------------------------------------------------------ */
int fetch_dlm_waiters(int n)
{
	int mntpt;
	char cmd[256];
	const char *fn;

	for (mntpt = 0; mntpt < node[n]->mount_count; mntpt++) {
		sprintf(cmd, "/bin/cat %s/dlm/%s_waiters",
			node[n]->debugfs, dlm_mntname(node[n]->mounts[mntpt]));
		if (verbose)
			fprintf(stderr, "fetching dlm waiters for %s: %s\n",
				node[n]->nodename,
				dlm_mntname(node[n]->mounts[mntpt]));
		fn = do_node_cmd(node[n]->nodename, cmd,
				 node[n]->mounts[mntpt], "waiters");
	}
	return 0;
}

/* ------------------------------------------------------------------------ */
/* decode_glock_flags - print a user-friendly version of the glock flags    */
/* ------------------------------------------------------------------------ */
void decode_glock_flags(const char *nodename, const char *gflags)
{
	const char *p;
	int flagi, count = 0;
	const char *flags = "lsDdpyfirqO?";
	const char *flagdesc[12] = {"locked",         "sticky",
				    "demote",         "pending demote",
				    "demote in prog", "dirty",
				    "lflush",         "invalidate in prog",
				    "reply pending",  "holder queued",
				    "callback owed",  "unknown flag" };

	p = strchr(gflags, ' ');
	if (p == gflags || p == gflags + 1)
		return;
	printf("%-10.10s:                         ", nodename);
	p = gflags;
	while (p && *p != ' ') {
		for (flagi = 0; flagi < 11; flagi++)
			if (*p == flags[flagi])
				break;
		printf("%s%s", count ? ", " : "(", flagdesc[flagi]);
		count++;
		p++;
	}
	printf(")\n");
}

/* ------------------------------------------------------------------------ */
/* get_lock_level - figure out the level of a lock based on its number      */
/* ------------------------------------------------------------------------ */
int get_lock_level(struct glockinfo *ginfo)
{
	if (!strcmp(ginfo->glock_id, "1/3"))
		return rename_glock;
	if (ginfo->glock_id[0] == '5')
		return iopen_glock;
	if (ginfo->glock_id[0] == '2')
		return inode_glock;
	if (ginfo->glock_id[0] == '3')
		return rgrp_glock;
	if (!strcmp(ginfo->glock_id, "1/2"))
		return trans_glock;
	fprintf(stderr, "Error: unable to detect glock type for: %s\n",
		ginfo->glock_id);
	exit(-1);
}

/* ------------------------------------------------------------------------ */
/* find_who_is_holding - find what pid is holding a given glock             */
/* ------------------------------------------------------------------------ */
void find_who_is_holding(struct glockinfo *ginfo, int this_level)
{
	int h;
	char holderflags[32], pid[32], *p;
	struct pidinfo *locker;

	for (h = 0; h < ginfo->holders; h++) {
		p = strstr(ginfo->holder[h].line, "f:");
		if (p)
			p += 2;
		strncpy(holderflags, p, 31);
		p = strchr(holderflags, ' ');
		if (p)
			*p = '\0';
		if (!strchr(holderflags, 'H'))
			continue;

		p = strstr(ginfo->holder[h].line, "p:");
		if (p)
			p += 2;
		strncpy(pid, p, 31);
		p = strchr(pid, ' ');
		if (p)
			*p = '\0';
		printf("which is held by pid %s", pid);
		locker = find_suspect(atoi(pid));
		locker->locks_held[this_level] = 1;
		return;
	}
	printf("but no holder was found.");
}

/* ------------------------------------------------------------------------ */
/* backtrack_waitingholders - find who's waiting for a glock and back-track */
/*                            what they're also waiting for.                */
/* ------------------------------------------------------------------------ */
void backtrack_waitingholders(void)
{
	int g, h, this_level, llevel;
	char *p;
	struct pidinfo *me;
	char waiting_pid[32];

	printf("There %s %d %s with waiters.\n",
	       (hung_glock_cnt == 1) ? "is" : "are", hung_glock_cnt,
	       (hung_glock_cnt == 1) ? "glock" : "glocks");
	for (g = 0; g < hung_glock_cnt; g++) {
		for (h = 0; h < hung_glocks[g].holders; h++) {
			if (!strchr(hung_glocks[g].holder[h].line, 'W'))
				continue;

			p = strstr(hung_glocks[g].holder[h].line, "p:");
			if (p)
				p += 2;
			strncpy(waiting_pid, p, 31);
			p = strchr(waiting_pid, ' ');
			if (p)
				*p = '\0';
			this_level = get_lock_level(&hung_glocks[g]);
			printf("%s, pid %s is waiting for glock %s, ",
			       hung_glocks[g].holder[h].nodename, waiting_pid,
			       hung_glocks[g].glock_id);
			find_who_is_holding(&hung_glocks[g], this_level);
			printf("\n");
		}
		if (hung_glocks[g].lkb_granted_pid)
			printf("         The dlm has granted lkb \"%s\" to "
			       "pid %d", glock_to_lkb(hung_glocks[g].glock_id),
			       hung_glocks[g].lkb_granted_pid);
		printf("\n\n");
	}
	/* Run through the glocks a second time.  This time we're trying to
	   determine if anyone has violated GFS2 lock ordering. */
	for (g = 0; g < hung_glock_cnt; g++) {
		for (h = 0; h < hung_glocks[g].holders; h++) {
			if (!strchr(hung_glocks[g].holder[h].line, 'W'))
				continue;

			p = strstr(hung_glocks[g].holder[h].line, "p:");
			if (p)
				p += 2;
			strncpy(waiting_pid, p, 31);
			p = strchr(waiting_pid, ' ');
			if (p)
				*p = '\0';
			this_level = get_lock_level(&hung_glocks[g]);
			/* Now let's check if this lock requestor has
			   violated locking protocol and asked for its
			   locks in the wrong order. */
			if (this_level == rename_glock)
				continue;

			me = find_suspect(atoi(waiting_pid));
			for (llevel = this_level - 1; llevel >= rename_glock;
			     llevel--) {
				if (!me->locks_held[llevel])
					continue;
				printf("Process %s is holding a higher lock "
				       "level than the one it's "
				       "requesting!\n", waiting_pid);
				printf("This could be a GFS2 lock ordering "
				       "problem.\n");
				printf("Please use sysrq-t to dump the "
				       "complete call trace for that pid.\n");
				break;
			}
		}
	}
}

/* ------------------------------------------------------------------------ */
/* process_line - process a line from a glock dump                          */
/* returns: 1 - a different glock was found, so caller should finish        */
/*          0 - working on the same glock, so caller can call again         */
/* ------------------------------------------------------------------------ */
int process_line(int n, int mntpt, const char *glock_id, char *line,
		 int *found_it)
{
	char *gf, *p;

	while ((p = strchr(line, '\n')))
		*p = '\0';
	while ((p = strchr(line, '\r')))
		*p = '\0';
	if (line[0] == 'G') {
		if (strstr(line, glock_id)) {
			*found_it = 1;
			printf("%-10.10s: %s: %s\n", node[n]->nodename,
			       dlm_mntname(node[n]->mounts[mntpt]),
			       line);
			gf = strstr(line, "f:");
			if (gf) {
				gf += 2;
				decode_glock_flags(node[n]->nodename, gf);
			}
		} else if (*found_it)
			return 1;
	} else if (*found_it) {
		if (line[1] == 'H') {
			int *h = &hung_glocks[hung_glock_cnt].holders;

			strcpy(hung_glocks[hung_glock_cnt].holder[*h].line,
			       line);
			hung_glocks[hung_glock_cnt].holder[*h].mountpt =
				node[n]->mounts[mntpt];
			hung_glocks[hung_glock_cnt].holder[*h].nodename =
				node[n]->nodename;
			*h = *h + 1;
		}
		printf("%-10.10s: %s: %s\n", node[n]->nodename,
		       dlm_mntname(node[n]->mounts[mntpt]), line);
	}
	return 0;
}

/* ------------------------------------------------------------------------ */
/* process_lkb - reformat an lkb so it's easier to read.                    */
/* returns: 1 - if this is a "granted" lkb, else 0                          */
/* ------------------------------------------------------------------------ */
void process_lkb(int n, const char *line, const char *mnt_id,
		 const char *glock_id)
{
	unsigned int lkb_id, lkb_nodeid, lkb_remid, lkb_ownpid;
	unsigned long long xid;
	unsigned int lkb_exflags, lkb_flags, lkb_status;
	unsigned int lkb_grmode, lkb_rqmode, waiting, res_nodeid, res_length;
	char res_name[64], *p;
	const char *sts_strings[] = {"zero", "wait", "grnt", "conv"};

	memset(res_name, 0, sizeof(res_name));
	p = strchr(line, '\"');
	if (p) {
		p++;
		strncpy(res_name, p, 63);
		p = strchr(res_name, '\"');
		if (p)
			*p = '\0';
	}
	sscanf(line, "%x %d %x %u %llu %x %x %d %d %d %u %u %d ",
	       &lkb_id, &lkb_nodeid, &lkb_remid, &lkb_ownpid, &xid,
	       &lkb_exflags, &lkb_flags, &lkb_status, &lkb_grmode,
	       &lkb_rqmode, &waiting, &res_nodeid, &res_length);
	printf("%-10.10s: %s: ", node[n]->nodename, dlm_mntname(mnt_id));
	printf("%8x %1d %8x %4u %5x %7x %4s %2d %2d %10u %1u %2d \"%s\"\n",
	       lkb_id, lkb_nodeid, lkb_remid, lkb_ownpid, lkb_exflags,
	       lkb_flags, sts_strings[lkb_status], lkb_grmode, lkb_rqmode,
	       waiting, res_nodeid, res_length, res_name);
	if (lkb_status == 2) {
		int l;

		/* Find the hung glock and add the granted lkb pid info: */
		for (l = 0; l <= hung_glock_cnt; l++) {
			struct glockinfo *g;

			g = &hung_glocks[l];
			if (!strcmp(g->glock_id, glock_id)) {
				g->lkb_granted_pid = lkb_ownpid;
				break;
			}
		}
	}
}

/* ------------------------------------------------------------------------ */
/* find_dump_lkb - find a given dlm lkb in all the dlm dumps and dump it    */
/* ------------------------------------------------------------------------ */
void find_dump_lkb(int n, int mntpt, const char *glock_id, const char *mnt_id)
{
	FILE *file;
	char line[1024], lkb[256], *p;
	int found_it;

	strcpy(lkb, glock_to_lkb(glock_id));

	file = fopen(fn_suffix(n, node[n]->mounts[mntpt], "dlm2"), "rt");
	found_it = 0;
	while (fgets(line, PATH_MAX, file)) {
		while ((p = strchr(line, '\n')))
			*p = '\0';
		while ((p = strchr(line, '\r')))
			*p = '\0';
		if (strstr(line, lkb)) {
			if (!found_it)
				printf("\n                      "
				       "  lkb_id N RemoteID "
				       " pid exflg lkbflgs "
				       "stat gr rq    waiting "
				       "n ln             "
				       "resource name\n");
			found_it = 1;
			process_lkb(n, line, mnt_id, glock_id);
		}
	}
	fclose(file);
}

/* ------------------------------------------------------------------------ */
/* find_dump_glock - find a given glock in all the glock dumps and dump it  */
/* ------------------------------------------------------------------------ */
void find_dump_glock(const char *glock_id, const char *mnt_id)
{
	int n;
	int mntpt;
	FILE *waiter;
	char line[1024];
	int found_it, finished;

	if (verbose)
		printf("Dumping all occurrences of glock %s\n", glock_id);
	for (n = 0; n < node_count; n++) {
		/* printf("%-10.10s: --------------------------------------"
		       "--------------------- %s\n",
		       node[n]->nodename, glock_id); */
		for (mntpt = 0; mntpt < node[n]->mount_count; mntpt++) {
			if (strcmp(node[n]->mounts[mntpt], mnt_id))
				continue;
			waiter = fopen(fn_suffix(n, node[n]->mounts[mntpt],
						 "glocks"), "rt");
			found_it = 0;
			while (waiter && fgets(line, PATH_MAX, waiter)) {
				finished = process_line(n, mntpt, glock_id,
							line, &found_it);
				memset(line, 0, sizeof(line));
				if (finished)
					break;
			}
			if (waiter)
				fclose(waiter);
			if (!found_it && has_filters[n]) {
				fetch_one_glock_dump(n, node[n]->mounts[mntpt],
						     glock_id);
				waiter = fopen(fn_suffix(n,
							node[n]->mounts[mntpt],
							 "glocks2"), "rt");
				found_it = 0;
				while (waiter &&
				       fgets(line, PATH_MAX, waiter)) {
					finished = process_line(n, mntpt,
								glock_id, line,
								&found_it);
					memset(line, 0, sizeof(line));
					if (finished)
						break;
				}
				if (waiter)
					fclose(waiter);
			}
			if (!found_it)
				printf("%-10.10s: %s: No glock refs found to %s\n",
				       node[n]->nodename, dlm_mntname(mnt_id),
				       glock_id);
			find_dump_lkb(n, mntpt, glock_id, mnt_id);
		}
	}
	printf("\n\n\n");
}

/* ------------------------------------------------------------------------ */
/* process_holder - We've found a holder that's waiting, so dump the glock  */
/* ------------------------------------------------------------------------ */
void process_holder(int n, int mntpt, char *line, char *last_glock)
{
	char holder[PATH_MAX];
	int glock_already_done;
	char glock_id[80], *p;
	int g;

	strcpy(holder, line);
	/* We have a waiter */
	if (verbose >= 2) {
		printf("Glock waited on: \n");
		printf("%s\n", last_glock);
		printf("%s\n", holder);
	}
	memset(glock_id, 0, sizeof(glock_id));
	strncpy(glock_id, &last_glock[11], 20);
	p = strchr(glock_id, ' ');
	if (p)
		*p = '\0';
	/* See if we've already processed it */
	glock_already_done = 0;
	for (g = 0; g < hung_glock_cnt; g++) {
		if (!strcmp(hung_glocks[g].glock_id, glock_id)) {
			glock_already_done = 1;
			break;
		}
	}
	if (!glock_already_done) {
		strcpy(hung_glocks[hung_glock_cnt].glock_id, glock_id);
		hung_glocks[hung_glock_cnt].holders = 0;
		find_dump_glock(glock_id, node[n]->mounts[mntpt]);
		hung_glock_cnt++;
	}
}

/* ------------------------------------------------------------------------ */
/* analyze_glocks                                                           */
/* ------------------------------------------------------------------------ */
int analyze_glocks(void)
{
	int n;
	int mntpt;
	FILE *waiter;
	char line[1024], *p;
	char last_glock[PATH_MAX];
	const char *fn;

	memset(last_glock, 0, sizeof(last_glock));
	memset(hung_glocks, 0, sizeof(hung_glocks));
	hung_glock_cnt = 0;
	/* find all the glocks that are in the wait state */
	for (n = 0; n < node_count; n++) {
		for (mntpt = 0; mntpt < node[n]->mount_count; mntpt++) {
			fn = fn_suffix(n, node[n]->mounts[mntpt], "glocks");
			waiter = fopen(fn, "rt");
			if (waiter == NULL) {
				perror(fn);
				continue;
			}
			while (fgets(line, PATH_MAX, waiter)) {
				while ((p = strchr(line, '\n')))
					*p = '\0';
				while ((p = strchr(line, '\r')))
					*p = '\0';
				if (line[0] == 'G') {
					strcpy(last_glock, line);
					continue;
				}
				if (line[1] == 'H' && strchr(line, 'W'))
					process_holder(n, mntpt, line,
						       last_glock);
			}
			fclose(waiter);
		}
	}
	if (!hung_glock_cnt)
		printf("No waiting glocks found on any node.\n");
	else
		backtrack_waitingholders();
	return 0;
}

/* ------------------------------------------------------------------------ */
/* main                                                                     */
/* ------------------------------------------------------------------------ */
int main(int argc, char **argv)
{
	int error, i;

	/* step 1 - analyze the cluster.conf file to determine the nodes.*/
	init(argc, argv);
	memset(node, 0, sizeof(node));
	memset(suspect, 0, sizeof(suspect));
	error = analyze_cluster_conf();

	for (i = 0; i < node_count; i++)
		find_debugfs(i);
	for (i = 0; i < node_count; i++)
		find_mounts(i);
	for (i = 0; i < node_count; i++)
		fetch_glock_dumps(i);
	for (i = 0; i < node_count; i++)
		fetch_dlm_dumps(i);
	for (i = 0; i < node_count; i++)
		fetch_dlm_dumps2(i);
	for (i = 0; i < node_count; i++)
		fetch_dlm_waiters(i);
	analyze_glocks();
	/* free up the memory we allocated for node names. */
	fflush(stdout);
	for (i = 0; i < node_count; i++)
		if (node[i]) {
			free(node[i]);
			node[i] = NULL;
		}
	exit(0);
}