Fix multi-thread access for MQTTClient

Bug: 474905

Makefile

@@ -78,13 +78,13 @@ SYNC_SAMPLES = ${addprefix ${blddir}/samples/,${SAMPLE_FILES_C}}
 SAMPLE_FILES_A = stdoutsuba MQTTAsync_subscribe MQTTAsync_publish
 ASYNC_SAMPLES = ${addprefix ${blddir}/samples/,${SAMPLE_FILES_A}}
 
-TEST_FILES_C = test1 sync_client_test test_mqtt4sync
+TEST_FILES_C = test1 test2 sync_client_test test_mqtt4sync 
 SYNC_TESTS = ${addprefix ${blddir}/test/,${TEST_FILES_C}}
 
 TEST_FILES_CS = test3
 SYNC_SSL_TESTS = ${addprefix ${blddir}/test/,${TEST_FILES_CS}}
 
-TEST_FILES_A = test4 test_mqtt4async
+TEST_FILES_A = test4 test_mqtt4async MQTTAsync_multi_pub
 ASYNC_TESTS = ${addprefix ${blddir}/test/,${TEST_FILES_A}}
 
 TEST_FILES_AS = test5
@@ -172,7 +172,7 @@ mkdir:
 	echo OSTYPE is $(OSTYPE)
 
 ${SYNC_TESTS}: ${blddir}/test/%: ${srcdir}/../test/%.c $(MQTTLIB_C_TARGET)
-	${CC} -g -o $@ $< -l${MQTTLIB_C} ${FLAGS_EXE}
+	${CC} -DNOSTACKTRACE $(srcdir)/Thread.c -g -o $@ $< -l${MQTTLIB_C} ${FLAGS_EXE}
 
 ${SYNC_SSL_TESTS}: ${blddir}/test/%: ${srcdir}/../test/%.c $(MQTTLIB_CS_TARGET)
 	${CC} -g -o $@ $< -l${MQTTLIB_CS} ${FLAGS_EXES}

build.xml

@@ -82,7 +82,7 @@
   </target>
 	
   <target name="test" >
-    <foreach target="runAtest" param="aTest" list="test1,test4"/>
+    <foreach target="runAtest" param="aTest" list="test1,test2,test4"/>
     <foreach target="runSSLtest" param="aTest" list="test3,test5"/>
   </target>
 

src/MQTTClient.c

 /*******************************************************************************
- * Copyright (c) 2009, 2014 IBM Corp.
+ * Copyright (c) 2009, 2015 IBM Corp.
  *
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
@@ -27,6 +27,7 @@
  *    Ian Craggs - fix for bug 443724 - stack corruption
  *    Ian Craggs - fix for bug 447672 - simultaneous access to socket structure
  *    Ian Craggs - fix for bug 459791 - deadlock in WaitForCompletion for bad client
+ *    Ian Craggs - fix for bug 474905 - insufficient synchronization for subscribe, unsubscribe, connect
  *******************************************************************************/
 
 /**
@@ -79,6 +80,9 @@ MQTTProtocol state;
 #if defined(WIN32) || defined(WIN64)
 static mutex_type mqttclient_mutex = NULL;
 static mutex_type socket_mutex = NULL;
+static mutex_type subscribe_mutex = NULL;
+static mutex_type unsubscribe_mutex = NULL;
+static mutex_type connect_mutex = NULL;
 extern mutex_type stack_mutex;
 extern mutex_type heap_mutex;
 extern mutex_type log_mutex;
@@ -93,6 +97,9 @@ BOOL APIENTRY DllMain(HANDLE hModule,
 			if (mqttclient_mutex == NULL)
 			{
 				mqttclient_mutex = CreateMutex(NULL, 0, NULL);
+				subscribe_mutex = CreateMutex(NULL, 0, NULL);
+				unsubscribe_mutex = CreateMutex(NULL, 0, NULL);
+				connect_mutex = CreateMutex(NULL, 0, NULL);
 				stack_mutex = CreateMutex(NULL, 0, NULL);
 				heap_mutex = CreateMutex(NULL, 0, NULL);
 				log_mutex = CreateMutex(NULL, 0, NULL);
@@ -110,9 +117,19 @@ BOOL APIENTRY DllMain(HANDLE hModule,
 #else
 static pthread_mutex_t mqttclient_mutex_store = PTHREAD_MUTEX_INITIALIZER;
 static mutex_type mqttclient_mutex = &mqttclient_mutex_store;
+
 static pthread_mutex_t socket_mutex_store = PTHREAD_MUTEX_INITIALIZER;
 static mutex_type socket_mutex = &socket_mutex_store;
 
+static pthread_mutex_t subscribe_mutex_store = PTHREAD_MUTEX_INITIALIZER;
+static mutex_type subscribe_mutex = &subscribe_mutex_store;
+
+static pthread_mutex_t unsubscribe_mutex_store = PTHREAD_MUTEX_INITIALIZER;
+static mutex_type unsubscribe_mutex = &unsubscribe_mutex_store;
+
+static pthread_mutex_t connect_mutex_store = PTHREAD_MUTEX_INITIALIZER;
+static mutex_type connect_mutex = &connect_mutex_store;
+
 void MQTTClient_init()
 {
 	pthread_mutexattr_t attr;
@@ -124,6 +141,12 @@ void MQTTClient_init()
 		printf("MQTTClient: error %d initializing client_mutex\n", rc);
 	if ((rc = pthread_mutex_init(socket_mutex, &attr)) != 0)
 		printf("MQTTClient: error %d initializing socket_mutex\n", rc);
+	if ((rc = pthread_mutex_init(subscribe_mutex, &attr)) != 0)
+		printf("MQTTClient: error %d initializing subscribe_mutex\n", rc);
+	if ((rc = pthread_mutex_init(unsubscribe_mutex, &attr)) != 0)
+		printf("MQTTClient: error %d initializing unsubscribe_mutex\n", rc);
+	if ((rc = pthread_mutex_init(connect_mutex, &attr)) != 0)
+		printf("MQTTClient: error %d initializing connect_mutex\n", rc);
 }
 
 #define WINAPI
@@ -1032,6 +1055,7 @@ int MQTTClient_connect(MQTTClient handle, MQTTClient_connectOptions* options)
 	int rc = SOCKET_ERROR;
 
 	FUNC_ENTRY;
+	Thread_lock_mutex(connect_mutex);
 	Thread_lock_mutex(mqttclient_mutex);
 
 	if (options == NULL)
@@ -1106,6 +1130,7 @@ exit:
 		m->c->will = NULL;
 	}
 	Thread_unlock_mutex(mqttclient_mutex);
+	Thread_unlock_mutex(connect_mutex);
 	FUNC_EXIT_RC(rc);
 	return rc;
 }
@@ -1217,13 +1242,14 @@ int MQTTClient_isConnected(MQTTClient handle)
 int MQTTClient_subscribeMany(MQTTClient handle, int count, char* const* topic, int* qos)
 {
 	MQTTClients* m = handle;
-	List* topics = ListInitialize();
-	List* qoss = ListInitialize();
+	List* topics = NULL;
+	List* qoss = NULL;
 	int i = 0;
 	int rc = MQTTCLIENT_FAILURE;
 	int msgid = 0;
 
 	FUNC_ENTRY;
+	Thread_lock_mutex(subscribe_mutex);
 	Thread_lock_mutex(mqttclient_mutex);
 
 	if (m == NULL || m->c == NULL)
@@ -1256,6 +1282,8 @@ int MQTTClient_subscribeMany(MQTTClient handle, int count, char* const* topic, i
 		goto exit;
 	}
 
+	topics = ListInitialize();
+	qoss = ListInitialize();
 	for (i = 0; i < count; i++)
 	{
 		ListAppend(topics, topic[i], strlen(topic[i]));
@@ -1297,6 +1325,7 @@ int MQTTClient_subscribeMany(MQTTClient handle, int count, char* const* topic, i
 
 exit:
 	Thread_unlock_mutex(mqttclient_mutex);
+	Thread_unlock_mutex(subscribe_mutex);
 	FUNC_EXIT_RC(rc);
 	return rc;
 }
@@ -1319,12 +1348,13 @@ int MQTTClient_subscribe(MQTTClient handle, const char* topic, int qos)
 int MQTTClient_unsubscribeMany(MQTTClient handle, int count, char* const* topic)
 {
 	MQTTClients* m = handle;
-	List* topics = ListInitialize();
+	List* topics = NULL;
 	int i = 0;
 	int rc = SOCKET_ERROR;
 	int msgid = 0;
 
 	FUNC_ENTRY;
+	Thread_lock_mutex(unsubscribe_mutex);
 	Thread_lock_mutex(mqttclient_mutex);
 
 	if (m == NULL || m->c == NULL)
@@ -1351,6 +1381,7 @@ int MQTTClient_unsubscribeMany(MQTTClient handle, int count, char* const* topic)
 		goto exit;
 	}
 
+	topics = ListInitialize();
 	for (i = 0; i < count; i++)
 		ListAppend(topics, topic[i], strlen(topic[i]));
 	rc = MQTTProtocol_unsubscribe(m->c, topics, msgid);
@@ -1377,6 +1408,7 @@ int MQTTClient_unsubscribeMany(MQTTClient handle, int count, char* const* topic)
 
 exit:
 	Thread_unlock_mutex(mqttclient_mutex);
+	Thread_unlock_mutex(unsubscribe_mutex);
 	FUNC_EXIT_RC(rc);
 	return rc;
 }

test/test2.c

+/*******************************************************************************
+ * Copyright (c) 2009, 2015 IBM Corp.
+ *
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * and Eclipse Distribution License v1.0 which accompany this distribution. 
+ *
+ * The Eclipse Public License is available at 
+ *    http://www.eclipse.org/legal/epl-v10.html
+ * and the Eclipse Distribution License is available at 
+ *   http://www.eclipse.org/org/documents/edl-v10.php.
+ *
+ * Contributors:
+ *    Ian Craggs - initial API and implementation and/or initial documentation
+ *******************************************************************************/
+
+/**
+ * @file
+ * Multi-threaded tests for the MQ Telemetry MQTT C client
+ */
+
+#include "MQTTClient.h"
+#include "Thread.h"
+#include <string.h>
+#include <stdlib.h>
+
+#if !defined(_WINDOWS)
+	#include <sys/time.h>
+  	#include <sys/socket.h>
+	#include <unistd.h>
+  	#include <errno.h>
+	#define WINAPI
+#else
+#include <winsock2.h>
+#include <ws2tcpip.h>
+#define MAXHOSTNAMELEN 256
+#define EAGAIN WSAEWOULDBLOCK
+#define EINTR WSAEINTR
+#define EINPROGRESS WSAEINPROGRESS
+#define EWOULDBLOCK WSAEWOULDBLOCK
+#define ENOTCONN WSAENOTCONN
+#define ECONNRESET WSAECONNRESET
+#define setenv(a, b, c) _putenv_s(a, b)
+#endif
+
+#define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0]))
+
+void usage()
+{
+	printf("help!!\n");
+	exit(-1);
+}
+
+struct Options
+{
+	char* connection;         /**< connection to system under test. */
+	char** haconnections;
+	int hacount;
+	int verbose;
+	int test_no;
+	int MQTTVersion;
+	int iterations;
+} options =
+{
+	"tcp://m2m.eclipse.org:1883",
+	NULL,
+	0,
+	0,
+	0,
+	MQTTVERSION_DEFAULT,
+	1,
+};
+
+void getopts(int argc, char** argv)
+{
+	int count = 1;
+	
+	while (count < argc)
+	{
+		if (strcmp(argv[count], "--test_no") == 0)
+		{
+			if (++count < argc)
+				options.test_no = atoi(argv[count]);
+			else
+				usage();
+		}
+		else if (strcmp(argv[count], "--connection") == 0)
+		{
+			if (++count < argc)
+			{
+				options.connection = argv[count];
+				printf("\nSetting connection to %s\n", options.connection);
+			}
+			else
+				usage();
+		}
+		else if (strcmp(argv[count], "--haconnections") == 0)
+		{
+			if (++count < argc)
+			{
+				char* tok = strtok(argv[count], " ");
+				options.hacount = 0;
+				options.haconnections = malloc(sizeof(char*) * 5);
+				while (tok)
+				{
+					options.haconnections[options.hacount] = malloc(strlen(tok) + 1);
+					strcpy(options.haconnections[options.hacount], tok);
+					options.hacount++;
+					tok = strtok(NULL, " ");
+				}
+			}
+			else
+				usage();
+		}
+		else if (strcmp(argv[count], "--MQTTversion") == 0)
+		{
+			if (++count < argc)
+			{
+				options.MQTTVersion = atoi(argv[count]);
+				printf("setting MQTT version to %d\n", options.MQTTVersion);
+			}
+			else
+				usage();
+		}
+		else if (strcmp(argv[count], "--iterations") == 0)
+		{
+			if (++count < argc)
+				options.iterations = atoi(argv[count]);
+			else
+				usage();
+		}
+		else if (strcmp(argv[count], "--verbose") == 0)
+		{
+			options.verbose = 1;
+			printf("\nSetting verbose on\n");
+		}
+		count++;
+	}
+}
+
+
+#define LOGA_DEBUG 0
+#define LOGA_INFO 1
+#include <stdarg.h>
+#include <time.h>
+#include <sys/timeb.h>
+void MyLog(int LOGA_level, char* format, ...)
+{
+	static char msg_buf[256];
+	va_list args;
+	struct timeb ts;
+
+	struct tm *timeinfo;
+
+	if (LOGA_level == LOGA_DEBUG && options.verbose == 0)
+	  return;
+	
+	ftime(&ts);
+	timeinfo = localtime(&ts.time);
+	strftime(msg_buf, 80, "%Y%m%d %H%M%S", timeinfo);
+
+	sprintf(&msg_buf[strlen(msg_buf)], ".%.3hu ", ts.millitm);
+
+	va_start(args, format);
+	vsnprintf(&msg_buf[strlen(msg_buf)], sizeof(msg_buf) - strlen(msg_buf), format, args);
+	va_end(args);
+
+	printf("%s\n", msg_buf);
+	fflush(stdout);
+}
+
+
+#if defined(WIN32) || defined(_WINDOWS)
+#define mqsleep(A) Sleep(1000*A)
+#define START_TIME_TYPE DWORD
+static DWORD start_time = 0;
+START_TIME_TYPE start_clock(void)
+{
+	return GetTickCount();
+}
+#elif defined(AIX)
+#define mqsleep sleep
+#define START_TIME_TYPE struct timespec
+START_TIME_TYPE start_clock(void)
+{
+	static struct timespec start;
+	clock_gettime(CLOCK_REALTIME, &start);
+	return start;
+}
+#else
+#define mqsleep sleep
+#define START_TIME_TYPE struct timeval
+/* TODO - unused - remove? static struct timeval start_time; */
+START_TIME_TYPE start_clock(void)
+{
+	struct timeval start_time;
+	gettimeofday(&start_time, NULL);
+	return start_time;
+}
+#endif
+
+
+#if defined(WIN32)
+long elapsed(START_TIME_TYPE start_time)
+{
+	return GetTickCount() - start_time;
+}
+#elif defined(AIX)
+#define assert(a)
+long elapsed(struct timespec start)
+{
+	struct timespec now, res;
+
+	clock_gettime(CLOCK_REALTIME, &now);
+	ntimersub(now, start, res);
+	return (res.tv_sec)*1000L + (res.tv_nsec)/1000000L;
+}
+#else
+long elapsed(START_TIME_TYPE start_time)
+{
+	struct timeval now, res;
+
+	gettimeofday(&now, NULL);
+	timersub(&now, &start_time, &res);
+	return (res.tv_sec)*1000 + (res.tv_usec)/1000;
+}
+#endif
+
+
+#define assert(a, b, c, d) myassert(__FILE__, __LINE__, a, b, c, d)
+#define assert1(a, b, c, d, e) myassert(__FILE__, __LINE__, a, b, c, d, e)
+
+int tests = 0;
+int failures = 0;
+FILE* xml;
+START_TIME_TYPE global_start_time;
+char output[3000];
+char* cur_output = output;
+
+
+void write_test_result()
+{
+	long duration = elapsed(global_start_time);
+
+	fprintf(xml, " time=\"%ld.%.3ld\" >\n", duration / 1000, duration % 1000); 
+	if (cur_output != output)
+	{
+		fprintf(xml, "%s", output);
+		cur_output = output;	
+	}
+	fprintf(xml, "</testcase>\n");
+}
+
+
+void myassert(char* filename, int lineno, char* description, int value, char* format, ...)
+{
+	++tests;
+	if (!value)
+	{
+		va_list args;
+
+		++failures;
+		MyLog(LOGA_INFO, "Assertion failed, file %s, line %d, description: %s\n", filename, lineno, description);
+
+		va_start(args, format);
+		vprintf(format, args);
+		va_end(args);
+
+		cur_output += sprintf(cur_output, "<failure type=\"%s\">file %s, line %d </failure>\n", 
+                        description, filename, lineno);
+	}
+    else
+    	MyLog(LOGA_DEBUG, "Assertion succeeded, file %s, line %d, description: %s", filename, lineno, description);  
+}
+
+
+/*********************************************************************
+
+Test1: multiple threads to single client object
+
+*********************************************************************/
+volatile int test1_arrivedcount = 0;
+volatile int test1_arrivedcount_qos[3] = {0, 0, 0};
+int test1_deliveryCompleted = 0;
+MQTTClient_message test1_pubmsg_check = MQTTClient_message_initializer;
+
+
+void test1_deliveryComplete(void* context, MQTTClient_deliveryToken dt)
+{
+	++test1_deliveryCompleted;
+}
+
+int test1_messageArrived(void* context, char* topicName, int topicLen, MQTTClient_message* m)
+{
+	++(test1_arrivedcount_qos[m->qos]);
+	++test1_arrivedcount;
+
+	printf("messageArrived qos %d, count %d\n", m->qos, test1_arrivedcount_qos[m->qos]);
+	MyLog(LOGA_DEBUG, "messageArrived: %d message received on topic %s is %.*s.",
+					test1_arrivedcount, topicName, m->payloadlen, (char*)(m->payload));
+	if (test1_pubmsg_check.payloadlen != m->payloadlen ||
+					memcmp(m->payload, test1_pubmsg_check.payload, m->payloadlen) != 0)
+	{
+		failures++;
+		MyLog(LOGA_INFO, "Error: wrong data received lengths %d %d\n", test1_pubmsg_check.payloadlen, m->payloadlen);
+	}
+	MQTTClient_free(topicName);
+	MQTTClient_freeMessage(&m);
+	return 1;
+}
+
+
+struct thread_parms
+{
+	MQTTClient* c;
+	int qos;
+	char* test_topic;
+};
+
+
+thread_return_type WINAPI test1_sendAndReceive(void* n)
+{
+	MQTTClient_deliveryToken dt;
+	int i = 0;
+	int iterations = 50;
+	int rc = 0;
+	int wait_seconds = 0;
+	MQTTClient_message test1_pubmsg = MQTTClient_message_initializer;
+	int subsqos = 2;
+
+	struct thread_parms *parms = n;
+	MQTTClient* c = parms->c;
+	int qos = parms->qos;
+	char* test_topic = parms->test_topic;
+
+	rc = MQTTClient_subscribe(c, test_topic, subsqos);
+	assert("Good rc from subscribe", rc == MQTTCLIENT_SUCCESS, "rc was %d", rc);
+
+	MyLog(LOGA_INFO, "Thread %d, %d messages at QoS %d", Thread_getid(), iterations, qos);
+	test1_pubmsg.payload = test1_pubmsg_check.payload;
+	test1_pubmsg.payloadlen = test1_pubmsg_check.payloadlen;
+	test1_pubmsg.retained = 0;
+	test1_pubmsg.qos = qos;
+	test1_deliveryCompleted = 0;
+
+	for (i = 1; i <= iterations; ++i)
+	{
+		if (i % 10 == 0)
+			rc = MQTTClient_publish(c, test_topic, test1_pubmsg.payloadlen, test1_pubmsg.payload,
+                   qos, test1_pubmsg.retained, NULL);
+		else
+			rc = MQTTClient_publishMessage(c, test_topic, &test1_pubmsg, &dt);
+		printf("published %d at qos %d %d\n", i, qos, parms->qos);
+		assert("Good rc from publish", rc == MQTTCLIENT_SUCCESS, "rc was %d", rc);
+
+		#if defined(WIN32)
+			Sleep(100);
+		#else
+			usleep(100000L);
+		#endif
+
+		wait_seconds = 10;
+		while ((test1_arrivedcount_qos[qos] < i) && (wait_seconds-- > 0))
+		{
+			MyLog(LOGA_DEBUG, "Arrived %d count %d", test1_arrivedcount_qos[qos], i);
+			#if defined(WIN32)
+				Sleep(1000);
+			#else
+				usleep(1000000L);
+			#endif
+		}
+		assert("Message Arrived", wait_seconds > 0,
+				"Timed out waiting for message %d\n", i);
+	}
+	if (qos > 0)
+	{
+		/* MQ Telemetry can send a message to a subscriber before the server has
+		   completed the QoS 2 handshake with the publisher. For QoS 1 and 2,
+		   allow time for the final delivery complete callback before checking
+		   that all expected callbacks have been made */
+		wait_seconds = 10;
+		while ((test1_deliveryCompleted < iterations*2) && (wait_seconds-- > 0))
+		{
+			MyLog(LOGA_DEBUG, "Delivery Completed %d count %d", test1_deliveryCompleted, i);
+			#if defined(WIN32)
+				Sleep(100);
+			#else
+				usleep(100000L);
+			#endif
+		}
+		assert("All Deliveries Complete", wait_seconds > 0,
+			   "Number of deliveryCompleted callbacks was %d\n",
+			   test1_deliveryCompleted);
+	}
+	return NULL;
+}
+
+
+int test1(struct Options options)
+{
+	MQTTClient c;
+	MQTTClient_connectOptions opts = MQTTClient_connectOptions_initializer;
+	MQTTClient_willOptions wopts = MQTTClient_willOptions_initializer;
+	int rc = 0;
+	char* test_topic = "C client test1";
+
+	fprintf(xml, "<testcase classname=\"test1\" name=\"multiple threads using same client object\"");
+	global_start_time = start_clock();
+	failures = 0;
+	MyLog(LOGA_INFO, "Starting test 1 - multiple threads using same client object");
+	
+	rc = MQTTClient_create(&c, options.connection, "single_object, multiple threads",
+			MQTTCLIENT_PERSISTENCE_NONE, NULL);
+	assert("good rc from create",  rc == MQTTCLIENT_SUCCESS, "rc was %d\n", rc);
+	if (rc != MQTTCLIENT_SUCCESS)
+	{
+		MQTTClient_destroy(&c);
+		goto exit;
+	}
+
+	rc = MQTTClient_setCallbacks(c, NULL, NULL, test1_messageArrived, test1_deliveryComplete);
+	assert("Good rc from setCallbacks", rc == MQTTCLIENT_SUCCESS, "rc was %d", rc);
+
+	opts.keepAliveInterval = 20;
+	opts.cleansession = 1;
+	opts.username = "testuser";
+	opts.password = "testpassword";
+	opts.MQTTVersion = options.MQTTVersion;
+	if (options.haconnections != NULL)
+	{
+		opts.serverURIs = options.haconnections;
+		opts.serverURIcount = options.hacount;
+	}
+
+	opts.will = &wopts;
+	opts.will->message = "will message";
+	opts.will->qos = 1;
+	opts.will->retained = 0;
+	opts.will->topicName = "will topic";
+	opts.will = NULL;
+
+	MyLog(LOGA_DEBUG, "Connecting");
+	rc = MQTTClient_connect(c, &opts);
+	assert("Good rc from connect", rc == MQTTCLIENT_SUCCESS, "rc was %d", rc);
+	if (rc != MQTTCLIENT_SUCCESS)
+		goto exit;
+
+	test1_pubmsg_check.payload = "a much longer message that we can shorten to the extent that we need to";
+	test1_pubmsg_check.payloadlen = 27;
+
+	struct thread_parms parms0 = {c, 0, test_topic};
+	Thread_start(test1_sendAndReceive, (void*)&parms0);
+
+	struct thread_parms parms1 = {c, 1, test_topic};
+	Thread_start(test1_sendAndReceive, (void*)&parms1);
+
+	struct thread_parms parms2 = {c, 2, test_topic};
+	Thread_start(test1_sendAndReceive, (void*)&parms2);
+
+	int wait_seconds = 90;
+	while ((test1_arrivedcount < 150) && (wait_seconds-- > 0))
+	{
+		#if defined(WIN32)
+			Sleep(1000);
+		#else
+			usleep(1000000L);
+		#endif
+	}
+	assert("Arrived count == 150", test1_arrivedcount == 150, "arrivedcount was %d", test1_arrivedcount);
+
+	MyLog(LOGA_DEBUG, "Stopping\n");
+
+	rc = MQTTClient_unsubscribe(c, test_topic);
+	assert("Unsubscribe successful", rc == MQTTCLIENT_SUCCESS, "rc was %d", rc);
+	rc = MQTTClient_disconnect(c, 0);
+	assert("Disconnect successful", rc == MQTTCLIENT_SUCCESS, "rc was %d", rc);
+
+	/* Just to make sure we can connect again */
+	rc = MQTTClient_connect(c, &opts);
+	assert("Connect successful",  rc == MQTTCLIENT_SUCCESS, "rc was %d", rc);
+	rc = MQTTClient_disconnect(c, 0);
+	assert("Disconnect successful", rc == MQTTCLIENT_SUCCESS, "rc was %d", rc);
+
+	MQTTClient_destroy(&c);
+
+exit:
+	MyLog(LOGA_INFO, "TEST1: test %s. %d tests run, %d failures.",
+			(failures == 0) ? "passed" : "failed", tests, failures);
+	write_test_result();
+	return failures;
+}
+
+
+/*********************************************************************
+
+Test2: multiple client objects used from multiple threads
+
+*********************************************************************/
+volatile int test2_arrivedcount = 0;
+int test2_deliveryCompleted = 0;
+MQTTClient_message test2_pubmsg = MQTTClient_message_initializer;
+
+void test2_deliveryComplete(void* context, MQTTClient_deliveryToken dt)
+{
+	++test2_deliveryCompleted;
+}
+
+int test2_messageArrived(void* context, char* topicName, int topicLen, MQTTClient_message* m)
+{
+	++test2_arrivedcount;
+	MyLog(LOGA_DEBUG, "Callback: %d message received on topic %s is %.*s.",
+					test2_arrivedcount, topicName, m->payloadlen, (char*)(m->payload));
+	if (test2_pubmsg.payloadlen != m->payloadlen ||
+					memcmp(m->payload, test2_pubmsg.payload, m->payloadlen) != 0)
+	{
+		failures++;
+		MyLog(LOGA_INFO, "Error: wrong data received lengths %d %d\n", test2_pubmsg.payloadlen, m->payloadlen);
+	}
+	MQTTClient_free(topicName);
+	MQTTClient_freeMessage(&m);
+	return 1;
+}
+
+
+void test2_sendAndReceive(MQTTClient* c, int qos, char* test_topic)
+{
+	MQTTClient_deliveryToken dt;
+	int i = 0;
+	int iterations = 50;
+	int rc = 0;
+	int wait_seconds = 0;
+
+	test2_deliveryCompleted = 0;
+
+	MyLog(LOGA_INFO, "%d messages at QoS %d", iterations, qos);
+	test2_pubmsg.payload = "a much longer message that we can shorten to the extent that we need to";
+	test2_pubmsg.payloadlen = 27;
+	test2_pubmsg.qos = qos;
+	test2_pubmsg.retained = 0;
+
+	for (i = 1; i <= iterations; ++i)
+	{
+		if (i % 10 == 0)
+			rc = MQTTClient_publish(c, test_topic, test2_pubmsg.payloadlen, test2_pubmsg.payload, 
+                   test2_pubmsg.qos, test2_pubmsg.retained, NULL);
+		else
+			rc = MQTTClient_publishMessage(c, test_topic, &test2_pubmsg, &dt);
+		assert("Good rc from publish", rc == MQTTCLIENT_SUCCESS, "rc was %d", rc);
+
+		#if defined(WIN32)
+			Sleep(100);
+		#else
+			usleep(100000L);
+		#endif
+
+		wait_seconds = 10;
+		while ((test2_arrivedcount < i) && (wait_seconds-- > 0))
+		{
+			MyLog(LOGA_DEBUG, "Arrived %d count %d", test2_arrivedcount, i);
+			#if defined(WIN32)
+				Sleep(1000);
+			#else
+				usleep(1000000L);
+			#endif
+		}
+		assert("Message Arrived", wait_seconds > 0, 
+				"Time out waiting for message %d\n", i );
+	}
+	if (qos > 0)
+	{
+		/* MQ Telemetry can send a message to a subscriber before the server has
+		   completed the QoS 2 handshake with the publisher. For QoS 1 and 2,
+		   allow time for the final delivery complete callback before checking
+		   that all expected callbacks have been made */ 
+		wait_seconds = 10;
+		while ((test2_deliveryCompleted < iterations) && (wait_seconds-- > 0))
+		{
+			MyLog(LOGA_DEBUG, "Delivery Completed %d count %d", test2_deliveryCompleted, i);
+			#if defined(WIN32)
+				Sleep(1000);
+			#else
+				usleep(1000000L);
+			#endif
+		}
+		assert("All Deliveries Complete", wait_seconds > 0, 
+			   "Number of deliveryCompleted callbacks was %d\n", 
+			   test2_deliveryCompleted);
+	}
+}
+
+
+int test2(struct Options options)
+{
+	char* testname = "test2";
+	int subsqos = 2;
+	/* TODO - usused - remove ? MQTTClient_deliveryToken* dt = NULL; */
+	MQTTClient c;
+	MQTTClient_connectOptions opts = MQTTClient_connectOptions_initializer;
+	int rc = 0;
+	char* test_topic = "C client test2";
+
+	fprintf(xml, "<testcase classname=\"test1\" name=\"multi-threaded client using callbacks\"");
+	MyLog(LOGA_INFO, "Starting test 2 - multi-threaded client using callbacks");
+	global_start_time = start_clock();
+	failures = 0;
+
+	MQTTClient_create(&c, options.connection, "multi_threaded_sample", MQTTCLIENT_PERSISTENCE_NONE, NULL);
+
+	opts.keepAliveInterval = 20;
+	opts.cleansession = 1;
+	opts.MQTTVersion = options.MQTTVersion;
+	if (options.haconnections != NULL)
+	{
+		opts.serverURIs = options.haconnections;
+		opts.serverURIcount = options.hacount;
+	}
+
+	rc = MQTTClient_setCallbacks(c, NULL, NULL, test2_messageArrived, test2_deliveryComplete);
+	assert("Good rc from setCallbacks", rc == MQTTCLIENT_SUCCESS, "rc was %d", rc);
+
+	MyLog(LOGA_DEBUG, "Connecting");
+	rc = MQTTClient_connect(c, &opts);
+	assert("Good rc from connect", rc == MQTTCLIENT_SUCCESS, "rc was %d", rc);
+	if (rc != MQTTCLIENT_SUCCESS)
+		goto exit;
+
+	rc = MQTTClient_subscribe(c, test_topic, subsqos);
+	assert("Good rc from subscribe", rc == MQTTCLIENT_SUCCESS, "rc was %d", rc);
+
+	test2_sendAndReceive(c, 0, test_topic);
+	test2_sendAndReceive(c, 1, test_topic);
+	test2_sendAndReceive(c, 2, test_topic);
+
+	MyLog(LOGA_DEBUG, "Stopping");
+
+	rc = MQTTClient_unsubscribe(c, test_topic);
+	assert("Unsubscribe successful", rc == MQTTCLIENT_SUCCESS, "rc was %d", rc);
+	rc = MQTTClient_disconnect(c, 0);
+	assert("Disconnect successful", rc == MQTTCLIENT_SUCCESS, "rc was %d", rc);
+
+	MQTTClient_destroy(&c);
+
+exit:
+	MyLog(LOGA_INFO, "%s: test %s. %d tests run, %d failures.",
+			(failures == 0) ? "passed" : "failed", testname, tests, failures);
+	write_test_result();
+	return failures;
+}
+
+
+int main(int argc, char** argv)
+{
+	int rc = 0;
+ 	int (*tests[])() = {NULL, test1};
+	int i;
+	
+	xml = fopen("TEST-test2.xml", "w");
+	fprintf(xml, "<testsuite name=\"test1\" tests=\"%d\">\n", (int)(ARRAY_SIZE(tests) - 1));
+
+	setenv("MQTT_C_CLIENT_TRACE", "ON", 1);
+	setenv("MQTT_C_CLIENT_TRACE_LEVEL", "ERROR", 1);
+
+	getopts(argc, argv);
+
+	for (i = 0; i < options.iterations; ++i)
+	{
+	 	if (options.test_no == 0)
+		{ /* run all the tests */
+ 		   	for (options.test_no = 1; options.test_no < ARRAY_SIZE(tests); ++options.test_no)
+				rc += tests[options.test_no](options); /* return number of failures.  0 = test succeeded */
+		}
+		else
+ 		   	rc = tests[options.test_no](options); /* run just the selected test */
+	}
+    	
+ 	if (rc == 0)
+		MyLog(LOGA_INFO, "verdict pass");
+	else
+		MyLog(LOGA_INFO, "verdict fail");
+
+	fprintf(xml, "</testsuite>\n");
+	fclose(xml);	
+	return rc;
+}