Merge remote-tracking branch 'upstream/master'

Doxyfile

@@ -409,13 +409,13 @@ LOOKUP_CACHE_SIZE      = 0
 # normally produced when WARNINGS is set to YES.
 # The default value is: NO.
 
-EXTRACT_ALL            = NO
+EXTRACT_ALL            = YES
 
 # If the EXTRACT_PRIVATE tag is set to YES all private members of a class will
 # be included in the documentation.
 # The default value is: NO.
 
-EXTRACT_PRIVATE        = NO
+EXTRACT_PRIVATE        = YES
 
 # If the EXTRACT_PACKAGE tag is set to YES all members with package or internal
 # scope will be included in the documentation.
@@ -427,7 +427,7 @@ EXTRACT_PACKAGE        = NO
 # included in the documentation.
 # The default value is: NO.
 
-EXTRACT_STATIC         = NO
+EXTRACT_STATIC         = YES
 
 # If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) defined
 # locally in source files will be included in the documentation. If set to NO
@@ -452,7 +452,7 @@ EXTRACT_LOCAL_METHODS  = NO
 # are hidden.
 # The default value is: NO.
 
-EXTRACT_ANON_NSPACES   = NO
+EXTRACT_ANON_NSPACES   = YES
 
 # If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all
 # undocumented members inside documented classes or files. If set to NO these
@@ -1902,7 +1902,7 @@ ENABLE_PREPROCESSING   = YES
 # The default value is: NO.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-MACRO_EXPANSION        = NO
+MACRO_EXPANSION        = YES
 
 # If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
 # the macro expansion is limited to the macros specified with the PREDEFINED and
@@ -1910,7 +1910,7 @@ MACRO_EXPANSION        = NO
 # The default value is: NO.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-EXPAND_ONLY_PREDEF     = NO
+EXPAND_ONLY_PREDEF     = YES
 
 # If the SEARCH_INCLUDES tag is set to YES the includes files in the
 # INCLUDE_PATH will be searched if a #include is found.
@@ -1942,7 +1942,7 @@ INCLUDE_FILE_PATTERNS  =
 # recursively expanded use the := operator instead of the = operator.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-PREDEFINED             =
+PREDEFINED             = "BLOCKCHAIN_DB=2" \ # DB_LMDB
 
 # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
 # tag can be used to specify a list of macro names that should be expanded. The

contrib/epee/include/net/abstract_tcp_server2.inl

@@ -1005,6 +1005,12 @@ POP_WARNINGS
     while(local_shared_context->ec == boost::asio::error::would_block)
     {
       bool r = local_shared_context->cond.timed_wait(lock, boost::get_system_time() + boost::posix_time::milliseconds(conn_timeout));
+      if (m_stop_signal_sent)
+      {
+        if (sock_.is_open())
+          sock_.close();
+        return false;
+      }
       if(local_shared_context->ec == boost::asio::error::would_block && !r)
       {
         //timeout
@@ -1018,6 +1024,8 @@ POP_WARNINGS
     if (ec || !sock_.is_open())
     {
       _dbg3("Some problems at connect, message: " << ec.message());
+      if (sock_.is_open())
+        sock_.close();
       return false;
     }
 

contrib/epee/include/storages/portable_storage_template_helper.h

@@ -72,7 +72,7 @@ namespace epee
     {
       std::string json_buff;
       store_t_to_json(str_in, json_buff, indent, insert_newlines);
-      return std::move(json_buff);
+      return json_buff;
     }
     //-----------------------------------------------------------------------------------------------------------
     template<class t_struct>
@@ -117,7 +117,7 @@ namespace epee
     {
       std::string binary_buff;
       store_t_to_binary(str_in, binary_buff, indent);
-      return std::move(binary_buff);
+      return binary_buff;
     }
   }
 }

external/CMakeLists.txt

@@ -37,9 +37,6 @@
 
 find_package(Miniupnpc QUIET)
 
-# FreeBSD doesn't play well with the local copy, so default to using shared
-set(USE_SHARED_MINIUPNPC false)
-
 # If we have the correct shared version and we're not building static, use it
 if(STATIC)
  set(USE_SHARED_MINIUPNPC false)

external/miniupnpc/CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required (VERSION 2.6)
 
 project (miniupnpc C)
 set (MINIUPNPC_VERSION 1.9)
-set (MINIUPNPC_API_VERSION 15)
+set (MINIUPNPC_API_VERSION 16)
 
 # - we comment out this block as we don't support these other build types
 if(0)
@@ -33,7 +33,7 @@ endif (NO_GETADDRINFO)
 
 if (NOT WIN32)
   add_definitions (-DMINIUPNPC_SET_SOCKET_TIMEOUT)
-  add_definitions (-D_BSD_SOURCE -D_DEFAULT_SOURCE -D_POSIX_C_SOURCE=200112L)
+  add_definitions (-D_BSD_SOURCE -D_DEFAULT_SOURCE)
 else (NOT WIN32)
   add_definitions (-D_WIN32_WINNT=0x0501) # XP or higher for getnameinfo and friends
 endif (NOT WIN32)

external/miniupnpc/Changelog.txt

@@ -1,6 +1,14 @@
-$Id: Changelog.txt,v 1.219 2015/10/26 17:05:06 nanard Exp $
+$Id: Changelog.txt,v 1.222 2016/01/24 17:24:35 nanard Exp $
 miniUPnP client Changelog.
 
+2016/01/24:
+  change miniwget to return HTTP status code
+  increments API_VERSION to 16
+
+2016/01/22:
+  Improve UPNPIGD_IsConnected() to check if WAN address is not private.
+  parse HTTP response status line in miniwget.c
+
 2015/10/26:
   snprintf() overflow check. check overflow in simpleUPnPcommand2()
 

external/miniupnpc/Makefile

@@ -1,4 +1,4 @@
-# $Id: Makefile,v 1.126 2015/08/28 12:14:18 nanard Exp $
+# $Id: Makefile,v 1.133 2016/01/24 17:24:35 nanard Exp $
 # MiniUPnP Project
 # http://miniupnp.free.fr/
 # http://miniupnp.tuxfamily.org/
@@ -66,7 +66,7 @@ ifeq (SunOS, $(OS))
 endif
 
 # APIVERSION is used to build SONAME
-APIVERSION = 15
+APIVERSION = 16
 
 SRCS = igd_desc_parse.c miniupnpc.c minixml.c minisoap.c miniwget.c \
        upnpc.c upnpcommands.c upnpreplyparse.c testminixml.c \

external/miniupnpc/Makefile.mingw

@@ -18,7 +18,8 @@ OBJS=miniwget.o minixml.o igd_desc_parse.o minisoap.o \
      upnpdev.o
 OBJSDLL=$(addprefix dll/, $(OBJS))
 
-all:	init upnpc-static upnpc-shared testminixml libminiupnpc.a miniupnpc.dll
+all:	init upnpc-static upnpc-shared testminixml libminiupnpc.a \
+	miniupnpc.dll listdevices
 
 init:
 	mkdir dll
@@ -66,6 +67,9 @@ upnpc-static:	upnpc.o libminiupnpc.a
 upnpc-shared:	dll/upnpc.o miniupnpc.lib
 	$(CC) -o $@ $^ $(LDLIBS)
 
+listdevices: listdevices.o libminiupnpc.a
+	$(CC) -o $@ $^ $(LDLIBS)
+
 wingenminiupnpcstrings:	wingenminiupnpcstrings.o
 
 wingenminiupnpcstrings.o:	wingenminiupnpcstrings.c

external/miniupnpc/apiversions.txt

@@ -1,7 +1,12 @@
-$Id: apiversions.txt,v 1.7 2015/07/23 20:40:08 nanard Exp $
+$Id: apiversions.txt,v 1.9 2016/01/24 17:24:36 nanard Exp $
 
 Differences in API between miniUPnPc versions
 
+API version 16
+  added "status_code" argument to getHTTPResponse(), miniwget() and miniwget_getaddr()
+  updated macro :
+    #define MINIUPNPC_API_VERSION 16
+
 API version 15
   changed "sameport" argument of upnpDiscover() upnpDiscoverAll() upnpDiscoverDevice()
     to "localport". When 0 or 1, behaviour is not changed, but it can take

external/miniupnpc/minissdpc.c

@@ -1,5 +1,6 @@
 /* $Id: minissdpc.c,v 1.28 2015/09/18 13:05:39 nanard Exp $ */
-/* Project : miniupnp
+/* vim: tabstop=4 shiftwidth=4 noexpandtab
+ * Project : miniupnp
  * Web : http://miniupnp.free.fr/
  * Author : Thomas BERNARD
  * copyright (c) 2005-2015 Thomas Bernard
@@ -67,6 +68,10 @@ struct sockaddr_un {
 #define HAS_IP_MREQN
 #endif
 
+#if !defined(HAS_IP_MREQN) && !defined(_WIN32)
+#include <sys/ioctl.h>
+#endif
+
 #if defined(HAS_IP_MREQN) && defined(NEED_STRUCT_IP_MREQN)
 /* Several versions of glibc don't define this structure,
  * define it here and compile with CFLAGS NEED_STRUCT_IP_MREQN */
@@ -647,11 +652,25 @@ ssdpDiscoverDevices(const char * const deviceTypes[],
 				{
 					PRINT_SOCKET_ERROR("setsockopt");
 				}
-#else
+#elif !defined(_WIN32)
+				struct ifreq ifr;
+				int ifrlen = sizeof(ifr);
+				strncpy(ifr.ifr_name, multicastif, IFNAMSIZ);
+				ifr.ifr_name[IFNAMSIZ-1] = '\0';
+				if(ioctl(sudp, SIOCGIFADDR, &ifr, &ifrlen) < 0)
+				{
+					PRINT_SOCKET_ERROR("ioctl(...SIOCGIFADDR...)");
+				}
+				mc_if.s_addr = ((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr.s_addr;
+				if(setsockopt(sudp, IPPROTO_IP, IP_MULTICAST_IF, (const char *)&mc_if, sizeof(mc_if)) < 0)
+				{
+					PRINT_SOCKET_ERROR("setsockopt");
+				}
+#else /* _WIN32 */
 #ifdef DEBUG
 				printf("Setting of multicast interface not supported with interface name.\n");
 #endif
-#endif
+#endif /* #ifdef HAS_IP_MREQN / !defined(_WIN32) */
 			}
 		}
 	}

external/miniupnpc/miniupnpc.c

@@ -1,9 +1,9 @@
-/* $Id: miniupnpc.c,v 1.135 2015/07/23 20:40:08 nanard Exp $ */
+/* $Id: miniupnpc.c,v 1.148 2016/01/24 17:24:36 nanard Exp $ */
-/* vim: tabstop=4 shiftwidth=4 noexpandtab */
+/* vim: tabstop=4 shiftwidth=4 noexpandtab
-/* Project : miniupnp
+ * Project : miniupnp
  * Web : http://miniupnp.free.fr/
  * Author : Thomas BERNARD
- * copyright (c) 2005-2015 Thomas Bernard
+ * copyright (c) 2005-2016 Thomas Bernard
  * This software is subjet to the conditions detailed in the
  * provided LICENSE file. */
 #include <stdlib.h>
@@ -72,6 +72,25 @@
 #define SERVICEPREFIX "u"
 #define SERVICEPREFIX2 'u'
 
+/* check if an ip address is a private (LAN) address
+ * see https://tools.ietf.org/html/rfc1918 */
+static int is_rfc1918addr(const char * addr)
+{
+	/* 192.168.0.0     -   192.168.255.255 (192.168/16 prefix) */
+	if(COMPARE(addr, "192.168."))
+		return 1;
+	/* 10.0.0.0        -   10.255.255.255  (10/8 prefix) */
+	if(COMPARE(addr, "10."))
+		return 1;
+	/* 172.16.0.0      -   172.31.255.255  (172.16/12 prefix) */
+	if(COMPARE(addr, "172.")) {
+		int i = atoi(addr + 4);
+		if((16 <= i) && (i <= 31))
+			return 1;
+	}
+	return 0;
+}
+
 /* root description parsing */
 MINIUPNP_LIBSPEC void parserootdesc(const char * buffer, int bufsize, struct IGDdatas * data)
 {
@@ -107,6 +126,7 @@ char * simpleUPnPcommand2(int s, const char * url, const char * service,
 	int soapbodylen;
 	char * buf;
 	int n;
+	int status_code;
 
 	*bufsize = 0;
 	snprintf(soapact, sizeof(soapact), "%s#%s", service, action);
@@ -210,11 +230,15 @@ char * simpleUPnPcommand2(int s, const char * url, const char * service,
 		return NULL;
 	}
 
-	buf = getHTTPResponse(s, bufsize);
+	buf = getHTTPResponse(s, bufsize, &status_code);
 #ifdef DEBUG
 	if(*bufsize > 0 && buf)
 	{
-		printf("SOAP Response :\n%.*s\n", *bufsize, buf);
+		printf("HTTP %d SOAP Response :\n%.*s\n", status_code, *bufsize, buf);
+	}
+	else
+	{
+		printf("HTTP %d, empty SOAP response. size=%d\n", status_code, *bufsize);
 	}
 #endif
 	closesocket(s);
@@ -526,7 +550,7 @@ UPNPIGD_IsConnected(struct UPNPUrls * urls, struct IGDdatas * data)
  *     3 = an UPnP device has been found but was not recognized as an IGD
  *
  * In any positive non zero return case, the urls and data structures
- * passed as parameters are set. Donc forget to call FreeUPNPUrls(urls) to
+ * passed as parameters are set. Dont forget to call FreeUPNPUrls(urls) to
  * free allocated memory.
  */
 MINIUPNP_LIBSPEC int
@@ -547,6 +571,8 @@ UPNP_GetValidIGD(struct UPNPDev * devlist,
 	int n_igd = 0;
 	char extIpAddr[16];
 	char myLanAddr[40];
+	int status_code = -1;
+
 	if(!devlist)
 	{
 #ifdef DEBUG
@@ -570,7 +596,7 @@ UPNP_GetValidIGD(struct UPNPDev * devlist,
 		 * with st == urn:schemas-upnp-org:device:InternetGatewayDevice:1 */
 		desc[i].xml = miniwget_getaddr(dev->descURL, &(desc[i].size),
 		                               myLanAddr, sizeof(myLanAddr),
-		                               dev->scope_id);
+		                               dev->scope_id, &status_code);
 #ifdef DEBUG
 		if(!desc[i].xml)
 		{
@@ -604,20 +630,25 @@ UPNP_GetValidIGD(struct UPNPDev * devlist,
 				parserootdesc(desc[i].xml, desc[i].size, data);
 				if(desc[i].is_igd || state >= 3 )
 				{
+				  int is_connected;
+
 				  GetUPNPUrls(urls, data, dev->descURL, dev->scope_id);
 
 				  /* in state 2 and 3 we dont test if device is connected ! */
 				  if(state >= 2)
 				    goto free_and_return;
+				  is_connected = UPNPIGD_IsConnected(urls, data);
 #ifdef DEBUG
 				  printf("UPNPIGD_IsConnected(%s) = %d\n",
-				     urls->controlURL,
+				     urls->controlURL, is_connected);
-			         UPNPIGD_IsConnected(urls, data));
 #endif
 				  /* checks that status is connected AND there is a external IP address assigned */
-				  if(UPNPIGD_IsConnected(urls, data)
+				  if(is_connected &&
-				     && (UPNP_GetExternalIPAddress(urls->controlURL,  data->first.servicetype, extIpAddr) == 0))
+				     (UPNP_GetExternalIPAddress(urls->controlURL,  data->first.servicetype, extIpAddr) == 0)) {
+					if(!is_rfc1918addr(extIpAddr) && (extIpAddr[0] != '\0')
+					   && (0 != strcmp(extIpAddr, "0.0.0.0")))
 					  goto free_and_return;
+				  }
 				  FreeUPNPUrls(urls);
 				  if(data->second.servicetype[0] != '\0') {
 #ifdef DEBUG
@@ -629,14 +660,17 @@ UPNP_GetValidIGD(struct UPNPDev * devlist,
 				    memcpy(&data->first, &data->second, sizeof(struct IGDdatas_service));
 				    memcpy(&data->second, &data->tmp, sizeof(struct IGDdatas_service));
 				    GetUPNPUrls(urls, data, dev->descURL, dev->scope_id);
+				    is_connected = UPNPIGD_IsConnected(urls, data);
 #ifdef DEBUG
 				    printf("UPNPIGD_IsConnected(%s) = %d\n",
-				       urls->controlURL,
+				       urls->controlURL, is_connected);
-			           UPNPIGD_IsConnected(urls, data));
 #endif
-				    if(UPNPIGD_IsConnected(urls, data)
+				    if(is_connected &&
-				       && (UPNP_GetExternalIPAddress(urls->controlURL,  data->first.servicetype, extIpAddr) == 0))
+				       (UPNP_GetExternalIPAddress(urls->controlURL,  data->first.servicetype, extIpAddr) == 0)) {
+					  if(!is_rfc1918addr(extIpAddr) && (extIpAddr[0] != '\0')
+					     && (0 != strcmp(extIpAddr, "0.0.0.0")))
 					    goto free_and_return;
+				    }
 				    FreeUPNPUrls(urls);
 				  }
 				}
@@ -670,8 +704,9 @@ UPNP_GetIGDFromUrl(const char * rootdescurl,
 {
 	char * descXML;
 	int descXMLsize = 0;
+
 	descXML = miniwget_getaddr(rootdescurl, &descXMLsize,
-	   	                       lanaddr, lanaddrlen, 0);
+	                           lanaddr, lanaddrlen, 0, NULL);
 	if(descXML) {
 		memset(data, 0, sizeof(struct IGDdatas));
 		memset(urls, 0, sizeof(struct UPNPUrls));

external/miniupnpc/miniupnpc.h

@@ -1,8 +1,8 @@
-/* $Id: miniupnpc.h,v 1.44 2015/07/23 20:40:10 nanard Exp $ */
+/* $Id: miniupnpc.h,v 1.49 2016/01/24 17:24:36 nanard Exp $ */
 /* Project: miniupnp
  * http://miniupnp.free.fr/
  * Author: Thomas Bernard
- * Copyright (c) 2005-2015 Thomas Bernard
+ * Copyright (c) 2005-2016 Thomas Bernard
  * This software is subjects to the conditions detailed
  * in the LICENCE file provided within this distribution */
 #ifndef MINIUPNPC_H_INCLUDED
@@ -20,7 +20,7 @@
 
 /* versions : */
 #define MINIUPNPC_VERSION	"1.9"
-#define MINIUPNPC_API_VERSION	15
+#define MINIUPNPC_API_VERSION	16
 
 /* Source port:
    Using "1" as an alias for 1900 for backwards compatability

external/miniupnpc/miniwget.c

@@ -1,8 +1,8 @@
-/* $Id: miniwget.c,v 1.70 2015/07/15 12:41:13 nanard Exp $ */
+/* $Id: miniwget.c,v 1.75 2016/01/24 17:24:36 nanard Exp $ */
 /* Project : miniupnp
  * Website : http://miniupnp.free.fr/
  * Author : Thomas Bernard
- * Copyright (c) 2005-2015 Thomas Bernard
+ * Copyright (c) 2005-2016 Thomas Bernard
  * This software is subject to the conditions detailed in the
  * LICENCE file provided in this distribution. */
 
@@ -65,7 +65,7 @@
  * to the length parameter.
  */
 void *
-getHTTPResponse(int s, int * size)
+getHTTPResponse(int s, int * size, int * status_code)
 {
 	char buf[2048];
 	int n;
@@ -83,7 +83,10 @@ getHTTPResponse(int s, int * size)
 	unsigned int content_buf_used = 0;
 	char chunksize_buf[32];
 	unsigned int chunksize_buf_index;
+	char * reason_phrase = NULL;
+	int reason_phrase_len = 0;
 
+	if(status_code) *status_code = -1;
 	header_buf = malloc(header_buf_len);
 	if(header_buf == NULL)
 	{
@@ -155,7 +158,7 @@ getHTTPResponse(int s, int * size)
 				continue;
 			/* parse header lines */
 			for(i = 0; i < endofheaders - 1; i++) {
-				if(colon <= linestart && header_buf[i]==':')
+				if(linestart > 0 && colon <= linestart && header_buf[i]==':')
 				{
 					colon = i;
 					while(i < (endofheaders-1)
@@ -166,7 +169,29 @@ getHTTPResponse(int s, int * size)
 				/* detecting end of line */
 				else if(header_buf[i]=='\r' || header_buf[i]=='\n')
 				{
-					if(colon > linestart && valuestart > colon)
+					if(linestart == 0 && status_code)
+					{
+						/* Status line
+						 * HTTP-Version SP Status-Code SP Reason-Phrase CRLF */
+						int sp;
+						for(sp = 0; sp < i; sp++)
+							if(header_buf[sp] == ' ')
+							{
+								if(*status_code < 0)
+									*status_code = atoi(header_buf + sp + 1);
+								else
+								{
+									reason_phrase = header_buf + sp + 1;
+									reason_phrase_len = i - sp - 1;
+									break;
+								}
+							}
+#ifdef DEBUG
+						printf("HTTP status code = %d, Reason phrase = %.*s\n",
+						       *status_code, reason_phrase_len, reason_phrase);
+#endif
+					}
+					else if(colon > linestart && valuestart > colon)
 					{
 #ifdef DEBUG
 						printf("header='%.*s', value='%.*s'\n",
@@ -337,7 +362,8 @@ static void *
 miniwget3(const char * host,
           unsigned short port, const char * path,
           int * size, char * addr_str, int addr_str_len,
-          const char * httpversion, unsigned int scope_id)
+          const char * httpversion, unsigned int scope_id,
+          int * status_code)
 {
 	char buf[2048];
     int s;
@@ -435,7 +461,7 @@ miniwget3(const char * host,
 			sent += n;
 		}
 	}
-	content = getHTTPResponse(s, size);
+	content = getHTTPResponse(s, size, status_code);
 	closesocket(s);
 	return content;
 }
@@ -446,16 +472,18 @@ static void *
 miniwget2(const char * host,
           unsigned short port, const char * path,
           int * size, char * addr_str, int addr_str_len,
-          unsigned int scope_id)
+          unsigned int scope_id, int * status_code)
 {
 	char * respbuffer;
 
 #if 1
 	respbuffer = miniwget3(host, port, path, size,
-	                       addr_str, addr_str_len, "1.1", scope_id);
+	                       addr_str, addr_str_len, "1.1",
+	                       scope_id, status_code);
 #else
 	respbuffer = miniwget3(host, port, path, size,
-	                       addr_str, addr_str_len, "1.0", scope_id);
+	                       addr_str, addr_str_len, "1.0",
+	                       scope_id, status_code);
 	if (*size == 0)
 	{
 #ifdef DEBUG
@@ -463,7 +491,8 @@ miniwget2(const char * host,
 #endif
 		free(respbuffer);
 		respbuffer = miniwget3(host, port, path, size,
-		                       addr_str, addr_str_len, "1.1", scope_id);
+		                       addr_str, addr_str_len, "1.1",
+		                       scope_id, status_code);
 	}
 #endif
 	return respbuffer;
@@ -588,7 +617,8 @@ parseURL(const char * url,
 }
 
 void *
-miniwget(const char * url, int * size, unsigned int scope_id)
+miniwget(const char * url, int * size,
+         unsigned int scope_id, int * status_code)
 {
 	unsigned short port;
 	char * path;
@@ -601,12 +631,13 @@ miniwget(const char * url, int * size, unsigned int scope_id)
 	printf("parsed url : hostname='%s' port=%hu path='%s' scope_id=%u\n",
 	       hostname, port, path, scope_id);
 #endif
-	return miniwget2(hostname, port, path, size, 0, 0, scope_id);
+	return miniwget2(hostname, port, path, size, 0, 0, scope_id, status_code);
 }
 
 void *
 miniwget_getaddr(const char * url, int * size,
-                 char * addr, int addrlen, unsigned int scope_id)
+                 char * addr, int addrlen, unsigned int scope_id,
+                 int * status_code)
 {
 	unsigned short port;
 	char * path;
@@ -621,6 +652,6 @@ miniwget_getaddr(const char * url, int * size,
 	printf("parsed url : hostname='%s' port=%hu path='%s' scope_id=%u\n",
 	       hostname, port, path, scope_id);
 #endif
-	return miniwget2(hostname, port, path, size, addr, addrlen, scope_id);
+	return miniwget2(hostname, port, path, size, addr, addrlen, scope_id, status_code);
 }
 

external/miniupnpc/miniwget.h

@@ -1,7 +1,7 @@
-/* $Id: miniwget.h,v 1.7 2012/06/23 22:35:59 nanard Exp $ */
+/* $Id: miniwget.h,v 1.12 2016/01/24 17:24:36 nanard Exp $ */
 /* Project : miniupnp
  * Author : Thomas Bernard
- * Copyright (c) 2005-2015 Thomas Bernard
+ * Copyright (c) 2005-2016 Thomas Bernard
  * This software is subject to the conditions detailed in the
  * LICENCE file provided in this distribution.
  * */
@@ -14,11 +14,11 @@
 extern "C" {
 #endif
 
-MINIUPNP_LIBSPEC void * getHTTPResponse(int s, int * size);
+MINIUPNP_LIBSPEC void * getHTTPResponse(int s, int * size, int * status_code);
 
-MINIUPNP_LIBSPEC void * miniwget(const char *, int *, unsigned int);
+MINIUPNP_LIBSPEC void * miniwget(const char *, int *, unsigned int, int *);
 
-MINIUPNP_LIBSPEC void * miniwget_getaddr(const char *, int *, char *, int, unsigned int);
+MINIUPNP_LIBSPEC void * miniwget_getaddr(const char *, int *, char *, int, unsigned int, int *);
 
 int parseURL(const char *, char *, unsigned short *, char * *, unsigned int *);
 

external/miniupnpc/testminiwget.c

@@ -1,7 +1,7 @@
-/* $Id: testminiwget.c,v 1.4 2012/06/23 22:35:59 nanard Exp $ */
+/* $Id: testminiwget.c,v 1.5 2016/01/24 17:24:36 nanard Exp $ */
 /* Project : miniupnp
  * Author : Thomas Bernard
- * Copyright (c) 2005-2012 Thomas Bernard
+ * Copyright (c) 2005-2016 Thomas Bernard
  * This software is subject to the conditions detailed in the
  * LICENCE file provided in this distribution.
  * */
@@ -20,15 +20,17 @@ int main(int argc, char * * argv)
 	int size, writtensize;
 	FILE *f;
 	char addr[64];
+	int status_code = -1;
 
 	if(argc < 3) {
 		fprintf(stderr, "Usage:\t%s url file\n", argv[0]);
 		fprintf(stderr, "Example:\t%s http://www.google.com/ out.html\n", argv[0]);
 		return 1;
 	}
-	data = miniwget_getaddr(argv[1], &size, addr, sizeof(addr), 0);
+	data = miniwget_getaddr(argv[1], &size, addr, sizeof(addr), 0, &status_code);
-	if(!data) {
+	if(!data || (status_code != 200)) {
-		fprintf(stderr, "Error fetching %s\n", argv[1]);
+		if(data) free(data);
+		fprintf(stderr, "Error %d fetching %s\n", status_code, argv[1]);
 		return 1;
 	}
 	printf("local address : %s\n", addr);

external/miniupnpc/upnpc.c

@@ -1,7 +1,7 @@
-/* $Id: upnpc.c,v 1.111 2015/07/23 20:40:10 nanard Exp $ */
+/* $Id: upnpc.c,v 1.114 2016/01/22 15:04:23 nanard Exp $ */
 /* Project : miniupnp
  * Author : Thomas Bernard
- * Copyright (c) 2005-2015 Thomas Bernard
+ * Copyright (c) 2005-2016 Thomas Bernard
  * This software is subject to the conditions detailed in the
  * LICENCE file provided in this distribution. */
 
@@ -66,7 +66,7 @@ static void DisplayInfos(struct UPNPUrls * urls,
 	char connectionType[64];
 	char status[64];
 	char lastconnerr[64];
-	unsigned int uptime;
+	unsigned int uptime = 0;
 	unsigned int brUp, brDown;
 	time_t timenow, timestarted;
 	int r;
@@ -82,9 +82,11 @@ static void DisplayInfos(struct UPNPUrls * urls,
 	else
 		printf("Status : %s, uptime=%us, LastConnectionError : %s\n",
 		       status, uptime, lastconnerr);
+	if(uptime > 0) {
 		timenow = time(NULL);
 		timestarted = timenow - uptime;
 		printf("  Time started : %s", ctime(&timestarted));
+	}
 	if(UPNP_GetLinkLayerMaxBitRates(urls->controlURL_CIF, data->CIF.servicetype,
 	                                &brDown, &brUp) != UPNPCOMMAND_SUCCESS) {
 		printf("GetLinkLayerMaxBitRates failed.\n");
@@ -538,7 +540,7 @@ int main(int argc, char ** argv)
 	char ** commandargv = 0;
 	int commandargc = 0;
 	struct UPNPDev * devlist = 0;
-	char lanaddr[64];	/* my ip address on the LAN */
+	char lanaddr[64] = "unset";	/* my ip address on the LAN */
 	int i;
 	const char * rootdescurl = 0;
 	const char * multicastif = 0;
@@ -560,7 +562,7 @@ int main(int argc, char ** argv)
 	}
 #endif
     printf("upnpc : miniupnpc library test client, version %s.\n", MINIUPNPC_VERSION_STRING);
-	printf(" (c) 2005-2015 Thomas Bernard.\n");
+	printf(" (c) 2005-2016 Thomas Bernard.\n");
     printf("Go to http://miniupnp.free.fr/ or http://miniupnp.tuxfamily.org/\n"
 	       "for more information.\n");
 	/* command line processing */

external/miniupnpc/upnpcommands.c

@@ -616,14 +616,14 @@ UPNP_GetGenericPortMappingEntry(const char * controlURL,
 		protocol[3] = '\0';
 	}
 	p = GetValueFromNameValueList(&pdata, "NewInternalClient");
-	if(p && intClient)
+	if(p)
 	{
 		strncpy(intClient, p, 16);
 		intClient[15] = '\0';
 		r = 0;
 	}
 	p = GetValueFromNameValueList(&pdata, "NewInternalPort");
-	if(p && intPort)
+	if(p)
 	{
 		strncpy(intPort, p, 6);
 		intPort[5] = '\0';

src/CMakeLists.txt

@@ -87,7 +87,6 @@ add_subdirectory(wallet)
 add_subdirectory(p2p)
 add_subdirectory(cryptonote_protocol)
 
-add_subdirectory(connectivity_tool)
 add_subdirectory(miner)
 add_subdirectory(simplewallet)
 add_subdirectory(daemonizer)

src/blockchain_db/berkeleydb/db_bdb.cpp

@@ -2180,6 +2180,12 @@ void BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, const std::v
     LOG_PRINT_L3("db3: " << db3);
 }
 
+std::map<uint64_t, uint64_t>::BlockchainBDB::get_output_histogram(const std::vector<uint64_t> &amounts) const
+{
+  LOG_PRINT_L3("BlockchainBDB::" << __func__);
+  throw1(DB_ERROR("Not implemented."));
+}
+
 void BlockchainBDB::set_hard_fork_starting_height(uint8_t version, uint64_t height)
 {
     LOG_PRINT_L3("BlockchainBDB::" << __func__);

src/blockchain_db/berkeleydb/db_bdb.h

@@ -341,6 +341,15 @@ public:
   virtual bool can_thread_bulk_indices() const { return false; }
 #endif
 
+  /**
+   * @brief return a histogram of outputs on the blockchain
+   *
+   * @param amounts optional set of amounts to lookup
+   *
+   * @return a set of amount/instances
+   */
+  std::map<uint64_t, uint64_t> get_output_histogram(const std::vector<uint64_t> &amounts) const;
+
 private:
   virtual void add_block( const block& blk
                 , const size_t& block_size

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -228,17 +228,26 @@ mdb_threadinfo::~mdb_threadinfo()
     mdb_txn_abort(m_ti_rtxn);
 }
 
-mdb_txn_safe::mdb_txn_safe() : m_txn(NULL)
+mdb_txn_safe::mdb_txn_safe(const bool check) : m_txn(NULL), m_tinfo(NULL), m_check(check)
+{
+  if (check)
   {
     while (creation_gate.test_and_set());
     num_active_txns++;
     creation_gate.clear();
   }
+}
 
 mdb_txn_safe::~mdb_txn_safe()
 {
+  if (!m_check)
+    return;
   LOG_PRINT_L3("mdb_txn_safe: destructor");
-  if (m_txn != nullptr)
+  if (m_tinfo != nullptr)
+  {
+    mdb_txn_reset(m_tinfo->m_ti_rtxn);
+    memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags));
+  } else if (m_txn != nullptr)
   {
     if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety
     {
@@ -536,11 +545,12 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const
   {
     MDB_val_copy<crypto::hash> parent_key(blk.prev_id);
     MDB_val parent_h;
-    if (mdb_cursor_get(m_cur_block_heights, &parent_key, &parent_h, MDB_SET))
+    int result = mdb_cursor_get(m_cur_block_heights, &parent_key, &parent_h, MDB_SET);
+    if (result)
     {
       LOG_PRINT_L3("m_height: " << m_height);
       LOG_PRINT_L3("parent_key: " << blk.prev_id);
-      throw0(DB_ERROR("Failed to get top block hash to check for new block's parent"));
+      throw0(DB_ERROR(lmdb_error("Failed to get top block hash to check for new block's parent: ", result).c_str()));
     }
     uint64_t parent_height = *(const uint64_t *)parent_h.mv_data;
     if (parent_height != m_height - 1)
@@ -597,6 +607,8 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const
 
 void BlockchainLMDB::remove_block()
 {
+  int result;
+
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
@@ -605,29 +617,29 @@ void BlockchainLMDB::remove_block()
 
   MDB_val_copy<uint64_t> k(m_height - 1);
   MDB_val h;
-  if (mdb_get(*m_write_txn, m_block_hashes, &k, &h))
+  if ((result = mdb_get(*m_write_txn, m_block_hashes, &k, &h)))
-      throw1(BLOCK_DNE("Attempting to remove block that's not in the db"));
+      throw1(BLOCK_DNE(lmdb_error("Attempting to remove block that's not in the db: ", result).c_str()));
 
-  if (mdb_del(*m_write_txn, m_blocks, &k, NULL))
+  if ((result = mdb_del(*m_write_txn, m_blocks, &k, NULL)))
-      throw1(DB_ERROR("Failed to add removal of block to db transaction"));
+      throw1(DB_ERROR(lmdb_error("Failed to add removal of block to db transaction: ", result).c_str()));
 
-  if (mdb_del(*m_write_txn, m_block_sizes, &k, NULL))
+  if ((result = mdb_del(*m_write_txn, m_block_sizes, &k, NULL)))
-      throw1(DB_ERROR("Failed to add removal of block size to db transaction"));
+      throw1(DB_ERROR(lmdb_error("Failed to add removal of block size to db transaction: ", result).c_str()));
 
-  if (mdb_del(*m_write_txn, m_block_diffs, &k, NULL))
+  if ((result = mdb_del(*m_write_txn, m_block_diffs, &k, NULL)))
-      throw1(DB_ERROR("Failed to add removal of block cumulative difficulty to db transaction"));
+      throw1(DB_ERROR(lmdb_error("Failed to add removal of block cumulative difficulty to db transaction: ", result).c_str()));
 
-  if (mdb_del(*m_write_txn, m_block_coins, &k, NULL))
+  if ((result = mdb_del(*m_write_txn, m_block_coins, &k, NULL)))
-      throw1(DB_ERROR("Failed to add removal of block total generated coins to db transaction"));
+      throw1(DB_ERROR(lmdb_error("Failed to add removal of block total generated coins to db transaction: ", result).c_str()));
 
-  if (mdb_del(*m_write_txn, m_block_timestamps, &k, NULL))
+  if ((result = mdb_del(*m_write_txn, m_block_timestamps, &k, NULL)))
-      throw1(DB_ERROR("Failed to add removal of block timestamp to db transaction"));
+      throw1(DB_ERROR(lmdb_error("Failed to add removal of block timestamp to db transaction: ", result).c_str()));
 
-  if (mdb_del(*m_write_txn, m_block_heights, &h, NULL))
+  if ((result = mdb_del(*m_write_txn, m_block_heights, &h, NULL)))
-      throw1(DB_ERROR("Failed to add removal of block height by hash to db transaction"));
+      throw1(DB_ERROR(lmdb_error("Failed to add removal of block height by hash to db transaction: ", result).c_str()));
 
-  if (mdb_del(*m_write_txn, m_block_hashes, &k, NULL))
+  if ((result = mdb_del(*m_write_txn, m_block_hashes, &k, NULL)))
-      throw1(DB_ERROR("Failed to add removal of block hash to db transaction"));
+      throw1(DB_ERROR(lmdb_error("Failed to add removal of block hash to db transaction: ", result).c_str()));
 }
 
 void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash)
@@ -665,6 +677,8 @@ void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const tr
 
 void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx)
 {
+  int result;
+
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
@@ -673,16 +687,16 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const
   if (mdb_get(*m_write_txn, m_txs, &val_h, &unused))
       throw1(TX_DNE("Attempting to remove transaction that isn't in the db"));
 
-  if (mdb_del(*m_write_txn, m_txs, &val_h, NULL))
+  if ((result = mdb_del(*m_write_txn, m_txs, &val_h, NULL)))
-      throw1(DB_ERROR("Failed to add removal of tx to db transaction"));
+      throw1(DB_ERROR(lmdb_error("Failed to add removal of tx to db transaction: ", result).c_str()));
-  if (mdb_del(*m_write_txn, m_tx_unlocks, &val_h, NULL))
+  if ((result = mdb_del(*m_write_txn, m_tx_unlocks, &val_h, NULL)))
-      throw1(DB_ERROR("Failed to add removal of tx unlock time to db transaction"));
+      throw1(DB_ERROR(lmdb_error("Failed to add removal of tx unlock time to db transaction: ", result).c_str()));
-  if (mdb_del(*m_write_txn, m_tx_heights, &val_h, NULL))
+  if ((result = mdb_del(*m_write_txn, m_tx_heights, &val_h, NULL)))
-      throw1(DB_ERROR("Failed to add removal of tx block height to db transaction"));
+      throw1(DB_ERROR(lmdb_error("Failed to add removal of tx block height to db transaction: ", result).c_str()));
 
   remove_tx_outputs(&val_h, tx);
 
-  auto result = mdb_del(*m_write_txn, m_tx_outputs, &val_h, NULL);
+  result = mdb_del(*m_write_txn, m_tx_outputs, &val_h, NULL);
   if (result == MDB_NOTFOUND)
     LOG_PRINT_L1("tx has no outputs to remove: " << tx_hash);
   else if (result)
@@ -732,8 +746,8 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou
 
     MDB_val_copy<output_data_t> data(od);
     //MDB_val_copy<crypto::public_key> val_pubkey(boost::get<txout_to_key>(tx_output.target).key);
-    if (mdb_cursor_put(m_cur_output_keys, &k, &data, MDB_APPEND))
+    if ((result = mdb_cursor_put(m_cur_output_keys, &k, &data, MDB_APPEND)))
-      throw0(DB_ERROR("Failed to add output pubkey to db transaction"));
+      throw0(DB_ERROR(lmdb_error("Failed to add output pubkey to db transaction: ", result).c_str()));
   }
   else
   {
@@ -758,7 +772,7 @@ void BlockchainLMDB::remove_tx_outputs(const MDB_val *tx_hash, const transaction
   }
   else if (result)
   {
-    throw0(DB_ERROR("DB error attempting to get an output"));
+    throw0(DB_ERROR(lmdb_error("DB error attempting to get an output", result).c_str()));
   }
   else
   {
@@ -870,9 +884,9 @@ void BlockchainLMDB::remove_amount_output_index(const uint64_t amount, const MDB
   {
     // found the amount output index
     // now delete it
-    result = mdb_cursor_del(m_cur_output_amounts, 0);
+    int result = mdb_cursor_del(m_cur_output_amounts, 0);
     if (result)
-      throw0(DB_ERROR(std::string("Error deleting amount output index ").append(boost::lexical_cast<std::string>(amount_output_index)).c_str()));
+      throw0(DB_ERROR(lmdb_error(std::string("Error deleting amount output index ").append(boost::lexical_cast<std::string>(amount_output_index).append(": ")).c_str(), result).c_str()));
   }
   else
   {
@@ -987,6 +1001,8 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions)
 
 void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
 {
+  int result;
+
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
 
   if (m_open)
@@ -1016,10 +1032,10 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
   m_folder = filename;
 
   // set up lmdb environment
-  if (mdb_env_create(&m_env))
+  if ((result = mdb_env_create(&m_env)))
-    throw0(DB_ERROR("Failed to create lmdb environment"));
+    throw0(DB_ERROR(lmdb_error("Failed to create lmdb environment: ", result).c_str()));
-  if (mdb_env_set_maxdbs(m_env, 20))
+  if ((result = mdb_env_set_maxdbs(m_env, 20)))
-    throw0(DB_ERROR("Failed to set max number of dbs"));
+    throw0(DB_ERROR(lmdb_error("Failed to set max number of dbs: ", result).c_str()));
 
   size_t mapsize = DEFAULT_MAPSIZE;
 
@@ -1095,14 +1111,14 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
 
   // get and keep current height
   MDB_stat db_stats;
-  if (mdb_stat(txn, m_blocks, &db_stats))
+  if ((result = mdb_stat(txn, m_blocks, &db_stats)))
-    throw0(DB_ERROR("Failed to query m_blocks"));
+    throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str()));
   LOG_PRINT_L2("Setting m_height to: " << db_stats.ms_entries);
   m_height = db_stats.ms_entries;
 
   // get and keep current number of outputs
-  if (mdb_stat(txn, m_output_indices, &db_stats))
+  if ((result = mdb_stat(txn, m_output_indices, &db_stats)))
-    throw0(DB_ERROR("Failed to query m_output_indices"));
+    throw0(DB_ERROR(lmdb_error("Failed to query m_output_indices: ", result).c_str()));
   m_num_outputs = db_stats.ms_entries;
 
   bool compatible = true;
@@ -1223,8 +1239,8 @@ void BlockchainLMDB::reset()
   check_open();
 
   mdb_txn_safe txn;
-  if (mdb_txn_begin(m_env, NULL, 0, txn))
+  if (auto result = mdb_txn_begin(m_env, NULL, 0, txn))
-    throw0(DB_ERROR("Failed to create a transaction for the db"));
+    throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
   mdb_drop(txn, m_blocks, 0);
   mdb_drop(txn, m_block_timestamps, 0);
   mdb_drop(txn, m_block_heights, 0);
@@ -1303,9 +1319,10 @@ void BlockchainLMDB::unlock()
 #define TXN_PREFIX_RDONLY() \
   MDB_txn *m_txn; \
   mdb_txn_cursors *m_cursors; \
-  bool my_rtxn = block_rtxn_start(&m_txn, &m_cursors);
+  bool my_rtxn = block_rtxn_start(&m_txn, &m_cursors); \
-#define TXN_POSTFIX_RDONLY() \
+  mdb_txn_safe auto_txn(my_rtxn); \
-  if (my_rtxn) block_rtxn_stop()
+  if (my_rtxn) auto_txn.m_tinfo = m_tinfo.get()
+#define TXN_POSTFIX_RDONLY()
 
 #define TXN_POSTFIX_SUCCESS() \
   do { \
@@ -2675,6 +2692,63 @@ void BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const std::
   LOG_PRINT_L3("db3: " << db3);
 }
 
+std::map<uint64_t, uint64_t> BlockchainLMDB::get_output_histogram(const std::vector<uint64_t> &amounts) const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+
+  TXN_PREFIX_RDONLY();
+  RCURSOR(output_amounts);
+
+  std::map<uint64_t, uint64_t> histogram;
+  MDB_val k;
+  MDB_val v;
+
+  if (amounts.empty())
+  {
+    MDB_cursor_op op = MDB_FIRST;
+    while (1)
+    {
+      int ret = mdb_cursor_get(m_cur_output_amounts, &k, &v, op);
+      op = MDB_NEXT_NODUP;
+      if (ret == MDB_NOTFOUND)
+        break;
+      if (ret)
+        throw0(DB_ERROR(lmdb_error("Failed to enumerate outputs: ", ret).c_str()));
+      mdb_size_t num_elems = 0;
+      mdb_cursor_count(m_cur_output_amounts, &num_elems);
+      uint64_t amount = *(const uint64_t*)k.mv_data;
+      histogram[amount] = num_elems;
+    }
+  }
+  else
+  {
+    for (const auto &amount: amounts)
+    {
+      MDB_val_copy<uint64_t> k(amount);
+      int ret = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET);
+      if (ret == MDB_NOTFOUND)
+      {
+        histogram[amount] = 0;
+      }
+      else if (ret == MDB_SUCCESS)
+      {
+        mdb_size_t num_elems = 0;
+        mdb_cursor_count(m_cur_output_amounts, &num_elems);
+        histogram[amount] = num_elems;
+      }
+      else
+      {
+        throw0(DB_ERROR(lmdb_error("Failed to enumerate outputs: ", ret).c_str()));
+      }
+    }
+  }
+
+  TXN_POSTFIX_RDONLY();
+
+  return histogram;
+}
+
 void BlockchainLMDB::check_hard_fork_info()
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);

src/blockchain_db/lmdb/db_lmdb.h

@@ -115,7 +115,7 @@ typedef struct mdb_threadinfo
 
 struct mdb_txn_safe
 {
-  mdb_txn_safe();
+  mdb_txn_safe(const bool check=true);
   ~mdb_txn_safe();
 
   void commit(std::string message = "");
@@ -142,8 +142,10 @@ struct mdb_txn_safe
   static void wait_no_active_txns();
   static void allow_new_txns();
 
+  mdb_threadinfo* m_tinfo;
   MDB_txn* m_txn;
   bool m_batch_txn = false;
+  bool m_check;
   static std::atomic<uint64_t> num_active_txns;
 
   // could use a mutex here, but this should be sufficient.
@@ -278,6 +280,16 @@ public:
   virtual void pop_block(block& blk, std::vector<transaction>& txs);
 
   virtual bool can_thread_bulk_indices() const { return true; }
+
+  /**
+   * @brief return a histogram of outputs on the blockchain
+   *
+   * @param amounts optional set of amounts to lookup
+   *
+   * @return a set of amount/instances
+   */
+  std::map<uint64_t, uint64_t> get_output_histogram(const std::vector<uint64_t> &amounts) const;
+
 private:
   void do_resize(uint64_t size_increase=0);
 

src/connectivity_tool/CMakeLists.txt

@@ -1,46 +0,0 @@
-# Copyright (c) 2014-2016, The Monero Project
-#
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without modification, are
-# permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice, this list of
-#    conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice, this list
-#    of conditions and the following disclaimer in the documentation and/or other
-#    materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors may be
-#    used to endorse or promote products derived from this software without specific
-#    prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
-# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
-# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
-# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
-# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-set(connectivity_tool_sources
-  conn_tool.cpp)
-
-set(connectivity_tool_private_headers)
-
-bitmonero_add_executable(connectivity_tool
-  ${connectivity_tool_sources}
-  ${connectivity_tool_private_headers})
-target_link_libraries(connectivity_tool
-  LINK_PRIVATE
-    cryptonote_core
-    crypto
-    common
-    ${CMAKE_THREAD_LIBS_INIT}
-    ${Boost_PROGRAM_OPTIONS_LIBRARY}
-    ${Boost_REGEX_LIBRARY}
-    ${Boost_CHRONO_LIBRARY}
-    ${Boost_SYSTEM_LIBRARY})

src/connectivity_tool/conn_tool.cpp

@@ -1,377 +0,0 @@
-// Copyright (c) 2014-2016, The Monero Project
-// 
-// All rights reserved.
-// 
-// Redistribution and use in source and binary forms, with or without modification, are
-// permitted provided that the following conditions are met:
-// 
-// 1. Redistributions of source code must retain the above copyright notice, this list of
-//    conditions and the following disclaimer.
-// 
-// 2. Redistributions in binary form must reproduce the above copyright notice, this list
-//    of conditions and the following disclaimer in the documentation and/or other
-//    materials provided with the distribution.
-// 
-// 3. Neither the name of the copyright holder nor the names of its contributors may be
-//    used to endorse or promote products derived from this software without specific
-//    prior written permission.
-// 
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
-// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
-// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
-// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
-// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-// 
-// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
-
-#include "include_base_utils.h"
-#include "version.h"
-
-using namespace epee;
-#include <boost/program_options.hpp>
-#include "p2p/p2p_protocol_defs.h"
-#include "common/command_line.h"
-#include "cryptonote_core/cryptonote_core.h"
-#include "cryptonote_protocol/cryptonote_protocol_handler.h"
-#include "net/levin_client.h"
-#include "storages/levin_abstract_invoke2.h"
-#include "cryptonote_core/cryptonote_core.h"
-#include "storages/portable_storage_template_helper.h"
-#include "crypto/crypto.h"
-#include "storages/http_abstract_invoke.h"
-#include "net/http_client.h"
-
-namespace po = boost::program_options;
-using namespace cryptonote;
-using namespace nodetool;
-
-namespace
-{
-  const command_line::arg_descriptor<std::string, true> arg_ip           = {"ip", "set ip"};
-  const command_line::arg_descriptor<size_t>      arg_port               = {"port", "set port"};
-  const command_line::arg_descriptor<size_t>      arg_rpc_port           = {"rpc_port", "set rpc port"};
-  const command_line::arg_descriptor<uint32_t, true> arg_timeout         = {"timeout", "set timeout"};
-  const command_line::arg_descriptor<std::string> arg_priv_key           = {"private_key", "private key to subscribe debug command", "", true};
-  const command_line::arg_descriptor<uint64_t>    arg_peer_id            = {"peer_id", "peer_id if known(if not - will be requested)", 0};
-  const command_line::arg_descriptor<bool>        arg_generate_keys      = {"generate_keys_pair", "generate private and public keys pair"};
-  const command_line::arg_descriptor<bool>        arg_request_stat_info  = {"request_stat_info", "request statistics information"};
-  const command_line::arg_descriptor<bool>        arg_request_net_state  = {"request_net_state", "request network state information (peer list, connections count)"};
-  const command_line::arg_descriptor<bool>        arg_get_daemon_info    = {"rpc_get_daemon_info", "request daemon state info vie rpc (--rpc_port option should be set ).", "", true};
-}
-
-typedef COMMAND_REQUEST_STAT_INFO_T<t_cryptonote_protocol_handler<core>::stat_info> COMMAND_REQUEST_STAT_INFO;
-
-struct response_schema
-{
-  std::string status;
-  std::string COMMAND_REQUEST_STAT_INFO_status;
-  std::string COMMAND_REQUEST_NETWORK_STATE_status;
-  enableable<COMMAND_REQUEST_STAT_INFO::response> si_rsp;
-  enableable<COMMAND_REQUEST_NETWORK_STATE::response> ns_rsp;
-
-  BEGIN_KV_SERIALIZE_MAP()
-    KV_SERIALIZE(status)
-    KV_SERIALIZE(COMMAND_REQUEST_STAT_INFO_status)
-    KV_SERIALIZE(COMMAND_REQUEST_NETWORK_STATE_status)
-    KV_SERIALIZE(si_rsp)
-    KV_SERIALIZE(ns_rsp)
-  END_KV_SERIALIZE_MAP() 
-};
-
-  std::string get_response_schema_as_json(response_schema& rs)
-  {
-    std::stringstream ss;
-    ss << "{" << ENDL 
-       << "  \"status\": \"" << rs.status << "\"," << ENDL
-       << "  \"COMMAND_REQUEST_NETWORK_STATE_status\": \"" << rs.COMMAND_REQUEST_NETWORK_STATE_status << "\"," << ENDL
-       << "  \"COMMAND_REQUEST_STAT_INFO_status\": \"" << rs.COMMAND_REQUEST_STAT_INFO_status <<  "\"";
-    if(rs.si_rsp.enabled)
-    {
-      ss << "," << ENDL << "  \"si_rsp\": " <<  epee::serialization::store_t_to_json(rs.si_rsp.v, 1);
-    }
-    if(rs.ns_rsp.enabled)
-    {
-      ss << "," << ENDL << "  \"ns_rsp\": {" << ENDL 
-        << "    \"local_time\": " <<  rs.ns_rsp.v.local_time << "," << ENDL 
-        << "    \"my_id\": \"" <<  rs.ns_rsp.v.my_id << "\"," << ENDL
-        << "    \"connections_list\": [" << ENDL;
-
-      size_t i = 0;
-      BOOST_FOREACH(const connection_entry& ce, rs.ns_rsp.v.connections_list)
-      {
-        ss <<  "      {\"peer_id\": \"" << ce.id << "\", \"ip\": \"" << string_tools::get_ip_string_from_int32(ce.adr.ip) << "\", \"port\": " << ce.adr.port << ", \"is_income\": "<< ce.is_income << "}";
-        if(rs.ns_rsp.v.connections_list.size()-1 != i)
-          ss << ",";
-        ss << ENDL; 
-        i++;
-      }
-      ss << "    ]," << ENDL;
-      ss << "    \"local_peerlist_white\": [" << ENDL;      
-      i = 0;
-      BOOST_FOREACH(const peerlist_entry& pe, rs.ns_rsp.v.local_peerlist_white)
-      {
-        ss <<  "      {\"peer_id\": \"" << pe.id << "\", \"ip\": \"" << string_tools::get_ip_string_from_int32(pe.adr.ip) << "\", \"port\": " << pe.adr.port << ", \"last_seen\": "<< rs.ns_rsp.v.local_time - pe.last_seen << "}";
-        if(rs.ns_rsp.v.local_peerlist_white.size()-1 != i)
-          ss << ",";
-        ss << ENDL; 
-        i++;
-      }
-      ss << "    ]," << ENDL;
-
-      ss << "    \"local_peerlist_gray\": [" << ENDL;      
-      i = 0;
-      BOOST_FOREACH(const peerlist_entry& pe, rs.ns_rsp.v.local_peerlist_gray)
-      {
-        ss <<  "      {\"peer_id\": \"" << pe.id << "\", \"ip\": \"" << string_tools::get_ip_string_from_int32(pe.adr.ip) << "\", \"port\": " << pe.adr.port << ", \"last_seen\": "<< rs.ns_rsp.v.local_time - pe.last_seen << "}";
-        if(rs.ns_rsp.v.local_peerlist_gray.size()-1 != i)
-          ss << ",";
-        ss << ENDL; 
-        i++;
-      }
-      ss << "    ]" << ENDL << "  }" << ENDL;
-    }
-    ss << "}";
-    return std::move(ss.str());
-  }
-//---------------------------------------------------------------------------------------------------------------
-bool print_COMMAND_REQUEST_STAT_INFO(const COMMAND_REQUEST_STAT_INFO::response& si)
-{
-  std::cout << " ------ COMMAND_REQUEST_STAT_INFO ------ " << ENDL;
-  std::cout << "Version:             " << si.version << ENDL;
-  std::cout << "OS Version:          " << si.os_version << ENDL;
-  std::cout << "Connections:          " << si.connections_count << ENDL;
-  std::cout << "INC Connections:     " << si.incoming_connections_count << ENDL;
-
-
-  std::cout << "Tx pool size:        " << si.payload_info.tx_pool_size << ENDL;
-  std::cout << "BC height:           " << si.payload_info.blockchain_height << ENDL;
-  std::cout << "Mining speed:          " << si.payload_info.mining_speed << ENDL;
-  std::cout << "Alternative blocks:  " << si.payload_info.alternative_blocks << ENDL;
-  std::cout << "Top block id:        " << si.payload_info.top_block_id_str << ENDL;
-  return true;
-}
-//---------------------------------------------------------------------------------------------------------------
-bool print_COMMAND_REQUEST_NETWORK_STATE(const COMMAND_REQUEST_NETWORK_STATE::response& ns)
-{
-  std::cout << " ------ COMMAND_REQUEST_NETWORK_STATE ------ " << ENDL;
-  std::cout << "Peer id: " << ns.my_id << ENDL;
-  std::cout << "Active connections:"  << ENDL;
-  BOOST_FOREACH(const connection_entry& ce, ns.connections_list)
-  {
-    std::cout <<  ce.id << "\t" << string_tools::get_ip_string_from_int32(ce.adr.ip) << ":" << ce.adr.port << (ce.is_income ? "(INC)":"(OUT)") << ENDL; 
-  }
-  
-  std::cout << "Peer list white:" << ns.my_id << ENDL;
-  BOOST_FOREACH(const peerlist_entry& pe, ns.local_peerlist_white)
-  {
-    std::cout <<  pe.id << "\t" << string_tools::get_ip_string_from_int32(pe.adr.ip) << ":" << pe.adr.port <<  "\t" << misc_utils::get_time_interval_string(ns.local_time - pe.last_seen) << ENDL; 
-  }
-
-  std::cout << "Peer list gray:" << ns.my_id << ENDL;
-  BOOST_FOREACH(const peerlist_entry& pe, ns.local_peerlist_gray)
-  {
-    std::cout <<  pe.id << "\t" << string_tools::get_ip_string_from_int32(pe.adr.ip) << ":" << pe.adr.port <<  "\t" << misc_utils::get_time_interval_string(ns.local_time - pe.last_seen) << ENDL; 
-  }
-
-
-  return true;
-}
-//---------------------------------------------------------------------------------------------------------------
-bool handle_get_daemon_info(po::variables_map& vm)
-{
-  if(!command_line::has_arg(vm, arg_rpc_port))
-  {
-    std::cout << "ERROR: rpc port not set" << ENDL;
-    return false;
-  }
-
-  epee::net_utils::http::http_simple_client http_client;
-
-  cryptonote::COMMAND_RPC_GET_INFO::request req = AUTO_VAL_INIT(req);
-  cryptonote::COMMAND_RPC_GET_INFO::response res = AUTO_VAL_INIT(res);
-  std::string daemon_addr = command_line::get_arg(vm, arg_ip) + ":" + std::to_string(command_line::get_arg(vm, arg_rpc_port));
-  bool r = net_utils::invoke_http_json_remote_command2(daemon_addr + "/getinfo", req, res, http_client, command_line::get_arg(vm, arg_timeout));
-  if(!r)
-  {
-    std::cout << "ERROR: failed to invoke request" << ENDL;
-    return false;
-  }
-  std::cout << "OK" << ENDL
-    << "height: " << res.height << ENDL
-  << "difficulty: " << res.difficulty << ENDL
-  << "tx_count: " << res.tx_count << ENDL
-  << "tx_pool_size: " << res.tx_pool_size << ENDL
-  << "alt_blocks_count: " << res.alt_blocks_count << ENDL
-  << "outgoing_connections_count: " << res.outgoing_connections_count << ENDL
-  << "incoming_connections_count: " << res.incoming_connections_count << ENDL
-  << "white_peerlist_size: " << res.white_peerlist_size << ENDL
-  << "grey_peerlist_size: " << res.grey_peerlist_size << ENDL;
-
-  return true;
-}
-//---------------------------------------------------------------------------------------------------------------
-bool handle_request_stat(po::variables_map& vm, peerid_type peer_id)
-{
-
-  if(!command_line::has_arg(vm, arg_priv_key))
-  {
-    std::cout << "{" << ENDL << "  \"status\": \"ERROR: " << "secret key not set \"" << ENDL << "}";
-    return false;
-  }
-  crypto::secret_key prvk = AUTO_VAL_INIT(prvk);
-  if(!string_tools::hex_to_pod(command_line::get_arg(vm, arg_priv_key) , prvk))
-  {
-    std::cout << "{" << ENDL << "  \"status\": \"ERROR: " << "wrong secret key set \"" << ENDL << "}";
-    return false;
-  }
-
-
-  response_schema rs = AUTO_VAL_INIT(rs);
-
-  levin::levin_client_impl2 transport;
-  if(!transport.connect(command_line::get_arg(vm, arg_ip), static_cast<int>(command_line::get_arg(vm, arg_port)), static_cast<int>(command_line::get_arg(vm, arg_timeout))))
-  {
-    std::cout << "{" << ENDL << "  \"status\": \"ERROR: " << "Failed to connect to " << command_line::get_arg(vm, arg_ip) << ":" << command_line::get_arg(vm, arg_port) << "\"" << ENDL << "}";
-    return false;
-  }else
-    rs.status = "OK";
-
-  if(!peer_id)
-  {
-    COMMAND_REQUEST_PEER_ID::request req = AUTO_VAL_INIT(req);
-    COMMAND_REQUEST_PEER_ID::response rsp = AUTO_VAL_INIT(rsp);
-    if(!net_utils::invoke_remote_command2(COMMAND_REQUEST_PEER_ID::ID, req, rsp, transport))
-    {
-      std::cout << "{" << ENDL << "  \"status\": \"ERROR: " << "Failed to connect to " << command_line::get_arg(vm, arg_ip) << ":" << command_line::get_arg(vm, arg_port) << "\"" << ENDL << "}";
-      return false;
-    }else
-    {
-      peer_id = rsp.my_id;
-    }
-  }
-
-
-  nodetool::proof_of_trust pot = AUTO_VAL_INIT(pot);
-  pot.peer_id = peer_id;
-  pot.time = time(NULL);
-  crypto::public_key pubk = AUTO_VAL_INIT(pubk);
-  string_tools::hex_to_pod(::config::P2P_REMOTE_DEBUG_TRUSTED_PUB_KEY, pubk);
-  crypto::hash h = tools::get_proof_of_trust_hash(pot);
-  crypto::generate_signature(h, pubk, prvk, pot.sign);
-
-  if(command_line::get_arg(vm, arg_request_stat_info))
-  {
-    COMMAND_REQUEST_STAT_INFO::request req = AUTO_VAL_INIT(req);
-    req.tr = pot;
-    if(!net_utils::invoke_remote_command2(COMMAND_REQUEST_STAT_INFO::ID, req, rs.si_rsp.v, transport))
-    {
-      std::stringstream ss;
-      ss << "ERROR: " << "Failed to invoke remote command COMMAND_REQUEST_STAT_INFO to " << command_line::get_arg(vm, arg_ip) << ":" << command_line::get_arg(vm, arg_port);
-      rs.COMMAND_REQUEST_STAT_INFO_status = ss.str();
-    }else
-    {
-      rs.si_rsp.enabled = true;
-      rs.COMMAND_REQUEST_STAT_INFO_status = "OK";
-    }
-  }
-
-
-  if(command_line::get_arg(vm, arg_request_net_state))
-  {
-    ++pot.time;
-    h = tools::get_proof_of_trust_hash(pot);
-    crypto::generate_signature(h, pubk, prvk, pot.sign);
-    COMMAND_REQUEST_NETWORK_STATE::request req = AUTO_VAL_INIT(req);    
-    req.tr = pot;
-    if(!net_utils::invoke_remote_command2(COMMAND_REQUEST_NETWORK_STATE::ID, req, rs.ns_rsp.v, transport))
-    {
-      std::stringstream ss;
-      ss << "ERROR: " << "Failed to invoke remote command COMMAND_REQUEST_NETWORK_STATE to " << command_line::get_arg(vm, arg_ip) << ":" << command_line::get_arg(vm, arg_port);
-      rs.COMMAND_REQUEST_NETWORK_STATE_status = ss.str();
-    }else
-    {
-      rs.ns_rsp.enabled = true;
-      rs.COMMAND_REQUEST_NETWORK_STATE_status = "OK";
-    }
-  }
-  std::cout << get_response_schema_as_json(rs);
-  return true;
-}
-//---------------------------------------------------------------------------------------------------------------
-bool generate_and_print_keys()
-{
-  crypto::public_key pk = AUTO_VAL_INIT(pk);
-  crypto::secret_key sk = AUTO_VAL_INIT(sk);
-  generate_keys(pk, sk);
-  std::cout << "PUBLIC KEY: " << epee::string_tools::pod_to_hex(pk) << ENDL 
-    << "PRIVATE KEY: " << epee::string_tools::pod_to_hex(sk);
-  return true;
-}
-int main(int argc, char* argv[])
-{
-  string_tools::set_module_name_and_folder(argv[0]);
-  log_space::get_set_log_detalisation_level(true, LOG_LEVEL_0);
-
-  // Declare the supported options.
-  po::options_description desc_general("General options");
-  command_line::add_arg(desc_general, command_line::arg_help);
-
-  po::options_description desc_params("Connectivity options");
-  command_line::add_arg(desc_params, arg_ip);
-  command_line::add_arg(desc_params, arg_port);
-  command_line::add_arg(desc_params, arg_rpc_port);
-  command_line::add_arg(desc_params, arg_timeout);
-  command_line::add_arg(desc_params, arg_request_stat_info);
-  command_line::add_arg(desc_params, arg_request_net_state);
-  command_line::add_arg(desc_params, arg_generate_keys);
-  command_line::add_arg(desc_params, arg_peer_id);
-  command_line::add_arg(desc_params, arg_priv_key);
-  command_line::add_arg(desc_params, arg_get_daemon_info);
-
-
-  po::options_description desc_all;
-  desc_all.add(desc_general).add(desc_params);
-
-  po::variables_map vm;
-  bool r = command_line::handle_error_helper(desc_all, [&]()
-  {
-    po::store(command_line::parse_command_line(argc, argv, desc_general, true), vm);
-    if (command_line::get_arg(vm, command_line::arg_help))
-    {
-      std::cout << desc_all << ENDL;
-      return false;
-    }
-
-    po::store(command_line::parse_command_line(argc, argv, desc_params, false), vm);
-    po::notify(vm);
-
-    return true;
-  });
-  if (!r)
-    return 1;
-
-  if(command_line::has_arg(vm, arg_request_stat_info) || command_line::has_arg(vm, arg_request_net_state))
-  {
-    return handle_request_stat(vm, command_line::get_arg(vm, arg_peer_id)) ? 0:1;
-  }
-  if(command_line::has_arg(vm, arg_get_daemon_info))
-  {
-    return handle_get_daemon_info(vm) ? 0:1;
-  }
-  else if(command_line::has_arg(vm, arg_generate_keys))
-  {
-    return  generate_and_print_keys() ? 0:1;
-  }
-  else
-  {
-    std::cerr << "Not enough arguments." << ENDL;
-    std::cerr << desc_all << ENDL;
-  }
-
-  return 1;
-}
-

src/cryptonote_core/CMakeLists.txt

@@ -31,7 +31,6 @@ set(cryptonote_core_sources
   blockchain_storage.cpp
   blockchain.cpp
   checkpoints.cpp
-  checkpoints_create.cpp
   cryptonote_basic_impl.cpp
   cryptonote_core.cpp
   cryptonote_format_utils.cpp
@@ -49,7 +48,6 @@ set(cryptonote_core_private_headers
   blockchain_storage_boost_serialization.h
   blockchain.h
   checkpoints.h
-  checkpoints_create.h
   connection_context.h
   cryptonote_basic.h
   cryptonote_basic_impl.h

src/cryptonote_core/blockchain.cpp

@@ -49,7 +49,7 @@
 #include "common/boost_serialization_helper.h"
 #include "warnings.h"
 #include "crypto/hash.h"
-#include "cryptonote_core/checkpoints_create.h"
+#include "cryptonote_core/checkpoints.h"
 #include "cryptonote_core/cryptonote_core.h"
 #if defined(PER_BLOCK_CHECKPOINT)
 #include "blocks/blocks.h"
@@ -81,6 +81,9 @@ static const struct {
 
   // version 2 starts from block 1009827, which is on or around the 20th of March, 2016. Fork time finalised on 2015-09-20. No fork voting occurs for the v2 fork.
   { 2, 1009827, 0, 1442763710 },
+
+  // version 3 starts from block 1141317, which is on or around the 24th of September, 2016. Fork time finalised on 2016-03-21.
+  { 3, 1141317, 0, 1458558528 },
 };
 static const uint64_t mainnet_hard_fork_version_1_till = 1009826;
 
@@ -837,7 +840,7 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<blocks_ext_by_hash::
     }
   }
 
-  //removing alt_chain entries from alternative chain
+  //removing alt_chain entries from alternative chains container
   for (auto ch_ent: alt_chain)
   {
     m_alternative_chains.erase(ch_ent);
@@ -945,7 +948,7 @@ bool Blockchain::prevalidate_miner_transaction(const block& b, uint64_t height)
 }
 //------------------------------------------------------------------
 // This function validates the miner transaction reward
-bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward)
+bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward, uint8_t version)
 {
   LOG_PRINT_L3("Blockchain::" << __func__);
   //validate reward
@@ -954,6 +957,15 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl
     money_in_use += o.amount;
   partial_block_reward = false;
 
+  if (version >= 3) {
+    for (auto &o: b.miner_tx.vout) {
+      if (!is_valid_decomposed_amount(o.amount)) {
+        LOG_PRINT_L1("miner tx output " << print_money(o.amount) << " is not a valid decomposed amount");
+        return false;
+      }
+    }
+  }
+
   std::vector<size_t> last_blocks_sizes;
   get_last_n_blocks_sizes(last_blocks_sizes, CRYPTONOTE_REWARD_BLOCKS_WINDOW);
   if (!get_block_reward(epee::misc_utils::median(last_blocks_sizes), cumulative_block_size, already_generated_coins, base_reward, get_current_hard_fork_version()))
@@ -981,15 +993,14 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl
     // to show the amount of coins that were actually generated, the remainder will be pushed back for later
     // emission. This modifies the emission curve very slightly.
     CHECK_AND_ASSERT_MES(money_in_use - fee <= base_reward, false, "base reward calculation bug");
-    base_reward = money_in_use - fee;
     if(base_reward + fee != money_in_use)
       partial_block_reward = true;
+    base_reward = money_in_use - fee;
   }
   return true;
 }
 //------------------------------------------------------------------
-// get the block sizes of the last <count> blocks, starting at <from_height>
+// get the block sizes of the last <count> blocks, and return by reference <sz>.
-// and return by reference <sz>.
 void Blockchain::get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) const
 {
   LOG_PRINT_L3("Blockchain::" << __func__);
@@ -1405,6 +1416,10 @@ bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list<block
 //TODO: This function *looks* like it won't need to be rewritten
 //      to use BlockchainDB, as it calls other functions that were,
 //      but it warrants some looking into later.
+//
+//FIXME: This function appears to want to return false if any transactions
+//       that belong with blocks are missing, but not if blocks themselves
+//       are missing.
 bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp)
 {
   LOG_PRINT_L3("Blockchain::" << __func__);
@@ -1418,6 +1433,9 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO
   {
     std::list<crypto::hash> missed_tx_ids;
     std::list<transaction> txs;
+
+    // FIXME: s/rsp.missed_ids/missed_tx_id/ ?  Seems like rsp.missed_ids
+    //        is for missed blocks, not missed transactions as well.
     get_transactions(bl.tx_hashes, txs, missed_tx_ids);
 
     if (missed_tx_ids.size() != 0)
@@ -1665,6 +1683,8 @@ uint64_t Blockchain::block_difficulty(uint64_t i) const
   return 0;
 }
 //------------------------------------------------------------------
+//TODO: return type should be void, throw on exception
+//       alternatively, return true only if no blocks missed
 template<class t_ids_container, class t_blocks_container, class t_missed_container>
 bool Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) const
 {
@@ -1689,6 +1709,8 @@ bool Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container
   return true;
 }
 //------------------------------------------------------------------
+//TODO: return type should be void, throw on exception
+//       alternatively, return true only if no transactions missed
 template<class t_ids_container, class t_tx_container, class t_missed_container>
 bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const
 {
@@ -1705,7 +1727,6 @@ bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container
     {
       missed_txs.push_back(tx_hash);
     }
-    //FIXME: is this the correct way to handle this?
     catch (const std::exception& e)
     {
       return false;
@@ -1962,6 +1983,11 @@ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<u
   return true;
 }
 //------------------------------------------------------------------
+//FIXME: it seems this function is meant to be merely a wrapper around
+//       another function of the same name, this one adding one bit of
+//       functionality.  Should probably move anything more than that
+//       (getting the hash of the block at height max_used_block_id)
+//       to the other function to keep everything in one place.
 // This function overloads its sister function with
 // an extra value (hash of highest block that holds an output used as input)
 // as a return-by-reference.
@@ -1972,6 +1998,7 @@ bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t& max_used_block
 
 #if defined(PER_BLOCK_CHECKPOINT)
   // check if we're doing per-block checkpointing
+  // FIXME: investigate why this block returns
   if (m_db->height() < m_blocks_hash_check.size() && kept_by_block)
   {
     TIME_MEASURE_START(a);
@@ -2035,6 +2062,10 @@ bool Blockchain::have_tx_keyimges_as_spent(const transaction &tx) const
 }
 //------------------------------------------------------------------
 // This function validates transaction inputs and their keys.
+// FIXME: consider moving functionality specific to one input into
+//        check_tx_input() rather than here, and use this function simply
+//        to iterate the inputs as necessary (splitting the task
+//        using threads, etc.)
 bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height)
 {
   LOG_PRINT_L3("Blockchain::" << __func__);
@@ -2317,7 +2348,7 @@ bool Blockchain::check_tx_input(const txin_to_key& txin, const crypto::hash& tx_
 
   output_keys.clear();
 
-  //check ring signature
+  // collect output keys
   outputs_visitor vi(output_keys, *this);
   if (!scan_outputkeys_for_indexes(txin, vi, tx_prefix_hash, pmax_related_block_height))
   {
@@ -2614,6 +2645,11 @@ leave:
     txs.push_back(tx);
     TIME_MEASURE_START(dd);
 
+    // FIXME: the storage should not be responsible for validation.
+    //        If it does any, it is merely a sanity check.
+    //        Validation is the purview of the Blockchain class
+    //        - TW
+    //
     // ND: this is not needed, db->add_block() checks for duplicate k_images and fails accordingly.
     // if (!check_for_double_spend(tx, keys))
     // {
@@ -2671,7 +2707,7 @@ leave:
   TIME_MEASURE_START(vmt);
   uint64_t base_reward = 0;
   uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0;
-  if(!validate_miner_transaction(bl, cumulative_block_size, fee_summary, base_reward, already_generated_coins, bvc.m_partial_block_reward))
+  if(!validate_miner_transaction(bl, cumulative_block_size, fee_summary, base_reward, already_generated_coins, bvc.m_partial_block_reward, m_hardfork->get_current_version()))
   {
     LOG_PRINT_L1("Block with id: " << id << " has incorrect miner transaction");
     bvc.m_verifivation_failed = true;
@@ -2794,10 +2830,13 @@ bool Blockchain::add_new_block(const block& bl_, block_verification_context& bvc
   return handle_block_to_main_chain(bl, id, bvc);
 }
 //------------------------------------------------------------------
+//TODO: Refactor, consider returning a failure height and letting
+//      caller decide course of action.
 void Blockchain::check_against_checkpoints(const checkpoints& points, bool enforce)
 {
   const auto& pts = points.get_points();
 
+  CRITICAL_REGION_LOCAL(m_blockchain_lock);
   m_db->batch_start();
   for (const auto& pt : pts)
   {
@@ -2830,7 +2869,7 @@ void Blockchain::check_against_checkpoints(const checkpoints& points, bool enfor
 // with an existing checkpoint.
 bool Blockchain::update_checkpoints(const std::string& file_path, bool check_dns)
 {
-  if (!cryptonote::load_checkpoints_from_json(m_checkpoints, file_path))
+  if (!m_checkpoints.load_checkpoints_from_json(file_path))
   {
       return false;
   }
@@ -2839,7 +2878,7 @@ bool Blockchain::update_checkpoints(const std::string& file_path, bool check_dns
   // if we're not hard-enforcing dns checkpoints, handle accordingly
   if (m_enforce_dns_checkpoints && check_dns)
   {
-    if (!cryptonote::load_checkpoints_from_dns(m_checkpoints))
+    if (!m_checkpoints.load_checkpoints_from_dns())
     {
       return false;
     }
@@ -2847,7 +2886,7 @@ bool Blockchain::update_checkpoints(const std::string& file_path, bool check_dns
   else if (check_dns)
   {
     checkpoints dns_points;
-    cryptonote::load_checkpoints_from_dns(dns_points);
+    dns_points.load_checkpoints_from_dns();
     if (m_checkpoints.check_for_conflicts(dns_points))
     {
       check_against_checkpoints(dns_points, false);
@@ -2874,6 +2913,8 @@ void Blockchain::block_longhash_worker(const uint64_t height, const std::vector<
   TIME_MEASURE_START(t);
   slow_hash_allocate_state();
 
+  //FIXME: height should be changing here, as get_block_longhash expects
+  //       the height of the block passed to it
   for (const auto & block : blocks)
   {
     crypto::hash id = get_block_hash(block);
@@ -2929,6 +2970,7 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync)
 }
 
 //------------------------------------------------------------------
+//FIXME: unused parameter txs
 void Blockchain::output_scan_worker(const uint64_t amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, std::unordered_map<crypto::hash, cryptonote::transaction> &txs) const
 {
   try
@@ -3280,6 +3322,11 @@ bool Blockchain::get_hard_fork_voting_info(uint8_t version, uint32_t &window, ui
   return m_hardfork->get_voting_info(version, window, votes, threshold, earliest_height, voting);
 }
 
+std::map<uint64_t, uint64_t> Blockchain:: get_output_histogram(const std::vector<uint64_t> &amounts) const
+{
+  return m_db->get_output_histogram(amounts);
+}
+
 void Blockchain::load_compiled_in_block_hashes()
 {
   if (m_fast_sync && get_blocks_dat_start(m_testnet) != nullptr)

src/cryptonote_core/blockchain_storage.cpp

@@ -48,7 +48,7 @@
 #include "common/boost_serialization_helper.h"
 #include "warnings.h"
 #include "crypto/hash.h"
-#include "cryptonote_core/checkpoints_create.h"
+#include "cryptonote_core/checkpoints.h"
 //#include "serialization/json_archive.h"
 #include "../../contrib/otshell_utils/utils.hpp"
 #include "../../src/p2p/data_logger.hpp"
@@ -1854,7 +1854,7 @@ void blockchain_storage::check_against_checkpoints(const checkpoints& points, bo
 // with an existing checkpoint.
 bool blockchain_storage::update_checkpoints(const std::string& file_path, bool check_dns)
 {
-  if (!cryptonote::load_checkpoints_from_json(m_checkpoints, file_path))
+  if (!m_checkpoints.load_checkpoints_from_json(file_path))
   {
     return false;
   }
@@ -1863,7 +1863,7 @@ bool blockchain_storage::update_checkpoints(const std::string& file_path, bool c
   // if we're not hard-enforcing dns checkpoints, handle accordingly
   if (m_enforce_dns_checkpoints && check_dns)
   {
-    if (!cryptonote::load_checkpoints_from_dns(m_checkpoints))
+    if (!m_checkpoints.load_checkpoints_from_dns())
     {
       return false;
     }
@@ -1871,7 +1871,7 @@ bool blockchain_storage::update_checkpoints(const std::string& file_path, bool c
   else if (check_dns)
   {
     checkpoints dns_points;
-    cryptonote::load_checkpoints_from_dns(dns_points, m_testnet);
+    dns_points.load_checkpoints_from_dns(m_testnet);
     if (m_checkpoints.check_for_conflicts(dns_points))
     {
       check_against_checkpoints(dns_points, false);

src/cryptonote_core/checkpoints.cpp

@@ -29,10 +29,40 @@
 // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
 
 #include "include_base_utils.h"
+
 using namespace epee;
 
 #include "checkpoints.h"
 
+#include "common/dns_utils.h"
+#include "include_base_utils.h"
+#include <sstream>
+#include <random>
+
+namespace
+{
+  bool dns_records_match(const std::vector<std::string>& a, const std::vector<std::string>& b)
+  {
+    if (a.size() != b.size()) return false;
+
+    for (const auto& record_in_a : a)
+    {
+      bool ok = false;
+      for (const auto& record_in_b : b)
+      {
+	if (record_in_a == record_in_b)
+	{
+	  ok = true;
+	  break;
+	}
+      }
+      if (!ok) return false;
+    }
+
+    return true;
+  }
+} // anonymous namespace
+
 namespace cryptonote
 {
   //---------------------------------------------------------------------------
@@ -84,10 +114,7 @@ namespace cryptonote
     return check_block(height, h, ignored);
   }
   //---------------------------------------------------------------------------
-  // this basically says if the blockchain is smaller than the first
+  //FIXME: is this the desired behavior?
-  // checkpoint then alternate blocks are allowed.  Alternatively, if the
-  // last checkpoint *before* the end of the current chain is also before
-  // the block to be added, then this is fine.
   bool checkpoints::is_alternative_block_allowed(uint64_t blockchain_height, uint64_t block_height) const
   {
     if (0 == block_height)
@@ -128,4 +155,206 @@ namespace cryptonote
     }
     return true;
   }
+
+  bool checkpoints::init_default_checkpoints()
+  {
+    ADD_CHECKPOINT(1,     "771fbcd656ec1464d3a02ead5e18644030007a0fc664c0a964d30922821a8148");
+    ADD_CHECKPOINT(10,    "c0e3b387e47042f72d8ccdca88071ff96bff1ac7cde09ae113dbb7ad3fe92381");
+    ADD_CHECKPOINT(100,   "ac3e11ca545e57c49fca2b4e8c48c03c23be047c43e471e1394528b1f9f80b2d");
+    ADD_CHECKPOINT(1000,  "5acfc45acffd2b2e7345caf42fa02308c5793f15ec33946e969e829f40b03876");
+    ADD_CHECKPOINT(10000, "c758b7c81f928be3295d45e230646de8b852ec96a821eac3fea4daf3fcac0ca2");
+    ADD_CHECKPOINT(22231, "7cb10e29d67e1c069e6e11b17d30b809724255fee2f6868dc14cfc6ed44dfb25");
+    ADD_CHECKPOINT(29556, "53c484a8ed91e4da621bb2fa88106dbde426fe90d7ef07b9c1e5127fb6f3a7f6");
+    ADD_CHECKPOINT(50000, "0fe8758ab06a8b9cb35b7328fd4f757af530a5d37759f9d3e421023231f7b31c");
+    ADD_CHECKPOINT(80000, "a62dcd7b536f22e003ebae8726e9e7276f63d594e264b6f0cd7aab27b66e75e3");
+    ADD_CHECKPOINT(202612, "bbd604d2ba11ba27935e006ed39c9bfdd99b76bf4a50654bc1e1e61217962698");
+    ADD_CHECKPOINT(202613, "e2aa337e78df1f98f462b3b1e560c6b914dec47b610698b7b7d1e3e86b6197c2");
+    ADD_CHECKPOINT(202614, "c29e3dc37d8da3e72e506e31a213a58771b24450144305bcba9e70fa4d6ea6fb");
+    ADD_CHECKPOINT(205000, "5d3d7a26e6dc7535e34f03def711daa8c263785f73ec1fadef8a45880fde8063");
+    ADD_CHECKPOINT(220000, "9613f455933c00e3e33ac315cc6b455ee8aa0c567163836858c2d9caff111553");
+    ADD_CHECKPOINT(230300, "bae7a80c46859db355556e3a9204a337ae8f24309926a1312323fdecf1920e61");
+    ADD_CHECKPOINT(230700, "93e631240ceac831da1aebfc5dac8f722c430463024763ebafa888796ceaeedf");
+    ADD_CHECKPOINT(231350, "b5add137199b820e1ea26640e5c3e121fd85faa86a1e39cf7e6cc097bdeb1131");
+    ADD_CHECKPOINT(232150, "955de8e6b6508af2c24f7334f97beeea651d78e9ade3ab18fec3763be3201aa8");
+    ADD_CHECKPOINT(249380, "654fb0a81ce3e5caf7e3264a70f447d4bd07586c08fa50f6638cc54da0a52b2d");
+    ADD_CHECKPOINT(460000, "75037a7aed3e765db96c75bcf908f59d690a5f3390baebb9edeafd336a1c4831");
+    ADD_CHECKPOINT(500000, "2428f0dbe49796be05ed81b347f53e1f7f44aed0abf641446ec2b94cae066b02");
+    ADD_CHECKPOINT(600000, "f5828ebf7d7d1cb61762c4dfe3ccf4ecab2e1aad23e8113668d981713b7a54c5");
+    ADD_CHECKPOINT(700000, "12be9b3d210b93f574d2526abb9c1ab2a881b479131fd0d4f7dac93875f503cd");
+    ADD_CHECKPOINT(825000, "56503f9ad766774b575be3aff73245e9d159be88132c93d1754764f28da2ff60");
+    ADD_CHECKPOINT(900000, "d9958d0e7dcf91a5a7b11de225927bf7efc6eb26240315ce12372be902cc1337");
+    ADD_CHECKPOINT(913193, "5292d5d56f6ba4de33a58d9a34d263e2cb3c6fee0aed2286fd4ac7f36d53c85f");
+
+    return true;
+  }
+
+  bool checkpoints::load_checkpoints_from_json(const std::string json_hashfile_fullpath)
+  {
+    boost::system::error_code errcode;
+    if (! (boost::filesystem::exists(json_hashfile_fullpath, errcode)))
+    {
+      LOG_PRINT_L1("Blockchain checkpoints file not found");
+      return true;
+    }
+
+    LOG_PRINT_L1("Adding checkpoints from blockchain hashfile");
+
+    uint64_t prev_max_height = get_max_height();
+    LOG_PRINT_L1("Hard-coded max checkpoint height is " << prev_max_height);
+    t_hash_json hashes;
+    epee::serialization::load_t_from_json_file(hashes, json_hashfile_fullpath);
+    for (std::vector<t_hashline>::const_iterator it = hashes.hashlines.begin(); it != hashes.hashlines.end(); )
+    {
+      uint64_t height;
+      height = it->height;
+      if (height <= prev_max_height) {
+	LOG_PRINT_L1("ignoring checkpoint height " << height);
+      } else {
+	std::string blockhash = it->hash;
+	LOG_PRINT_L1("Adding checkpoint height " << height << ", hash=" << blockhash);
+	ADD_CHECKPOINT(height, blockhash);
+      }
+      ++it;
+    }
+
+    return true;
+  }
+
+  bool checkpoints::load_checkpoints_from_dns(bool testnet)
+  {
+    // All four MoneroPulse domains have DNSSEC on and valid
+    static const std::vector<std::string> dns_urls = { "checkpoints.moneropulse.se"
+						     , "checkpoints.moneropulse.org"
+						     , "checkpoints.moneropulse.net"
+						     , "checkpoints.moneropulse.co"
+    };
+
+    static const std::vector<std::string> testnet_dns_urls = { "testpoints.moneropulse.se"
+							     , "testpoints.moneropulse.org"
+							     , "testpoints.moneropulse.net"
+							     , "testpoints.moneropulse.co"
+    };
+
+    std::vector<std::vector<std::string> > records;
+    records.resize(dns_urls.size());
+
+    std::random_device rd;
+    std::mt19937 gen(rd());
+    std::uniform_int_distribution<int> dis(0, dns_urls.size() - 1);
+    size_t first_index = dis(gen);
+
+    bool avail, valid;
+    size_t cur_index = first_index;
+    do
+    {
+      std::string url;
+      if (testnet)
+      {
+        url = testnet_dns_urls[cur_index];
+      }
+      else
+      {
+        url = dns_urls[cur_index];
+      }
+
+      records[cur_index] = tools::DNSResolver::instance().get_txt_record(url, avail, valid);
+      if (!avail)
+      {
+        records[cur_index].clear();
+        LOG_PRINT_L2("DNSSEC not available for checkpoint update at URL: " << url << ", skipping.");
+      }
+      if (!valid)
+      {
+        records[cur_index].clear();
+        LOG_PRINT_L2("DNSSEC validation failed for checkpoint update at URL: " << url << ", skipping.");
+      }
+
+      cur_index++;
+      if (cur_index == dns_urls.size())
+      {
+        cur_index = 0;
+      }
+      records[cur_index].clear();
+    } while (cur_index != first_index);
+
+    size_t num_valid_records = 0;
+
+    for( const auto& record_set : records)
+    {
+      if (record_set.size() != 0)
+      {
+        num_valid_records++;
+      }
+    }
+
+    if (num_valid_records < 2)
+    {
+      LOG_PRINT_L0("WARNING: no two valid MoneroPulse DNS checkpoint records were received");
+      return true;
+    }
+
+    int good_records_index = -1;
+    for (size_t i = 0; i < records.size() - 1; ++i)
+    {
+      if (records[i].size() == 0) continue;
+
+      for (size_t j = i + 1; j < records.size(); ++j)
+      {
+        if (dns_records_match(records[i], records[j]))
+        {
+    good_records_index = i;
+    break;
+        }
+      }
+      if (good_records_index >= 0) break;
+    }
+
+    if (good_records_index < 0)
+    {
+      LOG_PRINT_L0("WARNING: no two MoneroPulse DNS checkpoint records matched");
+      return true;
+    }
+
+    for (auto& record : records[good_records_index])
+    {
+      auto pos = record.find(":");
+      if (pos != std::string::npos)
+      {
+        uint64_t height;
+        crypto::hash hash;
+
+        // parse the first part as uint64_t,
+        // if this fails move on to the next record
+        std::stringstream ss(record.substr(0, pos));
+        if (!(ss >> height))
+        {
+    continue;
+        }
+
+        // parse the second part as crypto::hash,
+        // if this fails move on to the next record
+        std::string hashStr = record.substr(pos + 1);
+        if (!epee::string_tools::parse_tpod_from_hex_string(hashStr, hash))
+        {
+    continue;
+        }
+
+        ADD_CHECKPOINT(height, hashStr);
+      }
+    }
+    return true;
+  }
+
+  bool checkpoints::load_new_checkpoints(const std::string json_hashfile_fullpath, bool testnet, bool dns)
+  {
+    bool result;
+
+    result = load_checkpoints_from_json(json_hashfile_fullpath);
+    if (dns)
+    {
+      result &= load_checkpoints_from_dns(testnet);
+    }
+
+    return result;
+  }
 }

src/cryptonote_core/checkpoints.h

@@ -32,23 +32,186 @@
 #include <map>
 #include <vector>
 #include "cryptonote_basic_impl.h"
+#include "misc_log_ex.h"
+#include "storages/portable_storage_template_helper.h" // epee json include
+
+#define ADD_CHECKPOINT(h, hash)  CHECK_AND_ASSERT(add_checkpoint(h,  hash), false);
+#define JSON_HASH_FILE_NAME "checkpoints.json"
 
 
 namespace cryptonote
 {
+  /**
+   * @brief A container for blockchain checkpoints
+   *
+   * A checkpoint is a pre-defined hash for the block at a given height.
+   * Some of these are compiled-in, while others can be loaded at runtime
+   * either from a json file or via DNS from a checkpoint-hosting server.
+   */
   class checkpoints
   {
   public:
+
+    /**
+     * @brief default constructor
+     */
     checkpoints();
+
+    /**
+     * @brief adds a checkpoint to the container
+     *
+     * @param height the height of the block the checkpoint is for
+     * @param hash_str the hash of the block, as a string
+     *
+     * @return false if parsing the hash fails, or if the height is a duplicate
+     *         AND the existing checkpoint hash does not match the new one,
+     *         otherwise returns true
+     */
     bool add_checkpoint(uint64_t height, const std::string& hash_str);
+
+    /**
+     * @brief checks if there is a checkpoint in the future
+     *
+     * This function checks if the height passed is lower than the highest
+     * checkpoint.
+     *
+     * @param height the height to check against
+     *
+     * @return false if no checkpoints, otherwise returns whether or not
+     *         the height passed is lower than the highest checkpoint.
+     */
     bool is_in_checkpoint_zone(uint64_t height) const;
-    bool check_block(uint64_t height, const crypto::hash& h) const;
+
+    /**
+     * @brief checks if the given height and hash agree with the checkpoints
+     *
+     * This function checks if the given height and hash exist in the
+     * checkpoints container.  If so, it returns whether or not the passed
+     * parameters match the stored values.
+     *
+     * @param height the height to be checked
+     * @param h the hash to be checked
+     * @param is_a_checkpoint return-by-reference if there is a checkpoint at the given height
+     *
+     * @return true if there is no checkpoint at the given height,
+     *         true if the passed parameters match the stored checkpoint,
+     *         false otherwise
+     */
     bool check_block(uint64_t height, const crypto::hash& h, bool& is_a_checkpoint) const;
+
+    /**
+     * @overload
+     */
+    bool check_block(uint64_t height, const crypto::hash& h) const;
+
+    /**
+     * @brief checks if alternate chain blocks should be kept for a given height
+     *
+     * this basically says if the blockchain is smaller than the first
+     * checkpoint then alternate blocks are allowed.  Alternatively, if the
+     * last checkpoint *before* the end of the current chain is also before
+     * the block to be added, then this is fine.
+     *
+     * @param blockchain_height the current blockchain height
+     * @param block_height the height of the block to be added as alternate
+     *
+     * @return true if alternate blocks are allowed given the parameters,
+     *         otherwise false
+     */
     bool is_alternative_block_allowed(uint64_t blockchain_height, uint64_t block_height) const;
+
+    /**
+     * @brief gets the highest checkpoint height
+     *
+     * @return the height of the highest checkpoint
+     */
     uint64_t get_max_height() const;
+
+    /**
+     * @brief gets the checkpoints container
+     *
+     * @return a const reference to the checkpoints container
+     */
     const std::map<uint64_t, crypto::hash>& get_points() const;
+
+    /**
+     * @brief checks if our checkpoints container conflicts with another
+     *
+     * A conflict refers to a case where both checkpoint sets have a checkpoint
+     * for a specific height but their hashes for that height do not match.
+     *
+     * @param other the other checkpoints instance to check against
+     *
+     * @return false if any conflict is found, otherwise true
+     */
     bool check_for_conflicts(const checkpoints& other) const;
+
+    /**
+     * @brief loads the default main chain checkpoints
+     *
+     * @return true unless adding a checkpoint fails
+     */
+    bool init_default_checkpoints();
+
+    /**
+     * @brief load new checkpoints
+     *
+     * Loads new checkpoints from the specified json file, as well as
+     * (optionally) from DNS.
+     *
+     * @param json_hashfile_fullpath path to the json checkpoints file
+     * @param testnet whether to load testnet checkpoints or mainnet
+     * @param dns whether or not to load DNS checkpoints
+     *
+     * @return true if loading successful and no conflicts
+     */
+    bool load_new_checkpoints(const std::string json_hashfile_fullpath, bool testnet=false, bool dns=true);
+
+    /**
+     * @brief load new checkpoints from json
+     *
+     * @param json_hashfile_fullpath path to the json checkpoints file
+     *
+     * @return true if loading successful and no conflicts
+     */
+    bool load_checkpoints_from_json(const std::string json_hashfile_fullpath);
+
+    /**
+     * @brief load new checkpoints from DNS
+     *
+     * @param testnet whether to load testnet checkpoints or mainnet
+     *
+     * @return true if loading successful and no conflicts
+     */
+    bool load_checkpoints_from_dns(bool testnet = false);
+
   private:
-    std::map<uint64_t, crypto::hash> m_points;
+
+
+    /**
+     * @brief struct for loading a checkpoint from json
+     */
+  struct t_hashline
+  {
+    uint64_t height; //!< the height of the checkpoint
+    std::string hash; //!< the hash for the checkpoint
+        BEGIN_KV_SERIALIZE_MAP()
+          KV_SERIALIZE(height)
+          KV_SERIALIZE(hash)
+        END_KV_SERIALIZE_MAP()
+  };
+
+  /**
+   * @brief struct for loading many checkpoints from json
+   */
+  struct t_hash_json {
+    std::vector<t_hashline> hashlines; //!< the checkpoint lines from the file
+        BEGIN_KV_SERIALIZE_MAP()
+          KV_SERIALIZE(hashlines)
+        END_KV_SERIALIZE_MAP()
+  };
+
+    std::map<uint64_t, crypto::hash> m_points; //!< the checkpoints container
+
   };
 }

src/cryptonote_core/checkpoints_create.cpp

@@ -1,280 +0,0 @@
-// Copyright (c) 2014-2016, The Monero Project
-//
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without modification, are
-// permitted provided that the following conditions are met:
-//
-// 1. Redistributions of source code must retain the above copyright notice, this list of
-//    conditions and the following disclaimer.
-//
-// 2. Redistributions in binary form must reproduce the above copyright notice, this list
-//    of conditions and the following disclaimer in the documentation and/or other
-//    materials provided with the distribution.
-//
-// 3. Neither the name of the copyright holder nor the names of its contributors may be
-//    used to endorse or promote products derived from this software without specific
-//    prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
-// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
-// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
-// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
-// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
-
-#include "checkpoints_create.h"
-#include "common/dns_utils.h"
-#include "include_base_utils.h"
-#include <sstream>
-#include <random>
-#include "storages/portable_storage_template_helper.h" // epee json include
-
-namespace
-{
-  bool dns_records_match(const std::vector<std::string>& a, const std::vector<std::string>& b)
-  {
-    if (a.size() != b.size()) return false;
-
-    for (const auto& record_in_a : a)
-    {
-      bool ok = false;
-      for (const auto& record_in_b : b)
-      {
-        if (record_in_a == record_in_b)
-        {
-          ok = true;
-          break;
-        }
-      }
-      if (!ok) return false;
-    }
-
-    return true;
-  }
-} // anonymous namespace
-
-namespace cryptonote
-{
-
-struct t_hashline
-{
-  uint64_t height;
-  std::string hash;
-  BEGIN_KV_SERIALIZE_MAP()
-    KV_SERIALIZE(height)
-    KV_SERIALIZE(hash)
-  END_KV_SERIALIZE_MAP()
-};
-
-struct t_hash_json {
-  std::vector<t_hashline> hashlines;
-  BEGIN_KV_SERIALIZE_MAP()
-    KV_SERIALIZE(hashlines)
-  END_KV_SERIALIZE_MAP()
-};
-
-bool create_checkpoints(cryptonote::checkpoints& checkpoints)
-{
-  ADD_CHECKPOINT(1,     "771fbcd656ec1464d3a02ead5e18644030007a0fc664c0a964d30922821a8148");
-  ADD_CHECKPOINT(10,    "c0e3b387e47042f72d8ccdca88071ff96bff1ac7cde09ae113dbb7ad3fe92381");
-  ADD_CHECKPOINT(100,   "ac3e11ca545e57c49fca2b4e8c48c03c23be047c43e471e1394528b1f9f80b2d");
-  ADD_CHECKPOINT(1000,  "5acfc45acffd2b2e7345caf42fa02308c5793f15ec33946e969e829f40b03876");
-  ADD_CHECKPOINT(10000, "c758b7c81f928be3295d45e230646de8b852ec96a821eac3fea4daf3fcac0ca2");
-  ADD_CHECKPOINT(22231, "7cb10e29d67e1c069e6e11b17d30b809724255fee2f6868dc14cfc6ed44dfb25");
-  ADD_CHECKPOINT(29556, "53c484a8ed91e4da621bb2fa88106dbde426fe90d7ef07b9c1e5127fb6f3a7f6");
-  ADD_CHECKPOINT(50000, "0fe8758ab06a8b9cb35b7328fd4f757af530a5d37759f9d3e421023231f7b31c");
-  ADD_CHECKPOINT(80000, "a62dcd7b536f22e003ebae8726e9e7276f63d594e264b6f0cd7aab27b66e75e3");
-  ADD_CHECKPOINT(202612, "bbd604d2ba11ba27935e006ed39c9bfdd99b76bf4a50654bc1e1e61217962698");
-  ADD_CHECKPOINT(202613, "e2aa337e78df1f98f462b3b1e560c6b914dec47b610698b7b7d1e3e86b6197c2");
-  ADD_CHECKPOINT(202614, "c29e3dc37d8da3e72e506e31a213a58771b24450144305bcba9e70fa4d6ea6fb");
-  ADD_CHECKPOINT(205000, "5d3d7a26e6dc7535e34f03def711daa8c263785f73ec1fadef8a45880fde8063");
-  ADD_CHECKPOINT(220000, "9613f455933c00e3e33ac315cc6b455ee8aa0c567163836858c2d9caff111553");
-  ADD_CHECKPOINT(230300, "bae7a80c46859db355556e3a9204a337ae8f24309926a1312323fdecf1920e61");
-  ADD_CHECKPOINT(230700, "93e631240ceac831da1aebfc5dac8f722c430463024763ebafa888796ceaeedf");
-  ADD_CHECKPOINT(231350, "b5add137199b820e1ea26640e5c3e121fd85faa86a1e39cf7e6cc097bdeb1131");
-  ADD_CHECKPOINT(232150, "955de8e6b6508af2c24f7334f97beeea651d78e9ade3ab18fec3763be3201aa8");
-  ADD_CHECKPOINT(249380, "654fb0a81ce3e5caf7e3264a70f447d4bd07586c08fa50f6638cc54da0a52b2d");
-  ADD_CHECKPOINT(300000, "0c1cd46df6ccff90ec4ab493281f2583c344cd62216c427628990fe9db1bb8b6");
-  ADD_CHECKPOINT(400000, "1b2b0e7a30e59691491529a3d506d1ba3d6052d0f6b52198b7330b28a6f1b6ac");
-  ADD_CHECKPOINT(450000, "4d098b511ca97723e81737c448343cfd4e6dadb3d8a0e757c6e4d595e6e48357");
-  ADD_CHECKPOINT(460000, "75037a7aed3e765db96c75bcf908f59d690a5f3390baebb9edeafd336a1c4831");
-  ADD_CHECKPOINT(500000, "2428f0dbe49796be05ed81b347f53e1f7f44aed0abf641446ec2b94cae066b02");
-  ADD_CHECKPOINT(600000, "f5828ebf7d7d1cb61762c4dfe3ccf4ecab2e1aad23e8113668d981713b7a54c5");
-  ADD_CHECKPOINT(700000, "12be9b3d210b93f574d2526abb9c1ab2a881b479131fd0d4f7dac93875f503cd");
-  ADD_CHECKPOINT(825000, "56503f9ad766774b575be3aff73245e9d159be88132c93d1754764f28da2ff60");
-  ADD_CHECKPOINT(900000, "d9958d0e7dcf91a5a7b11de225927bf7efc6eb26240315ce12372be902cc1337");
-  ADD_CHECKPOINT(913193, "5292d5d56f6ba4de33a58d9a34d263e2cb3c6fee0aed2286fd4ac7f36d53c85f");
-
-  return true;
-}
-
-bool load_checkpoints_from_json(cryptonote::checkpoints& checkpoints, std::string json_hashfile_fullpath)
-{
-  boost::system::error_code errcode;
-  if (! (boost::filesystem::exists(json_hashfile_fullpath, errcode)))
-  {
-    LOG_PRINT_L1("Blockchain checkpoints file not found");
-    return true;
-  }
-
-  LOG_PRINT_L1("Adding checkpoints from blockchain hashfile");
-
-  uint64_t prev_max_height = checkpoints.get_max_height();
-  LOG_PRINT_L1("Hard-coded max checkpoint height is " << prev_max_height);
-  t_hash_json hashes;
-  epee::serialization::load_t_from_json_file(hashes, json_hashfile_fullpath);
-  for (std::vector<t_hashline>::const_iterator it = hashes.hashlines.begin(); it != hashes.hashlines.end(); )
-  {
-      uint64_t height;
-      height = it->height;
-      if (height <= prev_max_height) {
-        LOG_PRINT_L1("ignoring checkpoint height " << height);
-      } else {
-        std::string blockhash = it->hash;
-        LOG_PRINT_L1("Adding checkpoint height " << height << ", hash=" << blockhash);
-        ADD_CHECKPOINT(height, blockhash);
-      }
-      ++it;
-  }
-
-  return true;
-}
-
-bool load_checkpoints_from_dns(cryptonote::checkpoints& checkpoints, bool testnet)
-{
-  // All four MoneroPulse domains have DNSSEC on and valid
-  static const std::vector<std::string> dns_urls = { "checkpoints.moneropulse.se"
-                                                   , "checkpoints.moneropulse.org"
-                                                   , "checkpoints.moneropulse.net"
-                                                   , "checkpoints.moneropulse.co"
-  };
-
-  static const std::vector<std::string> testnet_dns_urls = { "testpoints.moneropulse.se"
-                                                           , "testpoints.moneropulse.org"
-                                                           , "testpoints.moneropulse.net"
-                                                           , "testpoints.moneropulse.co"
-  };
-
-  std::vector<std::vector<std::string> > records;
-  records.resize(dns_urls.size());
-
-  std::random_device rd;
-  std::mt19937 gen(rd());
-  std::uniform_int_distribution<int> dis(0, dns_urls.size() - 1);
-  size_t first_index = dis(gen);
-
-  bool avail, valid;
-  size_t cur_index = first_index;
-  do
-  {
-    std::string url;
-    if (testnet)
-    {
-      url = testnet_dns_urls[cur_index];
-    }
-    else
-    {
-      url = dns_urls[cur_index];
-    }
-
-    records[cur_index] = tools::DNSResolver::instance().get_txt_record(url, avail, valid);
-    if (!avail)
-    {
-      records[cur_index].clear();
-      LOG_PRINT_L2("DNSSEC not available for checkpoint update at URL: " << url << ", skipping.");
-    }
-    if (!valid)
-    {
-      records[cur_index].clear();
-      LOG_PRINT_L2("DNSSEC validation failed for checkpoint update at URL: " << url << ", skipping.");
-    }
-
-    cur_index++;
-    if (cur_index == dns_urls.size())
-    {
-      cur_index = 0;
-    }
-    records[cur_index].clear();
-  } while (cur_index != first_index);
-
-  size_t num_valid_records = 0;
-
-  for( const auto& record_set : records)
-  {
-    if (record_set.size() != 0)
-    {
-      num_valid_records++;
-    }
-  }
-
-  if (num_valid_records < 2)
-  {
-    LOG_PRINT_L0("WARNING: no two valid MoneroPulse DNS checkpoint records were received");
-    return true;
-  }
-
-  int good_records_index = -1;
-  for (size_t i = 0; i < records.size() - 1; ++i)
-  {
-    if (records[i].size() == 0) continue;
-
-    for (size_t j = i + 1; j < records.size(); ++j)
-    {
-      if (dns_records_match(records[i], records[j]))
-      {
-        good_records_index = i;
-        break;
-      }
-    }
-    if (good_records_index >= 0) break;
-  }
-
-  if (good_records_index < 0)
-  {
-    LOG_PRINT_L0("WARNING: no two MoneroPulse DNS checkpoint records matched");
-    return true;
-  }
-
-  for (auto& record : records[good_records_index])
-  {
-    auto pos = record.find(":");
-    if (pos != std::string::npos)
-    {
-      uint64_t height;
-      crypto::hash hash;
-
-      // parse the first part as uint64_t,
-      // if this fails move on to the next record
-      std::stringstream ss(record.substr(0, pos));
-      if (!(ss >> height))
-      {
-        continue;
-      }
-
-      // parse the second part as crypto::hash,
-      // if this fails move on to the next record
-      std::string hashStr = record.substr(pos + 1);
-      if (!epee::string_tools::parse_tpod_from_hex_string(hashStr, hash))
-      {
-        continue;
-      }
-
-      ADD_CHECKPOINT(height, hashStr);
-    }
-  }
-  return true;
-}
-
-bool load_new_checkpoints(cryptonote::checkpoints& checkpoints, std::string json_hashfile_fullpath)
-{
-  // TODO: replace hard-coded url with const string or #define
-  return (load_checkpoints_from_json(checkpoints, json_hashfile_fullpath) && load_checkpoints_from_dns(checkpoints));
-}
-
-}  // namespace cryptonote

src/cryptonote_core/checkpoints_create.h

@@ -1,48 +0,0 @@
-// Copyright (c) 2014-2016, The Monero Project
-// 
-// All rights reserved.
-// 
-// Redistribution and use in source and binary forms, with or without modification, are
-// permitted provided that the following conditions are met:
-// 
-// 1. Redistributions of source code must retain the above copyright notice, this list of
-//    conditions and the following disclaimer.
-// 
-// 2. Redistributions in binary form must reproduce the above copyright notice, this list
-//    of conditions and the following disclaimer in the documentation and/or other
-//    materials provided with the distribution.
-// 
-// 3. Neither the name of the copyright holder nor the names of its contributors may be
-//    used to endorse or promote products derived from this software without specific
-//    prior written permission.
-// 
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
-// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
-// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
-// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
-// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-// 
-// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
-
-#pragma once
-
-#include "checkpoints.h"
-#include "misc_log_ex.h"
-
-#define ADD_CHECKPOINT(h, hash)  CHECK_AND_ASSERT(checkpoints.add_checkpoint(h,  hash), false);
-#define JSON_HASH_FILE_NAME "checkpoints.json"
-
-namespace cryptonote
-{
-
-  bool create_checkpoints(cryptonote::checkpoints& checkpoints);
-
-  bool load_checkpoints_from_json(cryptonote::checkpoints& checkpoints, std::string json_hashfile_fullpath);
-  bool load_checkpoints_from_dns(cryptonote::checkpoints& checkpoints, bool testnet = false);
-  bool load_new_checkpoints(cryptonote::checkpoints& checkpoints, std::string json_hashfile_fullpath);
-
-}  // namespace cryptonote

src/cryptonote_core/cryptonote_core.cpp

@@ -42,7 +42,7 @@ using namespace epee;
 #include "cryptonote_format_utils.h"
 #include "misc_language.h"
 #include <csignal>
-#include "cryptonote_core/checkpoints_create.h"
+#include "cryptonote_core/checkpoints.h"
 #include "blockchain_db/blockchain_db.h"
 #include "blockchain_db/lmdb/db_lmdb.h"
 #if defined(BERKELEY_DB)
@@ -159,7 +159,7 @@ namespace cryptonote
     if (!m_testnet && !m_fakechain)
     {
       cryptonote::checkpoints checkpoints;
-      if (!cryptonote::create_checkpoints(checkpoints))
+      if (!checkpoints.init_default_checkpoints())
       {
         throw std::runtime_error("Failed to initialize checkpoints");
       }
@@ -415,7 +415,7 @@ namespace cryptonote
     CHECK_AND_ASSERT_MES(update_checkpoints(), false, "One or more checkpoints loaded from json or dns conflicted with existing checkpoints.");
 
     r = m_miner.init(vm, m_testnet);
-    CHECK_AND_ASSERT_MES(r, false, "Failed to initialize blockchain storage");
+    CHECK_AND_ASSERT_MES(r, false, "Failed to initialize miner instance");
 
     return load_state_data();
   }
@@ -634,11 +634,6 @@ namespace cryptonote
     return m_blockchain_storage.get_total_transactions();
   }
   //-----------------------------------------------------------------------------------------------
-  //bool core::get_outs(uint64_t amount, std::list<crypto::public_key>& pkeys)
-  //{
-  //  return m_blockchain_storage.get_outs(amount, pkeys);
-  //}
-  //-----------------------------------------------------------------------------------------------
   bool core::add_new_tx(const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed)
   {
     if(m_mempool.have_tx(tx_hash))
@@ -770,10 +765,6 @@ namespace cryptonote
   {
     m_miner.on_synchronized();
   }
-  //bool core::get_backward_blocks_sizes(uint64_t from_height, std::vector<size_t>& sizes, size_t count)
-  //{
-  //  return m_blockchain_storage.get_backward_blocks_sizes(from_height, sizes, count);
-  //}
   //-----------------------------------------------------------------------------------------------
   bool core::add_new_block(const block& b, block_verification_context& bvc)
   {
@@ -894,10 +885,6 @@ namespace cryptonote
     return m_blockchain_storage.get_block_by_hash(h, blk);
   }
   //-----------------------------------------------------------------------------------------------
-  //void core::get_all_known_block_ids(std::list<crypto::hash> &main, std::list<crypto::hash> &alt, std::list<crypto::hash> &invalid) {
-  //  m_blockchain_storage.get_all_known_block_ids(main, alt, invalid);
-  //}
-  //-----------------------------------------------------------------------------------------------
   std::string core::print_pool(bool short_format) const
   {
     return m_mempool.print_pool(short_format);

src/cryptonote_core/cryptonote_core.h

@@ -63,157 +63,739 @@ namespace cryptonote
   /************************************************************************/
   /*                                                                      */
   /************************************************************************/
+
+   /**
+    * @brief handles core cryptonote functionality
+    *
+    * This class coordinates cryptonote functionality including, but not
+    * limited to, communication among the Blockchain, the transaction pool,
+    * any miners, and the network.
+    */
    class core: public i_miner_handler
    {
    public:
+
+      /**
+       * @brief constructor
+       *
+       * sets member variables into a usable state
+       *
+       * @param pprotocol pre-constructed protocol object to store and use
+       */
      core(i_cryptonote_protocol* pprotocol);
+
+    /**
+     * @copydoc Blockchain::handle_get_objects
+     *
+     * @note see Blockchain::handle_get_objects()
+     * @param context connection context associated with the request
+     */
      bool handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp, cryptonote_connection_context& context);
+
+     /**
+      * @brief calls various idle routines
+      *
+      * @note see miner::on_idle and tx_memory_pool::on_idle
+      *
+      * @return true
+      */
      bool on_idle();
+
+     /**
+      * @brief handles an incoming transaction
+      *
+      * Parses an incoming transaction and, if nothing is obviously wrong,
+      * passes it along to the transaction pool
+      *
+      * @param tx_blob the tx to handle
+      * @param tvc metadata about the transaction's validity
+      * @param keeped_by_block if the transaction has been in a block
+      * @param relayed whether or not the transaction was relayed to us
+      *
+      * @return true if the transaction made it to the transaction pool, otherwise false
+      */
      bool handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed);
+
+     /**
+      * @brief handles an incoming block
+      *
+      * periodic update to checkpoints is triggered here
+      * Attempts to add the block to the Blockchain and, on success,
+      * optionally updates the miner's block template.
+      *
+      * @param block_blob the block to be added
+      * @param bvc return-by-reference metadata context about the block's validity
+      * @param update_miner_blocktemplate whether or not to update the miner's block template
+      *
+      * @return false if loading new checkpoints fails, or the block is not
+      * added, otherwise true
+      */
      bool handle_incoming_block(const blobdata& block_blob, block_verification_context& bvc, bool update_miner_blocktemplate = true);
+
+     /**
+      * @copydoc Blockchain::prepare_handle_incoming_blocks
+      *
+      * @note see Blockchain::prepare_handle_incoming_blocks
+      */
      bool prepare_handle_incoming_blocks(const std::list<block_complete_entry>  &blocks);
+
+     /**
+      * @copydoc Blockchain::cleanup_handle_incoming_blocks
+      *
+      * @note see Blockchain::cleanup_handle_incoming_blocks
+      */
      bool cleanup_handle_incoming_blocks(bool force_sync = false);
      	     	
+     /**
+      * @brief check the size of a block against the current maximum
+      *
+      * @param block_blob the block to check
+      *
+      * @return whether or not the block is too big
+      */
      bool check_incoming_block_size(const blobdata& block_blob) const;
+
+     /**
+      * @brief get the cryptonote protocol instance
+      *
+      * @return the instance
+      */
      i_cryptonote_protocol* get_protocol(){return m_pprotocol;}
 
      //-------------------- i_miner_handler -----------------------
+
+     /**
+      * @brief stores and relays a block found by a miner
+      *
+      * Updates the miner's target block, attempts to store the found
+      * block in Blockchain, and -- on success -- relays that block to
+      * the network.
+      *
+      * @param b the block found
+      *
+      * @return true if the block was added to the main chain, otherwise false
+      */
      virtual bool handle_block_found( block& b);
+
+     /**
+      * @copydoc Blockchain::create_block_template
+      *
+      * @note see Blockchain::create_block_template
+      */
      virtual bool get_block_template(block& b, const account_public_address& adr, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce);
 
 
+     /**
+      * @brief gets the miner instance
+      *
+      * @return a reference to the miner instance
+      */
      miner& get_miner(){return m_miner;}
+
+     /**
+      * @brief gets the miner instance (const)
+      *
+      * @return a const reference to the miner instance
+      */
      const miner& get_miner()const{return m_miner;}
+
+     /**
+      * @brief adds command line options to the given options set
+      *
+      * As of now, there are no command line options specific to core,
+      * so this function simply returns.
+      *
+      * @param desc return-by-reference the command line options set to add to
+      */
      static void init_options(boost::program_options::options_description& desc);
+
+     /**
+      * @brief initializes the core as needed
+      *
+      * This function initializes the transaction pool, the Blockchain, and
+      * a miner instance with parameters given on the command line (or defaults)
+      *
+      * @param vm command line parameters
+      * @param test_options configuration options for testing
+      *
+      * @return false if one of the init steps fails, otherwise true
+      */
      bool init(const boost::program_options::variables_map& vm, const test_options *test_options = NULL);
+
+     /**
+      * @copydoc Blockchain::reset_and_set_genesis_block
+      *
+      * @note see Blockchain::reset_and_set_genesis_block
+      */
      bool set_genesis_block(const block& b);
+
+     /**
+      * @brief performs safe shutdown steps for core and core components
+      *
+      * Uninitializes the miner instance, transaction pool, and Blockchain
+      *
+      * if m_fast_exit is set, the call to Blockchain::deinit() is not made.
+      *
+      * @return true
+      */
      bool deinit();
+
+     /**
+      * @brief sets fast exit flag
+      *
+      * @note see deinit()
+      */
      static void set_fast_exit();
+
+     /**
+      * @brief gets the current state of the fast exit flag
+      *
+      * @return the fast exit flag
+      *
+      * @note see deinit()
+      */
      static bool get_fast_exit();
+
+     /**
+      * @brief sets to drop blocks downloaded (for testing)
+      */
      void test_drop_download();
+
+     /**
+      * @brief sets to drop blocks downloaded below a certain height
+      *
+      * @param height height below which to drop blocks
+      */
      void test_drop_download_height(uint64_t height);
+
+     /**
+      * @brief gets whether or not to drop blocks (for testing)
+      *
+      * @return whether or not to drop blocks
+      */
      bool get_test_drop_download() const;
+
+     /**
+      * @brief gets whether or not to drop blocks
+      *
+      * If the current blockchain height <= our block drop threshold
+      * and test drop blocks is set, return true
+      *
+      * @return see above
+      */
      bool get_test_drop_download_height() const;
+
+     /**
+      * @copydoc Blockchain::get_current_blockchain_height
+      *
+      * @note see Blockchain::get_current_blockchain_height()
+      */
      uint64_t get_current_blockchain_height() const;
-     bool get_blockchain_top(uint64_t& heeight, crypto::hash& top_id) const;
+
+     /**
+      * @brief get the hash and height of the most recent block
+      *
+      * @param height return-by-reference height of the block
+      * @param top_id return-by-reference hash of the block
+      *
+      * @return true
+      */
+     bool get_blockchain_top(uint64_t& height, crypto::hash& top_id) const;
+
+     /**
+      * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::list<block>&, std::list<transaction>&) const
+      *
+      * @note see Blockchain::get_blocks(uint64_t, size_t, std::list<block>&, std::list<transaction>&) const
+      */
      bool get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks, std::list<transaction>& txs) const;
+
+     /**
+      * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::list<block>&) const
+      *
+      * @note see Blockchain::get_blocks(uint64_t, size_t, std::list<block>&) const
+      */
      bool get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks) const;
+
+     /**
+      * @copydoc Blockchain::get_blocks(const t_ids_container&, t_blocks_container&, t_missed_container&) const
+      *
+      * @note see Blockchain::get_blocks(const t_ids_container&, t_blocks_container&, t_missed_container&) const
+      */
      template<class t_ids_container, class t_blocks_container, class t_missed_container>
      bool get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) const
      {
        return m_blockchain_storage.get_blocks(block_ids, blocks, missed_bs);
      }
-     crypto::hash get_block_id_by_height(uint64_t height) const;
-     bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::list<transaction>& txs, std::list<crypto::hash>& missed_txs) const;
-     bool get_block_by_hash(const crypto::hash &h, block &blk) const;
-     //void get_all_known_block_ids(std::list<crypto::hash> &main, std::list<crypto::hash> &alt, std::list<crypto::hash> &invalid);
 
+     /**
+      * @copydoc Blockchain::get_block_id_by_height
+      *
+      * @note see Blockchain::get_block_id_by_height
+      */
+     crypto::hash get_block_id_by_height(uint64_t height) const;
+
+     /**
+      * @copydoc Blockchain::get_transactions
+      *
+      * @note see Blockchain::get_transactions
+      */
+     bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::list<transaction>& txs, std::list<crypto::hash>& missed_txs) const;
+
+     /**
+      * @copydoc Blockchain::get_block_by_hash
+      *
+      * @note see Blockchain::get_block_by_hash
+      */
+     bool get_block_by_hash(const crypto::hash &h, block &blk) const;
+
+     /**
+      * @copydoc Blockchain::get_alternative_blocks
+      *
+      * @note see Blockchain::get_alternative_blocks(std::list<block>&) const
+      */
      bool get_alternative_blocks(std::list<block>& blocks) const;
+
+     /**
+      * @copydoc Blockchain::get_alternative_blocks_count
+      *
+      * @note see Blockchain::get_alternative_blocks_count() const
+      */
      size_t get_alternative_blocks_count() const;
 
+     /**
+      * @brief set the pointer to the cryptonote protocol object to use
+      *
+      * @param pprotocol the pointer to set ours as
+      */
      void set_cryptonote_protocol(i_cryptonote_protocol* pprotocol);
+
+     /**
+      * @copydoc Blockchain::set_checkpoints
+      *
+      * @note see Blockchain::set_checkpoints()
+      */
      void set_checkpoints(checkpoints&& chk_pts);
+
+     /**
+      * @brief set the file path to read from when loading checkpoints
+      *
+      * @param path the path to set ours as
+      */
      void set_checkpoints_file_path(const std::string& path);
+
+     /**
+      * @brief set whether or not we enforce DNS checkpoints
+      *
+      * @param enforce_dns enforce DNS checkpoints or not
+      */
      void set_enforce_dns_checkpoints(bool enforce_dns);
 
+     /**
+      * @copydoc tx_memory_pool::get_transactions
+      *
+      * @note see tx_memory_pool::get_transactions
+      */
      bool get_pool_transactions(std::list<transaction>& txs) const;
+
+     /**
+      * @copydoc tx_memory_pool::get_pool_transactions_and_spent_keys_info
+      *
+      * @note see tx_memory_pool::get_pool_transactions_and_spent_keys_info
+      */
      bool get_pool_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos) const;
+
+     /**
+      * @copydoc tx_memory_pool::get_transactions_count
+      *
+      * @note see tx_memory_pool::get_transactions_count
+      */
      size_t get_pool_transactions_count() const;
+
+     /**
+      * @copydoc Blockchain::get_total_transactions
+      *
+      * @note see Blockchain::get_total_transactions
+      */
      size_t get_blockchain_total_transactions() const;
-     //bool get_outs(uint64_t amount, std::list<crypto::public_key>& pkeys);
+
+     /**
+      * @copydoc Blockchain::have_block
+      *
+      * @note see Blockchain::have_block
+      */
      bool have_block(const crypto::hash& id) const;
+
+     /**
+      * @copydoc Blockchain::get_short_chain_history
+      *
+      * @note see Blockchain::get_short_chain_history
+      */
      bool get_short_chain_history(std::list<crypto::hash>& ids) const;
+
+     /**
+      * @copydoc Blockchain::find_blockchain_supplement(const std::list<crypto::hash>&, NOTIFY_RESPONSE_CHAIN_ENTRY::request&) const
+      *
+      * @note see Blockchain::find_blockchain_supplement(const std::list<crypto::hash>&, NOTIFY_RESPONSE_CHAIN_ENTRY::request&) const
+      */
      bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const;
+
+     /**
+      * @copydoc Blockchain::find_blockchain_supplement(const uint64_t, const std::list<crypto::hash>&, std::list<std::pair<block, std::list<transaction> > >&, uint64_t&, uint64_t&, size_t) const
+      *
+      * @note see Blockchain::find_blockchain_supplement(const uint64_t, const std::list<crypto::hash>&, std::list<std::pair<block, std::list<transaction> > >&, uint64_t&, uint64_t&, size_t) const
+      */
      bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<block, std::list<transaction> > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) const;
+
+     /**
+      * @brief gets some stats about the daemon
+      *
+      * @param st_inf return-by-reference container for the stats requested
+      *
+      * @return true
+      */
      bool get_stat_info(core_stat_info& st_inf) const;
-     //bool get_backward_blocks_sizes(uint64_t from_height, std::vector<size_t>& sizes, size_t count);
+
+     /**
+      * @copydoc Blockchain::get_tx_outputs_gindexs
+      *
+      * @note see Blockchain::get_tx_outputs_gindexs
+      */
      bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const;
+
+     /**
+      * @copydoc Blockchain::get_tail_id
+      *
+      * @note see Blockchain::get_tail_id
+      */
      crypto::hash get_tail_id() const;
+
+     /**
+      * @copydoc Blockchain::get_random_outs_for_amounts
+      *
+      * @note see Blockchain::get_random_outs_for_amounts
+      */
      bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const;
+
+
+     /**
+      * @copydoc miner::pause
+      *
+      * @note see miner::pause
+      */
      void pause_mine();
+
+     /**
+      * @copydoc miner::resume
+      *
+      * @note see miner::resume
+      */
      void resume_mine();
+
 #if BLOCKCHAIN_DB == DB_LMDB
+     /**
+      * @brief gets the Blockchain instance
+      *
+      * @return a reference to the Blockchain instance
+      */
      Blockchain& get_blockchain_storage(){return m_blockchain_storage;}
+
+     /**
+      * @brief gets the Blockchain instance (const)
+      *
+      * @return a const reference to the Blockchain instance
+      */
      const Blockchain& get_blockchain_storage()const{return m_blockchain_storage;}
 #else
      blockchain_storage& get_blockchain_storage(){return m_blockchain_storage;}
      const blockchain_storage& get_blockchain_storage()const{return m_blockchain_storage;}
 #endif
-     //debug functions
+
+     /**
+      * @copydoc Blockchain::print_blockchain
+      *
+      * @note see Blockchain::print_blockchain
+      */
      void print_blockchain(uint64_t start_index, uint64_t end_index) const;
+
+     /**
+      * @copydoc Blockchain::print_blockchain_index
+      *
+      * @note see Blockchain::print_blockchain_index
+      */
      void print_blockchain_index() const;
+
+     /**
+      * @copydoc tx_memory_pool::print_pool
+      *
+      * @note see tx_memory_pool::print_pool
+      */
      std::string print_pool(bool short_format) const;
+
+     /**
+      * @copydoc Blockchain::print_blockchain_outs
+      *
+      * @note see Blockchain::print_blockchain_outs
+      */
      void print_blockchain_outs(const std::string& file);
+
+     /**
+      * @copydoc miner::on_synchronized
+      *
+      * @note see miner::on_synchronized
+      */
      void on_synchronized();
 
+     /**
+      * @brief sets the target blockchain height
+      *
+      * @param target_blockchain_height the height to set
+      */
      void set_target_blockchain_height(uint64_t target_blockchain_height);
+
+     /**
+      * @brief gets the target blockchain height
+      *
+      * @param target_blockchain_height the target height
+      */
      uint64_t get_target_blockchain_height() const;
 
+     /**
+      * @brief tells the Blockchain to update its checkpoints
+      *
+      * This function will check if enough time has passed since the last
+      * time checkpoints were updated and tell the Blockchain to update
+      * its checkpoints if it is time.  If updating checkpoints fails,
+      * the daemon is told to shut down.
+      *
+      * @note see Blockchain::update_checkpoints()
+      */
      bool update_checkpoints();
 
+     /**
+      * @brief tells the daemon to wind down operations and stop running
+      *
+      * Currently this function raises SIGTERM, allowing the installed signal
+      * handlers to do the actual stopping.
+      */
+     void graceful_exit();
+
+     /**
+      * @brief stops the daemon running
+      *
+      * @note see graceful_exit()
+      */
      void stop();
 
+     /**
+      * @copydoc Blockchain::have_tx_keyimg_as_spent
+      *
+      * @note see Blockchain::have_tx_keyimg_as_spent
+      */
      bool is_key_image_spent(const crypto::key_image& key_im) const;
+
+     /**
+      * @brief check if multiple key images are spent
+      *
+      * plural version of is_key_image_spent()
+      *
+      * @param key_im list of key images to check
+      * @param spent return-by-reference result for each image checked
+      *
+      * @return true
+      */
      bool are_key_images_spent(const std::vector<crypto::key_image>& key_im, std::vector<bool> &spent) const;
 
    private:
+
+     /**
+      * @copydoc add_new_tx(const transaction&, tx_verification_context&, bool)
+      *
+      * @param tx_hash the transaction's hash
+      * @param tx_prefix_hash the transaction prefix' hash
+      * @param blob_size the size of the transaction
+      * @param relayed whether or not the transaction was relayed to us
+      *
+      */
      bool add_new_tx(const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed);
+
+     /**
+      * @brief add a new transaction to the transaction pool
+      *
+      * Adds a new transaction to the transaction pool.
+      *
+      * @param tx the transaction to add
+      * @param tvc return-by-reference metadata about the transaction's verification process
+      * @param keeped_by_block whether or not the transaction has been in a block
+      * @param relayed whether or not the transaction was relayed to us
+      *
+      * @return true if the transaction is already in the transaction pool,
+      * is already in a block on the Blockchain, or is successfully added
+      * to the transaction pool
+      */
      bool add_new_tx(const transaction& tx, tx_verification_context& tvc, bool keeped_by_block, bool relayed);
+
+     /**
+      * @copydoc Blockchain::add_new_block
+      *
+      * @note see Blockchain::add_new_block
+      */
      bool add_new_block(const block& b, block_verification_context& bvc);
+
+     /**
+      * @brief load any core state stored on disk
+      *
+      * currently does nothing, but may have state to load in the future.
+      *
+      * @return true
+      */
      bool load_state_data();
+
+     /**
+      * @copydoc parse_tx_from_blob(transaction&, crypto::hash&, crypto::hash&, const blobdata&) const
+      *
+      * @note see parse_tx_from_blob(transaction&, crypto::hash&, crypto::hash&, const blobdata&) const
+      */
      bool parse_tx_from_blob(transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash, const blobdata& blob) const;
 
+     /**
+      * @brief check a transaction's syntax
+      *
+      * For now this does nothing, but it may check something about the tx
+      * in the future.
+      *
+      * @param tx the transaction to check
+      *
+      * @return true
+      */
      bool check_tx_syntax(const transaction& tx) const;
-     //check correct values, amounts and all lightweight checks not related with database
-     bool check_tx_semantic(const transaction& tx, bool keeped_by_block) const;
-     //check if tx already in memory pool or in main blockchain
 
-     bool check_tx_ring_signature(const txin_to_key& tx, const crypto::hash& tx_prefix_hash, const std::vector<crypto::signature>& sig) const;
+     /**
-     bool is_tx_spendtime_unlocked(uint64_t unlock_time) const;
+      * @brief validates some simple properties of a transaction
+      *
+      * Currently checks: tx has inputs,
+      *                   tx inputs all of supported type(s),
+      *                   tx outputs valid (type, key, amount),
+      *                   input and output total amounts don't overflow,
+      *                   output amount <= input amount,
+      *                   tx not too large,
+      *                   each input has a different key image.
+      *
+      * @param tx the transaction to check
+      * @param keeped_by_block if the transaction has been in a block
+      *
+      * @return true if all the checks pass, otherwise false
+      */
+     bool check_tx_semantic(const transaction& tx, bool keeped_by_block) const;
+
+     /**
+      * @copydoc miner::on_block_chain_update
+      *
+      * @note see miner::on_block_chain_update
+      *
+      * @return true
+      */
      bool update_miner_block_template();
+
+     /**
+      * @brief act on a set of command line options given
+      *
+      * @param vm the command line options
+      *
+      * @return true
+      */
      bool handle_command_line(const boost::program_options::variables_map& vm);
-     bool on_update_blocktemplate_interval();
+
+     /**
+      * @brief verify that each input key image in a transaction is unique
+      *
+      * @param tx the transaction to check
+      *
+      * @return false if any key image is repeated, otherwise true
+      */
      bool check_tx_inputs_keyimages_diff(const transaction& tx) const;
-     void graceful_exit();
+
+     /**
+      * @brief checks HardFork status and prints messages about it
+      *
+      * Checks the status of HardFork and logs/prints if an update to
+      * the daemon is necessary.
+      *
+      * @note see Blockchain::get_hard_fork_state and HardFork::State
+      *
+      * @return true
+      */
      bool check_fork_time();
+
+     /**
+      * @brief attempts to relay any transactions in the mempool which need it
+      *
+      * @return true
+      */
      bool relay_txpool_transactions();
+
+     /**
+      * @brief locks a file in the BlockchainDB directory
+      *
+      * @param path the directory in which to place the file
+      *
+      * @return true if lock acquired successfully, otherwise false
+      */
      bool lock_db_directory(const boost::filesystem::path &path);
+
+     /**
+      * @brief unlocks the db directory
+      *
+      * @note see lock_db_directory()
+      *
+      * @return true
+      */
      bool unlock_db_directory();
 
-     static std::atomic<bool> m_fast_exit;
+     static std::atomic<bool> m_fast_exit; //!< whether or not to deinit Blockchain on exit
-     bool m_test_drop_download = true;
-     uint64_t m_test_drop_download_height = 0;
 
-     tx_memory_pool m_mempool;
+     bool m_test_drop_download = true; //!< whether or not to drop incoming blocks (for testing)
+
+     uint64_t m_test_drop_download_height = 0; //!< height under which to drop incoming blocks, if doing so
+
+     tx_memory_pool m_mempool; //!< transaction pool instance
 #if BLOCKCHAIN_DB == DB_LMDB
-     Blockchain m_blockchain_storage;
+     Blockchain m_blockchain_storage; //!< Blockchain instance
 #else
      blockchain_storage m_blockchain_storage;
 #endif
-     i_cryptonote_protocol* m_pprotocol;
+
-     epee::critical_section m_incoming_tx_lock;
+     i_cryptonote_protocol* m_pprotocol; //!< cryptonote protocol instance
+
+     epee::critical_section m_incoming_tx_lock; //!< incoming transaction lock
+
      //m_miner and m_miner_addres are probably temporary here
-     miner m_miner;
+     miner m_miner; //!< miner instance
-     account_public_address m_miner_address;
+     account_public_address m_miner_address; //!< address to mine to (for miner instance)
-     std::string m_config_folder;
+
-     cryptonote_protocol_stub m_protocol_stub;
+     std::string m_config_folder; //!< folder to look in for configs and other files
-     epee::math_helper::once_a_time_seconds<60*60*12, false> m_store_blockchain_interval;
+
-     epee::math_helper::once_a_time_seconds<60*60*2, true> m_fork_moaner;
+     cryptonote_protocol_stub m_protocol_stub; //!< cryptonote protocol stub instance
+
+     epee::math_helper::once_a_time_seconds<60*60*12, false> m_store_blockchain_interval; //!< interval for manual storing of Blockchain, if enabled
+     epee::math_helper::once_a_time_seconds<60*60*2, false> m_fork_moaner; //!< interval for checking HardFork status
      epee::math_helper::once_a_time_seconds<60*2, false> m_txpool_auto_relayer; //!< interval for checking re-relaying txpool transactions
+
      friend class tx_validate_inputs;
-     std::atomic<bool> m_starter_message_showed;
+     std::atomic<bool> m_starter_message_showed; //!< has the "daemon will sync now" message been shown?
 
-     uint64_t m_target_blockchain_height;
+     uint64_t m_target_blockchain_height; //!< blockchain height target
 
-     bool m_testnet;
+     bool m_testnet; //!< are we on testnet?
-     bool m_fakechain;
-     std::string m_checkpoints_path;
-     time_t m_last_dns_checkpoints_update;
-     time_t m_last_json_checkpoints_update;
 
-     std::atomic_flag m_checkpoints_updating;
+     bool m_fakechain; //!< are we using a fake chain (for testing purposes)?
 
-     boost::interprocess::file_lock db_lock;
+     std::string m_checkpoints_path; //!< path to json checkpoints file
+     time_t m_last_dns_checkpoints_update; //!< time when dns checkpoints were last updated
+     time_t m_last_json_checkpoints_update; //!< time when json checkpoints were last updated
+
+     std::atomic_flag m_checkpoints_updating; //!< set if checkpoints are currently updating to avoid multiple threads attempting to update at once
+
+     boost::interprocess::file_lock db_lock; //!< a lock object for a file lock in the db directory
    };
 }
 

src/cryptonote_core/cryptonote_format_utils.cpp

@@ -126,20 +126,19 @@ namespace cryptonote
       return false;
     }
 
-    // from hard fork 2, we cut out the low significant digits. This makes the tx smaller, and
-    // keeps the paid amount almost the same. The unpaid remainder gets pushed back to the
-    // emission schedule
-    if (hard_fork_version >= 2)
-    {
-        block_reward = block_reward - block_reward % ::config::BASE_REWARD_CLAMP_THRESHOLD;
-    }
-
 #if defined(DEBUG_CREATE_BLOCK_TEMPLATE)
     LOG_PRINT_L1("Creating block template: reward " << block_reward <<
       ", fee " << fee)
 #endif
     block_reward += fee;
 
+    // from hard fork 2, we cut out the low significant digits. This makes the tx smaller, and
+    // keeps the paid amount almost the same. The unpaid remainder gets pushed back to the
+    // emission schedule
+    if (hard_fork_version >= 2) {
+      block_reward = block_reward - block_reward % ::config::BASE_REWARD_CLAMP_THRESHOLD;
+    }
+
     std::vector<uint64_t> out_amounts;
     decompose_amount_into_digits(block_reward, hard_fork_version >= 2 ? 0 : ::config::DEFAULT_DUST_THRESHOLD,
       [&out_amounts](uint64_t a_chunk) { out_amounts.push_back(a_chunk); },

src/cryptonote_core/difficulty.cpp

@@ -116,8 +116,8 @@ namespace cryptonote {
     return !carry;
   }
 
-  difficulty_type next_difficulty(vector<uint64_t> timestamps, vector<difficulty_type> cumulative_difficulties, size_t target_seconds) {
+  difficulty_type next_difficulty(std::vector<std::uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds) {
-    //cutoff DIFFICULTY_LAG
+
     if(timestamps.size() > DIFFICULTY_WINDOW)
     {
       timestamps.resize(DIFFICULTY_WINDOW);
@@ -151,6 +151,8 @@ namespace cryptonote {
     assert(total_work > 0);
     uint64_t low, high;
     mul(total_work, target_seconds, low, high);
+    // blockchain errors "difficulty overhead" if this function returns zero.
+    // TODO: consider throwing an exception instead
     if (high != 0 || low + time_span - 1 < low) {
       return 0;
     }

src/cryptonote_core/difficulty.h

@@ -39,6 +39,18 @@ namespace cryptonote
 {
     typedef std::uint64_t difficulty_type;
 
+    /**
+     * @brief checks if a hash fits the given difficulty
+     *
+     * The hash passes if (hash * difficulty) < 2^192.
+     * Phrased differently, if (hash * difficulty) fits without overflow into
+     * the least significant 192 bits of the 256 bit multiplication result.
+     *
+     * @param hash the hash to check
+     * @param difficulty the difficulty to check against
+     *
+     * @return true if valid, else false
+     */
     bool check_hash(const crypto::hash &hash, difficulty_type difficulty);
     difficulty_type next_difficulty(std::vector<std::uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds);
 }

src/cryptonote_protocol/cryptonote_protocol_handler.inl

@@ -281,7 +281,7 @@ namespace cryptonote
       << " [" << std::abs(diff) << " blocks (" << diff / (24 * 60 * 60 / DIFFICULTY_TARGET_V1) << " days) "
       << (0 <= diff ? std::string("behind") : std::string("ahead"))
       << "] " << ENDL << "SYNCHRONIZATION started", (is_inital ? LOG_LEVEL_0:LOG_LEVEL_1));
-    LOG_PRINT_L1("Remote top block height: " << hshd.current_height << ", id: " << hshd.top_id);
+    LOG_PRINT_L1("Remote blockchain height: " << hshd.current_height << ", id: " << hshd.top_id);
     context.m_state = cryptonote_connection_context::state_synchronizing;
     context.m_remote_blockchain_height = hshd.current_height;
     //let the socket to send response to handshake, but request callback, to let send request data after response

src/daemon/command_parser_executor.cpp

@@ -439,5 +439,23 @@ bool t_command_parser_executor::flush_txpool(const std::vector<std::string>& arg
   return m_executor.flush_txpool(txid);
 }
 
+bool t_command_parser_executor::output_histogram(const std::vector<std::string>& args)
+{
+  if (args.size() > 2) return false;
+
+  uint64_t min_count = 3;
+  uint64_t max_count = 0;
+
+  if (args.size() >= 1)
+  {
+    min_count = boost::lexical_cast<uint64_t>(args[0]);
+  }
+  if (args.size() >= 2)
+  {
+    max_count = boost::lexical_cast<uint64_t>(args[1]);
+  }
+  return m_executor.output_histogram(min_count, max_count);
+}
+
 
 } // namespace daemonize

src/daemon/command_parser_executor.h

@@ -114,6 +114,8 @@ public:
   bool unban(const std::vector<std::string>& args);
 
   bool flush_txpool(const std::vector<std::string>& args);
+
+  bool output_histogram(const std::vector<std::string>& args);
 };
 
 } // namespace daemonize

src/daemon/command_server.cpp

@@ -214,6 +214,11 @@ t_command_server::t_command_server(
     , std::bind(&t_command_parser_executor::flush_txpool, &m_parser, p::_1)
     , "Flush a transaction from the tx pool by its txid, or the whole tx pool"
     );
+    m_command_lookup.set_handler(
+      "output_histogram"
+    , std::bind(&t_command_parser_executor::output_histogram, &m_parser, p::_1)
+    , "Print output histogram (amount, instances)"
+    );
 }
 
 bool t_command_server::process_command_str(const std::string& cmd)

src/daemon/core.h

@@ -28,7 +28,6 @@
 
 #pragma once
 
-#include "cryptonote_core/checkpoints_create.h"
 #include "cryptonote_core/cryptonote_core.h"
 #include "cryptonote_protocol/cryptonote_protocol_handler.h"
 #include "misc_log_ex.h"

src/daemon/rpc_command_executor.cpp

@@ -558,6 +558,7 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash) {
 
   std::string fail_message = "Problem fetching transaction";
 
+  req.txs_hashes.push_back(epee::string_tools::pod_to_hex(transaction_hash));
   if (m_is_rpc)
   {
     if (!m_rpc_client->rpc_request(req, res, "/gettransactions", fail_message.c_str()))
@@ -567,7 +568,6 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash) {
   }
   else
   {
-    req.txs_hashes.push_back(epee::string_tools::pod_to_hex(transaction_hash));
     if (!m_rpc_server->on_get_transactions(req, res) || res.status != CORE_RPC_STATUS_OK)
     {
       tools::fail_msg_writer() << fail_message.c_str();
@@ -1203,5 +1203,41 @@ bool t_rpc_command_executor::flush_txpool(const std::string &txid)
     return true;
 }
 
+bool t_rpc_command_executor::output_histogram(uint64_t min_count, uint64_t max_count)
+{
+    cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request req;
+    cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response res;
+    std::string fail_message = "Unsuccessful";
+    epee::json_rpc::error error_resp;
+
+    req.min_count = min_count;
+    req.max_count = max_count;
+
+    if (m_is_rpc)
+    {
+        if (!m_rpc_client->json_rpc_request(req, res, "get_output_histogram", fail_message.c_str()))
+        {
+            return true;
+        }
+    }
+    else
+    {
+        if (!m_rpc_server->on_get_output_histogram(req, res, error_resp))
+        {
+            tools::fail_msg_writer() << fail_message.c_str();
+            return true;
+        }
+    }
+
+    std::sort(res.histogram.begin(), res.histogram.end(),
+        [](const cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::entry &e1, const cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::entry &e2)->bool { return e1.instances < e2.instances; });
+    for (const auto &e: res.histogram)
+    {
+        tools::msg_writer() << e.instances << "  " << cryptonote::print_money(e.amount);
+    }
+
+    return true;
+}
+
 
 }// namespace daemonize

src/daemon/rpc_command_executor.h

@@ -132,6 +132,8 @@ public:
   bool unban(const std::string &ip);
 
   bool flush_txpool(const std::string &txid);
+
+  bool output_histogram(uint64_t min_count, uint64_t max_count);
 };
 
 } // namespace daemonize

src/rpc/core_rpc_server.cpp

@@ -1041,6 +1041,38 @@ namespace cryptonote
     return true;
   }
   //------------------------------------------------------------------------------------------------------------------------------
+  bool core_rpc_server::on_get_output_histogram(const COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request& req, COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response& res, epee::json_rpc::error& error_resp)
+  {
+    if(!check_core_busy())
+    {
+      error_resp.code = CORE_RPC_ERROR_CODE_CORE_BUSY;
+      error_resp.message = "Core is busy.";
+      return false;
+    }
+
+    std::map<uint64_t, uint64_t> histogram;
+    try
+    {
+      histogram = m_core.get_blockchain_storage().get_output_histogram(req.amounts);
+    }
+    catch (const std::exception &e)
+    {
+      res.status = "Failed to get output histogram";
+      return true;
+    }
+
+    res.histogram.clear();
+    res.histogram.reserve(histogram.size());
+    for (const auto &i: histogram)
+    {
+      if (i.second >= req.min_count && (i.second <= req.max_count || req.max_count == 0))
+        res.histogram.push_back(COMMAND_RPC_GET_OUTPUT_HISTOGRAM::entry(i.first, i.second));
+    }
+
+    res.status = CORE_RPC_STATUS_OK;
+    return true;
+  }
+  //------------------------------------------------------------------------------------------------------------------------------
   bool core_rpc_server::on_fast_exit(const COMMAND_RPC_FAST_EXIT::request& req, COMMAND_RPC_FAST_EXIT::response& res)
   {
 	  cryptonote::core::set_fast_exit();

src/rpc/core_rpc_server.h

@@ -109,6 +109,7 @@ namespace cryptonote
         MAP_JON_RPC_WE("setbans",                on_set_bans,                   COMMAND_RPC_SETBANS)
         MAP_JON_RPC_WE("getbans",                on_get_bans,                   COMMAND_RPC_GETBANS)
         MAP_JON_RPC_WE("flush_txpool",           on_flush_txpool,               COMMAND_RPC_FLUSH_TRANSACTION_POOL)
+        MAP_JON_RPC_WE("get_output_histogram",   on_get_output_histogram,       COMMAND_RPC_GET_OUTPUT_HISTOGRAM)
       END_JSON_RPC_MAP()
     END_URI_MAP2()
 
@@ -149,6 +150,7 @@ namespace cryptonote
     bool on_set_bans(const COMMAND_RPC_SETBANS::request& req, COMMAND_RPC_SETBANS::response& res, epee::json_rpc::error& error_resp);
     bool on_get_bans(const COMMAND_RPC_GETBANS::request& req, COMMAND_RPC_GETBANS::response& res, epee::json_rpc::error& error_resp);
     bool on_flush_txpool(const COMMAND_RPC_FLUSH_TRANSACTION_POOL::request& req, COMMAND_RPC_FLUSH_TRANSACTION_POOL::response& res, epee::json_rpc::error& error_resp);
+    bool on_get_output_histogram(const COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request& req, COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response& res, epee::json_rpc::error& error_resp);
     //-----------------------
 
 private:

src/rpc/core_rpc_server_commands_defs.h

@@ -986,5 +986,46 @@ namespace cryptonote
       END_KV_SERIALIZE_MAP()
     };
   };
+
+  struct COMMAND_RPC_GET_OUTPUT_HISTOGRAM
+  {
+    struct request
+    {
+      std::vector<uint64_t> amounts;
+      uint64_t min_count;
+      uint64_t max_count;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(amounts);
+        KV_SERIALIZE(min_count);
+        KV_SERIALIZE(max_count);
+      END_KV_SERIALIZE_MAP()
+    };
+
+    struct entry
+    {
+      uint64_t amount;
+      uint64_t instances;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(amount);
+        KV_SERIALIZE(instances);
+      END_KV_SERIALIZE_MAP()
+
+      entry(uint64_t amount, uint64_t instances): amount(amount), instances(instances) {}
+      entry() {}
+    };
+
+    struct response
+    {
+      std::string status;
+      std::vector<entry> histogram;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(status)
+        KV_SERIALIZE(histogram)
+      END_KV_SERIALIZE_MAP()
+    };
+  };
 }
 

src/simplewallet/simplewallet.cpp

@@ -57,6 +57,7 @@
 #include "version.h"
 #include "crypto/crypto.h"  // for crypto::secret_key definition
 #include "mnemonics/electrum-words.h"
+#include "rapidjson/document.h"
 #include <stdexcept>
 
 #if defined(WIN32)
@@ -81,6 +82,7 @@ namespace
   const command_line::arg_descriptor<std::string> arg_generate_new_wallet = {"generate-new-wallet", sw::tr("Generate new wallet and save it to <arg> or <address>.wallet by default"), ""};
   const command_line::arg_descriptor<std::string> arg_generate_from_view_key = {"generate-from-view-key", sw::tr("Generate incoming-only wallet from view key"), ""};
   const command_line::arg_descriptor<std::string> arg_generate_from_keys = {"generate-from-keys", sw::tr("Generate wallet from private keys"), ""};
+  const command_line::arg_descriptor<std::string> arg_generate_from_json = {"generate-from-json", sw::tr("Generate wallet from JSON format file"), ""};
   const command_line::arg_descriptor<std::string> arg_daemon_address = {"daemon-address", sw::tr("Use daemon instance at <host>:<port>"), ""};
   const command_line::arg_descriptor<std::string> arg_daemon_host = {"daemon-host", sw::tr("Use daemon instance at host <arg> instead of localhost"), ""};
   const command_line::arg_descriptor<std::string> arg_password = {"password", sw::tr("Wallet password"), "", true};
@@ -539,7 +541,7 @@ simple_wallet::simple_wallet()
   m_cmd_binder.set_handler("bc_height", boost::bind(&simple_wallet::show_blockchain_height, this, _1), tr("Show blockchain height"));
   m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::transfer, this, _1), tr("transfer [<mixin_count>] <addr_1> <amount_1> [<addr_2> <amount_2> ... <addr_N> <amount_N>] [payment_id] - Transfer <amount_1>,... <amount_N> to <address_1>,... <address_N>, respectively. <mixin_count> is the number of extra inputs to include for untraceability (from 0 to maximum available)"));
   m_cmd_binder.set_handler("transfer_new", boost::bind(&simple_wallet::transfer_new, this, _1), tr("Same as transfer, but using a new transaction building algorithm"));
-  m_cmd_binder.set_handler("sweep_dust", boost::bind(&simple_wallet::sweep_dust, this, _1), tr("Send all dust outputs to yourself with mixin 0"));
+  m_cmd_binder.set_handler("sweep_unmixable", boost::bind(&simple_wallet::sweep_unmixable, this, _1), tr("Send all unmixable outputs to yourself with mixin 0"));
   m_cmd_binder.set_handler("set_log", boost::bind(&simple_wallet::set_log, this, _1), tr("set_log <level> - Change current log detail level, <0-4>"));
   m_cmd_binder.set_handler("address", boost::bind(&simple_wallet::print_address, this, _1), tr("Show current wallet public address"));
   m_cmd_binder.set_handler("integrated_address", boost::bind(&simple_wallet::print_integrated_address, this, _1), tr("integrated_address [PID] - Encode a payment ID into an integrated address for the current wallet public address (no argument uses a random payment ID), or decode an integrated address to standard address and payment ID"));
@@ -716,7 +718,7 @@ bool simple_wallet::ask_wallet_create_if_needed()
   // add logic to error out if new wallet requested but named wallet file exists
   if (keys_file_exists || wallet_file_exists)
   {
-    if (!m_generate_new.empty() || m_restore_deterministic_wallet || !m_generate_from_view_key.empty() || !m_generate_from_keys.empty())
+    if (!m_generate_new.empty() || m_restore_deterministic_wallet || !m_generate_from_view_key.empty() || !m_generate_from_keys.empty() || !m_generate_from_json.empty())
     {
       fail_msg_writer() << tr("attempting to generate or restore wallet, but specified file(s) exist.  Exiting to not risk overwriting.");
       return false;
@@ -760,6 +762,208 @@ void simple_wallet::print_seed(std::string seed)
   std::cout << seed << std::endl;
 }
 
+//----------------------------------------------------------------------------------------------------
+static bool get_password(const boost::program_options::variables_map& vm, bool allow_entry, tools::password_container &pwd_container)
+{
+  // has_arg returns true even when the parameter is not passed ??
+  const std::string gfj = command_line::get_arg(vm, arg_generate_from_json);
+  if (!gfj.empty()) {
+    // will be in the json file, if any
+    return true;
+  }
+
+  if (has_arg(vm, arg_password) && has_arg(vm, arg_password_file))
+  {
+    fail_msg_writer() << tr("can't specify more than one of --password and --password-file");
+    return false;
+  }
+
+  if (command_line::has_arg(vm, arg_password))
+  {
+    pwd_container.password(command_line::get_arg(vm, arg_password));
+    return true;
+  }
+
+  if (command_line::has_arg(vm, arg_password_file))
+  {
+    std::string password;
+    bool r = epee::file_io_utils::load_file_to_string(command_line::get_arg(vm, arg_password_file),
+                                                      password);
+    if (!r)
+    {
+      fail_msg_writer() << tr("the password file specified could not be read");
+      return false;
+    }
+
+    // Remove line breaks the user might have inserted
+    password.erase(std::remove(password.begin() - 1, password.end(), '\n'), password.end());
+    password.erase(std::remove(password.end() - 1, password.end(), '\r'), password.end());
+    pwd_container.password(password.c_str());
+    return true;
+  }
+
+  if (allow_entry)
+  {
+    bool r = pwd_container.read_password();
+    if (!r)
+    {
+      fail_msg_writer() << tr("failed to read wallet password");
+      return false;
+    }
+    return true;
+  }
+
+  fail_msg_writer() << tr("Wallet password not set.");
+  return false;
+}
+
+//----------------------------------------------------------------------------------------------------
+bool simple_wallet::generate_from_json(const boost::program_options::variables_map& vm, std::string &wallet_file, std::string &password)
+{
+  std::string buf;
+  bool r = epee::file_io_utils::load_file_to_string(m_generate_from_json, buf);
+  if (!r) {
+    fail_msg_writer() << tr("Failed to load file ") << m_generate_from_json;
+    return false;
+  }
+
+  rapidjson::Document json;
+  if (json.Parse(buf.c_str()).HasParseError()) {
+    fail_msg_writer() << tr("Failed to parse JSON");
+    return false;
+  }
+
+  if (!json.HasMember("version")) {
+    fail_msg_writer() << tr("Version not found in JSON");
+    return false;
+  }
+  unsigned int version = json["version"].GetUint();
+  const int current_version = 1;
+  if (version > current_version) {
+    fail_msg_writer() << boost::format(tr("Version %u too new, we can only grok up to %u")) % version % current_version;
+    return false;
+  }
+  if (!json.HasMember("filename")) {
+    fail_msg_writer() << tr("Filename not found in JSON");
+    return false;
+  }
+  std::string filename = json["filename"].GetString();
+
+  bool recover = false;
+  uint64_t scan_from_height = 0;
+  if (json.HasMember("scan_from_height")) {
+    scan_from_height = json["scan_from_height"].GetUint64();
+    recover = true;
+  }
+
+  password = "";
+  if (json.HasMember("password")) {
+    password = json["password"].GetString();
+  }
+
+  std::string viewkey_string("");
+  crypto::secret_key viewkey;
+  if (json.HasMember("viewkey")) {
+    viewkey_string = json["viewkey"].GetString();
+    cryptonote::blobdata viewkey_data;
+    if(!epee::string_tools::parse_hexstr_to_binbuff(viewkey_string, viewkey_data))
+    {
+      fail_msg_writer() << tr("failed to parse view key secret key");
+      return false;
+    }
+    viewkey = *reinterpret_cast<const crypto::secret_key*>(viewkey_data.data());
+  }
+
+  std::string spendkey_string("");
+  crypto::secret_key spendkey;
+  if (json.HasMember("spendkey")) {
+    spendkey_string = json["spendkey"].GetString();
+    cryptonote::blobdata spendkey_data;
+    if(!epee::string_tools::parse_hexstr_to_binbuff(spendkey_string, spendkey_data))
+    {
+      fail_msg_writer() << tr("failed to parse spend key secret key");
+      return false;
+    }
+    spendkey = *reinterpret_cast<const crypto::secret_key*>(spendkey_data.data());
+  }
+
+  std::string seed("");
+  std::string old_language;
+  if (json.HasMember("seed")) {
+    seed = json["seed"].GetString();
+    if (!crypto::ElectrumWords::words_to_bytes(seed, m_recovery_key, old_language))
+    {
+      fail_msg_writer() << tr("Electrum-style word list failed verification");
+      return false;
+    }
+    m_electrum_seed = seed;
+    m_restore_deterministic_wallet = true;
+  }
+
+  // compatibility checks
+  if (seed.empty() && viewkey_string.empty()) {
+    fail_msg_writer() << tr("At least one of Electrum-style word list and private view key must be specified");
+    return false;
+  }
+  if (!seed.empty() && (!viewkey_string.empty() || !spendkey_string.empty())) {
+    fail_msg_writer() << tr("Both Electrum-style word list and private key(s) specified");
+    return false;
+  }
+
+  m_wallet_file = filename;
+
+  bool was_deprecated_wallet = m_restore_deterministic_wallet && ((old_language == crypto::ElectrumWords::old_language_name) ||
+    crypto::ElectrumWords::get_is_old_style_seed(m_electrum_seed));
+  if (was_deprecated_wallet) {
+    fail_msg_writer() << tr("Cannot create deprecated wallets from JSON");
+    return false;
+  }
+
+  bool testnet = command_line::get_arg(vm, arg_testnet);
+  m_wallet.reset(new tools::wallet2(testnet));
+  m_wallet->callback(this);
+
+  try
+  {
+    if (!seed.empty())
+    {
+      m_wallet->generate(m_wallet_file, password, m_recovery_key, recover, false);
+    }
+    else
+    {
+      cryptonote::account_public_address address;
+      if (!crypto::secret_key_to_public_key(viewkey, address.m_view_public_key)) {
+        fail_msg_writer() << tr("failed to verify view key secret key");
+        return false;
+      }
+      if (!crypto::secret_key_to_public_key(spendkey, address.m_spend_public_key)) {
+        fail_msg_writer() << tr("failed to verify spend key secret key");
+        return false;
+      }
+
+      if (spendkey_string.empty())
+      {
+        m_wallet->generate(m_wallet_file, password, address, viewkey);
+      }
+      else
+      {
+        m_wallet->generate(m_wallet_file, password, address, spendkey, viewkey);
+      }
+    }
+  }
+  catch (const std::exception& e)
+  {
+    fail_msg_writer() << tr("failed to generate new wallet: ") << e.what();
+    return false;
+  }
+
+  m_wallet->set_refresh_from_block_height(scan_from_height);
+
+  wallet_file = m_wallet_file;
+
+  return r;
+}
+
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::init(const boost::program_options::variables_map& vm)
 {
@@ -772,12 +976,12 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm)
     return false;
   }
 
-  if((!m_generate_new.empty()) + (!m_wallet_file.empty()) + (!m_generate_from_view_key.empty()) + (!m_generate_from_keys.empty()) > 1)
+  if((!m_generate_new.empty()) + (!m_wallet_file.empty()) + (!m_generate_from_view_key.empty()) + (!m_generate_from_keys.empty()) + (!m_generate_from_json.empty()) > 1)
   {
-    fail_msg_writer() << tr("can't specify more than one of --generate-new-wallet=\"wallet_name\", --wallet-file=\"wallet_name\", --generate-from-view-key=\"wallet_name\" and --generate-from-keys=\"wallet_name\"");
+    fail_msg_writer() << tr("can't specify more than one of --generate-new-wallet=\"wallet_name\", --wallet-file=\"wallet_name\", --generate-from-view-key=\"wallet_name\", --generate-from-json=\"jsonfilename\" and --generate-from-keys=\"wallet_name\"");
     return false;
   }
-  else if (m_generate_new.empty() && m_wallet_file.empty() && m_generate_from_view_key.empty() && m_generate_from_keys.empty())
+  else if (m_generate_new.empty() && m_wallet_file.empty() && m_generate_from_view_key.empty() && m_generate_from_keys.empty() && m_generate_from_json.empty())
   {
     if(!ask_wallet_create_if_needed()) return false;
   }
@@ -795,44 +999,11 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm)
   if (m_daemon_address.empty())
     m_daemon_address = std::string("http://") + m_daemon_host + ":" + std::to_string(m_daemon_port);
 
-  if (has_arg(vm, arg_password) && has_arg(vm, arg_password_file))
-  {
-    fail_msg_writer() << tr("can't specify more than one of --password and --password-file");
-    return false;
-  }
-
   tools::password_container pwd_container;
-  if (command_line::has_arg(vm, arg_password))
+  if (!get_password(vm, true, pwd_container))
-  {
-    pwd_container.password(command_line::get_arg(vm, arg_password));
-  }
-  else if (command_line::has_arg(vm, arg_password_file))
-  {
-    std::string password;
-    bool r = epee::file_io_utils::load_file_to_string(command_line::get_arg(vm, arg_password_file),
-                                                      password);
-    if (!r)
-    {
-      fail_msg_writer() << tr("the password file specified could not be read");
     return false;
-    }
 
-    // Remove line breaks the user might have inserted
+  if (!m_generate_new.empty() || m_restore_deterministic_wallet || !m_generate_from_view_key.empty() || !m_generate_from_keys.empty() || !m_generate_from_json.empty())
-    password.erase(std::remove(password.begin() - 1, password.end(), '\n'), password.end());
-    password.erase(std::remove(password.end() - 1, password.end(), '\r'), password.end());
-    pwd_container.password(password.c_str());
-  }
-  else
-  {
-    bool r = pwd_container.read_password();
-    if (!r)
-    {
-      fail_msg_writer() << tr("failed to read wallet password");
-      return false;
-    }
-  }
-
-  if (!m_generate_new.empty() || m_restore_deterministic_wallet || !m_generate_from_view_key.empty() || !m_generate_from_keys.empty())
   {
     if (m_wallet_file.empty()) m_wallet_file = m_generate_new;  // alias for simplicity later
 
@@ -978,6 +1149,12 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm)
       bool r = new_wallet(m_wallet_file, pwd_container.password(), address, spendkey, viewkey, testnet);
       CHECK_AND_ASSERT_MES(r, false, tr("account creation failed"));
     }
+    else if (!m_generate_from_json.empty())
+    {
+      std::string wallet_file, password; // we don't need to remember them
+      if (!generate_from_json(vm, wallet_file, password))
+        return false;
+    }
     else
     {
       bool r = new_wallet(m_wallet_file, pwd_container.password(), m_recovery_key, m_restore_deterministic_wallet,
@@ -1008,6 +1185,7 @@ bool simple_wallet::handle_command_line(const boost::program_options::variables_
   m_generate_new                  = command_line::get_arg(vm, arg_generate_new_wallet);
   m_generate_from_view_key        = command_line::get_arg(vm, arg_generate_from_view_key);
   m_generate_from_keys            = command_line::get_arg(vm, arg_generate_from_keys);
+  m_generate_from_json            = command_line::get_arg(vm, arg_generate_from_json);
   m_daemon_address                = command_line::get_arg(vm, arg_daemon_address);
   m_daemon_host                   = command_line::get_arg(vm, arg_daemon_host);
   m_daemon_port                   = command_line::get_arg(vm, arg_daemon_port);
@@ -1544,8 +1722,7 @@ bool simple_wallet::refresh(const std::vector<std::string>& args)
 bool simple_wallet::show_balance(const std::vector<std::string>& args/* = std::vector<std::string>()*/)
 {
   success_msg_writer() << tr("Balance: ") << print_money(m_wallet->balance()) << ", "
-    << tr("unlocked balance: ") << print_money(m_wallet->unlocked_balance()) << ", "
+    << tr("unlocked balance: ") << print_money(m_wallet->unlocked_balance());
-    << tr("including unlocked dust: ") << print_money(m_wallet->unlocked_dust_balance(tools::tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD)));
   return true;
 }
 //----------------------------------------------------------------------------------------------------
@@ -2030,7 +2207,7 @@ bool simple_wallet::transfer_new(const std::vector<std::string> &args_)
 }
 
 //----------------------------------------------------------------------------------------------------
-bool simple_wallet::sweep_dust(const std::vector<std::string> &args_)
+bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_)
 {
   if (!try_connect_to_daemon())
     return true;
@@ -2043,28 +2220,37 @@ bool simple_wallet::sweep_dust(const std::vector<std::string> &args_)
 
   try
   {
-    uint64_t total_dust = m_wallet->unlocked_dust_balance(tools::tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD));
-
     // figure out what tx will be necessary
-    auto ptx_vector = m_wallet->create_dust_sweep_transactions();
+    auto ptx_vector = m_wallet->create_unmixable_sweep_transactions(m_trusted_daemon);
+
+    if (ptx_vector.empty())
+    {
+      fail_msg_writer() << tr("No unmixable outputs found");
+      return true;
+    }
 
     // give user total and fee, and prompt to confirm
-    uint64_t total_fee = 0;
+    uint64_t total_fee = 0, total_unmixable = 0;
     for (size_t n = 0; n < ptx_vector.size(); ++n)
     {
       total_fee += ptx_vector[n].fee;
+      for (const auto &vin: ptx_vector[n].tx.vin)
+      {
+        if (vin.type() == typeid(txin_to_key))
+          total_unmixable += boost::get<txin_to_key>(vin).amount;
+      }
     }
 
-    std::string prompt_str = tr("Sweeping ") + print_money(total_dust);
+    std::string prompt_str = tr("Sweeping ") + print_money(total_unmixable);
     if (ptx_vector.size() > 1) {
       prompt_str = (boost::format(tr("Sweeping %s in %llu transactions for a total fee of %s.  Is this okay?  (Y/Yes/N/No)")) %
-        print_money(total_dust) %
+        print_money(total_unmixable) %
         ((unsigned long long)ptx_vector.size()) %
         print_money(total_fee)).str();
     }
     else {
       prompt_str = (boost::format(tr("Sweeping %s for a total fee of %s.  Is this okay?  (Y/Yes/N/No)")) %
-        print_money(total_dust) %
+        print_money(total_unmixable) %
         print_money(total_fee)).str();
     }
     std::string accepted = command_line::input_line(prompt_str);
@@ -2107,11 +2293,12 @@ bool simple_wallet::sweep_dust(const std::vector<std::string> &args_)
   }
   catch (const tools::error::not_enough_money& e)
   {
-    fail_msg_writer() << boost::format(tr("not enough money to transfer, available only %s, transaction amount %s = %s + %s (fee)")) %
+    fail_msg_writer() << boost::format(tr("not enough money to transfer, available only %s, transaction amount %s = %s + %s (fee).\n%s")) %
       print_money(e.available()) %
       print_money(e.tx_amount() + e.fee()) %
       print_money(e.tx_amount()) %
-      print_money(e.fee());
+      print_money(e.fee()) %
+      tr("This is usually due to dust which is so small it cannot pay for itself in fees");
   }
   catch (const tools::error::not_enough_outs_to_mix& e)
   {
@@ -2560,6 +2747,7 @@ int main(int argc, char* argv[])
   command_line::add_arg(desc_params, arg_generate_new_wallet);
   command_line::add_arg(desc_params, arg_generate_from_view_key);
   command_line::add_arg(desc_params, arg_generate_from_keys);
+  command_line::add_arg(desc_params, arg_generate_from_json);
   command_line::add_arg(desc_params, arg_password);
   command_line::add_arg(desc_params, arg_password_file);
   command_line::add_arg(desc_params, arg_daemon_address);
@@ -2675,16 +2863,13 @@ int main(int argc, char* argv[])
       LOG_ERROR(sw::tr("Daemon address not set."));
       return 1;
     }
-    if(!command_line::has_arg(vm, arg_password) )
-    {
-      LOG_ERROR(sw::tr("Wallet password not set."));
-      return 1;
-    }
 
     bool testnet = command_line::get_arg(vm, arg_testnet);
     bool restricted = command_line::get_arg(vm, arg_restricted);
     std::string wallet_file     = command_line::get_arg(vm, arg_wallet_file);
-    std::string wallet_password = command_line::get_arg(vm, arg_password);
+    tools::password_container pwd_container;
+    if (!get_password(vm, false, pwd_container))
+      return 1;
     std::string daemon_address  = command_line::get_arg(vm, arg_daemon_address);
     std::string daemon_host = command_line::get_arg(vm, arg_daemon_host);
     int daemon_port = command_line::get_arg(vm, arg_daemon_port);
@@ -2695,11 +2880,21 @@ int main(int argc, char* argv[])
     if (daemon_address.empty())
       daemon_address = std::string("http://") + daemon_host + ":" + std::to_string(daemon_port);
 
+    std::string password;
+    const std::string gfj = command_line::get_arg(vm, arg_generate_from_json);
+    if (!gfj.empty()) {
+      if (!w.generate_from_json(vm, wallet_file, password))
+        return 1;
+    }
+    else {
+      password = pwd_container.password();
+    }
+
     tools::wallet2 wal(testnet,restricted);
     try
     {
       LOG_PRINT_L0(sw::tr("Loading wallet..."));
-      wal.load(wallet_file, wallet_password);
+      wal.load(wallet_file, password);
       wal.init(daemon_address);
       wal.refresh();
       LOG_PRINT_GREEN(sw::tr("Loaded ok"), LOG_LEVEL_0);

src/simplewallet/simplewallet.h

@@ -69,6 +69,7 @@ namespace cryptonote
     bool run();
     void stop();
     void interrupt();
+    bool generate_from_json(const boost::program_options::variables_map& vm, std::string &wallet_file, std::string &password);
 
     //wallet *create_wallet();
     bool process_command(const std::vector<std::string> &args);
@@ -120,7 +121,7 @@ namespace cryptonote
     bool transfer_main(bool new_algorithm, const std::vector<std::string> &args);
     bool transfer(const std::vector<std::string> &args);
     bool transfer_new(const std::vector<std::string> &args);
-    bool sweep_dust(const std::vector<std::string> &args);
+    bool sweep_unmixable(const std::vector<std::string> &args);
     std::vector<std::vector<cryptonote::tx_destination_entry>> split_amounts(
         std::vector<cryptonote::tx_destination_entry> dsts, size_t num_splits
     );
@@ -221,6 +222,7 @@ namespace cryptonote
     std::string m_generate_new;
     std::string m_generate_from_view_key;
     std::string m_generate_from_keys;
+    std::string m_generate_from_json;
     std::string m_import_path;
 
     std::string m_electrum_seed;  // electrum-style seed parameter

src/version.h.in

@@ -1,4 +1,4 @@
 #define MONERO_VERSION_TAG "@VERSIONTAG@"
-#define MONERO_VERSION "0.9.1.0"
+#define MONERO_VERSION "0.9.3.0"
 #define MONERO_RELEASE_NAME "Hydrogen Helix"
 #define MONERO_VERSION_FULL MONERO_VERSION "-" MONERO_VERSION_TAG

src/wallet/wallet2.cpp

@@ -92,6 +92,13 @@ void do_prepare_file_names(const std::string& file_path, std::string& keys_file,
   }
 }
 
+uint64_t calculate_fee(const cryptonote::blobdata &blob)
+{
+  uint64_t bytes = blob.size();
+  uint64_t kB = (bytes + 1023) / 1024;
+  return kB * FEE_PER_KB;
+}
+
 } //namespace
 
 namespace tools
@@ -490,7 +497,7 @@ void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cry
   //handle transactions from new block
     
   //optimization: seeking only for blocks that are not older then the wallet creation time plus 1 day. 1 day is for possible user incorrect time setup
-  if(b.timestamp + 60*60*24 > m_account.get_createtime())
+  if(b.timestamp + 60*60*24 > m_account.get_createtime() && height >= m_refresh_from_block_height)
   {
     TIME_MEASURE_START(miner_tx_handle_time);
     process_new_transaction(b.miner_tx, height, true);
@@ -1207,7 +1214,7 @@ void wallet2::generate(const std::string& wallet_, const std::string& password,
   m_account_public_address = account_public_address;
   m_watch_only = false;
 
-  bool r = store_keys(m_keys_file, password, true);
+  bool r = store_keys(m_keys_file, password, false);
   THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file);
 
   r = file_io_utils::save_string_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_testnet));
@@ -2014,13 +2021,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions(std::vector<crypto
 	{
 	  transfer(dst_vector, fake_outs_count, unlock_time, needed_fee, extra, tx, ptx);
 	  auto txBlob = t_serializable_object_to_blob(ptx.tx);
-	  uint64_t txSize = txBlob.size();
+          needed_fee = calculate_fee(txBlob);
-	  uint64_t numKB = txSize / 1024;
-	  if (txSize % 1024)
-	  {
-	    numKB++;
-	  }
-	  needed_fee = numKB * FEE_PER_KB;
 	} while (ptx.fee < needed_fee);
 
         ptx_vector.push_back(ptx);
@@ -2392,15 +2393,9 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp
       transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, unlock_time, needed_fee, extra,
         detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx);
       auto txBlob = t_serializable_object_to_blob(test_ptx.tx);
-      uint64_t txSize = txBlob.size();
+      needed_fee = calculate_fee(txBlob);
-      uint64_t numKB = txSize / 1024;
-      if (txSize % 1024)
-      {
-        numKB++;
-      }
-      needed_fee = numKB * FEE_PER_KB;
       available_for_fee = test_ptx.fee + test_ptx.change_dts.amount;
-      LOG_PRINT_L2("Made a " << numKB << " kB tx, with " << print_money(available_for_fee) << " available for fee (" <<
+      LOG_PRINT_L2("Made a " << txBlob.size() << " kB tx, with " << print_money(available_for_fee) << " available for fee (" <<
         print_money(needed_fee) << " needed)");
 
       if (needed_fee > available_for_fee && dsts[0].amount > 0)
@@ -2497,7 +2492,7 @@ uint64_t wallet2::unlocked_dust_balance(const tx_dust_policy &dust_policy) const
 }
 
 template<typename T>
-void wallet2::transfer_dust(size_t num_outputs, uint64_t unlock_time, uint64_t needed_fee, T destination_split_strategy, const tx_dust_policy& dust_policy, const std::vector<uint8_t> &extra, cryptonote::transaction& tx, pending_tx &ptx)
+void wallet2::transfer_from(const std::vector<size_t> &outs, size_t num_outputs, uint64_t unlock_time, uint64_t needed_fee, T destination_split_strategy, const tx_dust_policy& dust_policy, const std::vector<uint8_t> &extra, cryptonote::transaction& tx, pending_tx &ptx)
 {
   using namespace cryptonote;
 
@@ -2507,6 +2502,19 @@ void wallet2::transfer_dust(size_t num_outputs, uint64_t unlock_time, uint64_t n
   // throw if there are none
   uint64_t money = 0;
   std::list<transfer_container::iterator> selected_transfers;
+#if 1
+  for (size_t n = 0; n < outs.size(); ++n)
+  {
+    const transfer_details& td = m_transfers[outs[n]];
+    if (!td.m_spent)
+    {
+      selected_transfers.push_back (m_transfers.begin() + outs[n]);
+      money += td.amount();
+      if (selected_transfers.size() >= num_outputs)
+        break;
+    }
+  }
+#else
   for (transfer_container::iterator i = m_transfers.begin(); i != m_transfers.end(); ++i)
   {
     const transfer_details& td = *i;
@@ -2518,6 +2526,7 @@ void wallet2::transfer_dust(size_t num_outputs, uint64_t unlock_time, uint64_t n
         break;
     }
   }
+#endif
 
   // we don't allow no output to self, easier, but one may want to burn the dust if = fee
   THROW_WALLET_EXCEPTION_IF(money <= needed_fee, error::not_enough_money, money, needed_fee, needed_fee);
@@ -2611,8 +2620,8 @@ bool wallet2::use_fork_rules(uint8_t version)
   r = net_utils::invoke_http_json_remote_command2(m_daemon_address + "/json_rpc", req_t, resp_t, m_http_client);
   m_daemon_rpc_mutex.unlock();
   CHECK_AND_ASSERT_MES(r, false, "Failed to connect to daemon");
-  CHECK_AND_ASSERT_MES(res.status != CORE_RPC_STATUS_BUSY, false, "Failed to connect to daemon");
+  CHECK_AND_ASSERT_MES(resp_t.result.status != CORE_RPC_STATUS_BUSY, false, "Failed to connect to daemon");
-  CHECK_AND_ASSERT_MES(res.status == CORE_RPC_STATUS_OK, false, "Failed to get hard fork status");
+  CHECK_AND_ASSERT_MES(resp_t.result.status == CORE_RPC_STATUS_OK, false, "Failed to get hard fork status");
 
   bool close_enough = res.height >=  resp_t.result.earliest_height - 10; // start using the rules a bit beforehand
   if (close_enough)
@@ -2630,20 +2639,85 @@ uint64_t wallet2::get_upper_tranaction_size_limit()
   return ((full_reward_zone * 125) / 100) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE;
 }
 //----------------------------------------------------------------------------------------------------
-std::vector<wallet2::pending_tx> wallet2::create_dust_sweep_transactions()
+std::vector<size_t> wallet2::select_available_outputs(const std::function<bool(const transfer_details &td)> &f)
+{
+  std::vector<size_t> outputs;
+  size_t n = 0;
+  for (transfer_container::const_iterator i = m_transfers.begin(); i != m_transfers.end(); ++i, ++n)
+  {
+    if (i->m_spent)
+      continue;
+    if (!is_transfer_unlocked(*i))
+      continue;
+    if (f(*i))
+      outputs.push_back(n);
+  }
+  return outputs;
+}
+//----------------------------------------------------------------------------------------------------
+std::vector<uint64_t> wallet2::get_unspent_amounts_vector()
+{
+  std::set<uint64_t> set;
+  for (const auto &td: m_transfers)
+  {
+    if (!td.m_spent)
+      set.insert(td.amount());
+  }
+  std::vector<uint64_t> vector;
+  vector.reserve(set.size());
+  for (const auto &i: set)
+  {
+    vector.push_back(i);
+  }
+  return vector;
+}
+//----------------------------------------------------------------------------------------------------
+std::vector<size_t> wallet2::select_available_unmixable_outputs(bool trusted_daemon)
+{
+  // request all outputs with at least 3 instances, so we can use mixin 2 with
+  epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request> req_t = AUTO_VAL_INIT(req_t);
+  epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response, std::string> resp_t = AUTO_VAL_INIT(resp_t);
+  m_daemon_rpc_mutex.lock();
+  req_t.jsonrpc = "2.0";
+  req_t.id = epee::serialization::storage_entry(0);
+  req_t.method = "get_output_histogram";
+  if (trusted_daemon)
+    req_t.params.amounts = get_unspent_amounts_vector();
+  req_t.params.min_count = 3;
+  req_t.params.max_count = 0;
+  bool r = net_utils::invoke_http_json_remote_command2(m_daemon_address + "/json_rpc", req_t, resp_t, m_http_client);
+  m_daemon_rpc_mutex.unlock();
+  THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "select_available_unmixable_outputs");
+  THROW_WALLET_EXCEPTION_IF(resp_t.result.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_histogram");
+  THROW_WALLET_EXCEPTION_IF(resp_t.result.status != CORE_RPC_STATUS_OK, error::get_histogram_error, resp_t.result.status);
+
+  std::set<uint64_t> mixable;
+  for (const auto &i: resp_t.result.histogram)
+  {
+    mixable.insert(i.amount);
+  }
+
+  return select_available_outputs([mixable](const transfer_details &td) {
+    const uint64_t amount = td.amount();
+    if (mixable.find(amount) == mixable.end())
+      return true;
+    return false;
+  });
+}
+//----------------------------------------------------------------------------------------------------
+std::vector<wallet2::pending_tx> wallet2::create_unmixable_sweep_transactions(bool trusted_daemon)
 {
   // From hard fork 1, we don't consider small amounts to be dust anymore
   const bool hf1_rules = use_fork_rules(2); // first hard fork has version 2
   tx_dust_policy dust_policy(hf1_rules ? 0 : ::config::DEFAULT_DUST_THRESHOLD);
 
-  size_t num_dust_outputs = 0;
+  // may throw
-  for (transfer_container::const_iterator i = m_transfers.begin(); i != m_transfers.end(); ++i)
+  std::vector<size_t> unmixable_outputs = select_available_unmixable_outputs(trusted_daemon);
+  size_t num_dust_outputs = unmixable_outputs.size();
+
+  if (num_dust_outputs == 0)
   {
-    const transfer_details& td = *i;
+    return std::vector<wallet2::pending_tx>();
-    if (!td.m_spent && (td.amount() < dust_policy.dust_threshold || !is_valid_decomposed_amount(td.amount())) && is_transfer_unlocked(td))
-    {
-      num_dust_outputs++;
-    }
   }
 
   // failsafe split attempt counter
@@ -2666,23 +2740,18 @@ std::vector<wallet2::pending_tx> wallet2::create_dust_sweep_transactions()
 
 	// loop until fee is met without increasing tx size to next KB boundary.
 	uint64_t needed_fee = 0;
-	if (1)
+	do
 	{
-	  transfer_dust(num_outputs_per_tx, (uint64_t)0 /* unlock_time */, 0, detail::digit_split_strategy, dust_policy, extra, tx, ptx);
+	  transfer_from(unmixable_outputs, num_outputs_per_tx, (uint64_t)0 /* unlock_time */, 0, detail::digit_split_strategy, dust_policy, extra, tx, ptx);
 	  auto txBlob = t_serializable_object_to_blob(ptx.tx);
-	  uint64_t txSize = txBlob.size();
+          needed_fee = calculate_fee(txBlob);
-	  uint64_t numKB = txSize / 1024;
-	  if (txSize % 1024)
-	  {
-	    numKB++;
-	  }
-	  needed_fee = numKB * FEE_PER_KB;
 
           // reroll the tx with the actual amount minus the fee
           // if there's not enough for the fee, it'll throw
-	  transfer_dust(num_outputs_per_tx, (uint64_t)0 /* unlock_time */, needed_fee, detail::digit_split_strategy, dust_policy, extra, tx, ptx);
+	  transfer_from(unmixable_outputs, num_outputs_per_tx, (uint64_t)0 /* unlock_time */, needed_fee, detail::digit_split_strategy, dust_policy, extra, tx, ptx);
 	  txBlob = t_serializable_object_to_blob(ptx.tx);
-	}
+          needed_fee = calculate_fee(txBlob);
+	} while (ptx.fee < needed_fee);
 
         ptx_vector.push_back(ptx);
 

src/wallet/wallet2.h

@@ -88,10 +88,10 @@ namespace tools
     };
 
   private:
-    wallet2(const wallet2&) : m_run(true), m_callback(0), m_testnet(false), m_always_confirm_transfers (false), m_store_tx_info(true), m_default_mixin(0), m_refresh_type(RefreshOptimizeCoinbase), m_auto_refresh(true) {}
+    wallet2(const wallet2&) : m_run(true), m_callback(0), m_testnet(false), m_always_confirm_transfers (false), m_store_tx_info(true), m_default_mixin(0), m_refresh_type(RefreshOptimizeCoinbase), m_auto_refresh(true), m_refresh_from_block_height(0) {}
 
   public:
-    wallet2(bool testnet = false, bool restricted = false) : m_run(true), m_callback(0), m_testnet(testnet), m_restricted(restricted), is_old_file_format(false), m_store_tx_info(true), m_default_mixin(0), m_refresh_type(RefreshOptimizeCoinbase), m_auto_refresh(true) {}
+    wallet2(bool testnet = false, bool restricted = false) : m_run(true), m_callback(0), m_testnet(testnet), m_restricted(restricted), is_old_file_format(false), m_store_tx_info(true), m_default_mixin(0), m_refresh_type(RefreshOptimizeCoinbase), m_auto_refresh(true), m_refresh_from_block_height(0) {}
     struct transfer_details
     {
       uint64_t m_block_height;
@@ -226,6 +226,8 @@ namespace tools
     cryptonote::account_base& get_account(){return m_account;}
     const cryptonote::account_base& get_account()const{return m_account;}
 
+    void set_refresh_from_block_height(uint64_t height) {m_refresh_from_block_height = height;}
+
     // upper_transaction_size_limit as defined below is set to 
     // approximately 125% of the fixed minimum allowable penalty
     // free block size. TODO: fix this so that it actually takes
@@ -278,7 +280,7 @@ namespace tools
     void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, size_t fake_outputs_count, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra);
     void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, size_t fake_outputs_count, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx& ptx);
     template<typename T>
-    void transfer_dust(size_t num_outputs, uint64_t unlock_time, uint64_t needed_fee, T destination_split_strategy, const tx_dust_policy& dust_policy, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx);
+    void transfer_from(const std::vector<size_t> &outs, size_t num_outputs, uint64_t unlock_time, uint64_t needed_fee, T destination_split_strategy, const tx_dust_policy& dust_policy, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx);
     template<typename T>
     void transfer_selected(const std::vector<cryptonote::tx_destination_entry>& dsts, const std::list<transfer_container::iterator> selected_transfers, size_t fake_outputs_count,
       uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx &ptx);
@@ -287,7 +289,7 @@ namespace tools
     void commit_tx(std::vector<pending_tx>& ptx_vector);
     std::vector<pending_tx> create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee, const std::vector<uint8_t> extra);
     std::vector<wallet2::pending_tx> create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, const uint64_t fee_UNUSED, const std::vector<uint8_t> extra);
-    std::vector<pending_tx> create_dust_sweep_transactions();
+    std::vector<pending_tx> create_unmixable_sweep_transactions(bool trusted_daemon);
     bool check_connection();
     void get_transfers(wallet2::transfer_container& incoming_transfers) const;
     void get_payments(const crypto::hash& payment_id, std::list<wallet2::payment_details>& payments, uint64_t min_height = 0) const;
@@ -319,6 +321,9 @@ namespace tools
       if(ver < 9)
         return;
       a & m_confirmed_txs;
+      if(ver < 11)
+        return;
+      a & m_refresh_from_block_height;
     }
 
     /*!
@@ -398,6 +403,9 @@ namespace tools
     void parse_block_round(const cryptonote::blobdata &blob, cryptonote::block &bl, crypto::hash &bl_id, bool &error) const;
     uint64_t get_upper_tranaction_size_limit();
     void check_pending_txes();
+    std::vector<uint64_t> get_unspent_amounts_vector();
+    std::vector<size_t> select_available_outputs(const std::function<bool(const transfer_details &td)> &f);
+    std::vector<size_t> select_available_unmixable_outputs(bool trusted_daemon);
 
     cryptonote::account_base m_account;
     std::string m_daemon_address;
@@ -431,9 +439,10 @@ namespace tools
     uint32_t m_default_mixin;
     RefreshType m_refresh_type;
     bool m_auto_refresh;
+    uint64_t m_refresh_from_block_height;
   };
 }
-BOOST_CLASS_VERSION(tools::wallet2, 10)
+BOOST_CLASS_VERSION(tools::wallet2, 11)
 BOOST_CLASS_VERSION(tools::wallet2::payment_details, 0)
 BOOST_CLASS_VERSION(tools::wallet2::unconfirmed_transfer_details, 2)
 BOOST_CLASS_VERSION(tools::wallet2::confirmed_transfer_details, 1)

src/wallet/wallet_errors.h

@@ -76,6 +76,7 @@ namespace tools
     //         daemon_busy
     //         no_connection_to_daemon
     //         is_key_image_spent_error
+    //         get_histogram_error
     //       wallet_files_doesnt_correspond
     //
     // * - class with protected ctor
@@ -600,6 +601,14 @@ namespace tools
       }
     };
     //----------------------------------------------------------------------------------------------------
+    struct get_histogram_error : public wallet_rpc_error
+    {
+      explicit get_histogram_error(std::string&& loc, const std::string& request)
+        : wallet_rpc_error(std::move(loc), "failed to get output histogram", request)
+      {
+      }
+    };
+    //----------------------------------------------------------------------------------------------------
     struct wallet_files_doesnt_correspond : public wallet_logic_error
     {
       explicit wallet_files_doesnt_correspond(std::string&& loc, const std::string& keys_file, const std::string& wallet_file)

src/wallet/wallet_rpc_server.cpp

@@ -347,7 +347,7 @@ namespace tools
 
     try
     {
-      std::vector<wallet2::pending_tx> ptx_vector = m_wallet.create_dust_sweep_transactions();
+      std::vector<wallet2::pending_tx> ptx_vector = m_wallet.create_unmixable_sweep_transactions(req.trusted_daemon);
 
       m_wallet.commit_tx(ptx_vector);
 

src/wallet/wallet_rpc_server_commands_defs.h

@@ -178,9 +178,11 @@ namespace wallet_rpc
     struct request
     {
       bool get_tx_keys;
+      bool trusted_daemon;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(get_tx_keys)
+        KV_SERIALIZE(trusted_daemon)
       END_KV_SERIALIZE_MAP()
     };
 

tests/CMakeLists.txt

@@ -54,7 +54,7 @@ else ()
   endif()
 endif ()
 
-#add_subdirectory(core_tests)
+add_subdirectory(core_tests)
 add_subdirectory(crypto)
 add_subdirectory(functional_tests)
 add_subdirectory(performance_tests)

tests/core_tests/chaingen.h

@@ -472,7 +472,7 @@ inline bool replay_events_through_core(cryptonote::core& cr, const std::vector<t
 //--------------------------------------------------------------------------
 template<class t_test_class>
 struct get_test_options {
-  const std::pair<uint8_t, uint64_t> hard_forks[1] = {std::make_pair(1, 0)};
+  const std::pair<uint8_t, uint64_t> hard_forks[1] = {std::make_pair((uint8_t)1, (uint64_t)0)};
   const cryptonote::test_options test_options = {
     hard_forks
   };

tests/unit_tests/hardfork.cpp

@@ -108,6 +108,7 @@ public:
   virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>) const { return true; }
   virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const { return true; }
   virtual bool is_read_only() const { return false; }
+  virtual std::map<uint64_t, uint64_t> get_output_histogram(const std::vector<uint64_t> &amounts) const { return std::map<uint64_t, uint64_t>(); }
 
   virtual void add_block( const block& blk
                         , const size_t& block_size

utils/gpg_keys/aabramov.asc

@@ -1,31 +0,0 @@
------BEGIN PGP PUBLIC KEY BLOCK-----
-Version: GnuPG v1
-
-mQENBFQwICMBCACyHKYHaSEdNyE8kqYm+V37IsX5xxqMTkOCuVmaWt8i8FP2aPDz
-TgsITQ8tHoJubL0wh/y6Z/KQY8cSWT/JU39gP64I1XdGVgaPErF3kZ7n2jh+K5HY
-vBWrFvw4Xj2XvHaJZW6rVVX5dc4rxqnnOjfKTPI6/tQhT16dwbMuCyZOyMfStAF9
-q+14I9M2MAbMlgNO0el0aR3GMVjoDeaWBk/qWxTaHjyOBAVyryoFpwIl7Gmwkk1d
-+jv4542LjJOiiwBEDcWUmNvlsRk5sXMcXImJSpaCN1Ta9Eyu2NBIK/qAbuC6YgLe
-nn/0DaA7TOpeIMU/WNq/Dpbathst+fMJNK6BABEBAAG0JkFsZXhhbmRlciBBYnJh
-bW92IDxhYWJyYW1vdkBnbWFpbC5jb20+iQE+BBMBAgAoBQJUMCAjAhsDBQkDwmcA
-BgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRA/57nKbsMV+F/+B/9f7C1iiEFD
-18Af48WLJXZaJJTqxMeWzztWhZ2LS27rFOGJcj50UeON7aVu8LWU8gvyPJct6wJA
-T1hvPWc6TTsbYp8KHP/nUfUWWV+np0qVSj91eGHp7NjAxXYe4NnWt1JUtINxvNll
-BKY2l0MHHHKwuWxky6zug48CToB8Y0RBDUgFM0pmnCzmySs6OAJbLmnnvesTDFZk
-m6wycq4/SrbVByTZHVqBmZNnypMQxbqXTti9ltpXWqO89RpQQ+QBQQlrNy9QRHJq
-VW9Qn5eL4O4IdmXpHqln1ppSu2il+pqQCGc5O9xFxqG6YRHljQEngBfgKLxOsqa1
-OGJHu1pLyUYRuQENBFQwICMBCACmQ+1DRgm9AMNo4iDur47obDzUTQ/1g2ah1+AT
-IYNhhltBD3jURsRZ6lMeancxU2/6hyUzCon5ETKfnF0+ujqldSDuo+XY9CDgmXLC
-s3wPAKZvyjScwMFTyW1hGfyT9SFwYVDaXOeW+VrJ97Xh5FWHx8LU7eieYIl8tOUY
-YWEeIps/y5zBkk7bB4LVNVc3AlKPS4vb/py5szA5fMg8ix/ggvpq8be4BnImP3a1
-yrl+W9HUtcTs8jBnFLMoZYaUklZ9LR7yfnjhhtKqW833UB07qYQY//dJGy4kVnit
-M5LX9VKa+wnWsgP+w5jElPX0bweV6+V/yFRhjLLwV7dv/MJnABEBAAGJASUEGAEC
-AA8FAlQwICMCGwwFCQPCZwAACgkQP+e5ym7DFfgrvAf/bnQD5iR/HiNsH0HQE96G
-Wsrr9VW/ucnm06pd1D/r7ogYcVWuNIglx3QdvFla4uZDvYR0PhyjNrOyPzUgLWp/
-fjQw9buyI3BawAScbPuN06zHLUIvDQpaPq2+uBRq7sCcdsox6sMiUH6x5QcD3lbR
-nh6aAGwSkTabW5a/GVZTIHSicGbhXUaIIh3w+sQZ/y5xTVjgVi4I4XdrxOZoo5RN
-nF5oyn74n5p9euxJMWgvC/tz/bD7VKYWJHRnuyeumLTPudLjPFFQpTX+I+gMJ5FT
-Vs7byaOlrZbQxzXd4G0uBnCvJ1FH3cL6Ddqlsz/l2WzddMknMbVMn7+fSVMdpbIm
-hw==
-=7XDh
------END PGP PUBLIC KEY BLOCK-----

utils/gpg_keys/hyc.asc

@@ -0,0 +1,44 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mQGiBERuNuERBACY9KGBcxSvy6D9Sxk51L59/7ACWQf/Mk2B5jWCxcadEZHokKf4
+mYBBH963B1v04S8QKsm1avt4amS3dEEfxJ920RWtg5XkVCW0HfErsZ58XobIcPLD
+pd7u3btGHObQfJ5fzyRuIaWpBg59gGJAw13Rz/ILuDpXHBP0ppI9OOq6nwCgnG+I
+B56YTz9oGLiVXFsa8LItTI0D/jBTd5UWQ+BYCU8HdHq+fRm6l/N921yrFO4hF1jU
+4Mi6t9srHkQVW5ro9cxVhAWKCbVum5ogylcfH1WK//R2HkSdKJXNVxZdBieGD/tL
+YYSJj3qU7UA9ZTH5QGUkNqcHH7bV+o+5oZdhIhpcMuG1TFudGkQcnTWW0X69S6bR
+uBa1A/9Zg6Hk2uL0ytbHtASWKZHr80f1hhe6catT314nqJNU5rufYHHHCvQJVDPs
+NWkj4OfkBE5xG1VxMPulvMTJIKm1FdXK+1B9OfcXUk/+zyCiYLGbrAjuUlHDmVDH
+P1YQEn+a8S8QowXl+tAL62lO3mDCiPny6ohuIJiTfLLPrE1+bLQaSG93YXJkIENo
+dSA8aHljQHN5bWFzLmNvbT6IRgQQEQIABgUCRukakgAKCRCZSHtC+KtbW2rTAJ9/
+j5IiB0Fvq7gAs2Zf/eqc4wqLzwCeLicSAHhaXSRrKd7ZEFTnAFW/qWSIRgQQEQIA
+BgUCSDbdawAKCRBsEnep072iJdceAJ44GfpthT4ZPN8XEob5pn8qsFPKIgCfbcNI
+wsWoHybiovYiHBZTpW50/QqIRgQQEQIABgUCSDbe8wAKCRBDbTC1hOUd3UFUAKCn
+CAhM4Hpe2QNGZM8lYewn2bVRogCeKs5eOlG4vqzYsH1ro1evBsKFXLyIZAQTEQIA
+JAUCRG424QIbAwUJEswDAAYLCQgHAwIDFQIDAxYCAQIeAQIXgAAKCRD9KnC0SrEb
+p72rAJ44Keweij6d4EFjROG6PdF8J64vYACfZ99qH7+wzvBxaMuAIAw558lBaY2I
+XgQTEQIAHgIbAwYLCQgHAwIDFQIDAxYCAQIeAQIXgAUCVpAQGAAKCRD9KnC0SrEb
+p+3XAJwNjg2sFjNTwzf5LaTT/wl2sWwqbgCgiX9jv1OS+pzJHH4OpjTt5+lHuUO5
+AQ0ERG424RAEAIrOpREkyhB6FTmYo+Tw3r1/pfO0gHC1FcuTrL0lsRVOYwiwVwmk
+gVME4gvHRNqnT1gG/RMQm6kKGz07RvzVaRrrT7B/nOG9kbVMzRQIqBbk1o+u7TFr
+EYSvXF9xfl3Sq2/UYqIURCW9ZIOF6JLGcF0qGrhheg9eDq97tAfZDwvXAAMFA/4u
+38QVNe0Izqc9FTGkO6VijsfXvsn3tCKqm7v/vkJa8Rpu3dumChwMtiGwkzLCJMpC
+TbnL0dW4hChHjT/DOWKV84WZwP6vmuMlhNFFrsRh9wqi6LjBNdhVHHfjx2JU0R4L
+kID4FSiU20TH4Z/Vm+JSC1E272kw3xfacga4e3DpN4hPBBgRAgAPBQJEbjbhAhsM
+BQkSzAMAAAoJEP0qcLRKsRun3wAAn1BnosO00oGZ0sVX5N+m8haWI6YeAJ9HV/RG
+L4t04ykgAnU8LLAOaNCWV7kCDQRWkBA0EAgAmQ81PiXqJmk475469zKLWyxjrvG/
+utPTzE4jgah7Ib3I591J6cGmpoOWeM9bdaLc2DGxWqpjcZd/6dFo4CyIFLMPXsgv
+JH0iwh5gUt/YBUHkz3kVpN+UqoE+5d+6LVoJQrio/lHAUpFBtpnrx2BeolWRmG6m
+uTQwOkyNXUbFgSIwpKuN1rqe2U4KuvFEBTX1QCCzGIOEYtbmtBdGtg/FLwt/+pIF
+QHY4tQe/bykX+XLYyLCChoq5Kkq+KZv0Ebafg3vm+xUMwbeqQwUNFLPjIFpHeeKi
+nQk8BPNvFjf+MmZ/y/tbsA/156WfMldHsXOV/nD6JWk2HPv8/BL7Fz4bvwADBQf+
+M+6GDIWUpZFfix7eDBZhtyNbehQ6sqSGu97LljKZXb0SxOM5jTkAguuVmjOM8bi4
+cVs9LJt1nFD+PmNRLuju4tda4xk61fvueqv1T0c59fFak5STejzrpWhDeEJZmJMw
+EwSv4e0OTFsKlmWJSvfoOavPhbw+Tuhgza1WFLjedqo+RQnCQmXMNlcblD6otgSV
+X1uadZBWODmHX/nBmgyDzSILZsy3mJkspAIx/bH95oTMaGQAkVH++WLLGUmjr6tb
+uru3e8+mCUvpEptaaBcIq8minoVvfkIZKUgISKf4Yize1PoEuCQbnETcfekZyY94
+m/RwmduzBwMXzMikPOyRwYhPBBgRAgAPBQJWkBA0AhsMBQkSzAMAAAoJEP0qcLRK
+sRunqSUAnRenNWORvzTRRy0qmF5xVFlDIUGpAJ4pUZgkE7YRyjSzdhxcaRBOjAQa
+GQ==
+=sUzZ
+-----END PGP PUBLIC KEY BLOCK-----

utils/munin_plugins/alt_blocks_count

@@ -42,4 +42,4 @@ EOM
 esac
 
 printf "alt_blocks_count.value "
-/home/user/bitmonero/build/release/src/connectivity_tool --ip=127.0.0.1 --rpc_port=8081 --timeout=1000 --rpc_get_daemon_info | grep alt_blocks_count | cut -d ' ' -f2
+# rewrite using curl or similar: /home/user/bitmonero/build/release/src/connectivity_tool --ip=127.0.0.1 --rpc_port=8081 --timeout=1000 --rpc_get_daemon_info | grep alt_blocks_count | cut -d ' ' -f2

utils/munin_plugins/difficulty

@@ -42,4 +42,4 @@ EOM
 esac
 
 printf "difficulty.value "
-/home/user/bitmonero/build/release/src/connectivity_tool --ip=127.0.0.1 --rpc_port=8081 --timeout=1000 --rpc_get_daemon_info | grep difficulty | cut -d ' ' -f2
+# rewrite using curl or similar: /home/user/bitmonero/build/release/src/connectivity_tool --ip=127.0.0.1 --rpc_port=8081 --timeout=1000 --rpc_get_daemon_info | grep difficulty | cut -d ' ' -f2

utils/munin_plugins/grey_peerlist_size

@@ -42,4 +42,4 @@ EOM
 esac
 
 printf "grey_peerlist_size.value "
-/home/user/bitmonero/build/release/src/connectivity_tool --ip=127.0.0.1 --rpc_port=8081 --timeout=1000 --rpc_get_daemon_info | grep grey_peerlist_size | cut -d ' ' -f2
+# rewrite using curl or similar: /home/user/bitmonero/build/release/src/connectivity_tool --ip=127.0.0.1 --rpc_port=8081 --timeout=1000 --rpc_get_daemon_info | grep grey_peerlist_size | cut -d ' ' -f2

utils/munin_plugins/height

@@ -43,4 +43,4 @@ EOM
 esac
 
 printf "height.value "
-/home/user/bitmonero/build/release/src/connectivity_tool --ip=127.0.0.1 --rpc_port=8081 --timeout=1000 --rpc_get_daemon_info | grep height | cut -d ' ' -f2
+# rewrite using curl or similar: /home/user/bitmonero/build/release/src/connectivity_tool --ip=127.0.0.1 --rpc_port=8081 --timeout=1000 --rpc_get_daemon_info | grep height | cut -d ' ' -f2

utils/munin_plugins/incoming_connections_count

@@ -42,4 +42,4 @@ EOM
 esac
 
 printf "incoming_connections_count.value "
-/home/user/bitmonero/build/release/src/connectivity_tool --ip=127.0.0.1 --rpc_port=8081 --timeout=1000 --rpc_get_daemon_info | grep incoming_connections_count | cut -d ' ' -f2
+# rewrite using curl or similar: /home/user/bitmonero/build/release/src/connectivity_tool --ip=127.0.0.1 --rpc_port=8081 --timeout=1000 --rpc_get_daemon_info | grep incoming_connections_count | cut -d ' ' -f2

utils/munin_plugins/outgoing_connections_count

@@ -42,4 +42,4 @@ EOM
 esac
 
 printf "outgoing_connections_count.value "
-/home/user/bitmonero/build/release/src/connectivity_tool --ip=127.0.0.1 --rpc_port=8081 --timeout=1000 --rpc_get_daemon_info | grep outgoing_connections_count | cut -d ' ' -f2
+# rewrite using curl or similar: /home/user/bitmonero/build/release/src/connectivity_tool --ip=127.0.0.1 --rpc_port=8081 --timeout=1000 --rpc_get_daemon_info | grep outgoing_connections_count | cut -d ' ' -f2

utils/munin_plugins/tx_count

@@ -43,4 +43,4 @@ EOM
 esac
 
 printf "tx_count.value "
-/home/user/bitmonero/build/release/src/connectivity_tool --ip=127.0.0.1 --rpc_port=8081 --timeout=1000 --rpc_get_daemon_info | grep tx_count| cut -d ' ' -f2
+# rewrite using curl or similar: /home/user/bitmonero/build/release/src/connectivity_tool --ip=127.0.0.1 --rpc_port=8081 --timeout=1000 --rpc_get_daemon_info | grep tx_count| cut -d ' ' -f2

utils/munin_plugins/tx_pool_size

@@ -42,4 +42,4 @@ EOM
 esac
 
 printf "tx_pool_size.value "
-/home/user/bitmonero/build/release/src/connectivity_tool --ip=127.0.0.1 --rpc_port=8081 --timeout=1000 --rpc_get_daemon_info | grep tx_pool_size| cut -d ' ' -f2
+# rewrite using curl or similar: /home/user/bitmonero/build/release/src/connectivity_tool --ip=127.0.0.1 --rpc_port=8081 --timeout=1000 --rpc_get_daemon_info | grep tx_pool_size| cut -d ' ' -f2

utils/munin_plugins/white_peerlist_size

@@ -42,4 +42,4 @@ EOM
 esac
 
 printf "white_peerlist_size.value "
-/home/user/bitmonero/build/release/src/connectivity_tool --ip=127.0.0.1 --rpc_port=8081 --timeout=1000 --rpc_get_daemon_info | grep white_peerlist_size | cut -d ' ' -f2
+# rewrite using curl or similar: /home/user/bitmonero/build/release/src/connectivity_tool --ip=127.0.0.1 --rpc_port=8081 --timeout=1000 --rpc_get_daemon_info | grep white_peerlist_size | cut -d ' ' -f2