validation.js

/*jslint node: true */
"use strict";
var _ = require('lodash');
var async = require('async');
var storage = require('./storage.js');
var graph = require('./graph.js');
var main_chain = require('./main_chain.js');
const archiving = require('./archiving.js');
var paid_witnessing = require("./paid_witnessing.js");
var headers_commission = require("./headers_commission.js");
var mc_outputs = require("./mc_outputs.js");
var objectHash = require("./object_hash.js");
var objectLength = require("./object_length.js");
const { isTooDeeplyNestedOrHasTooManyNodes, isObjectWellFormed } = require("./string_utils.js");
var db = require('./db.js');
var mutex = require('./mutex.js');
var constants = require("./constants.js");
var ValidationUtils = require("./validation_utils.js");
var Definition = require("./definition.js");
var aa_validation = require("./aa_validation.js");
var conf = require('./conf.js');
var profiler = require('./profiler.js');
var breadcrumbs = require('./breadcrumbs.js');

var MAX_INT32 = Math.pow(2, 31) - 1;
const BOOT_DATE = new Date().toISOString().replace(/T/, ' ').slice(0, 19); // YYYY-MM-DD HH:MM:SS

var hasFieldsExcept = ValidationUtils.hasFieldsExcept;
var isNonemptyString = ValidationUtils.isNonemptyString;
var isNonemptyObject = ValidationUtils.isNonemptyObject;
var isArrayOfLength = ValidationUtils.isArrayOfLength;
var isStringOfLength = ValidationUtils.isStringOfLength;
var isInteger = ValidationUtils.isInteger;
var isNonnegativeInteger = ValidationUtils.isNonnegativeInteger;
var isPositiveInteger = ValidationUtils.isPositiveInteger;
var isNonemptyArray = ValidationUtils.isNonemptyArray;
var isValidAddress = ValidationUtils.isValidAddress;
var isValidBase64 = ValidationUtils.isValidBase64;

var assocWitnessListMci = {};

function hasValidHashes(objJoint){
	var objUnit = objJoint.unit;
	try {
		if (objectHash.getUnitHash(objUnit) !== objUnit.unit)
			return false;
	}
	catch(e){
		console.log("failed to calc unit hash: "+e);
		return false;
	}
	return true;
}

function hasValidPayloadHashes(objJoint) {
	try {
		if (!("messages" in objJoint.unit)) return true; // final-bad
		if (!isNonemptyArray(objJoint.unit.messages)) return false;
		for (let m of objJoint.unit.messages) {
			if (m.payload_location !== "inline") {
				if ("payload" in m)
					return false;
				continue;
			}
			const expected_payload_hash = objectHash.getBase64Hash(getPayloadForHash(m), objJoint.unit.version !== constants.versionWithoutTimestamp);
			if (expected_payload_hash !== m.payload_hash)
				return false;
			if (m.app === "temp_data" && "data" in m.payload) {
				const hash = objectHash.getBase64Hash(m.payload.data, true);
				if (hash !== m.payload.data_hash)
					return false;
			}
		}
	}
	catch (e) {
		console.log("failed to calc payload hash: " + e);
		return false;
	}
	return true;
}

function allHashesAreValid(objJoint) {
	return hasValidHashes(objJoint) && hasValidPayloadHashes(objJoint);
}


function getPayloadForHash(objMessage) {
	if (objMessage.app !== "temp_data")
		return objMessage.payload;
	let p = _.cloneDeep(objMessage.payload);
	delete p.data;
	return p;
}

const lightStableErrorMessage = "I'm light, can't accept stable unit without proof";

function validateLight(objJoint) {
	if (!conf.bLight) throw Error("validateLightUnstable called in non-light mode");
	let objJointCopy = { ...objJoint };
	objJointCopy.unit = { ...objJoint.unit };
	delete objJointCopy.unit.main_chain_index;
	delete objJointCopy.unit.actual_tps_fee;
	let error;
	validate(objJointCopy, {
		// all callbacks are called synchronously, so we can just assign the error and return it
		ifOk: () => {},
		ifUnitError: err => error = err,
		ifJointError: err => {
			if (err !== lightStableErrorMessage) // this means the unit is stable
				error = err;
		},
		ifTransientError: err => error = err,
		// all other callbacks are impossible in light mode and intentionally left out to cause a crash if still called
	});
	return error;
}

function validate(objJoint, callbacks, external_conn) {
	
	var objUnit = objJoint.unit;
	if (typeof objUnit !== "object" || objUnit === null)
		throw Error("no unit object");
	if (!objUnit.unit)
		throw Error("no unit");
	
	console.log("\nvalidating joint identified by unit "+objJoint.unit.unit);
	
	if (!isStringOfLength(objUnit.unit, constants.HASH_LENGTH))
		return callbacks.ifJointError("wrong unit length");
	
	try{
		// UnitError is linked to objUnit.unit, so we need to ensure objUnit.unit is true before we throw any UnitErrors
		if (objectHash.getUnitHash(objUnit) !== objUnit.unit)
			return callbacks.ifJointError("wrong unit hash: "+objectHash.getUnitHash(objUnit)+" != "+objUnit.unit);
	}
	catch(e){
		return callbacks.ifJointError("failed to calc unit hash: "+e);
	}

	const bGenesis = storage.isGenesisUnit(objUnit.unit);

	var bAA = false;
	if (objJoint.aa) {
		bAA = true;
		var aa_mci = objJoint.aa_mci;
		delete objJoint.aa;
		delete objJoint.aa_mci;
	}
	else {
		if (isArrayOfLength(objUnit.authors, 1) && !isNonemptyObject(objUnit.authors[0].authentifiers) && !objUnit.content_hash && !conf.bLight)
			return callbacks.ifTransientError("possible AA");
	}
	
	if (isTooDeeplyNestedOrHasTooManyNodes(objUnit))
		return bAA ? callbacks.ifUnitError("unit is too deeply nested") : callbacks.ifJointError("unit is too deeply nested");

	if (!isObjectWellFormed(objJoint))
		return bAA ? callbacks.ifUnitError("unit contains invalid string (lone surrogate or null byte)") : callbacks.ifJointError("unit contains invalid string (lone surrogate or null byte)");

	if (objJoint.unsigned){
		if (hasFieldsExcept(objJoint, ["unit", "unsigned"]))
			return callbacks.ifJointError("unknown fields in unsigned unit-joint");
	}
	else if ("ball" in objJoint){
		if (!isStringOfLength(objJoint.ball, constants.HASH_LENGTH))
			return callbacks.ifJointError("wrong ball length");
		if (hasFieldsExcept(objJoint, ["unit", "ball", "skiplist_units"]))
			return callbacks.ifJointError("unknown fields in ball-joint");
		if ("skiplist_units" in objJoint){
			if (!isNonemptyArray(objJoint.skiplist_units))
				return callbacks.ifJointError("missing or empty skiplist array");
			//if (objUnit.unit.charAt(0) !== "0")
			//    return callbacks.ifJointError("found skiplist while unit doesn't start with 0");
		}
	}
	else{
		if (hasFieldsExcept(objJoint, ["unit"]))
			return callbacks.ifJointError("unknown fields in unit-joint");
	}
	
	if ("content_hash" in objUnit){ // nonserial and stripped off content
		if (!isStringOfLength(objUnit.content_hash, constants.HASH_LENGTH))
			return callbacks.ifUnitError("wrong content_hash length");
		if (hasFieldsExcept(objUnit, ["unit", "version", "alt", "timestamp", "authors", "witness_list_unit", "witnesses", "content_hash", "parent_units", "last_ball", "last_ball_unit"]))
			return callbacks.ifJointError("unknown fields in nonserial unit");
		if (!objJoint.ball)
			return callbacks.ifJointError("content_hash allowed only in finished ball");
	}
	else{ // serial
		if (hasFieldsExcept(objUnit, ["unit", "version", "alt", "timestamp", "authors", "messages", "witness_list_unit", "witnesses", "earned_headers_commission_recipients", "last_ball", "last_ball_unit", "parent_units", "headers_commission", "payload_commission", "oversize_fee", "tps_fee", "burn_fee", "max_aa_responses"]))
			return callbacks.ifUnitError("unknown fields in unit");

		if (typeof objUnit.headers_commission !== "number")
			return callbacks.ifJointError("no headers_commission");
		if (typeof objUnit.payload_commission !== "number")
			return callbacks.ifJointError("no payload_commission");
		if ("oversize_fee" in objUnit && !isPositiveInteger(objUnit.oversize_fee))
			return callbacks.ifJointError("bad oversize_fee");
		if ("tps_fee" in objUnit && !isNonnegativeInteger(objUnit.tps_fee))
			return callbacks.ifUnitError("bad tps_fee");
		if ("burn_fee" in objUnit && !isPositiveInteger(objUnit.burn_fee))
			return callbacks.ifUnitError("bad burn_fee");
		if ("max_aa_responses" in objUnit) {
			if (!isNonnegativeInteger(objUnit.max_aa_responses))
				return callbacks.ifUnitError("bad max_aa_responses");
			if (objUnit.max_aa_responses > constants.MAX_RESPONSES_PER_PRIMARY_TRIGGER && !(constants.bTestnet && ['pLpKQaXTTgcpFrd8B1zvcw2dLcDAvaUkQuz1fY+ldPA=', 'J/8gMqNqIkq2LIPV7rQNjWk37u16/P7nToV+SSZxMDs='].includes(objUnit.unit)))
				return callbacks.ifTransientError("max_aa_responses too large"); // later: ifUnitError
			if (objUnit.max_aa_responses === 0 && !("ball" in objJoint))
				return callbacks.ifTransientError("max_aa_responses=0 not allowed in new units");
		}
		
		if (!isNonemptyArray(objUnit.messages))
			return callbacks.ifUnitError("missing or empty messages array");
		if (objUnit.messages.length > constants.MAX_MESSAGES_PER_UNIT && !bGenesis)
			return callbacks.ifUnitError("too many messages");
		if (!objUnit.messages.every(isNonemptyObject))
			return callbacks.ifUnitError("all messages must be non-empty objects");

		// validate payload hashes early to make sure payloads were not tampered with
		for (let m of objUnit.messages) {
			if (typeof m.payload_location !== "string")
				return callbacks.ifUnitError("bad payload_location");
			if (!["inline", "none", "uri"].includes(m.payload_location))
				return callbacks.ifUnitError("invalid payload_location: " + m.payload_location);
			if (!isStringOfLength(m.payload_hash, constants.HASH_LENGTH))
				return callbacks.ifUnitError("wrong payload hash size");
			if (m.payload_location !== "inline") {
				if ("payload" in m)
					return callbacks.ifJointError("payload must be absent when payload_location is not inline");
				continue;
			};
			if (!("payload" in m) || m.payload === null)
				return callbacks.ifJointError("missing payload");
			
			try {
				const expected_payload_hash = objectHash.getBase64Hash(getPayloadForHash(m), objUnit.version !== constants.versionWithoutTimestamp);
				if (expected_payload_hash !== m.payload_hash)
					return callbacks.ifJointError("wrong payload hash: expected " + expected_payload_hash + ", got " + m.payload_hash);
			}
			catch(e) {
				return callbacks.ifJointError("failed to calc payload hash: " + e);
			}
		}

		if (!objUnit.messages.every(m => {
			if (m.app === "payment" && m.payload)
				return isNonemptyArray(m.payload.outputs) &&
					(!("asset" in m.payload) || isStringOfLength(m.payload.asset, constants.HASH_LENGTH)) &&
					m.payload.outputs.every(o => isNonemptyObject(o) && (isValidAddress(o.address) || objUnit.version === constants.versionWithoutTimestamp && ValidationUtils.isValidAddressAnyCase(o.address)) && isPositiveInteger(o.amount) && o.amount <= constants.MAX_CAP) &&
					isNonemptyArray(m.payload.inputs) &&
					m.payload.inputs.every(i => isNonemptyObject(i) && (!("type" in i) || ["issue", "headers_commission", "witnessing"].includes(i.type)));
			else
				return true;
		}))
			return callbacks.ifUnitError("invalid payment message");

		if (objectLength.getHeadersSize(objUnit) !== objUnit.headers_commission)
			return callbacks.ifJointError("wrong headers commission, expected "+objectLength.getHeadersSize(objUnit));
		try {
			const payloadSize = objectLength.getTotalPayloadSize(objUnit);
			if (payloadSize !== objUnit.payload_commission)
				return callbacks.ifJointError("wrong payload commission, unit " + objUnit.unit + ", expected " + payloadSize);
		}
		catch (e) {
			return callbacks.ifJointError("failed to calculate payload commission: " + e);
		}
		if (objUnit.headers_commission + objUnit.payload_commission > constants.MAX_UNIT_LENGTH && !bGenesis && !bAA)
			return callbacks.ifUnitError("unit too large");
	}
	
	if (!isNonemptyArray(objUnit.authors))
		return callbacks.ifUnitError("missing or empty authors array");
	

	if (constants.supported_versions.indexOf(objUnit.version) === -1)
		return callbacks.ifUnitError("wrong version");
	if (objUnit.alt !== constants.alt)
		return callbacks.ifUnitError("wrong alt");

	if (objUnit.version !== constants.versionWithoutTimestamp) {
		if (!isPositiveInteger(objUnit.timestamp))
			return callbacks.ifUnitError("timestamp required in version " + objUnit.version);
		var current_ts = Math.round(Date.now() / 1000);
		var max_seconds_into_the_future_to_accept = conf.max_seconds_into_the_future_to_accept || 5;
		if (objUnit.timestamp > current_ts + max_seconds_into_the_future_to_accept)
			return callbacks.ifTransientError("timestamp is too far into the future");
	}


	if (bGenesis) {
		if ("oversize_fee" in objUnit)
			return callbacks.ifJointError("oversize_fee in genesis");
		if ("tps_fee" in objUnit)
			return callbacks.ifUnitError("tps_fee in genesis");
		if ("burn_fee" in objUnit)
			return callbacks.ifUnitError("burn_fee in genesis");
		if ("max_aa_responses" in objUnit)
			return callbacks.ifUnitError("max_aa_responses in genesis");
		if ("parent_units" in objUnit)
			return callbacks.ifUnitError("parent_units in genesis");
		if ("last_ball" in objUnit)
			return callbacks.ifUnitError("last_ball in genesis");
		if ("last_ball_unit" in objUnit)
			return callbacks.ifUnitError("last_ball_unit in genesis");
	}
	else {
		if (!isNonemptyArray(objUnit.parent_units))
			return callbacks.ifUnitError("missing or empty parent units array");
		if (!objUnit.parent_units.every(p => isStringOfLength(p, constants.HASH_LENGTH)))
			return callbacks.ifUnitError("wrong length of parent unit");
		if (!isStringOfLength(objUnit.last_ball, constants.HASH_LENGTH))
			return callbacks.ifUnitError("wrong length of last ball");
		if (!isStringOfLength(objUnit.last_ball_unit, constants.HASH_LENGTH))
			return callbacks.ifUnitError("wrong length of last ball unit");
	}
	
	
	if ("witness_list_unit" in objUnit && "witnesses" in objUnit)
		return callbacks.ifUnitError("ambiguous witnesses");
		
	var arrAuthorAddresses = objUnit.authors ? objUnit.authors.map(function(author) { return author.address; } ) : [];
	
	var objValidationState = {
		arrAdditionalQueries: [],
		arrDoubleSpendInputs: [],
		arrInputKeys: []
	};
	if (bGenesis) {
		objValidationState.last_ball_mci = 0;
		objValidationState.bGenesis = true;
	}
	if (objJoint.unsigned)
		objValidationState.bUnsigned = true;
	objValidationState.bAA = bAA;
	if (bAA)
		objValidationState.aa_mci = aa_mci;
	if (objJoint.ball)
		objValidationState.hasBall = true;

	if (conf.bLight){
		if (!isPositiveInteger(objUnit.timestamp) && !objJoint.unsigned)
			return callbacks.ifJointError("bad timestamp");
		if (objJoint.ball)
			return callbacks.ifJointError(lightStableErrorMessage);
		return objJoint.unsigned 
			? callbacks.ifOkUnsigned(true) 
			: callbacks.ifOk({sequence: 'good', arrDoubleSpendInputs: [], arrAdditionalQueries: []}, function(){});
	}
	else{
		if ("timestamp" in objUnit && !isPositiveInteger(objUnit.timestamp))
			return callbacks.ifJointError("bad timestamp");
	}

	if (!arrAuthorAddresses.every(isValidAddress))
		return callbacks.ifUnitError("invalid author address");

	mutex.lock(arrAuthorAddresses, function(unlock){
		
		var conn = null;
		var commit_fn = null;
		var start_time = null;

		async.series(
			[
				function(cb){
					if (external_conn) {
						conn = external_conn;
						start_time = Date.now();
						commit_fn = function (cb2) { cb2(); };
						return cb();
					}
					db.takeConnectionFromPool(function(new_conn){
						conn = new_conn;
						start_time = Date.now();
						commit_fn = function (cb2) {
							conn.query(objValidationState.bAdvancedLastStableMci ? "COMMIT" : "ROLLBACK", function () { cb2(); });
						};
						conn.query("BEGIN", function(){cb();});
					});
				},
				function(cb){
					profiler.start();
					checkDuplicate(conn, objUnit, cb);
				},
				function(cb){
					profiler.stop('validation-checkDuplicate');
					profiler.start();
					objUnit.content_hash ? cb() : validateHeadersCommissionRecipients(objUnit, cb);
				},
				function(cb){
					profiler.stop('validation-hc-recipients');
					profiler.start();
					!objUnit.parent_units
						? cb()
						: validateHashTreeBall(conn, objJoint, cb);
				},
				function(cb){
					profiler.stop('validation-hash-tree-ball');
					profiler.start();
					!objUnit.parent_units
						? cb()
						: validateParentsExistAndOrdered(conn, objUnit, cb);
				},
				function(cb){
					profiler.stop('validation-parents-exist');
					profiler.start();
					!objUnit.parent_units
						? cb()
						: validateHashTreeParentsAndSkiplist(conn, objJoint, cb);
				},
				function(cb){
					profiler.stop('validation-hash-tree-parents');
				//	profiler.start(); // conflicting with profiling in determineIfStableInLaterUnitsAndUpdateStableMcFlag
					!objUnit.parent_units
						? cb()
						: validateParents(conn, objJoint, objValidationState, cb);
				},
				function(cb){
				//	profiler.stop('validation-parents');
					profiler.start();
					!objJoint.skiplist_units
						? cb()
						: validateSkiplist(conn, objJoint.skiplist_units, cb);
				},
				function(cb){
					profiler.stop('validation-skiplist');
					validateWitnesses(conn, objUnit, objValidationState, cb);
				},
				function (cb) {
					validateAATrigger(conn, objUnit, objValidationState, cb);
				},
				function (cb) {
					validateTpsFee(conn, objJoint, objValidationState, cb);
				},
				function(cb){
					profiler.start();
					validateAuthors(conn, objUnit.authors, objUnit, objValidationState, cb);
				},
				function(cb){
					profiler.stop('validation-authors');
					profiler.start();
					objUnit.content_hash ? cb() : validateMessages(conn, objUnit.messages, objUnit, objValidationState, cb);
				}
			], 
			function(err){
				if(err){
					if (profiler.isStarted())
						profiler.stop('validation-advanced-stability');
					// We might have advanced the stability point and have to commit the changes as the caches are already updated.
					// There are no other updates/inserts/deletes during validation
					commit_fn(function(){
						var consumed_time = Date.now()-start_time;
						profiler.add_result('failed validation', consumed_time);
						console.log(objUnit.unit+" validation "+JSON.stringify(err)+" took "+consumed_time+"ms");
						if (!external_conn)
							conn.release();
						unlock();
						if (typeof err === "object"){
							if (err.error_code === "unresolved_dependency")
								callbacks.ifNeedParentUnits(err.arrMissingUnits, err.bRequestPrunedContent);
							else if (err.error_code === "need_hash_tree") // need to download hash tree to catch up
								callbacks.ifNeedHashTree();
							else if (err.error_code === "invalid_joint") // ball found in hash tree but with another unit
								callbacks.ifJointError(err.message);
							else if (err.error_code === "transient")
								callbacks.ifTransientError(err.message);
							else
								throw Error("unknown error code");
						}
						else
							callbacks.ifUnitError(err);
					});
				}
				else{
					profiler.stop('validation-messages');
					profiler.start();
					commit_fn(function(){
						var consumed_time = Date.now()-start_time;
						profiler.add_result('validation', consumed_time);
						console.log(objUnit.unit+" validation ok took "+consumed_time+"ms");
						if (!external_conn)
							conn.release();
						profiler.stop('validation-commit');
						if (objJoint.unsigned){
							unlock();
							callbacks.ifOkUnsigned(objValidationState.sequence === 'good');
						}
						else
							callbacks.ifOk(objValidationState, unlock);
					});
				}
			}
		); // async.series
		
	});
	
}



//  ----------------    


function checkDuplicate(conn, objUnit, cb){
	var unit = objUnit.unit;
	conn.query("SELECT sequence, main_chain_index FROM units WHERE unit=?", [unit], function (rows) {
		if (rows.length === 0) 
			return cb();
		var row = rows[0];
	//	if (row.sequence === 'final-bad' && row.main_chain_index !== null && row.main_chain_index < storage.getMinRetrievableMci() && objUnit.messages && !objUnit.content_hash) // already stripped locally but received a full version
	//		return cb();
		cb(createTransientError("unit "+unit+" already exists"));
	});
}

function validateHashTreeBall(conn, objJoint, callback){
	if (!objJoint.ball)
		return callback();
	var objUnit = objJoint.unit;
	var unit_by_hash_tree_ball = storage.assocHashTreeUnitsByBall[objJoint.ball];
//	conn.query("SELECT unit FROM hash_tree_balls WHERE ball=?", [objJoint.ball], function(rows){
		if (!unit_by_hash_tree_ball) 
			return callback({error_code: "need_hash_tree", message: "ball "+objJoint.ball+" is not known in hash tree"});
		if (unit_by_hash_tree_ball !== objUnit.unit)
			return callback(createJointError("ball "+objJoint.ball+" unit "+objUnit.unit+" contradicts hash tree"));
		callback();
//	});
}

function validateHashTreeParentsAndSkiplist(conn, objJoint, callback){
	if (!objJoint.ball)
		return callback();
	var objUnit = objJoint.unit;
	
	function validateBallHash(arrParentBalls, arrSkiplistBalls){
		var hash = objectHash.getBallHash(objUnit.unit, arrParentBalls, arrSkiplistBalls, !!objUnit.content_hash);
		if (hash !== objJoint.ball)
			return callback(createJointError("ball hash is wrong"));
		callback();
	}
	
	function readBallsByUnits(arrUnits, handleList){
		conn.query("SELECT ball FROM balls WHERE unit IN(?) ORDER BY ball", [arrUnits], function(rows){
			var arrBalls = rows.map(function(row){ return row.ball; });
			if (arrBalls.length === arrUnits.length)
				return handleList(arrBalls);
			// we have to check in hash_tree_balls too because if we were synced, went offline, and now starting to catch up, our parents will have no ball yet
			for (var ball in storage.assocHashTreeUnitsByBall){
				var unit = storage.assocHashTreeUnitsByBall[ball];
				if (arrUnits.indexOf(unit) >= 0 && arrBalls.indexOf(ball) === -1)
					arrBalls.push(ball);
			}
			arrBalls.sort();
			handleList(arrBalls);
		});
	}
	
	readBallsByUnits(objUnit.parent_units, function(arrParentBalls){
		if (arrParentBalls.length !== objUnit.parent_units.length)
			return callback(createJointError("some parents not found in balls nor in hash tree")); // while the child is found in hash tree
		if (!objJoint.skiplist_units)
			return validateBallHash(arrParentBalls, []);
		readBallsByUnits(objJoint.skiplist_units, function(arrSkiplistBalls){
			if (arrSkiplistBalls.length !== objJoint.skiplist_units.length)
				return callback(createJointError("some skiplist balls not found"));
			validateBallHash(arrParentBalls, arrSkiplistBalls);
		});
	});
}

// we cannot verify that skiplist units lie on MC if they are unstable yet, 
// but if they don't, we'll get unmatching ball hash when the current unit reaches stability
function validateSkiplist(conn, arrSkiplistUnits, callback){
	if (!arrSkiplistUnits.every(isNonemptyString))
		return callback(createJointError("skiplist units must be non-empty strings"));
	var prev = "";
	async.eachSeries(
		arrSkiplistUnits,
		function(skiplist_unit, cb){
			//if (skiplist_unit.charAt(0) !== "0")
			//    return cb("skiplist unit doesn't start with 0");
			if (skiplist_unit <= prev)
				return cb(createJointError("skiplist units not ordered"));
			conn.query("SELECT unit, is_stable, is_on_main_chain, main_chain_index FROM units WHERE unit=?", [skiplist_unit], function(rows){
				if (rows.length === 0)
					return cb(createJointError("skiplist unit "+skiplist_unit+" not found"));
				var objSkiplistUnitProps = rows[0];
				// if not stable, can't check that it is on MC as MC is not stable in its area yet
				if (objSkiplistUnitProps.is_stable === 1){
					if (objSkiplistUnitProps.is_on_main_chain !== 1)
						return cb(createJointError("skiplist unit "+skiplist_unit+" is not on MC"));
					if (objSkiplistUnitProps.main_chain_index % 10 !== 0)
						return cb(createJointError("skiplist unit "+skiplist_unit+" MCI is not divisible by 10"));
				}
				// we can't verify the choice of skiplist unit.
				// If we try to find a skiplist unit now, we might find something matching on unstable part of MC.
				// Again, we have another check when we reach stability
				cb();
			});
		},
		callback
	);
}

function validateParentsExistAndOrdered(conn, objUnit, callback){
	var prev = "";
	var arrMissingParentUnits = [];
	if (objUnit.parent_units.length > constants.MAX_PARENTS_PER_UNIT) // anti-spam
		return callback("too many parents: "+objUnit.parent_units.length);
	async.eachSeries(
		objUnit.parent_units,
		function(parent_unit, cb){
			if (parent_unit <= prev)
				return cb("parent units not ordered");
			prev = parent_unit;
			if (storage.assocUnstableUnits[parent_unit] || storage.assocStableUnits[parent_unit])
				return cb();
			storage.readStaticUnitProps(conn, parent_unit, function(objUnitProps){
				if (!objUnitProps)
					arrMissingParentUnits.push(parent_unit);
				cb();
			}, true);
		},
		function(err){
			if (err)
				return callback(err);
			if (arrMissingParentUnits.length > 0){
				conn.query("SELECT error FROM known_bad_joints WHERE unit IN(?) AND creation_date>=?", [arrMissingParentUnits, BOOT_DATE], function(rows){
					(rows.length > 0)
						? callback("some of the unit's parents are known bad: "+rows[0].error)
						: callback({error_code: "unresolved_dependency", arrMissingUnits: arrMissingParentUnits});
				});
				return;
			}
			callback();
		}
	);
}

function validateParents(conn, objJoint, objValidationState, callback){
	
	// avoid merging the obvious nonserials
	function checkNoSameAddressInDifferentParents(){
		if (objUnit.parent_units.length === 1)
			return callback();
		var assocAuthors = {};
		var found_address;
		async.eachSeries(
			objUnit.parent_units,
			function(parent_unit, cb){
				storage.readUnitAuthors(conn, parent_unit, function(arrAuthors){
					arrAuthors.forEach(function(address){
						if (assocAuthors[address])
							found_address = address;
						assocAuthors[address] = true;
					});
					cb(found_address);
				});
			},
			function(){
				if (found_address)
					return callback("some addresses found more than once in parents, e.g. "+found_address);
				return callback();
			}
		);
	}
	
	function readMaxParentLastBallMci(handleResult){
		storage.readMaxLastBallMci(conn, objUnit.parent_units, function(max_parent_last_ball_mci) {
			if (max_parent_last_ball_mci > objValidationState.last_ball_mci)
				return callback("last ball mci must not retreat, parents: "+objUnit.parent_units.join(', '));
			handleResult(max_parent_last_ball_mci);
		});
	}
	
	var objUnit = objJoint.unit;
	if (objValidationState.bAA && objUnit.parent_units.length > 2)
		throw Error("AA unit with more than 2 parents");
	// obsolete: when handling a ball, we can't trust parent list before we verify ball hash
	// obsolete: when handling a fresh unit, we can begin trusting parent list earlier, after we verify parents_hash
	// after this point, we can trust parent list as it either agrees with parents_hash or agrees with hash tree
	// hence, there are no more joint errors, except unordered parents or skiplist units
	var last_ball = objUnit.last_ball;
	var last_ball_unit = objUnit.last_ball_unit;
	var arrPrevParentUnitProps = [];
	objValidationState.max_parent_limci = 0;
	objValidationState.max_parent_wl = 0;
	let bHaveNonAAParent = false;
	async.eachSeries(
		objUnit.parent_units, 
		function(parent_unit, cb){
			storage.readUnitProps(conn, parent_unit, function(objParentUnitProps){
				if (objUnit.version !== constants.versionWithoutTimestamp && objUnit.timestamp < objParentUnitProps.timestamp)
					return cb("timestamp decreased from parent " + parent_unit);
				if (!objParentUnitProps.bAA)
					bHaveNonAAParent = true;
				if (objParentUnitProps.latest_included_mc_index > objValidationState.max_parent_limci)
					objValidationState.max_parent_limci = objParentUnitProps.latest_included_mc_index;
				if (objParentUnitProps.witnessed_level > objValidationState.max_parent_wl)
					objValidationState.max_parent_wl = objParentUnitProps.witnessed_level;
				async.eachSeries(
					arrPrevParentUnitProps, 
					function(objPrevParentUnitProps, cb2){
						graph.compareUnitsByProps(conn, objPrevParentUnitProps, objParentUnitProps, function(result){
							(result === null) ? cb2() : cb2("parent unit "+parent_unit+" is related to one of the other parent units");
						});
					},
					function(err){
						if (err)
							return cb(err);
						arrPrevParentUnitProps.push(objParentUnitProps);
						cb();
					}
				);
			});
		}, 
		function(err){
			if (err)
				return callback(err);
			conn.query(
				"SELECT is_stable, is_on_main_chain, main_chain_index, ball, timestamp, (SELECT MAX(main_chain_index) FROM units) AS max_known_mci \n\
				FROM units LEFT JOIN balls USING(unit) WHERE unit=?", 
				[last_ball_unit], 
				function(rows){
					if (rows.length !== 1) // at the same time, direct parents already received
						return callback("last ball unit "+last_ball_unit+" not found");
					var objLastBallUnitProps = rows[0];
					// it can be unstable and have a received (not self-derived) ball
					//if (objLastBallUnitProps.ball !== null && objLastBallUnitProps.is_stable === 0)
					//    throw "last ball "+last_ball+" is unstable";
					if (objLastBallUnitProps.ball === null && objLastBallUnitProps.is_stable === 1)
						throw Error("last ball unit "+last_ball_unit+" is stable but has no ball");
					if (objLastBallUnitProps.is_on_main_chain !== 1)
						return callback("last ball "+last_ball+" is not on MC");
					if (objLastBallUnitProps.ball && objLastBallUnitProps.ball !== last_ball)
						return callback("last_ball "+last_ball+" and last_ball_unit "+last_ball_unit+" do not match");
					objValidationState.last_ball_mci = objLastBallUnitProps.main_chain_index;
					objValidationState.last_ball_timestamp = objLastBallUnitProps.timestamp;
					objValidationState.max_known_mci = objLastBallUnitProps.max_known_mci;
					if (objValidationState.max_parent_limci < objValidationState.last_ball_mci)
						return callback("last ball unit "+last_ball_unit+" is not included in parents, unit "+objUnit.unit);
					if (!objValidationState.bAA && !bHaveNonAAParent && objValidationState.last_ball_mci >= constants.v4UpgradeMci)
						return callback("non-AA unit should have at least one non-AA parent");
					if (objUnit.headers_commission + objUnit.payload_commission > constants.MAX_UNIT_LENGTH && objValidationState.bAA && objValidationState.last_ball_mci < constants.pemCurvesFixMci)
						return callback("unit too large");
					
					var bRequiresTimestamp = (objValidationState.last_ball_mci >= constants.timestampUpgradeMci);
					if (bRequiresTimestamp && objUnit.version === constants.versionWithoutTimestamp)
						return callback("should be higher version at this mci");
					if (!bRequiresTimestamp && objUnit.version !== constants.versionWithoutTimestamp)
						return callback("should be version " + constants.versionWithoutTimestamp + " at this mci");
					
					var bWithKeys = (objValidationState.last_ball_mci >= constants.includeKeySizesUpgradeMci);
					var bWithKeysVersion = (objUnit.version !== constants.versionWithoutTimestamp && objUnit.version !== constants.versionWithoutKeySizes);
					if (bWithKeys !== bWithKeysVersion)
						return callback("wrong version, with keys mci = " + bWithKeys + ", with keys version = " + bWithKeysVersion);

					const bCommonOpList = (objValidationState.last_ball_mci >= constants.v4UpgradeMci);
					const fVersion = parseFloat(objUnit.version);
					if (fVersion >= constants.fVersion4) {
						if (!bCommonOpList)
							return callback("version 4.0+ should be used only since mci " + constants.v4UpgradeMci);
						if ("witnesses" in objUnit)
							return callback("should have no per-unit witnesses since version 4.0");
						if ("witness_list_unit" in objUnit)
							return callback("should have no witness_list_unit since version 4.0");
						if (!("content_hash" in objUnit)) { // serial unit
							const oversize_fee = storage.getOversizeFee(objUnit, objValidationState.last_ball_mci, objValidationState.bAA);
							if (oversize_fee) {
								if (objUnit.oversize_fee !== oversize_fee)
									return callback(createJointError(`oversize_fee mismatch: expected ${oversize_fee}, found ${objUnit.oversize_fee}`));
							}
							else {
								if ("oversize_fee" in objUnit)
									return callback(createJointError("zero oversize fee should be omitted"));
							}
							if (!("tps_fee" in objUnit) && !objValidationState.bAA)
								return callback("no tps_fee field");
						}
					}
					else { // < 4.0
						if (bCommonOpList)
							return callback("version 4.0 should be used since mci " + constants.v4UpgradeMci);
						if (!("witnesses" in objUnit) && !("witness_list_unit" in objUnit))
							return callback("should have either witnesses or witness_list_unit");
						if ("oversize_fee" in objUnit)
							return callback(createJointError("oversize_fee not charged before version 4.0"));
						if ("tps_fee" in objUnit)
							return callback("tps_fee not charged before version 4.0");
						if ("burn_fee" in objUnit)
							return callback("burn_fee not paid before version 4.0");
					}
					
					readMaxParentLastBallMci(function(max_parent_last_ball_mci){
						if (objLastBallUnitProps.is_stable === 1){
							// if it were not stable, we wouldn't have had the ball at all
							if (objLastBallUnitProps.ball !== last_ball)
								return callback("stable: last_ball "+last_ball+" and last_ball_unit "+last_ball_unit+" do not match");
							if (objValidationState.last_ball_mci <= constants.lastBallStableInParentsUpgradeMci || max_parent_last_ball_mci === objValidationState.last_ball_mci)
								return checkNoSameAddressInDifferentParents();
						}
						// Last ball is not stable yet in our view. Check if it is stable in view of the parents
						main_chain.determineIfStableInLaterUnitsAndUpdateStableMcFlag(conn, last_ball_unit, objUnit.parent_units, objLastBallUnitProps.is_stable, function(bStable, bAdvancedLastStableMci){
							/*if (!bStable && objLastBallUnitProps.is_stable === 1){
								var eventBus = require('./event_bus.js');
								eventBus.emit('nonfatal_error', "last ball is stable, but not stable in parents, unit "+objUnit.unit, new Error());
								return checkNoSameAddressInDifferentParents();
							}
							else */if (!bStable)
								return callback(objUnit.unit+": last ball unit "+last_ball_unit+" is not stable in view of your parents "+objUnit.parent_units);
							if (bAdvancedLastStableMci)
								return callback(createTransientError("last ball just advanced, try again"));
							if (!bAdvancedLastStableMci)
								return checkNoSameAddressInDifferentParents();
							conn.query("SELECT ball FROM balls WHERE unit=?", [last_ball_unit], function(ball_rows){
								if (ball_rows.length === 0)
									throw Error("last ball unit "+last_ball_unit+" just became stable but ball not found");
								if (ball_rows[0].ball !== last_ball)
									return callback("last_ball "+last_ball+" and last_ball_unit "+last_ball_unit
													+" do not match after advancing stability point");
								if (bAdvancedLastStableMci)
									objValidationState.bAdvancedLastStableMci = true; // not used
								checkNoSameAddressInDifferentParents();
							});
						});
					});
				}
			);
		}
	);
}


	
function checkNoReferencesInWitnessAddressDefinitions(conn, objValidationState, arrWitnesses, cb){
	profiler.start();
	var cross = (conf.storage === 'sqlite') ? 'CROSS' : ''; // correct the query planner
	conn.query(
		"SELECT 1 \n\
		FROM address_definition_changes \n\
		JOIN definitions USING(definition_chash) \n\
		JOIN units AS change_units USING(unit)   -- units where the change was declared \n\
		JOIN unit_authors USING(definition_chash) \n\
		JOIN units AS definition_units ON unit_authors.unit=definition_units.unit   -- units where the definition was disclosed \n\
		WHERE address_definition_changes.address IN(?) AND has_references=1 \n\
			AND change_units.is_stable=1 AND change_units.main_chain_index<=? AND +change_units.sequence='good' \n\
			AND definition_units.is_stable=1 AND definition_units.main_chain_index<=? AND +definition_units.sequence='good' \n\
		UNION \n\
		SELECT 1 \n\
		FROM definitions \n\
		"+cross+" JOIN unit_authors USING(definition_chash) \n\
		JOIN units AS definition_units ON unit_authors.unit=definition_units.unit   -- units where the definition was disclosed \n\
		WHERE definition_chash IN(?) AND has_references=1 \n\
			AND definition_units.is_stable=1 AND definition_units.main_chain_index<=? AND +definition_units.sequence='good' \n\
		LIMIT 1",
		[arrWitnesses, objValidationState.last_ball_mci, objValidationState.last_ball_mci, arrWitnesses, objValidationState.last_ball_mci],
		function(rows){
			profiler.stop('validation-witnesses-no-refs');
			(rows.length > 0) ? cb("some witnesses have references in their addresses") : cb();
		}
	);
}

function checkWitnessesKnownAndGood(conn, objValidationState, arrWitnesses, cb) {
	if (objValidationState.bGenesis)
		return cb();
	profiler.start();
	// check that all witnesses are already known and their units are good and stable
	conn.query(
		// address=definition_chash is true in the first appearence of the address
		// (not just in first appearence: it can return to its initial definition_chash sometime later)
		"SELECT COUNT(DISTINCT address) AS count_stable_good_witnesses \n\
		FROM unit_authors " + db.forceIndex(conf.storage === 'sqlite' ? 'byDefinitionChash' : 'unitAuthorsIndexByAddressDefinitionChash') + " \n\
		CROSS JOIN units USING(unit) \n\
		WHERE address=definition_chash AND +sequence='good' AND is_stable=1 AND main_chain_index<=? AND definition_chash IN(?)",
		[objValidationState.last_ball_mci, arrWitnesses],
		function(rows){
			profiler.stop('validation-witnesses-stable');
			if (rows[0].count_stable_good_witnesses !== constants.COUNT_WITNESSES)
				return cb("some witnesses are not stable, not serial, or don't come before last ball");
			cb();
		}
	);
}


function validateWitnesses(conn, objUnit, objValidationState, callback){

	function validateWitnessListMutations(arrWitnesses){
		if (!objUnit.parent_units) // genesis
			return callback();
		storage.determineIfHasWitnessListMutationsAlongMc(conn, objUnit, last_ball_unit, arrWitnesses, function(err){
			if (err && objValidationState.last_ball_mci >= 512000) // do not enforce before the || bug was fixed
				return callback(err);
			checkNoReferencesInWitnessAddressDefinitions(conn, objValidationState, arrWitnesses, err => {
				if (err)
					return callback(err);
				checkWitnessedLevelDidNotRetreat(arrWitnesses);
			});
		});
	}

	function checkWitnessedLevelDidNotRetreat(arrWitnesses){
		if (!objUnit.parent_units) // genesis
			return callback();
		storage.determineWitnessedLevelAndBestParent(conn, objUnit.parent_units, arrWitnesses, objUnit.version, objValidationState.last_ball_mci >= constants.bestParentPrefersOpUpgradeMci, function(witnessed_level, best_parent_unit){
			if (!best_parent_unit)
				return callback("no best parent");
			objValidationState.witnessed_level = witnessed_level;
			objValidationState.best_parent_unit = best_parent_unit;
			if (objValidationState.last_ball_mci < constants.witnessedLevelMustNotRetreatUpgradeMci) // not enforced
				return callback();
			if (typeof objValidationState.max_parent_wl === 'undefined')
				throw Error('no max_parent_wl');
			if (objValidationState.last_ball_mci >= constants.witnessedLevelMustNotRetreatFromAllParentsUpgradeMci)
				return (witnessed_level >= objValidationState.max_parent_wl) ? callback() : callback("witnessed level retreats from parent's "+objValidationState.max_parent_wl+" to "+witnessed_level);
			storage.readStaticUnitProps(conn, best_parent_unit, function(props){
				(witnessed_level >= props.witnessed_level) 
					? callback() 
					: callback("witnessed level retreats from "+props.witnessed_level+" to "+witnessed_level);
			});
		});
	}
	
	if (objValidationState.last_ball_mci >= constants.v4UpgradeMci)
		return checkWitnessedLevelDidNotRetreat(storage.getOpList(objValidationState.last_ball_mci));

	var last_ball_unit = objUnit.last_ball_unit;
	if (typeof objUnit.witness_list_unit === "string"){
		profiler.start();
		storage.readWitnessList(conn, objUnit.witness_list_unit, function(arrWitnesses){
			if (arrWitnesses.length === 0){
				profiler.stop('validation-witnesses-read-list');
				return callback("referenced witness list unit "+objUnit.witness_list_unit+" has no witnesses");
			}
			if (typeof assocWitnessListMci[objUnit.witness_list_unit] === 'number' && assocWitnessListMci[objUnit.witness_list_unit] <= objValidationState.last_ball_mci){
				profiler.stop('validation-witnesses-read-list');
				return validateWitnessListMutations(arrWitnesses);
			}
			conn.query("SELECT sequence, is_stable, main_chain_index FROM units WHERE unit=?", [objUnit.witness_list_unit], function(unit_rows){
				profiler.stop('validation-witnesses-read-list');
				if (unit_rows.length === 0)
					return callback("witness list unit "+objUnit.witness_list_unit+" not found");
				var objWitnessListUnitProps = unit_rows[0];
				if (objWitnessListUnitProps.sequence !== 'good')
					return callback("witness list unit "+objUnit.witness_list_unit+" is not serial");
				if (objWitnessListUnitProps.is_stable !== 1)
					return callback("witness list unit "+objUnit.witness_list_unit+" is not stable");
				if (objWitnessListUnitProps.main_chain_index > objValidationState.last_ball_mci)
					return callback("witness list unit "+objUnit.witness_list_unit+" must come before last ball");
				assocWitnessListMci[objUnit.witness_list_unit] = objWitnessListUnitProps.main_chain_index;
				validateWitnessListMutations(arrWitnesses);
			});
		}, true);
	}
	else if (Array.isArray(objUnit.witnesses) && objUnit.witnesses.length === constants.COUNT_WITNESSES){
		var prev_witness = objUnit.witnesses[0];
		for (var i=0; i<objUnit.witnesses.length; i++){
			var curr_witness = objUnit.witnesses[i];
			if (!isNonemptyString(curr_witness))
				return callback("witness address must be a string");
			if (!isValidAddress(curr_witness)) // the isNonemptyString before is to make string concatenation safe in the next line (curr_witness={toString: 42})
				return callback("witness address "+curr_witness+" is invalid");
			if (i === 0)
				continue;
			if (curr_witness <= prev_witness)
				return callback("wrong order of witnesses, or duplicates");
			prev_witness = curr_witness;
		}
		if (storage.isGenesisUnit(objUnit.unit)){
			// addresses might not be known yet, it's ok
			validateWitnessListMutations(objUnit.witnesses);
			return;
		}
		checkWitnessesKnownAndGood(conn, objValidationState, objUnit.witnesses, err => {
			if (err)
				return callback(err);
			validateWitnessListMutations(objUnit.witnesses);
		});
	}
	else
		return callback("no witnesses or not enough witnesses");
}


async function validateAATrigger(conn, objUnit, objValidationState, callback) {
	if (objValidationState.last_ball_mci < constants.v4UpgradeMci || objValidationState.bAA || !objValidationState.last_ball_mci) {
		if ("max_aa_responses" in objUnit)
			return callback(`max_aa_responses should not be there`);
		if (objValidationState.bAA || !objValidationState.last_ball_mci)
			return callback();
	}
	if ("content_hash" in objUnit) { // messages already stripped off
		objValidationState.count_primary_aa_triggers = 0;
		return callback();
	}
	if (objUnit.max_aa_responses === 0 && (objValidationState.last_ball_mci >= constants.pemCurvesFixMci || !objValidationState.hasBall && storage.getMinRetrievableMci() >= constants.pemCurvesFixMci))
		return callback(`max_aa_responses=0 is not allowed`);
	let outputCounts = {};
	for (let m of objUnit.messages) {
		if (m.app === 'payment' && m.payload) {
			const asset = m.payload.asset || 'base';
			for (let o of m.payload.outputs) {
				if (!outputCounts[o.address])
					outputCounts[o.address] = {};
				if (!outputCounts[o.address][asset])
					outputCounts[o.address][asset] = 0;
				outputCounts[o.address][asset]++;
			}
		}
	}
	const arrOutputAddresses = Object.keys(outputCounts);
	if (arrOutputAddresses.length === 0)
		return callback("no output addresses found in payment messages");

	// Look for AA triggers
	// There might be actually more triggers due to AAs defined between last_ball_mci and our unit, so our validation of tps fee might require a smaller fee than the fee actually charged when the trigger executes
	const rows = await conn.query("SELECT address FROM aa_addresses WHERE address IN (?) AND mci<=?", [arrOutputAddresses, objValidationState.last_ball_mci]);
	if (rows.length === 0) {
		if ("max_aa_responses" in objUnit)
			return callback(`no outputs to AAs, max_aa_responses should not be there`);
		return callback();
	}
	objValidationState.count_primary_aa_triggers = rows.length;
	if (objValidationState.count_primary_aa_triggers > 1) {
		if (objValidationState.last_ball_mci >= constants.pemCurvesFixMci)
			return callback(`more than 1 primary AA trigger (${objValidationState.count_primary_aa_triggers})`);
		if (storage.getMinRetrievableMci() > constants.pemCurvesFixMci)
			return callback(createTransientError(`more than 1 primary AA trigger (${objValidationState.count_primary_aa_triggers})`));
	}
	if ((objValidationState.last_ball_mci >= constants.pemCurvesFixMci || !objValidationState.hasBall && storage.getMinRetrievableMci() >= constants.pemCurvesFixMci) && objValidationState.count_primary_aa_triggers === 1) {
		const address = rows[0].address;
		for (let asset in outputCounts[address]) {
			if (outputCounts[address][asset] > 1)
				return callback(`more than 1 output to the same AA ${address} for asset ${asset}`);
		}
	}
	callback();
}


function isFromOP(author_addresses, mci) {
	const ops = storage.getOpList(mci);
	for (let a of author_addresses)
		if (ops.includes(a))
			return true;
	return false;
}

async function validateTpsFee(conn, objJoint, objValidationState, callback) {
	if (objValidationState.last_ball_mci < constants.v4UpgradeMci || !objValidationState.last_ball_mci)
		return callback();
	const objUnit = objJoint.unit;
	if (objValidationState.bAA) {
		if ("tps_fee" in objUnit)
			return callback("tps_fee in AA response");
		return callback();
	}
	if ("content_hash" in objUnit) // tps_fee and other unit fields have been already stripped
		return callback();
	const objUnitProps = {
		unit: objUnit.unit,
		parent_units: objUnit.parent_units,
		best_parent_unit: objValidationState.best_parent_unit,
		last_ball_unit: objUnit.last_ball_unit,
		timestamp: objUnit.timestamp,
		count_primary_aa_triggers: objValidationState.count_primary_aa_triggers,
		max_aa_responses: objUnit.max_aa_responses,
	};
	const count_units = storage.getCountUnitsPayingTpsFee(objUnitProps);
	const min_tps_fee = await storage.getLocalTpsFee(conn, objUnitProps, count_units);
	console.log('validation', {min_tps_fee}, objUnitProps)
	
	// compare against the current tps fee or soft-reject
	const current_tps_fee = objJoint.ball ? 0 : storage.getCurrentTpsFee(0, count_units); // very low while catching up
	const min_acceptable_tps_fee_multiplier = objJoint.ball ? 0 : storage.getMinAcceptableTpsFeeMultiplier();
	const min_acceptable_tps_fee = current_tps_fee * min_acceptable_tps_fee_multiplier * count_units;

	const author_addresses = objUnit.authors.map(a => a.address);
	const bFromOP = isFromOP(author_addresses, objValidationState.last_ball_mci);
	const recipients = storage.getTpsFeeRecipients(objValidationState.last_ball_mci < constants.tpsFeeRecipientsFixMci ? objUnit.earned_headers_commission_recipients : storage.ehcr2assoc(objUnit.earned_headers_commission_recipients), author_addresses);
	for (let address in recipients) {
		const share = recipients[address] / 100;
		if (!share)
			throw Error(`invalid share for address ${address}: ${share}`);
		const [row] = await conn.query("SELECT tps_fees_balance FROM tps_fees_balances WHERE address=? AND mci<=? ORDER BY mci DESC LIMIT 1", [address, objValidationState.last_ball_mci]);
		const tps_fees_balance = row ? row.tps_fees_balance : 0;
		if (tps_fees_balance + objUnit.tps_fee * share < min_tps_fee * share)
			return callback(createTransientError(`tps_fee ${objUnit.tps_fee} + tps fees balance ${tps_fees_balance} less than required ${min_tps_fee} for address ${address} whose share is ${share}`));
		const tps_fee = tps_fees_balance / share + objUnit.tps_fee;
		if (tps_fee < min_acceptable_tps_fee) {
			if (!bFromOP)
				return callback(createTransientError(`tps fee on address ${address} must be at least ${min_acceptable_tps_fee}, found ${tps_fee}`));
			console.log(`unit from OP, hence accepting despite low tps fee on address ${address} which must be at least ${min_acceptable_tps_fee} but found ${tps_fee}`);
		}
	}
	callback();
}


function validateHeadersCommissionRecipients(objUnit, cb){
	if (objUnit.authors.length > 1 && typeof objUnit.earned_headers_commission_recipients !== "object")
		return cb("must specify earned_headers_commission_recipients when more than 1 author");
	if ("earned_headers_commission_recipients" in objUnit){
		if (!isNonemptyArray(objUnit.earned_headers_commission_recipients))
			return cb("empty earned_headers_commission_recipients array");
		var total_earned_headers_commission_share = 0;
		var prev_address = "";
		for (var i=0; i<objUnit.earned_headers_commission_recipients.length; i++){
			var recipient = objUnit.earned_headers_commission_recipients[i];
			if (!isPositiveInteger(recipient.earned_headers_commission_share))
				return cb("earned_headers_commission_share must be positive integer");
			if (hasFieldsExcept(recipient, ["address", "earned_headers_commission_share"]))
				return cb("unknown fields in recipient");
			if (!isValidAddress(recipient.address))
				return cb("invalid recipient address checksum");
			if (recipient.address <= prev_address)
				return cb("recipient list must be sorted by address");
			total_earned_headers_commission_share += recipient.earned_headers_commission_share;
			prev_address = recipient.address;
		}
		if (total_earned_headers_commission_share !== 100)
			return cb("sum of earned_headers_commission_share is not 100");
	}
	cb();
}

function validateAuthors(conn, arrAuthors, objUnit, objValidationState, callback) {
	if (objValidationState.bAA && arrAuthors.length !== 1)
		throw Error("AA unit with multiple authors");
	if (arrAuthors.length > constants.MAX_AUTHORS_PER_UNIT) // this is anti-spam. Otherwise an attacker would send nonserial balls signed by zillions of authors.
		return callback("too many authors");
	objValidationState.arrAddressesWithForkedPath = [];
	var prev_address = "";
	for (var i=0; i<arrAuthors.length; i++){
		var objAuthor = arrAuthors[i];
		if (objAuthor.address <= prev_address)
			return callback("author addresses not sorted");
		prev_address = objAuthor.address;
	}
	
	objValidationState.unit_hash_to_sign = objectHash.getUnitHashToSign(objUnit);
	
	async.eachSeries(arrAuthors, function(objAuthor, cb){
		validateAuthor(conn, objAuthor, objUnit, objValidationState, cb);
	}, callback);
}

function validateAuthor(conn, objAuthor, objUnit, objValidationState, callback){
	if (objValidationState.bAA && hasFieldsExcept(objAuthor, ["address"]))
		throw Error("unknown fields in AA author");
	if (!objValidationState.bAA) {
		if (hasFieldsExcept(objAuthor, ["address", "authentifiers", "definition"]))
			return callback("unknown fields in author");
		if (!isNonemptyObject(objAuthor.authentifiers) && !objUnit.content_hash)
			return callback("no authentifiers");
		for (var path in objAuthor.authentifiers) {
			if (!isNonemptyString(objAuthor.authentifiers[path]))
				return callback("authentifiers must be nonempty strings");
			if (objAuthor.authentifiers[path].length > constants.MAX_AUTHENTIFIER_LENGTH)
				return callback("authentifier too long");
		}
	}
	
	var bNonserial = false;
	var bInitialDefinition = false;

	if (objValidationState.bAA) {
		storage.readAADefinition(conn, objAuthor.address, objValidationState.aa_mci, function (arrDefinition) {
			if (!arrDefinition)
				throw Error("AA definition not found " + objAuthor.address);
			checkSerialAddressUse();
		});
		return;
	}
	
	var arrAddressDefinition = objAuthor.definition;
	if (isNonemptyArray(arrAddressDefinition)){
		if (arrAddressDefinition[0] === 'autonomous agent')
			return callback('AA cannot be defined in authors');
		// todo: check that the address is really new?
		validateAuthentifiers(arrAddressDefinition);
	}
	else if (!("definition" in objAuthor)){
		if (objUnit.content_hash){ // nothing else to check
			objValidationState.sequence = 'final-bad';
			return callback();
		}
		// we check signatures using the latest address definition before last ball
		storage.readDefinitionByAddress(conn, objAuthor.address, objValidationState.last_ball_mci, {
			ifDefinitionNotFound: function(definition_chash){
				storage.readAADefinition(conn, objAuthor.address, objValidationState.last_ball_mci, function (arrAADefinition) {
					if (arrAADefinition)
						return callback(createTransientError("will not validate unit signed by AA"));
					if (objValidationState.last_ball_mci >= constants.pemCurvesFixMci || !objValidationState.hasBall && storage.getMinRetrievableMci() >= constants.pemCurvesFixMci)
						return callback("definition " + definition_chash + " bound to address " + objAuthor.address + " not found before last ball");
					findUnstableInitialDefinition(definition_chash, function (arrDefinition) {
						if (!arrDefinition)
							return callback("definition " + definition_chash + " bound to address " + objAuthor.address + " is not defined");
						bInitialDefinition = true;
						validateAuthentifiers(arrDefinition);
					});
				});
			},
			ifFound: function(arrAddressDefinition){
				validateAuthentifiers(arrAddressDefinition);
			}
		});
	}
	else
		return callback("bad type of definition");
	
	function findUnstableInitialDefinition(definition_chash, handleUnstableInitialDefinition) {
		if (objValidationState.last_ball_mci < constants.unstableInitialDefinitionUpgradeMci || definition_chash !== objAuthor.address)
			return handleUnstableInitialDefinition(null);
		conn.query("SELECT definition, main_chain_index, unit \n\
			FROM definitions \n\
			CROSS JOIN unit_authors USING(definition_chash) \n\
			CROSS JOIN units USING(unit) \n\
			WHERE definition_chash=?",
			[definition_chash],
			function (rows) {
				if (rows.length === 0)
					return handleUnstableInitialDefinition(null);
				if (rows.some(function (row) { return row.main_chain_index !== null && row.main_chain_index <= objValidationState.last_ball_mci })) // some are stable, maybe we returned to the initial definition
					return handleUnstableInitialDefinition(null);
				async.eachSeries(
					rows,
					function (row, cb) {
						graph.determineIfIncludedOrEqual(conn, row.unit, objUnit.parent_units, function (bIncluded) {
							console.log("unstable definition of " + definition_chash + " found in " + row.unit + ", included? " + bIncluded);
							bIncluded ? cb(JSON.parse(row.definition)) : cb();
						});
					},
					function (arrDefinition) {
						handleUnstableInitialDefinition(arrDefinition);
					}
				)
			}
		);
	}
	
	function validateAuthentifiers(arrAddressDefinition){
		Definition.validateAuthentifiers(
			conn, objAuthor.address, null, arrAddressDefinition, objUnit, objValidationState, objAuthor.authentifiers, 
			function(err, res){
				if (err) // error in address definition
					return callback(err);
				if (!res) // wrong signature or the like
					return callback("authentifier verification failed");
				checkSerialAddressUse();
			}
		);
	}
	
	
	function findConflictingUnits(handleConflictingUnits){
	//	var cross = (objValidationState.max_known_mci - objValidationState.max_parent_limci < 1000) ? 'CROSS' : '';
		var indexMySQL = conf.storage == "mysql" ? "USE INDEX(unitAuthorsIndexByAddressMci)" : "";
		conn.query( // _left_ join forces use of indexes in units
		/*	"SELECT unit, is_stable \n\
			FROM units \n\
			"+cross+" JOIN unit_authors USING(unit) \n\
			WHERE address=? AND (main_chain_index>? OR main_chain_index IS NULL) AND unit != ?",
			[objAuthor.address, objValidationState.max_parent_limci, objUnit.unit],*/
			// final-bad units are permanently voided and never come back to 'good', so they are not real competitors:
			// exclude them here rather than only when deciding bConflictsWithStableUnits below
			"SELECT unit, is_stable, sequence, level \n\
			FROM unit_authors "+indexMySQL+"\n\
			CROSS JOIN units USING(unit)\n\
			WHERE address=? AND _mci>? AND unit != ? AND sequence!='final-bad' \n\
			UNION \n\
			SELECT unit, is_stable, sequence, level \n\
			FROM unit_authors "+indexMySQL+"\n\
			CROSS JOIN units USING(unit)\n\
			WHERE address=? AND _mci IS NULL AND unit != ? AND sequence!='final-bad' \n\
			ORDER BY level DESC",
			[objAuthor.address, objValidationState.max_parent_limci, objUnit.unit, objAuthor.address, objUnit.unit],
			function(rows){
				if (rows.length === 0)
					return handleConflictingUnits([]);
				var bAllSerial = rows.every(function(row){ return (row.sequence === 'good'); });
				var arrConflictingUnitProps = [];
				async.eachSeries(
					rows,
					function(row, cb){
						graph.determineIfIncludedOrEqual(conn, row.unit, objUnit.parent_units, function(bIncluded){
							if (!bIncluded)
								arrConflictingUnitProps.push(row);
							else if (bAllSerial)
								return cb('done'); // all are serial and this one is included, therefore the earlier ones are included too
							cb();
						});
					},
					function(){
						handleConflictingUnits(arrConflictingUnitProps);
					}
				);
			}
		);
	}


	function checkSerialAddressUse(){
		var next = (objValidationState.last_ball_mci >= constants.pemCurvesFixMci || !objValidationState.hasBall && storage.getMinRetrievableMci() >= constants.pemCurvesFixMci) ? validateDefinition : checkNoPendingChangeOfDefinitionChash;
		findConflictingUnits(function(arrConflictingUnitProps){
			if (arrConflictingUnitProps.length === 0){ // no conflicting units
				// we can have 2 authors. If the 1st author gave bad sequence but the 2nd is good then don't overwrite
				objValidationState.sequence = objValidationState.sequence || 'good';
				return next();
			}
			var arrConflictingUnits = arrConflictingUnitProps.map(function(objConflictingUnitProps){ return objConflictingUnitProps.unit; });
			breadcrumbs.add("========== found conflicting units "+arrConflictingUnits+" =========");
			breadcrumbs.add("========== will accept a conflicting unit "+objUnit.unit+" =========");
			objValidationState.arrAddressesWithForkedPath.push(objAuthor.address);
			objValidationState.arrConflictingUnits = (objValidationState.arrConflictingUnits || []).concat(arrConflictingUnits);
			bNonserial = true;
			var arrUnstableConflictingUnitProps = arrConflictingUnitProps.filter(function(objConflictingUnitProps){
				return (objConflictingUnitProps.is_stable === 0);
			});
			// findConflictingUnits() already excludes final-bad rows, so any stable row left here is a real, good competitor
			var bConflictsWithStableUnits = arrConflictingUnitProps.some(function(objConflictingUnitProps){
				return (objConflictingUnitProps.is_stable === 1);
			});
			if (objValidationState.sequence !== 'final-bad') // if it were already final-bad because of 1st author, it can't become temp-bad due to 2nd author
				objValidationState.sequence = bConflictsWithStableUnits ? 'final-bad' : 'temp-bad';
			var arrUnstableConflictingUnits = arrUnstableConflictingUnitProps.map(function(objConflictingUnitProps){ return objConflictingUnitProps.unit; });
			// if we are (or already became, due to another author) final-bad, we are not a living competitor for this address either,
			// so there is no need to punish other pending units - they'll correctly resolve to 'good' on their own once stable
			if (objValidationState.sequence === 'final-bad')
				return next();
			if (arrUnstableConflictingUnits.length === 0)
				return next();
			conn.query("SELECT unit FROM units WHERE unit IN(?) AND +sequence='good'",[arrUnstableConflictingUnits],function(rows){
				if (rows.length > 0)
					objValidationState.arrUnitsGettingBadSequence = (objValidationState.arrUnitsGettingBadSequence || []).concat(rows.map(function(row){return row.unit}));
				// we don't modify the db during validation, schedule the update for the write
				objValidationState.arrAdditionalQueries.push(
				{sql: "UPDATE units SET sequence='temp-bad' WHERE unit IN(?) AND +sequence='good'", params: [arrUnstableConflictingUnits]});
				next();
				});
		});
	}
	
	// don't allow contradicting pending keychanges.
	// We don't trust pending keychanges even when they are serial, as another unit may arrive and make them nonserial
	function checkNoPendingChangeOfDefinitionChash(){
		var next = checkNoPendingDefinition;
		//var filter = bNonserial ? "AND sequence='good'" : "";
		conn.query(
			"SELECT unit FROM address_definition_changes JOIN units USING(unit) \n\
			WHERE address=? AND (is_stable=0 OR main_chain_index>? OR main_chain_index IS NULL)", 
			[objAuthor.address, objValidationState.last_ball_mci], 
			function(rows){
				if (rows.length === 0)
					return next();
				if (!bNonserial || objValidationState.arrAddressesWithForkedPath.indexOf(objAuthor.address) === -1)
					return callback("you can't send anything before your last keychange is stable and before last ball");
				// from this point, our unit is nonserial
				async.eachSeries(
					rows,
					function(row, cb){
						graph.determineIfIncludedOrEqual(conn, row.unit, objUnit.parent_units, function(bIncluded){
							if (bIncluded)
								console.log("checkNoPendingChangeOfDefinitionChash: unit "+row.unit+" is included");
							bIncluded ? cb("found") : cb();
						});
					},
					function(err){
						(err === "found") 
							? callback("you can't send anything before your last included keychange is stable and before last ball (self is nonserial)") 
							: next();
					}
				);
			}
		);
	}
	
	// We don't trust pending definitions even when they are serial, as another unit may arrive and make them nonserial, 
	// then the definition will be removed
	function checkNoPendingDefinition(){
		//var next = checkNoPendingOrRetrievableNonserialIncluded;
		var next = validateDefinition;
		if (bInitialDefinition)
			return next();
		//var filter = bNonserial ? "AND sequence='good'" : "";
	//	var cross = (objValidationState.max_known_mci - objValidationState.last_ball_mci < 1000) ? 'CROSS' : '';
		conn.query( // _left_ join forces use of indexes in units
		//	"SELECT unit FROM units "+cross+" JOIN unit_authors USING(unit) \n\
		//	WHERE address=? AND definition_chash IS NOT NULL AND ( /* is_stable=0 OR */ main_chain_index>? OR main_chain_index IS NULL)", 
		//	[objAuthor.address, objValidationState.last_ball_mci], 
			"SELECT unit FROM unit_authors WHERE address=? AND definition_chash IS NOT NULL AND _mci>?  \n\
			UNION \n\
			SELECT unit FROM unit_authors WHERE address=? AND definition_chash IS NOT NULL AND _mci IS NULL", 
			[objAuthor.address, objValidationState.last_ball_mci, objAuthor.address], 
			function(rows){
				if (rows.length === 0)
					return next();
				if (!bNonserial || objValidationState.arrAddressesWithForkedPath.indexOf(objAuthor.address) === -1)
					return callback("you can't send anything before your last definition is stable and before last ball");
				// from this point, our unit is nonserial
				async.eachSeries(
					rows,
					function(row, cb){
						graph.determineIfIncludedOrEqual(conn, row.unit, objUnit.parent_units, function(bIncluded){
							if (bIncluded)
								console.log("checkNoPendingDefinition: unit "+row.unit+" is included");
							bIncluded ? cb("found") : cb();
						});
					},
					function(err){
						(err === "found") 
							? callback("you can't send anything before your last included definition is stable and before last ball (self is nonserial)") 
							: next();
					}
				);
			}
		);
	}
	
	// This was bad idea.  An uncovered nonserial, if not archived, will block new units from this address forever.
	/*
	function checkNoPendingOrRetrievableNonserialIncluded(){
		var next = validateDefinition;
		conn.query(
			"SELECT lb_units.main_chain_index FROM units JOIN units AS lb_units ON units.last_ball_unit=lb_units.unit \n\
			WHERE units.is_on_main_chain=1 AND units.main_chain_index=?",
			[objValidationState.last_ball_mci],
			function(lb_rows){
				var last_ball_of_last_ball_mci = (lb_rows.length > 0) ? lb_rows[0].main_chain_index : 0;
				conn.query(
					"SELECT unit FROM unit_authors JOIN units USING(unit) \n\
					WHERE address=? AND (is_stable=0 OR main_chain_index>?) AND sequence!='good'", 
					[objAuthor.address, last_ball_of_last_ball_mci], 
					function(rows){
						if (rows.length === 0)
							return next();
						if (!bNonserial)
							return callback("you can't send anything before all your nonserial units are stable and before last ball of last ball");
						// from this point, the unit is nonserial
						async.eachSeries(
							rows,
							function(row, cb){
								graph.determineIfIncludedOrEqual(conn, row.unit, objUnit.parent_units, function(bIncluded){
									if (bIncluded)
										console.log("checkNoPendingOrRetrievableNonserialIncluded: unit "+row.unit+" is included");
									bIncluded ? cb("found") : cb();
								});
							},
							function(err){
								(err === "found") 
									? callback("you can't send anything before all your included nonserial units are stable \
											   and lie before last ball of last ball (self is nonserial)") 
									: next();
							}
						);
					}
				);
			}
		);
	}
	*/
	
	function validateDefinition(){
		if (!("definition" in objAuthor))
			return callback();
		// the rest assumes that the definition is explicitly defined
		var arrAddressDefinition = objAuthor.definition;
		storage.readDefinitionByAddress(conn, objAuthor.address, objValidationState.last_ball_mci, {
			ifDefinitionNotFound: function(definition_chash){ // first use of the definition_chash (in particular, of the address, when definition_chash=address)
				try {
					if (objectHash.getChash160(arrAddressDefinition) !== definition_chash)
						return callback("wrong definition: " + objectHash.getChash160(arrAddressDefinition) + "!==" + definition_chash);
				}
				catch (e) {
					return callback("definition hash failed: " + e.toString());
				}
				callback();
			},
			ifFound: function(arrAddressDefinition2){ // arrAddressDefinition2 can be different
				handleDuplicateAddressDefinition(arrAddressDefinition2);
			}
		});
	}
	
	function handleDuplicateAddressDefinition(arrAddressDefinition){
	//	if (!bNonserial || objValidationState.arrAddressesWithForkedPath.indexOf(objAuthor.address) === -1)
			return callback("duplicate definition of address "+objAuthor.address+", bNonserial="+bNonserial);
		// todo: investigate if this can split the nodes
		// in one particular case, the attacker changes his definition then quickly sends a new ball with the old definition - the new definition will not be active yet
		try {
			if (objectHash.getChash160(arrAddressDefinition) !== objectHash.getChash160(objAuthor.definition))
				return callback("unit definition doesn't match the stored definition");
		}
		catch (e) {
			return callback("handleDuplicateAddressDefinition definition hash failed: " + e.toString());
		}
		callback(); // let it be for now. Eventually, at most one of the balls will be declared good
	}
	
}

function validateMessages(conn, arrMessages, objUnit, objValidationState, callback){
	console.log("validateMessages "+objUnit.unit);
	let arrSortedMessages = arrMessages.map((objMessage, message_index) => ({ objMessage, message_index })); // shallow copy preserving the original indexes

	arrSortedMessages.sort(({ objMessage: a }, { objMessage: b }) => {
		if (a.app === 'payment' && b.app !== 'payment') return -1;
		if (a.app !== 'payment' && b.app === 'payment') return 1;
		if (a.app === 'payment' && b.app === 'payment') {
			if (!a.payload) return 1;
			if (!b.payload) return -1;
			if (!a.payload.asset) return -1;
			if (!b.payload.asset) return 1;
		}
		return 0;
	});
	async.forEachOfSeries(
		arrSortedMessages, 
		function({ objMessage, message_index }, _sorted_idx, cb){
			validateMessage(conn, objMessage, message_index, objUnit, objValidationState, cb); 
		}, 
		function(err){
			if (err)
				return callback(err);
			if (!objValidationState.bHasBasePayment)
				return callback("no base payment message");
			callback();
		}
	);
}

function validateMessage(conn, objMessage, message_index, objUnit, objValidationState, callback) {
	if (typeof objMessage.app !== "string")
		return callback("no app");
	if (hasFieldsExcept(objMessage, ["app", "payload_hash", "payload_location", "payload", "payload_uri", "payload_uri_hash", "spend_proofs"]))
		return callback("unknown fields in message");
	
	if ("spend_proofs" in objMessage){
		if (objValidationState.bAA)
			return callback("spend proofs in AA");
		if (objMessage.app !== "payment")
			return callback("spend proofs in non-payment message");
		if (objMessage.payload_location !== "none")
			return callback("spend proofs in message with payload");
		if (!Array.isArray(objMessage.spend_proofs) || objMessage.spend_proofs.length === 0 || objMessage.spend_proofs.length > constants.MAX_SPEND_PROOFS_PER_MESSAGE)
			return callback("spend_proofs must be non-empty array max "+constants.MAX_SPEND_PROOFS_PER_MESSAGE+" elements");
		var arrAuthorAddresses = objUnit.authors.map(function(author) { return author.address; } );
		// spend proofs are sorted in the same order as their corresponding inputs
		//var prev_spend_proof = "";
		for (var i=0; i<objMessage.spend_proofs.length; i++){
			var objSpendProof = objMessage.spend_proofs[i];
			if (typeof objSpendProof !== "object")
				return callback("spend_proof must be object");
			if (hasFieldsExcept(objSpendProof, ["spend_proof", "address"]))
				return callback("unknown fields in spend_proof");
			//if (objSpendProof.spend_proof <= prev_spend_proof)
			//    return callback("spend_proofs not sorted");
			
			if (!isValidBase64(objSpendProof.spend_proof, constants.HASH_LENGTH))
				return callback("spend proof " + JSON.stringify(objSpendProof.spend_proof) + " is not a valid base64");
			
			var address = null;
			if (arrAuthorAddresses.length === 1){
				if ("address" in objSpendProof)
					return callback("when single-authored, must not put address in spend proof");
				address = arrAuthorAddresses[0];
			}
			else{
				if (typeof objSpendProof.address !== "string")
					return callback("when multi-authored, must put address in spend_proofs");
				if (arrAuthorAddresses.indexOf(objSpendProof.address) === -1)
					return callback("spend proof address "+objSpendProof.address+" is not an author");
				address = objSpendProof.address;
			}
			
			if (objValidationState.arrInputKeys.indexOf(objSpendProof.spend_proof) >= 0)
				return callback("spend proof "+objSpendProof.spend_proof+" already used");
			objValidationState.arrInputKeys.push(objSpendProof.spend_proof);
			
			//prev_spend_proof = objSpendProof.spend_proof;
		}
		if (objMessage.payload_location === "inline")
			return callback("you don't need spend proofs when you have inline payload");
	}

	if (objMessage.payload_location === "none" && ("payload" in objMessage || "payload_uri" in objMessage || "payload_uri_hash" in objMessage))
		return callback(createJointError("must be no payload"));

	if (objMessage.payload_location === "uri"){
		if ("payload" in objMessage)
			return callback(createJointError("must not contain payload"));
		if (typeof objMessage.payload_uri !== "string")
			return callback(createJointError("no payload uri"));
		if (objMessage.payload_uri !== objMessage.payload_uri.trim())
			return callback(createJointError("payload uri must be trimmed"));
		if (!isStringOfLength(objMessage.payload_uri_hash, constants.HASH_LENGTH))
			return callback("wrong length of payload uri hash");
		if (objMessage.payload_uri.length > 500)
			return callback(createJointError("payload_uri too long"));
		if (objMessage.payload_uri.includes("\x00"))
			return callback(createJointError("payload_uri contains null byte"));
		if (objectHash.getBase64Hash(objMessage.payload_uri) !== objMessage.payload_uri_hash)
			return callback(createJointError("wrong payload_uri hash"));
	}
	else{
		if ("payload_uri_hash" in objMessage)
			return callback("must not contain payload_uri_hash");
		if ("payload_uri" in objMessage)
			return callback(createJointError("must not contain payload_uri"));
	}
	
	if (objMessage.app === "payment"){ // special requirements for payment
		if (objMessage.payload_location !== "inline" && objMessage.payload_location !== "none")
			return callback("payment location must be inline or none");
		if (objMessage.payload_location === "none" && !objMessage.spend_proofs)
			return callback("private payment must come with spend proof(s)");
	}
	
	var arrInlineOnlyApps = ["address_definition_change", "data_feed", "definition_template", "asset", "asset_attestors", "attestation", "poll", "vote", "definition", "system_vote", "system_vote_count", "temp_data"];
	if (arrInlineOnlyApps.indexOf(objMessage.app) >= 0 && objMessage.payload_location !== "inline")
		return callback(objMessage.app+" must be inline");

	if (objValidationState.bAA) {
		if (!aa_validation.aaApps.includes(objMessage.app))
			return callback("unsupported app in AA response: " + objMessage.app);
		if (objMessage.payload_location !== "inline")
			return callback("AA response must be inline");
	}
	
	
	function validatePayload(cb){
		if (objMessage.payload_location === "inline"){
			validateInlinePayload(conn, objMessage, message_index, objUnit, objValidationState, cb);
		}
		else{
			if (!isValidBase64(objMessage.payload_hash, constants.HASH_LENGTH))
				return cb("wrong payload hash");
			cb();
		}
	}
	
	function validateSpendProofs(cb){
		if (!("spend_proofs" in objMessage))
			return cb();
		var arrEqs = objMessage.spend_proofs.map(function(objSpendProof){
			return "spend_proof="+conn.escape(objSpendProof.spend_proof)+
				" AND address="+conn.escape(objSpendProof.address ? objSpendProof.address : objUnit.authors[0].address);
		});
		var doubleSpendIndexMySQL = conf.storage == "mysql" ? "USE INDEX(bySpendProof)" : "";
		checkForDoublespends(conn, "spend proof", 
			"SELECT address, unit, main_chain_index, sequence, is_stable FROM spend_proofs "+ doubleSpendIndexMySQL+" JOIN units USING(unit) WHERE unit != ? AND sequence!='final-bad' AND ("+arrEqs.join(" OR ")+")",
			[objUnit.unit], 
			objUnit, objValidationState, function(cb2){ cb2(); }, cb);
	}
	
	async.series([validateSpendProofs, validatePayload], callback);
}


function checkForDoublespends(conn, type, sql, arrSqlArgs, objUnit, objValidationState, onAcceptedDoublespends, cb){
	conn.query(
		sql, 
		arrSqlArgs,
		function(rows){
			if (rows.length === 0)
				return cb();
			var arrAuthorAddresses = objUnit.authors.map(function(author) { return author.address; } );
			async.eachSeries(
				rows,
				function(objConflictingRecord, cb2){
					if (arrAuthorAddresses.indexOf(objConflictingRecord.address) === -1)
						throw Error("conflicting "+type+" spent from another address?");
					if (conf.bLight) // we can't use graph in light wallet, the private payment can be resent and revalidated when stable
						return cb2(objUnit.unit+": conflicting "+type);
					graph.determineIfIncludedOrEqual(conn, objConflictingRecord.unit, objUnit.parent_units, function(bIncluded){
						if (bIncluded){
							var error = objUnit.unit+": conflicting "+type+" in inner unit "+objConflictingRecord.unit;

							// too young (serial or nonserial)
							if (objConflictingRecord.main_chain_index > objValidationState.last_ball_mci || objConflictingRecord.main_chain_index === null)
								return cb2(objConflictingRecord.is_stable ? error : createTransientError(error)); // if the conflict is not stable, our unit can be retried later. If the conflict stabilizes as good, the rejection becomes permanent. If as final-bad, the conflict is treated as non-existent.

							// in good sequence (final state); final-bad is excluded by the query and treated as non-existent
							if (objConflictingRecord.sequence === 'good')
								return cb2(error);

							throw Error("unreachable code, conflicting "+type+" in unit "+objConflictingRecord.unit);
						}
						else{ // arrAddressesWithForkedPath is not set when validating private payments
							if (objValidationState.arrAddressesWithForkedPath && objValidationState.arrAddressesWithForkedPath.indexOf(objConflictingRecord.address) === -1)
								throw Error("double spending "+type+" without double spending address?");
							cb2();
						}
					});
				},
				function(err){
					if (err)
						return cb(err);
					onAcceptedDoublespends(cb);
				}
			);
		}
	);
}

function validateInlinePayload(conn, objMessage, message_index, objUnit, objValidationState, callback){
	var payload = objMessage.payload;
	if (typeof payload === "undefined")
		return callback("no inline payload");

	switch (objMessage.app){

		case "text":
			if (typeof payload !== "string")
				return callback("payload must be string");
			return callback();

		case "address_definition_change":
			// Disabled now. A signer in a shared address could completely rewrite the rules when they get an opportunity to sign unilaterally under the current rules.
			if (objValidationState.last_ball_mci >= constants.pemCurvesFixMci || !objValidationState.hasBall && storage.getMinRetrievableMci() >= constants.pemCurvesFixMci)
				return callback("address_definition_change not allowed any more");
			if (!isNonemptyObject(payload))
				return callback("payload must be a non empty object");
			if (hasFieldsExcept(payload, ["definition_chash", "address"]))
				return callback("unknown fields in address_definition_change");
			var arrAuthorAddresses = objUnit.authors.map(function(author) { return author.address; } );
			var address;
			if (objUnit.authors.length > 1){
				if (!isValidAddress(payload.address))
					return callback("when multi-authored, must indicate address");
				if (arrAuthorAddresses.indexOf(payload.address) === -1)
					return callback("foreign address");
				address = payload.address;
			}
			else{
				if ('address' in payload)
					return callback("when single-authored, must not indicate address");
				address = arrAuthorAddresses[0];
			}
			if (!objValidationState.arrDefinitionChangeFlags)
				objValidationState.arrDefinitionChangeFlags = {};
			if (objValidationState.arrDefinitionChangeFlags[address])
				return callback("can be only one definition change per address");
			objValidationState.arrDefinitionChangeFlags[address] = true;
			if (!isValidAddress(payload.definition_chash))
				return callback("bad new definition_chash");
			return callback();

		case "definition": // for AAs only
			if (!isNonemptyObject(payload))
				return callback("payload must be a non empty object");
			if (hasFieldsExcept(payload, ["address", "definition"])) // AA definition cannot be changed and its address is also its definition_chash
				return callback("unknown fields in app definition");
			try{
				if (payload.address !== objectHash.getChash160(payload.definition))
					return callback("definition doesn't match the chash");
			}
			catch(e){
				return callback("bad definition");
			}
			if (constants.bTestnet && ['BD7RTYgniYtyCX0t/a/mmAAZEiK/ZhTvInCMCPG5B1k=', 'EHEkkpiLVTkBHkn8NhzZG/o4IphnrmhRGxp4uQdEkco=', 'bx8VlbNQm2WA2ruIhx04zMrlpQq3EChK6o3k5OXJ130=', '08t8w/xuHcsKlMpPWajzzadmMGv+S4AoeV/QL1F3kBM=', '4N5fsU9qJSn2FuS70cChKx8QqgcesPRPs0dNfzOhoXw='].indexOf(objUnit.unit) >= 0)
				return callback();
			const top_mci = objValidationState.aa_mci || objValidationState.last_ball_mci;
			var readGetterProps = function (aa_address, func_name, cb) {
				storage.readAAGetterProps(conn, aa_address, func_name, top_mci, cb);
			};
			if (!objValidationState.hasBall && !objValidationState.bAA && storage.getMinRetrievableMci() >= constants.pemCurvesFixMci && objValidationState.last_ball_mci < constants.pemCurvesFixMci)
				return callback(createTransientError("AA definition attached to an old part of the DAG"));
			aa_validation.validateAADefinition(payload.definition, readGetterProps, objValidationState.last_ball_mci, function (err) {
				if (err)
					return callback(err);
				var template = payload.definition[1];
				if (template.messages)
					return callback(); // regular AA
				// else parameterized AA
				storage.readAADefinition(conn, template.base_aa, top_mci, function (arrBaseDefinition) {
					if (!arrBaseDefinition)
						return callback("base AA not found");
					if (!arrBaseDefinition[1].messages)
						return callback("base AA must be a regular AA");
					callback();
				});
			});
			break;

		case "poll":
			if (objValidationState.bHasPoll)
				return callback("can be only one poll");
			objValidationState.bHasPoll = true;
			if (!isNonemptyObject(payload))
				return callback("poll payload must be a non empty object");
			if (hasFieldsExcept(payload, ["question", "choices"]))
				return callback("unknown fields in "+objMessage.app);
			if (typeof payload.question !== 'string')
				return callback("no question in poll");
			if (payload.question !== payload.question.trim())
				return callback("question must be trimmed");
			if (!isNonemptyArray(payload.choices))
				return callback("no choices in poll");
			if (payload.choices.length > constants.MAX_CHOICES_PER_POLL)
				return callback("too many choices in poll");
			let seenChoices = Object.create(null);
			for (var i=0; i<payload.choices.length; i++) {
				if (typeof payload.choices[i] !== 'string')
					return callback("all choices must be strings");
				if (payload.choices[i].trim().length === 0)
					return callback("all choices must be longer than 0 chars");
				if (payload.choices[i].length > constants.MAX_CHOICE_LENGTH)
					return callback("all choices must be "+ constants.MAX_CHOICE_LENGTH + " chars or less");
				if (seenChoices[payload.choices[i]])
					return callback("all choices must be different");
				seenChoices[payload.choices[i]] = true;
			}
			return callback();
			
		case "vote":
			if (objValidationState.bHasVote && (objValidationState.last_ball_mci >= constants.pemCurvesFixMci || !objValidationState.hasBall && storage.getMinRetrievableMci() >= constants.pemCurvesFixMci))
				return callback("can be only one vote");
			objValidationState.bHasVote = true;
			if (!isNonemptyObject(payload))
				return callback("payload must be a non empty object");
			if (!isStringOfLength(payload.unit, constants.HASH_LENGTH))
				return callback("invalid unit in vote");
			if (typeof payload.choice !== "string")
				return callback("choice must be string");
			if (hasFieldsExcept(payload, ["unit", "choice"]))
				return callback("unknown fields in "+objMessage.app);
			conn.query(
				"SELECT main_chain_index, sequence FROM polls JOIN poll_choices USING(unit) JOIN units USING(unit) WHERE unit=? AND choice=?", 
				[payload.unit, payload.choice],
				function(poll_unit_rows){
					if (poll_unit_rows.length > 1)
						throw Error("more than one poll?");
					if (poll_unit_rows.length === 0)
						return callback("invalid choice "+payload.choice+" or poll "+payload.unit);
					var objPollUnitProps = poll_unit_rows[0];
					if (objPollUnitProps.main_chain_index === null || objPollUnitProps.main_chain_index > objValidationState.last_ball_mci)
						return callback("poll unit must be before last ball");
					if (objPollUnitProps.sequence !== 'good')
						return callback("poll unit is not serial");
					return callback();
				}
			);
			break;

		case "system_vote":
			if (objValidationState.last_ball_mci < constants.v4UpgradeMci && !constants.bDevnet)
				return callback("cannot vote for system params yet");
			if (objValidationState.bAA)
				return callback("AA cannot cast system vote");
			if (objValidationState.bHasSystemVote)
				return callback("can be only one system vote");
			objValidationState.bHasSystemVote = true;
			if (!isNonemptyObject(payload))
				return callback("payload must be a non empty object");
			if (hasFieldsExcept(payload, ["subject", "value"]))
				return callback("unknown fields in " + objMessage.app);
			if (typeof payload.subject !== "string")
				return callback("subject must be string");
			if (!payload.value)
				return callback("no value in " + objMessage.app);
			switch (payload.subject) {
				case "op_list":
					const arrOPs = payload.value;
					if (!isArrayOfLength(arrOPs, constants.COUNT_WITNESSES))
						return callback("OP list must be an array of " + constants.COUNT_WITNESSES);
					if (!arrOPs.every(isValidAddress))
						return callback("all OPs must be valid addresses");
					let prev_op = arrOPs[0];
					for (let i = 1; i < arrOPs.length; i++){
						const op = arrOPs[i];
						if (op <= prev_op)
							return callback("OP list must be sorted and unique");
						prev_op = op;
					}
					checkNotAAs(conn, arrOPs, objValidationState.last_ball_mci, err => {
						if (err)
							return callback(err);
						checkWitnessesKnownAndGood(conn, objValidationState, arrOPs, err => {
							if (err)
								return callback(err);
							checkNoReferencesInWitnessAddressDefinitions(conn, objValidationState, arrOPs, callback);
						});
					});
					break;
				case "threshold_size":
					if (!isPositiveInteger(payload.value))
						return callback(payload.subject + " must be a positive integer");
					if (!constants.bTestnet || objValidationState.last_ball_mci > 3543000) {
						if (payload.value < 1000)
							return callback(payload.subject + " must be at least 1000");
					}
					callback();
					break;
				case "base_tps_fee":
				case "tps_interval":
				case "tps_fee_multiplier":
					if (!(typeof payload.value === 'number' && isFinite(payload.value) && payload.value > 0))
						return callback(payload.subject + " must be a positive number");
					if (objValidationState.last_ball_mci >= constants.pemCurvesFixMci || !objValidationState.hasBall && storage.getMinRetrievableMci() >= constants.pemCurvesFixMci) {
						if (payload.subject === "tps_interval" && payload.value < 0.1)
							return callback(payload.subject + " must be at least 0.1");
						if (payload.subject === "base_tps_fee" && payload.value > 1e8)
							return callback(payload.subject + " must be at most 1e8");
						if (payload.subject === "tps_fee_multiplier" && (payload.value < 1 || payload.value > 1000))
							return callback(payload.subject + " must be between 1 and 1000");
					}
					callback();
					break;
				default:
					return callback("unknown subject: " + payload.subject);
			}
			break;

		case "system_vote_count":
			if (objValidationState.last_ball_mci < constants.v4UpgradeMci && !constants.bDevnet)
				return callback("cannot count votes for system params yet");
			if (objValidationState.bAA)
				return callback("AA cannot trigger system vote count");
			if (objValidationState.bHasSystemVoteCount)
				return callback("can be only one system vote count");
			objValidationState.bHasSystemVoteCount = true;
			if (!["op_list", "threshold_size", "base_tps_fee", "tps_interval", "tps_fee_multiplier"].includes(payload))
				return callback("unknown subject in vote count");
			return callback();

		case "data_feed":
			if (objValidationState.bHasDataFeed)
				return callback("can be only one data feed");
			objValidationState.bHasDataFeed = true;
			if (!isNonemptyObject(payload))
				return callback("data feed payload must be non-empty object");
			if (Object.keys(payload).length * objUnit.authors.length > constants.MAX_DATA_FEEDS_PER_MESSAGE)
				return callback("too many data feeds in message");
			for (var feed_name in payload){
				if (feed_name.length > constants.MAX_DATA_FEED_NAME_LENGTH)
					return callback("feed name "+feed_name+" too long");
				if (feed_name.indexOf('\n') >=0 )
					return callback("feed name "+feed_name+" contains \\n");
				var value = payload[feed_name];
				if (typeof value === 'string'){
					if (value.length > constants.MAX_DATA_FEED_VALUE_LENGTH)
						return callback("data feed value too long: " + value);
					if (value.indexOf('\n') >=0 )
						return callback("value "+value+" of feed name "+feed_name+" contains \\n");
				}
				else if (typeof value === 'number'){
					if (!isInteger(value))
						return callback("fractional numbers not allowed in data feeds");
				}
				else
					return callback("data feed "+feed_name+" must be string or number");
			}
			return callback();

		case "profile":
			if (objUnit.authors.length !== 1)
				return callback("profile must be single-authored");
			if (objValidationState.bHasProfile)
				return callback("can be only one profile");
			objValidationState.bHasProfile = true;
			// no break, continuing
		case "data":
			if (typeof payload !== "object" || payload === null)
				return callback(objMessage.app+" payload must be object");
			return callback();

		case "temp_data":
			if (objValidationState.last_ball_mci < constants.v4UpgradeMci)
				return callback("cannot use temp_data yet");
			if (typeof payload !== "object" || payload === null)
				return callback("temp_data payload must be an object");
			if (Array.isArray(payload))
				return callback("temp_data payload must not be an array");
			if (hasFieldsExcept(payload, ["data_length", "data_hash", "data"]))
				return callback("unknown fields in " + objMessage.app);
			if (!isPositiveInteger(payload.data_length))
				return callback("bad data_length");
			if (!isValidBase64(payload.data_hash))
				return callback("bad data_hash");
			if ("data" in payload) {
				const createError = objValidationState.bAA ? err => err : createJointError;
				if (payload.data === null)
					return callback(createError("null data"));
				if (!payload.data && (objValidationState.last_ball_mci >= constants.pemCurvesFixMci || !objValidationState.hasBall && storage.getMinRetrievableMci() >= constants.pemCurvesFixMci))
					return callback(createError("falsy temp data: " + payload.data)); // getTotalPayloadSize() checks for thruthiness
				try {
					const len = objectLength.getLength(payload.data, true);
					if (len !== payload.data_length)
						return callback(createError(`data_length mismatch, expected ${payload.data_length}, got ${len}`));
					const hash = objectHash.getBase64Hash(payload.data, true);
					if (hash !== payload.data_hash)
						return callback(createError(`data_hash mismatch, expected ${payload.data_hash}, got ${hash}`));
				}
				catch (e) {
					return callback(createError("invalid temp data: " + e));
				}
			}
			else {
				if (Math.round(Date.now()/1000) - objUnit.timestamp < constants.TEMP_DATA_PURGE_TIMEOUT && !objValidationState.bAA)
					return callback(createTransientError("data not found in temp_data"))
			}
			return callback();

		case "definition_template":
			if (objValidationState.bHasDefinitionTemplate)
				return callback("can be only one definition template");
			objValidationState.bHasDefinitionTemplate = true;
			if (!ValidationUtils.isArrayOfLength(payload, 2))
				return callback(objMessage.app+" payload must be array of two elements");
			return callback();

		case "attestation":
			if (objUnit.authors.length !== 1)
				return callback("attestation must be single-authored");
			if (!isNonemptyObject(payload))
				return callback("payload must be a non empty object");
			if (hasFieldsExcept(payload, ["address", "profile"]))
				return callback("unknown fields in "+objMessage.app);
			if (!isValidAddress(payload.address))
				return callback("attesting an invalid address");
			if (!isNonemptyObject(payload.profile))
				return callback("attested profile must be non empty object");
			// it is ok if the address has never been used yet
			// it is also ok to attest oneself
			return callback();

		case "asset":
			if (objValidationState.bHasAssetDefinition)
				return callback("can be only one asset definition");
			objValidationState.bHasAssetDefinition = true;
			validateAssetDefinition(conn, payload, objUnit, objValidationState, callback);
			break;

		case "asset_attestors":
			if (!isStringOfLength(payload.asset, constants.HASH_LENGTH))
				return callback("invalid asset in attestor list update");
			if (!objValidationState.assocHasAssetAttestors)
				objValidationState.assocHasAssetAttestors = {};
			if (objValidationState.assocHasAssetAttestors[payload.asset])
				return callback("can be only one asset attestor list update per asset");
			objValidationState.assocHasAssetAttestors[payload.asset] = true;
			validateAttestorListUpdate(conn, payload, objUnit, objValidationState, callback);
			break;

		case "payment":
			validatePayment(conn, payload, message_index, objUnit, objValidationState, callback);
			break;

		default:
			return callback("unknown app: "+objMessage.app);
	}
}

function checkNotAAs(conn, arrOPs, mci, cb) {
	conn.query("SELECT address FROM aa_addresses WHERE address IN (?) AND mci<=?", [arrOPs, mci], rows => {
		rows.length ? cb("some OPs are AAs: " + rows.map(r => r.address).join(', ')) : cb();
	});
}

// used for both public and private payments
function validatePayment(conn, payload, message_index, objUnit, objValidationState, callback){

	if (!isNonemptyObject(payload))
		return callback("payment must be a non-empty object");
	if (!isNonemptyArray(payload.inputs))
		return callback("no inputs");
	if (!isNonemptyArray(payload.outputs))
		return callback("no outputs");

	if (!("asset" in payload)){ // base currency
		if (hasFieldsExcept(payload, ["inputs", "outputs"]))
			return callback("unknown fields in payment message");
		if (objValidationState.bHasBasePayment)
			return callback("can have only one base payment");
		objValidationState.bHasBasePayment = true;
		return validatePaymentInputsAndOutputs(conn, payload, null, message_index, objUnit, objValidationState, callback);
	}
	
	// asset
	if (!isStringOfLength(payload.asset, constants.HASH_LENGTH))
		return callback("invalid asset");
	
	var arrAuthorAddresses = objUnit.authors.map(function(author) { return author.address; } );
	// note that light clients cannot check attestations
	const mci = (objValidationState.bAA && (objValidationState.aa_mci >= constants.pemCurvesFixMci || constants.bTestnet && [1120088, 1230819].includes(objValidationState.aa_mci))) ? objValidationState.aa_mci : objValidationState.last_ball_mci;
	storage.loadAssetWithListOfAttestedAuthors(conn, payload.asset, mci, arrAuthorAddresses, objValidationState.bAA, function(err, objAsset){
		if (err)
			return callback(err);
		if (hasFieldsExcept(payload, ["inputs", "outputs", "asset", "denomination"]))
			return callback("unknown fields in payment message");
		if (objAsset.fixed_denominations){
			if (!isPositiveInteger(payload.denomination))
				return callback("no denomination");
		}
		else{
			if ("denomination" in payload)
				return callback("denomination in arbitrary-amounts asset")
		}
		if (!!objAsset.is_private !== !!objValidationState.bPrivate)
			return callback("asset privacy mismatch");
		var bIssue = (payload.inputs[0].type === "issue");
		var issuer_address;
		if (bIssue){
			if (arrAuthorAddresses.length === 1)
				issuer_address = arrAuthorAddresses[0];
			else{
				issuer_address = payload.inputs[0].address;
				if (arrAuthorAddresses.indexOf(issuer_address) === -1)
					return callback("issuer not among authors");
			}
			if (objAsset.issued_by_definer_only && issuer_address !== objAsset.definer_address)
				return callback("only definer can issue this asset");
		}
		if (objAsset.cosigned_by_definer && arrAuthorAddresses.indexOf(objAsset.definer_address) === -1)
			return callback("must be cosigned by definer");
		
		if (objAsset.spender_attested){
			if (conf.bLight && objAsset.is_private) // in light clients, we don't have the attestation data but if the asset is public, we trust witnesses to have checked attestations
				return callback("being light, I can't check attestations for private assets"); // TODO: request history
			if (objAsset.arrAttestedAddresses.length === 0)
				return callback("none of the authors is attested");
			if (bIssue && objAsset.arrAttestedAddresses.indexOf(issuer_address) === -1)
				return callback("issuer is not attested");
		}
		validatePaymentInputsAndOutputs(conn, payload, objAsset, message_index, objUnit, objValidationState, callback);
	});
}

// divisible assets (including base asset)
function validatePaymentInputsAndOutputs(conn, payload, objAsset, message_index, objUnit, objValidationState, callback){
	
//	if (objAsset)
//		profiler2.start();
	var denomination = payload.denomination || 1;
	var arrAuthorAddresses = objUnit.authors.map(function(author) { return author.address; } );
	var arrInputAddresses = []; // used for non-transferrable assets only
	var arrOutputAddresses = [];
	var total_input = 0;
	if (payload.inputs.length > constants.MAX_INPUTS_PER_PAYMENT_MESSAGE && !objValidationState.bAA)
		return callback("too many inputs");
	if (payload.outputs.length > constants.MAX_OUTPUTS_PER_PAYMENT_MESSAGE && !storage.isGenesisUnit(objUnit.unit))
		return callback("too many outputs");
	
	if (objAsset && objAsset.fixed_denominations && payload.inputs.length !== 1)
		return callback("fixed denominations payment must have 1 input");

	const isValidAddressWithCase = objValidationState.last_ball_mci >= constants.timestampUpgradeMci ? ValidationUtils.isValidAddress : ValidationUtils.isValidAddressAnyCase;

	var total_output = 0;
	var prev_address = ""; // if public, outputs must be sorted by address
	var prev_amount = 0;
	var count_open_outputs = 0;
	for (var i=0; i<payload.outputs.length; i++){
		var output = payload.outputs[i];
		if (!isNonemptyObject(output))
			return callback("output must be a non-empty object");
		if (hasFieldsExcept(output, ["address", "amount", "blinding", "output_hash"]))
			return callback("unknown fields in payment output");
		if (!isPositiveInteger(output.amount))
			return callback("amount must be positive integer, found "+JSON.stringify(output.amount));
		if (output.amount > constants.MAX_CAP)
			return callback("output too large: " + output.amount);
		if (objAsset && objAsset.fixed_denominations && output.amount % denomination !== 0)
			return callback("output amount must be divisible by denomination");
		if (objAsset && objAsset.is_private){
			if (("output_hash" in output) !== !!objAsset.fixed_denominations)
				return callback("output_hash must be present with fixed denominations only");
			if ("output_hash" in output && !isStringOfLength(output.output_hash, constants.HASH_LENGTH))
				return callback("invalid output hash");
			if (!objAsset.fixed_denominations && !(("blinding" in output) && ("address" in output)))
				return callback("no blinding or address");
			if ("blinding" in output && !isStringOfLength(output.blinding, 16))
				return callback("bad blinding");
			if (("blinding" in output) !== ("address" in output))
				return callback("address and blinding must come together");
			if ("address" in output && !isValidAddressWithCase(output.address))
				return callback("output address " + JSON.stringify(output.address) + " invalid");
			if (output.address)
				count_open_outputs++;
		}
		else{
			if ("blinding" in output)
				return callback("public output must not have blinding");
			if ("output_hash" in output)
				return callback("public output must not have output_hash");
			if (!isValidAddressWithCase(output.address))
				return callback("output address " + JSON.stringify(output.address) + " invalid");
			if (prev_address > output.address)
				return callback("output addresses not sorted");
			else if (prev_address === output.address && prev_amount > output.amount)
				return callback("output amounts for same address not sorted");
			prev_address = output.address;
			prev_amount = output.amount;
		}
		if (output.address && arrOutputAddresses.indexOf(output.address) === -1)
			arrOutputAddresses.push(output.address);
		total_output += output.amount;
		if (total_output > constants.MAX_CAP)
			return callback("total output too large: " + total_output);
	}
	if (objAsset && objAsset.is_private && count_open_outputs !== 1)
		return callback("found "+count_open_outputs+" open outputs, expected 1");

	var bIssue = false;
	var bHaveHeadersComissions = false;
	var bHaveWitnessings = false;
	
	// same for both public and private
	function validateIndivisibleIssue(input, cb){
	//	if (objAsset)
	//		profiler2.start();
		conn.query(
			"SELECT count_coins FROM asset_denominations WHERE asset=? AND denomination=?", 
			[payload.asset, denomination], 
			function(rows){
				if (rows.length === 0)
					return cb("invalid denomination: "+denomination);
				if (rows.length > 1)
					throw Error("more than one record per denomination?");
				var denomInfo = rows[0];
				if (denomInfo.count_coins === null){ // uncapped
					if (input.amount % denomination !== 0)
						return cb("issue amount must be multiple of denomination");
				}
				else{
					if (input.amount !== denomination * denomInfo.count_coins)
						return cb("wrong size of issue of denomination "+denomination);
				}
			//	if (objAsset)
			//		profiler2.stop('validateIndivisibleIssue');
				cb();
			}
		);
	}
	
//	if (objAsset)
//		profiler2.stop('validate outputs');
	
	// max 1 issue must come first, then transfers, then hc, then witnessings
	// no particular sorting order within the groups
	async.forEachOfSeries(
		payload.inputs,
		function(input, input_index, cb){
			if (!isNonemptyObject(input))
				return cb("input must be a non-empty object");
			if (objAsset){
				if ("type" in input && input.type !== "issue")
					return cb("non-base input can have only type=issue");
			}
			else{
				if ("type" in input && !["issue", "headers_commission", "witnessing"].includes(input.type))
					return cb("bad input type");
			}
			var type = input.type || "transfer";

			var doubleSpendFields = "unit, address, message_index, input_index, main_chain_index, sequence, is_stable";
			var doubleSpendWhere;
			var doubleSpendVars = [];
			var doubleSpendIndexMySQL = "";
			function checkInputDoubleSpend(cb2){
			//	if (objAsset)
			//		profiler2.start();
				doubleSpendWhere += " AND unit != " + conn.escape(objUnit.unit);
				if (objAsset){
					doubleSpendWhere += " AND asset=?";
					doubleSpendVars.push(payload.asset);
				}
				else
					doubleSpendWhere += " AND asset IS NULL";
				// final-bad units are treated as non-existent competitors (their inputs.is_unique is kept NULL)
				var doubleSpendQuery = "SELECT "+doubleSpendFields+" FROM inputs " + doubleSpendIndexMySQL + " JOIN units USING(unit) WHERE "+doubleSpendWhere+" AND sequence!='final-bad'";
				checkForDoublespends(
					conn, "divisible input", 
					doubleSpendQuery, doubleSpendVars, 
					objUnit, objValidationState, 
					function acceptDoublespends(cb3){
						console.log("--- accepting doublespend on unit "+objUnit.unit);
						var sql = "UPDATE inputs SET is_unique=NULL WHERE "+doubleSpendWhere+
							" AND (SELECT is_stable FROM units WHERE units.unit=inputs.unit)=0";
						if (!(objAsset && objAsset.is_private)){
							objValidationState.arrAdditionalQueries.push({sql: sql, params: doubleSpendVars});
							objValidationState.arrDoubleSpendInputs.push({message_index: message_index, input_index: input_index});
							return cb3();
						}
						mutex.lock(["private_write"], function(unlock){
							console.log("--- will ununique the conflicts of unit "+objUnit.unit);
							conn.query(
								sql, 
								doubleSpendVars, 
								function(){
									console.log("--- ununique done unit "+objUnit.unit);
									objValidationState.arrDoubleSpendInputs.push({message_index: message_index, input_index: input_index});
									unlock();
									cb3();
								}
							);
						});
					}, 
					function onDone(err){
						if (err && objAsset && objAsset.is_private && !conf.bLight)
							throw Error("spend proof didn't help: "+err);
					//	if (objAsset)
					//		profiler2.stop('checkInputDoubleSpend');
						cb2(err);
					}
				);
			}

			switch (type){
				case "issue":
				//	if (objAsset)
				//		profiler2.start();
					if (input_index !== 0)
						return cb("issue must come first");
					if (hasFieldsExcept(input, ["type", "address", "amount", "serial_number"]))
						return cb("unknown fields in issue input");
					if (!isPositiveInteger(input.amount))
						return cb("amount must be positive");
					if (input.amount > constants.MAX_CAP)
						return cb("issue amount too large: " + input.amount)
					if (!isPositiveInteger(input.serial_number))
						return cb("serial_number must be positive");
					if (!objAsset || objAsset.cap){
						if (input.serial_number !== 1)
							return cb("for capped asset serial_number must be 1");
					}
					if (bIssue)
						return cb("only one issue per message allowed");
					bIssue = true;
					
					var address = null;
					if (arrAuthorAddresses.length === 1){
						if ("address" in input)
							return cb("when single-authored, must not put address in issue input");
						address = arrAuthorAddresses[0];
					}
					else{
						if (typeof input.address !== "string")
							return cb("when multi-authored, must put address in issue input");
						if (arrAuthorAddresses.indexOf(input.address) === -1)
							return cb("issue input address "+input.address+" is not an author");
						address = input.address;
					}
					
					arrInputAddresses = [address];
					if (objAsset){
						if (objAsset.cap && !objAsset.fixed_denominations && input.amount !== objAsset.cap)
							return cb("issue must be equal to cap");
					}
					else{
						if (!storage.isGenesisUnit(objUnit.unit))
							return cb("only genesis can issue base asset");
						if (input.amount !== constants.TOTAL_WHITEBYTES)
							return cb("issue must be equal to cap");
					}
					total_input += input.amount;
					
					var input_key = (payload.asset || "base") + "-" + denomination + "-" + address + "-" + input.serial_number;
					if (objValidationState.arrInputKeys.indexOf(input_key) >= 0)
						return callback("input "+input_key+" already used");
					objValidationState.arrInputKeys.push(input_key);
					doubleSpendWhere = "type='issue'";
					doubleSpendVars = [];
				//	if (objAsset && objAsset.fixed_denominations){
						doubleSpendWhere += " AND denomination=?";
						doubleSpendVars.push(denomination);
				//	}
					if (objAsset){
						doubleSpendWhere += " AND serial_number=?";
						doubleSpendVars.push(input.serial_number);
					}
					if (objAsset && !objAsset.issued_by_definer_only){
						doubleSpendWhere += " AND address=?";
						doubleSpendVars.push(address);
					}
				//	if (objAsset)
				//		profiler2.stop('validate issue');
					if (objAsset && objAsset.fixed_denominations){
						validateIndivisibleIssue(input, function(err){
							if (err)
								return cb(err);
							checkInputDoubleSpend(cb);
						});
					}
					else
						checkInputDoubleSpend(cb);
					// attestations and issued_by_definer_only already checked before
					break;
					
				case "transfer":
				//	if (objAsset)
				//		profiler2.start();
					if (bHaveHeadersComissions || bHaveWitnessings)
						return cb("all transfers must come before hc and witnessings");
					if (hasFieldsExcept(input, ["type", "unit", "message_index", "output_index"]))
						return cb("unknown fields in payment input");
					if (!isStringOfLength(input.unit, constants.HASH_LENGTH))
						return cb("wrong unit length in payment input");
					if (!isNonnegativeInteger(input.message_index))
						return cb("no message_index in payment input");
					if (!isNonnegativeInteger(input.output_index))
						return cb("no output_index in payment input");
					
					var input_key = (payload.asset || "base") + "-" + input.unit + "-" + input.message_index + "-" + input.output_index;
					if (objValidationState.arrInputKeys.indexOf(input_key) >= 0)
						return cb("input "+input_key+" already used");
					objValidationState.arrInputKeys.push(input_key);
					
					doubleSpendWhere = "type=? AND src_unit=? AND src_message_index=? AND src_output_index=?";
					doubleSpendVars = [type, input.unit, input.message_index, input.output_index];
					if (conf.storage == "mysql")
						doubleSpendIndexMySQL = " FORCE INDEX(bySrcOutput) ";

					// for private fixed denominations assets, we can't look up src output in the database 
					// because we validate the entire chain before saving anything.
					// Instead we prepopulate objValidationState with denomination and src_output 
					if (objAsset && objAsset.is_private && objAsset.fixed_denominations){
						if (!objValidationState.src_coin)
							throw Error("no src_coin");
						var src_coin = objValidationState.src_coin;
						if (!src_coin.src_output)
							throw Error("no src_output");
						if (!isPositiveInteger(src_coin.denomination))
							throw Error("no denomination in src coin");
						if (!isPositiveInteger(src_coin.amount))
							throw Error("no src coin amount");
						var owner_address = src_coin.src_output.address;
						if (arrAuthorAddresses.indexOf(owner_address) === -1)
							return cb("output owner is not among authors");
						if (denomination !== src_coin.denomination)
							return cb("private denomination mismatch");
						if (objAsset.auto_destroy && owner_address === objAsset.definer_address)
							return cb("this output was destroyed by sending to definer address");
						if (objAsset.spender_attested && objAsset.arrAttestedAddresses.indexOf(owner_address) === -1)
							return cb("owner address is not attested");
						if (arrInputAddresses.indexOf(owner_address) === -1)
							arrInputAddresses.push(owner_address);
						total_input += src_coin.amount;
						console.log("-- val state "+JSON.stringify(objValidationState));
					//	if (objAsset)
					//		profiler2.stop('validate transfer');
						return checkInputDoubleSpend(cb);
					}
					
					conn.query(
						"SELECT amount, is_stable, sequence, address, main_chain_index, denomination, asset \n\
						FROM units \n\
						LEFT JOIN outputs ON units.unit=outputs.unit AND message_index=? AND output_index=? \n\
						WHERE units.unit=?",
						[input.message_index, input.output_index, input.unit],
						function(rows){
							if (rows.length > 1)
								throw Error("more than 1 src output");
							if (rows.length === 0)
								return cb("input unit "+input.unit+" not found");
							var src_output = rows[0];
							var bStableInParents = (src_output.main_chain_index !== null && src_output.main_chain_index <= objValidationState.last_ball_mci);
							if (bStableInParents) {
								if (src_output.sequence === 'temp-bad')
									throw Error("spending a stable temp-bad output " + input.unit);
								if (src_output.sequence === 'final-bad')
									return cb("spending a stable final-bad output " + input.unit);
							}
							if (!src_output.address) {
								if (src_output.sequence === 'final-bad' && src_output.main_chain_index < storage.getMinRetrievableMci()) { // already stripped locally
									const objCachedOutput = archiving.getCachedOutput(input.unit, input.message_index, input.output_index);
									if (!objCachedOutput) // ask the peer who sent this unit. If the peer doesn't respond but other nodes have accepted the unit, they'll share it with us, we'll get here again and request input.unit from them
										return cb({error_code: "unresolved_dependency", arrMissingUnits: [input.unit], bRequestPrunedContent: true});
									src_output.address = objCachedOutput.address;
									src_output.amount = objCachedOutput.amount;
									src_output.denomination = objCachedOutput.denomination;
									src_output.asset = objCachedOutput.asset;
								}
								else
									return cb("output being spent " + input.unit + " not found");
							}
							if (typeof src_output.amount !== 'number')
								throw Error("src output amount is not a number");
							if (!(!payload.asset && !src_output.asset || payload.asset === src_output.asset))
								return cb("asset mismatch");
							//if (src_output.is_stable !== 1) // we allow immediate spends, that's why the error is transient
							//    return cb(createTransientError("input unit is not on stable MC yet, unit "+objUnit.unit+", input "+input.unit));
							if (bStableInParents && src_output.sequence !== 'good')
								return cb("stable input unit "+input.unit+" is not serial");
							if (objValidationState.last_ball_mci < constants.spendUnconfirmedUpgradeMci){
								if (!objAsset || !objAsset.is_private){
									// for public payments, you can't spend unconfirmed transactions
									if (!bStableInParents)
										return cb("src output must be before last ball");
								}
								if (src_output.sequence !== 'good') // it is also stable or private
									return cb("input unit "+input.unit+" is not serial");
							}
							else{ // after this MCI, spending unconfirmed is allowed for public assets too, non-good sequence will be inherited
								if (src_output.sequence !== 'good'){
									console.log(objUnit.unit + ": inheriting sequence " + src_output.sequence + " from src output " + input.unit);
									if (objValidationState.sequence === 'good' || objValidationState.sequence === 'temp-bad')
										objValidationState.sequence = src_output.sequence;
								}
							}
							var owner_address = src_output.address;
							if (arrAuthorAddresses.indexOf(owner_address) === -1)
								return cb("output owner is not among authors");
							if (denomination !== src_output.denomination)
								return cb("denomination mismatch");
							if (objAsset && objAsset.auto_destroy && owner_address === objAsset.definer_address)
								return cb("this output was destroyed by sending it to definer address");
							if (objAsset && objAsset.spender_attested && objAsset.arrAttestedAddresses.indexOf(owner_address) === -1)
								return cb("owner address is not attested");
							if (arrInputAddresses.indexOf(owner_address) === -1)
								arrInputAddresses.push(owner_address);
							total_input += src_output.amount;
							
							if (bStableInParents)
								return checkInputDoubleSpend(cb);

							// the below is for unstable inputs only.
							// when divisible, the asset is also non-transferrable and auto-destroy, 
							// then this transfer is a transfer back to the issuer 
							// and input.unit is known both to payer and the payee (issuer), even if light
							graph.determineIfIncludedOrEqual(conn, input.unit, objUnit.parent_units, function(bIncluded){
								if (!bIncluded)
									return cb("input "+input.unit+" is not in your genes");
								checkInputDoubleSpend(cb);
							});
						}
					);
					break;


				case "headers_commission":
				case "witnessing":
					if (objValidationState.bAA)
						return cb(type+" in AA");
					if (type === "headers_commission"){
						if (bHaveWitnessings)
							return cb("all headers commissions must come before witnessings");
						bHaveHeadersComissions = true;
					}
					else
						bHaveWitnessings = true;
					if (objAsset)
						return cb("only base asset can have "+type);
					if (hasFieldsExcept(input, ["type", "from_main_chain_index", "to_main_chain_index", "address"]))
						return cb("unknown fields in witnessing input");
					if (!isNonnegativeInteger(input.from_main_chain_index))
						return cb("from_main_chain_index must be nonnegative int");
					if (!isNonnegativeInteger(input.to_main_chain_index))
						return cb("to_main_chain_index must be nonnegative int");
					if (input.from_main_chain_index > input.to_main_chain_index)
						return cb("from_main_chain_index > input.to_main_chain_index");
					if (input.to_main_chain_index > objValidationState.last_ball_mci)
						return cb("to_main_chain_index > last_ball_mci");
					if (input.from_main_chain_index > objValidationState.last_ball_mci)
						return cb("from_main_chain_index > last_ball_mci");

					var address = null;
					if (arrAuthorAddresses.length === 1){
						if ("address" in input)
							return cb("when single-authored, must not put address in "+type+" input");
						address = arrAuthorAddresses[0];
					}
					else{
						if (typeof input.address !== "string")
							return cb("when multi-authored, must put address in "+type+" input");
						if (arrAuthorAddresses.indexOf(input.address) === -1)
							return cb(type+" input address "+input.address+" is not an author");
						address = input.address;
					}

					var input_key = type + "-" + address;
					if (objValidationState.arrInputKeys.indexOf(input_key) >= 0)
						return cb("input "+input_key+" already used");
					objValidationState.arrInputKeys.push(input_key);
					
					doubleSpendWhere = "type=? AND from_main_chain_index=? AND address=? AND asset IS NULL";
					doubleSpendVars = [type, input.from_main_chain_index, address];
					if (conf.storage == "mysql")
						doubleSpendIndexMySQL = " USE INDEX (byIndexAddress) ";

					mc_outputs.readNextSpendableMcIndex(conn, type, address, objValidationState.arrConflictingUnits, function(next_spendable_mc_index, bStable){
						if (input.from_main_chain_index < next_spendable_mc_index) { // gaps allowed, in case a unit becomes bad due to another address being nonserial
							const error = type + " ranges must not overlap";
							return cb(bStable ? error : createTransientError(error)); // an unstable blocking unit can still become final-bad
						}
						var max_mci = (type === "headers_commission") 
							? headers_commission.getMaxSpendableMciForLastBallMci(objValidationState.last_ball_mci)
							: paid_witnessing.getMaxSpendableMciForLastBallMci(objValidationState.last_ball_mci);
						if (input.to_main_chain_index > max_mci)
							return cb(type+" to_main_chain_index is too large");

						var calcFunc = (type === "headers_commission") ? mc_outputs.calcEarnings : paid_witnessing.calcWitnessEarnings;
						calcFunc(conn, type, input.from_main_chain_index, input.to_main_chain_index, address, {
							ifError: function(err){
								throw Error(err);
							},
							ifOk: function(commission){
								if (commission === 0)
									return cb("zero "+type+" commission");
								total_input += commission;
								checkInputDoubleSpend(cb);
							}
						});
					});
					break;

				default:
					return cb("unrecognized input type: "+input.type);
			}
		},
		function(err){
			console.log("inputs done "+payload.asset, arrInputAddresses, arrOutputAddresses);
			if (err)
				return callback(err);
			if (total_input > constants.MAX_CAP)
				return callback("total input too large: " + total_input);
			if (objAsset){
				if (total_input !== total_output)
					return callback("inputs and outputs do not balance: "+total_input+" !== "+total_output);
				if (!objAsset.is_transferrable){ // the condition holds for issues too
					if (arrInputAddresses.length === 1 && arrInputAddresses[0] === objAsset.definer_address
					   || arrOutputAddresses.length === 1 && arrOutputAddresses[0] === objAsset.definer_address
						// sending payment to the definer and the change back to oneself
					   || !(objAsset.fixed_denominations && objAsset.is_private) 
							&& arrInputAddresses.length === 1 && arrOutputAddresses.length === 2 
							&& arrOutputAddresses.indexOf(objAsset.definer_address) >= 0
							&& arrOutputAddresses.indexOf(arrInputAddresses[0]) >= 0
					   ){
						// good
					}
					else
						return callback("the asset is not transferrable");
				}
				async.series([
					function(cb){
						if (!objAsset.spender_attested)
							return cb();
						storage.filterAttestedAddresses(
							conn, objAsset, objValidationState.last_ball_mci, arrOutputAddresses, 
							function(arrAttestedOutputAddresses){
								if (arrAttestedOutputAddresses.length !== arrOutputAddresses.length)
									return cb("some output addresses are not attested");
								cb();
							}
						);
					},
					function(cb){
						var arrCondition = bIssue ? objAsset.issue_condition : objAsset.transfer_condition;
						if (!arrCondition)
							return cb();
						Definition.evaluateAssetCondition(
							conn, payload.asset, arrCondition, objUnit, objValidationState, 
							function(cond_err, bSatisfiesCondition){
								if (cond_err)
									return cb(cond_err);
								if (!bSatisfiesCondition)
									return cb("transfer or issue condition not satisfied");
								console.log("validatePaymentInputsAndOutputs with transfer/issue conditions done");
								cb();
							}
						);
					}
				], callback);
			}
			else{ // base asset
				const vote_count_fee = objUnit.messages.find(m => m.app === 'system_vote_count') ? constants.SYSTEM_VOTE_COUNT_FEE : 0;
				const oversize_fee = objUnit.oversize_fee || 0;
				const tps_fee = objUnit.tps_fee || 0;
				const burn_fee = objUnit.burn_fee || 0;
				if (total_input !== total_output + objUnit.headers_commission + objUnit.payload_commission + oversize_fee + tps_fee + burn_fee + vote_count_fee)
					return callback("inputs and outputs do not balance: "+total_input+" !== "+total_output+" + "+objUnit.headers_commission+" + "+objUnit.payload_commission+" + "+oversize_fee+" + "+tps_fee+" + "+burn_fee+" + "+vote_count_fee);
				callback();
			}
		//	console.log("validatePaymentInputsAndOutputs done");
		//	if (objAsset)
		//		profiler2.stop('validate IO');
		//	callback();
		}
	);
}


function initPrivatePaymentValidationState(conn, unit, message_index, payload, onError, onDone){
	conn.query(
		"SELECT payload_hash, app, units.sequence, units.version, units.is_stable, lb_units.main_chain_index AS last_ball_mci, lb_units.timestamp AS last_ball_timestamp \n\
		FROM messages JOIN units USING(unit) \n\
		LEFT JOIN units AS lb_units ON units.last_ball_unit=lb_units.unit \n\
		WHERE messages.unit=? AND message_index=?", 
		[unit, message_index], 
		function(rows){
			if (rows.length > 1)
				throw Error("more than 1 message by index");
			if (rows.length === 0)
				return onError("message not found");
			var row = rows[0];
			if (row.sequence !== "good" && row.is_stable === 1)
				return onError("unit is final nonserial");
			if (row.sequence !== "good") // sender must resend if temp-bad becomes good after stabilization
				return onError("unit is unstable nonserial");
			var bStable = (row.is_stable === 1); // it's ok if the unit is not stable yet
			if (row.app !== "payment")
				return onError("invalid app");
			try{
				if (objectHash.getBase64Hash(payload, row.version !== constants.versionWithoutTimestamp) !== row.payload_hash)
					return onError("payload hash does not match");
			}
			catch(e){
				return onError("failed to calc payload hash: "+e);
			}
			var objValidationState = {
				last_ball_mci: row.last_ball_mci,
				last_ball_timestamp: row.last_ball_timestamp,
				arrDoubleSpendInputs: [],
				arrInputKeys: [],
				bPrivate: true
			};
			var objPartialUnit = {unit: unit};
			storage.readUnitAuthors(conn, unit, function(arrAuthors){
				objPartialUnit.authors = arrAuthors.map(function(address){ return {address: address}; }); // array of objects {address: address}
				// we need parent_units in checkForDoublespends in case it is a doublespend
				conn.query("SELECT parent_unit FROM parenthoods WHERE child_unit=? ORDER BY parent_unit", [unit], function(prows){
					objPartialUnit.parent_units = prows.map(function(prow){ return prow.parent_unit; });
					onDone(bStable, objPartialUnit, objValidationState);
				});
			});
		}
	);
}


function validateAssetDefinition(conn, payload, objUnit, objValidationState, callback){
	if (objUnit.authors.length !== 1)
		return callback("asset definition must be single-authored");
	if (!isNonemptyObject(payload))
		return callback("asset definition must be a non-empty object");
	if (hasFieldsExcept(payload, ["cap", "is_private", "is_transferrable", "auto_destroy", "fixed_denominations", "issued_by_definer_only", "cosigned_by_definer", "spender_attested", "issue_condition", "transfer_condition", "attestors", "denominations"]))
		return callback("unknown fields in asset definition");
	if (typeof payload.is_private !== "boolean" || typeof payload.is_transferrable !== "boolean" || typeof payload.auto_destroy !== "boolean" || typeof payload.fixed_denominations !== "boolean" || typeof payload.issued_by_definer_only !== "boolean" || typeof payload.cosigned_by_definer !== "boolean" || typeof payload.spender_attested !== "boolean")
		return callback("some required fields in asset definition are missing");

	if ("cap" in payload && !(isPositiveInteger(payload.cap) && payload.cap <= constants.MAX_CAP))
		return callback("invalid cap");

	if (objValidationState.bAA) {
		if (payload.cosigned_by_definer !== false)
			return callback("cosigned_by_definer must be false because AAs can't cosign");
		if (payload.issued_by_definer_only === true && (payload.is_private !== false || payload.fixed_denominations !== false))
			return callback("assets issued by AA definer cannot be private or fixed denominations");
	}

	// attestors
	var err;
	if ( payload.spender_attested && (err=checkAttestorList(payload.attestors)) )
		return callback(err);
	if (!payload.spender_attested && "attestors" in payload && (objValidationState.last_ball_mci >= constants.pemCurvesFixMci || !objValidationState.hasBall && storage.getMinRetrievableMci() >= constants.pemCurvesFixMci))
		return callback("attestors should not be defined when spender_attested is false");

	// denominations
	if (payload.fixed_denominations && !isNonemptyArray(payload.denominations))
		return callback("denominations not defined");
	if (!payload.fixed_denominations && "denominations" in payload)
		return callback("denominations should not be defined when fixed");
	if (payload.denominations){
		if (payload.denominations.length > constants.MAX_DENOMINATIONS_PER_ASSET_DEFINITION)
			return callback("too many denominations");
		var total_cap_from_denominations = 0;
		var bHasUncappedDenominations = false;
		var prev_denom = 0;
		for (var i=0; i<payload.denominations.length; i++){
			var denomInfo = payload.denominations[i];
			if (!isNonemptyObject(denomInfo))
				return callback("denomination must be a non-empty object: " + JSON.stringify(denomInfo));
			if (hasFieldsExcept(denomInfo, ["denomination", "count_coins"]))
				return callback("unknown fields in denomination: " + JSON.stringify(denomInfo));
			if (!isPositiveInteger(denomInfo.denomination))
				return callback("invalid denomination");
			if (denomInfo.denomination > constants.MAX_CAP && (objValidationState.last_ball_mci >= constants.pemCurvesFixMci || !objValidationState.hasBall && storage.getMinRetrievableMci() >= constants.pemCurvesFixMci))
				return callback("denomination exceeds MAX_CAP");
			if (denomInfo.denomination <= prev_denom)
				return callback("denominations unsorted");
			if ("count_coins" in denomInfo){
				if (!isPositiveInteger(denomInfo.count_coins))
					return callback("invalid count_coins");
				total_cap_from_denominations += denomInfo.count_coins * denomInfo.denomination;
			}
			else
				bHasUncappedDenominations = true;
			prev_denom = denomInfo.denomination;
		}
		if (bHasUncappedDenominations && total_cap_from_denominations)
			return callback("some denominations are capped, some uncapped");
		if (bHasUncappedDenominations && payload.cap)
			return callback("has cap but some denominations are uncapped");
		if (total_cap_from_denominations && !payload.cap)
			return callback("has no cap but denominations are capped");
		if (total_cap_from_denominations && payload.cap !== total_cap_from_denominations)
			return callback("cap doesn't match sum of denominations");
	}
	
	if (payload.is_private && payload.is_transferrable && !payload.fixed_denominations)
		return callback("if private and transferrable, must have fixed denominations");
	if (payload.is_private && !payload.fixed_denominations){
		if (!(payload.auto_destroy && !payload.is_transferrable))
			return callback("if private and divisible, must also be auto-destroy and non-transferrable");
	}
	if (payload.is_private && ("issue_condition" in payload || "transfer_condition" in payload) && (objValidationState.last_ball_mci >= constants.noPrivateAssetsWithConditionsUpgradeMci || !objValidationState.hasBall && storage.getMinRetrievableMci() >= constants.noPrivateAssetsWithConditionsUpgradeMci))
		return callback("if private, cannot have issue or transfer conditions");
	if (payload.cap && !payload.issued_by_definer_only)
		return callback("if capped, must be issued by definer only");
	
	// possible: definer is like black hole
	//if (!payload.issued_by_definer_only && payload.auto_destroy)
	//    return callback("if issued by anybody, cannot auto-destroy");
	
	// possible: the entire issue should go to the definer
	//if (!payload.issued_by_definer_only && !payload.is_transferrable)
	//    return callback("if issued by anybody, must be transferrable");
	
	objValidationState.bDefiningPrivateAsset = payload.is_private;
	
	async.series([
		function(cb){
			if (!("issue_condition" in payload))
				return cb();
			Definition.validateDefinition(conn, payload.issue_condition, objUnit, objValidationState, null, true, cb);
		},
		function(cb){
			if (!("transfer_condition" in payload))
				return cb();
			Definition.validateDefinition(conn, payload.transfer_condition, objUnit, objValidationState, null, true, cb);
		}
	], callback);
}

function validateAttestorListUpdate(conn, payload, objUnit, objValidationState, callback){
	if (objUnit.authors.length !== 1)
		return callback("attestor list must be single-authored");
	if (!isNonemptyObject(payload))
		return callback("attestor update must be a non-empty object");
	if (hasFieldsExcept(payload, ['asset', 'attestors']))
		return callback("foreign fields in attestor list update");
	storage.readAsset(conn, payload.asset, objValidationState.last_ball_mci, false, function(err, objAsset){
		if (err)
			return callback(err);
		if (!objAsset.spender_attested)
			return callback("this asset does not require attestors");
		if (objUnit.authors[0].address !== objAsset.definer_address)
			return callback("attestor list can be edited only by definer");
		err = checkAttestorList(payload.attestors);
		if (err)
			return callback(err);
		callback();
	});
}

function checkAttestorList(arrAttestors){
	if (!isNonemptyArray(arrAttestors))
		return "attestors not defined";
	if (arrAttestors.length > constants.MAX_ATTESTORS_PER_ASSET)
		return "too many attestors";
	var prev="";
	for (var i=0; i<arrAttestors.length; i++){
		if (!isValidAddress(arrAttestors[i]))
			return "invalid attestor address: "+JSON.stringify(arrAttestors[i]);
		if (arrAttestors[i] <= prev)
			return "attestors not sorted";
		prev = arrAttestors[i];
	}
	return null;
}



function validateAuthorSignaturesWithoutReferences(objAuthor, objUnit, arrAddressDefinition, callback){
	if (!isNonemptyObject(objAuthor.authentifiers))
		return callback("no authentifiers");
	try {
		var unit_hash_to_sign = objectHash.getUnitHashToSign(objUnit);
	}
	catch(e) {
		return callback("failed to calc unit hash to sign: "+e);
	}
	var objValidationState = {
		unit_hash_to_sign,
		last_ball_mci: -1,
		bNoReferences: true
	};
	Definition.validateAuthentifiers(
		null, objAuthor.address, null, arrAddressDefinition, objUnit, objValidationState, objAuthor.authentifiers, 
		function(err, res){
			if (err) // error in address definition
				return callback(err);
			if (!res) // wrong signature or the like
				return callback("authentifier verification failed");
			callback();
		}
	);
}


function createTransientError(err){
	return {
		error_code: "transient", 
		message: err
	};
}

// A problem is with the joint rather than with the unit. That is, the field that has an issue is not covered by unit hash.
function createJointError(err){
	return {
		error_code: "invalid_joint", 
		message: err
	};
}

// the function is here for compatibility only, it'll be eventually removed, use its counterpart in signed_message.js instead
function validateSignedMessage(objSignedMessage, handleResult){
	var signed_message = require('./signed_message.js');
	signed_message.validateSignedMessage(objSignedMessage, handleResult);
}

function validateSignedMessageSync(objSignedMessage){
	var signed_message = require('./signed_message.js');
	return signed_message.validateSignedMessageSync(objSignedMessage);
}

exports.validate = validate;
exports.hasValidHashes = hasValidHashes;
exports.hasValidPayloadHashes = hasValidPayloadHashes;
exports.allHashesAreValid = allHashesAreValid;
exports.validateLight = validateLight;
exports.validateAuthorSignaturesWithoutReferences = validateAuthorSignaturesWithoutReferences;
exports.validatePayment = validatePayment;
exports.initPrivatePaymentValidationState = initPrivatePaymentValidationState;
exports.validateSignedMessage = validateSignedMessage;
exports.validateSignedMessageSync = validateSignedMessageSync;