#include "learn.h"
#include <string.h>

//Back Function Prototype Flashes
BOOL forceBackpropLearning=FALSE;


NSLearningNetwork::NSLearningNetwork()
{
	m_dpDesiredExemplarData = NULL;
	m_cpBestWeightsPathName = new char[255];
	strcpy(m_cpBestWeightsPathName, "");
	m_bCrossValidationEnabled = false;
	m_bSaveBestWeightsEnabled = false;
	m_bSaveBestWeightsForTraining = true;
	m_iUpdateMethod =  1 ;
	m_iCustomExemplarsPerUpdate = 45;
	m_iNumberOfEpochsTrained = 0;
	m_iEpochOfBestCost = 0;
	m_iGradientDescentMethod = 0; //Indicates the type of gradient descent method used on breadboard
	m_iNumberOfConjugateGradientComponents = 0;
	m_fInputNormMin = -9.000000000000000E-01;
	m_fInputNormMax = 9.000000000000000E-01;
	m_bAutoComputeInputNormCoeff = true;
	m_bNormalizeInputByChannel = true;
	m_fOutputNormMin = -9.000000000000000E-01;
	m_fOutputNormMax = 9.000000000000000E-01;
	m_bAutoComputeOutputNormCoeff = true;
	m_bNormalizeOutputByChannel = true;
	m_iSamplesPerExemplar = 50;
	m_iSamplesPerExemplarBackprop = 40;

	m_dpCostData = NULL;
	m_dpCVCostData = NULL;
	onEpochEnded = NULL;
	m_fCurrentCost = 0.0;
	m_fCurrentCVCost = 0.0;
}

int NSLearningNetwork::createNetwork()
{
	NSRecallNetwork::createNetwork();
	return 0;
}

int NSLearningNetwork::createComponents()
{
	NSRecallNetwork::createComponents();
    inputAxonBackprop = new BackAxon;
    hidden1SynapseBackprop = new BackFullSynapse;
    hidden1SynapseBackpropGradient = new Momentum;
    hidden1AxonBackprop = new BackTanhAxon;
    hidden1AxonBackpropGradient = new Momentum;
    hidden2SynapseBackprop = new BackFullSynapse;
    hidden2SynapseBackpropGradient = new Momentum;
    hidden2AxonBackprop = new BackTanhAxon;
    hidden2AxonBackpropGradient = new Momentum;
    hidden3SynapseBackprop = new BackFullSynapse;
    hidden3SynapseBackpropGradient = new Momentum;
    hidden3AxonBackprop = new BackTanhAxon;
    hidden3AxonBackpropGradient = new Momentum;
    hidden4SynapseBackprop = new BackFullSynapse;
    hidden4SynapseBackpropGradient = new Momentum;
    hidden4AxonBackprop = new BackTanhAxon;
    hidden4AxonBackpropGradient = new Momentum;
    hidden5SynapseBackprop = new BackFullSynapse;
    hidden5SynapseBackpropGradient = new Momentum;
    hidden5AxonBackprop = new BackTanhAxon;
    hidden5AxonBackpropGradient = new Momentum;
    outputSynapseBackprop = new BackFullSynapse;
    outputSynapseBackpropGradient = new Momentum;
    outputAxonBackprop = new BackTanhAxon;
    outputAxonBackpropGradient = new Momentum;
    criterion = new L2Criterion;
    criterionBackprop = new BackCriteriaControl;
    controlBackprop = new BackDynamicControl;
    recurrentSynapseBackprop = new BackFullSynapse;
    recurrentSynapseBackpropGradient = new Momentum;
    forwardSynapseBackprop = new BackFullSynapse;
    forwardSynapseBackpropGradient = new Momentum;

	desiredDataFunctionIO = new FunctionIO;
	return 0;
}

int NSLearningNetwork::setInitializations()
{
	NSRecallNetwork::setInitializations();
inputAxonBackprop->setRows(36);
hidden1SynapseBackpropGradient->setDefaultMomentum(6.9999999999999996e-001);
hidden1SynapseBackpropGradient->setDefaultStepSize(1.0000000000000000e-002);
hidden1SynapseBackpropGradient->setStepDivisor(1800);
hidden1AxonBackprop->setOffset(1.0000000000000000e-003);
hidden1AxonBackprop->setTemporalDimension(2);
hidden1AxonBackprop->setRows(72);
hidden1AxonBackpropGradient->setDefaultMomentum(6.9999999999999996e-001);
hidden1AxonBackpropGradient->setDefaultStepSize(1.0000000000000000e-002);
hidden1AxonBackpropGradient->setStepDivisor(1800);
hidden2SynapseBackprop->setTemporalDimension(1);
hidden2SynapseBackpropGradient->setDefaultMomentum(6.9999999999999996e-001);
hidden2SynapseBackpropGradient->setDefaultStepSize(1.0000000000000001e-001);
hidden2SynapseBackpropGradient->setStepDivisor(1800);
hidden2AxonBackprop->setOffset(1.0000000000000000e-003);
hidden2AxonBackprop->setTemporalDimension(2);
hidden2AxonBackprop->setRows(54);
hidden2AxonBackpropGradient->setDefaultMomentum(6.9999999999999996e-001);
hidden2AxonBackpropGradient->setDefaultStepSize(1.0000000000000001e-001);
hidden2AxonBackpropGradient->setStepDivisor(1800);
hidden3SynapseBackpropGradient->setDefaultMomentum(6.9999999999999996e-001);
hidden3SynapseBackpropGradient->setDefaultStepSize(1.0000000000000000e-002);
hidden3SynapseBackpropGradient->setStepDivisor(1800);
hidden3AxonBackprop->setOffset(1.0000000000000000e-003);
hidden3AxonBackprop->setRows(36);
hidden3AxonBackpropGradient->setDefaultMomentum(6.9999999999999996e-001);
hidden3AxonBackpropGradient->setDefaultStepSize(1.0000000000000000e-002);
hidden3AxonBackpropGradient->setStepDivisor(1800);
hidden4SynapseBackpropGradient->setDefaultMomentum(6.9999999999999996e-001);
hidden4SynapseBackpropGradient->setDefaultStepSize(1.0000000000000000e-003);
hidden4SynapseBackpropGradient->setStepDivisor(1800);
hidden4AxonBackprop->setOffset(1.0000000000000000e-003);
hidden4AxonBackprop->setRows(36);
hidden4AxonBackpropGradient->setDefaultMomentum(6.9999999999999996e-001);
hidden4AxonBackpropGradient->setDefaultStepSize(1.0000000000000000e-003);
hidden4AxonBackpropGradient->setStepDivisor(1800);
hidden5SynapseBackpropGradient->setDefaultMomentum(6.9999999999999996e-001);
hidden5SynapseBackpropGradient->setDefaultStepSize(1.0000000000000000e-004);
hidden5SynapseBackpropGradient->setStepDivisor(1800);
hidden5AxonBackprop->setOffset(1.0000000000000000e-003);
hidden5AxonBackprop->setRows(18);
hidden5AxonBackpropGradient->setDefaultMomentum(6.9999999999999996e-001);
hidden5AxonBackpropGradient->setDefaultStepSize(1.0000000000000000e-004);
hidden5AxonBackpropGradient->setStepDivisor(1800);
outputSynapseBackpropGradient->setDefaultMomentum(6.9999999999999996e-001);
outputSynapseBackpropGradient->setDefaultStepSize(1.0000000000000001e-005);
outputSynapseBackpropGradient->setStepDivisor(1800);
outputAxonBackprop->setOffset(1.0000000000000000e-003);
outputAxonBackprop->setRows(2);
outputAxonBackpropGradient->setDefaultMomentum(6.9999999999999996e-001);
outputAxonBackpropGradient->setDefaultStepSize(1.0000000000000001e-005);
outputAxonBackpropGradient->setStepDivisor(1800);
criterion->setAutoSave(TRUE);
criterion->setCheckCostEvery(1);
criterion->setConfusionThreshold(0.0000000000000000e+000);
criterion->setOnIncrease(TRUE);
criterion->setBestCost(1.0000000000000000e+009);
criterion->setTotalNetworkWeights(17750);
criterion->setTemporalDimension(40);
criterion->setRows(2);
criterionBackprop->setRows(2);
controlBackprop->setGradientClassName("Momentum");
recurrentSynapseBackprop->setTemporalDimension(1);
recurrentSynapseBackpropGradient->setDefaultMomentum(6.9999999999999996e-001);
recurrentSynapseBackpropGradient->setDefaultStepSize(1.0000000000000001e-001);
recurrentSynapseBackpropGradient->setStepDivisor(1800);
forwardSynapseBackpropGradient->setDefaultMomentum(6.9999999999999996e-001);
forwardSynapseBackpropGradient->setDefaultStepSize(1.0000000000000001e-001);
forwardSynapseBackpropGradient->setStepDivisor(1800);
	desiredDataFunctionIO->setFunction((void*)setFunctionIO, (void*)this);
	return 0;
}

void setFunctionIO(NSFloat *data, int rows, int cols, void *aNN)
{
	if (aNN)
		((NSLearningNetwork *)aNN)->setDesiredFunctionIO(data);
}





int NSLearningNetwork::setInterconnections()
{
	NSRecallNetwork::setInterconnections();

inputAxonBackprop->setDual(inputAxon);
hidden1SynapseBackprop->setDual(hidden1Synapse);
hidden1AxonBackprop->setDual(hidden1Axon);
hidden2SynapseBackprop->setDual(hidden2Synapse);
hidden2AxonBackprop->setDual(hidden2Axon);
hidden3SynapseBackprop->setDual(hidden3Synapse);
hidden3AxonBackprop->setDual(hidden3Axon);
hidden4SynapseBackprop->setDual(hidden4Synapse);
hidden4AxonBackprop->setDual(hidden4Axon);
hidden5SynapseBackprop->setDual(hidden5Synapse);
hidden5AxonBackprop->setDual(hidden5Axon);
outputSynapseBackprop->setDual(outputSynapse);
outputAxonBackprop->setDual(outputAxon);
criterionBackprop->setDual(criterion);
recurrentSynapseBackprop->setDual(recurrentSynapse);
forwardSynapseBackprop->setDual(forwardSynapse);

inputAxonBackprop->setLast(hidden1SynapseBackprop);
inputAxonBackprop->setLast(forwardSynapseBackprop);
hidden1SynapseBackprop->setLast(hidden1AxonBackprop);
hidden1SynapseBackprop->setNext(inputAxonBackprop);
hidden1AxonBackprop->setLast(hidden2SynapseBackprop);
hidden1AxonBackprop->setLast(recurrentSynapseBackprop);
hidden1AxonBackprop->setNext(hidden1SynapseBackprop);
hidden1AxonBackprop->setNext(recurrentSynapseBackprop);
hidden2SynapseBackprop->setLast(hidden2AxonBackprop);
hidden2SynapseBackprop->setNext(hidden1AxonBackprop);
hidden2AxonBackprop->setLast(hidden3SynapseBackprop);
hidden2AxonBackprop->setNext(hidden2SynapseBackprop);
hidden2AxonBackprop->setNext(forwardSynapseBackprop);
hidden3SynapseBackprop->setLast(hidden3AxonBackprop);
hidden3SynapseBackprop->setNext(hidden2AxonBackprop);
hidden3AxonBackprop->setLast(hidden4SynapseBackprop);
hidden3AxonBackprop->setNext(hidden3SynapseBackprop);
hidden4SynapseBackprop->setLast(hidden4AxonBackprop);
hidden4SynapseBackprop->setNext(hidden3AxonBackprop);
hidden4AxonBackprop->setLast(hidden5SynapseBackprop);
hidden4AxonBackprop->setNext(hidden4SynapseBackprop);
hidden5SynapseBackprop->setLast(hidden5AxonBackprop);
hidden5SynapseBackprop->setNext(hidden4AxonBackprop);
hidden5AxonBackprop->setLast(outputSynapseBackprop);
hidden5AxonBackprop->setNext(hidden5SynapseBackprop);
outputSynapseBackprop->setLast(outputAxonBackprop);
outputSynapseBackprop->setNext(hidden5AxonBackprop);
outputAxonBackprop->setLast(criterionBackprop);
outputAxonBackprop->setNext(outputSynapseBackprop);
criterion->setLast(outputAxon);
criterionBackprop->setNext(outputAxonBackprop);
recurrentSynapseBackprop->setLast(hidden1AxonBackprop);
recurrentSynapseBackprop->setNext(hidden1AxonBackprop);
forwardSynapseBackprop->setLast(hidden2AxonBackprop);
forwardSynapseBackprop->setNext(inputAxonBackprop);
outputAxon->setNext(criterion);
hidden1SynapseBackpropGradient->setErrorSoma(hidden1SynapseBackprop);
hidden1SynapseBackpropGradient->setForwardSoma(hidden1Synapse);
hidden1SynapseBackpropGradient->setIndividualSteps(FALSE);
hidden1AxonBackpropGradient->setErrorSoma(hidden1AxonBackprop);
hidden1AxonBackpropGradient->setForwardSoma(hidden1Axon);
hidden1AxonBackpropGradient->setIndividualSteps(FALSE);
hidden2SynapseBackpropGradient->setErrorSoma(hidden2SynapseBackprop);
hidden2SynapseBackpropGradient->setForwardSoma(hidden2Synapse);
hidden2SynapseBackpropGradient->setIndividualSteps(FALSE);
hidden2AxonBackpropGradient->setErrorSoma(hidden2AxonBackprop);
hidden2AxonBackpropGradient->setForwardSoma(hidden2Axon);
hidden2AxonBackpropGradient->setIndividualSteps(FALSE);
hidden3SynapseBackpropGradient->setErrorSoma(hidden3SynapseBackprop);
hidden3SynapseBackpropGradient->setForwardSoma(hidden3Synapse);
hidden3SynapseBackpropGradient->setIndividualSteps(FALSE);
hidden3AxonBackpropGradient->setErrorSoma(hidden3AxonBackprop);
hidden3AxonBackpropGradient->setForwardSoma(hidden3Axon);
hidden3AxonBackpropGradient->setIndividualSteps(FALSE);
hidden4SynapseBackpropGradient->setErrorSoma(hidden4SynapseBackprop);
hidden4SynapseBackpropGradient->setForwardSoma(hidden4Synapse);
hidden4SynapseBackpropGradient->setIndividualSteps(FALSE);
hidden4AxonBackpropGradient->setErrorSoma(hidden4AxonBackprop);
hidden4AxonBackpropGradient->setForwardSoma(hidden4Axon);
hidden4AxonBackpropGradient->setIndividualSteps(FALSE);
hidden5SynapseBackpropGradient->setErrorSoma(hidden5SynapseBackprop);
hidden5SynapseBackpropGradient->setForwardSoma(hidden5Synapse);
hidden5SynapseBackpropGradient->setIndividualSteps(FALSE);
hidden5AxonBackpropGradient->setErrorSoma(hidden5AxonBackprop);
hidden5AxonBackpropGradient->setForwardSoma(hidden5Axon);
hidden5AxonBackpropGradient->setIndividualSteps(FALSE);
outputSynapseBackpropGradient->setErrorSoma(outputSynapseBackprop);
outputSynapseBackpropGradient->setForwardSoma(outputSynapse);
outputSynapseBackpropGradient->setIndividualSteps(FALSE);
outputAxonBackpropGradient->setErrorSoma(outputAxonBackprop);
outputAxonBackpropGradient->setForwardSoma(outputAxon);
outputAxonBackpropGradient->setIndividualSteps(FALSE);
recurrentSynapseBackpropGradient->setErrorSoma(recurrentSynapseBackprop);
recurrentSynapseBackpropGradient->setForwardSoma(recurrentSynapse);
recurrentSynapseBackpropGradient->setIndividualSteps(FALSE);
forwardSynapseBackpropGradient->setErrorSoma(forwardSynapseBackprop);
forwardSynapseBackpropGradient->setForwardSoma(forwardSynapse);
forwardSynapseBackpropGradient->setIndividualSteps(FALSE);
criterion->setDesiredAccess(desiredDataFunctionIO);

	return 0;
}

int NSLearningNetwork::postInterconnections()
{
control->prepareToFire(controlBackprop);

	preRunGetReady();
control->preFireGetReady(TRUE);

	postFireGetReady();

	return 0;
}

int NSLearningNetwork::preRunGetReady()
{
	NSRecallNetwork::preRunGetReady();

inputAxonBackprop->setStaticControl(control);
inputAxonBackprop->preFireGetReady();
hidden1SynapseBackprop->setStaticControl(control);
hidden1SynapseBackprop->preFireGetReady();
hidden1SynapseBackpropGradient->setStaticControl(control);
hidden1SynapseBackpropGradient->preFireGetReady();
hidden1AxonBackprop->setStaticControl(control);
hidden1AxonBackprop->preFireGetReady();
hidden1AxonBackpropGradient->setStaticControl(control);
hidden1AxonBackpropGradient->preFireGetReady();
hidden2SynapseBackprop->setStaticControl(control);
hidden2SynapseBackprop->preFireGetReady();
hidden2SynapseBackpropGradient->setStaticControl(control);
hidden2SynapseBackpropGradient->preFireGetReady();
hidden2AxonBackprop->setStaticControl(control);
hidden2AxonBackprop->preFireGetReady();
hidden2AxonBackpropGradient->setStaticControl(control);
hidden2AxonBackpropGradient->preFireGetReady();
hidden3SynapseBackprop->setStaticControl(control);
hidden3SynapseBackprop->preFireGetReady();
hidden3SynapseBackpropGradient->setStaticControl(control);
hidden3SynapseBackpropGradient->preFireGetReady();
hidden3AxonBackprop->setStaticControl(control);
hidden3AxonBackprop->preFireGetReady();
hidden3AxonBackpropGradient->setStaticControl(control);
hidden3AxonBackpropGradient->preFireGetReady();
hidden4SynapseBackprop->setStaticControl(control);
hidden4SynapseBackprop->preFireGetReady();
hidden4SynapseBackpropGradient->setStaticControl(control);
hidden4SynapseBackpropGradient->preFireGetReady();
hidden4AxonBackprop->setStaticControl(control);
hidden4AxonBackprop->preFireGetReady();
hidden4AxonBackpropGradient->setStaticControl(control);
hidden4AxonBackpropGradient->preFireGetReady();
hidden5SynapseBackprop->setStaticControl(control);
hidden5SynapseBackprop->preFireGetReady();
hidden5SynapseBackpropGradient->setStaticControl(control);
hidden5SynapseBackpropGradient->preFireGetReady();
hidden5AxonBackprop->setStaticControl(control);
hidden5AxonBackprop->preFireGetReady();
hidden5AxonBackpropGradient->setStaticControl(control);
hidden5AxonBackpropGradient->preFireGetReady();
outputSynapseBackprop->setStaticControl(control);
outputSynapseBackprop->preFireGetReady();
outputSynapseBackpropGradient->setStaticControl(control);
outputSynapseBackpropGradient->preFireGetReady();
outputAxonBackprop->setStaticControl(control);
outputAxonBackprop->preFireGetReady();
outputAxonBackpropGradient->setStaticControl(control);
outputAxonBackpropGradient->preFireGetReady();
criterion->setStaticControl(control);
criterion->preFireGetReady();
criterionBackprop->setStaticControl(control);
criterionBackprop->preFireGetReady();
controlBackprop->setStaticControl(control);
controlBackprop->preFireGetReady();
recurrentSynapseBackprop->setStaticControl(control);
recurrentSynapseBackprop->preFireGetReady();
recurrentSynapseBackpropGradient->setStaticControl(control);
recurrentSynapseBackpropGradient->preFireGetReady();
forwardSynapseBackprop->setStaticControl(control);
forwardSynapseBackprop->preFireGetReady();
forwardSynapseBackpropGradient->setStaticControl(control);
forwardSynapseBackpropGradient->preFireGetReady();


	return 0;
}

int NSLearningNetwork::loadComponentWeights(FILE *&weightsFile)
{
	int lcw_errorcode = 0;

	if(lcw_errorcode = NSRecallNetwork::loadComponentWeights(weightsFile))
		return lcw_errorcode;

	if(!seekComponentReturnNull(weightsFile, "BackFullSynapse", "hidden1SynapseBackprop"))
		lcw_errorcode = 1;
	else
		hidden1SynapseBackprop->loadWeights(weightsFile, m_usiWeightFileVersion);

	if (hidden1SynapseBackpropGradient->isWeightedComponent())
	{
		if(!seekComponentReturnNull(weightsFile,hidden1SynapseBackpropGradient->className(), "hidden1SynapseBackpropGradient"))
			lcw_errorcode = 1;
		else
			hidden1SynapseBackpropGradient->loadWeights(weightsFile, m_usiWeightFileVersion);
	}

	if (hidden1AxonBackpropGradient->isWeightedComponent())
	{
		if(!seekComponentReturnNull(weightsFile,hidden1AxonBackpropGradient->className(), "hidden1AxonBackpropGradient"))
			lcw_errorcode = 1;
		else
			hidden1AxonBackpropGradient->loadWeights(weightsFile, m_usiWeightFileVersion);
	}

	if(!seekComponentReturnNull(weightsFile, "BackFullSynapse", "hidden2SynapseBackprop"))
		lcw_errorcode = 1;
	else
		hidden2SynapseBackprop->loadWeights(weightsFile, m_usiWeightFileVersion);

	if (hidden2SynapseBackpropGradient->isWeightedComponent())
	{
		if(!seekComponentReturnNull(weightsFile,hidden2SynapseBackpropGradient->className(), "hidden2SynapseBackpropGradient"))
			lcw_errorcode = 1;
		else
			hidden2SynapseBackpropGradient->loadWeights(weightsFile, m_usiWeightFileVersion);
	}

	if (hidden2AxonBackpropGradient->isWeightedComponent())
	{
		if(!seekComponentReturnNull(weightsFile,hidden2AxonBackpropGradient->className(), "hidden2AxonBackpropGradient"))
			lcw_errorcode = 1;
		else
			hidden2AxonBackpropGradient->loadWeights(weightsFile, m_usiWeightFileVersion);
	}

	if(!seekComponentReturnNull(weightsFile, "BackFullSynapse", "hidden3SynapseBackprop"))
		lcw_errorcode = 1;
	else
		hidden3SynapseBackprop->loadWeights(weightsFile, m_usiWeightFileVersion);

	if (hidden3SynapseBackpropGradient->isWeightedComponent())
	{
		if(!seekComponentReturnNull(weightsFile,hidden3SynapseBackpropGradient->className(), "hidden3SynapseBackpropGradient"))
			lcw_errorcode = 1;
		else
			hidden3SynapseBackpropGradient->loadWeights(weightsFile, m_usiWeightFileVersion);
	}

	if (hidden3AxonBackpropGradient->isWeightedComponent())
	{
		if(!seekComponentReturnNull(weightsFile,hidden3AxonBackpropGradient->className(), "hidden3AxonBackpropGradient"))
			lcw_errorcode = 1;
		else
			hidden3AxonBackpropGradient->loadWeights(weightsFile, m_usiWeightFileVersion);
	}

	if(!seekComponentReturnNull(weightsFile, "BackFullSynapse", "hidden4SynapseBackprop"))
		lcw_errorcode = 1;
	else
		hidden4SynapseBackprop->loadWeights(weightsFile, m_usiWeightFileVersion);

	if (hidden4SynapseBackpropGradient->isWeightedComponent())
	{
		if(!seekComponentReturnNull(weightsFile,hidden4SynapseBackpropGradient->className(), "hidden4SynapseBackpropGradient"))
			lcw_errorcode = 1;
		else
			hidden4SynapseBackpropGradient->loadWeights(weightsFile, m_usiWeightFileVersion);
	}

	if (hidden4AxonBackpropGradient->isWeightedComponent())
	{
		if(!seekComponentReturnNull(weightsFile,hidden4AxonBackpropGradient->className(), "hidden4AxonBackpropGradient"))
			lcw_errorcode = 1;
		else
			hidden4AxonBackpropGradient->loadWeights(weightsFile, m_usiWeightFileVersion);
	}

	if(!seekComponentReturnNull(weightsFile, "BackFullSynapse", "hidden5SynapseBackprop"))
		lcw_errorcode = 1;
	else
		hidden5SynapseBackprop->loadWeights(weightsFile, m_usiWeightFileVersion);

	if (hidden5SynapseBackpropGradient->isWeightedComponent())
	{
		if(!seekComponentReturnNull(weightsFile,hidden5SynapseBackpropGradient->className(), "hidden5SynapseBackpropGradient"))
			lcw_errorcode = 1;
		else
			hidden5SynapseBackpropGradient->loadWeights(weightsFile, m_usiWeightFileVersion);
	}

	if (hidden5AxonBackpropGradient->isWeightedComponent())
	{
		if(!seekComponentReturnNull(weightsFile,hidden5AxonBackpropGradient->className(), "hidden5AxonBackpropGradient"))
			lcw_errorcode = 1;
		else
			hidden5AxonBackpropGradient->loadWeights(weightsFile, m_usiWeightFileVersion);
	}

	if(!seekComponentReturnNull(weightsFile, "BackFullSynapse", "outputSynapseBackprop"))
		lcw_errorcode = 1;
	else
		outputSynapseBackprop->loadWeights(weightsFile, m_usiWeightFileVersion);

	if (outputSynapseBackpropGradient->isWeightedComponent())
	{
		if(!seekComponentReturnNull(weightsFile,outputSynapseBackpropGradient->className(), "outputSynapseBackpropGradient"))
			lcw_errorcode = 1;
		else
			outputSynapseBackpropGradient->loadWeights(weightsFile, m_usiWeightFileVersion);
	}

	if (outputAxonBackpropGradient->isWeightedComponent())
	{
		if(!seekComponentReturnNull(weightsFile,outputAxonBackpropGradient->className(), "outputAxonBackpropGradient"))
			lcw_errorcode = 1;
		else
			outputAxonBackpropGradient->loadWeights(weightsFile, m_usiWeightFileVersion);
	}

	if(!seekComponentReturnNull(weightsFile, "L2Criterion", "criterion"))
		lcw_errorcode = 1;
	else
		criterion->loadWeights(weightsFile, m_usiWeightFileVersion);

	if(!seekComponentReturnNull(weightsFile, "BackFullSynapse", "recurrentSynapseBackprop"))
		lcw_errorcode = 1;
	else
		recurrentSynapseBackprop->loadWeights(weightsFile, m_usiWeightFileVersion);

	if (recurrentSynapseBackpropGradient->isWeightedComponent())
	{
		if(!seekComponentReturnNull(weightsFile,recurrentSynapseBackpropGradient->className(), "recurrentSynapseBackpropGradient"))
			lcw_errorcode = 1;
		else
			recurrentSynapseBackpropGradient->loadWeights(weightsFile, m_usiWeightFileVersion);
	}

	if(!seekComponentReturnNull(weightsFile, "BackFullSynapse", "forwardSynapseBackprop"))
		lcw_errorcode = 1;
	else
		forwardSynapseBackprop->loadWeights(weightsFile, m_usiWeightFileVersion);

	if (forwardSynapseBackpropGradient->isWeightedComponent())
	{
		if(!seekComponentReturnNull(weightsFile,forwardSynapseBackpropGradient->className(), "forwardSynapseBackpropGradient"))
			lcw_errorcode = 1;
		else
			forwardSynapseBackpropGradient->loadWeights(weightsFile, m_usiWeightFileVersion);
	}


	return lcw_errorcode;
}

int NSLearningNetwork::saveComponentWeights(FILE *&weightsFile)
{
	NSRecallNetwork::saveComponentWeights(weightsFile);
	hidden1SynapseBackprop->saveWeights(putComponent(weightsFile, "BackFullSynapse", "hidden1SynapseBackprop"));
	hidden1SynapseBackpropGradient->saveWeights(putComponent(weightsFile,hidden1SynapseBackpropGradient->className(), "hidden1SynapseBackpropGradient"));
	hidden1AxonBackpropGradient->saveWeights(putComponent(weightsFile,hidden1AxonBackpropGradient->className(), "hidden1AxonBackpropGradient"));
	hidden2SynapseBackprop->saveWeights(putComponent(weightsFile, "BackFullSynapse", "hidden2SynapseBackprop"));
	hidden2SynapseBackpropGradient->saveWeights(putComponent(weightsFile,hidden2SynapseBackpropGradient->className(), "hidden2SynapseBackpropGradient"));
	hidden2AxonBackpropGradient->saveWeights(putComponent(weightsFile,hidden2AxonBackpropGradient->className(), "hidden2AxonBackpropGradient"));
	hidden3SynapseBackprop->saveWeights(putComponent(weightsFile, "BackFullSynapse", "hidden3SynapseBackprop"));
	hidden3SynapseBackpropGradient->saveWeights(putComponent(weightsFile,hidden3SynapseBackpropGradient->className(), "hidden3SynapseBackpropGradient"));
	hidden3AxonBackpropGradient->saveWeights(putComponent(weightsFile,hidden3AxonBackpropGradient->className(), "hidden3AxonBackpropGradient"));
	hidden4SynapseBackprop->saveWeights(putComponent(weightsFile, "BackFullSynapse", "hidden4SynapseBackprop"));
	hidden4SynapseBackpropGradient->saveWeights(putComponent(weightsFile,hidden4SynapseBackpropGradient->className(), "hidden4SynapseBackpropGradient"));
	hidden4AxonBackpropGradient->saveWeights(putComponent(weightsFile,hidden4AxonBackpropGradient->className(), "hidden4AxonBackpropGradient"));
	hidden5SynapseBackprop->saveWeights(putComponent(weightsFile, "BackFullSynapse", "hidden5SynapseBackprop"));
	hidden5SynapseBackpropGradient->saveWeights(putComponent(weightsFile,hidden5SynapseBackpropGradient->className(), "hidden5SynapseBackpropGradient"));
	hidden5AxonBackpropGradient->saveWeights(putComponent(weightsFile,hidden5AxonBackpropGradient->className(), "hidden5AxonBackpropGradient"));
	outputSynapseBackprop->saveWeights(putComponent(weightsFile, "BackFullSynapse", "outputSynapseBackprop"));
	outputSynapseBackpropGradient->saveWeights(putComponent(weightsFile,outputSynapseBackpropGradient->className(), "outputSynapseBackpropGradient"));
	outputAxonBackpropGradient->saveWeights(putComponent(weightsFile,outputAxonBackpropGradient->className(), "outputAxonBackpropGradient"));
	criterion->saveWeights(putComponent(weightsFile, "L2Criterion", "criterion"));
	recurrentSynapseBackprop->saveWeights(putComponent(weightsFile, "BackFullSynapse", "recurrentSynapseBackprop"));
	recurrentSynapseBackpropGradient->saveWeights(putComponent(weightsFile,recurrentSynapseBackpropGradient->className(), "recurrentSynapseBackpropGradient"));
	forwardSynapseBackprop->saveWeights(putComponent(weightsFile, "BackFullSynapse", "forwardSynapseBackprop"));
	forwardSynapseBackpropGradient->saveWeights(putComponent(weightsFile,forwardSynapseBackpropGradient->className(), "forwardSynapseBackpropGradient"));


	return 0;
}

int NSLearningNetwork::getResponse(int exemplars, double *inputData, double *outputData)
{
	m_dpDesiredExemplarData = NULL;
	return NSRecallNetwork::getResponse(exemplars, inputData, outputData);
}

int NSLearningNetwork::getSensitivity(int exemplars, double *inputData, double *&sensitivityData, double dither)
{
	m_dpDesiredExemplarData = NULL;
	return NSRecallNetwork::getSensitivity(exemplars, inputData, sensitivityData, dither);
}

int NSLearningNetwork::getInput(double *inputData, double *desiredData, double deltaThreshhold, double *newInputData)
{
	//Set pointer to desired output
	m_dpDesiredExemplarData = desiredData;

	//Initialize inputs
	for (int pe = 0; pe < m_iNumberOfInputs; pe++)
		newInputData[pe] = inputData[pe];

	//Loop (updating the inputs) until the sum of the absolute value of the input
	//deltas is less than deltaThreshhold or the maxIteration has been reached
	double sumOfAbsOfInputDeltas = 0;
	int maxIteration = 10000;
	int iterationNum = 0;
	do 
	{
		setInput(0, newInputData);
		control->fireEngines();

		startBackprop();
		recurrentSynapseBackprop->fire();
hidden2SynapseBackprop->fire();
criterionBackprop->fire();

		completeBackprop();

		//Update inputs
		sumOfAbsOfInputDeltas = 0;
		inputAxonBackprop->decrement();

		double inputDelta;
		if (m_ipInputDataMap)
		{
			for (int pe = 0; pe < m_iNumberOfInputsAfterMask; pe++)
			{
				inputDelta = 0.1 * inputAxonBackprop->V(pe);
				newInputData[m_ipInputDataMap[pe]] += inputDelta;
				sumOfAbsOfInputDeltas += (double) fabs(inputDelta);
			}
		}
		else
		{
			for (int pe = 0; pe < m_iNumberOfInputs; pe++)
			{
				inputDelta = 0.1 * inputAxonBackprop->V(pe);
				newInputData[pe] += inputDelta;
				sumOfAbsOfInputDeltas += (double) fabs(inputDelta);
			}
		}

		inputAxonBackprop->increment();


		iterationNum += 1;

	} while ((sumOfAbsOfInputDeltas > deltaThreshhold) && (iterationNum < maxIteration));

	return 0;
}

int NSLearningNetwork::train(int epochs, int exemplars, double *inputData, double *desiredData, int CVExemplars, double *CVInputData, double *CVDesiredData)
{
	char note[255];
	char number[10];
	int saveWeightsReturnValue = 0;
	int exemplarsPerUpdate;
	int epochsPerCV = 1;
	int saveWeightsNow;


	if (m_bAutoComputeInputNormCoeff)
		computeInputNormCoeff(inputData, exemplars);

	if (m_bAutoComputeOutputNormCoeff)
		computeOutputNormCoeff(desiredData, exemplars);


	int samples = m_iSamplesPerExemplar;
	int samplesBackprop = m_iSamplesPerExemplarBackprop;


	if (exemplars % samples == 0)
		exemplars = exemplars / samples;
	else
		return 3;


	criterion->setCheckCostEvery(epochs);

	switch(m_iUpdateMethod)
	{
	case ONLINE:
		exemplarsPerUpdate = 1;
		break;
	case BATCH:
		exemplarsPerUpdate = exemplars;
		break;
	case CUSTOM:
		exemplarsPerUpdate = m_iCustomExemplarsPerUpdate;
		break;
	}


	setDefaultStepSizeDivisor(exemplarsPerUpdate * samplesBackprop);


	//Allocate memory for cost data and cross validation cost data
	if (m_dpCostData)
		delete m_dpCostData;
	m_dpCostData = new double[epochs];


	if (m_dpCVCostData)
		delete m_dpCVCostData;
	if (m_bCrossValidationEnabled)
		m_dpCVCostData = new double[epochs];
	else
		m_dpCVCostData = NULL;


	m_iNumberOfEpochsTrained = 0;




	control->prepareToFire(controlBackprop);

	preRunGetReady();
	control->preFireGetReady(TRUE);

	postFireGetReady();
	hidden1SynapseBackpropGradient->fireGetReady();
	hidden1AxonBackpropGradient->fireGetReady();
	hidden2SynapseBackpropGradient->fireGetReady();
	hidden2AxonBackpropGradient->fireGetReady();
	hidden3SynapseBackpropGradient->fireGetReady();
	hidden3AxonBackpropGradient->fireGetReady();
	hidden4SynapseBackpropGradient->fireGetReady();
	hidden4AxonBackpropGradient->fireGetReady();
	hidden5SynapseBackpropGradient->fireGetReady();
	hidden5AxonBackpropGradient->fireGetReady();
	outputSynapseBackpropGradient->fireGetReady();
	outputAxonBackpropGradient->fireGetReady();
	criterion->fireGetReady();
int updateCounter=0;
	recurrentSynapseBackpropGradient->fireGetReady();
	forwardSynapseBackpropGradient->fireGetReady();



control->setBackpropLearningOn(TRUE);
control->setActiveNumberOfExemplars(exemplars);
control->postFireGetReady();
control->setEpochsPerTest(epochsPerCV);
control->setEpochs(epochs);
control->setExemplars(exemplars);
control->setEpochCounter(0);
for (; control->epochCounter()<control->epochs();) { 
int phases = control->backpropLearningOn() ? control->phases() : 1;
control->setPhaseCounter(0);
while (control->phaseCounter()<phases) { 
m_dpDesiredExemplarData = desiredData;
control->setExemplarCounter(0);
for (; control->exemplarCounter()<control->exemplars(); control->incrementExemplarCounter()) { 
control->setSampleCounter(0);
for (; control->sampleCounter()<samples; control->incrementSampleCounter()) { 
setInput((control->exemplarCounter() * samples) + control->sampleCounter(), inputData);
control->fireEngines();
m_dpDesiredExemplarData += m_iNumberOfOutputs;
}
if (control->backpropLearningOn() || forceBackpropLearning) {
inputAxonBackprop->backpropStarting();
hidden1SynapseBackprop->backpropStarting();
hidden1AxonBackprop->backpropStarting();
hidden2SynapseBackprop->backpropStarting();
hidden2AxonBackprop->backpropStarting();
hidden3SynapseBackprop->backpropStarting();
hidden3AxonBackprop->backpropStarting();
hidden4SynapseBackprop->backpropStarting();
hidden4AxonBackprop->backpropStarting();
hidden5SynapseBackprop->backpropStarting();
hidden5AxonBackprop->backpropStarting();
outputSynapseBackprop->backpropStarting();
outputAxonBackprop->backpropStarting();
criterionBackprop->backpropStarting();
recurrentSynapseBackprop->backpropStarting();
forwardSynapseBackprop->backpropStarting();
for (int backOffset=0; backOffset<samplesBackprop; backOffset++) {
	recurrentSynapseBackprop->fire();
	hidden2SynapseBackprop->fire();
	criterionBackprop->fire();
}
inputAxonBackprop->backpropComplete();
hidden1SynapseBackprop->backpropComplete();
hidden1AxonBackprop->backpropComplete();
hidden2SynapseBackprop->backpropComplete();
hidden2AxonBackprop->backpropComplete();
hidden3SynapseBackprop->backpropComplete();
hidden3AxonBackprop->backpropComplete();
hidden4SynapseBackprop->backpropComplete();
hidden4AxonBackprop->backpropComplete();
hidden5SynapseBackprop->backpropComplete();
hidden5AxonBackprop->backpropComplete();
outputSynapseBackprop->backpropComplete();
outputAxonBackprop->backpropComplete();
criterionBackprop->backpropComplete();
recurrentSynapseBackprop->backpropComplete();
forwardSynapseBackprop->backpropComplete();
control->backpropComplete();
if (++updateCounter >= exemplarsPerUpdate) {
control->updateNetworkWeights();
hidden1SynapseBackpropGradient->updateWeights(hidden1Synapse);
hidden1AxonBackpropGradient->updateWeights(hidden1Axon);
hidden2SynapseBackpropGradient->updateWeights(hidden2Synapse);
hidden2AxonBackpropGradient->updateWeights(hidden2Axon);
hidden3SynapseBackpropGradient->updateWeights(hidden3Synapse);
hidden3AxonBackpropGradient->updateWeights(hidden3Axon);
hidden4SynapseBackpropGradient->updateWeights(hidden4Synapse);
hidden4AxonBackpropGradient->updateWeights(hidden4Axon);
hidden5SynapseBackpropGradient->updateWeights(hidden5Synapse);
hidden5AxonBackpropGradient->updateWeights(hidden5Axon);
outputSynapseBackpropGradient->updateWeights(outputSynapse);
outputAxonBackpropGradient->updateWeights(outputAxon);
recurrentSynapseBackpropGradient->updateWeights(recurrentSynapse);
forwardSynapseBackpropGradient->updateWeights(forwardSynapse);
control->networkWeightsUpdated();
updateCounter = 0;
}
forceBackpropLearning = FALSE;
}
if (control->networkPaused()) {
	criterion->reportCost();
	goto ConcludeFiring;
}
}
control->incrementPhaseCounter();
if (control->phaseCounter() < phases) {
if (control->phases() > 1) {
	control->phaseEnded(control->phaseCounter());
}
}
}
control->incrementEpochCounter();



		//Store learning curve and keep track of number of epochs trained.  Exclude unsupervised epochs (cost = 0.0)
		m_fCurrentCost = criterion->reportCost();
		criterion->m_bIgnoreNextError = FALSE;
		if (m_fCurrentCost != 0.0)
		{
			m_dpCostData[m_iNumberOfEpochsTrained] = m_fCurrentCost;
			m_iNumberOfEpochsTrained++;
		}


		if ((m_bSaveBestWeightsForTraining) || (!m_bCrossValidationEnabled))
		{
			//Updates best training cost and returns whether or not weights need to be saved
			saveWeightsNow = criterion->checkForSaveBest(m_fCurrentCost, FALSE);
			if (saveWeightsNow)
			{
				m_iEpochOfBestCost = m_iNumberOfEpochsTrained;
				if (m_bSaveBestWeightsEnabled)
				{
					strcpy(note, "// Saved During Training, Epoch ");
					_itoa(m_iNumberOfEpochsTrained, number, 10);
					strcat(note, number);
					saveWeightsReturnValue = saveWeights(m_cpBestWeightsPathName, note);
				}
			}
		}

		if ((m_bCrossValidationEnabled) && (control->epochCounter() % epochsPerCV == 0) && (control->backpropLearningOn()))
		{
			m_dpDesiredExemplarData = CVDesiredData;
			startTesting();
			for (int testExemplar = 0; testExemplar < CVExemplars; testExemplar++)
			{
				setInput(testExemplar, CVInputData);
				control->fireEngines();

				m_dpDesiredExemplarData += m_iNumberOfOutputs;
			}
			m_fCurrentCVCost = criterion->reportCost(); //Must be placed before endTesting() because endTesting() zeros the costCounter which results in criterion->reportCost() returning zero if called after endTesting()
			if (m_iNumberOfEpochsTrained > 0)
				m_dpCVCostData[m_iNumberOfEpochsTrained - 1] = m_fCurrentCVCost;
			endTesting();

			if (!m_bSaveBestWeightsForTraining)
			{
				//Updates best cross validation cost and returns whether or not weights need to be saved
				saveWeightsNow = criterion->checkForSaveBest(m_fCurrentCVCost, FALSE);
				if (saveWeightsNow)
				{
					m_iEpochOfBestCost = m_iNumberOfEpochsTrained;
					if (m_bSaveBestWeightsEnabled)
					{
						strcpy(note, "// Saved During Cross Validation, Epoch ");
						_itoa(m_iNumberOfEpochsTrained, number, 10);
						strcat(note, number);
						saveWeightsReturnValue = saveWeights(m_cpBestWeightsPathName, note);
					}
				}
			}
		}
criterion->epochEnded();
control->epochEnded();
if ((control->phases() > 1) && control->backpropLearningOn()) {
}
if (control->networkPaused())
	goto ConcludeFiring;


		if (onEpochEnded)
			if (onEpochEnded((void *) this))
				goto ConcludeFiring;
	}

	if (epochs <= 0)
		return 4;

	ConcludeFiring:
	control->setNetworkPaused(FALSE);
	fireConclude();
	saveWeightsNow = criterion->checkForSaveBest(criterion->lastUncheckedCost(), TRUE);
	if (saveWeightsNow)
	{
		m_iEpochOfBestCost = m_iNumberOfEpochsTrained;
		if (m_bSaveBestWeightsEnabled)
		{
			if (m_bCrossValidationEnabled)
				strcpy(note, "// Saved During Cross Validation, Epoch ");
			else
				strcpy(note, "// Saved During Training, Epoch ");
			_itoa(m_iNumberOfEpochsTrained, number, 10);
			strcat(note, number);
			saveWeightsReturnValue = saveWeights(m_cpBestWeightsPathName, note);
		}
	}

	if (saveWeightsReturnValue != 0)
		return 2;
	return 0;
}


int NSLearningNetwork::fireConclude()
{
	NSRecallNetwork::fireConclude();

	return 0;
}

int NSLearningNetwork::startBackprop()
{
	inputAxonBackprop->backpropStarting();
	hidden1SynapseBackprop->backpropStarting();
	hidden1AxonBackprop->backpropStarting();
	hidden2SynapseBackprop->backpropStarting();
	hidden2AxonBackprop->backpropStarting();
	hidden3SynapseBackprop->backpropStarting();
	hidden3AxonBackprop->backpropStarting();
	hidden4SynapseBackprop->backpropStarting();
	hidden4AxonBackprop->backpropStarting();
	hidden5SynapseBackprop->backpropStarting();
	hidden5AxonBackprop->backpropStarting();
	outputSynapseBackprop->backpropStarting();
	outputAxonBackprop->backpropStarting();
	criterionBackprop->backpropStarting();
	recurrentSynapseBackprop->backpropStarting();
	forwardSynapseBackprop->backpropStarting();

	return 0;
}

int NSLearningNetwork::completeBackprop()
{
	inputAxonBackprop->backpropComplete();
	hidden1SynapseBackprop->backpropComplete();
	hidden1AxonBackprop->backpropComplete();
	hidden2SynapseBackprop->backpropComplete();
	hidden2AxonBackprop->backpropComplete();
	hidden3SynapseBackprop->backpropComplete();
	hidden3AxonBackprop->backpropComplete();
	hidden4SynapseBackprop->backpropComplete();
	hidden4AxonBackprop->backpropComplete();
	hidden5SynapseBackprop->backpropComplete();
	hidden5AxonBackprop->backpropComplete();
	outputSynapseBackprop->backpropComplete();
	outputAxonBackprop->backpropComplete();
	criterionBackprop->backpropComplete();
	recurrentSynapseBackprop->backpropComplete();
	forwardSynapseBackprop->backpropComplete();

	return 0;
}

int NSLearningNetwork::updateWeights()
{
	hidden1SynapseBackpropGradient->updateWeights(hidden1Synapse);
	hidden1AxonBackpropGradient->updateWeights(hidden1Axon);
	hidden2SynapseBackpropGradient->updateWeights(hidden2Synapse);
	hidden2AxonBackpropGradient->updateWeights(hidden2Axon);
	hidden3SynapseBackpropGradient->updateWeights(hidden3Synapse);
	hidden3AxonBackpropGradient->updateWeights(hidden3Axon);
	hidden4SynapseBackpropGradient->updateWeights(hidden4Synapse);
	hidden4AxonBackpropGradient->updateWeights(hidden4Axon);
	hidden5SynapseBackpropGradient->updateWeights(hidden5Synapse);
	hidden5AxonBackpropGradient->updateWeights(hidden5Axon);
	outputSynapseBackpropGradient->updateWeights(outputSynapse);
	outputAxonBackpropGradient->updateWeights(outputAxon);
	recurrentSynapseBackpropGradient->updateWeights(recurrentSynapse);
	forwardSynapseBackpropGradient->updateWeights(forwardSynapse);
	return 0;
}

void NSLearningNetwork::setDesiredFunctionIO(NSFloat *desiredFunctionIOData)
{
	if (m_dpDesiredExemplarData)
	        if (m_bNormalizeOutput)
	        { 
	                for (int output = 0; output < m_iNumberOfOutputs; output++)
	                {
	                        desiredFunctionIOData[output] = m_dpDesiredExemplarData[output] * m_dpOutputNormAmp[output] + m_dpOutputNormOffset[output];
	                }
	        }
	        else
	        {
	                for (int output = 0; output < m_iNumberOfOutputs; output++)
	                {
	                        desiredFunctionIOData[output] = m_dpDesiredExemplarData[output];
	                }
	        }
}

int NSLearningNetwork::startTesting()
{
	control->setCrossValidationActive(TRUE);

	criterion->setTesting(TRUE);

	return 0;
}

int NSLearningNetwork::endTesting()
{
	criterion->setTesting(FALSE);


	control->setCrossValidationActive(FALSE);
	return 0;
}

int NSLearningNetwork::computeInputNormCoeff(double *inputData, int exemplars)
{
	//Allocate memory for holding input normalization coefficients
	if (m_dpInputNormAmp)
		delete m_dpInputNormAmp;
	m_dpInputNormAmp = new double[getNumberOfInputs()];

	if (m_dpInputNormOffset)
		delete m_dpInputNormOffset;
	m_dpInputNormOffset = new double[getNumberOfInputs()];

	//Compute and store input normalization coefficients
	if (m_ipInputDataMap)
		computeInputNormCoeffAfterMask(m_dpInputNormAmp, m_dpInputNormOffset, inputData, exemplars, m_fInputNormMin, m_fInputNormMax, m_bNormalizeInputByChannel);
	else
		computeNormCoeff(m_dpInputNormAmp, m_dpInputNormOffset, inputData, exemplars, m_iNumberOfInputs, m_fInputNormMin, m_fInputNormMax, m_bNormalizeInputByChannel);

	m_bNormalizeInput = true;

	return 0;
}

int NSLearningNetwork::computeOutputNormCoeff(double *desiredData, int exemplars)
{
	//Allocate memory for holding output normalization coefficients
	if (m_dpOutputNormAmp)
		delete m_dpOutputNormAmp;
	m_dpOutputNormAmp = new double[m_iNumberOfOutputs];

	if (m_dpOutputNormOffset)
		delete m_dpOutputNormOffset;
	m_dpOutputNormOffset = new double[m_iNumberOfOutputs];

	//Compute and store output normalization coefficients
	computeNormCoeff(m_dpOutputNormAmp, m_dpOutputNormOffset, desiredData, exemplars, m_iNumberOfOutputs, m_fOutputNormMin, m_fOutputNormMax, m_bNormalizeOutputByChannel);

	m_bNormalizeOutput = true;

	return 0;
}

int NSLearningNetwork::computeNormCoeff(double *normAmp, double *normOff, double *data, int exemplars, int pes, double normMin, double normMax, bool normalizeByChannel)
{
	int pe;
	if (normalizeByChannel)
	{
		//Compute the amplitude and offset for each column using the max and min of that column
		for (pe = 0; pe < pes; pe++)
		{
			//Initialize dataMin and dataMax to the first value in the current column
			double dataMin = *(data + pe);
			double dataMax = dataMin;

			for (int exemplar = 1; exemplar < exemplars; exemplar++)
			{
				double currentDataValue = *(data + (exemplar * pes) + pe);

				if (currentDataValue < dataMin)
					dataMin = currentDataValue;
				else
					if (currentDataValue > dataMax)
						dataMax = currentDataValue;
			}

			double normAmplitude;
			double normOffset;
			computeAmplitudeAndOffset(normAmplitude, normOffset, dataMin, dataMax, normMin, normMax);

			normAmp[pe] = normAmplitude;
			normOff[pe] = normOffset;
		}
	}
	else
	{
		//Initialize dataMin and dataMax to the first value in the dataset
		double dataMin = *data;
		double dataMax = dataMin;

		//Compute one amplitude and offset pair to for use with all columns using the max and min of the entire dataset
		for (pe = 0; pe < pes; pe++)
		{
			for (int exemplar = 0; exemplar < exemplars; exemplar++)
			{
				double currentDataValue = *(data + (exemplar * pes) + pe);

				if (currentDataValue < dataMin)
					dataMin = currentDataValue;
				else
					if (currentDataValue > dataMax)
						dataMax = currentDataValue;
			}
		}

		double normAmplitude;
		double normOffset;
		computeAmplitudeAndOffset(normAmplitude, normOffset, dataMin, dataMax, normMin, normMax);

		for (pe = 0; pe < pes; pe++)
		{
			normAmp[pe] = normAmplitude;
			normOff[pe] = normOffset;
		}
	}
	return 0;
}

int NSLearningNetwork::computeInputNormCoeffAfterMask(double *normAmp, double *normOff, double *data, int exemplars, double normMin, double normMax, bool normalizeByChannel)
{
	int pe;
	if (normalizeByChannel)
	{
		//Compute the amplitude and offset for each masked column using the max and min of that column
		for (pe = 0; pe < m_iNumberOfInputsAfterMask; pe++)
		{
			//Initialize dataMin and dataMax to the first value in the current column
			double dataMin = *(data + m_ipInputDataMap[pe]);
			double dataMax = dataMin;

			for (int exemplar = 1; exemplar < exemplars; exemplar++)
			{
				double currentDataValue = *(data + (exemplar * m_iNumberOfInputs) + m_ipInputDataMap[pe]);

				if (currentDataValue < dataMin)
					dataMin = currentDataValue;
				else
					if (currentDataValue > dataMax)
						dataMax = currentDataValue;
			}

			double normAmplitude;
			double normOffset;
			computeAmplitudeAndOffset(normAmplitude, normOffset, dataMin, dataMax, normMin, normMax);

			normAmp[pe] = normAmplitude;
			normOff[pe] = normOffset;
		}
	}
	else
	{
		//Initialize dataMin and dataMax to the first masked value in the dataset
		double dataMin = *(data + m_ipInputDataMap[0]);
		double dataMax = dataMin;

		//Compute one amplitude and offset pair to for use with all columns using the max and min of the entire dataset
		for (pe = 0; pe < m_iNumberOfInputsAfterMask; pe++)
		{
			for (int exemplar = 0; exemplar < exemplars; exemplar++)
			{
				double currentDataValue = *(data + (exemplar * m_iNumberOfInputs) + m_ipInputDataMap[pe]);

				if (currentDataValue < dataMin)
					dataMin = currentDataValue;
				else
					if (currentDataValue > dataMax)
						dataMax = currentDataValue;
			}
		}

		double normAmplitude;
		double normOffset;
		computeAmplitudeAndOffset(normAmplitude, normOffset, dataMin, dataMax, normMin, normMax);

		for (pe = 0; pe < m_iNumberOfInputsAfterMask; pe++)
		{
			normAmp[pe] = normAmplitude;
			normOff[pe] = normOffset;
		}
	}
	return 0;
}

int NSLearningNetwork::computeAmplitudeAndOffset(double &normAmplitude, double &normOffset, double dataMin, double dataMax, double normMin, double normMax)
{
	//Compute the amplitude and offset
	if (dataMin - dataMax == 0)
	{
		normAmplitude = 1.0;
		normOffset = normMax - dataMax;
	}
	else
	{
		normAmplitude = (normMax - normMin) / (dataMax - dataMin);
		normOffset = normMax - ((normMax - normMin)/(dataMax - dataMin))*dataMax;
	}
	return 0;
}

int NSLearningNetwork::setAutoComputeInputNormCoeff(bool newVal)
{
	m_bAutoComputeInputNormCoeff = newVal;
	return 0;
}

int NSLearningNetwork::getAutoComputeInputNormCoeff(bool &val)
{
	val = m_bAutoComputeInputNormCoeff;
	return 0;
}

int NSLearningNetwork::setAutoComputeOutputNormCoeff(bool newVal)
{
	m_bAutoComputeOutputNormCoeff = newVal;
	return 0;
}

int NSLearningNetwork::getAutoComputeOutputNormCoeff(bool &val)
{
	val = m_bAutoComputeOutputNormCoeff;
	return 0;
}

int NSLearningNetwork::removeInputNormalization()
{
	m_bNormalizeInput = false;
	m_bAutoComputeInputNormCoeff = false;
	return 0;
}

int NSLearningNetwork::removeOutputNormalization()
{
	m_bNormalizeOutput = false;
	m_bAutoComputeOutputNormCoeff = false;
	return 0;
}

int NSLearningNetwork::setInputNormMin(double newVal)
{
	m_fInputNormMin = newVal;
	return 0;
}

int NSLearningNetwork::getInputNormMin(double &val)
{
	val = m_fInputNormMin;
	return 0;
}

int NSLearningNetwork::setInputNormMax(double newVal)
{
	m_fInputNormMax = newVal;
	return 0;
}

int NSLearningNetwork::getInputNormMax(double &val)
{
	val = m_fInputNormMax;
	return 0;
}

int NSLearningNetwork::setOutputNormMin(double newVal)
{
	m_fOutputNormMin = newVal;
	return 0;
}

int NSLearningNetwork::getOutputNormMin(double &val)
{
	val = m_fOutputNormMin;
	return 0;
}

int NSLearningNetwork::setOutputNormMax(double newVal)
{
	m_fOutputNormMax = newVal;
	return 0;
}

int NSLearningNetwork::getOutputNormMax(double &val)
{
	val = m_fOutputNormMax;
	return 0;
}

int NSLearningNetwork::setNormalizeInputByChannel(bool newVal)
{
	m_bNormalizeInputByChannel = newVal;
	return 0;
}

int NSLearningNetwork::getNormalizeInputByChannel(bool &val)
{
	val = m_bNormalizeInputByChannel;
	return 0;
}

int NSLearningNetwork::setNormalizeOutputByChannel(bool newVal)
{
	m_bNormalizeOutputByChannel = newVal;
	return 0;
}

int NSLearningNetwork::getNormalizeOutputByChannel(bool &val)
{
	val = m_bNormalizeOutputByChannel;
	return 0;
}

int NSLearningNetwork::getCostData(double *val)
{
	if ((m_dpCostData) && (m_iNumberOfEpochsTrained))
		for (int epoch = 0; epoch < m_iNumberOfEpochsTrained; epoch++)
			val[epoch] = m_dpCostData[epoch];
	else
		return 2;

	return 0;
}

int NSLearningNetwork::getCrossValidationCostData(double *val)
{
	if ((m_dpCVCostData) && (m_iNumberOfEpochsTrained))
		for (int epoch = 0; epoch < m_iNumberOfEpochsTrained; epoch++)
			val[epoch] = m_dpCVCostData[epoch];
	else
		return 2;

	return 0;
}

int NSLearningNetwork::getNumberOfEpochsTrained(int &val)
{
	val = m_iNumberOfEpochsTrained;
	return 0;
}

int NSLearningNetwork::setBestCost(double newVal)
{
	criterion->setBestCost(newVal);
	return 0;
}

int NSLearningNetwork::getBestCost(double &val)
{
	val = criterion->bestCost();
	return 0;
}

int NSLearningNetwork::getEpochOfBestCost(int &val)
{
	val = m_iEpochOfBestCost;
	return 0;
}

int NSLearningNetwork::setBestWeightsPathName(char *newVal)
{
	strcpy(m_cpBestWeightsPathName, newVal);
	return 0;
}

int NSLearningNetwork::getBestWeightsPathName(char *val, int bufferLength, int &pathNameLength)
{
	pathNameLength = strlen(m_cpBestWeightsPathName);
	if (val)
	{
		if (bufferLength >= (pathNameLength + 1))
			strcpy(val, m_cpBestWeightsPathName);
		else
			return 2;
	}
	return 0;
}

int NSLearningNetwork::setDefaultStepSizeDivisor(int exemplars)
{

	recurrentSynapseBackpropGradient->setStepDivisor(exemplars);
	forwardSynapseBackpropGradient->setStepDivisor(exemplars);
	outputSynapseBackpropGradient->setStepDivisor(exemplars);
	outputAxonBackpropGradient->setStepDivisor(exemplars);
	hidden1SynapseBackpropGradient->setStepDivisor(exemplars);
	hidden1AxonBackpropGradient->setStepDivisor(exemplars);
	hidden2SynapseBackpropGradient->setStepDivisor(exemplars);
	hidden2AxonBackpropGradient->setStepDivisor(exemplars);
	hidden3SynapseBackpropGradient->setStepDivisor(exemplars);
	hidden3AxonBackpropGradient->setStepDivisor(exemplars);
	hidden4SynapseBackpropGradient->setStepDivisor(exemplars);
	hidden4AxonBackpropGradient->setStepDivisor(exemplars);
	hidden5SynapseBackpropGradient->setStepDivisor(exemplars);
	hidden5AxonBackpropGradient->setStepDivisor(exemplars);
	return 0;
}

int  NSLearningNetwork::setSaveBestWeightsEnabled(bool newVal)
{
	m_bSaveBestWeightsEnabled = newVal;
	return 0;
}

int  NSLearningNetwork::getSaveBestWeightsEnabled(bool &val)
{
	val = m_bSaveBestWeightsEnabled;
	return 0;
}

int  NSLearningNetwork::setSaveBestWeightsForTraining(bool newVal)
{
	m_bSaveBestWeightsForTraining = newVal;
	return 0;
}

int  NSLearningNetwork::getSaveBestWeightsForTraining(bool &val)
{
	val = m_bSaveBestWeightsForTraining;
	return 0;
}

int  NSLearningNetwork::setCrossValidationEnabled(bool newVal)
{
	m_bCrossValidationEnabled = newVal;
	return 0;
}

int  NSLearningNetwork::getCrossValidationEnabled(bool &val)
{
	val = m_bCrossValidationEnabled;
	return 0;
}

int NSLearningNetwork::randomizeWeights()
{
	NSRecallNetwork::randomizeWeights();
	inputAxonBackprop->networkRandomize();
	hidden1SynapseBackprop->networkRandomize();
	hidden1SynapseBackpropGradient->networkRandomize();
	hidden1AxonBackprop->networkRandomize();
	hidden1AxonBackpropGradient->networkRandomize();
	hidden2SynapseBackprop->networkRandomize();
	hidden2SynapseBackpropGradient->networkRandomize();
	hidden2AxonBackprop->networkRandomize();
	hidden2AxonBackpropGradient->networkRandomize();
	hidden3SynapseBackprop->networkRandomize();
	hidden3SynapseBackpropGradient->networkRandomize();
	hidden3AxonBackprop->networkRandomize();
	hidden3AxonBackpropGradient->networkRandomize();
	hidden4SynapseBackprop->networkRandomize();
	hidden4SynapseBackpropGradient->networkRandomize();
	hidden4AxonBackprop->networkRandomize();
	hidden4AxonBackpropGradient->networkRandomize();
	hidden5SynapseBackprop->networkRandomize();
	hidden5SynapseBackpropGradient->networkRandomize();
	hidden5AxonBackprop->networkRandomize();
	hidden5AxonBackpropGradient->networkRandomize();
	outputSynapseBackprop->networkRandomize();
	outputSynapseBackpropGradient->networkRandomize();
	outputAxonBackprop->networkRandomize();
	outputAxonBackpropGradient->networkRandomize();
	criterion->networkRandomize();
	criterionBackprop->networkRandomize();
	recurrentSynapseBackprop->networkRandomize();
	recurrentSynapseBackpropGradient->networkRandomize();
	forwardSynapseBackprop->networkRandomize();
	forwardSynapseBackpropGradient->networkRandomize();

	return 0;
}

int NSLearningNetwork::resetNetwork()
{
	NSRecallNetwork::resetNetwork();

	criterion->setBestCost(1.000000000000000e+09);
	m_iEpochOfBestCost = 0;
	m_iNumberOfEpochsTrained = 0;
	m_fCurrentCost = 0.0;
	m_fCurrentCVCost = 0.0;

	inputAxonBackprop->networkReset();
	hidden1SynapseBackprop->networkReset();
	hidden1SynapseBackpropGradient->networkReset();
	hidden1AxonBackprop->networkReset();
	hidden1AxonBackpropGradient->networkReset();
	hidden2SynapseBackprop->networkReset();
	hidden2SynapseBackpropGradient->networkReset();
	hidden2AxonBackprop->networkReset();
	hidden2AxonBackpropGradient->networkReset();
	hidden3SynapseBackprop->networkReset();
	hidden3SynapseBackpropGradient->networkReset();
	hidden3AxonBackprop->networkReset();
	hidden3AxonBackpropGradient->networkReset();
	hidden4SynapseBackprop->networkReset();
	hidden4SynapseBackpropGradient->networkReset();
	hidden4AxonBackprop->networkReset();
	hidden4AxonBackpropGradient->networkReset();
	hidden5SynapseBackprop->networkReset();
	hidden5SynapseBackpropGradient->networkReset();
	hidden5AxonBackprop->networkReset();
	hidden5AxonBackpropGradient->networkReset();
	outputSynapseBackprop->networkReset();
	outputSynapseBackpropGradient->networkReset();
	outputAxonBackprop->networkReset();
	outputAxonBackpropGradient->networkReset();
	criterion->networkReset();
	criterionBackprop->networkReset();
	control->setBackpropLearningOn(TRUE);

	recurrentSynapseBackprop->networkReset();
	recurrentSynapseBackpropGradient->networkReset();
	forwardSynapseBackprop->networkReset();
	forwardSynapseBackpropGradient->networkReset();

	inputAxonBackprop->networkRandomize();
	hidden1SynapseBackprop->networkRandomize();
	hidden1SynapseBackpropGradient->networkRandomize();
	hidden1AxonBackprop->networkRandomize();
	hidden1AxonBackpropGradient->networkRandomize();
	hidden2SynapseBackprop->networkRandomize();
	hidden2SynapseBackpropGradient->networkRandomize();
	hidden2AxonBackprop->networkRandomize();
	hidden2AxonBackpropGradient->networkRandomize();
	hidden3SynapseBackprop->networkRandomize();
	hidden3SynapseBackpropGradient->networkRandomize();
	hidden3AxonBackprop->networkRandomize();
	hidden3AxonBackpropGradient->networkRandomize();
	hidden4SynapseBackprop->networkRandomize();
	hidden4SynapseBackpropGradient->networkRandomize();
	hidden4AxonBackprop->networkRandomize();
	hidden4AxonBackpropGradient->networkRandomize();
	hidden5SynapseBackprop->networkRandomize();
	hidden5SynapseBackpropGradient->networkRandomize();
	hidden5AxonBackprop->networkRandomize();
	hidden5AxonBackpropGradient->networkRandomize();
	outputSynapseBackprop->networkRandomize();
	outputSynapseBackpropGradient->networkRandomize();
	outputAxonBackprop->networkRandomize();
	outputAxonBackpropGradient->networkRandomize();
	criterion->networkRandomize();
	criterionBackprop->networkRandomize();
	recurrentSynapseBackprop->networkRandomize();
	recurrentSynapseBackpropGradient->networkRandomize();
	forwardSynapseBackprop->networkRandomize();
	forwardSynapseBackpropGradient->networkRandomize();

	return 0;
}

int NSLearningNetwork::seedRandom(unsigned int seed)
{
	srand(seed);
	return 0;
}

int NSLearningNetwork::destroyNetwork()
{
	NSRecallNetwork::destroyNetwork();
inputAxonBackprop->setStaticControl(NULL);
hidden1SynapseBackprop->setStaticControl(NULL);
hidden1SynapseBackpropGradient->setStaticControl(NULL);
hidden1AxonBackprop->setStaticControl(NULL);
hidden1AxonBackpropGradient->setStaticControl(NULL);
hidden2SynapseBackprop->setStaticControl(NULL);
hidden2SynapseBackpropGradient->setStaticControl(NULL);
hidden2AxonBackprop->setStaticControl(NULL);
hidden2AxonBackpropGradient->setStaticControl(NULL);
hidden3SynapseBackprop->setStaticControl(NULL);
hidden3SynapseBackpropGradient->setStaticControl(NULL);
hidden3AxonBackprop->setStaticControl(NULL);
hidden3AxonBackpropGradient->setStaticControl(NULL);
hidden4SynapseBackprop->setStaticControl(NULL);
hidden4SynapseBackpropGradient->setStaticControl(NULL);
hidden4AxonBackprop->setStaticControl(NULL);
hidden4AxonBackpropGradient->setStaticControl(NULL);
hidden5SynapseBackprop->setStaticControl(NULL);
hidden5SynapseBackpropGradient->setStaticControl(NULL);
hidden5AxonBackprop->setStaticControl(NULL);
hidden5AxonBackpropGradient->setStaticControl(NULL);
outputSynapseBackprop->setStaticControl(NULL);
outputSynapseBackpropGradient->setStaticControl(NULL);
outputAxonBackprop->setStaticControl(NULL);
outputAxonBackpropGradient->setStaticControl(NULL);
criterion->setStaticControl(NULL);
criterionBackprop->setStaticControl(NULL);
controlBackprop->setStaticControl(NULL);
recurrentSynapseBackprop->setStaticControl(NULL);
recurrentSynapseBackpropGradient->setStaticControl(NULL);
forwardSynapseBackprop->setStaticControl(NULL);
forwardSynapseBackpropGradient->setStaticControl(NULL);

	inputAxonBackprop->freeObject();
	delete inputAxonBackprop;
	hidden1SynapseBackprop->freeObject();
	delete hidden1SynapseBackprop;
	hidden1SynapseBackpropGradient->freeObject();
	delete hidden1SynapseBackpropGradient;
	hidden1AxonBackprop->freeObject();
	delete hidden1AxonBackprop;
	hidden1AxonBackpropGradient->freeObject();
	delete hidden1AxonBackpropGradient;
	hidden2SynapseBackprop->freeObject();
	delete hidden2SynapseBackprop;
	hidden2SynapseBackpropGradient->freeObject();
	delete hidden2SynapseBackpropGradient;
	hidden2AxonBackprop->freeObject();
	delete hidden2AxonBackprop;
	hidden2AxonBackpropGradient->freeObject();
	delete hidden2AxonBackpropGradient;
	hidden3SynapseBackprop->freeObject();
	delete hidden3SynapseBackprop;
	hidden3SynapseBackpropGradient->freeObject();
	delete hidden3SynapseBackpropGradient;
	hidden3AxonBackprop->freeObject();
	delete hidden3AxonBackprop;
	hidden3AxonBackpropGradient->freeObject();
	delete hidden3AxonBackpropGradient;
	hidden4SynapseBackprop->freeObject();
	delete hidden4SynapseBackprop;
	hidden4SynapseBackpropGradient->freeObject();
	delete hidden4SynapseBackpropGradient;
	hidden4AxonBackprop->freeObject();
	delete hidden4AxonBackprop;
	hidden4AxonBackpropGradient->freeObject();
	delete hidden4AxonBackpropGradient;
	hidden5SynapseBackprop->freeObject();
	delete hidden5SynapseBackprop;
	hidden5SynapseBackpropGradient->freeObject();
	delete hidden5SynapseBackpropGradient;
	hidden5AxonBackprop->freeObject();
	delete hidden5AxonBackprop;
	hidden5AxonBackpropGradient->freeObject();
	delete hidden5AxonBackpropGradient;
	outputSynapseBackprop->freeObject();
	delete outputSynapseBackprop;
	outputSynapseBackpropGradient->freeObject();
	delete outputSynapseBackpropGradient;
	outputAxonBackprop->freeObject();
	delete outputAxonBackprop;
	outputAxonBackpropGradient->freeObject();
	delete outputAxonBackpropGradient;
	criterion->freeObject();
	delete criterion;
	criterionBackprop->freeObject();
	delete criterionBackprop;
	controlBackprop->freeObject();
	delete controlBackprop;
	recurrentSynapseBackprop->freeObject();
	delete recurrentSynapseBackprop;
	recurrentSynapseBackpropGradient->freeObject();
	delete recurrentSynapseBackpropGradient;
	forwardSynapseBackprop->freeObject();
	delete forwardSynapseBackprop;
	forwardSynapseBackpropGradient->freeObject();
	delete forwardSynapseBackpropGradient;

	delete desiredDataFunctionIO;
	delete m_cpBestWeightsPathName;
	if (m_dpCostData)
		delete m_dpCostData;
	if (m_dpCVCostData)
		delete m_dpCVCostData;
	return 0;
}

