% Multi-layer perceptron
% Roderick Murray-Smith 2000

%close all

% set up training inputs and outputs for model

fprintf('Start loading data\n');
fid = fopen('Dataset.data','r');
if fid == -1
   error('Could not open file');
end
maxiter = 300;			% maximum number of training iterations
N = 20000;
Nletters = 26;			% actual number of classes
inputs = zeros(N,16);
targets = zeros(N,Nletters);   
for i =1:N
   line = fgets(fid);
   ordinalpos = line(1) - 'A' + 1;
   targets(i, ordinalpos)  = 1;
   inputs(i,:) = str2num(line(2:end));
end
% normalise the inputs
inputs  = inputs - ones(N,1)*mean(inputs);
inputs  = inputs./(ones(N,1)*std(inputs));

% split data into training and test sets.
trainset = 1:9999;
testset = 10000:N;
testinputs  = inputs(testset,:);
testtargets = targets(testset,:);
inputs = inputs(trainset,:);
targets = targets(trainset,:);
fprintf('Finished loading\n');

% dimensions of training data
[N,diminputs]  = size(inputs);
[N,dimoutputs] = size(targets);

% initialise the weight vector (add one to include the bias term
% w0 as first entry of input
ninput = diminputs + 1;
nhidden = 10; 
noutput = Nletters;
w1 = 0.1*randn(ninput,nhidden);
w2 = 0.1*randn(nhidden+1,noutput);  % add one for a bias term (first entry)

eta = 0.1;    % learning rate
errorhistory = zeros(maxiter,1);
testerrorhistory = zeros(maxiter,1);

fprintf('Start learning\n');
for batch = 1:maxiter
   for t = 1:N
      [outputs, hidden] = mlpforward(inputs(t,:), w1, w2);
      [deltaout, deltahidden] = mlpbackward(hidden, outputs, targets(t,:), w2);
      
      changew2 = [1 hidden]'*deltaout;
      changew1 = [1 inputs(t,:)]'*deltahidden;
      
      w2 = w2 - eta*changew2;
      w1 = w1 - eta*changew1;
   end
   
   % collect statistics on learning performance
   % this is not needed to learn - only for evaluation purposes
   [outputs, hidden] = mlpforward(inputs, w1, w2);
   errorsum = sum(sum((outputs - targets).^2));
   %fprintf('Batch %d, Error = %f\n', batch, errorsum);
   errorhistory(batch) = errorsum;
   
   % now calculate test results
   [testoutputs, testhidden] = mlpforward(testinputs, w1, w2);
   errorsum = sum(sum((testoutputs - testtargets).^2));
   testerrorhistory(batch) = errorsum;
   figure(1)
   plot(1:batch,errorhistory(1:batch),1:batch,testerrorhistory(1:batch));
   legend('training error','test error');
   pause(0.01);
   % calculate confusion matrix
   Ctraining = confusionmulti(round(outputs), targets, Nletters);
   Ctest = confusionmulti(round(testoutputs), testtargets, Nletters);
   figure(2)
   subplot(2,1,1)
   mesh(Ctraining);
   title('Training confusion matrix')
   subplot(2,1,2);
   mesh(Ctest);
   title('Test confusion matrix');
end
Ctraining
Ctest




