MultiLayer perceptron.
https://github.com/kaifishr/CppMultilayerPerceptron/blob/main/src/neural_network.cpp
https://github.com/Asynchronousx/CUDA-MLP/blob/main/gpu/data.h

The Fully Connected layer is a traditional Multi Layer Perceptron that uses a softmax activation function.

принимает список произвольных чисел и преобразует их в список вероятностей пропорциональных этим числам.
То есть:
Допустим, у нас есть список [1, 2, 3]:

softmax([1, 2, 3]) = [0.09003, 0.24473, 0.66524]

0.09003, 0.24473, 0.66524 summ= 1.0

softmax(x) { return np.exp(x) / np.sum(np.exp(x) }

-------------
// Pool1 layer
  cudnnPoolingForward(cudnnHandle, poolDesc, &alpha, conv1Tensor,
  	conv1, &beta, pool1Tensor, pool1);

  // Conv2 layer
  cudnnConvolutionForward(cudnnHandle, &alpha, pool1Tensor,
  	pool1, conv2filterDesc, pconv2, conv2Desc,
  	conv2algo, workspace, m_workspaceSize, &beta,
  	conv2Tensor, conv2);
  cudnnAddTensor(cudnnHandle, &alpha, conv2BiasTensor,
  	pconv2bias, &alpha, conv2Tensor, conv2);

  // Pool2 layer
  cudnnPoolingForward(cudnnHandle, poolDesc, &alpha, conv2Tensor,
  	conv2, &beta, pool2Tensor, pool2);

  // FC1 layer
  // Forward propagate neurons using weights (fc1 = pfc1'*pool2)
  cublasSgemm(cublasHandle, CUBLAS_OP_T, CUBLAS_OP_N,
  	ref_fc1.outputs, m_batchSize, ref_fc1.inputs,
  	&alpha,
  	pfc1, ref_fc1.inputs,
  	pool2, ref_fc1.inputs,
  	&beta,
  	fc1, ref_fc1.outputs);
  // Add bias using GEMM's "beta" (fc1 += pfc1bias*1_vec')
  cublasSgemm(cublasHandle, CUBLAS_OP_N, CUBLAS_OP_N,
  	ref_fc1.outputs, m_batchSize, 1,
  	&alpha,
  	pfc1bias, ref_fc1.outputs,
  	onevec, 1,
  	&alpha,
  	fc1, ref_fc1.outputs);

  // ReLU activation
  cudnnActivationForward(cudnnHandle, fc1Activation, &alpha,
  	fc1Tensor, fc1, &beta, fc1Tensor, fc1relu);

  // FC2 layer
  // Forward propagate neurons using weights (fc2 = pfc2'*fc1relu)
  cublasSgemm(cublasHandle, CUBLAS_OP_T, CUBLAS_OP_N,
  	ref_fc2.outputs, m_batchSize, ref_fc2.inputs,
  	&alpha,
  	pfc2, ref_fc2.inputs,
  	fc1relu, ref_fc2.inputs,
  	&beta,
  	fc2, ref_fc2.outputs);
  // Add bias using GEMM's "beta" (fc2 += pfc2bias*1_vec')
  cublasSgemm(cublasHandle, CUBLAS_OP_N, CUBLAS_OP_N,
  	ref_fc2.outputs, m_batchSize, 1,
  	&alpha,
  	pfc2bias, ref_fc2.outputs,
  	onevec, 1,
  	&alpha,
  	fc2, ref_fc2.outputs);

  // Softmax loss
  cudnnSoftmaxForward(cudnnHandle, CUDNN_SOFTMAX_ACCURATE, CUDNN_SOFTMAX_MODE_CHANNEL,
  	&alpha, fc2Tensor, fc2, &beta, fc2Tensor, result);