Download code/engine/neural_network/lstm.cpp from HDTenEightyP/SeafarerEngine: direct link, hf CLI and curl.
- Browser
- Download file 16 kB
-
https://huggingface.co/HDTenEightyP/SeafarerEngine/resolve/main/code/engine/neural_network/lstm.cpp
- Command line
-
hf download hf://HDTenEightyP/SeafarerEngine/code/engine/neural_network/lstm.cpp
-
curl -L -o lstm.cpp https://huggingface.co/HDTenEightyP/SeafarerEngine/resolve/main/code/engine/neural_network/lstm.cpp
16 kB
| // Activation Functions & Derivatives | |
| static inline float sigmoid(float x) { return 1.0f / (1.0f + expf(-x)); } | |
| static inline float sigmoid_deriv(float s) { return s * (1.0f - s); } | |
| static inline float tanh_deriv(float t) { return 1.0f - t * t; } | |
| // --- INITIALIZATION AND MEMORY MANAGEMENT --- | |
| void init_weight(float *w, int rows, int cols) { | |
| float scale = sqrtf(6.0f / (rows + cols)); | |
| for (int i = 0; i < rows * cols; i++) { | |
| w[i] = ((float)rand() / RAND_MAX) * 2.0f * scale - scale; | |
| } | |
| } | |
| LSTMNetwork* lstm_create(int input_dim, int hidden_dim) { | |
| LSTMNetwork *nn = (LSTMNetwork*)malloc(sizeof(LSTMNetwork)); | |
| nn->input_dim = input_dim; | |
| nn->hidden_dim = hidden_dim; | |
| int w_size = hidden_dim * input_dim; | |
| int u_size = hidden_dim * hidden_dim; | |
| int b_size = hidden_dim; | |
| // Allocate parameters with explicit casts | |
| nn->W_i = (float*)malloc(w_size * sizeof(float)); nn->W_f = (float*)malloc(w_size * sizeof(float)); | |
| nn->W_c = (float*)malloc(w_size * sizeof(float)); nn->W_o = (float*)malloc(w_size * sizeof(float)); | |
| nn->U_i = (float*)malloc(u_size * sizeof(float)); nn->U_f = (float*)malloc(u_size * sizeof(float)); | |
| nn->U_c = (float*)malloc(u_size * sizeof(float)); nn->U_o = (float*)malloc(u_size * sizeof(float)); | |
| nn->b_i = (float*)calloc(b_size, sizeof(float)); nn->b_f = (float*)calloc(b_size, sizeof(float)); | |
| nn->b_c = (float*)calloc(b_size, sizeof(float)); nn->b_o = (float*)calloc(b_size, sizeof(float)); | |
| // Allocate gradients with explicit casts | |
| nn->dW_i = (float*)malloc(w_size * sizeof(float)); nn->dW_f = (float*)malloc(w_size * sizeof(float)); | |
| nn->dW_c = (float*)malloc(w_size * sizeof(float)); nn->dW_o = (float*)malloc(w_size * sizeof(float)); | |
| nn->dU_i = (float*)malloc(u_size * sizeof(float)); nn->dU_f = (float*)malloc(u_size * sizeof(float)); | |
| nn->dU_c = (float*)malloc(u_size * sizeof(float)); nn->dU_o = (float*)malloc(u_size * sizeof(float)); | |
| nn->db_i = (float*)malloc(b_size * sizeof(float)); nn->db_f = (float*)malloc(b_size * sizeof(float)); | |
| nn->db_c = (float*)malloc(b_size * sizeof(float)); nn->db_o = (float*)malloc(b_size * sizeof(float)); | |
| // Initialize weights | |
| init_weight(nn->W_i, hidden_dim, input_dim); init_weight(nn->W_f, hidden_dim, input_dim); | |
| init_weight(nn->W_c, hidden_dim, input_dim); init_weight(nn->W_o, hidden_dim, input_dim); | |
| init_weight(nn->U_i, hidden_dim, hidden_dim); init_weight(nn->U_f, hidden_dim, hidden_dim); | |
| init_weight(nn->U_c, hidden_dim, hidden_dim); init_weight(nn->U_o, hidden_dim, hidden_dim); | |
| // Forget gate bias initialization trick | |
| for (int i = 0; i < hidden_dim; i++) nn->b_f[i] = 1.0f; | |
| return nn; | |
| } | |
| LSTMHistory* lstm_create_history(int seq_len, int input_dim, int hidden_dim) { | |
| LSTMHistory *hist = (LSTMHistory*)malloc(sizeof(LSTMHistory)); | |
| hist->seq_len = seq_len; | |
| hist->input_dim = input_dim; | |
| hist->hidden_dim = hidden_dim; | |
| hist->x = (float*)calloc(seq_len * input_dim, sizeof(float)); | |
| hist->i_gate = (float*)calloc(seq_len * hidden_dim, sizeof(float)); | |
| hist->f_gate = (float*)calloc(seq_len * hidden_dim, sizeof(float)); | |
| hist->c_cand = (float*)calloc(seq_len * hidden_dim, sizeof(float)); | |
| hist->c = (float*)calloc(seq_len * hidden_dim, sizeof(float)); | |
| hist->o_gate = (float*)calloc(seq_len * hidden_dim, sizeof(float)); | |
| hist->h = (float*)calloc(seq_len * hidden_dim, sizeof(float)); | |
| return hist; | |
| } | |
| LSTMDenseLayer* dense_create(int input_dim, int output_dim) { | |
| LSTMDenseLayer *layer = (LSTMDenseLayer*)malloc(sizeof(LSTMDenseLayer)); | |
| layer->input_dim = input_dim; | |
| layer->output_dim = output_dim; | |
| layer->W = (float*)malloc(output_dim * input_dim * sizeof(float)); | |
| layer->b = (float*)calloc(output_dim, sizeof(float)); | |
| layer->dW = (float*)calloc(output_dim * input_dim, sizeof(float)); | |
| layer->db = (float*)calloc(output_dim, sizeof(float)); | |
| init_weight(layer->W, output_dim, input_dim); | |
| return layer; | |
| } | |
| void lstm_zero_grads(LSTMNetwork *nn) { | |
| int w_size = nn->hidden_dim * nn->input_dim; | |
| int u_size = nn->hidden_dim * nn->hidden_dim; | |
| int b_size = nn->hidden_dim; | |
| memset(nn->dW_i, 0, w_size * sizeof(float)); memset(nn->dW_f, 0, w_size * sizeof(float)); | |
| memset(nn->dW_c, 0, w_size * sizeof(float)); memset(nn->dW_o, 0, w_size * sizeof(float)); | |
| memset(nn->dU_i, 0, u_size * sizeof(float)); memset(nn->dU_f, 0, u_size * sizeof(float)); | |
| memset(nn->dU_c, 0, u_size * sizeof(float)); memset(nn->dU_o, 0, u_size * sizeof(float)); | |
| memset(nn->db_i, 0, b_size * sizeof(float)); memset(nn->db_f, 0, b_size * sizeof(float)); | |
| memset(nn->db_c, 0, b_size * sizeof(float)); memset(nn->db_o, 0, b_size * sizeof(float)); | |
| } | |
| // --- ALGORITHMIC EXECUTION PIPELINES --- | |
| void lstm_forward(LSTMNetwork *nn, LSTMHistory *hist, const float *sequence_inputs) { | |
| int H = nn->hidden_dim; | |
| int I = nn->input_dim; | |
| memcpy(hist->x, sequence_inputs, hist->seq_len * I * sizeof(float)); | |
| for (int t = 0; t < hist->seq_len; t++) { | |
| float *x_t = &hist->x[t * I]; | |
| float *h_prev = (t == 0) ? NULL : &hist->h[(t - 1) * H]; | |
| float *c_prev = (t == 0) ? NULL : &hist->c[(t - 1) * H]; | |
| float *i_t = &hist->i_gate[t * H]; | |
| float *f_t = &hist->f_gate[t * H]; | |
| float *cc_t = &hist->c_cand[t * H]; | |
| float *c_t = &hist->c[t * H]; | |
| float *o_t = &hist->o_gate[t * H]; | |
| float *h_t = &hist->h[t * H]; | |
| for (int j = 0; j < H; j++) { | |
| float net_i = nn->b_i[j]; float net_f = nn->b_f[j]; | |
| float net_c = nn->b_c[j]; float net_o = nn->b_o[j]; | |
| for (int k = 0; k < I; k++) { | |
| net_i += nn->W_i[j * I + k] * x_t[k]; | |
| net_f += nn->W_f[j * I + k] * x_t[k]; | |
| net_c += nn->W_c[j * I + k] * x_t[k]; | |
| net_o += nn->W_o[j * I + k] * x_t[k]; | |
| } | |
| if (h_prev) { | |
| for (int k = 0; k < H; k++) { | |
| net_i += nn->U_i[j * H + k] * h_prev[k]; | |
| net_f += nn->U_f[j * H + k] * h_prev[k]; | |
| net_c += nn->U_c[j * H + k] * h_prev[k]; | |
| net_o += nn->U_o[j * H + k] * h_prev[k]; | |
| } | |
| } | |
| i_t[j] = sigmoid(net_i); | |
| f_t[j] = sigmoid(net_f); | |
| cc_t[j] = tanhf(net_c); | |
| o_t[j] = sigmoid(net_o); | |
| c_t[j] = i_t[j] * cc_t[j]; | |
| if (c_prev) { | |
| c_t[j] += f_t[j] * c_prev[j]; | |
| } | |
| h_t[j] = o_t[j] * tanhf(c_t[j]); | |
| } | |
| } | |
| } | |
| void lstm_backward(LSTMNetwork *nn, LSTMHistory *hist, const float *dh_sequence, float *dx_sequence) { | |
| int H = nn->hidden_dim; | |
| int I = nn->input_dim; | |
| float *dh_next = (float*)calloc(H, sizeof(float)); | |
| float *dc_next = (float*)calloc(H, sizeof(float)); | |
| float *dh_prev_step = (float*)calloc(H, sizeof(float)); | |
| float *zero_buf = (float*)calloc(H, sizeof(float)); | |
| lstm_zero_grads(nn); | |
| if (dx_sequence) { | |
| memset(dx_sequence, 0, hist->seq_len * I * sizeof(float)); | |
| } | |
| for (int t = hist->seq_len - 1; t >= 0; t--) { | |
| float *x_t = &hist->x[t * I]; | |
| float *i_t = &hist->i_gate[t * H]; | |
| float *f_t = &hist->f_gate[t * H]; | |
| float *cc_t = &hist->c_cand[t * H]; | |
| float *c_t = &hist->c[t * H]; | |
| float *o_t = &hist->o_gate[t * H]; | |
| float *c_prev = (t == 0) ? zero_buf : &hist->c[(t - 1) * H]; | |
| float *h_prev = (t == 0) ? zero_buf : &hist->h[(t - 1) * H]; | |
| memset(dh_prev_step, 0, H * sizeof(float)); | |
| for (int j = 0; j < H; j++) { | |
| float dh_total = dh_sequence[t * H + j] + dh_next[j]; | |
| float dc_curr = dh_total * o_t[j] * tanh_deriv(tanhf(c_t[j])) + dc_next[j]; | |
| float dnet_o = dh_total * tanhf(c_t[j]) * sigmoid_deriv(o_t[j]); | |
| float dnet_c = dc_curr * i_t[j] * tanh_deriv(cc_t[j]); | |
| float dnet_i = dc_curr * cc_t[j] * sigmoid_deriv(i_t[j]); | |
| float dnet_f = dc_curr * c_prev[j] * sigmoid_deriv(f_t[j]); | |
| nn->db_o[j] += dnet_o; nn->db_c[j] += dnet_c; | |
| nn->db_i[j] += dnet_i; nn->db_f[j] += dnet_f; | |
| for (int k = 0; k < I; k++) { | |
| nn->dW_o[j * I + k] += dnet_o * x_t[k]; | |
| nn->dW_c[j * I + k] += dnet_c * x_t[k]; | |
| nn->dW_i[j * I + k] += dnet_i * x_t[k]; | |
| nn->dW_f[j * I + k] += dnet_f * x_t[k]; | |
| if (dx_sequence) { | |
| dx_sequence[t * I + k] += dnet_i * nn->W_i[j * I + k] + | |
| dnet_f * nn->W_f[j * I + k] + | |
| dnet_c * nn->W_c[j * I + k] + | |
| dnet_o * nn->W_o[j * I + k]; | |
| } | |
| } | |
| for (int k = 0; k < H; k++) { | |
| nn->dU_o[j * H + k] += dnet_o * h_prev[k]; | |
| nn->dU_c[j * H + k] += dnet_c * h_prev[k]; | |
| nn->dU_i[j * H + k] += dnet_i * h_prev[k]; | |
| nn->dU_f[j * H + k] += dnet_f * h_prev[k]; | |
| dh_prev_step[k] += dnet_i * nn->U_i[j * H + k] + | |
| dnet_f * nn->U_f[j * H + k] + | |
| dnet_c * nn->U_c[j * H + k] + | |
| dnet_o * nn->U_o[j * H + k]; | |
| } | |
| dc_next[j] = dc_curr * f_t[j]; | |
| } | |
| memcpy(dh_next, dh_prev_step, H * sizeof(float)); | |
| } | |
| free(dh_next); free(dc_next); free(dh_prev_step); free(zero_buf); | |
| } | |
| void lstm_update(LSTMNetwork *nn, float lr) { | |
| int w_size = nn->hidden_dim * nn->input_dim; | |
| int u_size = nn->hidden_dim * nn->hidden_dim; | |
| int b_size = nn->hidden_dim; | |
| for (int i = 0; i < w_size; i++) { | |
| nn->W_i[i] -= lr * nn->dW_i[i]; nn->W_f[i] -= lr * nn->dW_f[i]; | |
| nn->W_c[i] -= lr * nn->dW_c[i]; nn->W_o[i] -= lr * nn->dW_o[i]; | |
| } | |
| for (int i = 0; i < u_size; i++) { | |
| nn->U_i[i] -= lr * nn->dU_i[i]; nn->U_f[i] -= lr * nn->dU_f[i]; | |
| nn->U_c[i] -= lr * nn->dU_c[i]; nn->U_o[i] -= lr * nn->dU_o[i]; | |
| } | |
| for (int i = 0; i < b_size; i++) { | |
| nn->b_i[i] -= lr * nn->db_i[i]; nn->b_f[i] -= lr * nn->db_f[i]; | |
| nn->b_c[i] -= lr * nn->db_c[i]; nn->b_o[i] -= lr * nn->db_o[i]; | |
| } | |
| } | |
| // --- CLEANUP MANAGEMENT --- | |
| void lstm_free_history(LSTMHistory *hist) { | |
| free(hist->x); free(hist->i_gate); free(hist->f_gate); | |
| free(hist->c_cand); free(hist->c); free(hist->o_gate); free(hist->h); | |
| free(hist); | |
| } | |
| void lstm_free(LSTMNetwork *nn) { | |
| free(nn->W_i); free(nn->W_f); free(nn->W_c); free(nn->W_o); | |
| free(nn->U_i); free(nn->U_f); free(nn->U_c); free(nn->U_o); | |
| free(nn->b_i); free(nn->b_f); free(nn->b_c); free(nn->b_o); | |
| free(nn->dW_i); free(nn->dW_f); free(nn->dW_c); free(nn->dW_o); | |
| free(nn->dU_i); free(nn->dU_f); free(nn->dU_c); free(nn->dU_o); | |
| free(nn->db_i); free(nn->db_f); free(nn->db_c); free(nn->db_o); | |
| free(nn); | |
| } | |
| void dense_free(LSTMDenseLayer *layer) { | |
| free(layer->W); free(layer->b); | |
| free(layer->dW); free(layer->db); | |
| free(layer); | |
| } | |
| // --- MAIN ARCHITECTURE ASSEMBLY --- | |
| int CreateLSTM() { | |
| srand((unsigned int)time(NULL)); | |
| // Structural Hyperparameters | |
| int seq_len = 5; | |
| int input_dim = 384; | |
| int hidden1 = 256; | |
| int hidden2 = 128; | |
| int output_dim = 16; | |
| float lr = 0.01f; | |
| // Layer Object Allocations | |
| LSTMNetwork *layer1 = lstm_create(input_dim, hidden1); | |
| LSTMNetwork *layer2 = lstm_create(hidden1, hidden2); | |
| LSTMDenseLayer *dense = dense_create(hidden2, output_dim); | |
| // Context Sequence Memory History Structures | |
| LSTMHistory *hist1 = lstm_create_history(seq_len, input_dim, hidden1); | |
| LSTMHistory *hist2 = lstm_create_history(seq_len, hidden1, hidden2); | |
| // Variable Buffers | |
| float *raw_inputs = (float*)calloc(seq_len * input_dim, sizeof(float)); | |
| float *dense_outputs = (float*)calloc(seq_len * output_dim, sizeof(float)); | |
| float *target_outs = (float*)calloc(seq_len * output_dim, sizeof(float)); | |
| // Recurrent Backward Gradient Trackers | |
| float *d_dense_out = (float*)calloc(seq_len * output_dim, sizeof(float)); | |
| float *dh_layer2 = (float*)calloc(seq_len * hidden2, sizeof(float)); | |
| float *dh_layer1 = (float*)calloc(seq_len * hidden1, sizeof(float)); | |
| // Populate mock training sequence data | |
| for (int i = 0; i < seq_len * input_dim; i++) raw_inputs[i] = ((float)rand() / RAND_MAX) * 0.5f; | |
| for (int i = 0; i < seq_len * output_dim; i++) target_outs[i] = ((float)rand() / RAND_MAX); | |
| printf("Executing Deep Stacked Network Architecture Training Loop...\n\n"); | |
| for (int epoch = 0; epoch <= 100; epoch++) { | |
| // ---------------------------------------------------- | |
| // A. FORWARD PIPELINE | |
| // ---------------------------------------------------- | |
| lstm_forward(layer1, hist1, raw_inputs); | |
| lstm_forward(layer2, hist2, hist1->h); | |
| // Project Output from Hidden Layer 2 through Dense Weights | |
| for (int t = 0; t < seq_len; t++) { | |
| for (int i = 0; i < output_dim; i++) { | |
| float net = dense->b[i]; | |
| for (int j = 0; j < hidden2; j++) { | |
| net += dense->W[i * hidden2 + j] * hist2->h[t * hidden2 + j]; | |
| } | |
| dense_outputs[t * output_dim + i] = net; | |
| } | |
| } | |
| // Calculate loss (MSE Evaluation) | |
| float total_loss = 0.0f; | |
| for (int i = 0; i < seq_len * output_dim; i++) { | |
| float diff = dense_outputs[i] - target_outs[i]; | |
| total_loss += 0.5f * diff * diff; | |
| d_dense_out[i] = diff; // Backprop seeding derivative | |
| } | |
| if (epoch % 20 == 0) { | |
| printf("Epoch %3d | Aggregated Sequence MSE Loss: %f\n", epoch, total_loss / seq_len); | |
| } | |
| // ---------------------------------------------------- | |
| // B. BACKPROPAGATION THROUGH TIME (BPTT) CHAIN | |
| // ---------------------------------------------------- | |
| // Step 1: Initialize Dense Gradients | |
| memset(dense->db, 0, output_dim * sizeof(float)); | |
| memset(dense->dW, 0, output_dim * hidden2 * sizeof(float)); | |
| memset(dh_layer2, 0, seq_len * hidden2 * sizeof(float)); | |
| // Step 2: Clear and push dense deviations back down to hidden states | |
| for (int t = 0; t < seq_len; t++) { | |
| for (int i = 0; i < output_dim; i++) { | |
| float d_out = d_dense_out[t * output_dim + i]; | |
| dense->db[i] += d_out; | |
| for (int j = 0; j < hidden2; j++) { | |
| dense->dW[i * hidden2 + j] += d_out * hist2->h[t * hidden2 + j]; | |
| dh_layer2[t * hidden2 + j] += d_out * dense->W[i * hidden2 + j]; | |
| } | |
| } | |
| } | |
| // Step 3: Recurrent step extraction back through Layer 2 | |
| lstm_backward(layer2, hist2, dh_layer2, dh_layer1); | |
| // Step 4: Recurrent step extraction back through Layer 1 | |
| lstm_backward(layer1, hist1, dh_layer1, NULL); | |
| // ---------------------------------------------------- | |
| // C. DESCENT WEIGHT PARAMETER OPTIMIZATION | |
| // ---------------------------------------------------- | |
| lstm_update(layer1, lr); | |
| lstm_update(layer2, lr); | |
| for (int i = 0; i < output_dim * hidden2; i++) dense->W[i] -= lr * dense->dW[i]; | |
| for (int i = 0; i < output_dim; i++) dense->b[i] -= lr * dense->db[i]; | |
| } | |
| // Free all assigned dynamic spaces safely | |
| free(raw_inputs); free(dense_outputs); free(target_outs); | |
| free(d_dense_out); free(dh_layer2); free(dh_layer1); | |
| lstm_free_history(hist1); lstm_free_history(hist2); | |
| lstm_free(layer1); lstm_free(layer2); | |
| dense_free(dense); | |
| printf("\nTraining terminated. Free routines executed cleanly.\n"); | |
| return 0; | |
| } | |