Matrix multiplication implementations with different optimization strategies. More...

Classes
struct	TrainingPairList

struct	PreloadedTrainingData

Macros
#define	ATTN_SCORES_ACCESS(scores_ptr, h, i, j, aligned_context_window) scores_ptr[((h) * (aligned_context_window) * (aligned_context_window)) + ((i) * (aligned_context_window)) + (j)]

Functions
void	gemm_naive_parallel (const float A, const float B, const float bias, float C, int M, int N, int K)
	Naive parallel GEMM implementation (reference baseline)

void	gemm_avx512_parallel (const float A, const float B, const float bias, float C, int M, int N, int K)
	AVX-512 optimized GEMM with vectorized inner loops.

void	gemm_fine_grained_parallel (const float A, const float B, const float bias, float C, int M, int N, int K)
	Cache-blocked GEMM with fine-grained parallelism.

void	gemm_blocked_serial (const float A, const float B, const float bias, float C, int M, int N, int K)

void	layernorm_naive_serial (const float input, const float gamma, const float beta, float output, float mean_cache, float rstd_cache, int tokens, int d_model, int aligned_embed_dim, float eps)

void	layernorm_forward_rolled_slice (const float __restrict input_slice_base, const float __restrict gamma, const float __restrict beta, float __restrict output_slice_base, float __restrict mean_cache_slice, float __restrict rstd_cache_slice, int num_tokens_in_slice, int d_model, int aligned_embed_dim, float eps)

void	layernorm_forward_unrolled_slice (const float __restrict input_slice_base, const float __restrict gamma, const float __restrict beta, float __restrict output_slice_base, float __restrict mean_cache_slice, float __restrict rstd_cache_slice, int num_tokens_in_slice, int d_model, float eps)

void	layernorm_token_parallel (TransformerModel *M, size_t input_offset, size_t weight_offset, size_t bias_offset, size_t mean_cache_offset, size_t rstd_cache_offset, size_t output_offset, float eps)
	Token-parallel Layer Normalization with AVX-512 optimization.

void	layernorm_naive_serial_matched_precision (const float input, const float gamma, const float beta, float output, float mean_cache, float rstd_cache, int tokens, int d_model, float eps)

static void *	aligned_alloc_64 (size_t size)

void	debug_math_comparison (TransformerModel *M)

void	run_layernorm_benchmark_precision_matched (TransformerModel *M)

void	run_layernorm_benchmark_performance (TransformerModel *M)

static void	qkv_micro_kernel_blocked_4x16_polished (const float __restrict input_token, const float __restrict Q_weights_block, const float __restrict K_weights_block, const float __restrict V_weights_block, const float __restrict Q_bias_4, const float __restrict K_bias_4, const float __restrict V_bias_4, float __restrict Q_output_4, float __restrict K_output_4, float __restrict V_output_4, int embed_dim)

static void	qkv_token_kernel_4x16_blocked_polished (const float __restrict input_token, const float __restrict Q_weights, const float __restrict K_weights, const float __restrict V_weights, const float __restrict Q_bias, const float __restrict K_bias, const float __restrict V_bias, float __restrict Q_output, float __restrict K_output, float __restrict V_output, int embed_dim)

void	qkv_projection (TransformerModel *M, size_t layer_idx)

static void	qkv_micro_kernel_head_major_4x16 (const float __restrict input_token, const float __restrict Q_weights_block, const float __restrict K_weights_block, const float __restrict V_weights_block, const float __restrict Q_bias_4, const float __restrict K_bias_4, const float __restrict V_bias_4, TransformerModel M, float __restrict q_output_base, float __restrict k_output_base, float *__restrict v_output_base, int embed_dim, int token_idx, int output_start_dim)

static void	qkv_token_kernel_head_major_4x16 (const float __restrict input_token, const float __restrict Q_weights, const float __restrict K_weights, const float __restrict V_weights, const float __restrict Q_bias, const float __restrict K_bias, const float __restrict V_bias, TransformerModel M, float __restrict q_output_base, float __restrict k_output_base, float *__restrict v_output_base, int embed_dim, int token_idx)

void	qkv_projection_head_major (TransformerModel *M, int layer_idx)

double	compare_arrays (const float a, const float b, size_t size, const char *name)

void	convert_token_major_to_head_major_layer (const float token_major_base, float head_major_base, TransformerModel *M)

void	benchmark_qkv_dual_comparison (TransformerModel *M)

void	compute_attention_scores_head_major (TransformerModel *M, int layer_idx)

void	apply_causal_softmax_head_major (TransformerModel *M, int layer_idx)

void	compute_attention_output_head_major (TransformerModel *M, int layer_idx)

void	attention_head_major_complete (TransformerModel *M, int layer_idx)
	Complete multi-head attention with head-major layout (self-attention)

void	test_attention_head_major_after_qkv (TransformerModel *M)

void	attention_projection_with_concat (TransformerModel *M, int layer_idx)
	Production attention projection with concat: Head-major → Token-major → GEMM.

void	benchmark_attention_projection_complete (TransformerModel *M)

void	add_gpt2_token_and_positional_embeddings (TransformerModel *M, size_t token_ids_offset, size_t output_offset)

void	residual_add_token_parallel (TransformerModel *M, size_t input_offset, size_t residual_offset, size_t output_offset)

void	gelu_activation_token_parallel (TransformerModel *M, size_t data_offset)

void	mlp_token_parallel (TransformerModel *M, int layer_idx, size_t input_offset, size_t fc1_weight_offset, size_t fc1_bias_offset, size_t fc1_output_offset, size_t fc2_weight_offset, size_t fc2_bias_offset, size_t output_offset)

void	embed_tokens (TransformerModel M, int32_t token_ids, int num_tokens)

static int	effective_token_count (const TransformerModel *M)

static void	reset_kv_cache (TransformerModel *M)

static void	kv_cache_store_layer (TransformerModel *M, int layer_idx, int tokens)

static void	apply_seq_cls_head (TransformerModel *M, int active_tokens)

static float	compute_seq_cls_loss (TransformerModel *M, int32_t label)

static void	backward_seq_cls_head (TransformerModel *M, int active_tokens)

void	compute_logits_last_token_optimized (TransformerModel *M, int position)

void	transformer_layer_forward (TransformerModel *M, int layer_idx, size_t layer_input_offset)

void	run_comprehensive_benchmark (TransformerModel *M)

int	save_model_weights (TransformerModel M, const char weight_file)
	Load weights into already-allocated TransformerModel.

int	load_model_weights (TransformerModel M, const char weight_file)

int	read_model_metadata (TransformerModel M, const char weight_file)
	Read model metadata from weight file header.

int	sample_token (float *logits, int vocab_size, float temperature)

void	generate (TransformerModel M, int prompt, int prompt_len, int max_tokens)

static void	debug_forward_dump_logits (TransformerModel M, int32_t prompt, int prompt_len, int top_k)

void	zero_gradients (TransformerModel *M)

void	cache_forward_activations (TransformerModel *M)
	Copy forward pass activations to gradient storage for backward pass This preserves the forward computations needed for gradient calculation.

void	backward_residual_connection (TransformerModel *M, size_t d_output_offset, size_t d_input_offset, size_t d_transform_offset)

void	backward_embedding_layer (TransformerModel *M)

void	backward_final_layernorm (TransformerModel *M)

static void	backward_fc2_reference (TransformerModel *M, size_t d_output_offset, size_t fc2_input_copy_offset, size_t fc2_weight_offset, size_t fc2_bias_offset, size_t d_input_offset, size_t d_weight_offset, size_t d_bias_offset)

void	backward_gelu (TransformerModel *M, size_t d_output_offset, size_t input_copy_offset, size_t d_input_offset)

static void	backward_fc2_feature_parallel (TransformerModel *M, size_t d_output_offset, size_t fc2_input_copy_offset, size_t fc2_weight_offset, size_t fc2_bias_offset, size_t d_input_offset, size_t d_weight_offset, size_t d_bias_offset)

void	backward_fc2 (TransformerModel *M, size_t d_output_offset, size_t fc2_input_copy_offset, size_t fc2_weight_offset, size_t fc2_bias_offset, size_t d_input_offset, size_t d_weight_offset, size_t d_bias_offset)

static void	backward_fc1_reference (TransformerModel *M, size_t d_output_offset, size_t fc1_input_copy_offset, size_t fc1_weight_offset, size_t fc1_bias_offset, size_t d_input_offset, size_t d_weight_offset, size_t d_bias_offset)

static void	backward_fc1_feature_parallel (TransformerModel *M, size_t d_output_offset, size_t fc1_input_copy_offset, size_t fc1_weight_offset, size_t fc1_bias_offset, size_t d_input_offset, size_t d_weight_offset, size_t d_bias_offset)

void	backward_fc1 (TransformerModel *M, size_t d_output_offset, size_t fc1_input_copy_offset, size_t fc1_weight_offset, size_t fc1_bias_offset, size_t d_input_offset, size_t d_weight_offset, size_t d_bias_offset)

void	backward_gelu_fast (TransformerModel *M, size_t d_output_offset, size_t input_copy_offset, size_t d_input_offset)

void	backward_layernorm (TransformerModel *M, size_t d_output_offset, size_t input_copy_offset, size_t gamma_copy_offset, size_t beta_copy_offset, size_t mean_copy_offset, size_t rstd_copy_offset, size_t d_input_offset, size_t d_gamma_offset, size_t d_beta_offset)

void	add_gradient (TransformerModel *M, size_t source_offset, size_t dest_offset)

static void	accumulate_linear_weight_bias_feature_parallel (TransformerModel M, float input, float scratch, float d_weights, float *d_bias, int aligned_in, int aligned_out, int T)

void	backward_attention_projection (TransformerModel *M, size_t d_output_offset, size_t attention_output_copy_offset, size_t proj_weight_offset, size_t proj_bias_offset, size_t d_attention_token_offset, size_t d_attention_head_offset, size_t d_weight_offset, size_t d_bias_offset)

void	backward_attention_weighted_values (TransformerModel *M, size_t d_output_offset, size_t attention_weights_offset, size_t v_output_offset, size_t d_weights_offset, size_t d_v_offset)

void	backward_causal_softmax (TransformerModel *M, size_t d_scores_offset, size_t weights_copy_offset)
	Computes the backward pass for the causal softmax function.

void	backward_qk_matmul (TransformerModel *M, size_t d_scores_offset, size_t q_copy_offset, size_t k_copy_offset, size_t d_q_offset, size_t d_k_offset)

static void	reshape_qkv_grad_to_token_major (TransformerModel *M, size_t d_output_offset, size_t scratch_offset)

static void	backward_linear_reference (TransformerModel *M, size_t d_output_offset, size_t input_copy_offset, size_t weight_offset, size_t bias_offset, size_t d_input_offset, size_t d_weight_offset, size_t d_bias_offset, size_t scratch_offset)

static void	backward_linear_feature_parallel (TransformerModel *M, size_t d_output_offset, size_t input_copy_offset, size_t weight_offset, size_t bias_offset, size_t d_input_offset, size_t d_weight_offset, size_t d_bias_offset, size_t scratch_offset)

void	backward_linear (TransformerModel *M, size_t d_output_offset, size_t input_copy_offset, size_t weight_offset, size_t bias_offset, size_t d_input_offset, size_t d_weight_offset, size_t d_bias_offset, size_t scratch_offset)

void	backward_lm_head (TransformerModel *M)

void	backward_transformer_layer (TransformerModel *M, int layer_idx)
	Performs the backward pass for a single transformer layer.

void	compute_cross_entropy_loss (TransformerModel M, int32_t target_tokens, int active_tokens, float *loss_out)
	Compute cross-entropy loss and gradients w.r.t logits.

static void	free_training_pair_list (TrainingPairList *list)

static int	compare_path_strings (const void a, const void b)

static bool	build_training_pair_list (const char dir_path, TrainingPairList out)

static void	initialize_optimizer_state (TransformerModel *M)

static void	apply_optimizer_update (TransformerModel *M, float learning_rate)

static float	compute_scheduled_lr (const TransformerModel *M, float base_lr, int step_index)

static void	debug_forward_dump_hidden (TransformerModel M, int32_t prompt, int prompt_len)

static void	debug_forward_dump_embed (TransformerModel M, int32_t prompt, int prompt_len)

static void	debug_forward_dump_layer_output (TransformerModel M, int32_t prompt, int prompt_len, int layer_idx)

static void	debug_backward_dump_grads_lm (TransformerModel M, int32_t prompt, int prompt_len, int layer_idx)

static bool	shuffle_training_pairs (TrainingPairList *list)

static bool	preload_all_training_windows (const TrainingPairList list, TransformerModel M, PreloadedTrainingData *out, int num_threads)

float	training_step (TransformerModel M, int32_t input_tokens, int32_t *target_tokens, int32_t seq_label, int ctx_len, float learning_rate)

static void	run_training_loop (TransformerModel M, const char train_dir, TrainingPairList list, int total_steps, float learning_rate, int log_interval, const char checkpoint_dir, int checkpoint_interval)

static void	initialize_optimizer_tensor (float *base, size_t weight_offset, size_t count, size_t m_offset, size_t v_offset, size_t ema_offset, bool zero_moments, bool init_ema)

static float	clip_grad_value (float grad, float limit)

static void	sgd_update_tensor (TransformerModel *M, size_t weight_offset, size_t grad_offset, size_t ema_offset, size_t count, float learning_rate, float weight_decay, float ema_decay)

static void	adam_update_tensor (TransformerModel *M, size_t weight_offset, size_t grad_offset, size_t m_offset, size_t v_offset, size_t ema_offset, size_t count, float lr_t, float beta1, float beta2, float eps, float weight_decay, float ema_decay)

static void	update_all_weights_sgd (TransformerModel *M, float learning_rate)

static void	update_all_weights_adam (TransformerModel *M, float learning_rate)

int	main (int argc, char **argv)

Variables
size_t	TrulyOptimalLayer::layer_start_canary_offset

size_t	TrulyOptimalLayer::ln1_weight_offset

size_t	TrulyOptimalLayer::ln1_bias_offset

size_t	TrulyOptimalLayer::ln1_mean_offset

size_t	TrulyOptimalLayer::ln1_rstd_offset

size_t	TrulyOptimalLayer::layer_input_offset

size_t	TrulyOptimalLayer::ln1_output_offset

size_t	TrulyOptimalLayer::q_weight_offset

size_t	TrulyOptimalLayer::q_bias_offset

size_t	TrulyOptimalLayer::q_output_offset

size_t	TrulyOptimalLayer::k_weight_offset

size_t	TrulyOptimalLayer::k_bias_offset

size_t	TrulyOptimalLayer::k_output_offset

size_t	TrulyOptimalLayer::v_weight_offset

size_t	TrulyOptimalLayer::v_bias_offset

size_t	TrulyOptimalLayer::v_output_offset

size_t	TrulyOptimalLayer::attention_scores_offset

size_t	TrulyOptimalLayer::proj_weight_offset

size_t	TrulyOptimalLayer::proj_bias_offset

size_t	TrulyOptimalLayer::attention_output_offset

size_t	TrulyOptimalLayer::residual1_output_offset

size_t	TrulyOptimalLayer::ln2_weight_offset

size_t	TrulyOptimalLayer::ln2_bias_offset

size_t	TrulyOptimalLayer::ln2_mean_offset

size_t	TrulyOptimalLayer::ln2_rstd_offset

size_t	TrulyOptimalLayer::ln2_output_offset

size_t	TrulyOptimalLayer::fc1_weight_offset

size_t	TrulyOptimalLayer::fc1_bias_offset

size_t	TrulyOptimalLayer::fc1_output_offset

size_t	TrulyOptimalLayer::fc2_weight_offset

size_t	TrulyOptimalLayer::fc2_bias_offset

size_t	TrulyOptimalLayer::mlp_output_offset

size_t	TrulyOptimalLayer::residual2_output_offset

size_t	TrulyOptimalLayer::layer_end_canary_offset

size_t	TrulyOptimalLayer::k_cache_offset

size_t	TrulyOptimalLayer::v_cache_offset

size_t	LayerGradients::residual2_copy_offset

size_t	LayerGradients::d_residual2_offset

size_t	LayerGradients::mlp_output_copy_offset

size_t	LayerGradients::d_mlp_output_offset

size_t	LayerGradients::fc2_input_copy_offset

size_t	LayerGradients::fc2_weights_copy_offset

size_t	LayerGradients::fc2_bias_copy_offset

size_t	LayerGradients::d_fc2_input_offset

size_t	LayerGradients::d_fc2_weights_offset

size_t	LayerGradients::d_fc2_bias_offset

size_t	LayerGradients::fc1_output_copy_offset

size_t	LayerGradients::d_fc1_output_offset

size_t	LayerGradients::ln2_output_copy_offset

size_t	LayerGradients::fc1_weights_copy_offset

size_t	LayerGradients::fc1_bias_copy_offset

size_t	LayerGradients::d_ln2_output_offset

size_t	LayerGradients::d_fc1_weights_offset

size_t	LayerGradients::d_fc1_bias_offset

size_t	LayerGradients::ln2_input_copy_offset

size_t	LayerGradients::ln2_mean_copy_offset

size_t	LayerGradients::ln2_rstd_copy_offset

size_t	LayerGradients::ln2_gamma_copy_offset

size_t	LayerGradients::ln2_beta_copy_offset

size_t	LayerGradients::d_ln2_input_offset

size_t	LayerGradients::d_ln2_gamma_offset

size_t	LayerGradients::d_ln2_beta_offset

size_t	LayerGradients::residual1_copy_offset

size_t	LayerGradients::d_residual1_offset

size_t	LayerGradients::attention_output_copy_offset

size_t	LayerGradients::proj_weights_copy_offset

size_t	LayerGradients::proj_bias_copy_offset

size_t	LayerGradients::d_attention_output_offset

size_t	LayerGradients::d_attention_token_offset

size_t	LayerGradients::d_attention_head_offset

size_t	LayerGradients::d_proj_weights_offset

size_t	LayerGradients::d_proj_bias_offset

size_t	LayerGradients::attention_weights_copy_offset

size_t	LayerGradients::v_output_copy_offset

size_t	LayerGradients::d_attention_weights_offset

size_t	LayerGradients::d_v_output_offset

size_t	LayerGradients::d_attention_scores_offset

size_t	LayerGradients::q_output_copy_offset

size_t	LayerGradients::k_output_copy_offset

size_t	LayerGradients::d_q_output_offset

size_t	LayerGradients::d_k_output_offset

size_t	LayerGradients::ln1_output_copy_offset

size_t	LayerGradients::q_weights_copy_offset

size_t	LayerGradients::q_bias_copy_offset

size_t	LayerGradients::k_weights_copy_offset

size_t	LayerGradients::k_bias_copy_offset

size_t	LayerGradients::v_weights_copy_offset

size_t	LayerGradients::v_bias_copy_offset

size_t	LayerGradients::d_ln1_output_offset

size_t	LayerGradients::d_q_weights_offset

size_t	LayerGradients::d_q_bias_offset

size_t	LayerGradients::d_k_weights_offset

size_t	LayerGradients::d_k_bias_offset

size_t	LayerGradients::d_v_weights_offset

size_t	LayerGradients::d_v_bias_offset

size_t	LayerGradients::qkv_scratch_offset

size_t	LayerGradients::ln1_input_copy_offset

size_t	LayerGradients::ln1_mean_copy_offset

size_t	LayerGradients::ln1_rstd_copy_offset

size_t	LayerGradients::ln1_gamma_copy_offset

size_t	LayerGradients::ln1_beta_copy_offset

size_t	LayerGradients::d_ln1_input_offset

size_t	LayerGradients::d_ln1_gamma_offset

size_t	LayerGradients::d_ln1_beta_offset

size_t	LayerGradients::ln1_gamma_m_offset

size_t	LayerGradients::ln1_gamma_v_offset

size_t	LayerGradients::ln1_gamma_ema_offset

size_t	LayerGradients::ln1_beta_m_offset

size_t	LayerGradients::ln1_beta_v_offset

size_t	LayerGradients::ln1_beta_ema_offset

size_t	LayerGradients::q_weight_m_offset

size_t	LayerGradients::q_weight_v_offset

size_t	LayerGradients::q_weight_ema_offset

size_t	LayerGradients::q_bias_m_offset

size_t	LayerGradients::q_bias_v_offset

size_t	LayerGradients::q_bias_ema_offset

size_t	LayerGradients::k_weight_m_offset

size_t	LayerGradients::k_weight_v_offset

size_t	LayerGradients::k_weight_ema_offset

size_t	LayerGradients::k_bias_m_offset

size_t	LayerGradients::k_bias_v_offset

size_t	LayerGradients::k_bias_ema_offset

size_t	LayerGradients::v_weight_m_offset

size_t	LayerGradients::v_weight_v_offset

size_t	LayerGradients::v_weight_ema_offset

size_t	LayerGradients::v_bias_m_offset

size_t	LayerGradients::v_bias_v_offset

size_t	LayerGradients::v_bias_ema_offset

size_t	LayerGradients::proj_weight_m_offset

size_t	LayerGradients::proj_weight_v_offset

size_t	LayerGradients::proj_weight_ema_offset

size_t	LayerGradients::proj_bias_m_offset

size_t	LayerGradients::proj_bias_v_offset

size_t	LayerGradients::proj_bias_ema_offset

size_t	LayerGradients::ln2_gamma_m_offset

size_t	LayerGradients::ln2_gamma_v_offset

size_t	LayerGradients::ln2_gamma_ema_offset

size_t	LayerGradients::ln2_beta_m_offset

size_t	LayerGradients::ln2_beta_v_offset

size_t	LayerGradients::ln2_beta_ema_offset

size_t	LayerGradients::fc1_weight_m_offset

size_t	LayerGradients::fc1_weight_v_offset

size_t	LayerGradients::fc1_weight_ema_offset

size_t	LayerGradients::fc1_bias_m_offset

size_t	LayerGradients::fc1_bias_v_offset

size_t	LayerGradients::fc1_bias_ema_offset

size_t	LayerGradients::fc2_weight_m_offset

size_t	LayerGradients::fc2_weight_v_offset

size_t	LayerGradients::fc2_weight_ema_offset

size_t	LayerGradients::fc2_bias_m_offset

size_t	LayerGradients::fc2_bias_v_offset

size_t	LayerGradients::fc2_bias_ema_offset

size_t	GradientStorage::backprop_base

size_t	GradientStorage::total_gradient_floats

size_t	GradientStorage::logits_copy_offset

size_t	GradientStorage::actual_tokens_offset

size_t	GradientStorage::d_logits_offset

size_t	GradientStorage::final_output_copy_offset

size_t	GradientStorage::d_final_output_offset

size_t	GradientStorage::d_embed_weights_offset

size_t	GradientStorage::final_ln_input_copy_offset

size_t	GradientStorage::final_ln_mean_copy_offset

size_t	GradientStorage::final_ln_rstd_copy_offset

size_t	GradientStorage::final_ln_gamma_copy_offset

size_t	GradientStorage::final_ln_beta_copy_offset

size_t	GradientStorage::d_final_ln_input_offset

size_t	GradientStorage::d_final_ln_gamma_offset

size_t	GradientStorage::d_final_ln_beta_offset

LayerGradients *	GradientStorage::layers

size_t	GradientStorage::d_pos_embed_offset

size_t	GradientStorage::training_pair_tokens_offset

size_t	GradientStorage::training_pair_tokens_count

size_t	GradientStorage::training_pair_tokens_stride

size_t	GradientStorage::training_pair_capacity

size_t	GradientStorage::token_emb_m_offset

size_t	GradientStorage::token_emb_v_offset

size_t	GradientStorage::token_emb_ema_offset

size_t	GradientStorage::pos_emb_m_offset

size_t	GradientStorage::pos_emb_v_offset

size_t	GradientStorage::pos_emb_ema_offset

size_t	GradientStorage::final_ln_gamma_m_offset

size_t	GradientStorage::final_ln_gamma_v_offset

size_t	GradientStorage::final_ln_gamma_ema_offset

size_t	GradientStorage::final_ln_beta_m_offset

size_t	GradientStorage::final_ln_beta_v_offset

size_t	GradientStorage::final_ln_beta_ema_offset

size_t	GradientStorage::seq_cls_weight_m_offset

size_t	GradientStorage::seq_cls_weight_v_offset

size_t	GradientStorage::seq_cls_weight_ema_offset

size_t	GradientStorage::seq_cls_bias_m_offset

size_t	GradientStorage::seq_cls_bias_v_offset

size_t	GradientStorage::seq_cls_bias_ema_offset

size_t	GradientStorage::seq_cls_logits_offset

size_t	GradientStorage::d_seq_cls_logits_offset

size_t	GradientStorage::seq_cls_pooled_offset

size_t	GradientStorage::d_seq_cls_pooled_offset

size_t	GradientStorage::d_seq_cls_weight_offset

size_t	GradientStorage::d_seq_cls_bias_offset

size_t	GradientStorage::layer_backprop_stride

char	TransformerModel::magic [8]
	Magic string "BUMPWGT2" for file validation.

uint32_t	TransformerModel::version
	Weight file format version.

uint32_t	TransformerModel::model_type
	Model architecture: 0=GPT2, 1=LLAMA, etc.

int	TransformerModel::num_layers
	Number of transformer layers (e.g., 12 for GPT-2)

int	TransformerModel::vocab_size
	Vocabulary size (e.g., 50257 for GPT-2)

int	TransformerModel::embed_dim
	Embedding dimension (e.g., 768 for GPT-2 small)

int	TransformerModel::context_window
	Maximum sequence length (e.g., 1024)

size_t	TransformerModel::aligned_embed_dim
	embed_dim rounded up to 64-byte alignment (in floats)

size_t	TransformerModel::aligned_head_dim
	head_dim rounded up to 64-byte alignment (in floats)

size_t	TransformerModel::aligned_attn_context_window
	context_window padded to prevent false sharing

int	TransformerModel::num_cores
	Number of CPU cores to use (OpenMP threads)

int	TransformerModel::tokens_per_core
	Tokens assigned per core: context_window / num_cores.

int	TransformerModel::num_attention_heads
	Number of attention heads (e.g., 12 for GPT-2)

int	TransformerModel::head_dim
	Dimension per head: embed_dim / num_attention_heads.

float *	TransformerModel::memory_base
	Base pointer to single contiguous memory block.

size_t	TransformerModel::total_floats
	Total size of memory block in float elements.

size_t	TransformerModel::layer_stride
	Byte offset between consecutive layer memory blocks.

size_t	TransformerModel::token_emb_offset
	Token embedding table [vocab_size × aligned_embed_dim].

size_t	TransformerModel::pos_emb_offset
	Positional embedding table [context_window × aligned_embed_dim].

size_t	TransformerModel::embedded_input_offset
	Combined token+pos embeddings [context_window × aligned_embed_dim].

size_t	TransformerModel::layers_start_offset
	Start of first transformer layer memory.

TrulyOptimalLayer *	TransformerModel::layers
	Array of per-layer offset structures.

size_t	TransformerModel::final_ln_weight_offset
	Final LayerNorm gamma [aligned_embed_dim].

size_t	TransformerModel::final_ln_bias_offset
	Final LayerNorm beta [aligned_embed_dim].

size_t	TransformerModel::final_ln_mean_offset
	Final LayerNorm mean [context_window].

size_t	TransformerModel::final_ln_rstd_offset
	Final LayerNorm rstd [context_window].

size_t	TransformerModel::final_output_offset
	Final normalized output [context_window × aligned_embed_dim].

size_t	TransformerModel::lm_head_weight_offset
	Language model head (weight-tied to token_emb_offset)

size_t	TransformerModel::logits_offset
	Output logits [context_window × vocab_size].

GradientStorage	TransformerModel::gradients
	Gradient and activation cache memory (training only)

bool	TransformerModel::training_enabled
	Whether gradient storage is allocated.

float	TransformerModel::learning_rate
	SGD learning rate for weight updates.

int	TransformerModel::lr_warmup_steps
	Number of steps to linearly warm up the LR.

float	TransformerModel::lr_warmup_init
	Starting LR for warmup (defaults to 0 when unset)

float	TransformerModel::grad_clip
	Clip gradients to +/- this value (0 disables)

size_t	TransformerModel::training_cache_samples
	Number of training windows cached inside arena.

int	TransformerModel::active_tokens
	Number of tokens populated in current forward pass.

TaskType	TransformerModel::task_type
	Active training/inference task.

OptimizerType	TransformerModel::optimizer
	Selected optimizer.

uint64_t	TransformerModel::optimizer_step
	Number of optimization steps taken.

float	TransformerModel::adam_beta1

float	TransformerModel::adam_beta2

float	TransformerModel::adam_eps

float	TransformerModel::weight_decay

bool	TransformerModel::ema_enabled

float	TransformerModel::ema_decay

bool	TransformerModel::optimizer_state_initialized

bool	TransformerModel::seq_cls_enabled

int	TransformerModel::seq_cls_num_classes

int	TransformerModel::seq_cls_pooling
	0=final,1=cls,2=mean

size_t	TransformerModel::seq_cls_weight_offset

size_t	TransformerModel::seq_cls_bias_offset

bool	TransformerModel::kv_cache_enabled
	Whether KV cache regions are allocated/used.

int	TransformerModel::kv_cache_capacity
	Maximum tokens stored in cache.

int	TransformerModel::kv_cache_tokens
	Number of valid tokens currently cached.

long *	TransformerModel::training_data_buffer
	Pointer to in-memory buffer for the entire training dataset.

long	TransformerModel::num_training_tokens
	Total number of tokens in the preloaded dataset.

uint8_t	TransformerModel::checksum [32]
	SHA256 checksum of weight file.

uint8_t	TransformerModel::reserved [32]
	Reserved for future extensions.

char **	TrainingPairList::paths

size_t	TrainingPairList::count

uint32_t *	PreloadedTrainingData::cache_base

size_t	PreloadedTrainingData::count

size_t	PreloadedTrainingData::tokens_per_pair

size_t	PreloadedTrainingData::stride

uint16_t *	PreloadedTrainingData::ctx_lengths

uint16_t *	PreloadedTrainingData::target_lengths

Detailed Description

Matrix multiplication implementations with different optimization strategies.

Macro Definition Documentation

◆ ATTN_SCORES_ACCESS

#define ATTN_SCORES_ACCESS	(	scores_ptr,
		h,
		i,
		j,
		aligned_context_window
	)	scores_ptr[((h) * (aligned_context_window) * (aligned_context_window)) + ((i) * (aligned_context_window)) + (j)]

Function Documentation

◆ accumulate_linear_weight_bias_feature_parallel()

static void accumulate_linear_weight_bias_feature_parallel	(	TransformerModel *	M,
		float *	input,
		float *	scratch,
		float *	d_weights,
		float *	d_bias,
		int	aligned_in,
		int	aligned_out,
		int	T
	)

inlinestatic

◆ adam_update_tensor()

static void adam_update_tensor	(	TransformerModel *	M,
		size_t	weight_offset,
		size_t	grad_offset,
		size_t	m_offset,
		size_t	v_offset,
		size_t	ema_offset,
		size_t	count,
		float	lr_t,
		float	beta1,
		float	beta2,
		float	eps,
		float	weight_decay,
		float	ema_decay
	)

inlinestatic

◆ add_gpt2_token_and_positional_embeddings()

void add_gpt2_token_and_positional_embeddings	(	TransformerModel *	M,
		size_t	token_ids_offset,
		size_t	output_offset
	)

◆ add_gradient()

void add_gradient	(	TransformerModel *	M,
		size_t	source_offset,
		size_t	dest_offset
	)

ADD GRADIENT (accumulate gradients from residual path)

This is used when gradients from multiple paths need to be summed. For example, at a residual connection, gradients flow through both:

The transformation path (MLP or attention)
The skip connection path

Both gradients need to be added together.

◆ aligned_alloc_64()

static void * aligned_alloc_64 ( size_t size )

inlinestatic

◆ apply_causal_softmax_head_major()

void apply_causal_softmax_head_major	(	TransformerModel *	M,
		int	layer_idx
	)

◆ apply_optimizer_update()

static void apply_optimizer_update	(	TransformerModel *	M,
		float	learning_rate
	)

inlinestatic

◆ apply_seq_cls_head()

static void apply_seq_cls_head	(	TransformerModel *	M,
		int	active_tokens
	)

static

◆ attention_head_major_complete()

void attention_head_major_complete	(	TransformerModel *	M,
		int	layer_idx
	)

Complete multi-head attention with head-major layout (self-attention)

Computes scaled dot-product attention using head-major memory layout for optimal cache locality. Each attention head operates independently, enabling head-level parallelism and cache-efficient processing.

Parameters

M	Transformer model with memory layout and configuration
layer_idx	Layer index for accessing Q/K/V tensors

Head-Level Parallelism Strategy: Unlike token-parallel operations (LayerNorm, GELU), attention parallelizes across HEADS because each head's computation is independent.

Memory Layout (Head-Major):

Q, K, V Tensors: [num_heads][context_window][head_dim]
 
Head 0:  [Token0: 64f] [Token1: 64f] ... [TokenN: 64f]
Head 1:  [Token0: 64f] [Token1: 64f] ... [TokenN: 64f]
...
Head 11: [Token0: 64f] [Token1: 64f] ... [TokenN: 64f]
 
Each head's data is CONTIGUOUS (no interleaving with other heads)

Why Head-Major Layout?:

✅ Perfect Locality for Q·K^T: All data for one head fits in L2 cache
✅ Attention Matrix in Cache: For 64-dim head, 1024 tokens:
- Score matrix: 1024 × 1024 × 4 bytes = 4MB (fits in L3)
- Per-head Q: 1024 × 64 × 4 bytes = 256KB (fits in L2)
- Per-head K: 1024 × 64 × 4 bytes = 256KB (fits in L2)
✅ No Strided Access: Sequential reads within each head
✅ Head Parallelism: 12 independent heads = 12 parallel tasks

Three-Phase Attention Algorithm:

Phase 1: Compute Attention Scores (Q·K^T / √d_k)

For each head h in parallel:
  For each query token i:
    For each key token j (where j <= i for causal masking):
      scores[h][i][j] = (Q[h][i] · K[h][j]) / sqrt(head_dim)

FLOPs: num_heads × T × (T+1)/2 × head_dim × 2
Memory: Streaming reads of Q and K
Cache: Each head's scores fit in L1 (1024×1024 floats = 4KB)

Phase 2: Causal Softmax

For each head h in parallel:
  For each query token i:
    scores[h][i][0:i+1] = softmax(scores[h][i][0:i+1])
    scores[h][i][i+1:T] = 0  (causal mask)

Prevents attending to future tokens (autoregressive)
Row-wise softmax for numerical stability
FLOPs: num_heads × T × (T+1)/2 × 5 (exp, sum, divide, max)

Phase 3: Weighted Sum of Values (Softmax·V)

For each head h in parallel:
  For each query token i:
    output[h][i] = Σ_{j=0}^{i} scores[h][i][j] * V[h][j]

FLOPs: num_heads × T × (T+1)/2 × head_dim × 2
Produces per-head attention output in head-major layout

Stride Pattern Access: Access to Q[h][t][d]:

offset = h * (context_window * aligned_head_dim) +
         t * aligned_head_dim +
         d

aligned_head_dim ensures 64-byte alignment (prevents false sharing)
Sequential access within a head (hardware prefetcher friendly)
Each head occupies separate cache lines

Performance Characteristics:

Compute: O(num_heads × T² × head_dim)
Memory: O(num_heads × T²) for attention scores
Parallelism: Scales with min(num_heads, num_cores)
Cache: L2/L3 critical (must fit score matrix + Q/K/V for one head)

Comparison: Token-Parallel vs Head-Parallel:

Operation	Parallelism	Memory Pattern	Cache Footprint
LayerNorm	Token	Sequential	3KB per token
Attention	Head	Strided	512KB per head
MLP	Token	Sequential	3KB per token

Why NOT Token-Parallel for Attention?:

Attention requires ALL token pairs (Q[i] · K[j] for all i,j)
Token parallelism would require synchronization at score matrix
Head-major layout allows independent head computation

Note: This function orchestrates all three attention phases

See also: compute_attention_scores_head_major Phase 1: Q·K^T; apply_causal_softmax_head_major Phase 2: Softmax with causal mask; compute_attention_output_head_major Phase 3: Attention·V; Q_ACCESS Head-major memory access macro

@performance Achieves 100-200 GFLOPS on attention computation (Xeon Gold 6248)

◆ attention_projection_with_concat()

void attention_projection_with_concat	(	TransformerModel *	M,
		int	layer_idx
	)

Production attention projection with concat: Head-major → Token-major → GEMM.

This function implements the concat strategy that proved 3.2x faster than direct head-major projection on hyperthreaded systems.

Parameters

M	Transformer model
layer_idx	Layer index to process

Memory flow (important for understanding residual usage):

Input (read-only): L->attention_output_offset = head-major attention tensor [head][token][head_dim]
Scratch (temporary): L->residual1_output_offset = used here as a concat buffer for the projected heads in token-major layout [token][embed_dim]
Output of c_proj (before residual add): L->residual2_output_offset = token-major projection result [token][embed_dim]

Later in transformer_layer_forward, we do: RES1 = h_in + (data at residual2_output_offset) and write that sum to residual1_output_offset. After the MLP block, residual2_output_offset is reused to hold the final layer output RES2.

◆ backward_attention_projection()

void backward_attention_projection	(	TransformerModel *	M,
		size_t	d_output_offset,
		size_t	attention_output_copy_offset,
		size_t	proj_weight_offset,
		size_t	proj_bias_offset,
		size_t	d_attention_token_offset,
		size_t	d_attention_head_offset,
		size_t	d_weight_offset,
		size_t	d_bias_offset
	)

BACKWARD THROUGH ATTENTION OUTPUT PROJECTION

Forward: output[T×D] = attention[T×D] @ W_proj[D×D] + b_proj[D]

This is after concatenating all heads back to [T×D] format

Backward computes:

d_attention[T×D] = d_output[T×D] @ W_proj^T[D×D]
d_W_proj[D×D] = attention^T[D×T] @ d_output[T×D]
d_b_proj[D] = sum(d_output) over T

◆ backward_attention_weighted_values()

void backward_attention_weighted_values	(	TransformerModel *	M,
		size_t	d_output_offset,
		size_t	attention_weights_offset,
		size_t	v_output_offset,
		size_t	d_weights_offset,
		size_t	d_v_offset
	)

BACKWARD THROUGH ATTENTION WEIGHTED VALUES

Forward: attention_output[h,t,d] = sum_over_s(attention_weights[h,t,s] * V[h,s,d])

This operates in HEAD-MAJOR layout

Backward computes:

d_attention_weights[h,t,s] = sum_over_d(d_output[h,t,d] * V[h,s,d])
d_V[h,s,d] = sum_over_t(attention_weights[h,t,s] * d_output[h,t,d])

◆ backward_causal_softmax()

void backward_causal_softmax	(	TransformerModel *	M,
		size_t	d_scores_offset,
		size_t	weights_copy_offset
	)

Computes the backward pass for the causal softmax function.

Parameters

M	The transformer model.
d_scores_offset	Offset to the gradient of the attention weights (dL/dWeights), which is both the input and output of this function (dL/dScores).
weights_copy_offset	Offset to the cached attention weights (the output of the forward softmax).

This function calculates the gradient of the loss with respect to the pre-softmax attention scores (dL/dScores) given the gradient with respect to the post-softmax attention weights (dL/dWeights). It does this for each row of the attention matrix independently, taking the causal mask into account.

The operation is performed in-place, meaning the input d_scores_offset buffer is overwritten with the output gradients.

Mathematical Derivation: The Softmax Jacobian-Vector Product

The core of this function is calculating the Jacobian-vector product for the softmax function. Let y = softmax(x), where x is a vector of pre-softmax scores and y is the vector of resulting probabilities. The backward pass requires us to compute dL/dx, given dL/dy.

By the chain rule: dL/dx = (dL/dy) * (dy/dx). The term dy/dx is the Jacobian matrix J of the softmax function.

The Jacobian J of softmax has a specific structure: J[i, j] = y[i] * (kronecker_delta(i, j) - y[j]) where kronecker_delta(i, j) is 1 if i == j and 0 otherwise.

This means:

On the diagonal (i=j): J[i, i] = y[i] * (1 - y[i])
Off the diagonal (i!=j): J[i, j] = -y[i] * y[j]

A naive backward pass would be to form this N x N matrix J and multiply it by the N x 1 vector dL/dy. This is computationally expensive (O(N^2)).

The Efficient Method (Jacobian-Vector Product):

We can compute the product J @ (dL/dy) without ever forming J. Let's expand the product for a single element i of the output dL/dx:

(dL/dx)[i] = sum_j ( J[i, j] * (dL/dy)[j] ) = sum_j ( y[i] * (delta(i, j) - y[j]) * (dL/dy)[j] ) = y[i] * sum_j ( (delta(i, j) - y[j]) * (dL/dy)[j] ) = y[i] * ( (delta(i, i) - y[i]) * (dL/dy)[i] + sum_{j!=i} (-y[j] * (dL/dy)[j]) ) = y[i] * ( (dL/dy)[i] - y[i]*(dL/dy)[i] - sum_{j!=i} (y[j] * (dL/dy)[j]) )

Notice that y[i]*(dL/dy)[i] + sum_{j!=i} (y[j] * (dL/dy)[j]) is simply the dot product of the vector y and the vector dL/dy. Let's call this dot(y, dL/dy).

So, the equation simplifies to: (dL/dx)[i] = y[i] * ( (dL/dy)[i] - dot(y, dL/dy) )

This is the formula implemented in the code. It's an O(N) operation, as it only requires one pass to compute the dot product and a second pass to apply the formula.

In our code:

x are the pre-softmax scores (scores).
y are the post-softmax probabilities (weights).
dL/dy is the incoming gradient (d_scores_inout).
dot(y, dL/dy) is dot_product.
(dL/dx)[i] is the final value written back to d_scores_inout.

Implementation Details

Parallelism: The computation is parallelized across heads (h) and query tokens (i) using omp parallel for collapse(2). This is safe because each row i of the attention matrix for each head h is processed independently.
Causal Mask: The loops for j run from 0 to i. This correctly applies the causal mask, ensuring that the dot product and the final gradient calculation only include terms from tokens that were attended to in the forward pass.
In-Place Operation: The code reads from d_scores_inout to calculate the dot_product, then reads from it again inside the final loop while overwriting it with the new value. This is safe because the dot_product is computed and stored in a local variable before the overwrite loop begins.

See also: backward_attention_weighted_values Where the input gradient dL/dWeights comes from.; backward_qk_matmul Where the output gradient dL/dScores is consumed.

◆ backward_embedding_layer()

void backward_embedding_layer ( TransformerModel * M )

◆ backward_fc1()

void backward_fc1	(	TransformerModel *	M,
		size_t	d_output_offset,
		size_t	fc1_input_copy_offset,
		size_t	fc1_weight_offset,
		size_t	fc1_bias_offset,
		size_t	d_input_offset,
		size_t	d_weight_offset,
		size_t	d_bias_offset
	)

◆ backward_fc1_feature_parallel()

static void backward_fc1_feature_parallel	(	TransformerModel *	M,
		size_t	d_output_offset,
		size_t	fc1_input_copy_offset,
		size_t	fc1_weight_offset,
		size_t	fc1_bias_offset,
		size_t	d_input_offset,
		size_t	d_weight_offset,
		size_t	d_bias_offset
	)

static

◆ backward_fc1_reference()

static void backward_fc1_reference	(	TransformerModel *	M,
		size_t	d_output_offset,
		size_t	fc1_input_copy_offset,
		size_t	fc1_weight_offset,
		size_t	fc1_bias_offset,
		size_t	d_input_offset,
		size_t	d_weight_offset,
		size_t	d_bias_offset
	)

static

BACKWARD THROUGH FC1 (Feed-Forward Layer 1)

FORWARD PASS (for reference): ──────────────────────────────────────────────────────────────── Input: fc1_input [T × D] (output from LayerNorm2) Weight: W_fc1 [D × 4D] (projects from D to 4D) Bias: b_fc1 [4D] Output: fc1_output [T × 4D] = fc1_input @ W_fc1 + b_fc1

BACKWARD PASS: ──────────────────────────────────────────────────────────────── Given: d_output [T × 4D] (gradient from GELU backward)

Need to compute:

d_input [T × D] = d_output @ W_fc1^T
d_W_fc1 [D × 4D] = fc1_input^T @ d_output (accumulated)
d_b_fc1 [4D] = sum over T of d_output (accumulated)

DIMENSION FLOW: d_output[T×4D] ──┬──> @ W_fc1^T[4D×D] ──> d_input[T×D] │ ├──> fc1_input^T[D×T] @ ──> d_W_fc1[D×4D] │ └──> sum_over_T ──> d_b_fc1[4D]

HPC CONSIDERATIONS:

FC1 expands dimensions (D -> 4D), so weight matrix is large
Memory bandwidth critical for weight gradient accumulation
Consider chunking for better cache reuse
Token parallelism for d_input computation

◆ backward_fc2()

void backward_fc2	(	TransformerModel *	M,
		size_t	d_output_offset,
		size_t	fc2_input_copy_offset,
		size_t	fc2_weight_offset,
		size_t	fc2_bias_offset,
		size_t	d_input_offset,
		size_t	d_weight_offset,
		size_t	d_bias_offset
	)

◆ backward_fc2_feature_parallel()

static void backward_fc2_feature_parallel	(	TransformerModel *	M,
		size_t	d_output_offset,
		size_t	fc2_input_copy_offset,
		size_t	fc2_weight_offset,
		size_t	fc2_bias_offset,
		size_t	d_input_offset,
		size_t	d_weight_offset,
		size_t	d_bias_offset
	)

static

◆ backward_fc2_reference()

static void backward_fc2_reference	(	TransformerModel *	M,
		size_t	d_output_offset,
		size_t	fc2_input_copy_offset,
		size_t	fc2_weight_offset,
		size_t	fc2_bias_offset,
		size_t	d_input_offset,
		size_t	d_weight_offset,
		size_t	d_bias_offset
	)

static

BACKWARD THROUGH FC2 (Feed-Forward Layer 2)

FORWARD PASS (for reference): ──────────────────────────────────────────────────────────────── Input: fc2_input [T × 4D] (after GELU activation) Weight: W_fc2 [4D × D] (projects from 4D back to D) Bias: b_fc2 [D] Output: fc2_output [T × D] = fc2_input @ W_fc2 + b_fc2

BACKWARD PASS: ──────────────────────────────────────────────────────────────── Given: d_output [T × D] (gradient from residual connection)

Need to compute:

d_input [T × 4D] = d_output @ W_fc2^T
d_W_fc2 [4D × D] = fc2_input^T @ d_output (accumulated)
d_b_fc2 [D] = sum over T of d_output (accumulated)

DIMENSION FLOW: d_output[T×D] ──┬──> @ W_fc2^T[D×4D] ──> d_input[T×4D] │ ├──> fc2_input^T[4D×T] @ ──> d_W_fc2[4D×D] │ └──> sum_over_T ──> d_b_fc2[D]

HPC CONSIDERATIONS:

Token-parallel for d_input computation (each thread handles tokens)
Reduction required for weight/bias gradients (atomic ops or local accumulation)
Memory bandwidth bound due to large weight matrix (4D×D)
Cache blocking beneficial for weight gradient accumulation

◆ backward_final_layernorm()

void backward_final_layernorm ( TransformerModel * M )

◆ backward_gelu()

void backward_gelu	(	TransformerModel *	M,
		size_t	d_output_offset,
		size_t	input_copy_offset,
		size_t	d_input_offset
	)

BACKWARD THROUGH GELU ACTIVATION

FORWARD PASS (for reference): ──────────────────────────────────────────────────────────────── GELU(x) = 0.5 * x * (1 + tanh(√(2/π) * (x + 0.044715 * x³)))

Approximation used in practice: GELU(x) ≈ 0.5 * x * (1 + tanh(0.7978845608 * (x + 0.044715 * x³)))

BACKWARD PASS (derivative): ──────────────────────────────────────────────────────────────── d/dx[GELU(x)] = 0.5 * tanh(g(x)) + 0.5 * x * sech²(g(x)) * g'(x) + 0.5 where: g(x) = 0.7978845608 * (x + 0.044715 * x³) g'(x) = 0.7978845608 * (1 + 3 * 0.044715 * x²)

Simplified form: GELU'(x) = 0.5 * (1 + tanh(g(x))) + 0.5 * x * sech²(g(x)) * g'(x) = 0.5 * (1 + tanh(g(x))) + 0.5 * x * (1 - tanh²(g(x))) * g'(x)

DIMENSION FLOW: d_output [T × 4D] ──> × GELU'(input) ──> d_input [T × 4D]

HPC CONSIDERATIONS:

Element-wise operation (embarrassingly parallel)
Compute bound (tanh is expensive)
Can fuse with surrounding operations for better cache usage
Consider using fast tanh approximations for speed

◆ backward_gelu_fast()

void backward_gelu_fast	(	TransformerModel *	M,
		size_t	d_output_offset,
		size_t	input_copy_offset,
		size_t	d_input_offset
	)

Alternative: Fast approximation using precomputed GELU derivative This version trades accuracy for speed by using a simpler approximation

◆ backward_layernorm()

void backward_layernorm	(	TransformerModel *	M,
		size_t	d_output_offset,
		size_t	input_copy_offset,
		size_t	gamma_copy_offset,
		size_t	beta_copy_offset,
		size_t	mean_copy_offset,
		size_t	rstd_copy_offset,
		size_t	d_input_offset,
		size_t	d_gamma_offset,
		size_t	d_beta_offset
	)

BACKWARD THROUGH LAYERNORM

FORWARD (for reference): x_hat = (x - mean) / rstd y = gamma * x_hat + beta

BACKWARD: Given d_y, compute d_x, d_gamma, d_beta

The math (per token): d_x = (rstd/D) * [D * d_y * gamma - sum(d_y * gamma) - x_hat * sum(d_y * gamma * x_hat)] d_gamma = sum_over_tokens(d_y * x_hat) d_beta = sum_over_tokens(d_y)

◆ backward_linear()

void backward_linear	(	TransformerModel *	M,
		size_t	d_output_offset,
		size_t	input_copy_offset,
		size_t	weight_offset,
		size_t	bias_offset,
		size_t	d_input_offset,
		size_t	d_weight_offset,
		size_t	d_bias_offset,
		size_t	scratch_offset
	)

◆ backward_linear_feature_parallel()

static void backward_linear_feature_parallel	(	TransformerModel *	M,
		size_t	d_output_offset,
		size_t	input_copy_offset,
		size_t	weight_offset,
		size_t	bias_offset,
		size_t	d_input_offset,
		size_t	d_weight_offset,
		size_t	d_bias_offset,
		size_t	scratch_offset
	)

static

◆ backward_linear_reference()

static void backward_linear_reference	(	TransformerModel *	M,
		size_t	d_output_offset,
		size_t	input_copy_offset,
		size_t	weight_offset,
		size_t	bias_offset,
		size_t	d_input_offset,
		size_t	d_weight_offset,
		size_t	d_bias_offset,
		size_t	scratch_offset
	)

static

BACKWARD THROUGH LINEAR LAYER (GENERIC)

Forward: output = input @ W + bias

This handles Q, K, V projections which are all linear layers Note: For QKV, the output is in head-major format but input is token-major

Backward: d_input += d_output @ W^T (accumulate because QKV all contribute) d_W += input^T @ d_output d_bias += sum(d_output)

◆ backward_lm_head()

void backward_lm_head ( TransformerModel * M )

◆ backward_qk_matmul()

void backward_qk_matmul	(	TransformerModel *	M,
		size_t	d_scores_offset,
		size_t	q_copy_offset,
		size_t	k_copy_offset,
		size_t	d_q_offset,
		size_t	d_k_offset
	)

BACKWARD THROUGH Q @ K^T

Forward: scores[h,i,j] = sum_d(Q[h,i,d] * K[h,j,d]) / sqrt(head_dim)

Backward: d_Q[h,i,d] = sum_j(d_scores[h,i,j] * K[h,j,d]) / sqrt(head_dim) d_K[h,j,d] = sum_i(d_scores[h,i,j] * Q[h,i,d]) / sqrt(head_dim)

Note: Causal mask means d_scores[h,i,j] = 0 for j > i

◆ backward_residual_connection()

void backward_residual_connection	(	TransformerModel *	M,
		size_t	d_output_offset,
		size_t	d_input_offset,
		size_t	d_transform_offset
	)

RESIDUAL CONNECTION - FORWARD & BACKWARD PROPAGATION

CONCEPT: A residual connection (skip connection) allows gradients to flow directly through the network by adding the input to the output of a transformation. This addresses the vanishing gradient problem in deep networks.

FORWARD PASS: ──────────────────────────────────────────────────────────────── input (x) │ ├────────────────┐ (identity path / skip connection) │ │ ▼ │ ┌─────────┐ │ │ F(x) │ │ │ (trans- │ │ │ form) │ │ └─────────┘ │ │ │ ▼ ▼ F(x) + x └────────┬───────┘ ▼ output = F(x) + x

Mathematical form: output = input + transform(input)

In transformers specifically: output = input + MultiHeadAttention(LayerNorm(input)) output = input + FFN(LayerNorm(input))

BACKWARD PASS: ────────────────────────────────────────────────────────────────

Given: d_output = ∂L/∂output (gradient from layer above) Need: d_input = ∂L/∂input and d_transform = ∂L/∂transform

Since output = input + transform, by the chain rule: ∂output/∂input = 1 (derivative of input w.r.t itself) ∂output/∂transform = 1 (derivative of transform w.r.t itself)

Therefore: d_input = d_output × 1 = d_output d_transform = d_output × 1 = d_output

BACKWARD FLOW: d_output │ ┌──────────┴──────────┐ │ │ ▼ ▼ d_transform d_input (gradient flows (gradient flows to transform) directly through)

KEY INSIGHT: The gradient d_output flows EQUALLY through both paths:

Through the transformation (to update its parameters)
Directly to the input (skip connection)

This is why residual connections help with vanishing gradients: even if the transformation has small gradients, the skip path ensures gradients can flow directly to earlier layers.

IMPLEMENTATION NOTES:

Both gradients receive the SAME value (d_output)
Use += (accumulation) not = (assignment) in case gradients already exist from other paths
This simple operation is crucial for training deep networks

Parameters

M	Transformer model
d_output_offset	Gradient from the layer above
d_input_offset	Where to accumulate gradient for input path
d_transform_offset	Where to accumulate gradient for transform path

◆ backward_seq_cls_head()

static void backward_seq_cls_head	(	TransformerModel *	M,
		int	active_tokens
	)

static

◆ backward_transformer_layer()

void backward_transformer_layer	(	TransformerModel *	M,
		int	layer_idx
	)

Performs the backward pass for a single transformer layer.

Parameters

M	The transformer model.
layer_idx	The index of the layer to process.

This function orchestrates the backpropagation of gradients through one entire transformer layer, from its output back to its input. It follows the chain rule in reverse order of the forward pass operations.

End-to-End Gradient Flow

The gradient flows from the output of the layer (d_layer_output) back to its input (d_ln1_input), accumulating gradients for all weights and biases along the way.

*
*                                d_layer_output (from layer L+1 or final LN)
*                                      │
*                                      ▼
*      [ 8. Backward through 2nd Residual Connection (d_residual2) ]
*         /                                       \
*        ▼                                         ▼
*  d_residual1 (accumulated)                  d_mlp_output
*        │                                         │
*        │                                         ▼
*        │                             [ 7. Backward through MLP Block ]
*        │                                (FC2 -> GELU -> FC1)
*        │                                         │
*        │                                         ▼
*        │                                    d_ln2_output
*        │                                         │
*        │                                         ▼
*        │                           [ 6. Backward through 2nd LayerNorm ]
*        │                                         │
*        │                                         ▼
*        └──────────────────────────────────> d_ln2_input (accumulated to d_residual1)
*                                                  │
*                                                  ▼
*      [ 5. Backward through 1st Residual Connection (d_residual1) ]
*         /                                       \
*        ▼                                         ▼
*  d_ln1_input (to layer L-1)               d_attention_output
*                                                  │
*                                                  ▼
*                               [ 4. Backward through Attention Projection ]
*                                                  │
*                                                  ▼
*                                        d_attention_heads
*                                                  │
*                                                  ▼
*                               [ 3. Backward through Attention Mechanism ]
*                               (Scores*V -> Softmax -> QK^T)
*                                    /         |           \
*                                   ▼          ▼            ▼
*                                 d_Q        d_K          d_V
*                                   │          │            │
*                                   ▼          ▼            ▼
*                         [ 2. Backward through Q, K, V Projections ]
*                                   (Gradients accumulate into d_ln1_output)
*                                                  │
*                                                  ▼
*                            [ 1. Backward through 1st LayerNorm ]
*                                                  │
*                                                  ▼
*                                        d_ln1_input (Final output gradient for this layer)
*
*

Key Steps:

Residual Connections: The backward_residual_connection function splits the incoming gradient, sending it down both the skip-connection path and the transformation path. The gradient from the skip path is added to the gradient from the transformation's input.
MLP & Attention Blocks: Each block (MLP, Attention) takes an incoming gradient and computes gradients for its own weights/biases and for its input.
LayerNorm: The backward_layernorm function computes gradients for gamma/beta and for the LayerNorm's input.
QKV Projections: The backward_linear function is used for each of the Q, K, and V projections. A crucial detail is that the gradients for their inputs (d_ln1_output) are accumulated, because all three projections originated from the same source tensor in the forward pass.
Final Output: The final gradient d_ln1_input becomes the input gradient for the preceding layer (layer_idx - 1), continuing the chain rule.

See also: backward_residual_connection; backward_mlp (conceptual, composed of backward_fc2, backward_gelu, backward_fc1); backward_attention_mechanism (conceptual, composed of multiple functions); backward_layernorm

◆ benchmark_attention_projection_complete()

void benchmark_attention_projection_complete ( TransformerModel * M )

◆ benchmark_qkv_dual_comparison()

void benchmark_qkv_dual_comparison ( TransformerModel * M )

◆ build_training_pair_list()

static bool build_training_pair_list	(	const char *	dir_path,
		TrainingPairList *	out
	)

static

◆ cache_forward_activations()

void cache_forward_activations ( TransformerModel * M )

Copy forward pass activations to gradient storage for backward pass This preserves the forward computations needed for gradient calculation.

◆ clip_grad_value()

static float clip_grad_value	(	float	grad,
		float	limit
	)

inlinestatic

◆ compare_arrays()

double compare_arrays	(	const float *	a,
		const float *	b,
		size_t	size,
		const char *	name
	)

◆ compare_path_strings()

static int compare_path_strings	(	const void *	a,
		const void *	b
	)

static

◆ compute_attention_output_head_major()

void compute_attention_output_head_major	(	TransformerModel *	M,
		int	layer_idx
	)

◆ compute_attention_scores_head_major()

void compute_attention_scores_head_major	(	TransformerModel *	M,
		int	layer_idx
	)

◆ compute_cross_entropy_loss()

void compute_cross_entropy_loss	(	TransformerModel *	M,
		int32_t *	target_tokens,
		int	active_tokens,
		float *	loss_out
	)

Compute cross-entropy loss and gradients w.r.t logits.

Loss = -sum(log(p[correct])) / context_length Gradient: dL/dlogit[i] = p[i] - 1 (for correct token) dL/dlogit[i] = p[i] (for other tokens)

◆ compute_logits_last_token_optimized()

void compute_logits_last_token_optimized	(	TransformerModel *	M,
		int	position
	)

◆ compute_scheduled_lr()

static float compute_scheduled_lr	(	const TransformerModel *	M,
		float	base_lr,
		int	step_index
	)

inlinestatic

◆ compute_seq_cls_loss()

static float compute_seq_cls_loss	(	TransformerModel *	M,
		int32_t	label
	)

static

◆ convert_token_major_to_head_major_layer()

void convert_token_major_to_head_major_layer	(	const float *	token_major_base,
		float *	head_major_base,
		TransformerModel *	M
	)

◆ debug_backward_dump_grads_lm()

static void debug_backward_dump_grads_lm	(	TransformerModel *	M,
		int32_t *	prompt,
		int	prompt_len,
		int	layer_idx
	)

static

◆ debug_forward_dump_embed()

static void debug_forward_dump_embed	(	TransformerModel *	M,
		int32_t *	prompt,
		int	prompt_len
	)

static

◆ debug_forward_dump_hidden()

static void debug_forward_dump_hidden	(	TransformerModel *	M,
		int32_t *	prompt,
		int	prompt_len
	)

static

◆ debug_forward_dump_layer_output()

static void debug_forward_dump_layer_output	(	TransformerModel *	M,
		int32_t *	prompt,
		int	prompt_len,
		int	layer_idx
	)

static

◆ debug_forward_dump_logits()

static void debug_forward_dump_logits	(	TransformerModel *	M,
		int32_t *	prompt,
		int	prompt_len,
		int	top_k
	)

static

◆ debug_math_comparison()

void debug_math_comparison ( TransformerModel * M )

◆ effective_token_count()

static int effective_token_count ( const TransformerModel * M )

inlinestatic

◆ embed_tokens()

void embed_tokens	(	TransformerModel *	M,
		int32_t *	token_ids,
		int	num_tokens
	)

◆ free_training_pair_list()

static void free_training_pair_list ( TrainingPairList * list )

static

◆ gelu_activation_token_parallel()

void gelu_activation_token_parallel	(	TransformerModel *	M,
		size_t	data_offset
	)

◆ gemm_avx512_parallel()

void gemm_avx512_parallel	(	const float *	A,
		const float *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

AVX-512 optimized GEMM with vectorized inner loops.

Parameters

A	Input matrix A [M x K]
B	Input matrix B [N x K] (transposed)
bias	Bias vector [N]
C	Output matrix C [M x N]
M	Number of rows in A and C
N	Number of columns in B and C
K	Inner dimension

@performance

Target: 200-400 GFLOPS on modern Xeon
16-wide SIMD operations
FMA instruction utilization

@optimization_details

Uses _mm512_fmadd_ps for 3 FLOPs per instruction
16-element vectorization of inner loop
Handles remainder elements with scalar code

See also: gemm_naive_parallel for reference implementation

◆ gemm_blocked_serial()

void gemm_blocked_serial	(	const float *	A,
		const float *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

◆ gemm_fine_grained_parallel()

void gemm_fine_grained_parallel	(	const float *	A,
		const float *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Cache-blocked GEMM with fine-grained parallelism.

Parameters

A	Input matrix A [M x K]
B	Input matrix B [N x K] (transposed)
bias	Bias vector [N]
C	Output matrix C [M x N]
M	Number of rows in A and C
N	Number of columns in B and C
K	Inner dimension

@performance

Target: 300-500 GFLOPS
Best performance for large matrices
Optimal cache utilization

@implementation_notes

64x64 blocking for L1 cache optimization
Collapse(3) OpenMP directive for maximum parallelism
Atomic updates for thread safety

@benchmark_results Tested on 8192x8192 matrices:

Naive: 85 GFLOPS
This impl: 474 GFLOPS (5.6x speedup)

◆ gemm_naive_parallel()

void gemm_naive_parallel	(	const float *	A,
		const float *	B,
		const float *	bias,
		float *	C,
		int	M,
		int	N,
		int	K
	)

Naive parallel GEMM implementation (reference baseline)

Parameters

A	Input matrix A [M x K]
B	Input matrix B [N x K] (transposed)
bias	Bias vector [N]
C	Output matrix C [M x N]
M	Number of rows in A and C
N	Number of columns in B and C
K	Inner dimension (columns of A, rows of B)

@performance

Baseline performance: ~50-100 GFLOPS
Used as reference for accuracy validation
Simple OpenMP parallelization

Note: This is the golden reference - all other implementations are validated against this

◆ generate()

void generate	(	TransformerModel *	M,
		int *	prompt,
		int	prompt_len,
		int	max_tokens
	)

◆ initialize_optimizer_state()

static void initialize_optimizer_state ( TransformerModel * M )

static

◆ initialize_optimizer_tensor()

static void initialize_optimizer_tensor	(	float *	base,
		size_t	weight_offset,
		size_t	count,
		size_t	m_offset,
		size_t	v_offset,
		size_t	ema_offset,
		bool	zero_moments,
		bool	init_ema
	)

inlinestatic

◆ kv_cache_store_layer()

static void kv_cache_store_layer	(	TransformerModel *	M,
		int	layer_idx,
		int	tokens
	)

static

◆ layernorm_forward_rolled_slice()

void layernorm_forward_rolled_slice	(	const float *__restrict	input_slice_base,
		const float *__restrict	gamma,
		const float *__restrict	beta,
		float *__restrict	output_slice_base,
		float *__restrict	mean_cache_slice,
		float *__restrict	rstd_cache_slice,
		int	num_tokens_in_slice,
		int	d_model,
		int	aligned_embed_dim,
		float	eps
	)

◆ layernorm_forward_unrolled_slice()

void layernorm_forward_unrolled_slice	(	const float *__restrict	input_slice_base,
		const float *__restrict	gamma,
		const float *__restrict	beta,
		float *__restrict	output_slice_base,
		float *__restrict	mean_cache_slice,
		float *__restrict	rstd_cache_slice,
		int	num_tokens_in_slice,
		int	d_model,
		float	eps
	)

◆ layernorm_naive_serial()

void layernorm_naive_serial	(	const float *	input,
		const float *	gamma,
		const float *	beta,
		float *	output,
		float *	mean_cache,
		float *	rstd_cache,
		int	tokens,
		int	d_model,
		int	aligned_embed_dim,
		float	eps
	)

◆ layernorm_naive_serial_matched_precision()

void layernorm_naive_serial_matched_precision	(	const float *	input,
		const float *	gamma,
		const float *	beta,
		float *	output,
		float *	mean_cache,
		float *	rstd_cache,
		int	tokens,
		int	d_model,
		float	eps
	)

◆ layernorm_token_parallel()

void layernorm_token_parallel	(	TransformerModel *	M,
		size_t	input_offset,
		size_t	weight_offset,
		size_t	bias_offset,
		size_t	mean_cache_offset,
		size_t	rstd_cache_offset,
		size_t	output_offset,
		float	eps
	)

Token-parallel Layer Normalization with AVX-512 optimization.

Performs Layer Normalization across tokens using token-level parallelism. Each CPU core processes a contiguous slice of tokens independently, achieving perfect cache locality and zero synchronization overhead.

Parameters

M	Transformer model containing memory layout and parallelism config
input_offset	Offset to input tensor [context_window × aligned_embed_dim]
weight_offset	Offset to gamma weights [aligned_embed_dim]
bias_offset	Offset to beta biases [aligned_embed_dim]
mean_cache_offset	Offset to mean cache [context_window] (for backward pass)
rstd_cache_offset	Offset to rstd cache [context_window] (for backward pass)
output_offset	Offset to output tensor [context_window × aligned_embed_dim]
eps	Epsilon for numerical stability (typically 1e-5)

Token-Level Parallelism Strategy:

Memory Layout (Token-Major):
┌──────────────┬──────────────┬──────────────┬──────────────┐
│ Token 0      │ Token 1      │ Token 2      │ Token 3      │
│ [768 floats] │ [768 floats] │ [768 floats] │ [768 floats] │
└──────────────┴──────────────┴──────────────┴──────────────┘
 │<─ Core 0 ──>│<─ Core 1 ──>│<─ Core 2 ──>│<─ Core 3 ──>│

Why Token-Parallel?:

✅ Perfect Locality: Each token's data (768 floats) is contiguous in memory
✅ Zero Sync: Tokens are independent, no barriers or atomics needed
✅ Cache Efficiency: Each core streams through sequential memory
✅ Linear Scaling: Speedup = num_cores (measured 7.8x on 8 cores)

Memory Access Pattern (per core):

Core 0 processes tokens [0, tokens_per_core):
  - Read:  input[0*768], input[1*768], ..., input[N*768]  (sequential)
  - Write: output[0*768], output[1*768], ..., output[N*768] (sequential)
  - Gamma/Beta: Shared read-only (broadcast to all cores)

Algorithm (per token):

Pass 1: Compute mean across embed_dim using AVX-512
Pass 2: Compute variance using FMA (fused multiply-add)
Pass 3: Normalize, scale by gamma, shift by beta

AVX-512 Optimization:

Processes 16 floats per instruction (4x16 unrolling)
Uses FMA for variance: acc = diff * diff + acc (2 FLOPs per cycle)
Aligned loads: _mm512_load_ps (requires 64-byte alignment)
Prefetching: Hints to load next cache line while computing current

Performance Characteristics:

Compute: 9 * embed_dim FLOPs per token
Memory: 3 * embed_dim reads + embed_dim writes per token
Bandwidth: Achieves 50-100 GB/s per core (streaming bandwidth)
Latency: ~5 μs per token on modern Xeon (768-dim, 3.0 GHz)

Cache Behavior:

L1 Data Cache: Holds ~4 tokens (768 floats = 3KB per token, 32KB L1)
L2 Cache: Holds ~80 tokens (256KB L2)
Prefetcher: Detects sequential pattern, hides DRAM latency

Why Aligned Embed Dim?: Padding to 64-byte boundaries ensures:

No false sharing between cores writing adjacent tokens
Aligned SIMD loads (faster than unaligned)
Clean cache line ownership (no partial cache line reads)

Note: This is a core building block used in every transformer layer

See also: layernorm_forward_rolled_slice Per-core slice processing kernel; TrulyOptimalLayer For offset definitions within a layer

@performance Measured 7.8x speedup on 8-core Xeon vs serial baseline

◆ load_model_weights()

int load_model_weights	(	TransformerModel *	M,
		const char *	weight_file
	)

◆ main()

int main	(	int	argc,
		char **	argv
	)

◆ mlp_token_parallel()

void mlp_token_parallel	(	TransformerModel *	M,
		int	layer_idx,
		size_t	input_offset,
		size_t	fc1_weight_offset,
		size_t	fc1_bias_offset,
		size_t	fc1_output_offset,
		size_t	fc2_weight_offset,
		size_t	fc2_bias_offset,
		size_t	output_offset
	)

◆ preload_all_training_windows()

static bool preload_all_training_windows	(	const TrainingPairList *	list,
		TransformerModel *	M,
		PreloadedTrainingData *	out,
		int	num_threads
	)

static

◆ qkv_micro_kernel_blocked_4x16_polished()

static void qkv_micro_kernel_blocked_4x16_polished	(	const float *__restrict	input_token,
		const float *__restrict	Q_weights_block,
		const float *__restrict	K_weights_block,
		const float *__restrict	V_weights_block,
		const float *__restrict	Q_bias_4,
		const float *__restrict	K_bias_4,
		const float *__restrict	V_bias_4,
		float *__restrict	Q_output_4,
		float *__restrict	K_output_4,
		float *__restrict	V_output_4,
		int	embed_dim
	)

inlinestatic

◆ qkv_micro_kernel_head_major_4x16()

static void qkv_micro_kernel_head_major_4x16	(	const float *__restrict	input_token,
		const float *__restrict	Q_weights_block,
		const float *__restrict	K_weights_block,
		const float *__restrict	V_weights_block,
		const float *__restrict	Q_bias_4,
		const float *__restrict	K_bias_4,
		const float *__restrict	V_bias_4,
		TransformerModel *	M,
		float *__restrict	q_output_base,
		float *__restrict	k_output_base,
		float *__restrict	v_output_base,
		int	embed_dim,
		int	token_idx,
		int	output_start_dim
	)

inlinestatic

◆ qkv_projection()

void qkv_projection	(	TransformerModel *	M,
		size_t	layer_idx
	)

◆ qkv_projection_head_major()

void qkv_projection_head_major	(	TransformerModel *	M,
		int	layer_idx
	)

◆ qkv_token_kernel_4x16_blocked_polished()

static void qkv_token_kernel_4x16_blocked_polished	(	const float *__restrict	input_token,
		const float *__restrict	Q_weights,
		const float *__restrict	K_weights,
		const float *__restrict	V_weights,
		const float *__restrict	Q_bias,
		const float *__restrict	K_bias,
		const float *__restrict	V_bias,
		float *__restrict	Q_output,
		float *__restrict	K_output,
		float *__restrict	V_output,
		int	embed_dim
	)

static

◆ qkv_token_kernel_head_major_4x16()

static void qkv_token_kernel_head_major_4x16	(	const float *__restrict	input_token,
		const float *__restrict	Q_weights,
		const float *__restrict	K_weights,
		const float *__restrict	V_weights,
		const float *__restrict	Q_bias,
		const float *__restrict	K_bias,
		const float *__restrict	V_bias,
		TransformerModel *	M,
		float *__restrict	q_output_base,
		float *__restrict	k_output_base,
		float *__restrict	v_output_base,
		int	embed_dim,
		int	token_idx
	)

static

◆ read_model_metadata()

int read_model_metadata	(	TransformerModel *	M,
		const char *	weight_file
	)

Read model metadata from weight file header.

This function ONLY reads the header and populates model dimensions. It does NOT allocate memory - that's your decision to make separately.

Parameters

M	Pointer to zero-initialized TransformerModel struct
weight_file	Path to the .weights file

Returns: 0 on success, -1 on failure

◆ reset_kv_cache()

static void reset_kv_cache ( TransformerModel * M )

static

◆ reshape_qkv_grad_to_token_major()

static void reshape_qkv_grad_to_token_major	(	TransformerModel *	M,
		size_t	d_output_offset,
		size_t	scratch_offset
	)

static

◆ residual_add_token_parallel()

void residual_add_token_parallel	(	TransformerModel *	M,
		size_t	input_offset,
		size_t	residual_offset,
		size_t	output_offset
	)

◆ run_comprehensive_benchmark()

void run_comprehensive_benchmark ( TransformerModel * M )

◆ run_layernorm_benchmark_performance()

void run_layernorm_benchmark_performance ( TransformerModel * M )

◆ run_layernorm_benchmark_precision_matched()

void run_layernorm_benchmark_precision_matched ( TransformerModel * M )

◆ run_training_loop()

static void run_training_loop	(	TransformerModel *	M,
		const char *	train_dir,
		TrainingPairList *	list,
		int	total_steps,
		float	learning_rate,
		int	log_interval,
		const char *	checkpoint_dir,
		int	checkpoint_interval
	)

static

◆ sample_token()

int sample_token	(	float *	logits,
		int	vocab_size,
		float	temperature
	)

◆ save_model_weights()

int save_model_weights	(	TransformerModel *	M,
		const char *	weight_file
	)

Load weights into already-allocated TransformerModel.

This assumes:

read_model_metadata() has been called
layout_transformer() has been called to allocate memory
You've verified you have enough RAM

Parameters

M	Pointer to initialized and allocated TransformerModel
weight_file	Path to the .weights file

Returns: 0 on success, -1 on failure

◆ sgd_update_tensor()

static void sgd_update_tensor	(	TransformerModel *	M,
		size_t	weight_offset,
		size_t	grad_offset,
		size_t	ema_offset,
		size_t	count,
		float	learning_rate,
		float	weight_decay,
		float	ema_decay
	)

inlinestatic

◆ shuffle_training_pairs()

static bool shuffle_training_pairs ( TrainingPairList * list )

static

◆ test_attention_head_major_after_qkv()

void test_attention_head_major_after_qkv ( TransformerModel * M )

◆ training_step()

float training_step	(	TransformerModel *	M,
		int32_t *	input_tokens,
		int32_t *	target_tokens,
		int32_t	seq_label,
		int	ctx_len,
		float	learning_rate
	)

◆ transformer_layer_forward()

void transformer_layer_forward	(	TransformerModel *	M,
		int	layer_idx,
		size_t	layer_input_offset
	)

◆ update_all_weights_adam()

static void update_all_weights_adam	(	TransformerModel *	M,
		float	learning_rate
	)

static

◆ update_all_weights_sgd()

static void update_all_weights_sgd	(	TransformerModel *	M,
		float	learning_rate
	)

static

◆ zero_gradients()

void zero_gradients ( TransformerModel * M )

Variable Documentation

◆ active_tokens

int TransformerModel::active_tokens

Number of tokens populated in current forward pass.

◆ actual_tokens_offset

size_t GradientStorage::actual_tokens_offset

◆ adam_beta1

float TransformerModel::adam_beta1

◆ adam_beta2

float TransformerModel::adam_beta2

◆ adam_eps

float TransformerModel::adam_eps

◆ aligned_attn_context_window

size_t TransformerModel::aligned_attn_context_window

context_window padded to prevent false sharing

◆ aligned_embed_dim

size_t TransformerModel::aligned_embed_dim

embed_dim rounded up to 64-byte alignment (in floats)

◆ aligned_head_dim

size_t TransformerModel::aligned_head_dim

head_dim rounded up to 64-byte alignment (in floats)

◆ attention_output_copy_offset

size_t LayerGradients::attention_output_copy_offset

◆ attention_output_offset

size_t TrulyOptimalLayer::attention_output_offset

◆ attention_scores_offset

size_t TrulyOptimalLayer::attention_scores_offset

◆ attention_weights_copy_offset

size_t LayerGradients::attention_weights_copy_offset

◆ backprop_base

size_t GradientStorage::backprop_base

◆ cache_base

uint32_t* PreloadedTrainingData::cache_base

◆ checksum

uint8_t TransformerModel::checksum[32]

SHA256 checksum of weight file.

◆ context_window

int TransformerModel::context_window

Maximum sequence length (e.g., 1024)

◆ count [1/2]

size_t TrainingPairList::count

◆ count [2/2]

size_t PreloadedTrainingData::count

◆ ctx_lengths

uint16_t* PreloadedTrainingData::ctx_lengths

◆ d_attention_head_offset

size_t LayerGradients::d_attention_head_offset

◆ d_attention_output_offset

size_t LayerGradients::d_attention_output_offset

◆ d_attention_scores_offset

size_t LayerGradients::d_attention_scores_offset

◆ d_attention_token_offset

size_t LayerGradients::d_attention_token_offset

◆ d_attention_weights_offset

size_t LayerGradients::d_attention_weights_offset

◆ d_embed_weights_offset

size_t GradientStorage::d_embed_weights_offset

◆ d_fc1_bias_offset

size_t LayerGradients::d_fc1_bias_offset

◆ d_fc1_output_offset

size_t LayerGradients::d_fc1_output_offset

◆ d_fc1_weights_offset

size_t LayerGradients::d_fc1_weights_offset

◆ d_fc2_bias_offset

size_t LayerGradients::d_fc2_bias_offset

◆ d_fc2_input_offset

size_t LayerGradients::d_fc2_input_offset

◆ d_fc2_weights_offset

size_t LayerGradients::d_fc2_weights_offset

◆ d_final_ln_beta_offset

size_t GradientStorage::d_final_ln_beta_offset

◆ d_final_ln_gamma_offset

size_t GradientStorage::d_final_ln_gamma_offset

◆ d_final_ln_input_offset

size_t GradientStorage::d_final_ln_input_offset

◆ d_final_output_offset

size_t GradientStorage::d_final_output_offset

◆ d_k_bias_offset

size_t LayerGradients::d_k_bias_offset

◆ d_k_output_offset

size_t LayerGradients::d_k_output_offset

◆ d_k_weights_offset

size_t LayerGradients::d_k_weights_offset

◆ d_ln1_beta_offset

size_t LayerGradients::d_ln1_beta_offset

◆ d_ln1_gamma_offset

size_t LayerGradients::d_ln1_gamma_offset

◆ d_ln1_input_offset

size_t LayerGradients::d_ln1_input_offset

◆ d_ln1_output_offset

size_t LayerGradients::d_ln1_output_offset

◆ d_ln2_beta_offset

size_t LayerGradients::d_ln2_beta_offset

◆ d_ln2_gamma_offset

size_t LayerGradients::d_ln2_gamma_offset

◆ d_ln2_input_offset

size_t LayerGradients::d_ln2_input_offset

◆ d_ln2_output_offset

size_t LayerGradients::d_ln2_output_offset

◆ d_logits_offset

size_t GradientStorage::d_logits_offset

◆ d_mlp_output_offset

size_t LayerGradients::d_mlp_output_offset

◆ d_pos_embed_offset

size_t GradientStorage::d_pos_embed_offset

◆ d_proj_bias_offset

size_t LayerGradients::d_proj_bias_offset

◆ d_proj_weights_offset

size_t LayerGradients::d_proj_weights_offset

◆ d_q_bias_offset

size_t LayerGradients::d_q_bias_offset

◆ d_q_output_offset

size_t LayerGradients::d_q_output_offset

◆ d_q_weights_offset

size_t LayerGradients::d_q_weights_offset

◆ d_residual1_offset

size_t LayerGradients::d_residual1_offset

◆ d_residual2_offset

size_t LayerGradients::d_residual2_offset

◆ d_seq_cls_bias_offset

size_t GradientStorage::d_seq_cls_bias_offset

◆ d_seq_cls_logits_offset

size_t GradientStorage::d_seq_cls_logits_offset

◆ d_seq_cls_pooled_offset

size_t GradientStorage::d_seq_cls_pooled_offset

◆ d_seq_cls_weight_offset

size_t GradientStorage::d_seq_cls_weight_offset

◆ d_v_bias_offset

size_t LayerGradients::d_v_bias_offset

◆ d_v_output_offset

size_t LayerGradients::d_v_output_offset

◆ d_v_weights_offset

size_t LayerGradients::d_v_weights_offset

◆ ema_decay

float TransformerModel::ema_decay

◆ ema_enabled

bool TransformerModel::ema_enabled

◆ embed_dim

int TransformerModel::embed_dim

Embedding dimension (e.g., 768 for GPT-2 small)

◆ embedded_input_offset

size_t TransformerModel::embedded_input_offset

Combined token+pos embeddings [context_window × aligned_embed_dim].

◆ fc1_bias_copy_offset

size_t LayerGradients::fc1_bias_copy_offset

◆ fc1_bias_ema_offset

size_t LayerGradients::fc1_bias_ema_offset

◆ fc1_bias_m_offset

size_t LayerGradients::fc1_bias_m_offset

◆ fc1_bias_offset

size_t TrulyOptimalLayer::fc1_bias_offset

◆ fc1_bias_v_offset

size_t LayerGradients::fc1_bias_v_offset

◆ fc1_output_copy_offset

size_t LayerGradients::fc1_output_copy_offset

◆ fc1_output_offset

size_t TrulyOptimalLayer::fc1_output_offset

◆ fc1_weight_ema_offset

size_t LayerGradients::fc1_weight_ema_offset

◆ fc1_weight_m_offset

size_t LayerGradients::fc1_weight_m_offset

◆ fc1_weight_offset

size_t TrulyOptimalLayer::fc1_weight_offset

◆ fc1_weight_v_offset

size_t LayerGradients::fc1_weight_v_offset

◆ fc1_weights_copy_offset

size_t LayerGradients::fc1_weights_copy_offset

◆ fc2_bias_copy_offset

size_t LayerGradients::fc2_bias_copy_offset

◆ fc2_bias_ema_offset

size_t LayerGradients::fc2_bias_ema_offset

◆ fc2_bias_m_offset

size_t LayerGradients::fc2_bias_m_offset

◆ fc2_bias_offset

size_t TrulyOptimalLayer::fc2_bias_offset

◆ fc2_bias_v_offset

size_t LayerGradients::fc2_bias_v_offset

◆ fc2_input_copy_offset

size_t LayerGradients::fc2_input_copy_offset

◆ fc2_weight_ema_offset

size_t LayerGradients::fc2_weight_ema_offset

◆ fc2_weight_m_offset

size_t LayerGradients::fc2_weight_m_offset

◆ fc2_weight_offset

size_t TrulyOptimalLayer::fc2_weight_offset

◆ fc2_weight_v_offset

size_t LayerGradients::fc2_weight_v_offset

◆ fc2_weights_copy_offset

size_t LayerGradients::fc2_weights_copy_offset

◆ final_ln_beta_copy_offset

size_t GradientStorage::final_ln_beta_copy_offset

◆ final_ln_beta_ema_offset

size_t GradientStorage::final_ln_beta_ema_offset

◆ final_ln_beta_m_offset

size_t GradientStorage::final_ln_beta_m_offset

◆ final_ln_beta_v_offset

size_t GradientStorage::final_ln_beta_v_offset

◆ final_ln_bias_offset

size_t TransformerModel::final_ln_bias_offset

Final LayerNorm beta [aligned_embed_dim].

◆ final_ln_gamma_copy_offset

size_t GradientStorage::final_ln_gamma_copy_offset

◆ final_ln_gamma_ema_offset

size_t GradientStorage::final_ln_gamma_ema_offset

◆ final_ln_gamma_m_offset

size_t GradientStorage::final_ln_gamma_m_offset

◆ final_ln_gamma_v_offset

size_t GradientStorage::final_ln_gamma_v_offset

◆ final_ln_input_copy_offset

size_t GradientStorage::final_ln_input_copy_offset

◆ final_ln_mean_copy_offset

size_t GradientStorage::final_ln_mean_copy_offset

◆ final_ln_mean_offset

size_t TransformerModel::final_ln_mean_offset

Final LayerNorm mean [context_window].

◆ final_ln_rstd_copy_offset

size_t GradientStorage::final_ln_rstd_copy_offset

◆ final_ln_rstd_offset

size_t TransformerModel::final_ln_rstd_offset

Final LayerNorm rstd [context_window].

◆ final_ln_weight_offset

size_t TransformerModel::final_ln_weight_offset

Final LayerNorm gamma [aligned_embed_dim].

◆ final_output_copy_offset

size_t GradientStorage::final_output_copy_offset

◆ final_output_offset

size_t TransformerModel::final_output_offset

Final normalized output [context_window × aligned_embed_dim].

◆ grad_clip

float TransformerModel::grad_clip

Clip gradients to +/- this value (0 disables)

◆ gradients

GradientStorage TransformerModel::gradients

Gradient and activation cache memory (training only)

◆ head_dim

int TransformerModel::head_dim

Dimension per head: embed_dim / num_attention_heads.

◆ k_bias_copy_offset

size_t LayerGradients::k_bias_copy_offset

◆ k_bias_ema_offset

size_t LayerGradients::k_bias_ema_offset

◆ k_bias_m_offset

size_t LayerGradients::k_bias_m_offset

◆ k_bias_offset

size_t TrulyOptimalLayer::k_bias_offset

◆ k_bias_v_offset

size_t LayerGradients::k_bias_v_offset

◆ k_cache_offset

size_t TrulyOptimalLayer::k_cache_offset

◆ k_output_copy_offset

size_t LayerGradients::k_output_copy_offset

◆ k_output_offset

size_t TrulyOptimalLayer::k_output_offset

◆ k_weight_ema_offset

size_t LayerGradients::k_weight_ema_offset

◆ k_weight_m_offset

size_t LayerGradients::k_weight_m_offset

◆ k_weight_offset

size_t TrulyOptimalLayer::k_weight_offset

◆ k_weight_v_offset

size_t LayerGradients::k_weight_v_offset

◆ k_weights_copy_offset

size_t LayerGradients::k_weights_copy_offset

◆ kv_cache_capacity

int TransformerModel::kv_cache_capacity

Maximum tokens stored in cache.

◆ kv_cache_enabled

bool TransformerModel::kv_cache_enabled

Whether KV cache regions are allocated/used.

◆ kv_cache_tokens

int TransformerModel::kv_cache_tokens

Number of valid tokens currently cached.

◆ layer_backprop_stride

size_t GradientStorage::layer_backprop_stride

◆ layer_end_canary_offset

size_t TrulyOptimalLayer::layer_end_canary_offset

◆ layer_input_offset

size_t TrulyOptimalLayer::layer_input_offset

◆ layer_start_canary_offset

size_t TrulyOptimalLayer::layer_start_canary_offset

◆ layer_stride

size_t TransformerModel::layer_stride

Byte offset between consecutive layer memory blocks.

◆ layers [1/2]

LayerGradients* GradientStorage::layers

◆ layers [2/2]

TrulyOptimalLayer* TransformerModel::layers

Array of per-layer offset structures.

◆ layers_start_offset

size_t TransformerModel::layers_start_offset

Start of first transformer layer memory.

◆ learning_rate

float TransformerModel::learning_rate

SGD learning rate for weight updates.

◆ lm_head_weight_offset

size_t TransformerModel::lm_head_weight_offset

Language model head (weight-tied to token_emb_offset)

◆ ln1_beta_copy_offset

size_t LayerGradients::ln1_beta_copy_offset

◆ ln1_beta_ema_offset

size_t LayerGradients::ln1_beta_ema_offset

◆ ln1_beta_m_offset

size_t LayerGradients::ln1_beta_m_offset

◆ ln1_beta_v_offset

size_t LayerGradients::ln1_beta_v_offset

◆ ln1_bias_offset

size_t TrulyOptimalLayer::ln1_bias_offset

◆ ln1_gamma_copy_offset

size_t LayerGradients::ln1_gamma_copy_offset

◆ ln1_gamma_ema_offset

size_t LayerGradients::ln1_gamma_ema_offset

◆ ln1_gamma_m_offset

size_t LayerGradients::ln1_gamma_m_offset

◆ ln1_gamma_v_offset

size_t LayerGradients::ln1_gamma_v_offset

◆ ln1_input_copy_offset

size_t LayerGradients::ln1_input_copy_offset

◆ ln1_mean_copy_offset

size_t LayerGradients::ln1_mean_copy_offset

◆ ln1_mean_offset

size_t TrulyOptimalLayer::ln1_mean_offset

◆ ln1_output_copy_offset

size_t LayerGradients::ln1_output_copy_offset

◆ ln1_output_offset

size_t TrulyOptimalLayer::ln1_output_offset

◆ ln1_rstd_copy_offset

size_t LayerGradients::ln1_rstd_copy_offset

◆ ln1_rstd_offset

size_t TrulyOptimalLayer::ln1_rstd_offset

◆ ln1_weight_offset

size_t TrulyOptimalLayer::ln1_weight_offset

◆ ln2_beta_copy_offset

size_t LayerGradients::ln2_beta_copy_offset

◆ ln2_beta_ema_offset

size_t LayerGradients::ln2_beta_ema_offset

◆ ln2_beta_m_offset

size_t LayerGradients::ln2_beta_m_offset

◆ ln2_beta_v_offset

size_t LayerGradients::ln2_beta_v_offset

◆ ln2_bias_offset

size_t TrulyOptimalLayer::ln2_bias_offset

◆ ln2_gamma_copy_offset

size_t LayerGradients::ln2_gamma_copy_offset

◆ ln2_gamma_ema_offset

size_t LayerGradients::ln2_gamma_ema_offset

◆ ln2_gamma_m_offset

size_t LayerGradients::ln2_gamma_m_offset

◆ ln2_gamma_v_offset

size_t LayerGradients::ln2_gamma_v_offset

◆ ln2_input_copy_offset

size_t LayerGradients::ln2_input_copy_offset

◆ ln2_mean_copy_offset

size_t LayerGradients::ln2_mean_copy_offset

◆ ln2_mean_offset

size_t TrulyOptimalLayer::ln2_mean_offset

◆ ln2_output_copy_offset

size_t LayerGradients::ln2_output_copy_offset

◆ ln2_output_offset

size_t TrulyOptimalLayer::ln2_output_offset

◆ ln2_rstd_copy_offset

size_t LayerGradients::ln2_rstd_copy_offset

◆ ln2_rstd_offset

size_t TrulyOptimalLayer::ln2_rstd_offset

◆ ln2_weight_offset

size_t TrulyOptimalLayer::ln2_weight_offset

◆ logits_copy_offset

size_t GradientStorage::logits_copy_offset

◆ logits_offset

size_t TransformerModel::logits_offset

Output logits [context_window × vocab_size].

◆ lr_warmup_init

float TransformerModel::lr_warmup_init

Starting LR for warmup (defaults to 0 when unset)

◆ lr_warmup_steps

int TransformerModel::lr_warmup_steps

Number of steps to linearly warm up the LR.

◆ magic

char TransformerModel::magic[8]

Magic string "BUMPWGT2" for file validation.

◆ memory_base

float* TransformerModel::memory_base

Base pointer to single contiguous memory block.

◆ mlp_output_copy_offset

size_t LayerGradients::mlp_output_copy_offset

◆ mlp_output_offset

size_t TrulyOptimalLayer::mlp_output_offset

◆ model_type

uint32_t TransformerModel::model_type

Model architecture: 0=GPT2, 1=LLAMA, etc.

◆ num_attention_heads

int TransformerModel::num_attention_heads

Number of attention heads (e.g., 12 for GPT-2)

◆ num_cores

int TransformerModel::num_cores

Number of CPU cores to use (OpenMP threads)

◆ num_layers

int TransformerModel::num_layers

Number of transformer layers (e.g., 12 for GPT-2)

◆ num_training_tokens

long TransformerModel::num_training_tokens

Total number of tokens in the preloaded dataset.

◆ optimizer

OptimizerType TransformerModel::optimizer

Selected optimizer.

◆ optimizer_state_initialized

bool TransformerModel::optimizer_state_initialized

◆ optimizer_step

uint64_t TransformerModel::optimizer_step

Number of optimization steps taken.

◆ paths

char** TrainingPairList::paths

◆ pos_emb_ema_offset

size_t GradientStorage::pos_emb_ema_offset

◆ pos_emb_m_offset

size_t GradientStorage::pos_emb_m_offset

◆ pos_emb_offset

size_t TransformerModel::pos_emb_offset

Positional embedding table [context_window × aligned_embed_dim].

◆ pos_emb_v_offset

size_t GradientStorage::pos_emb_v_offset

◆ proj_bias_copy_offset

size_t LayerGradients::proj_bias_copy_offset

◆ proj_bias_ema_offset

size_t LayerGradients::proj_bias_ema_offset

◆ proj_bias_m_offset

size_t LayerGradients::proj_bias_m_offset

◆ proj_bias_offset

size_t TrulyOptimalLayer::proj_bias_offset

◆ proj_bias_v_offset

size_t LayerGradients::proj_bias_v_offset

◆ proj_weight_ema_offset

size_t LayerGradients::proj_weight_ema_offset

◆ proj_weight_m_offset

size_t LayerGradients::proj_weight_m_offset

◆ proj_weight_offset

size_t TrulyOptimalLayer::proj_weight_offset

◆ proj_weight_v_offset

size_t LayerGradients::proj_weight_v_offset

◆ proj_weights_copy_offset

size_t LayerGradients::proj_weights_copy_offset

◆ q_bias_copy_offset

size_t LayerGradients::q_bias_copy_offset

◆ q_bias_ema_offset

size_t LayerGradients::q_bias_ema_offset

◆ q_bias_m_offset

size_t LayerGradients::q_bias_m_offset

◆ q_bias_offset

size_t TrulyOptimalLayer::q_bias_offset

◆ q_bias_v_offset

size_t LayerGradients::q_bias_v_offset

◆ q_output_copy_offset

size_t LayerGradients::q_output_copy_offset

◆ q_output_offset

size_t TrulyOptimalLayer::q_output_offset

◆ q_weight_ema_offset

size_t LayerGradients::q_weight_ema_offset

◆ q_weight_m_offset

size_t LayerGradients::q_weight_m_offset

◆ q_weight_offset

size_t TrulyOptimalLayer::q_weight_offset

◆ q_weight_v_offset

size_t LayerGradients::q_weight_v_offset

◆ q_weights_copy_offset

size_t LayerGradients::q_weights_copy_offset

◆ qkv_scratch_offset

size_t LayerGradients::qkv_scratch_offset

◆ reserved

uint8_t TransformerModel::reserved[32]

Reserved for future extensions.

◆ residual1_copy_offset

size_t LayerGradients::residual1_copy_offset

◆ residual1_output_offset

size_t TrulyOptimalLayer::residual1_output_offset

◆ residual2_copy_offset

size_t LayerGradients::residual2_copy_offset

◆ residual2_output_offset

size_t TrulyOptimalLayer::residual2_output_offset

◆ seq_cls_bias_ema_offset

size_t GradientStorage::seq_cls_bias_ema_offset

◆ seq_cls_bias_m_offset

size_t GradientStorage::seq_cls_bias_m_offset

◆ seq_cls_bias_offset

size_t TransformerModel::seq_cls_bias_offset

◆ seq_cls_bias_v_offset

size_t GradientStorage::seq_cls_bias_v_offset

◆ seq_cls_enabled

bool TransformerModel::seq_cls_enabled

◆ seq_cls_logits_offset

size_t GradientStorage::seq_cls_logits_offset

◆ seq_cls_num_classes

int TransformerModel::seq_cls_num_classes

◆ seq_cls_pooled_offset

size_t GradientStorage::seq_cls_pooled_offset

◆ seq_cls_pooling

int TransformerModel::seq_cls_pooling

0=final,1=cls,2=mean

◆ seq_cls_weight_ema_offset

size_t GradientStorage::seq_cls_weight_ema_offset

◆ seq_cls_weight_m_offset

size_t GradientStorage::seq_cls_weight_m_offset

◆ seq_cls_weight_offset

size_t TransformerModel::seq_cls_weight_offset

◆ seq_cls_weight_v_offset

size_t GradientStorage::seq_cls_weight_v_offset

◆ stride

size_t PreloadedTrainingData::stride

◆ target_lengths

uint16_t* PreloadedTrainingData::target_lengths

◆ task_type

TaskType TransformerModel::task_type

Active training/inference task.

◆ token_emb_ema_offset

size_t GradientStorage::token_emb_ema_offset

◆ token_emb_m_offset

size_t GradientStorage::token_emb_m_offset

◆ token_emb_offset

size_t TransformerModel::token_emb_offset

Token embedding table [vocab_size × aligned_embed_dim].

◆ token_emb_v_offset

size_t GradientStorage::token_emb_v_offset

◆ tokens_per_core

int TransformerModel::tokens_per_core

Tokens assigned per core: context_window / num_cores.

◆ tokens_per_pair

size_t PreloadedTrainingData::tokens_per_pair

◆ total_floats

size_t TransformerModel::total_floats

Total size of memory block in float elements.

◆ total_gradient_floats

size_t GradientStorage::total_gradient_floats

◆ training_cache_samples

size_t TransformerModel::training_cache_samples

Number of training windows cached inside arena.

◆ training_data_buffer

long* TransformerModel::training_data_buffer

Pointer to in-memory buffer for the entire training dataset.

◆ training_enabled

bool TransformerModel::training_enabled

Whether gradient storage is allocated.

◆ training_pair_capacity

size_t GradientStorage::training_pair_capacity

◆ training_pair_tokens_count

size_t GradientStorage::training_pair_tokens_count

◆ training_pair_tokens_offset

size_t GradientStorage::training_pair_tokens_offset

◆ training_pair_tokens_stride

size_t GradientStorage::training_pair_tokens_stride

◆ v_bias_copy_offset

size_t LayerGradients::v_bias_copy_offset

◆ v_bias_ema_offset

size_t LayerGradients::v_bias_ema_offset

◆ v_bias_m_offset

size_t LayerGradients::v_bias_m_offset

◆ v_bias_offset

size_t TrulyOptimalLayer::v_bias_offset

◆ v_bias_v_offset

size_t LayerGradients::v_bias_v_offset

◆ v_cache_offset

size_t TrulyOptimalLayer::v_cache_offset

◆ v_output_copy_offset

size_t LayerGradients::v_output_copy_offset

◆ v_output_offset

size_t TrulyOptimalLayer::v_output_offset

◆ v_weight_ema_offset

size_t LayerGradients::v_weight_ema_offset

◆ v_weight_m_offset

size_t LayerGradients::v_weight_m_offset

◆ v_weight_offset

size_t TrulyOptimalLayer::v_weight_offset

◆ v_weight_v_offset

size_t LayerGradients::v_weight_v_offset

◆ v_weights_copy_offset

size_t LayerGradients::v_weights_copy_offset

◆ version

uint32_t TransformerModel::version

Weight file format version.

◆ vocab_size

int TransformerModel::vocab_size

Vocabulary size (e.g., 50257 for GPT-2)

◆ weight_decay

float TransformerModel::weight_decay

Classes

Macros

Functions

Variables

Detailed Description

Macro Definition Documentation

◆ ATTN_SCORES_ACCESS

Function Documentation

◆ accumulate_linear_weight_bias_feature_parallel()

◆ adam_update_tensor()

◆ add_gpt2_token_and_positional_embeddings()

◆ add_gradient()

ADD GRADIENT (accumulate gradients from residual path)

◆ aligned_alloc_64()

◆ apply_causal_softmax_head_major()

◆ apply_optimizer_update()

◆ apply_seq_cls_head()

◆ attention_head_major_complete()

◆ attention_projection_with_concat()

◆ backward_attention_projection()

◆ backward_attention_weighted_values()

◆ backward_causal_softmax()

Mathematical Derivation: The Softmax Jacobian-Vector Product

Implementation Details

◆ backward_embedding_layer()

◆ backward_fc1()

◆ backward_fc1_feature_parallel()

◆ backward_fc1_reference()

BACKWARD THROUGH FC1 (Feed-Forward Layer 1)

◆ backward_fc2()

◆ backward_fc2_feature_parallel()

◆ backward_fc2_reference()

BACKWARD THROUGH FC2 (Feed-Forward Layer 2)

◆ backward_final_layernorm()

◆ backward_gelu()

BACKWARD THROUGH GELU ACTIVATION

◆ backward_gelu_fast()

◆ backward_layernorm()

BACKWARD THROUGH LAYERNORM

◆ backward_linear()

◆ backward_linear_feature_parallel()

◆ backward_linear_reference()

◆ backward_lm_head()

◆ backward_qk_matmul()

◆ backward_residual_connection()

RESIDUAL CONNECTION - FORWARD & BACKWARD PROPAGATION

◆ backward_seq_cls_head()

◆ backward_transformer_layer()

End-to-End Gradient Flow

◆ benchmark_attention_projection_complete()

◆ benchmark_qkv_dual_comparison()

◆ build_training_pair_list()

◆ cache_forward_activations()

◆ clip_grad_value()

◆ compare_arrays()

◆ compare_path_strings()

◆ compute_attention_output_head_major()

◆ compute_attention_scores_head_major()

◆ compute_cross_entropy_loss()

◆ compute_logits_last_token_optimized()

◆ compute_scheduled_lr()

◆ compute_seq_cls_loss()

◆ convert_token_major_to_head_major_layer()

◆ debug_backward_dump_grads_lm()

◆ debug_forward_dump_embed()

◆ debug_forward_dump_hidden()

◆ debug_forward_dump_layer_output()

◆ debug_forward_dump_logits()

◆ debug_math_comparison()

◆ effective_token_count()

◆ embed_tokens()

◆ free_training_pair_list()

◆ gelu_activation_token_parallel()

◆ gemm_avx512_parallel()

◆ gemm_blocked_serial()

◆ gemm_fine_grained_parallel()

◆ gemm_naive_parallel()

◆ generate()

◆ initialize_optimizer_state()

◆ initialize_optimizer_tensor()