[trace 018472931] forward_pass
model: XPR4.8-30t
architecture: AndishehMoE
parameter_class: 30T
parameter_count: 29837964623872
active_parameters_per_token: 615007141888
active_parameter_count_convention: total minus inactive routed-expert weights; includes full embedding and output-projection tables
num_hidden_layers: 144
hidden_size: 16384
num_attention_heads: 128
num_key_value_heads: 16
head_dim: 128
vocab_size: 152064
num_routed_experts_per_layer: 512
num_shared_experts_per_layer: 1
num_routed_experts_per_token: 8
moe_intermediate_size: 8192
moe_layer_frequency: every_layer
expert_activation: swiglu
normalization: rmsnorm
rms_norm_eps: 0.000001
position_embedding: rope
attention_bias: false
mlp_bias: false
router_bias: false
tie_word_embeddings: false
batch_size: 1
sequence_length: 6
indices: zero_based
trace_scope: selected embedding, attention, and router slices; next-token candidates
input.text = "tell me what you see ?"
input.input_ids = tensor(
[[7261, 502, 1148, 499, 3817, 949]],
dtype=int64
)
input.attention_mask = tensor(
[[1, 1, 1, 1, 1, 1]],
dtype=int64
)
input.position_ids = tensor(
[[0, 1, 2, 3, 4, 5]],
dtype=int64
)
[trace 018472931] model.embed_tokens
output.shape = Shape([1, 6, 16384])
output[0, 0, :16] = tensor(
[-0.4658, -0.0250, -0.0559, -0.0478,
-0.0667, -0.0720, 0.1398, 0.4059,
0.3100, -0.0913, 0.6262, -0.2638,
0.0796, -0.2100, -0.2634, -0.4783],
dtype=float32
)
slice_info: batch=0, token_position=0, hidden_dimensions=0:16
[trace 018472931] model.layers.12.self_attn
attention_weights.shape = Shape([1, 128, 6, 6])
attention_weights[0, 5, 5, :] = tensor(
[0.0362, 0.0374, 0.7049, 0.0399, 0.0164, 0.1652],
dtype=float32
)
slice_info: batch=0, head=5, query_position=5, key_positions=0:6
attention_row_sum = 1.000000
causal_mask = enabled
attention_dropout = 0.0
[trace 018472931] model.layers.12.mlp.gate
router_logits.shape = Shape([1, 6, 512])
query_position = 5
selected_expert_ids[0, 5, :] = tensor(
[17, 94, 128, 203, 287, 331, 406, 489],
dtype=int64
)
selected_expert_weights[0, 5, :] = tensor(
[0.2500, 0.2000, 0.1500, 0.1200,
0.1000, 0.0800, 0.0600, 0.0400],
dtype=float32
)
routed_weight_normalization = over_selected_8_experts
routed_weight_sum = 1.000000
shared_experts_evaluated = 1
shared_expert_combine = add_to_weighted_routed_output
[trace 018472931] lm_head
output.logits.shape = Shape([1, 6, 152064])
generation_step = 0
prediction_position = 5
candidate_selection = topk(output.logits[0, -1, :], k=25)
temperature = 1.0
candidate_probabilities = softmax(candidate_logits / temperature)
probability_scope = conditional_on_selected_25_candidates
full_vocabulary_probabilities = not_logged
numeric_display = rounded
rank token_id logit p_candidate
1 5172 9.0948 0.402151
2 19269 8.6327 0.253338
3 16757 8.0965 0.148194
4 21884 7.3612 0.071039
5 11704 6.8906 0.044373
6 9555 6.3238 0.025174
7 10012 5.9680 0.017637
8 17714 5.5685 0.011829
9 31473 5.0705 0.007189
10 24579 4.7714 0.005330
11 30223 4.3653 0.003551
12 25622 4.0674 0.002636
13 32827 3.6704 0.001773
14 38205 3.4318 0.001396
15 24314 3.1516 0.001055
16 5582 2.8312 0.000766
17 18008 2.5584 0.000583
18 30508 2.3553 0.000476
19 13811 2.0972 0.000368
20 12560 1.8897 0.000299
21 28044 1.7007 0.000247
22 775 1.4311 0.000189
23 3234 1.2359 0.000155
24 29221 1.1350 0.000140
25 23624 0.9076 0.000112
candidate_probability_sum = 1.000000
[trace 018472931] sampling
method = multinomial
selected_rank = 3
selected_token_id = 16757
selected_candidate_probability = 0.148194
[trace 018472931] end
0000 3EE6