{
  "torch_version": "2.14.0+cpu",
  "device": "cpu",
  "tokens": [
    "小猫",
    "喜欢",
    "小鱼"
  ],
  "tiny": {
    "q": [
      [
        1.0,
        0.0
      ],
      [
        0.0,
        1.0
      ],
      [
        1.0,
        1.0
      ]
    ],
    "k": [
      [
        1.0,
        0.0
      ],
      [
        0.0,
        1.0
      ],
      [
        1.0,
        1.0
      ]
    ],
    "v": [
      [
        1.0,
        0.0
      ],
      [
        0.0,
        2.0
      ],
      [
        3.0,
        3.0
      ]
    ],
    "raw": [
      [
        1.0,
        0.0,
        1.0
      ],
      [
        0.0,
        1.0,
        1.0
      ],
      [
        1.0,
        1.0,
        2.0
      ]
    ],
    "scaled": [
      [
        0.7071067690849304,
        0.0,
        0.7071067690849304
      ],
      [
        0.0,
        0.7071067690849304,
        0.7071067690849304
      ],
      [
        0.7071067690849304,
        0.7071067690849304,
        1.4142135381698608
      ]
    ],
    "full_weights": [
      [
        0.40111207962036133,
        0.19777581095695496,
        0.40111207962036133
      ],
      [
        0.19777581095695496,
        0.40111207962036133,
        0.40111207962036133
      ],
      [
        0.24825508892536163,
        0.24825508892536163,
        0.5034898519515991
      ]
    ],
    "full_output": [
      [
        1.6044483184814453,
        1.5988879203796387
      ],
      [
        1.4011120796203613,
        2.0055603981018066
      ],
      [
        1.7587246894836426,
        2.0069797039031982
      ]
    ],
    "blocked_future": [
      [
        false,
        true,
        true
      ],
      [
        false,
        false,
        true
      ],
      [
        false,
        false,
        false
      ]
    ],
    "causal_weights": [
      [
        1.0,
        0.0,
        0.0
      ],
      [
        0.3302384614944458,
        0.6697615385055542,
        0.0
      ],
      [
        0.24825508892536163,
        0.24825508892536163,
        0.5034898519515991
      ]
    ],
    "causal_output": [
      [
        1.0,
        0.0
      ],
      [
        0.3302384614944458,
        1.3395230770111084
      ],
      [
        1.7587246894836426,
        2.0069797039031982
      ]
    ]
  },
  "multihead": {
    "shapes": {
      "input": [
        2,
        4,
        4
      ],
      "qkv_after_split": [
        2,
        2,
        4,
        2
      ],
      "scores": [
        2,
        2,
        4,
        4
      ],
      "weights": [
        2,
        2,
        4,
        4
      ],
      "per_head_output": [
        2,
        2,
        4,
        2
      ],
      "joined": [
        2,
        4,
        4
      ],
      "final": [
        2,
        4,
        4
      ]
    },
    "parameters": 64,
    "weights_first_batch": [
      [
        [
          1.0,
          0.0,
          0.0,
          0.0
        ],
        [
          0.8916287422180176,
          0.10837122797966003,
          0.0,
          0.0
        ],
        [
          0.26552140712738037,
          0.3447895348072052,
          0.3896890878677368,
          0.0
        ],
        [
          0.25264307856559753,
          0.2575526833534241,
          0.23844699561595917,
          0.2513572871685028
        ]
      ],
      [
        [
          1.0,
          0.0,
          0.0,
          0.0
        ],
        [
          0.29014793038368225,
          0.7098521590232849,
          0.0,
          0.0
        ],
        [
          0.4041063189506531,
          0.25306442379951477,
          0.34282928705215454,
          0.0
        ],
        [
          0.2024928480386734,
          0.295119047164917,
          0.22887106239795685,
          0.27351701259613037
        ]
      ]
    ],
    "first_output": [
      [
        -0.6527839303016663,
        0.30837586522102356,
        0.5315218567848206,
        -0.5971643924713135
      ],
      [
        -0.217147096991539,
        0.0044269561767578125,
        0.2474042773246765,
        -0.04232285916805267
      ],
      [
        -0.0019415616989135742,
        -0.0028208494186401367,
        -0.14507582783699036,
        0.31840041279792786
      ],
      [
        0.1453532725572586,
        -0.09161196649074554,
        -0.16982969641685486,
        0.30263498425483704
      ]
    ],
    "unmasked_earlier_change_after_future_edit": 2.821084976196289,
    "causal_earlier_change_after_future_edit": 0.0,
    "future_gradient_nonzero_count": 0,
    "past_gradient_nonzero_count": 8
  },
  "positions": {
    "sinusoidal_4x4": [
      [
        0.0,
        1.0,
        0.0,
        1.0
      ],
      [
        0.8414709568023682,
        0.5403023362159729,
        0.009999833069741726,
        0.9999499917030334
      ],
      [
        0.9092974066734314,
        -0.416146844625473,
        0.019998665899038315,
        0.9998000264167786
      ],
      [
        0.14112000167369843,
        -0.9899924993515015,
        0.029995499178767204,
        0.9995500445365906
      ]
    ],
    "unmasked_no_position_permutation_max_diff": 2.9802322387695312e-08,
    "with_position_permutation_max_diff": 0.2814931869506836
  }
}