{
  "_name_or_path": "NRJ-350",
  "activation": "softmax",
  "alpha": 0.1,
  "architectures": [
    "BertEnergyModelForMaskedLM"
  ],
  "auto_map": {
    "AutoModel": "mlm.BertEnergyModelForMaskedLM"
  },
  "beta": 0.125,
  "bias": true,
  "block_size": 512,
  "compile": false,
  "dropout": 0.1,
  "embedding_dim": 768,
  "forward_memories": 3072,
  "layer_norm": 1e-12,
  "model_type": "bert_energy",
  "num_heads": 12,
  "num_layers": 12,
  "pad_idx": null,
  "positional": true,
  "share_layers": false,
  "tie_weights": false,
  "torch_dtype": "float32",
  "transformers_version": "4.47.0",
  "vocabulary_size": 30000
}