bert-large.json

{
    "name": "bing-bert-large",
    "bert_token_file": "bert-large-uncased",
    "bert_model_file": "bert-large-uncased",
    "bert_model_config": {
        "vocab_size_or_config_json_file": 119547,
        "hidden_size": 1024,
        "num_hidden_layers": 24,
        "num_attention_heads": 16,
        "intermediate_size": 4096,
        "hidden_act": "gelu",
        "hidden_dropout_prob": 0.1,
        "attention_probs_dropout_prob": 0.1,
        "max_position_embeddings": 512,
        "type_vocab_size": 2,
        "initializer_range": 0.02
    },
    "data": {
        "datasets": {
            "wiki_pretrain_dataset": "placeholder/512/wiki_pretrain"
        }
    },
    "training": {
        "num_epochs": 250,
        "warmup_proportion": 0.02,
        "learning_rate": 2e-4,
        "num_workers": 0,
        "decay_rate": 0.99,
        "decay_step": 1000,
        "total_training_steps": 187000
    },
    "validation": {
        "path": "placeholder/validation_512_only"
    }
}