[GPT-3] Add GPT configs 89B and 175B (#2504)

* Add configs for 89B and 175B

Author: LiYuRio

examples/language_model/gpt-3/dygraph/modeling.py

@@ -552,6 +552,36 @@ class GPTPretrainedModel(PretrainedModel):
             "num_partitions": 1,
             "use_recompute": False,
         },
+        "gpt3-89B-en": { # 89B
+            "vocab_size": 51200,
+            "hidden_size": 12288,
+            "num_hidden_layers": 48,
+            "num_attention_heads": 96,
+            "intermediate_size": 49152,
+            "hidden_act": "gelu",
+            "hidden_dropout_prob": 0.1,
+            "attention_probs_dropout_prob": 0.1,
+            "max_position_embeddings": 1024,
+            "type_vocab_size": 1,  # no use
+            "initializer_range": 0.02,
+            "eos_token_id": 50256,
+            "eol_token_id": 198,
+        },
+        "gpt3-175B-en": { # 175B
+            "vocab_size": 51200,
+            "hidden_size": 12288,
+            "num_hidden_layers": 96,
+            "num_attention_heads": 96,
+            "intermediate_size": 49152,
+            "hidden_act": "gelu",
+            "hidden_dropout_prob": 0.1,
+            "attention_probs_dropout_prob": 0.1,
+            "max_position_embeddings": 1024,
+            "type_vocab_size": 1,  # no use
+            "initializer_range": 0.02,
+            "eos_token_id": 50256,
+            "eol_token_id": 198,
+        },
         "gpt3-13B-en": { # 13B
             "vocab_size": 50304,
             "hidden_size": 5120,

examples/language_model/gpt-3/static/modeling.py

@@ -627,6 +627,36 @@ class GPTPretrainedModel(PretrainedModel):
             "bos_token_id": 0,
             "eol_token_id": 3,
         },
+        "gpt3-89B-en": { # 89B
+            "vocab_size": 51200,
+            "hidden_size": 12288,
+            "num_hidden_layers": 48,
+            "num_attention_heads": 96,
+            "intermediate_size": 49152,
+            "hidden_act": "gelu",
+            "hidden_dropout_prob": 0.1,
+            "attention_probs_dropout_prob": 0.1,
+            "max_position_embeddings": 1024,
+            "type_vocab_size": 1,  # no use
+            "initializer_range": 0.02,
+            "eos_token_id": 50256,
+            "eol_token_id": 198,
+        },
+        "gpt3-175B-en": { # 175B
+            "vocab_size": 51200,
+            "hidden_size": 12288,
+            "num_hidden_layers": 96,
+            "num_attention_heads": 96,
+            "intermediate_size": 49152,
+            "hidden_act": "gelu",
+            "hidden_dropout_prob": 0.1,
+            "attention_probs_dropout_prob": 0.1,
+            "max_position_embeddings": 1024,
+            "type_vocab_size": 1,  # no use
+            "initializer_range": 0.02,
+            "eos_token_id": 50256,
+            "eol_token_id": 198,
+        },
         "gpt3-13B-en": { # 13B
             "vocab_size": 50304,
             "hidden_size": 5120,

paddlenlp/transformers/gpt/tokenizer.py

@@ -323,6 +323,8 @@ class GPTTokenizer(PretrainedTokenizer):
     gpt_merges_link = "http://bj.bcebos.com/paddlenlp/models/transformers/gpt/gpt-en-merges.txt"
     pretrained_resource_files_map = {
         "vocab_file": {
+            "gpt3-175B-en": gpt_vocab_link,
+            "gpt3-89B-en": gpt_vocab_link,
             "gpt3-13B-en": gpt_vocab_link,
             "gpt3-1.3B-en": gpt_vocab_link,
             "gpt2-xl-en": gpt_vocab_link,
@@ -332,6 +334,8 @@ class GPTTokenizer(PretrainedTokenizer):
             "gpt2-small-en": gpt_vocab_link,
         },
         "merges_file": {
+            "gpt3-175B-en": gpt_merges_link,
+            "gpt3-89B-en": gpt_merges_link,
             "gpt3-13B-en": gpt_merges_link,
             "gpt3-1.3B-en": gpt_merges_link,
             "gpt2-xl-en": gpt_merges_link,
@@ -342,6 +346,8 @@ class GPTTokenizer(PretrainedTokenizer):
         }
     }
     pretrained_init_configuration = {
+        "gpt3-175B-en": {},
+        "gpt3-89B-en": {},
         "gpt3-13B-en": {},
         "gpt3-1.3B-en": {},
         "gpt2-xl-en": {},