TCR chain type special tokens (#127)

Co-authored-by: VADIM RATNER [email protected] <[email protected]> Co-authored-by: VADIM RATNER [email protected] <[email protected]> Co-authored-by: VADIM RATNER [email protected] <[email protected]> Co-authored-by: VADIM RATNER [email protected] <[email protected]> Co-authored-by: VADIM RATNER [email protected] <[email protected]> Co-authored-by: VADIM RATNER [email protected] <[email protected]> Co-authored-by: VADIM RATNER [email protected] <[email protected]> Co-authored-by: VADIM RATNER [email protected] <[email protected]>
BiomedSciAI · Jun 2, 2024 · 7fc56d9 · 7fc56d9
1 parent 4a1208a
commit 7fc56d9
Show file tree

Hide file tree

Showing 11 changed files with 507 additions and 2 deletions.
diff --git a/...tokenizer/bpe_tokenizer_trained_on_chembl_zinc_with_aug_4272372_samples_balanced_1_1.json b/...tokenizer/bpe_tokenizer_trained_on_chembl_zinc_with_aug_4272372_samples_balanced_1_1.json
@@ -2702,6 +2702,51 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 300,
+      "content": "<MOLECULAR_ENTITY_TCR_ALPHA_CDR3>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 301,
+      "content": "<MOLECULAR_ENTITY_TCR_DELTA_CDR3>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 302,
+      "content": "<MOLECULAR_ENTITY_TCR_DELTA_VAR>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 303,
+      "content": "<MOLECULAR_ENTITY_TCR_GAMMA_CDR3>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 304,
+      "content": "<MOLECULAR_ENTITY_TCR_GAMMA_VAR>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,
@@ -3017,6 +3062,11 @@
       "<GENERAL_CHAIN>": 297,
       "<SUBMOLECULAR_ENTITY>": 298,
       "<MUTATED>": 299,
+      "<MOLECULAR_ENTITY_TCR_ALPHA_CDR3>": 300,
+      "<MOLECULAR_ENTITY_TCR_DELTA_CDR3>": 301,
+      "<MOLECULAR_ENTITY_TCR_DELTA_VAR>": 302,
+      "<MOLECULAR_ENTITY_TCR_GAMMA_CDR3>": 303,
+      "<MOLECULAR_ENTITY_TCR_GAMMA_VAR>": 304,
       "#": 527,
       "%": 528,
       "(": 529,

diff --git a/...izer/pretrained_tokenizers/bmfm_extended_modular_tokenizer/cell_attributes_tokenizer.json b/...izer/pretrained_tokenizers/bmfm_extended_modular_tokenizer/cell_attributes_tokenizer.json
@@ -2702,6 +2702,51 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 300,
+      "content": "<MOLECULAR_ENTITY_TCR_ALPHA_CDR3>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 301,
+      "content": "<MOLECULAR_ENTITY_TCR_DELTA_CDR3>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 302,
+      "content": "<MOLECULAR_ENTITY_TCR_DELTA_VAR>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 303,
+      "content": "<MOLECULAR_ENTITY_TCR_GAMMA_CDR3>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 304,
+      "content": "<MOLECULAR_ENTITY_TCR_GAMMA_VAR>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,
@@ -3023,6 +3068,11 @@
       "<GENERAL_CHAIN>": 297,
       "<SUBMOLECULAR_ENTITY>": 298,
       "<MUTATED>": 299,
+      "<MOLECULAR_ENTITY_TCR_ALPHA_CDR3>": 300,
+      "<MOLECULAR_ENTITY_TCR_DELTA_CDR3>": 301,
+      "<MOLECULAR_ENTITY_TCR_DELTA_VAR>": 302,
+      "<MOLECULAR_ENTITY_TCR_GAMMA_CDR3>": 303,
+      "<MOLECULAR_ENTITY_TCR_GAMMA_VAR>": 304,
       "[CL:0000499]": 3522,
       "[CL:2000060]": 3523,
       "[CL:0000235]": 3524,

diff --git a/...odulartokenizer/pretrained_tokenizers/bmfm_extended_modular_tokenizer/gene_tokenizer.json b/...odulartokenizer/pretrained_tokenizers/bmfm_extended_modular_tokenizer/gene_tokenizer.json
@@ -2702,6 +2702,51 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 300,
+      "content": "<MOLECULAR_ENTITY_TCR_ALPHA_CDR3>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 301,
+      "content": "<MOLECULAR_ENTITY_TCR_DELTA_CDR3>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 302,
+      "content": "<MOLECULAR_ENTITY_TCR_DELTA_VAR>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 303,
+      "content": "<MOLECULAR_ENTITY_TCR_GAMMA_CDR3>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 304,
+      "content": "<MOLECULAR_ENTITY_TCR_GAMMA_VAR>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,
@@ -3023,6 +3068,11 @@
       "<GENERAL_CHAIN>": 297,
       "<SUBMOLECULAR_ENTITY>": 298,
       "<MUTATED>": 299,
+      "<MOLECULAR_ENTITY_TCR_ALPHA_CDR3>": 300,
+      "<MOLECULAR_ENTITY_TCR_DELTA_CDR3>": 301,
+      "<MOLECULAR_ENTITY_TCR_DELTA_VAR>": 302,
+      "<MOLECULAR_ENTITY_TCR_GAMMA_CDR3>": 303,
+      "<MOLECULAR_ENTITY_TCR_GAMMA_VAR>": 304,
       "[100130093]": 5000,
       "[100133445]": 5001,
       "[100286793]": 5002,

diff --git a/...enizer/pretrained_tokenizers/bmfm_extended_modular_tokenizer/t5_tokenizer_AA_special.json b/...enizer/pretrained_tokenizers/bmfm_extended_modular_tokenizer/t5_tokenizer_AA_special.json
@@ -2702,6 +2702,51 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 300,
+      "content": "<MOLECULAR_ENTITY_TCR_ALPHA_CDR3>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 301,
+      "content": "<MOLECULAR_ENTITY_TCR_DELTA_CDR3>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 302,
+      "content": "<MOLECULAR_ENTITY_TCR_DELTA_VAR>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 303,
+      "content": "<MOLECULAR_ENTITY_TCR_GAMMA_CDR3>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 304,
+      "content": "<MOLECULAR_ENTITY_TCR_GAMMA_VAR>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,
@@ -3023,6 +3068,11 @@
       "<GENERAL_CHAIN>": 297,
       "<SUBMOLECULAR_ENTITY>": 298,
       "<MUTATED>": 299,
+      "<MOLECULAR_ENTITY_TCR_ALPHA_CDR3>": 300,
+      "<MOLECULAR_ENTITY_TCR_DELTA_CDR3>": 301,
+      "<MOLECULAR_ENTITY_TCR_DELTA_VAR>": 302,
+      "<MOLECULAR_ENTITY_TCR_GAMMA_CDR3>": 303,
+      "<MOLECULAR_ENTITY_TCR_GAMMA_VAR>": 304,
       "A": 501,
       "B": 502,
       "C": 503,

diff --git a/...tokenizer/bpe_tokenizer_trained_on_chembl_zinc_with_aug_4272372_samples_balanced_1_1.json b/...tokenizer/bpe_tokenizer_trained_on_chembl_zinc_with_aug_4272372_samples_balanced_1_1.json
@@ -2702,6 +2702,51 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 300,
+      "content": "<MOLECULAR_ENTITY_TCR_ALPHA_CDR3>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 301,
+      "content": "<MOLECULAR_ENTITY_TCR_DELTA_CDR3>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 302,
+      "content": "<MOLECULAR_ENTITY_TCR_DELTA_VAR>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 303,
+      "content": "<MOLECULAR_ENTITY_TCR_GAMMA_CDR3>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 304,
+      "content": "<MOLECULAR_ENTITY_TCR_GAMMA_VAR>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,
@@ -3017,6 +3062,11 @@
       "<GENERAL_CHAIN>": 297,
       "<SUBMOLECULAR_ENTITY>": 298,
       "<MUTATED>": 299,
+      "<MOLECULAR_ENTITY_TCR_ALPHA_CDR3>": 300,
+      "<MOLECULAR_ENTITY_TCR_DELTA_CDR3>": 301,
+      "<MOLECULAR_ENTITY_TCR_DELTA_VAR>": 302,
+      "<MOLECULAR_ENTITY_TCR_GAMMA_CDR3>": 303,
+      "<MOLECULAR_ENTITY_TCR_GAMMA_VAR>": 304,
       "#": 527,
       "%": 528,
       "(": 529,

diff --git a/...ulartokenizer/pretrained_tokenizers/bmfm_modular_tokenizer/cell_attributes_tokenizer.json b/...ulartokenizer/pretrained_tokenizers/bmfm_modular_tokenizer/cell_attributes_tokenizer.json
@@ -2702,6 +2702,51 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 300,
+      "content": "<MOLECULAR_ENTITY_TCR_ALPHA_CDR3>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 301,
+      "content": "<MOLECULAR_ENTITY_TCR_DELTA_CDR3>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 302,
+      "content": "<MOLECULAR_ENTITY_TCR_DELTA_VAR>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 303,
+      "content": "<MOLECULAR_ENTITY_TCR_GAMMA_CDR3>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 304,
+      "content": "<MOLECULAR_ENTITY_TCR_GAMMA_VAR>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,
@@ -3023,6 +3068,11 @@
       "<GENERAL_CHAIN>": 297,
       "<SUBMOLECULAR_ENTITY>": 298,
       "<MUTATED>": 299,
+      "<MOLECULAR_ENTITY_TCR_ALPHA_CDR3>": 300,
+      "<MOLECULAR_ENTITY_TCR_DELTA_CDR3>": 301,
+      "<MOLECULAR_ENTITY_TCR_DELTA_VAR>": 302,
+      "<MOLECULAR_ENTITY_TCR_GAMMA_CDR3>": 303,
+      "<MOLECULAR_ENTITY_TCR_GAMMA_VAR>": 304,
       "[CL:0000499]": 3522,
       "[CL:2000060]": 3523,
       "[CL:0000235]": 3524,