improve and clean tokenizer

edenia · Mar 25, 2024 · 42e770f · 42e770f
1 parent f3b442d
commit 42e770f
Showing 1 changed file with 2 additions and 4 deletions.
diff --git a/analysis/ethereum_smart_contracts/GPT_tokenizer.ipynb b/analysis/ethereum_smart_contracts/GPT_tokenizer.ipynb
@@ -6,7 +6,7 @@
     "id": "FPuIaYYJLjvJ"
    },
    "source": [
-    "# Training a new tokenizer based on smart contract disassembled code"
+    "# Train a new tokenizer based on smart contract opcodes"
    ]
   },
   {
@@ -90,9 +90,7 @@
    "outputs": [],
    "source": [
     "tokenizer.save_pretrained(\"/data/forta/ethereum/tokenizer\")\n",
-    "print(tokenizer)\n",
-    "encode = old_tokenizer.tokenize(\"PUSH1 PUSH1 MSTORE PUSH1 CALLDATASIZE LT PUSH2\", return_tensors=\"pt\")\n",
-    "print(type(encode))"
+    "print(tokenizer)"
    ]
   },
   {