Add gpt-tokenizer package

package-lock.json

@@ -14,6 +14,7 @@
         "crawlee": "^3.0.0",
         "cross-env": "^7.0.3",
         "glob": "^10.3.10",
+        "gpt-tokenizer": "^2.1.2",
         "inquirer": "^9.2.12",
         "playwright": "*",
         "prettier": "^3.1.0",
@@ -1982,6 +1983,14 @@
         "url": "https://github.com/sindresorhus/is?sponsor=1"
       }
     },
+    "node_modules/gpt-tokenizer": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/gpt-tokenizer/-/gpt-tokenizer-2.1.2.tgz",
+      "integrity": "sha512-HSuI5d6uey+c7x/VzQlPfCoGrfLyAc28vxWofKbjR9PJHm0AjQGSWkKw/OJnb+8S1g7nzgRsf0WH3dK+NNWYbg==",
+      "dependencies": {
+        "rfc4648": "^1.5.2"
+      }
+    },
     "node_modules/graceful-fs": {
       "version": "4.2.11",
       "license": "ISC"
@@ -2922,6 +2931,11 @@
         "node": ">= 4"
       }
     },
+    "node_modules/rfc4648": {
+      "version": "1.5.3",
+      "resolved": "https://registry.npmjs.org/rfc4648/-/rfc4648-1.5.3.tgz",
+      "integrity": "sha512-MjOWxM065+WswwnmNONOT+bD1nXzY9Km6u3kzvnx8F8/HXGZdz3T6e6vZJ8Q/RIMUSp/nxqjH3GwvJDy8ijeQQ=="
+    },
     "node_modules/rrweb-cssom": {
       "version": "0.6.0",
       "license": "MIT"

package.json

@@ -11,6 +11,7 @@
     "crawlee": "^3.0.0",
     "cross-env": "^7.0.3",
     "glob": "^10.3.10",
+    "gpt-tokenizer": "^2.1.2",
     "inquirer": "^9.2.12",
     "playwright": "*",
     "prettier": "^3.1.0",