Add optional maxFileSize and maxTokens to Config

README.md

@@ -78,6 +78,10 @@ type Config = {
   maxPagesToCrawl: number;
   /** File name for the finished data */
   outputFileName: string;
+  /** Optional maximum file size in bytes to include in the output file */
+  maxFileSize?: number().,
+  /** Optional maximum number tokens to include in the output file */
+  maxTokens?: number().,
 };
 ```
 

config.ts

@@ -5,4 +5,6 @@ export const defaultConfig: Config = {
   match: "https://www.builder.io/c/docs/**",
   maxPagesToCrawl: 50,
   outputFileName: "output.json",
+  maxFileSize: 1000,
+  maxTokens: 5000
 };

src/config.ts

@@ -51,6 +51,16 @@ export const configSchema = z.object({
       .optional(),
   /** Optional timeout for waiting for a selector to appear */
   waitForSelectorTimeout: z.number().int().nonnegative().optional(),
+
+
+  /** Optional maximum file size in bytes to include in the output file 
+   * @example 1000
+  */
+  maxFileSize: z.number().int().positive().optional(),
+  /** Optional maximum number tokens to include in the output file 
+   * @example 5000
+  */
+  maxTokens: z.number().int().positive().optional(),
 });
 
 export type Config = z.infer<typeof configSchema>;