Merge pull request #40 from 86/86/multiple-globs

chore: support multiple globs in config

config.ts

@@ -3,7 +3,7 @@ type Config = {
   /** URL to start the crawl */
   url: string;
   /** Pattern to match against for links on a page to subsequently crawl */
-  match: string;
+  match: string | string[];
   /** Selector to grab the inner text from */
   selector: string;
   /** Don't crawl more than this many pages */

src/main.ts

@@ -51,7 +51,7 @@ if (process.env.NO_CRAWL !== "true") {
       // Extract links from the current page
       // and add them to the crawling queue.
       await enqueueLinks({
-        globs: [config.match],
+        globs: typeof config.match === "string" ? [config.match] : config.match,
       });
     },
     // Comment this option to scrape the full website.