LLamaRecipes/end-to-end-use-cases/structured_parser/src/config.yaml

# Database Configuration
database:
  sql_db_path: "sqlite3.db"
  vector_db_path: "chroma.db"

# Model Configuration
model:
  backend: openai-compat  #  [offline-vllm, openai-compat]

  # for offline-vllm, use the following
  path: "/path/to/checkpoint"
  tensor_parallel_size: 4
  max_model_len: 32000
  max_num_seqs: 32

  # for openai-compat, use the following
  base_url: "https://api.llama.com/compat/v1"
  api_key: ""
  model_id: "Llama-4-Maverick-17B-128E-Instruct-FP8"


# Inference Parameters
extraction_inference:
  temperature: 0.2
  top_p: 0.9
  max_completion_tokens: 32000
  seed: 42

# Artifact configuration
artifacts:
  text:
    prompts:
      system: "You are an OCR expert. Your task is to extract all the various text sections (main text, body text, titles, captions etc.) from the following document into the provided TARGET SCHEMA. \n\nDo not extract any tables or any text inside pictures and graphs.\n\nEnsure your final answer is appropriately formatted as a JSON object and wrapped in a ```json\n\n``` block."
      user: "TARGET SCHEMA:\n```json\n{schema}\n```"
    output_schema: {
      "type": "object",
      "properties": {
        "text": {
          "type": "array",
          "items": {
            "type": "object",
            "properties": {
              "content": {
                "type": "string",
                "description": "The main text content extracted from the document"
              },
              "notes": {
                "type": "string",
                "description": "Any additional notes or observations about the text"
              }
            },
            "required": [
              "content",
              "notes"
            ]
          }
        }
      },
      "required": [
        "text"
      ]
    }
    use_json_decoding: true

  images:
    prompts:
      system: "You are an OCR expert. (Note: Do not extract tables)\n\n1.Your task is to extract images, pictures, charts and diagrams only from the following document.\n 2. For each extracted image, you must write\n a) a caption as given in the document\n	b) a detailed description of the image; utilize the surrounding text for this. Your descriptions should be very informative so that a human can understand what is in the image without ever seeing the document. Think step-by-step and write a JSON that corresponds to the schema and the information in the document\n\nIf there is nothing to extract, simply return an empty JSON {\"images\": []}. \nIf the image is a table, simply return an empty JSON {\"images\": []}. \n\nIf the image is a chart or a graph then you must convert them to JSON outputs.\n\n# Instructions to convert charts & graphs to JSON\nYour task is to: Analyze and describe the chart or graph. Summarize the type of chart/graph (e.g., bar chart, line graph, pie chart). Identify the axes, labels, categories, and any notable trends or patterns. Provide a brief textual description of what the chart/graph represents. Extract and structure the data:\n1. Capture all relevant values and data points from the chart/graph.\n2. Organize the extracted data into a clear and logical JSON structure.\n\n# Output format:\n\nYour response should be captured in the chart_data attribute of the JSON schema. Ensure your final answer is appropriately formatted as a JSON object and wrapped in a ```json\n\n``` block."
      user: "TARGET SCHEMA:\n```json\n{schema}\n```"
    output_schema: {
      "type": "object",
      "properties": {
        "images": {
          "type": "array",
          "items": {
            "type": "object",
            "properties": {
              "position_top": {
                "type": "string",
                "description": "Approximate position from top (e.g., 'top', 'middle', 'bottom')"
              },
              "position_left": {
                "type": "string",
                "description": "Approximate position from left (e.g., 'left', 'center', 'right')"
              },
              "description": {
                "type": "string",
                "description": "Detailed description of what the image shows"
              },
              "caption": {
                "type": "string",
                "description": "Caption or label associated with the image, if any"
              },
              "image_type": {
                "type": "string",
                "description": "Type of image (e.g., 'photograph', 'chart', 'diagram', 'illustration')"
              },
              "chart_data": {
                  "type": "object",
                  "properties": {
                    "type": {
                      "type": "string",
                      "enum": ["bar", "line", "pie", "scatter", "area"]
                    },
                    "title": {
                      "type": "string"
                    },
                    "subtitle": {
                      "type": "string"
                    },
                    "xAxis": {
                      "type": "object",
                      "properties": {
                        "title": { "type": "string" },
                        "labels": {
                          "type": "array",
                          "items": { "type": "string" }
                        }
                      },
                      "required": ["title", "labels"]
                    },
                    "yAxis": {
                      "type": "object",
                      "properties": {
                        "title": { "type": "string" },
                        "labels": {
                          "type": "array",
                          "items": { "type": "string" }
                        }
                      },
                      "required": ["title", "labels"]
                    },
                    "data": {
                      "type": "array",
                      "items": {
                        "oneOf": [
                          {
                            "type": "object",
                            "properties": {
                              "label": { "type": "string" },
                              "values": {
                                "type": "array",
                                "items": { "type": "number" }
                              }
                            },
                            "required": ["label", "values"]
                          },
                          {
                            "type": "object",
                            "properties": {
                              "x": { "type": "number" },
                              "y": { "type": "number" }
                            },
                            "required": ["x", "y"]
                          }
                        ]
                      }
                    },
                    "options": {
                      "type": "object",
                      "properties": {
                        "legend": { "type": "boolean" },
                        "rtl": { "type": "boolean" },
                        "responsive": { "type": "boolean" },
                        "animation": { "type": "boolean" }
                      }
                    }
                  },
                  "required": ["type", "title", "xAxis", "yAxis", "data"]
              }
            },
            "required": [
              "description",
              "caption",
              "image_type"
            ]
          }
        }
      },
      "required": [
        "images"
      ]
    }
    use_json_decoding: false

  tables:
    prompts:
      system: "Your task is to extract all tables in this document and prepare it in a structured format. If there are multiple tables in a document, extract each table separately. Your goal is to accurately extract the data so it can be loaded as a pandas dataframe. \n\n1. First you must ensure that there exists some tabular information in the document. If there isn't, simply return an empty JSON \n\n```json\n{\"tables\": []}\n```. 2. Next you must understand the table structure:\n\n  * how many columns are there?\n  * are any of the rows actually section headers that you should extract as a separate table?\n  * how should the numbers be interpreted? (e.g. are they percentages, currency, etc.)\n  * are there any special characters or formatting that needs to be preserved?\n\n3. After writing down your observations about the structure and nuances, you must extract the data from the table and format it as a JSON object. Each table or sub-table in the document should be a separate JSON object."
      user: "Think step-by-step and write a JSON that corresponds to the schema and the information in the document. Use the following JSON structure for your output:\n\n```json\n{\"tables\": [{\"table_contents\": { < dict, valid json of extracted table> },    \"table_info\": <str, a detailed description of the table and the information it represents>}, {\"table_contents\": { < dict, valid json of extracted table> },    \"table_info\": <str, a detailed description of the table and the information it represents>} ...]\n}\n```\n\nIf there is nothing to extract, simply return an empty JSON \n\n```json\n{\"tables\": []}\n```"
    use_json_decoding: false

  charts:
    prompts:
      system: "You are an OCR expert. Your task is to extract charts, graphs and other graphical visualizations of data from the following document into JSON. Do not extract tables, only analyze and extract the charts in this document.\n\n### Steps:\n\n1. Identify the type of chart/graph (e.g., bar chart, line graph, pie chart).\n2. Identify the axes, labels, categories, and any notable trends or patterns.\n3. Provide a brief textual description of what the chart/graph represents; use the caption and surrounding text in the document to write this.\n4. Extract and structure the data into a valid and logical JSON structure.\n5. If there are no charts or graphs to extract, simply return an empty JSON {\"charts\": []}. Ensure your final answer is appropriately formatted as a JSON object and wrapped in a ```json\n\n``` block."
      user: "TARGET SCHEMA:\n```json\n{schema}\n```"
    output_schema: {
      "type": "object",
      "properties": {
        "charts": {
          "type": "array",
          "items": {
            "type": "object",
            "properties": {
              "chart_type": {
                "type": "string",
                "enum": ["bar", "line", "pie", "scatter", "area", "sankey", "heatmap"]
              },
              "description": {
                "type": "string",
                "description": "Detailed description of what the image shows"
              },
              "caption": {
                "type": "string",
                "description": "Caption or label associated with the image, if any"
              },
              "data": {
                "type": "object",
                "description": "Data in the chart; use appropriate keys and values to best represent the information in the chart",
              }
            },
            "required": [
              "chart_type",
              "description",
              "caption",
              "data"
            ]
          }
        }
      },
      "required": [
        "charts"
      ]
    }
    use_json_decoding: false