llm-evals/llmeval/commons/apps.py

import os
from django.apps import AppConfig
from django.db import transaction
from django.db.models.signals import post_migrate


def import_llm_backends(sender, **kwargs):
    backend_model = sender.get_model('LLMBackend')

    if backend_model.objects.count() != 0:
        return

    backend_model.objects.create(
        name='OpenAI',
        parameters={
            'api_key': 'sk-1234567890abcdef1234567890abcdef',
        },
        client_type='openai'
    )
    backend_model.objects.create(
        name='Ollama',
        parameters={
            'host': 'http://localhost:11434',
        },
        client_type='ollama'
    )
    backend_model.objects.create(
        name='Anyscale',
        parameters={
            "api_key": "esecret_12345abcde", 
            "base_url": "https://api.endpoints.anyscale.com/v1"
        },
        client_type='openai'
    )
    backend_model.objects.create(
        name='TogheterAI',
        parameters={
            "api_key": "sk-12345abcde" 
        },
        client_type='togheter'
    )
    backend_model.objects.create(
        name='Groq',
        parameters={
            "api_key": "sk-12345abcde" 
        },
        client_type='groq'
    )
    backend_model.objects.create(
        name='Anthropic',
        parameters={
            "api_key": "sk-12345abcde"
        },
        client_type='anthropic'
    )
    backend_model.objects.create(
        name='GoogleGenAI',
        parameters={
            "api_key": "sk-12345abcde"
        },
        client_type='genai'
    )
    backend_model.objects.create(
        name='LMStudio',
        parameters={
            "api_key": "lm-studio",
            "base_url": "http://localhost:1234/v1"

        },
        client_type='openai'
    )

def import_llm_models(sender, **kwargs):
    backend_model = sender.get_model('LLMBackend')
    llm_model = sender.get_model('LLMModel')

    if llm_model.objects.count() != 0:
        return

    openai_backend = backend_model.objects.get(name='OpenAI')
    llm_model.objects.create(
        name='gpt-4o',
        backend=openai_backend,
    )
    llm_model.objects.get_or_create(
        name='gpt-4o-mini',
        backend=openai_backend,
    )
    
    ollama_backend = backend_model.objects.get(name='Ollama')
    llm_model.objects.get_or_create(
        name='llama3',
        backend=ollama_backend,
    )

    anhropic_backend = backend_model.objects.get(name='Anthropic')
    llm_model.objects.get_or_create(
        name='claude-3-5-sonnet-20240620',
        backend=anhropic_backend,
    )

    anyscale_backend = backend_model.objects.get(name='Anyscale')
    llm_model.objects.get_or_create(
        name='meta-llama/Meta-Llama-3-70B-Instruct',
        backend=anyscale_backend,
    )

def import_datasets(sender, **kwargs):
    dataset_model = sender.get_model('Dataset')

    if dataset_model.objects.count() != 0:
        return

    dataset_model.objects.create(
        name='medqa',
        description='MedQA test dataset - 1273 questions',
    )
    
    os.system('python manage.py import_medqa --file datasets//medqa/test.jsonl --dataset medqa --target test')

    dataset_model.objects.create(
        name='pubmedqa',
        description='PubMedQA test[expert] dataset - 1000 questions',
    )
    os.system('python manage.py import_pubmedqa --file datasets//pubmedqa/ori_pqal.json --file-test-ground-truth=datasets/pubmedqa/test_ground_truth.json --dataset pubmedqa --target test')

    # dataset_model.objects.create(
    #     name='mmlu',
    #     description='MMLU test dataset - all 14012 questions',
    # )
    # os.system('python manage.py import_mmlu --subject all --dataset mmlu --target test')


def import_configs(sender, **kwargs):
    config_model = sender.get_model('EvalConfig')
    dataset = sender.get_model('Dataset')

    # get the medqa dataset
    medqa_dataset = dataset.objects.get(name='medqa')

    if config_model.objects.count() != 0:
        return
    
    with transaction.atomic():
        config_model.objects.create(
            name='MedQA - zero shot CoT',
            description='MedQA test dataset evaluation',
            dataset=medqa_dataset,
            sys_prompt="""
You are a helpful assistant designed to answer questions. 
Provide concise and accurate responses, ensuring you follow the provided instructions thoroughly.""".strip(),
            final_answer_pattern = '(?i)Final Answer\s*:\s*([A-E])'
        )

        # add role messages
        role_message_model = sender.get_model('RoleMessage')
        config = config_model.objects.get(name='MedQA - zero shot CoT')
        role_message_model.objects.create(
            role='user',
            eval_config=config,
            content = """
Answer the following multiple choice question. The last line of your response should be of the following format: 'Final Answer: $LETTER' (without quotes) 
where LETTER is the index of the correct answers from given choices. Think step by step before answering.""".strip()
        )

def import_sessions(sender, **kwargs):
    session_model = sender.get_model('EvalSession')
    config = sender.get_model('EvalConfig')
    llm_model = sender.get_model('LLMModel')

    if session_model.objects.count() != 0:
        return

    medqa_config = config.objects.get(name='MedQA - zero shot CoT')
    gpt4o = llm_model.objects.get(name='gpt-4o')
    session_model.objects.create(
        name='MedQA - gpt-4o',
        config=medqa_config,
        llm_model=gpt4o,
        dataset_target='test'
    )

def answer_interpreters(sender, **kwargs):
    answer_interpreter_model = sender.get_model('AnswerInterpreter')

    if answer_interpreter_model.objects.count() != 0:
        return
    
    llm_model = sender.get_model('LLMModel')
    gpt4o_mini = llm_model.objects.get(name='gpt-4o-mini')

    answer_interpreter_model.objects.create(
        name='YesNoMaybe Interpreter',
        prompt="""
You are a helpful assistant who will respond concisely to user requests. Your task is to examine the answer given by another virtual assistant and classify it as 'yes', 'no' or 'maybe'. Your final line of response should be 'Final Answer: yes | no | maybe' (without quotes).
Here is the assistant's answer:
"$QUESTION"
The final answer:""",
        llm_model=gpt4o_mini,
    )

    answer_interpreter_model.objects.create(
        name='MultiChoice Interpreter',
        prompt="""
You are a helpful assistant who will respond concisely to user requests. Your task is to examine the answer given by another virtual assistant and extract the letter corresponding to the selected option from multiple choices. Your final line of response should be "Final Answer: $LETTER" where $LETTER is the letter of the answer chosen by the assistant.
Here is the assistant's answer:
"$QUESTION"
The final answer:""",
        llm_model=gpt4o_mini,
    )

class CommonsConfig(AppConfig):
    default_auto_field = 'django.db.models.BigAutoField'
    name = 'commons'


    def ready(self):
        post_migrate.connect(import_llm_backends, sender=self)
        post_migrate.connect(import_llm_models, sender=self)
        post_migrate.connect(import_datasets, sender=self)
        post_migrate.connect(import_configs, sender=self)
        post_migrate.connect(import_sessions, sender=self)
        post_migrate.connect(answer_interpreters, sender=self)