Browse Source

fix issue with jsonl encoding

Elena Paraschiv 8 months ago
parent
commit
97fa791e3a
1 changed files with 1 additions and 1 deletions
  1. 1 1
      llmeval/commons/management/commands/import_medqa.py

+ 1 - 1
llmeval/commons/management/commands/import_medqa.py

@@ -29,7 +29,7 @@ class Command(BaseCommand):
         # check if dataset exists
         # check if dataset exists
         dataset, _ = Dataset.objects.get_or_create(name=dataset_name)
         dataset, _ = Dataset.objects.get_or_create(name=dataset_name)
         
         
-        with open(file_path, 'r') as f:
+        with open(file_path, 'r', encoding='utf-8') as f:
             lines = f.readlines()
             lines = f.readlines()
             for line in tqdm(lines, desc="Importing MedQA data"):
             for line in tqdm(lines, desc="Importing MedQA data"):
                 data = json.loads(line)
                 data = json.loads(line)