Consolidate results from multiple LLMs into a CSV file

Once you run the same data extraction (for example ONSET_DATE) using multiple Gemini models, you can consolidate all the results into a CSV file.

This is what the Python code look like

import json
import pandas as pd

def get_summary(k1, v1):
    summary_str = f'''
    <details>
  <summary>{k1}</summary>
  {v1["explanation"]}
</details>
    '''
    return summary_str

model_list = [
    'gemini-1.5-flash',
    'gemini-1.5-pro',
    'gemini-2.0-flash-lite',
    'gemini-2.5-flash-preview-05-20'
]

experiment = 'had_covid19'
key_name = 'had_covid19'

df: pd.DataFrame = pd.read_csv(f'../csv/llm/japan_100.csv')
vaers_id_list = df['vaers_id'].unique()

output_file_name = f'../json/majority_vote/majority_{experiment}.json'

output_obj = {}
for model_name in model_list:
    file_name = f'../json/{experiment}/{experiment}_{model_name}.json'
    with open(file_name, 'r') as f:
        file_obj = json.load(f)
    for vaers_id in vaers_id_list:
        vaers_id_str = str(vaers_id)
        if not vaers_id_str in output_obj.keys():
            output_obj[vaers_id_str] = {}
        curr_obj = file_obj[vaers_id_str]
        parsed_obj = curr_obj['parsed']
        curr_val = parsed_obj[key_name]
        curr_expl = parsed_obj[f'{key_name}_explanation']
        output_obj[vaers_id_str][model_name] = {
            "value": curr_val,
            "explanation": curr_expl
        }

data = []
for key, value in output_obj.items():
    curr_obj = {
        "vaers_id": key
    }
    value_map = {}
    expl_sentences = ''
    for k1, v1 in value.items():
        curr_val = str(v1['value']).strip().lower()
        expl_sentences += get_summary(k1, v1)
        curr_obj[f'val_{k1}'] = v1['value']
        if curr_val not in value_map.keys():
            value_map[curr_val] = 0
        value_map[curr_val] += 1

    has_majority = 'No'
    majority_value = ''
    max_votes = 0
    for k1, v1 in value_map.items():
        if v1 > max_votes:
            max_votes = v1
        if v1 >= 3:
            has_majority = 'Yes'
            majority_value = k1
    curr_obj['majority value'] = majority_value
    curr_obj['expl_sentences'] = expl_sentences
    output_obj[key]['correct_value'] = majority_value
    data.append(curr_obj)

df1: pd.DataFrame = pd.DataFrame(data)
df1.to_csv(f'../csv/majority_vote/{experiment}.csv', index=False)

with open(output_file_name, 'w+') as f:
    json.dump(output_obj, f, indent=2)

The basic idea is that for each VAERS_ID you will get the corresponding value extracted by each Gemini model and check to see if there is a majority vote (3 or more LLMs agree on the same value).

By doing this you have constructed comparison data which you can inspect using my custom LLM Eval tool.

Leave a Reply Cancel reply