Consolidate results from multiple LLMs into a CSV file
Once you run the same data extraction (for example ONSET_DATE) using multiple Gemini models, you can consolidate all the results into a CSV file.
This is what the Python code look like
import json
import pandas as pd
def get_summary(k1, v1):
summary_str = f'''
<details>
<summary>{k1}</summary>
{v1["explanation"]}
</details>
'''
return summary_str
model_list = [
'gemini-1.5-flash',
'gemini-1.5-pro',
'gemini-2.0-flash-lite',
'gemini-2.5-flash-preview-05-20'
]
experiment = 'had_covid19'
key_name = 'had_covid19'
df: pd.DataFrame = pd.read_csv(f'../csv/llm/japan_100.csv')
vaers_id_list = df['vaers_id'].unique()
output_file_name = f'../json/majority_vote/majority_{experiment}.json'
output_obj = {}
for model_name in model_list:
file_name = f'../json/{experiment}/{experiment}_{model_name}.json'
with open(file_name, 'r') as f:
file_obj = json.load(f)
for vaers_id in vaers_id_list:
vaers_id_str = str(vaers_id)
if not vaers_id_str in output_obj.keys():
output_obj[vaers_id_str] = {}
curr_obj = file_obj[vaers_id_str]
parsed_obj = curr_obj['parsed']
curr_val = parsed_obj[key_name]
curr_expl = parsed_obj[f'{key_name}_explanation']
output_obj[vaers_id_str][model_name] = {
"value": curr_val,
"explanation": curr_expl
}
data = []
for key, value in output_obj.items():
curr_obj = {
"vaers_id": key
}
value_map = {}
expl_sentences = ''
for k1, v1 in value.items():
curr_val = str(v1['value']).strip().lower()
expl_sentences += get_summary(k1, v1)
curr_obj[f'val_{k1}'] = v1['value']
if curr_val not in value_map.keys():
value_map[curr_val] = 0
value_map[curr_val] += 1
has_majority = 'No'
majority_value = ''
max_votes = 0
for k1, v1 in value_map.items():
if v1 > max_votes:
max_votes = v1
if v1 >= 3:
has_majority = 'Yes'
majority_value = k1
curr_obj['majority value'] = majority_value
curr_obj['expl_sentences'] = expl_sentences
output_obj[key]['correct_value'] = majority_value
data.append(curr_obj)
df1: pd.DataFrame = pd.DataFrame(data)
df1.to_csv(f'../csv/majority_vote/{experiment}.csv', index=False)
with open(output_file_name, 'w+') as f:
json.dump(output_obj, f, indent=2)
The basic idea is that for each VAERS_ID you will get the corresponding value extracted by each Gemini model and check to see if there is a majority vote (3 or more LLMs agree on the same value).
By doing this you have constructed comparison data which you can inspect using my custom LLM Eval tool.