Skip to content

Commit

Permalink
LLMFactCheck
Browse files Browse the repository at this point in the history
  • Loading branch information
mlupei committed Feb 27, 2024
1 parent 360071a commit b8eb9e2
Show file tree
Hide file tree
Showing 6 changed files with 2,524 additions and 1,603 deletions.
1,114 changes: 1,014 additions & 100 deletions data/1.csv

Large diffs are not rendered by default.

1,001 changes: 501 additions & 500 deletions false_labeled_records.csv

Large diffs are not rendered by default.

993 changes: 497 additions & 496 deletions false_sentence_data.csv

Large diffs are not rendered by default.

1,001 changes: 501 additions & 500 deletions false_triple_data.csv

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion src/get_result.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ def get_result(model_info, prompt, model_type):
model = model_info
full_prompt = prompt
prompt = full_prompt
prompt_template=f'''SYSTEM: You are scientist. Read carefully and answer only yes or no.
prompt_template=f'''SYSTEM: You are computational biologist. Read carefully and please start your answer with 'Yes' or 'No' followed by a justification of why you chose that answer.
USER: {prompt}
Expand Down
16 changes: 10 additions & 6 deletions util/chembl/convert.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ def create_csvs(json_path: str, name: str):
"OBJECT_CUI", "OBJECT_NAME", "OBJECT_SEMTYPE", "OBJECT_NOVELTY",
"Column", "Column", "Column"]
triple_records = []
sentence_id = 1000
sentence_id = 0
n=100
for item in data:
segments = item['p3']['segments'][0]

Expand All @@ -37,14 +38,15 @@ def create_csvs(json_path: str, name: str):
print(sentence)
sentence_id = sentence_id+1
predicate_id = predicate_data['id']


labeled_records.append({
"Predicate ID": predicate_id,
"Triple": f"{subject_name} {predicate} {object_name}",
"Sentence ID": sentence_id,
"Sentence ID": sentence_id+n,
"Sentence": sentence,
"Question": f"Is the triple \"{subject_name} {predicate} {object_name}\" supported by the sentence: \"{sentence}\"?",
"Label": True,
"Label": False,
"Reference": None
})
sentence_records.append({
Expand All @@ -62,7 +64,7 @@ def create_csvs(json_path: str, name: str):

triple_records.append({
"PREDICATION_ID": predicate_id,
"SENTENCE_ID": sentence_id,
"SENTENCE_ID": sentence_id+n,
"PMID": None,
"PREDICATE": predicate,
"SUBJECT_CUI": None,
Expand All @@ -77,13 +79,15 @@ def create_csvs(json_path: str, name: str):
"Column": None,
"Column": None
})
if sentence_id==400:
n=-400
labeled_records_df=pd.DataFrame(labeled_records, columns=labeled_columns)
sentence_df = pd.DataFrame(sentence_records, columns=sentence_columns)
triple_df = pd.DataFrame(triple_records, columns=triple_columns)

labeled_records_df.to_csv(f"{name}_labeled_records.csv", index=False)
sentence_df.to_csv(f"{name}_sentence_data.csv", index=False)
triple_df.to_csv(f"{name}_triple_data.csv", index=False)



create_csvs('json/neo4j.json', 'true')
create_csvs('json/neo4j_false.json', 'false')

0 comments on commit b8eb9e2

Please sign in to comment.