####### A test function for a single tweet
    
text = "#HurricaneHarvey INTENSE eye wall of category 4 Hurricane Harvey from Rockport, TX" 
#"someone placed a car here for ins purpose. parking lot of I-10 highway. Terry Hershey Park Parking lot. …"

#"A cattle drive takes to the streets of Dayton, Tx. due to Harvey txwx Video via Rebekah Brand" 
#"someone placed a car here for ins purpose. parking lot of I-10 highway. Terry Hershey Park Parking lot. …"


this_conv_prompt = conversational_recog_prompt_syn.copy()
this_conv_prompt.append({'role': 'user','content': 'Sentence: '+text+' \nQ: Which parts of this sentence represent location descriptions? '})
#print(this_conv_prompt)

completion = openai.ChatCompletion.create(
  model=model_name,
  messages=this_conv_prompt,
  max_tokens =  n_tokens,
    temperature=temp,
    top_p=1,
    n=1,
    presence_penalty=0,
    frequency_penalty=0
)

answer = completion.choices[0].message["content"]
    
answer

len(cur_data_list)

# Start to run the code
count = 0
for i, row in (pbar := tqdm(enumerate(cur_data_list), total= len(cur_data_list) )):
    
    
    res_dict = get_place_recognition_answer_logits_conversational(row, conversational_recog_prompt_syn, text_key = "tweetFulltext")
    
    with open(res_file, 'a') as f:
        f.write(json.dumps(res_dict) + "\n")
      
    #print the result for a preview
#     print("Tweet ID is: "+ res_dict['tweetFileID'])
#     print("Tweet text is: "+ res_dict['tweetFulltext'])
#     print("Annotation is: "+ str(res_dict['tweetAnnotation']))
#     print("Answer is: "+ res_dict['answer'])
    
    count += 1
    time.sleep(0.5)
    
#     if count == 2:
#         break

f.close()

res_file

# read the result

import pandas as pd
res_dict_list = read_res_file(res_file)
res_dict_list
result_df = pd.DataFrame.from_records(res_dict_list)
result_df = result_df[["tweetFileID","tweetFulltext","tweetAnnotation","answer"]]
result_df.head()   

# Evaluation without considering categories: Precision, Recall, and F-score

def evaluate_performance_disregard_category(df, model_name):
    
    eval_output_dir = "Output/" 
    if not os.path.exists(eval_output_dir):
        os.mkdir(eval_output_dir)

    total_tp = 0
    total_predicted = 0
    total_true = 0
    
    df = df.reset_index()  # make sure indexes pair with number of rows
    
    # this is the code to output corret and incorrect location descriptions; useful for error analysis
    correctAnalysisFile = open('Output/'+model_name+'-evaluation-string_matched.csv','w', encoding="utf-8")
    correctAnalysisFile.write("annotationDesc|predictDesc\n")
    
    incorrectAnalysisFile = open('Output/'+model_name+'-evaluation-string_not_matched.csv','w', encoding="utf-8")
    incorrectAnalysisFile.write("TweetID|TweetFullText|annotationDesc|predictDesc\n")
    
    
    for index, row in df.iterrows():
        tweetID = row["tweetFileID"].replace("Data/","")
        
          
        this_prediction_list = row['answer'].strip(" ;")
        this_prediction_list = this_prediction_list.split(";")
       
        
        if type(row['tweetAnnotation']) == str:
            this_annotation_list = json.loads(row['tweetAnnotation'])
        else:
            this_annotation_list = row['tweetAnnotation']
        
        #print(row['tweetFulltext'])
        #print(str(this_prediction_list).encode("utf-8"))
        #print(this_annotation_list)
        #print("---")
        
        recognized = 0
        for this_annotation_item in this_annotation_list:
            this_annotation_cate = this_annotation_item['locationCate']
            this_annotation_desc = this_annotation_item['locationDesc']
            this_annotation_desc = this_annotation_desc.strip(".")  # disregard the cases when the difference between the true and predicted is only a .
            
            for this_prediction_item in this_prediction_list:
                this_prediction_item_cleaned = this_prediction_item.strip(" \n")
                this_prediction_result = this_prediction_item_cleaned.split(":")
                
                # check to ensure this is a valid prediction
                if len(this_prediction_result) != 2:
                    this_prediction_list.remove(this_prediction_item)
                    continue
                # check to ensure this is a valid prediction
                
                this_prediction_cate = this_prediction_result[0]
                this_prediction_desc = this_prediction_result[1].strip()
                this_prediction_desc = this_prediction_desc.strip(".")
                
                if this_annotation_desc.lower() == this_prediction_desc.lower():  #an alternative fuzzy way to evaluate: fuzz.ratio(this_annotation_desc.lower(), this_prediction_desc.lower()) >= 75:
                    total_tp += 1
                    total_predicted += 1
                    
                    correctAnalysisFile.write(this_annotation_cate+"|"+this_annotation_desc+"|"+this_prediction_desc+"\n")
                        
                    this_prediction_list.remove(this_prediction_item)  # remove this item from the prediction list once a match is found
                    recognized += 1
                    break
                
            total_true += 1
            
        # those that remain in the prediction list are false positives, i.e., predicted
        for this_prediction_item in this_prediction_list:
            total_predicted += 1
            
            
        # write errors
        if (recognized != len(this_annotation_list)) or (len(this_prediction_list)>0):
            incorrectAnalysisFile.write(tweetID+"|"+row['tweetFulltext']+"|"+str(row['tweetAnnotation'])+"|"+row['answer']+"\n")
                
    
    
    # after all the counts are done, calculate the metrics
    outputResultFile = open('Output/'+model_name+'-evaluation-not_categorized.csv','w')
    outputResultFile.write("Precision,Recall,Fscore\n")
    
    
    total_precision = total_tp*1.0/total_predicted
    total_recall = total_tp*1.0/total_true
    total_fscore = (2 * total_precision * total_recall) / (total_precision + total_recall)
    print("Total precision is "+str(total_precision)+", total recall is "+str(total_recall)+", total fscore is "+str(total_fscore))
    outputResultFile.write(str(total_precision)+","+str(total_recall)+","+str(total_fscore))
    
    
evaluate_performance_disregard_category(result_df,model_name)

# Evaluating across categories

def evaluate_performance_with_category(df, model_name):
    
    # create values for the confusion matrix
    total_valid_cate = 11
    
    result_dict={}  # a dictionary used to store the results
    
    df = df.reset_index()
    for index, row in df.iterrows():
        
        tweetID = row["tweetFileID"].replace("Data/","")
       
        this_prediction_list = row['answer'].split(";")
        
        if type(row['tweetAnnotation']) == str:
            this_annotation_list = json.loads(row['tweetAnnotation'])
        else:
            this_annotation_list = row['tweetAnnotation']
        
    
        #print(row['tweetFulltext'])
        #print(str(this_prediction_list).encode("utf-8"))
        #print(this_annotation_list)
        #print("---")
    
        for this_annotation_item in this_annotation_list:
            this_annotation_cate = this_annotation_item['locationCate']
            this_annotation_desc = this_annotation_item['locationDesc']
            this_annotation_desc = this_annotation_desc.strip(".")
            
            matched = False
            for this_prediction_item in this_prediction_list:
                this_prediction_item_cleaned = this_prediction_item.strip(" \n")
                this_prediction_result = this_prediction_item_cleaned.split(":")
    
                # check to ensure this is a valid prediction
                if len(this_prediction_result) != 2:
                    this_prediction_list.remove(this_prediction_item)
                    continue
                # check to ensure this is a valid prediction
    
                this_prediction_cate = this_prediction_result[0]
                this_prediction_desc = this_prediction_result[1].strip()
                this_prediction_desc = this_prediction_desc.strip(".")
                this_prediction_cate_number = int(this_prediction_cate[1:])
    
                if this_annotation_desc.lower() == this_prediction_desc.lower():#(fuzz.ratio(this_annotation_desc.lower(), this_prediction_desc.lower()) >= 75) and (this_prediction_cate_number <= total_valid_cate):
                    combined_cate = this_annotation_cate+this_prediction_cate  #create a unique string for the category id
                    if combined_cate in result_dict.keys():
                        result_dict[combined_cate] = result_dict[combined_cate] + 1
                    else:
                        result_dict[combined_cate] = 1
    
                    this_prediction_list.remove(this_prediction_item)  # remove this item from the prediction list once a match is found
                    matched = True
                    break
    
    
            if not matched:
                combined_cate = this_annotation_cate+"C12"
                if combined_cate in result_dict.keys():
                    result_dict[combined_cate] = result_dict[combined_cate] + 1
                else:        
                    result_dict[combined_cate] = 1
    
    
    
        # those that remain in the prediction list are false positives, i.e., predicted
        for this_prediction_item in this_prediction_list:
            this_prediction_item_cleaned = this_prediction_item.strip(" \n")
            this_prediction_result = this_prediction_item_cleaned.split(":")
            
            # check to ensure this is a valid prediction
            if len(this_prediction_result) != 2:
                this_prediction_list.remove(this_prediction_item)
                continue
            # check to ensure this is a valid prediction
    
            this_prediction_cate = this_prediction_result[0].strip()
            this_prediction_desc = this_prediction_result[1].strip()
            
            if not this_prediction_cate.startswith('C'):
                this_prediction_list.remove(this_prediction_item)
                continue
            
            this_prediction_cate_number = int(this_prediction_cate[1:])
    
            if this_prediction_cate_number <= total_valid_cate:
                combined_cate = "C12"+ this_prediction_cate
            else:
                combined_cate = "C12C12"
                
            if combined_cate in result_dict.keys():
                result_dict[combined_cate] = result_dict[combined_cate] + 1
            else:        
                result_dict[combined_cate] = 1

    
    # after all the counts are done, calculate the metrics
    outputResultFile = open('Output/'+model_name+'-evaluation-categorized.csv','w')
    outputResultFile.write("Model,Category,Precision,Recall,Fscore\n")
    
    

    for cate_num in range(1, total_valid_cate+1):
        value_key = "C"+str(cate_num)+"C"+str(cate_num)
        if value_key in result_dict.keys():
            value = result_dict[value_key]
        else:
            value = 0
        this_tp_count = value
        
        this_true_count = 0
        for pred_cate_num in range(1, total_valid_cate+2):
            value_key = "C"+str(cate_num)+"C"+str(pred_cate_num)
            if value_key in result_dict.keys():
                value = result_dict[value_key]
            else:
                value = 0
            this_true_count += value
        
        this_predict_count = 0
        for true_cate_num in range(1, total_valid_cate+2):
            value_key = "C"+str(true_cate_num)+"C"+str(cate_num)
            if value_key in result_dict.keys():
                value = result_dict[value_key]
            else:
                value = 0
            this_predict_count += value
        
        
        precision = this_tp_count*1.0/(this_predict_count + 0.000001)  # 0.000001 is to avoid the denominator to be 0
        recall = this_tp_count*1.0/(this_true_count + 0.000001)
        fscore = (2 * precision * recall) / (precision + recall + 0.000001)
        print("For category C"+str(cate_num)+": precision is "+str(precision)+", recall is "+str(recall)+", fscore is "+str(fscore))
        outputResultFile.write(model_name+",C"+str(cate_num)+","+str(precision)+","+str(recall)+","+str(fscore)+"\n")
        
        
        
evaluate_performance_with_category(result_df,model_name)     

# Visualization
df = pd.read_csv("Output/"+model_name+"-evaluation-categorized.csv")
df.head(15)

# importing the required library
import seaborn as sns
import matplotlib.pyplot as plt
 
# read a titanic.csv file
# from seaborn library
#df = sns.load_dataset('titanic')
 
 
# Show the plot
plt.figure(figsize=(9,3))
# who v/s fare barplot
ax = sns.barplot(x = 'Category',
            y = 'Precision',
            hue = 'Model',
            data = df, order = ["C1","C2","C3","C4","C5","C6","C7","C8","C9","C10","C11"], palette="muted", zorder=3)
#plt.show()
#plt.legend(bbox_to_anchor=(1.02, 1), loc='upper left', borderaxespad=0)
sns.move_legend(
    ax, "lower center",
    bbox_to_anchor=(0.5, -.55),
    ncol=5,
    title=None, frameon=False,
)
ax.grid(axis='y',linestyle='--', linewidth=0.5, zorder=0)
plt.ylim([0,1])
#plt.savefig("Precision.png", dpi=300, format='png',bbox_extra_artists=(ax,), bbox_inches='tight')

# Show the plot
plt.figure(figsize=(9,3))
# who v/s fare barplot
ax = sns.barplot(x = 'Category',
            y = 'Recall',
            hue = 'Model',
            data = df, order = ["C1","C2","C3","C4","C5","C6","C7","C8","C9","C10","C11"], palette="muted", zorder=3)
#plt.show()
#plt.legend(bbox_to_anchor=(1.02, 1), loc='upper left', borderaxespad=0)
sns.move_legend(
    ax, "lower center",
    bbox_to_anchor=(0.5, -.55),
    ncol=5,
    title=None, frameon=False,
)
ax.grid(axis='y',linestyle='--', linewidth=0.5, zorder=0)
plt.ylim([0,1])

# Show the plot
plt.figure(figsize=(9,3))
# who v/s fare barplot
ax = sns.barplot(x = 'Category',
            y = 'Fscore',
            hue = 'Model',
            data = df, order = ["C1","C2","C3","C4","C5","C6","C7","C8","C9","C10","C11"], palette="muted", zorder=3)
#plt.show()
#plt.legend(bbox_to_anchor=(1.02, 1), loc='upper left', borderaxespad=0)
sns.move_legend(
    ax, "lower center",
    bbox_to_anchor=(0.5, -.55),
    ncol=5,
    title=None, frameon=False,
)
ax.grid(axis='y',linestyle='--', linewidth=0.5, zorder=0)
plt.ylim([0,1])