CC_Evaluation_Thread.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import threading\n",
    "from openai import OpenAI\n",
    "\n",
    "def read_csv_file(file_path):\n",
    "    \"\"\"Reads the content of a given CSV file.\"\"\"\n",
    "    try:\n",
    "        with open(file_path, mode='r') as csvfile:\n",
    "            reader = csv.DictReader(csvfile)\n",
    "            rows = [row for row in reader]\n",
    "            return rows\n",
    "    except IOError as e:\n",
    "        print(f\"Error reading file {file_path}: {e}\")\n",
    "        return None\n",
    "\n",
    "def write_csv_file(file_path, data):\n",
    "    \"\"\"Writes the content to a CSV file.\"\"\"\n",
    "    try:\n",
    "        with open(file_path, mode='w', newline='') as csvfile:\n",
    "            fieldnames = ['response', 'file_content', 'evaluation', 'score', 'reason']\n",
    "            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n",
    "            writer.writeheader()\n",
    "            writer.writerows(data)\n",
    "    except IOError as e:\n",
    "        print(f\"Error writing to file {file_path}: {e}\")\n",
    "\n",
    "def evaluate_response(client, response, file_content, prompt):\n",
    "    \"\"\"Evaluates the response using OpenAI's API and returns the evaluation.\"\"\"\n",
    "    try:\n",
    "        full_prompt = prompt.replace(\"response\", response).replace(\"file_content\", file_content)\n",
    "        api_response = client.chat.completions.create(\n",
    "            model=\"gpt-4-turbo\",\n",
    "            messages=[\n",
    "                {\"role\": \"user\", \"content\": full_prompt},\n",
    "            ],\n",
    "        )\n",
    "        evaluation_content = api_response.choices[0].message.content.strip()\n",
    "        return evaluation_content\n",
    "    except Exception as e:\n",
    "        print(f\"Failed to evaluate response: {e}\")\n",
    "        return None\n",
    "\n",
    "def process_row(client, row, prompt, results, index, lock):\n",
    "    \"\"\"Process a single row and store the result.\"\"\"\n",
    "    response = row.get(\"response\", \"\")\n",
    "    file_content = row.get(\"file_content\", \"\")\n",
    "    if response and file_content:\n",
    "        evaluation = evaluate_response(client, response, file_content, prompt)\n",
    "        if evaluation:\n",
    "            # Parse evaluation to extract score and reason\n",
    "            evaluation_parts = evaluation.split('|')\n",
    "            if len(evaluation_parts) >= 2:\n",
    "                score = evaluation_parts[0].strip()\n",
    "                reason = evaluation_parts[1].strip()\n",
    "            else:\n",
    "                score = \"N/A\"\n",
    "                reason = \"No reason provided\"\n",
    "            \n",
    "            result = {\n",
    "                'response': response,\n",
    "                'file_content': file_content,\n",
    "                'evaluation': evaluation,\n",
    "                'score': score,\n",
    "                'reason': reason,\n",
    "            }\n",
    "        else:\n",
    "            result = {\n",
    "                'response': response,\n",
    "                'file_content': file_content,\n",
    "                'evaluation': 'Failed to evaluate',\n",
    "                'score': 'N/A',\n",
    "                'reason': 'Failed to evaluate response',\n",
    "            }\n",
    "        with lock:\n",
    "            results[index] = result\n",
    "\n",
    "def process_csv_file(input_file, output_file, prompt):\n",
    "    \"\"\"Processes the input CSV file and writes the evaluation to the output CSV file.\"\"\"\n",
    "    client = OpenAI(api_key=\"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\")\n",
    "\n",
    "    rows = read_csv_file(input_file)\n",
    "    if rows is None:\n",
    "        return\n",
    "\n",
    "    results = [None] * len(rows)\n",
    "    threads = []\n",
    "    lock = threading.Lock()\n",
    "\n",
    "    for index, row in enumerate(rows):\n",
    "        thread = threading.Thread(target=process_row, args=(client, row, prompt, results, index, lock))\n",
    "        threads.append(thread)\n",
    "        thread.start()\n",
    "\n",
    "    for thread in threads:\n",
    "        thread.join()\n",
    "\n",
    "    write_csv_file(output_file, results)\n",
    "\n",
    "# Main execution block\n",
    "if __name__ == \"__main__\":\n",
    "    input_file = \"/Users/parthagarwal/Desktop/Concept_Cards/bio2.csv\"\n",
    "    output_file = \"/Users/parthagarwal/Desktop/Concept_Cards/evaluated_responses_bio.csv\"\n",
    "    user_prompt = \"Evaluate the concept cards in the \\\"response\\\" column of the CSV file based on correctness, completeness, formula accuracy, and clarity from a NEET biology exam preparation perspective. Determine if the content accurately reflects the context provided, comprehensively covers all key concepts and data, accurately presents LaTeX formulas, and if the information is clear and understandable to a student. Assign a score from 1 to 100 for each parameter, with 1 being the worst and 100 being the best, considering a slightly strict scoring criteria. Provide an overall score by averaging the parameter scores. Additionally, assign a confidence score indicating your level of confidence in the provided scoring.The evaluation should be strict and large penalties on score should be given if parameter requirements are not satisfied. Finally, conclude with one of the following reasons for the overall score: \\\"There is scope for improvement in the response,\" \"The response generated is good,\" or \"The response is not good.\\\"\"\n",
    "    process_csv_file(input_file, output_file, user_prompt)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}