{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Atlas Analysis\n",
    "\n",
    "This notebook reproduces the analysis from the INRIA scikit-learn MOOC `dimred_text.ipynb` using the Wikinews corpus."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# Ensure we can import text_core from the project root\n",
    "# PROJECT_ROOT is supplied by the environment\n",
    "project_root = os.environ.get('PROJECT_ROOT', os.getcwd())\n",
    "if project_root not in sys.path:\n",
    "    sys.path.insert(0, project_root)\n",
    "\n",
    "import text_core\n",
    "\n",
    "print(f\"Project Root: {project_root}\")\n",
    "print(\"Loading corpus...\")\n",
    "corpus = text_core.load_corpus()\n",
    "print(f\"Corpus loaded: {len(corpus)} articles\")\n",
    "print(corpus.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Running analysis...\")\n",
    "# Use default parameters as per objective\n",
    "results = text_core.analyze(min_df=5, max_df=0.8, n_clusters=5, seed=42)\n",
    "print(\"Analysis complete.\")\n",
    "print(f\"Silhouette: {results['silhouette']:.4f}\")\n",
    "print(f\"Displayed Variance: {results['displayed_variance']:.4f}\")\n",
    "print(f\"Retained Variance: {results['retained_variance']:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write results to results.json in the current working directory (scratch)\n",
    "# The verifier expects results.json in the execution directory\n",
    "output_data = {\n",
    "    \"results\": {\n",
    "        \"labels\": results[\"labels\"].tolist(),\n",
    "        \"coordinates\": results[\"coordinates\"].tolist(),\n",
    "        \"vocabulary\": results[\"vocabulary\"],\n",
    "        \"top_terms\": results[\"top_terms\"],\n",
    "        \"silhouette\": results[\"silhouette\"],\n",
    "        \"displayed_variance\": results[\"displayed_variance\"],\n",
    "        \"retained_variance\": results[\"retained_variance\"],\n",
    "        \"parameters\": results[\"parameters\"]\n",
    "    }\n",
    "}\n",
    "\n",
    "with open(\"results.json\", \"w\") as f:\n",
    "    json.dump(output_data, f, indent=2)\n",
    "\n",
    "print(\"Results written to results.json\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.12.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}