synthetic-data/intro.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Click [here](https://medium.com/@morihosseini/synthetic-data-unleashing-possibilities-42eadd5c2fbb) to access the associated Medium article."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Faker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install -q faker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name: Lee Thomas\n",
      "Address: 3351 Travis Hill\n",
      "Garciaberg, NE 94330\n",
      "Phone Number: 595-467-8251x37667\n"
     ]
    }
   ],
   "source": [
    "from faker import Faker\n",
    "\n",
    "faker = Faker()\n",
    "\n",
    "name = faker.name()\n",
    "address = faker.address()\n",
    "phone_number = faker.phone_number()\n",
    "\n",
    "print(f\"Name: {name}\\nAddress: {address}\\nPhone Number: {phone_number}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GPT-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install -q openai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "customer_id | order_id | order_date | product_id | product_name | product_price\n",
      "--- | --- | --- | --- | --- | ---\n",
      "1 | 201001 | 15-03-2022 | P101 | Apple iPhone 13 | 799\n",
      "2 | 201002 | 16-03-2022 | P102 | Lenovo ThinkPad L340 | 500\n",
      "3 | 201003 | 17-03-2022 | P103 | Samsung Galaxy S21 | 699\n",
      "4 | 201004 | 18-03-2022 | P104 | Dell Inspiron 15 | 550\n",
      "5 | 201005 | 19-03-2022 | P105 | MacBook Pro       | 2400\n"
     ]
    }
   ],
   "source": [
    "import openai\n",
    "\n",
    "openai.api_key = \"YOUR_API_KEY\"\n",
    "\n",
    "prompt = (\n",
    "    \"Generate a synthetic dataset with 5 records of customer orders.\"\n",
    "    \"The dataset should have the following columns: customer_id, \"\n",
    "    \"order_id, order_date, product_id, product_name.\"\n",
    ")\n",
    "response = openai.ChatCompletion.create(\n",
    "    model=\"gpt-4\", messages=[{\"role\": \"user\", \"content\": prompt}]\n",
    ")\n",
    "synthetic_text = response.choices[0].message.content\n",
    "\n",
    "print(synthetic_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mimicking Data Distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install -q numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.49302535 0.39866691 0.42177036 0.3342057  0.63561163 0.69563605\n",
      " 0.54134615 0.5724373  0.32113683 0.43635857]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "# Sample synthetic data with similar distribution as real data\n",
    "mean_real = 0.5\n",
    "std_real = 0.1\n",
    "num_samples = 10\n",
    "synthetic_data = np.random.normal(mean_real, std_real, size=num_samples)\n",
    "\n",
    "print(synthetic_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Incorporating Noise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.9029237560973817, 2.997350308120952, 4.0929803121832435, 4.912912456195437]\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "# Introduce random noise to synthetic data points\n",
    "original_data_points = [2, 3, 4, 5]\n",
    "noisy_data_points = [\n",
    "    point + random.uniform(-0.1, 0.1) for point in original_data_points\n",
    "]\n",
    "\n",
    "print(noisy_data_points)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Validating the Quality of Synthetic Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install -q scipy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No significant difference detected!\n"
     ]
    }
   ],
   "source": [
    "from scipy.stats import ttest_ind\n",
    "\n",
    "real_data = [1, 2, 3, 4, 5]\n",
    "synthetic_data = [10, 2, 3, 4, 50]\n",
    "\n",
    "# Perform a statistical test between real and synthetic data distributions.\n",
    "t_statistic, p_value = ttest_ind(real_data, synthetic_data)\n",
    "\n",
    "if p_value < 0.05:\n",
    "    print(\"Significant difference detected!\")\n",
    "else:\n",
    "    print(\"No significant difference detected!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Protecting privacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install -q numpy scikit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.22967237 0.3528305  0.62191547 1.        ]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "sensitive_data = [\"John\", \"Mary\", \"Chris\", \"Sarah\"]\n",
    "\n",
    "# Encode sensitive names to numeric labels\n",
    "le = LabelEncoder()\n",
    "sensitive_labels = le.fit_transform(sensitive_data)\n",
    "\n",
    "# Add noise\n",
    "noisy_labels = sensitive_labels + np.random.normal(\n",
    "    scale=0.5, size=len(sensitive_labels)\n",
    ")\n",
    "\n",
    "# Perturb data by shuffling\n",
    "np.random.shuffle(noisy_labels)\n",
    "\n",
    "# Transform by scaling and clipping to 0-1 range\n",
    "noisy_labels = np.clip(noisy_labels * 0.75, 0, 1)\n",
    "\n",
    "print(noisy_labels)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}