{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b9db28d4-ea49-443c-a5d6-e1c44bfe4942",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "source": [
    "# Data Analytics with JS\n",
    "\n",
    "The dataset contains all the information about cars, a name of a manufacturer,\n",
    "all car's technical parameters and a sale price of a car.\n",
    "\n",
    "Libraries:\n",
    "\n",
    "- nodejs-polars\n",
    "- @observable/plot"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d163c580-6aa0-4e8c-a780-5cc931003dc8",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "source": [
    "## Exploring Data\n",
    "\n",
    "Use [pola-rs](https://pola-rs.github.io/nodejs-polars/modules.html) dataframes\n",
    "to read and manipulate data.\n",
    "\n",
    "- use `df.head(n)` to get n first rows\n",
    "- use `df.sample(n)` to get n random rows from the dataset\n",
    "- use `df.describe()` to get mean, std, min, max\n",
    "- use `df.select(...cols)` to get new dataframe for columns\n",
    "- use `df.groupBy(..cols).agg()` to group data and get aggragates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1cb3589a-bffe-422a-8e32-a853161b93c4",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.dataresource+json": {
       "bytes": null,
       "data": [
        {
         "CarName": "alfa-romero giulia",
         "aspiration": "std",
         "boreratio": 3.47,
         "car_ID": 1,
         "carbody": "convertible",
         "carheight": 48.8,
         "carlength": 168.8,
         "carwidth": 64.1,
         "citympg": 21,
         "compressionratio": 9,
         "curbweight": 2548,
         "cylindernumber": "four",
         "doornumber": "two",
         "drivewheel": "rwd",
         "enginelocation": "front",
         "enginesize": 130,
         "enginetype": "dohc",
         "fuelsystem": "mpfi",
         "fueltype": "gas",
         "highwaympg": 27,
         "horsepower": 111,
         "peakrpm": 5000,
         "price": 13495,
         "stroke": 2.68,
         "symboling": 3,
         "wheelbase": 88.6
        },
        {
         "CarName": "alfa-romero stelvio",
         "aspiration": "std",
         "boreratio": 3.47,
         "car_ID": 2,
         "carbody": "convertible",
         "carheight": 48.8,
         "carlength": 168.8,
         "carwidth": 64.1,
         "citympg": 21,
         "compressionratio": 9,
         "curbweight": 2548,
         "cylindernumber": "four",
         "doornumber": "two",
         "drivewheel": "rwd",
         "enginelocation": "front",
         "enginesize": 130,
         "enginetype": "dohc",
         "fuelsystem": "mpfi",
         "fueltype": "gas",
         "highwaympg": 27,
         "horsepower": 111,
         "peakrpm": 5000,
         "price": 16500,
         "stroke": 2.68,
         "symboling": 3,
         "wheelbase": 88.6
        },
        {
         "CarName": "alfa-romero Quadrifoglio",
         "aspiration": "std",
         "boreratio": 2.68,
         "car_ID": 3,
         "carbody": "hatchback",
         "carheight": 52.4,
         "carlength": 171.2,
         "carwidth": 65.5,
         "citympg": 19,
         "compressionratio": 9,
         "curbweight": 2823,
         "cylindernumber": "six",
         "doornumber": "two",
         "drivewheel": "rwd",
         "enginelocation": "front",
         "enginesize": 152,
         "enginetype": "ohcv",
         "fuelsystem": "mpfi",
         "fueltype": "gas",
         "highwaympg": 26,
         "horsepower": 154,
         "peakrpm": 5000,
         "price": 16500,
         "stroke": 3.47,
         "symboling": 1,
         "wheelbase": 94.5
        },
        {
         "CarName": "audi 100 ls",
         "aspiration": "std",
         "boreratio": 3.19,
         "car_ID": 4,
         "carbody": "sedan",
         "carheight": 54.3,
         "carlength": 176.6,
         "carwidth": 66.2,
         "citympg": 24,
         "compressionratio": 10,
         "curbweight": 2337,
         "cylindernumber": "four",
         "doornumber": "four",
         "drivewheel": "fwd",
         "enginelocation": "front",
         "enginesize": 109,
         "enginetype": "ohc",
         "fuelsystem": "mpfi",
         "fueltype": "gas",
         "highwaympg": 30,
         "horsepower": 102,
         "peakrpm": 5500,
         "price": 13950,
         "stroke": 3.4,
         "symboling": 2,
         "wheelbase": 99.8
        },
        {
         "CarName": "audi 100ls",
         "aspiration": "std",
         "boreratio": 3.19,
         "car_ID": 5,
         "carbody": "sedan",
         "carheight": 54.3,
         "carlength": 176.6,
         "carwidth": 66.4,
         "citympg": 18,
         "compressionratio": 8,
         "curbweight": 2824,
         "cylindernumber": "five",
         "doornumber": "four",
         "drivewheel": "4wd",
         "enginelocation": "front",
         "enginesize": 136,
         "enginetype": "ohc",
         "fuelsystem": "mpfi",
         "fueltype": "gas",
         "highwaympg": 22,
         "horsepower": 115,
         "peakrpm": 5500,
         "price": 17450,
         "stroke": 3.4,
         "symboling": 2,
         "wheelbase": 99.4
        }
       ],
       "description": null,
       "dialect": null,
       "encoding": null,
       "format": null,
       "hash": null,
       "homepage": null,
       "licenses": null,
       "mediatype": null,
       "path": null,
       "schema": {
        "fields": [
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "car_ID",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "symboling",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "CarName",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "fueltype",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "aspiration",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "doornumber",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "carbody",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "drivewheel",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "enginelocation",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "wheelbase",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "carlength",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "carwidth",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "carheight",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "curbweight",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "enginetype",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "cylindernumber",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "enginesize",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "fuelsystem",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "boreratio",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "stroke",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "compressionratio",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "horsepower",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "peakrpm",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "citympg",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "highwaympg",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "price",
          "rdfType": null,
          "title": null,
          "type": "number"
         }
        ],
        "foreignKeys": null,
        "missingValues": null,
        "primaryKey": null
       },
       "sources": null,
       "title": null
      },
      "text/html": [
       "<table><thead><tr><th>car_ID</th><th>symboling</th><th>CarName</th><th>fueltype</th><th>aspiration</th><th>doornumber</th><th>carbody</th><th>drivewheel</th><th>enginelocation</th><th>wheelbase</th><th>carlength</th><th>carwidth</th><th>carheight</th><th>curbweight</th><th>enginetype</th><th>cylindernumber</th><th>enginesize</th><th>fuelsystem</th><th>boreratio</th><th>stroke</th><th>compressionratio</th><th>horsepower</th><th>peakrpm</th><th>citympg</th><th>highwaympg</th><th>price</th></tr></thead><tbody><tr><td>1</td><td>3</td><td>alfa-romero giulia</td><td>gas</td><td>std</td><td>two</td><td>convertible</td><td>rwd</td><td>front</td><td>88.6</td><td>168.8</td><td>64.1</td><td>48.8</td><td>2548</td><td>dohc</td><td>four</td><td>130</td><td>mpfi</td><td>3.47</td><td>2.68</td><td>9</td><td>111</td><td>5000</td><td>21</td><td>27</td><td>13495</td></tr><tr><td>2</td><td>3</td><td>alfa-romero stelvio</td><td>gas</td><td>std</td><td>two</td><td>convertible</td><td>rwd</td><td>front</td><td>88.6</td><td>168.8</td><td>64.1</td><td>48.8</td><td>2548</td><td>dohc</td><td>four</td><td>130</td><td>mpfi</td><td>3.47</td><td>2.68</td><td>9</td><td>111</td><td>5000</td><td>21</td><td>27</td><td>16500</td></tr><tr><td>3</td><td>1</td><td>alfa-romero Quadrifoglio</td><td>gas</td><td>std</td><td>two</td><td>hatchback</td><td>rwd</td><td>front</td><td>94.5</td><td>171.2</td><td>65.5</td><td>52.4</td><td>2823</td><td>ohcv</td><td>six</td><td>152</td><td>mpfi</td><td>2.68</td><td>3.47</td><td>9</td><td>154</td><td>5000</td><td>19</td><td>26</td><td>16500</td></tr><tr><td>4</td><td>2</td><td>audi 100 ls</td><td>gas</td><td>std</td><td>four</td><td>sedan</td><td>fwd</td><td>front</td><td>99.8</td><td>176.6</td><td>66.2</td><td>54.3</td><td>2337</td><td>ohc</td><td>four</td><td>109</td><td>mpfi</td><td>3.19</td><td>3.4</td><td>10</td><td>102</td><td>5500</td><td>24</td><td>30</td><td>13950</td></tr><tr><td>5</td><td>2</td><td>audi 100ls</td><td>gas</td><td>std</td><td>four</td><td>sedan</td><td>4wd</td><td>front</td><td>99.4</td><td>176.6</td><td>66.4</td><td>54.3</td><td>2824</td><td>ohc</td><td>five</td><td>136</td><td>mpfi</td><td>3.19</td><td>3.4</td><td>8</td><td>115</td><td>5500</td><td>18</td><td>22</td><td>17450</td></tr></tbody></table>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import { display } from \"https://deno.land/x/display@v0.1.1/mod.ts\";\n",
    "import pl from \"npm:nodejs-polars\";\n",
    "\n",
    "let data = await Deno.readTextFile(\"assets/CarPrice_Assignment.csv\");\n",
    "let df = pl.readCSV(data, { sep: \",\" });\n",
    "\n",
    "await display(df.head(5));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a82e43fd-04a6-4bc8-a752-3b03abd6e983",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.dataresource+json": {
       "bytes": null,
       "data": [
        {
         "describe": "mean",
         "enginesize": 126.90731707317073,
         "horsepower": 104.1170731707317,
         "price": 13276.710570731706
        },
        {
         "describe": "std",
         "enginesize": 41.642693438179855,
         "horsepower": 39.54416680936116,
         "price": 7988.852331743148
        },
        {
         "describe": "min",
         "enginesize": 61,
         "horsepower": 48,
         "price": 5118
        },
        {
         "describe": "max",
         "enginesize": 326,
         "horsepower": 288,
         "price": 45400
        },
        {
         "describe": "median",
         "enginesize": 120,
         "horsepower": 95,
         "price": 10295
        }
       ],
       "description": null,
       "dialect": null,
       "encoding": null,
       "format": null,
       "hash": null,
       "homepage": null,
       "licenses": null,
       "mediatype": null,
       "path": null,
       "schema": {
        "fields": [
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "describe",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "enginesize",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "horsepower",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "price",
          "rdfType": null,
          "title": null,
          "type": "number"
         }
        ],
        "foreignKeys": null,
        "missingValues": null,
        "primaryKey": null
       },
       "sources": null,
       "title": null
      },
      "text/html": [
       "<table><thead><tr><th>describe</th><th>enginesize</th><th>horsepower</th><th>price</th></tr></thead><tbody><tr><td>mean</td><td>126.90731707317073</td><td>104.1170731707317</td><td>13276.710570731706</td></tr><tr><td>std</td><td>41.642693438179855</td><td>39.54416680936116</td><td>7988.852331743148</td></tr><tr><td>min</td><td>61</td><td>48</td><td>5118</td></tr><tr><td>max</td><td>326</td><td>288</td><td>45400</td></tr><tr><td>median</td><td>120</td><td>95</td><td>10295</td></tr></tbody></table>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "await display(\n",
    "  df.select(\n",
    "    \"enginesize\",\n",
    "    \"horsepower\",\n",
    "    \"price\",\n",
    "  ).describe(),\n",
    ");"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c931435-0e19-4d32-8ca8-ae81a9cb43c8",
   "metadata": {},
   "source": [
    "## Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e70e5f22-bdde-4140-9015-a6281b8478bf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[33mfalse\u001b[39m"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "// check for duplicates\n",
    "const hasDups = df.select(\"car_ID\").isDuplicated().toArray().includes(true);\n",
    "// if there are duplicates, use df.filter()\n",
    "hasDups;"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7fb89a88-f3ad-457d-9211-2b75eef6096f",
   "metadata": {},
   "source": [
    "Resolve categorical values. For this dataset it is a brand name:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ffb4509e-1930-4e2e-954c-51e919084514",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.dataresource+json": {
       "bytes": null,
       "data": [
        {
         "brand": "subaru"
        },
        {
         "brand": "renault"
        },
        {
         "brand": "buick"
        },
        {
         "brand": "vw"
        },
        {
         "brand": "chevrolet"
        },
        {
         "brand": "audi"
        },
        {
         "brand": "honda"
        },
        {
         "brand": "plymouth"
        },
        {
         "brand": "toyota"
        },
        {
         "brand": "alfa-romero"
        },
        {
         "brand": "saab"
        },
        {
         "brand": "mazda"
        },
        {
         "brand": "mercury"
        },
        {
         "brand": "peugeot"
        },
        {
         "brand": "nissan"
        },
        {
         "brand": "vokswagen"
        },
        {
         "brand": "volkswagen"
        },
        {
         "brand": "maxda"
        },
        {
         "brand": "bmw"
        },
        {
         "brand": "porcshce"
        },
        {
         "brand": "mitsubishi"
        },
        {
         "brand": "dodge"
        },
        {
         "brand": "isuzu"
        },
        {
         "brand": "porsche"
        },
        {
         "brand": "toyouta"
        },
        {
         "brand": "volvo"
        },
        {
         "brand": "jaguar"
        }
       ],
       "description": null,
       "dialect": null,
       "encoding": null,
       "format": null,
       "hash": null,
       "homepage": null,
       "licenses": null,
       "mediatype": null,
       "path": null,
       "schema": {
        "fields": [
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "brand",
          "rdfType": null,
          "title": null,
          "type": "string"
         }
        ],
        "foreignKeys": null,
        "missingValues": null,
        "primaryKey": null
       },
       "sources": null,
       "title": null
      },
      "text/html": [
       "<table><thead><tr><th>brand</th></tr></thead><tbody><tr><td>subaru</td></tr><tr><td>renault</td></tr><tr><td>buick</td></tr><tr><td>vw</td></tr><tr><td>chevrolet</td></tr><tr><td>audi</td></tr><tr><td>honda</td></tr><tr><td>plymouth</td></tr><tr><td>toyota</td></tr><tr><td>alfa-romero</td></tr><tr><td>saab</td></tr><tr><td>mazda</td></tr><tr><td>mercury</td></tr><tr><td>peugeot</td></tr><tr><td>nissan</td></tr><tr><td>vokswagen</td></tr><tr><td>volkswagen</td></tr><tr><td>maxda</td></tr><tr><td>bmw</td></tr><tr><td>porcshce</td></tr><tr><td>mitsubishi</td></tr><tr><td>dodge</td></tr><tr><td>isuzu</td></tr><tr><td>porsche</td></tr><tr><td>toyouta</td></tr><tr><td>volvo</td></tr><tr><td>jaguar</td></tr></tbody></table>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "// get brand names from `CarName`\n",
    "let brandNameTable = df.select(\"CarName\").map((row) => {\n",
    "  const [carName] = row;\n",
    "  const brand = carName.split(\" \")[0].toLowerCase();\n",
    "  return brand;\n",
    "});\n",
    "\n",
    "// create a dataframe from brand names\n",
    "let brandDf = pl.DataFrame({\n",
    "  \"brand\": brandNameTable,\n",
    "});\n",
    "await display(brandDf.unique());"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7f91cbc3-364c-49c9-97e3-370f78b708f8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.dataresource+json": {
       "bytes": null,
       "data": [
        {
         "brand": "subaru"
        },
        {
         "brand": "peugeot"
        },
        {
         "brand": "plymouth"
        },
        {
         "brand": "mitsubishi"
        },
        {
         "brand": "porsche"
        },
        {
         "brand": "bmw"
        },
        {
         "brand": "chevrolet"
        },
        {
         "brand": "dodge"
        },
        {
         "brand": "toyota"
        },
        {
         "brand": "mazda"
        },
        {
         "brand": "mercury"
        },
        {
         "brand": "isuzu"
        },
        {
         "brand": "renault"
        },
        {
         "brand": "buick"
        },
        {
         "brand": "audi"
        },
        {
         "brand": "nissan"
        },
        {
         "brand": "honda"
        },
        {
         "brand": "jaguar"
        },
        {
         "brand": "volkswagen"
        },
        {
         "brand": "alfa-romero"
        },
        {
         "brand": "saab"
        },
        {
         "brand": "volvo"
        }
       ],
       "description": null,
       "dialect": null,
       "encoding": null,
       "format": null,
       "hash": null,
       "homepage": null,
       "licenses": null,
       "mediatype": null,
       "path": null,
       "schema": {
        "fields": [
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "brand",
          "rdfType": null,
          "title": null,
          "type": "string"
         }
        ],
        "foreignKeys": null,
        "missingValues": null,
        "primaryKey": null
       },
       "sources": null,
       "title": null
      },
      "text/html": [
       "<table><thead><tr><th>brand</th></tr></thead><tbody><tr><td>subaru</td></tr><tr><td>peugeot</td></tr><tr><td>plymouth</td></tr><tr><td>mitsubishi</td></tr><tr><td>porsche</td></tr><tr><td>bmw</td></tr><tr><td>chevrolet</td></tr><tr><td>dodge</td></tr><tr><td>toyota</td></tr><tr><td>mazda</td></tr><tr><td>mercury</td></tr><tr><td>isuzu</td></tr><tr><td>renault</td></tr><tr><td>buick</td></tr><tr><td>audi</td></tr><tr><td>nissan</td></tr><tr><td>honda</td></tr><tr><td>jaguar</td></tr><tr><td>volkswagen</td></tr><tr><td>alfa-romero</td></tr><tr><td>saab</td></tr><tr><td>volvo</td></tr></tbody></table>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "// transform to remove duplicates\n",
    "brandNameTable = brandNameTable.map((name) => {\n",
    "  name = name\n",
    "    .replace(\"maxda\", \"mazda\")\n",
    "    .replace(\"porcshce\", \"porsche\")\n",
    "    .replace(\"toyouta\", \"toyota\")\n",
    "    .replace(/(vw|vokswagen)/ig, \"volkswagen\");\n",
    "  return name;\n",
    "});\n",
    "\n",
    "brandDf = pl.DataFrame({\n",
    "  brand: brandNameTable,\n",
    "});\n",
    "await display(brandDf.unique());"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "eb19de54-11bb-40af-8e27-e5e3a6bd11f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.dataresource+json": {
       "bytes": null,
       "data": [
        {
         "CarName": "alfa-romero giulia",
         "aspiration": "std",
         "boreratio": 3.47,
         "brand": "alfa-romero",
         "car_ID": 1,
         "carbody": "convertible",
         "carheight": 48.8,
         "carlength": 168.8,
         "carwidth": 64.1,
         "citympg": 21,
         "compressionratio": 9,
         "curbweight": 2548,
         "cylindernumber": "four",
         "doornumber": "two",
         "drivewheel": "rwd",
         "enginelocation": "front",
         "enginesize": 130,
         "enginetype": "dohc",
         "fuelsystem": "mpfi",
         "fueltype": "gas",
         "highwaympg": 27,
         "horsepower": 111,
         "peakrpm": 5000,
         "price": 13495,
         "stroke": 2.68,
         "symboling": 3,
         "wheelbase": 88.6
        },
        {
         "CarName": "alfa-romero stelvio",
         "aspiration": "std",
         "boreratio": 3.47,
         "brand": "alfa-romero",
         "car_ID": 2,
         "carbody": "convertible",
         "carheight": 48.8,
         "carlength": 168.8,
         "carwidth": 64.1,
         "citympg": 21,
         "compressionratio": 9,
         "curbweight": 2548,
         "cylindernumber": "four",
         "doornumber": "two",
         "drivewheel": "rwd",
         "enginelocation": "front",
         "enginesize": 130,
         "enginetype": "dohc",
         "fuelsystem": "mpfi",
         "fueltype": "gas",
         "highwaympg": 27,
         "horsepower": 111,
         "peakrpm": 5000,
         "price": 16500,
         "stroke": 2.68,
         "symboling": 3,
         "wheelbase": 88.6
        },
        {
         "CarName": "alfa-romero Quadrifoglio",
         "aspiration": "std",
         "boreratio": 2.68,
         "brand": "alfa-romero",
         "car_ID": 3,
         "carbody": "hatchback",
         "carheight": 52.4,
         "carlength": 171.2,
         "carwidth": 65.5,
         "citympg": 19,
         "compressionratio": 9,
         "curbweight": 2823,
         "cylindernumber": "six",
         "doornumber": "two",
         "drivewheel": "rwd",
         "enginelocation": "front",
         "enginesize": 152,
         "enginetype": "ohcv",
         "fuelsystem": "mpfi",
         "fueltype": "gas",
         "highwaympg": 26,
         "horsepower": 154,
         "peakrpm": 5000,
         "price": 16500,
         "stroke": 3.47,
         "symboling": 1,
         "wheelbase": 94.5
        },
        {
         "CarName": "audi 100 ls",
         "aspiration": "std",
         "boreratio": 3.19,
         "brand": "audi",
         "car_ID": 4,
         "carbody": "sedan",
         "carheight": 54.3,
         "carlength": 176.6,
         "carwidth": 66.2,
         "citympg": 24,
         "compressionratio": 10,
         "curbweight": 2337,
         "cylindernumber": "four",
         "doornumber": "four",
         "drivewheel": "fwd",
         "enginelocation": "front",
         "enginesize": 109,
         "enginetype": "ohc",
         "fuelsystem": "mpfi",
         "fueltype": "gas",
         "highwaympg": 30,
         "horsepower": 102,
         "peakrpm": 5500,
         "price": 13950,
         "stroke": 3.4,
         "symboling": 2,
         "wheelbase": 99.8
        },
        {
         "CarName": "audi 100ls",
         "aspiration": "std",
         "boreratio": 3.19,
         "brand": "audi",
         "car_ID": 5,
         "carbody": "sedan",
         "carheight": 54.3,
         "carlength": 176.6,
         "carwidth": 66.4,
         "citympg": 18,
         "compressionratio": 8,
         "curbweight": 2824,
         "cylindernumber": "five",
         "doornumber": "four",
         "drivewheel": "4wd",
         "enginelocation": "front",
         "enginesize": 136,
         "enginetype": "ohc",
         "fuelsystem": "mpfi",
         "fueltype": "gas",
         "highwaympg": 22,
         "horsepower": 115,
         "peakrpm": 5500,
         "price": 17450,
         "stroke": 3.4,
         "symboling": 2,
         "wheelbase": 99.4
        }
       ],
       "description": null,
       "dialect": null,
       "encoding": null,
       "format": null,
       "hash": null,
       "homepage": null,
       "licenses": null,
       "mediatype": null,
       "path": null,
       "schema": {
        "fields": [
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "brand",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "car_ID",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "symboling",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "CarName",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "fueltype",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "aspiration",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "doornumber",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "carbody",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "drivewheel",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "enginelocation",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "wheelbase",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "carlength",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "carwidth",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "carheight",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "curbweight",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "enginetype",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "cylindernumber",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "enginesize",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "fuelsystem",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "boreratio",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "stroke",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "compressionratio",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "horsepower",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "peakrpm",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "citympg",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "highwaympg",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "price",
          "rdfType": null,
          "title": null,
          "type": "number"
         }
        ],
        "foreignKeys": null,
        "missingValues": null,
        "primaryKey": null
       },
       "sources": null,
       "title": null
      },
      "text/html": [
       "<table><thead><tr><th>brand</th><th>car_ID</th><th>symboling</th><th>CarName</th><th>fueltype</th><th>aspiration</th><th>doornumber</th><th>carbody</th><th>drivewheel</th><th>enginelocation</th><th>wheelbase</th><th>carlength</th><th>carwidth</th><th>carheight</th><th>curbweight</th><th>enginetype</th><th>cylindernumber</th><th>enginesize</th><th>fuelsystem</th><th>boreratio</th><th>stroke</th><th>compressionratio</th><th>horsepower</th><th>peakrpm</th><th>citympg</th><th>highwaympg</th><th>price</th></tr></thead><tbody><tr><td>alfa-romero</td><td>1</td><td>3</td><td>alfa-romero giulia</td><td>gas</td><td>std</td><td>two</td><td>convertible</td><td>rwd</td><td>front</td><td>88.6</td><td>168.8</td><td>64.1</td><td>48.8</td><td>2548</td><td>dohc</td><td>four</td><td>130</td><td>mpfi</td><td>3.47</td><td>2.68</td><td>9</td><td>111</td><td>5000</td><td>21</td><td>27</td><td>13495</td></tr><tr><td>alfa-romero</td><td>2</td><td>3</td><td>alfa-romero stelvio</td><td>gas</td><td>std</td><td>two</td><td>convertible</td><td>rwd</td><td>front</td><td>88.6</td><td>168.8</td><td>64.1</td><td>48.8</td><td>2548</td><td>dohc</td><td>four</td><td>130</td><td>mpfi</td><td>3.47</td><td>2.68</td><td>9</td><td>111</td><td>5000</td><td>21</td><td>27</td><td>16500</td></tr><tr><td>alfa-romero</td><td>3</td><td>1</td><td>alfa-romero Quadrifoglio</td><td>gas</td><td>std</td><td>two</td><td>hatchback</td><td>rwd</td><td>front</td><td>94.5</td><td>171.2</td><td>65.5</td><td>52.4</td><td>2823</td><td>ohcv</td><td>six</td><td>152</td><td>mpfi</td><td>2.68</td><td>3.47</td><td>9</td><td>154</td><td>5000</td><td>19</td><td>26</td><td>16500</td></tr><tr><td>audi</td><td>4</td><td>2</td><td>audi 100 ls</td><td>gas</td><td>std</td><td>four</td><td>sedan</td><td>fwd</td><td>front</td><td>99.8</td><td>176.6</td><td>66.2</td><td>54.3</td><td>2337</td><td>ohc</td><td>four</td><td>109</td><td>mpfi</td><td>3.19</td><td>3.4</td><td>10</td><td>102</td><td>5500</td><td>24</td><td>30</td><td>13950</td></tr><tr><td>audi</td><td>5</td><td>2</td><td>audi 100ls</td><td>gas</td><td>std</td><td>four</td><td>sedan</td><td>4wd</td><td>front</td><td>99.4</td><td>176.6</td><td>66.4</td><td>54.3</td><td>2824</td><td>ohc</td><td>five</td><td>136</td><td>mpfi</td><td>3.19</td><td>3.4</td><td>8</td><td>115</td><td>5500</td><td>18</td><td>22</td><td>17450</td></tr></tbody></table>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "// add new column `brand` to our dataframe\n",
    "df = brandDf.hstack(df);\n",
    "await display(df.head(5));"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81abdb0d-41ce-445c-9a75-f9494eb1f9d3",
   "metadata": {},
   "source": [
    "Drop unnecessary values, and write to the new file:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6da96c06-b0b6-46b7-be2d-824af909aec4",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.dataresource+json": {
       "bytes": null,
       "data": [
        {
         "aspiration": "std",
         "boreratio": 3.47,
         "brand": "alfa-romero",
         "carbody": "convertible",
         "carheight": 48.8,
         "carlength": 168.8,
         "carwidth": 64.1,
         "citympg": 21,
         "compressionratio": 9,
         "curbweight": 2548,
         "cylindernumber": "four",
         "doornumber": "two",
         "drivewheel": "rwd",
         "enginelocation": "front",
         "enginesize": 130,
         "enginetype": "dohc",
         "fuelsystem": "mpfi",
         "fueltype": "gas",
         "highwaympg": 27,
         "horsepower": 111,
         "peakrpm": 5000,
         "price": 13495,
         "stroke": 2.68,
         "wheelbase": 88.6
        },
        {
         "aspiration": "std",
         "boreratio": 3.47,
         "brand": "alfa-romero",
         "carbody": "convertible",
         "carheight": 48.8,
         "carlength": 168.8,
         "carwidth": 64.1,
         "citympg": 21,
         "compressionratio": 9,
         "curbweight": 2548,
         "cylindernumber": "four",
         "doornumber": "two",
         "drivewheel": "rwd",
         "enginelocation": "front",
         "enginesize": 130,
         "enginetype": "dohc",
         "fuelsystem": "mpfi",
         "fueltype": "gas",
         "highwaympg": 27,
         "horsepower": 111,
         "peakrpm": 5000,
         "price": 16500,
         "stroke": 2.68,
         "wheelbase": 88.6
        },
        {
         "aspiration": "std",
         "boreratio": 2.68,
         "brand": "alfa-romero",
         "carbody": "hatchback",
         "carheight": 52.4,
         "carlength": 171.2,
         "carwidth": 65.5,
         "citympg": 19,
         "compressionratio": 9,
         "curbweight": 2823,
         "cylindernumber": "six",
         "doornumber": "two",
         "drivewheel": "rwd",
         "enginelocation": "front",
         "enginesize": 152,
         "enginetype": "ohcv",
         "fuelsystem": "mpfi",
         "fueltype": "gas",
         "highwaympg": 26,
         "horsepower": 154,
         "peakrpm": 5000,
         "price": 16500,
         "stroke": 3.47,
         "wheelbase": 94.5
        }
       ],
       "description": null,
       "dialect": null,
       "encoding": null,
       "format": null,
       "hash": null,
       "homepage": null,
       "licenses": null,
       "mediatype": null,
       "path": null,
       "schema": {
        "fields": [
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "brand",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "fueltype",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "aspiration",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "doornumber",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "carbody",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "drivewheel",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "enginelocation",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "wheelbase",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "carlength",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "carwidth",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "carheight",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "curbweight",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "enginetype",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "cylindernumber",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "enginesize",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "fuelsystem",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "boreratio",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "stroke",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "compressionratio",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "horsepower",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "peakrpm",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "citympg",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "highwaympg",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "price",
          "rdfType": null,
          "title": null,
          "type": "number"
         }
        ],
        "foreignKeys": null,
        "missingValues": null,
        "primaryKey": null
       },
       "sources": null,
       "title": null
      },
      "text/html": [
       "<table><thead><tr><th>brand</th><th>fueltype</th><th>aspiration</th><th>doornumber</th><th>carbody</th><th>drivewheel</th><th>enginelocation</th><th>wheelbase</th><th>carlength</th><th>carwidth</th><th>carheight</th><th>curbweight</th><th>enginetype</th><th>cylindernumber</th><th>enginesize</th><th>fuelsystem</th><th>boreratio</th><th>stroke</th><th>compressionratio</th><th>horsepower</th><th>peakrpm</th><th>citympg</th><th>highwaympg</th><th>price</th></tr></thead><tbody><tr><td>alfa-romero</td><td>gas</td><td>std</td><td>two</td><td>convertible</td><td>rwd</td><td>front</td><td>88.6</td><td>168.8</td><td>64.1</td><td>48.8</td><td>2548</td><td>dohc</td><td>four</td><td>130</td><td>mpfi</td><td>3.47</td><td>2.68</td><td>9</td><td>111</td><td>5000</td><td>21</td><td>27</td><td>13495</td></tr><tr><td>alfa-romero</td><td>gas</td><td>std</td><td>two</td><td>convertible</td><td>rwd</td><td>front</td><td>88.6</td><td>168.8</td><td>64.1</td><td>48.8</td><td>2548</td><td>dohc</td><td>four</td><td>130</td><td>mpfi</td><td>3.47</td><td>2.68</td><td>9</td><td>111</td><td>5000</td><td>21</td><td>27</td><td>16500</td></tr><tr><td>alfa-romero</td><td>gas</td><td>std</td><td>two</td><td>hatchback</td><td>rwd</td><td>front</td><td>94.5</td><td>171.2</td><td>65.5</td><td>52.4</td><td>2823</td><td>ohcv</td><td>six</td><td>152</td><td>mpfi</td><td>2.68</td><td>3.47</td><td>9</td><td>154</td><td>5000</td><td>19</td><td>26</td><td>16500</td></tr></tbody></table>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.drop(\"car_ID\", \"symboling\", \"CarName\");\n",
    "df.head(3);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f827b40-99b5-445d-b339-2bb55b9d6686",
   "metadata": {},
   "source": [
    "- Use df.groupBy(...cols) in order to get aggregates form the dataset:\n",
    "\n",
    "Count cars by brand:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7512602c-6ce4-4167-9f74-78bc5e8c0332",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.dataresource+json": {
       "bytes": null,
       "data": [
        {
         "brand": "mercury",
         "brand_count": 1
        },
        {
         "brand": "renault",
         "brand_count": 2
        },
        {
         "brand": "jaguar",
         "brand_count": 3
        },
        {
         "brand": "chevrolet",
         "brand_count": 3
        },
        {
         "brand": "alfa-romero",
         "brand_count": 3
        },
        {
         "brand": "isuzu",
         "brand_count": 4
        },
        {
         "brand": "porsche",
         "brand_count": 5
        },
        {
         "brand": "saab",
         "brand_count": 6
        },
        {
         "brand": "plymouth",
         "brand_count": 7
        },
        {
         "brand": "audi",
         "brand_count": 7
        },
        {
         "brand": "buick",
         "brand_count": 8
        },
        {
         "brand": "bmw",
         "brand_count": 8
        },
        {
         "brand": "dodge",
         "brand_count": 9
        },
        {
         "brand": "peugeot",
         "brand_count": 11
        },
        {
         "brand": "volvo",
         "brand_count": 11
        },
        {
         "brand": "subaru",
         "brand_count": 12
        },
        {
         "brand": "volkswagen",
         "brand_count": 12
        },
        {
         "brand": "mitsubishi",
         "brand_count": 13
        },
        {
         "brand": "honda",
         "brand_count": 13
        },
        {
         "brand": "mazda",
         "brand_count": 17
        },
        {
         "brand": "nissan",
         "brand_count": 18
        },
        {
         "brand": "toyota",
         "brand_count": 32
        }
       ],
       "description": null,
       "dialect": null,
       "encoding": null,
       "format": null,
       "hash": null,
       "homepage": null,
       "licenses": null,
       "mediatype": null,
       "path": null,
       "schema": {
        "fields": [
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "brand",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "brand_count",
          "rdfType": null,
          "title": null,
          "type": "integer"
         }
        ],
        "foreignKeys": null,
        "missingValues": null,
        "primaryKey": null
       },
       "sources": null,
       "title": null
      },
      "text/html": [
       "<table><thead><tr><th>brand</th><th>brand_count</th></tr></thead><tbody><tr><td>mercury</td><td>1</td></tr><tr><td>renault</td><td>2</td></tr><tr><td>jaguar</td><td>3</td></tr><tr><td>chevrolet</td><td>3</td></tr><tr><td>alfa-romero</td><td>3</td></tr><tr><td>isuzu</td><td>4</td></tr><tr><td>porsche</td><td>5</td></tr><tr><td>saab</td><td>6</td></tr><tr><td>plymouth</td><td>7</td></tr><tr><td>audi</td><td>7</td></tr><tr><td>buick</td><td>8</td></tr><tr><td>bmw</td><td>8</td></tr><tr><td>dodge</td><td>9</td></tr><tr><td>peugeot</td><td>11</td></tr><tr><td>volvo</td><td>11</td></tr><tr><td>subaru</td><td>12</td></tr><tr><td>volkswagen</td><td>12</td></tr><tr><td>mitsubishi</td><td>13</td></tr><tr><td>honda</td><td>13</td></tr><tr><td>mazda</td><td>17</td></tr><tr><td>nissan</td><td>18</td></tr><tr><td>toyota</td><td>32</td></tr></tbody></table>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "let brandCount = df.groupBy(\"brand\").len().sort(\"brand_count\");\n",
    "brandCount;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "44b5baf0-cbd2-48d7-a28b-e871682bbaf0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.dataresource+json": {
       "bytes": null,
       "data": [
        {
         "brand": null,
         "describe": "mean",
         "price": 15079.877371916704
        },
        {
         "brand": null,
         "describe": "std",
         "price": 8738.804703803093
        },
        {
         "brand": "alfa-romero",
         "describe": "min",
         "price": 6007
        },
        {
         "brand": "volvo",
         "describe": "max",
         "price": 34600
        },
        {
         "brand": null,
         "describe": "median",
         "price": 10534.274509803921
        }
       ],
       "description": null,
       "dialect": null,
       "encoding": null,
       "format": null,
       "hash": null,
       "homepage": null,
       "licenses": null,
       "mediatype": null,
       "path": null,
       "schema": {
        "fields": [
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "describe",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "brand",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "price",
          "rdfType": null,
          "title": null,
          "type": "number"
         }
        ],
        "foreignKeys": null,
        "missingValues": null,
        "primaryKey": null
       },
       "sources": null,
       "title": null
      },
      "text/html": [
       "<table><thead><tr><th>describe</th><th>brand</th><th>price</th></tr></thead><tbody><tr><td>mean</td><td>null</td><td>15079.877371916704</td></tr><tr><td>std</td><td>null</td><td>8738.804703803093</td></tr><tr><td>min</td><td>alfa-romero</td><td>6007</td></tr><tr><td>max</td><td>volvo</td><td>34600</td></tr><tr><td>median</td><td>null</td><td>10534.274509803921</td></tr></tbody></table>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "let avgPricePerBrand = df.groupBy(\"brand\").agg({ \"price\": [\"mean\"] });\n",
    "avgPricePerBrand.describe();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "98a18f05-665f-4143-b5d5-6ecda68ca2df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "  porsche: \u001b[33m31400.5\u001b[39m,\n",
       "  volkswagen: \u001b[33m10077.5\u001b[39m,\n",
       "  buick: \u001b[33m33647\u001b[39m,\n",
       "  plymouth: \u001b[33m7963.428571428572\u001b[39m,\n",
       "  peugeot: \u001b[33m15489.09090909091\u001b[39m,\n",
       "  renault: \u001b[33m9595\u001b[39m,\n",
       "  volvo: \u001b[33m18063.18181818182\u001b[39m,\n",
       "  jaguar: \u001b[33m34600\u001b[39m,\n",
       "  mitsubishi: \u001b[33m9239.76923076923\u001b[39m,\n",
       "  saab: \u001b[33m15223.333333333334\u001b[39m,\n",
       "  honda: \u001b[33m8184.692307692308\u001b[39m,\n",
       "  audi: \u001b[33m17859.166714285715\u001b[39m,\n",
       "  mercury: \u001b[33m16503\u001b[39m,\n",
       "  chevrolet: \u001b[33m6007\u001b[39m,\n",
       "  dodge: \u001b[33m7875.444444444444\u001b[39m,\n",
       "  subaru: \u001b[33m8541.25\u001b[39m,\n",
       "  nissan: \u001b[33m10415.666666666666\u001b[39m,\n",
       "  isuzu: \u001b[33m8916.5\u001b[39m,\n",
       "  \u001b[32m\"alfa-romero\"\u001b[39m: \u001b[33m15498.333333333334\u001b[39m,\n",
       "  mazda: \u001b[33m10652.882352941177\u001b[39m,\n",
       "  toyota: \u001b[33m9885.8125\u001b[39m,\n",
       "  bmw: \u001b[33m26118.75\u001b[39m\n",
       "}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "// map brand name to price\n",
    "avgPricePerBrand = avgPricePerBrand\n",
    "  .toRecords()\n",
    "  .reduce((acc, rec) => ({ ...acc, [rec.brand]: rec.price }), {});"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9e7dd2b4-4356-41ad-bd1e-20b7938f55f0",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.dataresource+json": {
       "bytes": null,
       "data": [
        {
         "brand_category": "Mid_Range"
        },
        {
         "brand_category": "Mid_Range"
        },
        {
         "brand_category": "Luxury"
        },
        {
         "brand_category": "Luxury"
        },
        {
         "brand_category": "Mid_Range"
        }
       ],
       "description": null,
       "dialect": null,
       "encoding": null,
       "format": null,
       "hash": null,
       "homepage": null,
       "licenses": null,
       "mediatype": null,
       "path": null,
       "schema": {
        "fields": [
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "brand_category",
          "rdfType": null,
          "title": null,
          "type": "string"
         }
        ],
        "foreignKeys": null,
        "missingValues": null,
        "primaryKey": null
       },
       "sources": null,
       "title": null
      },
      "text/html": [
       "<table><thead><tr><th>brand_category</th></tr></thead><tbody><tr><td>Mid_Range</td></tr><tr><td>Mid_Range</td></tr><tr><td>Luxury</td></tr><tr><td>Luxury</td></tr><tr><td>Mid_Range</td></tr></tbody></table>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "// create brand categories by budget\n",
    "let brandCategory = df.brand.toArray().map((brand) => {\n",
    "  const avgPrice = avgPricePerBrand[brand];\n",
    "  return avgPrice < 10000\n",
    "    ? \"Budget\"\n",
    "    : avgPrice > 20000\n",
    "    ? \"Luxury\"\n",
    "    : \"Mid_Range\";\n",
    "});\n",
    "let catDf = pl.DataFrame({\n",
    "  \"brand_category\": brandCategory,\n",
    "});\n",
    "\n",
    "catDf.sample(5);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14930aef-1fc0-4baf-92bf-8a093c3d2e86",
   "metadata": {},
   "source": [
    "Write the cleaned dataset to a new file:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "350c57c7-ec9e-400d-a8e7-19301c720c56",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = catDf.hstack(df);\n",
    "df.writeCSV(\"assets/cleaned_car_prices.csv\");"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ea22d38-cb20-47a0-8f92-422bc5925abf",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "source": [
    "## Exploratory Data Analysis\n",
    "\n",
    "For plotting we use [@observable/plot](https://observablehq.com/plot) and\n",
    "configured shotcuts for jupyter notebooks imported from\n",
    "[l12.xyz/x/shortcuts](https://l12.xyz/x/shortcuts)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2fed060e-1785-4dc1-ba44-4bcd5cee92f6",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\n",
      "  { brand: \"mercury\", brand_count: 1 },\n",
      "  { brand: \"renault\", brand_count: 2 },\n",
      "  { brand: \"jaguar\", brand_count: 3 },\n",
      "  { brand: \"chevrolet\", brand_count: 3 },\n",
      "  { brand: \"alfa-romero\", brand_count: 3 },\n",
      "  { brand: \"isuzu\", brand_count: 4 },\n",
      "  { brand: \"porsche\", brand_count: 5 },\n",
      "  { brand: \"saab\", brand_count: 6 },\n",
      "  { brand: \"plymouth\", brand_count: 7 },\n",
      "  { brand: \"audi\", brand_count: 7 },\n",
      "  { brand: \"buick\", brand_count: 8 },\n",
      "  { brand: \"bmw\", brand_count: 8 },\n",
      "  { brand: \"dodge\", brand_count: 9 },\n",
      "  { brand: \"peugeot\", brand_count: 11 },\n",
      "  { brand: \"volvo\", brand_count: 11 },\n",
      "  { brand: \"subaru\", brand_count: 12 },\n",
      "  { brand: \"volkswagen\", brand_count: 12 },\n",
      "  { brand: \"mitsubishi\", brand_count: 13 },\n",
      "  { brand: \"honda\", brand_count: 13 },\n",
      "  { brand: \"mazda\", brand_count: 17 },\n",
      "  { brand: \"nissan\", brand_count: 18 },\n",
      "  { brand: \"toyota\", brand_count: 32 }\n",
      "]\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<svg style=\"background-color:#fff\" viewBox=\"0 0 640 500\" height=\"500\" width=\"640\" text-anchor=\"middle\" font-size=\"10\" font-family=\"system-ui, sans-serif\" fill=\"currentColor\" class=\"plot-d6a7b5\"><style>:where(.plot-d6a7b5) {\n",
       "  --plot-background: white;\n",
       "  display: block;\n",
       "  height: auto;\n",
       "  height: intrinsic;\n",
       "  max-width: 100%;\n",
       "}\n",
       ":where(.plot-d6a7b5 text),\n",
       ":where(.plot-d6a7b5 tspan) {\n",
       "  white-space: pre;\n",
       "}</style><g transform=\"translate(0,9)\" stroke=\"currentColor\" fill=\"none\" aria-hidden=\"true\" aria-label=\"y-axis tick\"><path d=\"M0,0L-6,0\" transform=\"translate(80,26)\" /><path d=\"M0,0L-6,0\" transform=\"translate(80,46)\" /><path d=\"M0,0L-6,0\" transform=\"translate(80,66)\" /><path d=\"M0,0L-6,0\" transform=\"translate(80,86)\" /><path d=\"M0,0L-6,0\" transform=\"translate(80,106)\" /><path d=\"M0,0L-6,0\" transform=\"translate(80,126)\" /><path d=\"M0,0L-6,0\" transform=\"translate(80,146)\" /><path d=\"M0,0L-6,0\" transform=\"translate(80,166)\" /><path d=\"M0,0L-6,0\" transform=\"translate(80,186)\" /><path d=\"M0,0L-6,0\" transform=\"translate(80,206)\" /><path d=\"M0,0L-6,0\" transform=\"translate(80,226)\" /><path d=\"M0,0L-6,0\" transform=\"translate(80,246)\" /><path d=\"M0,0L-6,0\" transform=\"translate(80,266)\" /><path d=\"M0,0L-6,0\" transform=\"translate(80,286)\" /><path d=\"M0,0L-6,0\" transform=\"translate(80,306)\" /><path d=\"M0,0L-6,0\" transform=\"translate(80,326)\" /><path d=\"M0,0L-6,0\" transform=\"translate(80,346)\" /><path d=\"M0,0L-6,0\" transform=\"translate(80,366)\" /><path d=\"M0,0L-6,0\" transform=\"translate(80,386)\" /><path d=\"M0,0L-6,0\" transform=\"translate(80,406)\" /><path d=\"M0,0L-6,0\" transform=\"translate(80,426)\" /><path d=\"M0,0L-6,0\" transform=\"translate(80,446)\" /></g><g transform=\"translate(-9,9)\" text-anchor=\"end\" aria-label=\"y-axis tick label\"><text transform=\"translate(80,26)\" y=\"0.32em\">toyota</text><text transform=\"translate(80,46)\" y=\"0.32em\">nissan</text><text transform=\"translate(80,66)\" y=\"0.32em\">mazda</text><text transform=\"translate(80,86)\" y=\"0.32em\">honda</text><text transform=\"translate(80,106)\" y=\"0.32em\">mitsubishi</text><text transform=\"translate(80,126)\" y=\"0.32em\">subaru</text><text transform=\"translate(80,146)\" y=\"0.32em\">volkswagen</text><text transform=\"translate(80,166)\" y=\"0.32em\">peugeot</text><text transform=\"translate(80,186)\" y=\"0.32em\">volvo</text><text transform=\"translate(80,206)\" y=\"0.32em\">dodge</text><text transform=\"translate(80,226)\" y=\"0.32em\">bmw</text><text transform=\"translate(80,246)\" y=\"0.32em\">buick</text><text transform=\"translate(80,266)\" y=\"0.32em\">audi</text><text transform=\"translate(80,286)\" y=\"0.32em\">plymouth</text><text transform=\"translate(80,306)\" y=\"0.32em\">saab</text><text transform=\"translate(80,326)\" y=\"0.32em\">porsche</text><text transform=\"translate(80,346)\" y=\"0.32em\">isuzu</text><text transform=\"translate(80,366)\" y=\"0.32em\">alfa-romero</text><text transform=\"translate(80,386)\" y=\"0.32em\">chevrolet</text><text transform=\"translate(80,406)\" y=\"0.32em\">jaguar</text><text transform=\"translate(80,426)\" y=\"0.32em\">renault</text><text transform=\"translate(80,446)\" y=\"0.32em\">mercury</text></g><g transform=\"translate(-77,0)\" aria-label=\"y-axis label\"><text transform=\"translate(80,245) rotate(-90)\" y=\"0.71em\">brand</text></g><g stroke=\"currentColor\" fill=\"none\" aria-hidden=\"true\" aria-label=\"x-axis tick\"><path d=\"M0,0L0,6\" transform=\"translate(80,470)\" /><path d=\"M0,0L0,6\" transform=\"translate(164.375,470)\" /><path d=\"M0,0L0,6\" transform=\"translate(248.75,470)\" /><path d=\"M0,0L0,6\" transform=\"translate(333.125,470)\" /><path d=\"M0,0L0,6\" transform=\"translate(417.5,470)\" /><path d=\"M0,0L0,6\" transform=\"translate(501.875,470)\" /><path d=\"M0,0L0,6\" transform=\"translate(586.25,470)\" /></g><g transform=\"translate(0,9)\" font-variant=\"tabular-nums\" aria-label=\"x-axis tick label\"><text transform=\"translate(80,470)\" y=\"0.71em\">0</text><text transform=\"translate(164.375,470)\" y=\"0.71em\">5</text><text transform=\"translate(248.75,470)\" y=\"0.71em\">10</text><text transform=\"translate(333.125,470)\" y=\"0.71em\">15</text><text transform=\"translate(417.5,470)\" y=\"0.71em\">20</text><text transform=\"translate(501.875,470)\" y=\"0.71em\">25</text><text transform=\"translate(586.25,470)\" y=\"0.71em\">30</text></g><g transform=\"translate(17,27)\" text-anchor=\"end\" aria-label=\"x-axis label\"><text transform=\"translate(620,470)\">brand_count →</text></g><g aria-label=\"bar\"><rect height=\"18\" y=\"446\" width=\"16.875\" x=\"80\" /><rect height=\"18\" y=\"426\" width=\"33.75\" x=\"80\" /><rect height=\"18\" y=\"406\" width=\"50.625\" x=\"80\" /><rect height=\"18\" y=\"386\" width=\"50.625\" x=\"80\" /><rect height=\"18\" y=\"366\" width=\"50.625\" x=\"80\" /><rect height=\"18\" y=\"346\" width=\"67.5\" x=\"80\" /><rect height=\"18\" y=\"326\" width=\"84.375\" x=\"80\" /><rect height=\"18\" y=\"306\" width=\"101.25\" x=\"80\" /><rect height=\"18\" y=\"286\" width=\"118.125\" x=\"80\" /><rect height=\"18\" y=\"266\" width=\"118.125\" x=\"80\" /><rect height=\"18\" y=\"246\" width=\"135\" x=\"80\" /><rect height=\"18\" y=\"226\" width=\"135\" x=\"80\" /><rect height=\"18\" y=\"206\" width=\"151.875\" x=\"80\" /><rect height=\"18\" y=\"166\" width=\"185.625\" x=\"80\" /><rect height=\"18\" y=\"186\" width=\"185.625\" x=\"80\" /><rect height=\"18\" y=\"126\" width=\"202.5\" x=\"80\" /><rect height=\"18\" y=\"146\" width=\"202.5\" x=\"80\" /><rect height=\"18\" y=\"106\" width=\"219.375\" x=\"80\" /><rect height=\"18\" y=\"86\" width=\"219.375\" x=\"80\" /><rect height=\"18\" y=\"66\" width=\"286.875\" x=\"80\" /><rect height=\"18\" y=\"46\" width=\"303.75\" x=\"80\" /><rect height=\"18\" y=\"26\" width=\"540\" x=\"80\" /></g></svg>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import { document, Plot } from \"https://l12.xyz/x/shortcuts/raw/plots.ts\";\n",
    "\n",
    "const brandCountRecords = brandCount.toRecords();\n",
    "console.log(brandCountRecords);\n",
    "const brandCountPlot = Plot.plot({\n",
    "  marginLeft: 80,\n",
    "  style: {\n",
    "    backgroundColor: \"#fff\",\n",
    "  },\n",
    "  x: { padding: 0.4 },\n",
    "  marks: [\n",
    "    Plot.barX(brandCountRecords, {\n",
    "      x: \"brand_count\",\n",
    "      y: \"brand\",\n",
    "      sort: { y: \"x\", order: \"descending\" },\n",
    "    }),\n",
    "  ],\n",
    "  document,\n",
    "});\n",
    "await display(brandCountPlot);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e21f5bdf-a5a8-43b2-855b-0eb079656da7",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\n",
       "  \u001b[32m\"wheelbase\"\u001b[39m,        \u001b[32m\"carlength\"\u001b[39m,\n",
       "  \u001b[32m\"carwidth\"\u001b[39m,         \u001b[32m\"carheight\"\u001b[39m,\n",
       "  \u001b[32m\"curbweight\"\u001b[39m,       \u001b[32m\"enginesize\"\u001b[39m,\n",
       "  \u001b[32m\"boreratio\"\u001b[39m,        \u001b[32m\"stroke\"\u001b[39m,\n",
       "  \u001b[32m\"compressionratio\"\u001b[39m, \u001b[32m\"horsepower\"\u001b[39m,\n",
       "  \u001b[32m\"peakrpm\"\u001b[39m,          \u001b[32m\"citympg\"\u001b[39m,\n",
       "  \u001b[32m\"highwaympg\"\u001b[39m\n",
       "]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "let numericColumns = df.columns.filter((col) =>\n",
    "  df[col].isNumeric() && col !== \"price\"\n",
    ");\n",
    "numericColumns;"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c8d3b90-b4b1-4da6-9e7c-f8fd7cffb77d",
   "metadata": {},
   "source": [
    "Sometimes we can get some intuitive insight seeing the data plotted from\n",
    "different dimensions. It is an optional step, but it might help to get some\n",
    "assumtions about the relationships in the dataset. Below is an example for\n",
    "drawing plots side-by-side."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9a82f623-b39a-473a-91b9-0a0347c64c63",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "          <section style=\"display:grid;grid-template-columns: repeat(3, 1fr);\">\n",
       "              <img title=\"wheelbase / price\" src='data:image/svg+xml;base64,[encoded data, 27820 characters]'><img title=\"carlength / price\" src='data:image/svg+xml;base64,[encoded data, 27448 characters]'><img title=\"carwidth / price\" src='data:image/svg+xml;base64,[encoded data, 26752 characters]'><img title=\"carheight / price\" src='data:image/svg+xml;base64,[encoded data, 26808 characters]'><img title=\"curbweight / price\" src='data:image/svg+xml;base64,[encoded data, 27500 characters]'><img title=\"enginesize / price\" src='data:image/svg+xml;base64,[encoded data, 27268 characters]'><img title=\"boreratio / price\" src='data:image/svg+xml;base64,[encoded data, 27488 characters]'><img title=\"stroke / price\" src='data:image/svg+xml;base64,[encoded data, 28532 characters]'><img title=\"compressionratio / price\" src='data:image/svg+xml;base64,[encoded data, 25100 characters]'><img title=\"horsepower / price\" src='data:image/svg+xml;base64,[encoded data, 23556 characters]'><img title=\"peakrpm / price\" src='data:image/svg+xml;base64,[encoded data, 27108 characters]'><img title=\"citympg / price\" src='data:image/svg+xml;base64,[encoded data, 23872 characters]'><img title=\"highwaympg / price\" src='data:image/svg+xml;base64,[encoded data, 27704 characters]'>\n",
       "          </section>\n",
       "    "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import { sideBySidePlot } from \"https://l12.xyz/x/shortcuts/raw/plots.ts\";\n",
    "\n",
    "let records = df.toRecords();\n",
    "\n",
    "const plt = sideBySidePlot({\n",
    "  x: numericColumns,\n",
    "  y: [\"price\"],\n",
    "  marks: [\n",
    "    (x, y) => Plot.dot(records, { x, y }),\n",
    "    (x, y) => Plot.linearRegressionY(records, { x, y, stroke: \"red\" }),\n",
    "  ],\n",
    "  cols: 3,\n",
    "});\n",
    "\n",
    "await display(\n",
    "  plt,\n",
    ");"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53d04850-df05-474c-8568-a5f7c97146f3",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "source": [
    "Let's view the list of top features that have high correlation coefficient. The\n",
    "pearsonCorr() function calculates the Pearson'r correlation coefficients with\n",
    "respect to the 'price'."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "924838be-1b4d-4edf-abe5-8bf02e02cf11",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.dataresource+json": {
       "bytes": null,
       "data": [
        {
         "Variable": "enginesize",
         "idx (price)": 0.8741448025245117
        },
        {
         "Variable": "curbweight",
         "idx (price)": 0.8353048793372955
        },
        {
         "Variable": "horsepower",
         "idx (price)": 0.8081388225362217
        },
        {
         "Variable": "carwidth",
         "idx (price)": 0.7593252997414263
        },
        {
         "Variable": "carlength",
         "idx (price)": 0.6829200156779843
        },
        {
         "Variable": "wheelbase",
         "idx (price)": 0.5778155982921477
        },
        {
         "Variable": "boreratio",
         "idx (price)": 0.5531732367984261
        },
        {
         "Variable": "carheight",
         "idx (price)": 0.11933622657047727
        },
        {
         "Variable": "stroke",
         "idx (price)": 0.07944308388192935
        },
        {
         "Variable": "compressionratio",
         "idx (price)": 0.06798350579944248
        },
        {
         "Variable": "peakrpm",
         "idx (price)": -0.0852671502778569
        },
        {
         "Variable": "citympg",
         "idx (price)": -0.6857513360270401
        },
        {
         "Variable": "highwaympg",
         "idx (price)": -0.6975990916465564
        }
       ],
       "description": null,
       "dialect": null,
       "encoding": null,
       "format": null,
       "hash": null,
       "homepage": null,
       "licenses": null,
       "mediatype": null,
       "path": null,
       "schema": {
        "fields": [
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "Variable",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "idx (price)",
          "rdfType": null,
          "title": null,
          "type": "number"
         }
        ],
        "foreignKeys": null,
        "missingValues": null,
        "primaryKey": null
       },
       "sources": null,
       "title": null
      },
      "text/html": [
       "<table><thead><tr><th>Variable</th><th>idx (price)</th></tr></thead><tbody><tr><td>enginesize</td><td>0.8741448025245117</td></tr><tr><td>curbweight</td><td>0.8353048793372955</td></tr><tr><td>horsepower</td><td>0.8081388225362217</td></tr><tr><td>carwidth</td><td>0.7593252997414263</td></tr><tr><td>carlength</td><td>0.6829200156779843</td></tr><tr><td>wheelbase</td><td>0.5778155982921477</td></tr><tr><td>boreratio</td><td>0.5531732367984261</td></tr><tr><td>carheight</td><td>0.11933622657047727</td></tr><tr><td>stroke</td><td>0.07944308388192935</td></tr><tr><td>compressionratio</td><td>0.06798350579944248</td></tr><tr><td>peakrpm</td><td>-0.0852671502778569</td></tr><tr><td>citympg</td><td>-0.6857513360270401</td></tr><tr><td>highwaympg</td><td>-0.6975990916465564</td></tr></tbody></table>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "// we select pearon's idx from dataframe for all numeric cols,\n",
    "// then we transpose result so that columns become rows,\n",
    "// then we sort by the idx column\n",
    "df.select(\n",
    "  ...numericColumns.map((col) => pl.pearsonCorr(col, \"price\")),\n",
    ")\n",
    "  .transpose({\n",
    "    columnNames: [\"idx (price)\"],\n",
    "    headerName: \"Variable\",\n",
    "    includeHeader: true,\n",
    "  })\n",
    "  .sort(\"idx (price)\", true);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63bc6b92-5ab9-43e0-a957-57d895bc60f9",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "source": [
    "### Linearity Assumption\n",
    "\n",
    "Linear regression needs the relationship between independent variable and the\n",
    "dependent variable to be linear. We can test this assumption with some scatter\n",
    "plots and regression lines.\n",
    "\n",
    "** Here we use the same side-by-side plot shortcut, but for selected varisbles\n",
    "with high correlation coefficent."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2d5a635a-c819-4989-9818-a1927363993d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "          <section style=\"display:grid;grid-template-columns: repeat(2, 1fr);\">\n",
       "              <img title=\"enginesize / price\" src='data:image/svg+xml;base64,[encoded data, 27268 characters]'><img title=\"curbweight / price\" src='data:image/svg+xml;base64,[encoded data, 27500 characters]'><img title=\"horsepower / price\" src='data:image/svg+xml;base64,[encoded data, 23556 characters]'><img title=\"carwidth / price\" src='data:image/svg+xml;base64,[encoded data, 26752 characters]'>\n",
       "          </section>\n",
       "    "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "const plt = sideBySidePlot({\n",
    "  x: [\"enginesize\", \"curbweight\", \"horsepower\", \"carwidth\"],\n",
    "  y: [\"price\"],\n",
    "  marks: [\n",
    "    (x, y) => Plot.dot(records, { x, y }),\n",
    "    (x, y) => Plot.linearRegressionY(records, { x, y, stroke: \"red\" }),\n",
    "  ],\n",
    "  cols: 2,\n",
    "});\n",
    "\n",
    "await display(\n",
    "  plt,\n",
    ");"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a351f17-309b-444b-b4a8-26500b077e47",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "source": [
    "### Homoscedasticity\n",
    "\n",
    "The assumption of homoscedasticity (constant variance), is crucial to linear\n",
    "regression models. Homoscedasticity describes a situation in which the error\n",
    "term or variance or the \"noise\" or random disturbance in the relationship\n",
    "between the independent variables and the dependent variable is the same across\n",
    "all values of the independent variable. In other words, there is a constant\n",
    "variance present in the response variable as the predictor variable increases.\n",
    "If the \"noise\" is not the same across the values of an independent variable, we\n",
    "call it heteroscedasticity, opposite of homoscedasticity.\n",
    "\n",
    "#### Residuals\n",
    "\n",
    "Next we apply residual expression to 'price' and 'enginesize' varibles in order\n",
    "to check this assumption.\n",
    "[The residuals function](https://l12.xyz/x/shortcuts/src/branch/main/expr.ts)\n",
    "uses mean squared."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "fc9a31c3-6ac0-4058-b5bb-341eb1c40dd1",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<svg viewBox=\"0 0 640 400\" height=\"400\" width=\"640\" text-anchor=\"middle\" font-size=\"10\" font-family=\"system-ui, sans-serif\" fill=\"currentColor\" class=\"plot-d6a7b5\"><style>:where(.plot-d6a7b5) {\n",
       "  --plot-background: white;\n",
       "  display: block;\n",
       "  height: auto;\n",
       "  height: intrinsic;\n",
       "  max-width: 100%;\n",
       "}\n",
       ":where(.plot-d6a7b5 text),\n",
       ":where(.plot-d6a7b5 tspan) {\n",
       "  white-space: pre;\n",
       "}</style><g stroke=\"currentColor\" fill=\"none\" aria-hidden=\"true\" aria-label=\"y-axis tick\"><path d=\"M0,0L-6,0\" transform=\"translate(40,360.676690955552)\" /><path d=\"M0,0L-6,0\" transform=\"translate(40,332.6045058220467)\" /><path d=\"M0,0L-6,0\" transform=\"translate(40,304.5323206885414)\" /><path d=\"M0,0L-6,0\" transform=\"translate(40,276.4601355550362)\" /><path d=\"M0,0L-6,0\" transform=\"translate(40,248.3879504215309)\" /><path d=\"M0,0L-6,0\" transform=\"translate(40,220.31576528802563)\" /><path d=\"M0,0L-6,0\" transform=\"translate(40,192.24358015452034)\" /><path d=\"M0,0L-6,0\" transform=\"translate(40,164.17139502101506)\" /><path d=\"M0,0L-6,0\" transform=\"translate(40,136.0992098875098)\" /><path d=\"M0,0L-6,0\" transform=\"translate(40,108.02702475400451)\" /><path d=\"M0,0L-6,0\" transform=\"translate(40,79.95483962049926)\" /><path d=\"M0,0L-6,0\" transform=\"translate(40,51.882654486993985)\" /><path d=\"M0,0L-6,0\" transform=\"translate(40,23.810469353488706)\" /></g><g transform=\"translate(-9,0)\" font-variant=\"tabular-nums\" text-anchor=\"end\" aria-label=\"y-axis tick label\"><text transform=\"translate(40,360.676690955552)\" y=\"0.32em\">−10,000</text><text transform=\"translate(40,332.6045058220467)\" y=\"0.32em\">−8,000</text><text transform=\"translate(40,304.5323206885414)\" y=\"0.32em\">−6,000</text><text transform=\"translate(40,276.4601355550362)\" y=\"0.32em\">−4,000</text><text transform=\"translate(40,248.3879504215309)\" y=\"0.32em\">−2,000</text><text transform=\"translate(40,220.31576528802563)\" y=\"0.32em\">0</text><text transform=\"translate(40,192.24358015452034)\" y=\"0.32em\">2,000</text><text transform=\"translate(40,164.17139502101506)\" y=\"0.32em\">4,000</text><text transform=\"translate(40,136.0992098875098)\" y=\"0.32em\">6,000</text><text transform=\"translate(40,108.02702475400451)\" y=\"0.32em\">8,000</text><text transform=\"translate(40,79.95483962049926)\" y=\"0.32em\">10,000</text><text transform=\"translate(40,51.882654486993985)\" y=\"0.32em\">12,000</text><text transform=\"translate(40,23.810469353488706)\" y=\"0.32em\">14,000</text></g><g transform=\"translate(-37,-17)\" text-anchor=\"start\" aria-label=\"y-axis label\"><text transform=\"translate(40,20)\" y=\"0.71em\">↑ price</text></g><g stroke=\"currentColor\" fill=\"none\" aria-hidden=\"true\" aria-label=\"x-axis tick\"><path d=\"M0,0L0,6\" transform=\"translate(125.35849056603774,370)\" /><path d=\"M0,0L0,6\" transform=\"translate(234.79245283018867,370)\" /><path d=\"M0,0L0,6\" transform=\"translate(344.22641509433964,370)\" /><path d=\"M0,0L0,6\" transform=\"translate(453.6603773584905,370)\" /><path d=\"M0,0L0,6\" transform=\"translate(563.0943396226415,370)\" /></g><g transform=\"translate(0,9)\" font-variant=\"tabular-nums\" aria-label=\"x-axis tick label\"><text transform=\"translate(125.35849056603774,370)\" y=\"0.71em\">100</text><text transform=\"translate(234.79245283018867,370)\" y=\"0.71em\">150</text><text transform=\"translate(344.22641509433964,370)\" y=\"0.71em\">200</text><text transform=\"translate(453.6603773584905,370)\" y=\"0.71em\">250</text><text transform=\"translate(563.0943396226415,370)\" y=\"0.71em\">300</text></g><g transform=\"translate(17,27)\" text-anchor=\"end\" aria-label=\"x-axis label\"><text transform=\"translate(620,370)\">enginesize →</text></g><g stroke-width=\"1.5\" stroke=\"currentColor\" fill=\"none\" aria-label=\"dot\"><circle r=\"3\" cy=\"224.53148603945945\" cx=\"191.0188679245283\" /><circle r=\"3\" cy=\"182.35302787636778\" cx=\"191.0188679245283\" /><circle r=\"3\" cy=\"234.13729878380033\" cx=\"239.16981132075472\" /><circle r=\"3\" cy=\"168.7146235099468\" cx=\"145.0566037735849\" /><circle r=\"3\" cy=\"183.14172291270714\" cx=\"204.1509433962264\" /><circle r=\"3\" cy=\"214.0211265595629\" cx=\"204.1509433962264\" /><circle r=\"3\" cy=\"179.49233884535147\" cx=\"204.1509433962264\" /><circle r=\"3\" cy=\"162.50866683958074\" cx=\"204.1509433962264\" /><circle r=\"3\" cy=\"81.19067569235949\" cx=\"193.20754716981133\" /><circle r=\"3\" cy=\"165.62946454648466\" cx=\"193.20754716981133\" /><circle r=\"3\" cy=\"131.55128344860788\" cx=\"142.86792452830187\" /><circle r=\"3\" cy=\"124.60341762806534\" cx=\"142.86792452830187\" /><circle r=\"3\" cy=\"199.64193095992476\" cx=\"265.433962264151\" /><circle r=\"3\" cy=\"197.74705846341314\" cx=\"265.433962264151\" /><circle r=\"3\" cy=\"149.182178182449\" cx=\"265.433962264151\" /><circle r=\"3\" cy=\"168.15095704207405\" cx=\"363.92452830188677\" /><circle r=\"3\" cy=\"20\" cx=\"363.92452830188677\" /><circle r=\"3\" cy=\"82.25007053354793\" cx=\"363.92452830188677\" /><circle r=\"3\" cy=\"179.23433820676848\" cx=\"40\" /><circle r=\"3\" cy=\"231.4381326883828\" cx=\"103.47169811320755\" /><circle r=\"3\" cy=\"227.50802676969204\" cx=\"103.47169811320755\" /><circle r=\"3\" cy=\"241.58622761414492\" cx=\"103.47169811320755\" /><circle r=\"3\" cy=\"230.2871730979091\" cx=\"103.47169811320755\" /><circle r=\"3\" cy=\"226.940790808779\" cx=\"120.98113207547169\" /><circle r=\"3\" cy=\"232.36451479778844\" cx=\"103.47169811320755\" /><circle r=\"3\" cy=\"225.86580393938198\" cx=\"103.47169811320755\" /><circle r=\"3\" cy=\"212.99470705566978\" cx=\"103.47169811320755\" /><circle r=\"3\" cy=\"218.5050991761607\" cx=\"120.98113207547169\" /><circle r=\"3\" cy=\"269.9019294734469\" cx=\"173.50943396226415\" /><circle r=\"3\" cy=\"293.1842440830073\" cx=\"247.92452830188677\" /><circle r=\"3\" cy=\"233.5631526476851\" cx=\"107.84905660377359\" /><circle r=\"3\" cy=\"228.28558184258608\" cx=\"107.84905660377359\" /><circle r=\"3\" cy=\"218.12233617447683\" cx=\"79.39622641509433\" /><circle r=\"3\" cy=\"232.86134801934747\" cx=\"107.84905660377359\" /><circle r=\"3\" cy=\"224.43969247929584\" cx=\"107.84905660377359\" /><circle r=\"3\" cy=\"222.10970111321492\" cx=\"107.84905660377359\" /><circle r=\"3\" cy=\"222.10970111321492\" cx=\"107.84905660377359\" /><circle r=\"3\" cy=\"256.0569944974264\" cx=\"147.24528301886792\" /><circle r=\"3\" cy=\"239.21368341732324\" cx=\"147.24528301886792\" /><circle r=\"3\" cy=\"242.72270655901139\" cx=\"147.24528301886792\" /><circle r=\"3\" cy=\"222.37037233722006\" cx=\"147.24528301886792\" /><circle r=\"3\" cy=\"185.1747270353256\" cx=\"147.24528301886792\" /><circle r=\"3\" cy=\"221.66856770888245\" cx=\"147.24528301886792\" /><circle r=\"3\" cy=\"273.9908877423142\" cx=\"149.43396226415095\" /><circle r=\"3\" cy=\"194.64251602464074\" cx=\"103.47169811320755\" /><circle r=\"3\" cy=\"194.64251602464074\" cx=\"103.47169811320755\" /><circle r=\"3\" cy=\"232.98566909658686\" cx=\"166.9433962264151\" /><circle r=\"3\" cy=\"262.5748734114398\" cx=\"471.16981132075466\" /><circle r=\"3\" cy=\"216.2557679411561\" cx=\"471.16981132075466\" /><circle r=\"3\" cy=\"370\" cx=\"620\" /><circle r=\"3\" cy=\"249.2316650076031\" cx=\"105.66037735849058\" /><circle r=\"3\" cy=\"236.59918169752572\" cx=\"105.66037735849058\" /><circle r=\"3\" cy=\"226.7739169007989\" cx=\"105.66037735849058\" /><circle r=\"3\" cy=\"228.17752615747415\" cx=\"105.66037735849058\" /><circle r=\"3\" cy=\"218.3522613607473\" cx=\"105.66037735849058\" /><circle r=\"3\" cy=\"119.09369233713522\" cx=\"59.698113207547166\" /><circle r=\"3\" cy=\"106.46120902705786\" cx=\"59.698113207547166\" /><circle r=\"3\" cy=\"81.19624240690311\" cx=\"59.698113207547166\" /><circle r=\"3\" cy=\"76.66236223132172\" cx=\"81.58490566037736\" /><circle r=\"3\" cy=\"270.96867250852006\" cx=\"173.50943396226415\" /><circle r=\"3\" cy=\"275.8813049068835\" cx=\"173.50943396226415\" /><circle r=\"3\" cy=\"246.40551051670298\" cx=\"173.50943396226415\" /><circle r=\"3\" cy=\"251.3181429150664\" cx=\"173.50943396226415\" /><circle r=\"3\" cy=\"243.59829200335244\" cx=\"173.50943396226415\" /><circle r=\"3\" cy=\"237.28205034831376\" cx=\"173.50943396226415\" /><circle r=\"3\" cy=\"180.90708806547198\" cx=\"212.9056603773585\" /><circle r=\"3\" cy=\"165.8857951664455\" cx=\"199.77358490566036\" /><circle r=\"3\" cy=\"180.05133423911963\" cx=\"307.0188679245283\" /><circle r=\"3\" cy=\"142.2100286791545\" cx=\"307.0188679245283\" /><circle r=\"3\" cy=\"143.22062734396067\" cx=\"307.0188679245283\" /><circle r=\"3\" cy=\"95.16104639539968\" cx=\"307.0188679245283\" /><circle r=\"3\" cy=\"178.9371384883228\" cx=\"418.64150943396226\" /><circle r=\"3\" cy=\"166.6976657701145\" cx=\"418.64150943396226\" /><circle r=\"3\" cy=\"258.01203194464387\" cx=\"580.6037735849056\" /><circle r=\"3\" cy=\"186.2764589650926\" cx=\"571.8490566037736\" /><circle r=\"3\" cy=\"205.84922455659142\" cx=\"212.9056603773585\" /><circle r=\"3\" cy=\"248.86249354544546\" cx=\"107.84905660377359\" /><circle r=\"3\" cy=\"237.63361949204335\" cx=\"107.84905660377359\" /><circle r=\"3\" cy=\"230.89629506000207\" cx=\"107.84905660377359\" /><circle r=\"3\" cy=\"230.70246361666875\" cx=\"120.98113207547169\" /><circle r=\"3\" cy=\"227.08649943964895\" cx=\"147.24528301886792\" /><circle r=\"3\" cy=\"275.82516053661647\" cx=\"173.50943396226415\" /><circle r=\"3\" cy=\"297.88633509286944\" cx=\"247.92452830188677\" /><circle r=\"3\" cy=\"266.44548774334345\" cx=\"247.92452830188677\" /><circle r=\"3\" cy=\"271.77920291870953\" cx=\"247.92452830188677\" /><circle r=\"3\" cy=\"297.019660312413\" cx=\"173.50943396226415\" /><circle r=\"3\" cy=\"280.1763492323098\" cx=\"173.50943396226415\" /><circle r=\"3\" cy=\"236.63104238504073\" cx=\"147.24528301886792\" /><circle r=\"3\" cy=\"236.63104238504073\" cx=\"147.24528301886792\" /><circle r=\"3\" cy=\"259.08767584206464\" cx=\"118.79245283018867\" /><circle r=\"3\" cy=\"250.75291071001476\" cx=\"131.92452830188677\" /><circle r=\"3\" cy=\"242.9461693902991\" cx=\"118.79245283018867\" /><circle r=\"3\" cy=\"240.1389508769486\" cx=\"118.79245283018867\" /><circle r=\"3\" cy=\"233.12090459357228\" cx=\"118.79245283018867\" /><circle r=\"3\" cy=\"233.8227092219099\" cx=\"118.79245283018867\" /><circle r=\"3\" cy=\"226.8046629385336\" cx=\"118.79245283018867\" /><circle r=\"3\" cy=\"231.01549070855938\" cx=\"118.79245283018867\" /><circle r=\"3\" cy=\"223.997444425183\" cx=\"118.79245283018867\" /><circle r=\"3\" cy=\"220.4884212834949\" cx=\"118.79245283018867\" /><circle r=\"3\" cy=\"264.801257889993\" cx=\"169.13207547169813\" /><circle r=\"3\" cy=\"256.3796023499415\" cx=\"169.13207547169813\" /><circle r=\"3\" cy=\"344.5206969546044\" cx=\"302.6415094339623\" /><circle r=\"3\" cy=\"331.888213644527\" cx=\"302.6415094339623\" /><circle r=\"3\" cy=\"344.5206969546044\" cx=\"302.6415094339623\" /><circle r=\"3\" cy=\"292.5871544576196\" cx=\"302.6415094339623\" /><circle r=\"3\" cy=\"257.496923040738\" cx=\"302.6415094339623\" /><circle r=\"3\" cy=\"275.7438433775165\" cx=\"302.6415094339623\" /><circle r=\"3\" cy=\"223.38074872550598\" cx=\"169.13207547169813\" /><circle r=\"3\" cy=\"280.4564042540841\" cx=\"239.16981132075472\" /><circle r=\"3\" cy=\"215.80125873945954\" cx=\"169.13207547169813\" /><circle r=\"3\" cy=\"271.1925831600273\" cx=\"239.16981132075472\" /><circle r=\"3\" cy=\"171.7279280798563\" cx=\"169.13207547169813\" /><circle r=\"3\" cy=\"228.5228617570993\" cx=\"239.16981132075472\" /><circle r=\"3\" cy=\"156.07768486792708\" cx=\"169.13207547169813\" /><circle r=\"3\" cy=\"226.06654555791764\" cx=\"239.16981132075472\" /><circle r=\"3\" cy=\"156.990030884766\" cx=\"169.13207547169813\" /><circle r=\"3\" cy=\"213.784964562009\" cx=\"239.16981132075472\" /><circle r=\"3\" cy=\"168.6087971243955\" cx=\"199.77358490566036\" /><circle r=\"3\" cy=\"241.58622761414492\" cx=\"103.47169811320755\" /><circle r=\"3\" cy=\"226.940790808779\" cx=\"120.98113207547169\" /><circle r=\"3\" cy=\"232.36451479778844\" cx=\"103.47169811320755\" /><circle r=\"3\" cy=\"225.86580393938198\" cx=\"103.47169811320755\" /><circle r=\"3\" cy=\"231.82535102200896\" cx=\"120.98113207547169\" /><circle r=\"3\" cy=\"269.9019294734469\" cx=\"173.50943396226415\" /><circle r=\"3\" cy=\"295.99146259635774\" cx=\"247.92452830188677\" /><circle r=\"3\" cy=\"154.33230950466694\" cx=\"236.9811320754717\" /><circle r=\"3\" cy=\"108.02768794716955\" cx=\"331.0943396226415\" /><circle r=\"3\" cy=\"86.97354909704056\" cx=\"331.0943396226415\" /><circle r=\"3\" cy=\"44.865271396782674\" cx=\"331.0943396226415\" /><circle r=\"3\" cy=\"145.03785677831465\" cx=\"350.7924528301887\" /><circle r=\"3\" cy=\"288.19073581140526\" cx=\"195.39622641509436\" /><circle r=\"3\" cy=\"279.7690802713537\" cx=\"195.39622641509436\" /><circle r=\"3\" cy=\"226.436383849636\" cx=\"171.32075471698116\" /><circle r=\"3\" cy=\"221.94483422827514\" cx=\"171.32075471698116\" /><circle r=\"3\" cy=\"181.6612485616951\" cx=\"171.32075471698116\" /><circle r=\"3\" cy=\"175.06428505532136\" cx=\"171.32075471698116\" /><circle r=\"3\" cy=\"138.00900067909438\" cx=\"171.32075471698116\" /><circle r=\"3\" cy=\"131.41203717272066\" cx=\"171.32075471698116\" /><circle r=\"3\" cy=\"264.43542710999736\" cx=\"118.79245283018867\" /><circle r=\"3\" cy=\"263.1677234470473\" cx=\"142.86792452830187\" /><circle r=\"3\" cy=\"255.44787253533343\" cx=\"142.86792452830187\" /><circle r=\"3\" cy=\"262.1430886896744\" cx=\"142.86792452830187\" /><circle r=\"3\" cy=\"253.0336646138519\" cx=\"142.86792452830187\" /><circle r=\"3\" cy=\"222.36480235549743\" cx=\"142.86792452830187\" /><circle r=\"3\" cy=\"232.5690416515266\" cx=\"142.86792452830187\" /><circle r=\"3\" cy=\"204.1319181112857\" cx=\"142.86792452830187\" /><circle r=\"3\" cy=\"257.41292549467875\" cx=\"142.86792452830187\" /><circle r=\"3\" cy=\"219.02421232461032\" cx=\"142.86792452830187\" /><circle r=\"3\" cy=\"249.69307458296478\" cx=\"142.86792452830187\" /><circle r=\"3\" cy=\"198.02621784474837\" cx=\"142.86792452830187\" /><circle r=\"3\" cy=\"249.4379733406823\" cx=\"107.84905660377359\" /><circle r=\"3\" cy=\"235.54224169959718\" cx=\"107.84905660377359\" /><circle r=\"3\" cy=\"233.43682781458432\" cx=\"107.84905660377359\" /><circle r=\"3\" cy=\"227.40130801088065\" cx=\"107.84905660377359\" /><circle r=\"3\" cy=\"213.64593729546309\" cx=\"107.84905660377359\" /><circle r=\"3\" cy=\"201.29417583672077\" cx=\"107.84905660377359\" /><circle r=\"3\" cy=\"241.24356913429997\" cx=\"120.98113207547169\" /><circle r=\"3\" cy=\"237.59418506694428\" cx=\"120.98113207547169\" /><circle r=\"3\" cy=\"256.0148862197261\" cx=\"147.24528301886792\" /><circle r=\"3\" cy=\"257.5588564020689\" cx=\"147.24528301886792\" /><circle r=\"3\" cy=\"230.01469508089787\" cx=\"120.98113207547169\" /><circle r=\"3\" cy=\"221.3123176895112\" cx=\"120.98113207547169\" /><circle r=\"3\" cy=\"208.67983437943386\" cx=\"120.98113207547169\" /><circle r=\"3\" cy=\"225.52314545953698\" cx=\"120.98113207547169\" /><circle r=\"3\" cy=\"222.99664879752154\" cx=\"120.98113207547169\" /><circle r=\"3\" cy=\"208.1183906767637\" cx=\"120.98113207547169\" /><circle r=\"3\" cy=\"204.7497284607431\" cx=\"120.98113207547169\" /><circle r=\"3\" cy=\"333.0188970639715\" cx=\"226.0377358490566\" /><circle r=\"3\" cy=\"316.31594690953585\" cx=\"226.0377358490566\" /><circle r=\"3\" cy=\"311.4033145111724\" cx=\"226.0377358490566\" /><circle r=\"3\" cy=\"294.4196425054018\" cx=\"226.0377358490566\" /><circle r=\"3\" cy=\"289.50701010703835\" cx=\"226.0377358490566\" /><circle r=\"3\" cy=\"203.60612359851223\" cx=\"226.0377358490566\" /><circle r=\"3\" cy=\"269.5229549741445\" cx=\"173.50943396226415\" /><circle r=\"3\" cy=\"216.71382703281878\" cx=\"147.24528301886792\" /><circle r=\"3\" cy=\"254.9254187047218\" cx=\"173.50943396226415\" /><circle r=\"3\" cy=\"242.15257446897692\" cx=\"173.50943396226415\" /><circle r=\"3\" cy=\"237.2399420706135\" cx=\"173.50943396226415\" /><circle r=\"3\" cy=\"278.04598483498415\" cx=\"280.7547169811321\" /><circle r=\"3\" cy=\"285.9061966723656\" cx=\"280.7547169811321\" /><circle r=\"3\" cy=\"290.22931318292547\" cx=\"280.7547169811321\" /><circle r=\"3\" cy=\"265.8488426709964\" cx=\"258.8679245283019\" /><circle r=\"3\" cy=\"227.14152916013566\" cx=\"118.79245283018867\" /><circle r=\"3\" cy=\"252.5802765962938\" cx=\"145.0566037735849\" /><circle r=\"3\" cy=\"224.05358879545005\" cx=\"118.79245283018867\" /><circle r=\"3\" cy=\"249.4923362316082\" cx=\"145.0566037735849\" /><circle r=\"3\" cy=\"245.2815084615824\" cx=\"145.0566037735849\" /><circle r=\"3\" cy=\"202.99944994532106\" cx=\"118.79245283018867\" /><circle r=\"3\" cy=\"224.22736961145347\" cx=\"145.0566037735849\" /><circle r=\"3\" cy=\"201.76962150464925\" cx=\"145.0566037735849\" /><circle r=\"3\" cy=\"224.4379109999548\" cx=\"145.0566037735849\" /><circle r=\"3\" cy=\"241.46168752756432\" cx=\"204.1509433962264\" /><circle r=\"3\" cy=\"141.94244727994715\" cx=\"118.79245283018867\" /><circle r=\"3\" cy=\"192.01453717075617\" cx=\"145.0566037735849\" /><circle r=\"3\" cy=\"258.21365286772345\" cx=\"215.0943396226415\" /><circle r=\"3\" cy=\"251.54650889851598\" cx=\"215.0943396226415\" /><circle r=\"3\" cy=\"215.4737510019617\" cx=\"215.0943396226415\" /><circle r=\"3\" cy=\"208.03462194158277\" cx=\"215.0943396226415\" /><circle r=\"3\" cy=\"155.40373014820273\" cx=\"191.0188679245283\" /><circle r=\"3\" cy=\"147.96460108782384\" cx=\"191.0188679245283\" /><circle r=\"3\" cy=\"203.4027113945544\" cx=\"215.0943396226415\" /><circle r=\"3\" cy=\"172.52330774769862\" cx=\"215.0943396226415\" /><circle r=\"3\" cy=\"213.5978177501787\" cx=\"285.1320754716981\" /><circle r=\"3\" cy=\"133.86501268974038\" cx=\"223.84905660377356\" /><circle r=\"3\" cy=\"122.27409635872414\" cx=\"215.0943396226415\" /></g><g stroke=\"#ccc\" aria-label=\"rule\"><line y2=\"220.31576528802563\" y1=\"220.31576528802563\" x2=\"620\" x1=\"40\" /></g></svg>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import { residuals } from \"https://l12.xyz/x/shortcuts/raw/expr.ts\";\n",
    "\n",
    "let residualDf = df.select(\n",
    "  \"enginesize\",\n",
    "  residuals(pl.col(\"enginesize\"), pl.col(\"price\")),\n",
    ");\n",
    "\n",
    "let residPlot = Plot.plot({\n",
    "  x: \"enginesize\",\n",
    "  y: \"price\",\n",
    "  marks: [\n",
    "    Plot.dot(residualDf.toRecords(), { x: \"enginesize\", y: \"price\" }),\n",
    "    Plot.ruleY([0], { stroke: \"#ccc\" }),\n",
    "  ],\n",
    "  document,\n",
    "});\n",
    "\n",
    "await display(residPlot);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "29a0ec4e-084c-4af6-9af6-a14ae5cee4e7",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "source": [
    "From the above plot, we can tell the error variance across the true line is\n",
    "dispersed somewhat not uniformly, but in a funnel like shape. So, the assumption\n",
    "of the _homoscedasticity_ is more likely not met."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33d787f3-2a7b-421c-9d9b-a86719a92f9a",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "source": [
    "## Normality\n",
    "\n",
    "The linear regression analysis requires the dependent variable, 'price', to be\n",
    "normally distributed. A histogram, box plot, or a Q-Q-Plot can check if the\n",
    "target variable is normally distributed. The goodness of fit test, e.g., the\n",
    "Kolmogorov-Smirnov test can check for normality in the dependent variable.\n",
    "[This documentation](https://towardsdatascience.com/normality-tests-in-python-31e04aa4f411)\n",
    "contains more information on the normality assumption.\n",
    "\n",
    "Let's display all three charts to show how our target variable, 'price' behaves."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b94d0d8d-ae95-41f5-b17f-0ab9b3aa27ec",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <section style=\"display:flex;flex-direction: column; gap: 1em;\">\n",
       "              <div style=\"gap:0.5em; display:flex; flex-direction: row; border: 1px solid black;\">\n",
       "                  <img src=\"data:image/svg+xml;base64,[encoded data, 6880 characters]\" style=\"width: 100%;\">\n",
       "                  <img src=\"data:image/svg+xml;base64,[encoded data, 4960 characters]\" style=\"width: 65%\">\n",
       "              </div>\n",
       "              <div style=\"padding:1em; border: 1px solid black;\">\n",
       "                  <img src=\"data:image/svg+xml;base64,[encoded data, 53880 characters]\" style=\"width: 100%\">\n",
       "              </div>\n",
       "            </section>\n",
       "            "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import { threeChart } from \"https://l12.xyz/x/shortcuts/raw/plots.ts\";\n",
    "\n",
    "await display(threeChart(records, \"price\"));"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81fe890e-56c4-4076-b57c-38fe438c0ead",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "source": [
    "These three charts above can tell us a lot about our target variable:\n",
    "\n",
    "- Our target variable, 'price' is not normally distributed\n",
    "- Our target variable is right-skewed\n",
    "- There are some outliers in the variable\n",
    "\n",
    "The right-skewed plot means that most prices in the dataset are on the lower end\n",
    "(below 15,000). The 'max' value is very far from the '75%' quantile statistic.\n",
    "All these plots show that the assumption for accurate linear regression modeling\n",
    "is not met.\n",
    "\n",
    "Next, we will perform the log transformation to correct our target variable and\n",
    "to make it more normally distributed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "20b0317a-5807-4f28-9008-8540e5ee056a",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[33m0.8006648595531076\u001b[39m"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import { ShapiroWilkW } from \"https://l12.xyz/x/shortcuts/raw/shapiro.ts\";\n",
    "\n",
    "ShapiroWilkW(df.price.sort());"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "8f8c6044-d3a1-4f8e-89ba-365fea3fbd8b",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <section style=\"display:flex;flex-direction: column; gap: 1em;\">\n",
       "              <div style=\"gap:0.5em; display:flex; flex-direction: row; border: 1px solid black;\">\n",
       "                  <img src=\"data:image/svg+xml;base64,[encoded data, 7784 characters]\" style=\"width: 100%;\">\n",
       "                  <img src=\"data:image/svg+xml;base64,[encoded data, 4720 characters]\" style=\"width: 65%\">\n",
       "              </div>\n",
       "              <div style=\"padding:1em; border: 1px solid black;\">\n",
       "                  <img src=\"data:image/svg+xml;base64,[encoded data, 55872 characters]\" style=\"width: 100%\">\n",
       "              </div>\n",
       "            </section>\n",
       "            "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "let log2df = df.select(pl.col(\"price\").log());\n",
    "\n",
    "await display(threeChart(log2df.toRecords(), \"price\"));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "9bdeba3c-2eac-44bc-8601-8241c5bfe502",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[33m0.948674582545282\u001b[39m"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ShapiroWilkW(log2df.price.sort());"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09a9d9c0-7fb6-4353-91f8-4899520b36d7",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "source": [
    "## Data Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "79f05578-b839-4278-afb1-0123c1d32d17",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.dataresource+json": {
       "bytes": null,
       "data": [
        {
         "aspiration": "std",
         "boreratio": 3.47,
         "brand": "alfa-romero",
         "brand_category": "Mid_Range",
         "carbody": "convertible",
         "carheight": 48.8,
         "carlength": 168.8,
         "carwidth": 64.1,
         "citympg": 21,
         "compressionratio": 9,
         "curbweight": 2548,
         "cylindernumber": "four",
         "doornumber": "two",
         "drivewheel": "rwd",
         "enginelocation": "front",
         "enginesize": 130,
         "enginetype": "dohc",
         "fuelsystem": "mpfi",
         "fueltype": "gas",
         "highwaympg": 27,
         "horsepower": 111,
         "peakrpm": 5000,
         "price": 13495,
         "stroke": 2.68,
         "wheelbase": 88.6
        },
        {
         "aspiration": "std",
         "boreratio": 3.47,
         "brand": "alfa-romero",
         "brand_category": "Mid_Range",
         "carbody": "convertible",
         "carheight": 48.8,
         "carlength": 168.8,
         "carwidth": 64.1,
         "citympg": 21,
         "compressionratio": 9,
         "curbweight": 2548,
         "cylindernumber": "four",
         "doornumber": "two",
         "drivewheel": "rwd",
         "enginelocation": "front",
         "enginesize": 130,
         "enginetype": "dohc",
         "fuelsystem": "mpfi",
         "fueltype": "gas",
         "highwaympg": 27,
         "horsepower": 111,
         "peakrpm": 5000,
         "price": 16500,
         "stroke": 2.68,
         "wheelbase": 88.6
        },
        {
         "aspiration": "std",
         "boreratio": 2.68,
         "brand": "alfa-romero",
         "brand_category": "Mid_Range",
         "carbody": "hatchback",
         "carheight": 52.4,
         "carlength": 171.2,
         "carwidth": 65.5,
         "citympg": 19,
         "compressionratio": 9,
         "curbweight": 2823,
         "cylindernumber": "six",
         "doornumber": "two",
         "drivewheel": "rwd",
         "enginelocation": "front",
         "enginesize": 152,
         "enginetype": "ohcv",
         "fuelsystem": "mpfi",
         "fueltype": "gas",
         "highwaympg": 26,
         "horsepower": 154,
         "peakrpm": 5000,
         "price": 16500,
         "stroke": 3.47,
         "wheelbase": 94.5
        },
        {
         "aspiration": "std",
         "boreratio": 3.19,
         "brand": "audi",
         "brand_category": "Mid_Range",
         "carbody": "sedan",
         "carheight": 54.3,
         "carlength": 176.6,
         "carwidth": 66.2,
         "citympg": 24,
         "compressionratio": 10,
         "curbweight": 2337,
         "cylindernumber": "four",
         "doornumber": "four",
         "drivewheel": "fwd",
         "enginelocation": "front",
         "enginesize": 109,
         "enginetype": "ohc",
         "fuelsystem": "mpfi",
         "fueltype": "gas",
         "highwaympg": 30,
         "horsepower": 102,
         "peakrpm": 5500,
         "price": 13950,
         "stroke": 3.4,
         "wheelbase": 99.8
        },
        {
         "aspiration": "std",
         "boreratio": 3.19,
         "brand": "audi",
         "brand_category": "Mid_Range",
         "carbody": "sedan",
         "carheight": 54.3,
         "carlength": 176.6,
         "carwidth": 66.4,
         "citympg": 18,
         "compressionratio": 8,
         "curbweight": 2824,
         "cylindernumber": "five",
         "doornumber": "four",
         "drivewheel": "4wd",
         "enginelocation": "front",
         "enginesize": 136,
         "enginetype": "ohc",
         "fuelsystem": "mpfi",
         "fueltype": "gas",
         "highwaympg": 22,
         "horsepower": 115,
         "peakrpm": 5500,
         "price": 17450,
         "stroke": 3.4,
         "wheelbase": 99.4
        }
       ],
       "description": null,
       "dialect": null,
       "encoding": null,
       "format": null,
       "hash": null,
       "homepage": null,
       "licenses": null,
       "mediatype": null,
       "path": null,
       "schema": {
        "fields": [
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "brand_category",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "brand",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "fueltype",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "aspiration",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "doornumber",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "carbody",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "drivewheel",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "enginelocation",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "wheelbase",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "carlength",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "carwidth",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "carheight",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "curbweight",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "enginetype",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "cylindernumber",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "enginesize",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "fuelsystem",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "boreratio",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "stroke",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "compressionratio",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "horsepower",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "peakrpm",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "citympg",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "highwaympg",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "price",
          "rdfType": null,
          "title": null,
          "type": "number"
         }
        ],
        "foreignKeys": null,
        "missingValues": null,
        "primaryKey": null
       },
       "sources": null,
       "title": null
      },
      "text/html": [
       "<table><thead><tr><th>brand_category</th><th>brand</th><th>fueltype</th><th>aspiration</th><th>doornumber</th><th>carbody</th><th>drivewheel</th><th>enginelocation</th><th>wheelbase</th><th>carlength</th><th>carwidth</th><th>carheight</th><th>curbweight</th><th>enginetype</th><th>cylindernumber</th><th>enginesize</th><th>fuelsystem</th><th>boreratio</th><th>stroke</th><th>compressionratio</th><th>horsepower</th><th>peakrpm</th><th>citympg</th><th>highwaympg</th><th>price</th></tr></thead><tbody><tr><td>Mid_Range</td><td>alfa-romero</td><td>gas</td><td>std</td><td>two</td><td>convertible</td><td>rwd</td><td>front</td><td>88.6</td><td>168.8</td><td>64.1</td><td>48.8</td><td>2548</td><td>dohc</td><td>four</td><td>130</td><td>mpfi</td><td>3.47</td><td>2.68</td><td>9</td><td>111</td><td>5000</td><td>21</td><td>27</td><td>13495</td></tr><tr><td>Mid_Range</td><td>alfa-romero</td><td>gas</td><td>std</td><td>two</td><td>convertible</td><td>rwd</td><td>front</td><td>88.6</td><td>168.8</td><td>64.1</td><td>48.8</td><td>2548</td><td>dohc</td><td>four</td><td>130</td><td>mpfi</td><td>3.47</td><td>2.68</td><td>9</td><td>111</td><td>5000</td><td>21</td><td>27</td><td>16500</td></tr><tr><td>Mid_Range</td><td>alfa-romero</td><td>gas</td><td>std</td><td>two</td><td>hatchback</td><td>rwd</td><td>front</td><td>94.5</td><td>171.2</td><td>65.5</td><td>52.4</td><td>2823</td><td>ohcv</td><td>six</td><td>152</td><td>mpfi</td><td>2.68</td><td>3.47</td><td>9</td><td>154</td><td>5000</td><td>19</td><td>26</td><td>16500</td></tr><tr><td>Mid_Range</td><td>audi</td><td>gas</td><td>std</td><td>four</td><td>sedan</td><td>fwd</td><td>front</td><td>99.8</td><td>176.6</td><td>66.2</td><td>54.3</td><td>2337</td><td>ohc</td><td>four</td><td>109</td><td>mpfi</td><td>3.19</td><td>3.4</td><td>10</td><td>102</td><td>5500</td><td>24</td><td>30</td><td>13950</td></tr><tr><td>Mid_Range</td><td>audi</td><td>gas</td><td>std</td><td>four</td><td>sedan</td><td>4wd</td><td>front</td><td>99.4</td><td>176.6</td><td>66.4</td><td>54.3</td><td>2824</td><td>ohc</td><td>five</td><td>136</td><td>mpfi</td><td>3.19</td><td>3.4</td><td>8</td><td>115</td><td>5500</td><td>18</td><td>22</td><td>17450</td></tr></tbody></table>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "let carData = pl.readCSV(\n",
    "  await Deno.readTextFile(\"assets/cleaned_car_prices.csv\"),\n",
    "  { sep: \",\" },\n",
    ");\n",
    "\n",
    "carData.head(5);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af6eb67c-574a-430a-a04c-24d0ef9f35fa",
   "metadata": {},
   "source": [
    "We'll drop some unnecessary columns:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "e2df0c3f-dbc0-4820-b807-373ce3787645",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.dataresource+json": {
       "bytes": null,
       "data": [
        {
         "aspiration": "std",
         "boreratio": 3.47,
         "brand_category": "Mid_Range",
         "carheight": 48.8,
         "carlength": 168.8,
         "carwidth": 64.1,
         "citympg": 21,
         "curbweight": 2548,
         "cylindernumber": "four",
         "doornumber": "two",
         "drivewheel": "rwd",
         "enginesize": 130,
         "enginetype": "dohc",
         "fuelsystem": "mpfi",
         "fueltype": "gas",
         "highwaympg": 27,
         "horsepower": 111,
         "price": 13495,
         "wheelbase": 88.6
        },
        {
         "aspiration": "std",
         "boreratio": 3.47,
         "brand_category": "Mid_Range",
         "carheight": 48.8,
         "carlength": 168.8,
         "carwidth": 64.1,
         "citympg": 21,
         "curbweight": 2548,
         "cylindernumber": "four",
         "doornumber": "two",
         "drivewheel": "rwd",
         "enginesize": 130,
         "enginetype": "dohc",
         "fuelsystem": "mpfi",
         "fueltype": "gas",
         "highwaympg": 27,
         "horsepower": 111,
         "price": 16500,
         "wheelbase": 88.6
        },
        {
         "aspiration": "std",
         "boreratio": 2.68,
         "brand_category": "Mid_Range",
         "carheight": 52.4,
         "carlength": 171.2,
         "carwidth": 65.5,
         "citympg": 19,
         "curbweight": 2823,
         "cylindernumber": "six",
         "doornumber": "two",
         "drivewheel": "rwd",
         "enginesize": 152,
         "enginetype": "ohcv",
         "fuelsystem": "mpfi",
         "fueltype": "gas",
         "highwaympg": 26,
         "horsepower": 154,
         "price": 16500,
         "wheelbase": 94.5
        },
        {
         "aspiration": "std",
         "boreratio": 3.19,
         "brand_category": "Mid_Range",
         "carheight": 54.3,
         "carlength": 176.6,
         "carwidth": 66.2,
         "citympg": 24,
         "curbweight": 2337,
         "cylindernumber": "four",
         "doornumber": "four",
         "drivewheel": "fwd",
         "enginesize": 109,
         "enginetype": "ohc",
         "fuelsystem": "mpfi",
         "fueltype": "gas",
         "highwaympg": 30,
         "horsepower": 102,
         "price": 13950,
         "wheelbase": 99.8
        },
        {
         "aspiration": "std",
         "boreratio": 3.19,
         "brand_category": "Mid_Range",
         "carheight": 54.3,
         "carlength": 176.6,
         "carwidth": 66.4,
         "citympg": 18,
         "curbweight": 2824,
         "cylindernumber": "five",
         "doornumber": "four",
         "drivewheel": "4wd",
         "enginesize": 136,
         "enginetype": "ohc",
         "fuelsystem": "mpfi",
         "fueltype": "gas",
         "highwaympg": 22,
         "horsepower": 115,
         "price": 17450,
         "wheelbase": 99.4
        }
       ],
       "description": null,
       "dialect": null,
       "encoding": null,
       "format": null,
       "hash": null,
       "homepage": null,
       "licenses": null,
       "mediatype": null,
       "path": null,
       "schema": {
        "fields": [
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "brand_category",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "fueltype",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "aspiration",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "doornumber",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "drivewheel",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "wheelbase",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "carlength",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "carwidth",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "carheight",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "curbweight",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "enginetype",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "cylindernumber",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "enginesize",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "fuelsystem",
          "rdfType": null,
          "title": null,
          "type": "string"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "boreratio",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "horsepower",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "citympg",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "highwaympg",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "price",
          "rdfType": null,
          "title": null,
          "type": "number"
         }
        ],
        "foreignKeys": null,
        "missingValues": null,
        "primaryKey": null
       },
       "sources": null,
       "title": null
      },
      "text/html": [
       "<table><thead><tr><th>brand_category</th><th>fueltype</th><th>aspiration</th><th>doornumber</th><th>drivewheel</th><th>wheelbase</th><th>carlength</th><th>carwidth</th><th>carheight</th><th>curbweight</th><th>enginetype</th><th>cylindernumber</th><th>enginesize</th><th>fuelsystem</th><th>boreratio</th><th>horsepower</th><th>citympg</th><th>highwaympg</th><th>price</th></tr></thead><tbody><tr><td>Mid_Range</td><td>gas</td><td>std</td><td>two</td><td>rwd</td><td>88.6</td><td>168.8</td><td>64.1</td><td>48.8</td><td>2548</td><td>dohc</td><td>four</td><td>130</td><td>mpfi</td><td>3.47</td><td>111</td><td>21</td><td>27</td><td>13495</td></tr><tr><td>Mid_Range</td><td>gas</td><td>std</td><td>two</td><td>rwd</td><td>88.6</td><td>168.8</td><td>64.1</td><td>48.8</td><td>2548</td><td>dohc</td><td>four</td><td>130</td><td>mpfi</td><td>3.47</td><td>111</td><td>21</td><td>27</td><td>16500</td></tr><tr><td>Mid_Range</td><td>gas</td><td>std</td><td>two</td><td>rwd</td><td>94.5</td><td>171.2</td><td>65.5</td><td>52.4</td><td>2823</td><td>ohcv</td><td>six</td><td>152</td><td>mpfi</td><td>2.68</td><td>154</td><td>19</td><td>26</td><td>16500</td></tr><tr><td>Mid_Range</td><td>gas</td><td>std</td><td>four</td><td>fwd</td><td>99.8</td><td>176.6</td><td>66.2</td><td>54.3</td><td>2337</td><td>ohc</td><td>four</td><td>109</td><td>mpfi</td><td>3.19</td><td>102</td><td>24</td><td>30</td><td>13950</td></tr><tr><td>Mid_Range</td><td>gas</td><td>std</td><td>four</td><td>4wd</td><td>99.4</td><td>176.6</td><td>66.4</td><td>54.3</td><td>2824</td><td>ohc</td><td>five</td><td>136</td><td>mpfi</td><td>3.19</td><td>115</td><td>18</td><td>22</td><td>17450</td></tr></tbody></table>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "let carDataGeneralized = carData.drop(\n",
    "  \"brand\",\n",
    "  \"carbody\",\n",
    "  \"enginelocation\",\n",
    "  \"stroke\",\n",
    "  \"compressionratio\",\n",
    "  \"peakrpm\",\n",
    ");\n",
    "carDataGeneralized.head(5);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "826ab213-65a2-43d0-b9ce-fecdebd79eb8",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "source": [
    "Next we use one hot (binary) encoding. We assume that all non-numeric colums are\n",
    "categorical."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "9bc95dd8-0240-45c6-a0f9-21b1873ae04b",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.dataresource+json": {
       "bytes": null,
       "data": [
        {
         "aspiration_std": 1,
         "aspiration_turbo": 0,
         "boreratio": 3.47,
         "brand_category_Budget": 0,
         "brand_category_Luxury": 0,
         "brand_category_Mid_Range": 1,
         "carheight": 48.8,
         "carlength": 168.8,
         "carwidth": 64.1,
         "citympg": 21,
         "curbweight": 2548,
         "cylindernumber_eight": 0,
         "cylindernumber_five": 0,
         "cylindernumber_four": 1,
         "cylindernumber_six": 0,
         "cylindernumber_three": 0,
         "cylindernumber_twelve": 0,
         "cylindernumber_two": 0,
         "doornumber_four": 0,
         "doornumber_two": 1,
         "drivewheel_4wd": 0,
         "drivewheel_fwd": 0,
         "drivewheel_rwd": 1,
         "enginesize": 130,
         "enginetype_dohc": 1,
         "enginetype_dohcv": 0,
         "enginetype_l": 0,
         "enginetype_ohc": 0,
         "enginetype_ohcf": 0,
         "enginetype_ohcv": 0,
         "enginetype_rotor": 0,
         "fuelsystem_1bbl": 0,
         "fuelsystem_2bbl": 0,
         "fuelsystem_4bbl": 0,
         "fuelsystem_idi": 0,
         "fuelsystem_mfi": 0,
         "fuelsystem_mpfi": 1,
         "fuelsystem_spdi": 0,
         "fuelsystem_spfi": 0,
         "fueltype_diesel": 0,
         "fueltype_gas": 1,
         "highwaympg": 27,
         "horsepower": 111,
         "price": 13495,
         "wheelbase": 88.6
        },
        {
         "aspiration_std": 1,
         "aspiration_turbo": 0,
         "boreratio": 3.47,
         "brand_category_Budget": 0,
         "brand_category_Luxury": 0,
         "brand_category_Mid_Range": 1,
         "carheight": 48.8,
         "carlength": 168.8,
         "carwidth": 64.1,
         "citympg": 21,
         "curbweight": 2548,
         "cylindernumber_eight": 0,
         "cylindernumber_five": 0,
         "cylindernumber_four": 1,
         "cylindernumber_six": 0,
         "cylindernumber_three": 0,
         "cylindernumber_twelve": 0,
         "cylindernumber_two": 0,
         "doornumber_four": 0,
         "doornumber_two": 1,
         "drivewheel_4wd": 0,
         "drivewheel_fwd": 0,
         "drivewheel_rwd": 1,
         "enginesize": 130,
         "enginetype_dohc": 1,
         "enginetype_dohcv": 0,
         "enginetype_l": 0,
         "enginetype_ohc": 0,
         "enginetype_ohcf": 0,
         "enginetype_ohcv": 0,
         "enginetype_rotor": 0,
         "fuelsystem_1bbl": 0,
         "fuelsystem_2bbl": 0,
         "fuelsystem_4bbl": 0,
         "fuelsystem_idi": 0,
         "fuelsystem_mfi": 0,
         "fuelsystem_mpfi": 1,
         "fuelsystem_spdi": 0,
         "fuelsystem_spfi": 0,
         "fueltype_diesel": 0,
         "fueltype_gas": 1,
         "highwaympg": 27,
         "horsepower": 111,
         "price": 16500,
         "wheelbase": 88.6
        },
        {
         "aspiration_std": 1,
         "aspiration_turbo": 0,
         "boreratio": 2.68,
         "brand_category_Budget": 0,
         "brand_category_Luxury": 0,
         "brand_category_Mid_Range": 1,
         "carheight": 52.4,
         "carlength": 171.2,
         "carwidth": 65.5,
         "citympg": 19,
         "curbweight": 2823,
         "cylindernumber_eight": 0,
         "cylindernumber_five": 0,
         "cylindernumber_four": 0,
         "cylindernumber_six": 1,
         "cylindernumber_three": 0,
         "cylindernumber_twelve": 0,
         "cylindernumber_two": 0,
         "doornumber_four": 0,
         "doornumber_two": 1,
         "drivewheel_4wd": 0,
         "drivewheel_fwd": 0,
         "drivewheel_rwd": 1,
         "enginesize": 152,
         "enginetype_dohc": 0,
         "enginetype_dohcv": 0,
         "enginetype_l": 0,
         "enginetype_ohc": 0,
         "enginetype_ohcf": 0,
         "enginetype_ohcv": 1,
         "enginetype_rotor": 0,
         "fuelsystem_1bbl": 0,
         "fuelsystem_2bbl": 0,
         "fuelsystem_4bbl": 0,
         "fuelsystem_idi": 0,
         "fuelsystem_mfi": 0,
         "fuelsystem_mpfi": 1,
         "fuelsystem_spdi": 0,
         "fuelsystem_spfi": 0,
         "fueltype_diesel": 0,
         "fueltype_gas": 1,
         "highwaympg": 26,
         "horsepower": 154,
         "price": 16500,
         "wheelbase": 94.5
        },
        {
         "aspiration_std": 1,
         "aspiration_turbo": 0,
         "boreratio": 3.19,
         "brand_category_Budget": 0,
         "brand_category_Luxury": 0,
         "brand_category_Mid_Range": 1,
         "carheight": 54.3,
         "carlength": 176.6,
         "carwidth": 66.2,
         "citympg": 24,
         "curbweight": 2337,
         "cylindernumber_eight": 0,
         "cylindernumber_five": 0,
         "cylindernumber_four": 1,
         "cylindernumber_six": 0,
         "cylindernumber_three": 0,
         "cylindernumber_twelve": 0,
         "cylindernumber_two": 0,
         "doornumber_four": 1,
         "doornumber_two": 0,
         "drivewheel_4wd": 0,
         "drivewheel_fwd": 1,
         "drivewheel_rwd": 0,
         "enginesize": 109,
         "enginetype_dohc": 0,
         "enginetype_dohcv": 0,
         "enginetype_l": 0,
         "enginetype_ohc": 1,
         "enginetype_ohcf": 0,
         "enginetype_ohcv": 0,
         "enginetype_rotor": 0,
         "fuelsystem_1bbl": 0,
         "fuelsystem_2bbl": 0,
         "fuelsystem_4bbl": 0,
         "fuelsystem_idi": 0,
         "fuelsystem_mfi": 0,
         "fuelsystem_mpfi": 1,
         "fuelsystem_spdi": 0,
         "fuelsystem_spfi": 0,
         "fueltype_diesel": 0,
         "fueltype_gas": 1,
         "highwaympg": 30,
         "horsepower": 102,
         "price": 13950,
         "wheelbase": 99.8
        },
        {
         "aspiration_std": 1,
         "aspiration_turbo": 0,
         "boreratio": 3.19,
         "brand_category_Budget": 0,
         "brand_category_Luxury": 0,
         "brand_category_Mid_Range": 1,
         "carheight": 54.3,
         "carlength": 176.6,
         "carwidth": 66.4,
         "citympg": 18,
         "curbweight": 2824,
         "cylindernumber_eight": 0,
         "cylindernumber_five": 1,
         "cylindernumber_four": 0,
         "cylindernumber_six": 0,
         "cylindernumber_three": 0,
         "cylindernumber_twelve": 0,
         "cylindernumber_two": 0,
         "doornumber_four": 1,
         "doornumber_two": 0,
         "drivewheel_4wd": 1,
         "drivewheel_fwd": 0,
         "drivewheel_rwd": 0,
         "enginesize": 136,
         "enginetype_dohc": 0,
         "enginetype_dohcv": 0,
         "enginetype_l": 0,
         "enginetype_ohc": 1,
         "enginetype_ohcf": 0,
         "enginetype_ohcv": 0,
         "enginetype_rotor": 0,
         "fuelsystem_1bbl": 0,
         "fuelsystem_2bbl": 0,
         "fuelsystem_4bbl": 0,
         "fuelsystem_idi": 0,
         "fuelsystem_mfi": 0,
         "fuelsystem_mpfi": 1,
         "fuelsystem_spdi": 0,
         "fuelsystem_spfi": 0,
         "fueltype_diesel": 0,
         "fueltype_gas": 1,
         "highwaympg": 22,
         "horsepower": 115,
         "price": 17450,
         "wheelbase": 99.4
        }
       ],
       "description": null,
       "dialect": null,
       "encoding": null,
       "format": null,
       "hash": null,
       "homepage": null,
       "licenses": null,
       "mediatype": null,
       "path": null,
       "schema": {
        "fields": [
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "brand_category_Budget",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "brand_category_Luxury",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "brand_category_Mid_Range",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "fueltype_diesel",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "fueltype_gas",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "aspiration_std",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "aspiration_turbo",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "doornumber_four",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "doornumber_two",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "drivewheel_4wd",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "drivewheel_fwd",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "drivewheel_rwd",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "wheelbase",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "carlength",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "carwidth",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "carheight",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "curbweight",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "enginetype_dohc",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "enginetype_dohcv",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "enginetype_l",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "enginetype_ohc",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "enginetype_ohcf",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "enginetype_ohcv",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "enginetype_rotor",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "cylindernumber_eight",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "cylindernumber_five",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "cylindernumber_four",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "cylindernumber_six",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "cylindernumber_three",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "cylindernumber_twelve",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "cylindernumber_two",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "enginesize",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "fuelsystem_1bbl",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "fuelsystem_2bbl",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "fuelsystem_4bbl",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "fuelsystem_idi",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "fuelsystem_mfi",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "fuelsystem_mpfi",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "fuelsystem_spdi",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "fuelsystem_spfi",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "boreratio",
          "rdfType": null,
          "title": null,
          "type": "number"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "horsepower",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "citympg",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "highwaympg",
          "rdfType": null,
          "title": null,
          "type": "integer"
         },
         {
          "constraints": null,
          "description": null,
          "example": null,
          "format": null,
          "name": "price",
          "rdfType": null,
          "title": null,
          "type": "number"
         }
        ],
        "foreignKeys": null,
        "missingValues": null,
        "primaryKey": null
       },
       "sources": null,
       "title": null
      },
      "text/html": [
       "<table><thead><tr><th>brand_category_Budget</th><th>brand_category_Luxury</th><th>brand_category_Mid_Range</th><th>fueltype_diesel</th><th>fueltype_gas</th><th>aspiration_std</th><th>aspiration_turbo</th><th>doornumber_four</th><th>doornumber_two</th><th>drivewheel_4wd</th><th>drivewheel_fwd</th><th>drivewheel_rwd</th><th>wheelbase</th><th>carlength</th><th>carwidth</th><th>carheight</th><th>curbweight</th><th>enginetype_dohc</th><th>enginetype_dohcv</th><th>enginetype_l</th><th>enginetype_ohc</th><th>enginetype_ohcf</th><th>enginetype_ohcv</th><th>enginetype_rotor</th><th>cylindernumber_eight</th><th>cylindernumber_five</th><th>cylindernumber_four</th><th>cylindernumber_six</th><th>cylindernumber_three</th><th>cylindernumber_twelve</th><th>cylindernumber_two</th><th>enginesize</th><th>fuelsystem_1bbl</th><th>fuelsystem_2bbl</th><th>fuelsystem_4bbl</th><th>fuelsystem_idi</th><th>fuelsystem_mfi</th><th>fuelsystem_mpfi</th><th>fuelsystem_spdi</th><th>fuelsystem_spfi</th><th>boreratio</th><th>horsepower</th><th>citympg</th><th>highwaympg</th><th>price</th></tr></thead><tbody><tr><td>0</td><td>0</td><td>1</td><td>0</td><td>1</td><td>1</td><td>0</td><td>0</td><td>1</td><td>0</td><td>0</td><td>1</td><td>88.6</td><td>168.8</td><td>64.1</td><td>48.8</td><td>2548</td><td>1</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>1</td><td>0</td><td>0</td><td>0</td><td>0</td><td>130</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>1</td><td>0</td><td>0</td><td>3.47</td><td>111</td><td>21</td><td>27</td><td>13495</td></tr><tr><td>0</td><td>0</td><td>1</td><td>0</td><td>1</td><td>1</td><td>0</td><td>0</td><td>1</td><td>0</td><td>0</td><td>1</td><td>88.6</td><td>168.8</td><td>64.1</td><td>48.8</td><td>2548</td><td>1</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>1</td><td>0</td><td>0</td><td>0</td><td>0</td><td>130</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>1</td><td>0</td><td>0</td><td>3.47</td><td>111</td><td>21</td><td>27</td><td>16500</td></tr><tr><td>0</td><td>0</td><td>1</td><td>0</td><td>1</td><td>1</td><td>0</td><td>0</td><td>1</td><td>0</td><td>0</td><td>1</td><td>94.5</td><td>171.2</td><td>65.5</td><td>52.4</td><td>2823</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>1</td><td>0</td><td>0</td><td>0</td><td>0</td><td>1</td><td>0</td><td>0</td><td>0</td><td>152</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>1</td><td>0</td><td>0</td><td>2.68</td><td>154</td><td>19</td><td>26</td><td>16500</td></tr><tr><td>0</td><td>0</td><td>1</td><td>0</td><td>1</td><td>1</td><td>0</td><td>1</td><td>0</td><td>0</td><td>1</td><td>0</td><td>99.8</td><td>176.6</td><td>66.2</td><td>54.3</td><td>2337</td><td>0</td><td>0</td><td>0</td><td>1</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>1</td><td>0</td><td>0</td><td>0</td><td>0</td><td>109</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>1</td><td>0</td><td>0</td><td>3.19</td><td>102</td><td>24</td><td>30</td><td>13950</td></tr><tr><td>0</td><td>0</td><td>1</td><td>0</td><td>1</td><td>1</td><td>0</td><td>1</td><td>0</td><td>1</td><td>0</td><td>0</td><td>99.4</td><td>176.6</td><td>66.4</td><td>54.3</td><td>2824</td><td>0</td><td>0</td><td>0</td><td>1</td><td>0</td><td>0</td><td>0</td><td>0</td><td>1</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>136</td><td>0</td><td>0</td><td>0</td><td>0</td><td>0</td><td>1</td><td>0</td><td>0</td><td>3.19</td><td>115</td><td>18</td><td>22</td><td>17450</td></tr></tbody></table>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import { oneHotEncoding } from \"https://l12.xyz/x/shortcuts/raw/encoding.ts\";\n",
    "\n",
    "let encodedCarData = oneHotEncoding(carDataGeneralized);\n",
    "encodedCarData.head(5);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "aff549ce-6736-43c7-8d40-b09b9ca7fa59",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "encodedCarData.writeCSV(\"assets/encoded_car_data.csv\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec799133-584f-450d-bd8b-6b443fbf5fb5",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Deno",
   "language": "typescript",
   "name": "deno"
  },
  "language_info": {
   "codemirror_mode": "typescript",
   "file_extension": ".ts",
   "mimetype": "text/x.typescript",
   "name": "typescript",
   "nbconvert_exporter": "script",
   "pygments_lexer": "typescript",
   "version": "5.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
