{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initial Data Exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import polars as pl\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from io import StringIO\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pl.read_csv('output.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['',\n",
       " 'time',\n",
       " 'AVG_PACK_SOC',\n",
       " 'AvgSOC',\n",
       " 'AvgSOH',\n",
       " 'BATT_PACK_1_DISCONN',\n",
       " 'BATT_PACK_1_FAULT',\n",
       " 'BATT_PACK_1_WARN',\n",
       " 'BATT_PACK_2_DISCONN',\n",
       " 'BATT_PACK_2_FAULT',\n",
       " 'BATT_PACK_2_WARN',\n",
       " 'BATT_PACK_3_DISCONN',\n",
       " 'BATT_PACK_3_FAULT',\n",
       " 'BATT_PACK_3_WARN',\n",
       " 'BATT_PACK_4_DISCONN',\n",
       " 'BATT_PACK_4_FAULT',\n",
       " 'BATT_PACK_4_WARN',\n",
       " 'BATT_PACK_5_DISCONN',\n",
       " 'BATT_PACK_5_FAULT',\n",
       " 'BATT_PACK_5_WARN',\n",
       " 'BATT_PACK_6_DISCONN',\n",
       " 'BATT_PACK_6_FAULT',\n",
       " 'BATT_PACK_6_WARN',\n",
       " 'BATT_PACK_7_DISCONN',\n",
       " 'BATT_PACK_7_FAULT',\n",
       " 'BATT_PACK_7_WARN',\n",
       " 'BATT_ROOM_TEMP',\n",
       " 'Current',\n",
       " 'MAX_CELL_TEMP',\n",
       " 'MAX_CELL_VOL',\n",
       " 'MAX_PACK_SOC',\n",
       " 'MAX_UNCONN_PACK_VOL',\n",
       " 'MIN_CELL_TEMP',\n",
       " 'MIN_CELL_VOL',\n",
       " 'MIN_PACK_SOC',\n",
       " 'MIN_UNCONN_PACK_VOL',\n",
       " 'NUMBER_OF_PACKS_CONN',\n",
       " 'NUMBER_OF_PACKS_FAULTED',\n",
       " 'PACK1_CRIDATA_AVG_CELL_TEMP',\n",
       " 'PACK1_CRIDATA_AVG_CELL_VOL',\n",
       " 'PACK1_CRIDATA_BATT_VOL',\n",
       " 'PACK1_CRIDATA_BUS_VOL',\n",
       " 'PACK1_CRIDATA_CHARGE_CURR_LIMIT',\n",
       " 'PACK1_CRIDATA_CURR',\n",
       " 'PACK1_CRIDATA_DISCHARGE_CURR_LIMIT',\n",
       " 'PACK1_CRIDATA_SOC',\n",
       " 'PACK1_CRIDATA_SOH',\n",
       " 'PACK2_CRIDATA_AVG_CELL_TEMP',\n",
       " 'PACK2_CRIDATA_AVG_CELL_VOL',\n",
       " 'PACK2_CRIDATA_BATT_VOL',\n",
       " 'PACK2_CRIDATA_BUS_VOL',\n",
       " 'PACK2_CRIDATA_CHARGE_CURR_LIMIT',\n",
       " 'PACK2_CRIDATA_CURR',\n",
       " 'PACK2_CRIDATA_DISCHARGE_CURR_LIMIT',\n",
       " 'PACK2_CRIDATA_SOC',\n",
       " 'PACK2_CRIDATA_SOH',\n",
       " 'PACK3_CRIDATA_AVG_CELL_TEMP',\n",
       " 'PACK3_CRIDATA_AVG_CELL_VOL',\n",
       " 'PACK3_CRIDATA_BATT_VOL',\n",
       " 'PACK3_CRIDATA_BUS_VOL',\n",
       " 'PACK3_CRIDATA_CHARGE_CURR_LIMIT',\n",
       " 'PACK3_CRIDATA_CURR',\n",
       " 'PACK3_CRIDATA_DISCHARGE_CURR_LIMIT',\n",
       " 'PACK3_CRIDATA_SOC',\n",
       " 'PACK3_CRIDATA_SOH',\n",
       " 'PACK4_CRIDATA_AVG_CELL_TEMP',\n",
       " 'PACK4_CRIDATA_AVG_CELL_VOL',\n",
       " 'PACK4_CRIDATA_BATT_VOL',\n",
       " 'PACK4_CRIDATA_BUS_VOL',\n",
       " 'PACK4_CRIDATA_CHARGE_CURR_LIMIT',\n",
       " 'PACK4_CRIDATA_CURR',\n",
       " 'PACK4_CRIDATA_DISCHARGE_CURR_LIMIT',\n",
       " 'PACK4_CRIDATA_SOC',\n",
       " 'PACK4_CRIDATA_SOH',\n",
       " 'PACK5_CRIDATA_AVG_CELL_TEMP',\n",
       " 'PACK5_CRIDATA_AVG_CELL_VOL',\n",
       " 'PACK5_CRIDATA_BATT_VOL',\n",
       " 'PACK5_CRIDATA_BUS_VOL',\n",
       " 'PACK5_CRIDATA_CHARGE_CURR_LIMIT',\n",
       " 'PACK5_CRIDATA_CURR',\n",
       " 'PACK5_CRIDATA_DISCHARGE_CURR_LIMIT',\n",
       " 'PACK5_CRIDATA_SOC',\n",
       " 'PACK5_CRIDATA_SOH',\n",
       " 'PACK6_CRIDATA_AVG_CELL_TEMP',\n",
       " 'PACK6_CRIDATA_AVG_CELL_VOL',\n",
       " 'PACK6_CRIDATA_BATT_VOL',\n",
       " 'PACK6_CRIDATA_BUS_VOL',\n",
       " 'PACK6_CRIDATA_CHARGE_CURR_LIMIT',\n",
       " 'PACK6_CRIDATA_CURR',\n",
       " 'PACK6_CRIDATA_DISCHARGE_CURR_LIMIT',\n",
       " 'PACK6_CRIDATA_SOC',\n",
       " 'PACK6_CRIDATA_SOH',\n",
       " 'PACK7_CRIDATA_AVG_CELL_TEMP',\n",
       " 'PACK7_CRIDATA_AVG_CELL_VOL',\n",
       " 'PACK7_CRIDATA_BATT_VOL',\n",
       " 'PACK7_CRIDATA_BUS_VOL',\n",
       " 'PACK7_CRIDATA_CHARGE_CURR_LIMIT',\n",
       " 'PACK7_CRIDATA_CURR',\n",
       " 'PACK7_CRIDATA_DISCHARGE_CURR_LIMIT',\n",
       " 'PACK7_CRIDATA_SOC',\n",
       " 'PACK7_CRIDATA_SOH',\n",
       " 'PACK_1_STATUS',\n",
       " 'PACK_2_STATUS',\n",
       " 'PACK_3_STATUS',\n",
       " 'PACK_4_STATUS',\n",
       " 'PACK_5_STATUS',\n",
       " 'PACK_6_STATUS',\n",
       " 'PACK_7_STATUS',\n",
       " 'RunningState',\n",
       " 'Voltage',\n",
       " 'valuestreamname']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = df['PACK1_CRIDATA_BATT_VOL']\n",
    "s1 = s.cast(pl.Float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fb2b99838b0>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 19076 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(range(len(s1)), s1, marker='o', s=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 's1' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[24], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m plt\u001b[39m.\u001b[39mscatter(s1\u001b[39m.\u001b[39mfill_null(strategy\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mbackward\u001b[39m\u001b[39m'\u001b[39m))\n",
      "\u001b[0;31mNameError\u001b[0m: name 's1' is not defined"
     ]
    }
   ],
   "source": [
    "plt.scatter(s1.fill_null(strategy='backward'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "size = len(s1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_index = 0\n",
    "# end_index = math.floor(size/2)\n",
    "end_index = 60000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7feea4e44910>]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 14888 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(s1.fill_null(strategy='forward')[100:end_index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7feea49b2680>]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 17504 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "plt.plot(s1.fill_null(strategy='zero')[57700:57720])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Analysis: The voltage drops from 800 to 0 in a very short span of time. I doubt that it is due to the battery discharge, since it quickly resumes to its normal voltage"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What is the time?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-10-19T05:19:00Z\n",
      "2022-10-19T05:20:00Z\n",
      "2022-10-19T05:21:00Z\n"
     ]
    }
   ],
   "source": [
    "print(df['time'][57700])\n",
    "print(df['time'][57701])\n",
    "print(df['time'][57702])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns.index('PACK1_CRIDATA_BATT_VOL')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['',\n",
       " 'time',\n",
       " 'AVG_PACK_SOC',\n",
       " 'AvgSOC',\n",
       " 'AvgSOH',\n",
       " 'BATT_PACK_1_DISCONN',\n",
       " 'BATT_PACK_1_FAULT',\n",
       " 'BATT_PACK_1_WARN',\n",
       " 'BATT_PACK_2_DISCONN',\n",
       " 'BATT_PACK_2_FAULT',\n",
       " 'BATT_PACK_2_WARN',\n",
       " 'BATT_PACK_3_DISCONN',\n",
       " 'BATT_PACK_3_FAULT',\n",
       " 'BATT_PACK_3_WARN',\n",
       " 'BATT_PACK_4_DISCONN',\n",
       " 'BATT_PACK_4_FAULT',\n",
       " 'BATT_PACK_4_WARN',\n",
       " 'BATT_PACK_5_DISCONN',\n",
       " 'BATT_PACK_5_FAULT',\n",
       " 'BATT_PACK_5_WARN',\n",
       " 'BATT_PACK_6_DISCONN',\n",
       " 'BATT_PACK_6_FAULT',\n",
       " 'BATT_PACK_6_WARN',\n",
       " 'BATT_PACK_7_DISCONN',\n",
       " 'BATT_PACK_7_FAULT',\n",
       " 'BATT_PACK_7_WARN',\n",
       " 'BATT_ROOM_TEMP',\n",
       " 'Current',\n",
       " 'MAX_CELL_TEMP',\n",
       " 'MAX_CELL_VOL',\n",
       " 'MAX_PACK_SOC',\n",
       " 'MAX_UNCONN_PACK_VOL',\n",
       " 'MIN_CELL_TEMP',\n",
       " 'MIN_CELL_VOL',\n",
       " 'MIN_PACK_SOC',\n",
       " 'MIN_UNCONN_PACK_VOL',\n",
       " 'NUMBER_OF_PACKS_CONN',\n",
       " 'NUMBER_OF_PACKS_FAULTED',\n",
       " 'PACK1_CRIDATA_AVG_CELL_TEMP',\n",
       " 'PACK1_CRIDATA_AVG_CELL_VOL',\n",
       " 'PACK1_CRIDATA_BATT_VOL',\n",
       " 'PACK1_CRIDATA_BUS_VOL',\n",
       " 'PACK1_CRIDATA_CHARGE_CURR_LIMIT',\n",
       " 'PACK1_CRIDATA_CURR',\n",
       " 'PACK1_CRIDATA_DISCHARGE_CURR_LIMIT',\n",
       " 'PACK1_CRIDATA_SOC',\n",
       " 'PACK1_CRIDATA_SOH',\n",
       " 'PACK2_CRIDATA_AVG_CELL_TEMP',\n",
       " 'PACK2_CRIDATA_AVG_CELL_VOL',\n",
       " 'PACK2_CRIDATA_BATT_VOL',\n",
       " 'PACK2_CRIDATA_BUS_VOL',\n",
       " 'PACK2_CRIDATA_CHARGE_CURR_LIMIT',\n",
       " 'PACK2_CRIDATA_CURR',\n",
       " 'PACK2_CRIDATA_DISCHARGE_CURR_LIMIT',\n",
       " 'PACK2_CRIDATA_SOC',\n",
       " 'PACK2_CRIDATA_SOH',\n",
       " 'PACK3_CRIDATA_AVG_CELL_TEMP',\n",
       " 'PACK3_CRIDATA_AVG_CELL_VOL',\n",
       " 'PACK3_CRIDATA_BATT_VOL',\n",
       " 'PACK3_CRIDATA_BUS_VOL',\n",
       " 'PACK3_CRIDATA_CHARGE_CURR_LIMIT',\n",
       " 'PACK3_CRIDATA_CURR',\n",
       " 'PACK3_CRIDATA_DISCHARGE_CURR_LIMIT',\n",
       " 'PACK3_CRIDATA_SOC',\n",
       " 'PACK3_CRIDATA_SOH',\n",
       " 'PACK4_CRIDATA_AVG_CELL_TEMP',\n",
       " 'PACK4_CRIDATA_AVG_CELL_VOL',\n",
       " 'PACK4_CRIDATA_BATT_VOL',\n",
       " 'PACK4_CRIDATA_BUS_VOL',\n",
       " 'PACK4_CRIDATA_CHARGE_CURR_LIMIT',\n",
       " 'PACK4_CRIDATA_CURR',\n",
       " 'PACK4_CRIDATA_DISCHARGE_CURR_LIMIT',\n",
       " 'PACK4_CRIDATA_SOC',\n",
       " 'PACK4_CRIDATA_SOH',\n",
       " 'PACK5_CRIDATA_AVG_CELL_TEMP',\n",
       " 'PACK5_CRIDATA_AVG_CELL_VOL',\n",
       " 'PACK5_CRIDATA_BATT_VOL',\n",
       " 'PACK5_CRIDATA_BUS_VOL',\n",
       " 'PACK5_CRIDATA_CHARGE_CURR_LIMIT',\n",
       " 'PACK5_CRIDATA_CURR',\n",
       " 'PACK5_CRIDATA_DISCHARGE_CURR_LIMIT',\n",
       " 'PACK5_CRIDATA_SOC',\n",
       " 'PACK5_CRIDATA_SOH',\n",
       " 'PACK6_CRIDATA_AVG_CELL_TEMP',\n",
       " 'PACK6_CRIDATA_AVG_CELL_VOL',\n",
       " 'PACK6_CRIDATA_BATT_VOL',\n",
       " 'PACK6_CRIDATA_BUS_VOL',\n",
       " 'PACK6_CRIDATA_CHARGE_CURR_LIMIT',\n",
       " 'PACK6_CRIDATA_CURR',\n",
       " 'PACK6_CRIDATA_DISCHARGE_CURR_LIMIT',\n",
       " 'PACK6_CRIDATA_SOC',\n",
       " 'PACK6_CRIDATA_SOH',\n",
       " 'PACK7_CRIDATA_AVG_CELL_TEMP',\n",
       " 'PACK7_CRIDATA_AVG_CELL_VOL',\n",
       " 'PACK7_CRIDATA_BATT_VOL',\n",
       " 'PACK7_CRIDATA_BUS_VOL',\n",
       " 'PACK7_CRIDATA_CHARGE_CURR_LIMIT',\n",
       " 'PACK7_CRIDATA_CURR',\n",
       " 'PACK7_CRIDATA_DISCHARGE_CURR_LIMIT',\n",
       " 'PACK7_CRIDATA_SOC',\n",
       " 'PACK7_CRIDATA_SOH',\n",
       " 'PACK_1_STATUS',\n",
       " 'PACK_2_STATUS',\n",
       " 'PACK_3_STATUS',\n",
       " 'PACK_4_STATUS',\n",
       " 'PACK_5_STATUS',\n",
       " 'PACK_6_STATUS',\n",
       " 'PACK_7_STATUS',\n",
       " 'RunningState',\n",
       " 'Voltage',\n",
       " 'valuestreamname']"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['PACK1_CRIDATA_AVG_CELL_TEMP',\n",
       " 'PACK1_CRIDATA_AVG_CELL_VOL',\n",
       " 'PACK1_CRIDATA_BATT_VOL',\n",
       " 'PACK1_CRIDATA_BUS_VOL',\n",
       " 'PACK1_CRIDATA_CHARGE_CURR_LIMIT',\n",
       " 'PACK1_CRIDATA_CURR',\n",
       " 'PACK1_CRIDATA_DISCHARGE_CURR_LIMIT',\n",
       " 'PACK1_CRIDATA_SOC',\n",
       " 'PACK1_CRIDATA_SOH']"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns[38:47]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method DataFrame.melt of shape: (3, 9)\n",
       "┌───────────┬───────────┬───────────┬───────────┬───┬───────────┬───────────┬───────────┬──────────┐\n",
       "│ PACK1_CRI ┆ PACK1_CRI ┆ PACK1_CRI ┆ PACK1_CRI ┆ … ┆ PACK1_CRI ┆ PACK1_CRI ┆ PACK1_CRI ┆ PACK1_CR │\n",
       "│ DATA_AVG_ ┆ DATA_AVG_ ┆ DATA_BATT ┆ DATA_BUS_ ┆   ┆ DATA_CURR ┆ DATA_DISC ┆ DATA_SOC  ┆ IDATA_SO │\n",
       "│ CELL_TEMP ┆ CELL_VOL  ┆ _VOL      ┆ VOL       ┆   ┆ ---       ┆ HARGE_CUR ┆ ---       ┆ H        │\n",
       "│ ---       ┆ ---       ┆ ---       ┆ ---       ┆   ┆ str       ┆ R_LIM…    ┆ str       ┆ ---      │\n",
       "│ str       ┆ str       ┆ str       ┆ str       ┆   ┆           ┆ ---       ┆           ┆ str      │\n",
       "│           ┆           ┆           ┆           ┆   ┆           ┆ str       ┆           ┆          │\n",
       "╞═══════════╪═══════════╪═══════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪══════════╡\n",
       "│ 26.0      ┆ 3961.0    ┆ 904.0     ┆ 0.0       ┆ … ┆ 0.0       ┆ 471.0     ┆ 80.0      ┆ 100.0    │\n",
       "│ 0.0       ┆ 0.0       ┆ 0.0       ┆ 0.0       ┆ … ┆ 0.0       ┆ 0.0       ┆ 0.0       ┆ 0.0      │\n",
       "│ 26.0      ┆ 3961.0    ┆ 904.0     ┆ 0.0       ┆ … ┆ 0.0       ┆ 471.0     ┆ 80.0      ┆ 100.0    │\n",
       "└───────────┴───────────┴───────────┴───────────┴───┴───────────┴───────────┴───────────┴──────────┘>"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[57712:57715, 38:47].melt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method DataFrame.melt of shape: (3, 3)\n",
       "┌─────────────────────┬───────────────────┬──────────────────┐\n",
       "│ BATT_PACK_1_DISCONN ┆ BATT_PACK_1_FAULT ┆ BATT_PACK_1_WARN │\n",
       "│ ---                 ┆ ---               ┆ ---              │\n",
       "│ str                 ┆ str               ┆ str              │\n",
       "╞═════════════════════╪═══════════════════╪══════════════════╡\n",
       "│ True                ┆ False             ┆ False            │\n",
       "│ null                ┆ null              ┆ null             │\n",
       "│ True                ┆ False             ┆ False            │\n",
       "└─────────────────────┴───────────────────┴──────────────────┘>"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[57712:57715, 5:8].melt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Turns out it was just a missing data row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr > th,\n",
       ".dataframe > tbody > tr > td {\n",
       "  text-align: right;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>BATT_PACK_1_DISCONN</th><th>counts</th></tr><tr><td>str</td><td>u32</td></tr></thead><tbody><tr><td>&quot;False&quot;</td><td>150260</td></tr><tr><td>&quot;True&quot;</td><td>51769</td></tr><tr><td>null</td><td>71932</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 2)\n",
       "┌─────────────────────┬────────┐\n",
       "│ BATT_PACK_1_DISCONN ┆ counts │\n",
       "│ ---                 ┆ ---    │\n",
       "│ str                 ┆ u32    │\n",
       "╞═════════════════════╪════════╡\n",
       "│ False               ┆ 150260 │\n",
       "│ True                ┆ 51769  │\n",
       "│ null                ┆ 71932  │\n",
       "└─────────────────────┴────────┘"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['BATT_PACK_1_DISCONN'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A strategy we can employ is to try to convert all 0 values in voltage to their neighbor values (interpolate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Finding anomalies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr > th,\n",
       ".dataframe > tbody > tr > td {\n",
       "  text-align: right;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (5, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>PACK_1_STATUS</th><th>counts</th></tr><tr><td>str</td><td>u32</td></tr></thead><tbody><tr><td>null</td><td>66165</td></tr><tr><td>&quot;1.0&quot;</td><td>56</td></tr><tr><td>&quot;274.0&quot;</td><td>1</td></tr><tr><td>&quot;100.0&quot;</td><td>1</td></tr><tr><td>&quot;0.0&quot;</td><td>207738</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (5, 2)\n",
       "┌───────────────┬────────┐\n",
       "│ PACK_1_STATUS ┆ counts │\n",
       "│ ---           ┆ ---    │\n",
       "│ str           ┆ u32    │\n",
       "╞═══════════════╪════════╡\n",
       "│ null          ┆ 66165  │\n",
       "│ 1.0           ┆ 56     │\n",
       "│ 274.0         ┆ 1      │\n",
       "│ 100.0         ┆ 1      │\n",
       "│ 0.0           ┆ 207738 │\n",
       "└───────────────┴────────┘"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['PACK_1_STATUS'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr > th,\n",
       ".dataframe > tbody > tr > td {\n",
       "  text-align: right;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>BATT_PACK_1_DISCONN</th><th>counts</th></tr><tr><td>str</td><td>u32</td></tr></thead><tbody><tr><td>&quot;False&quot;</td><td>150260</td></tr><tr><td>null</td><td>71932</td></tr><tr><td>&quot;True&quot;</td><td>51769</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 2)\n",
       "┌─────────────────────┬────────┐\n",
       "│ BATT_PACK_1_DISCONN ┆ counts │\n",
       "│ ---                 ┆ ---    │\n",
       "│ str                 ┆ u32    │\n",
       "╞═════════════════════╪════════╡\n",
       "│ False               ┆ 150260 │\n",
       "│ null                ┆ 71932  │\n",
       "│ True                ┆ 51769  │\n",
       "└─────────────────────┴────────┘"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['BATT_PACK_1_DISCONN'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr > th,\n",
       ".dataframe > tbody > tr > td {\n",
       "  text-align: right;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>BATT_PACK_1_FAULT</th><th>counts</th></tr><tr><td>str</td><td>u32</td></tr></thead><tbody><tr><td>&quot;False&quot;</td><td>137042</td></tr><tr><td>null</td><td>136820</td></tr><tr><td>&quot;True&quot;</td><td>99</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 2)\n",
       "┌───────────────────┬────────┐\n",
       "│ BATT_PACK_1_FAULT ┆ counts │\n",
       "│ ---               ┆ ---    │\n",
       "│ str               ┆ u32    │\n",
       "╞═══════════════════╪════════╡\n",
       "│ False             ┆ 137042 │\n",
       "│ null              ┆ 136820 │\n",
       "│ True              ┆ 99     │\n",
       "└───────────────────┴────────┘"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['BATT_PACK_1_FAULT'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr > th,\n",
       ".dataframe > tbody > tr > td {\n",
       "  text-align: right;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>BATT_PACK_1_WARN</th><th>counts</th></tr><tr><td>str</td><td>u32</td></tr></thead><tbody><tr><td>&quot;True&quot;</td><td>148</td></tr><tr><td>null</td><td>131051</td></tr><tr><td>&quot;False&quot;</td><td>142762</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 2)\n",
       "┌──────────────────┬────────┐\n",
       "│ BATT_PACK_1_WARN ┆ counts │\n",
       "│ ---              ┆ ---    │\n",
       "│ str              ┆ u32    │\n",
       "╞══════════════════╪════════╡\n",
       "│ True             ┆ 148    │\n",
       "│ null             ┆ 131051 │\n",
       "│ False            ┆ 142762 │\n",
       "└──────────────────┴────────┘"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['BATT_PACK_1_WARN'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 75680 characters]",
      "text/plain": [
       "<Figure size 1000x1600 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "def convert_values(values):\n",
    "    numerical_values = []\n",
    "    for value in values:\n",
    "        if value == 'False':\n",
    "            numerical_values.append(0)\n",
    "        elif value == 'True':\n",
    "            numerical_values.append(1)\n",
    "        else:\n",
    "            numerical_values.append(-1)\n",
    "    return numerical_values\n",
    "\n",
    "\n",
    "numerical_values1 = convert_values(df['BATT_PACK_1_DISCONN'])\n",
    "numerical_values2 = convert_values(df['BATT_PACK_1_WARN'])\n",
    "numerical_values3 = convert_values(df['BATT_PACK_1_FAULT'])\n",
    "numerical_values4 = df['PACK1_CRIDATA_BATT_VOL'].cast(pl.Float32).fill_null(strategy='backward')\n",
    "\n",
    "fig, axs = plt.subplots(nrows=2, ncols=2, figsize=(10,10))\n",
    "axs[0,0].scatter(range(len(numerical_values1)), numerical_values1)\n",
    "axs[0,0].set_title(\"disconnected\")\n",
    "axs[0,1].scatter(range(len(numerical_values2)), numerical_values2)\n",
    "axs[0,1].set_title(\"warning\")\n",
    "axs[1,0].scatter(range(len(numerical_values3)), numerical_values3)\n",
    "axs[1,0].set_title(\"fault\")\n",
    "axs[1,1].scatter(range(len(numerical_values4)), numerical_values4)\n",
    "axs[1,1].set_title(\"voltage\")\n",
    "\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(800.0, 1000.0)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 78604 characters]",
      "text/plain": [
       "<Figure size 1000x1600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "start = 125000\n",
    "\n",
    "fig, axs = plt.subplots(nrows=2, ncols=1, figsize=(10,16))\n",
    "\n",
    "numerical_values1 = df['PACK1_CRIDATA_BATT_VOL'].cast(pl.Float32).fill_null(strategy='mean')[start:start+100000]\n",
    "numerical_values2 = convert_values(df['BATT_PACK_1_FAULT'])\n",
    "axs[0].scatter(range(len(numerical_values1)), numerical_values1, s=10)\n",
    "axs[1].scatter(range(len(numerical_values2)), numerical_values2, s=10)\n",
    "axs[0].set_ylim(800,1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## charge discharge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fb3381abf10>]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 18348 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(df['PACK1_CRIDATA_CURR'].cast(pl.Float32).fill_null(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'voltage')"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 36332 characters]",
      "text/plain": [
       "<Figure size 1000x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "start = 115000\n",
    "end = start + 10000\n",
    "\n",
    "fig, axs = plt.subplots(nrows=2, ncols=1, figsize=(10,8))\n",
    "\n",
    "numerical_values0 = convert_values(df['BATT_PACK_1_FAULT'][start:end])\n",
    "numerical_values1 = df['PACK1_CRIDATA_BATT_VOL'].cast(pl.Float32).fill_null(-100)[start:end]\n",
    "axs[0].scatter(range(len(numerical_values0)), numerical_values0, s=10)\n",
    "axs[0].set_title(\"fault\")\n",
    "axs[1].scatter(range(len(numerical_values2)), numerical_values2, s=10)\n",
    "axs[1].set_title(\"voltage\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'voltage')"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 20220 characters]",
      "text/plain": [
       "<Figure size 800x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "start = 50000\n",
    "end = start + 100000\n",
    "plt.figure(figsize=(8, 6))\n",
    "numerical_values = df['PACK1_CRIDATA_BATT_VOL'].cast(pl.Float32).fill_null(-100)[start:end]\n",
    "plt.scatter(range(len(numerical_values)), numerical_values, s=10)\n",
    "plt.title(\"voltage\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'voltage')"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 16952 characters]",
      "text/plain": [
       "<Figure size 800x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "start = 189000\n",
    "end = start + 1000\n",
    "plt.figure(figsize=(8, 6))\n",
    "numerical_values = df['PACK1_CRIDATA_BATT_VOL'].cast(pl.Float32).fill_null(-100)[start:end]\n",
    "plt.scatter(range(len(numerical_values)), numerical_values, s=10)\n",
    "plt.title(\"voltage\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'temperature')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 72488 characters]",
      "text/plain": [
       "<Figure size 1000x2000 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "end = start + 10000\n",
    "\n",
    "fig, axs = plt.subplots(nrows=4, ncols=1, figsize=(10,20))\n",
    "\n",
    "numerical_values0 = convert_values(df['BATT_PACK_1_FAULT'][start:end])\n",
    "numerical_values1 = convert_values(df['BATT_PACK_1_DISCONN'][start:end])\n",
    "numerical_values2 = df['PACK1_CRIDATA_BATT_VOL'].cast(pl.Float32).fill_null(-100)[start:end]\n",
    "numerical_values3 = df['PACK1_CRIDATA_AVG_CELL_TEMP'].cast(pl.Float32).fill_null(-100)[start:end]\n",
    "axs[0].scatter(range(len(numerical_values0)), numerical_values0, s=10)\n",
    "axs[0].set_title(\"fault\")\n",
    "axs[1].scatter(range(len(numerical_values1)), numerical_values1, s=10)\n",
    "axs[1].set_title(\"disconnect\")\n",
    "axs[2].scatter(range(len(numerical_values2)), numerical_values2, s=10)\n",
    "axs[2].set_title(\"voltage\")\n",
    "axs[3].scatter(range(len(numerical_values3)), numerical_values3, s=10)\n",
    "axs[3].set_title(\"temperature\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'temperature')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 72488 characters]",
      "text/plain": [
       "<Figure size 1000x2000 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "start = 115000\n",
    "end = start + 10000\n",
    "\n",
    "fig, axs = plt.subplots(nrows=4, ncols=1, figsize=(10,20))\n",
    "\n",
    "numerical_values0 = convert_values(df['BATT_PACK_1_FAULT'][start:end])\n",
    "numerical_values1 = convert_values(df['BATT_PACK_1_DISCONN'][start:end])\n",
    "numerical_values2 = df['PACK1_CRIDATA_BATT_VOL'].cast(pl.Float32).fill_null(-100)[start:end]\n",
    "numerical_values3 = df['PACK1_CRIDATA_AVG_CELL_TEMP'].cast(pl.Float32).fill_null(-100)[start:end]\n",
    "axs[0].scatter(range(len(numerical_values0)), numerical_values0, s=10)\n",
    "axs[0].set_title(\"fault\")\n",
    "axs[1].scatter(range(len(numerical_values1)), numerical_values1, s=10)\n",
    "axs[1].set_title(\"disconnect\")\n",
    "axs[2].scatter(range(len(numerical_values2)), numerical_values2, s=10)\n",
    "axs[2].set_title(\"voltage\")\n",
    "axs[3].scatter(range(len(numerical_values3)), numerical_values3, s=10)\n",
    "axs[3].set_title(\"temperature\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## temperature?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fb2b58497e0>]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 19772 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "plt.plot(df['PACK1_CRIDATA_AVG_CELL_TEMP'].cast(pl.Float32).fill_null(strategy='mean'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(800.0, 1000.0)"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 45660 characters]",
      "text/plain": [
       "<Figure size 1000x1600 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "start = 50000 + 7500\n",
    "end = start + 1000\n",
    "\n",
    "fig, axs = plt.subplots(nrows=3, ncols=1, figsize=(10,16))\n",
    "\n",
    "numerical_values1 = df['PACK1_CRIDATA_BATT_VOL'].cast(pl.Float32).fill_null(strategy='mean')[start:end]\n",
    "numerical_values2 = convert_values(df['BATT_PACK_1_FAULT'])[start:end]\n",
    "numerical_values3 = df['PACK1_CRIDATA_AVG_CELL_TEMP'].cast(pl.Float32).fill_null(strategy='mean')[start:end]\n",
    "axs[0].scatter(range(len(numerical_values1)), numerical_values1, s=10)\n",
    "axs[1].scatter(range(len(numerical_values2)), numerical_values2, s=10)\n",
    "axs[2].scatter(range(len(numerical_values3)), numerical_values3, s=10)\n",
    "axs[0].set_ylim(800,1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SOH and SOC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fb2b5a8e8c0>"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 27464 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "series = df['PACK1_CRIDATA_SOC'].cast(pl.Float32).fill_null(strategy='mean')\n",
    "plt.scatter(range(len(series)), series, s=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fb2aec98ac0>"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 16952 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "series = df['PACK1_CRIDATA_SOH'].cast(pl.Float32).fill_null(strategy='mean')\n",
    "plt.scatter(range(len(series)), series, s=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We need to remove outlier data points (e.g. 4000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## find discharge zones"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The idea is to have 2 moving average windows, and then find if the moving window ahead is lower than the moving window before."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "window_size = 10\n",
    "series = df['PACK1_CRIDATA_SOC'].cast(pl.Float32).fill_null(strategy='mean')\n",
    "moving_avg = series.rolling_mean(window_size=window_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape: (9, 2)\n",
      "┌────────────┬───────────┐\n",
      "│ statistic  ┆ value     │\n",
      "│ ---        ┆ ---       │\n",
      "│ str        ┆ f64       │\n",
      "╞════════════╪═══════════╡\n",
      "│ count      ┆ 273961.0  │\n",
      "│ null_count ┆ 9.0       │\n",
      "│ mean       ┆ 80.893529 │\n",
      "│ std        ┆ 13.902372 │\n",
      "│ min        ┆ 0.0       │\n",
      "│ 25%        ┆ 80.893616 │\n",
      "│ 50%        ┆ 85.0      │\n",
      "│ 75%        ┆ 85.0      │\n",
      "│ max        ┆ 88.0      │\n",
      "└────────────┴───────────┘\n",
      "shape: (9, 2)\n",
      "┌────────────┬───────────┐\n",
      "│ statistic  ┆ value     │\n",
      "│ ---        ┆ ---       │\n",
      "│ str        ┆ f64       │\n",
      "╞════════════╪═══════════╡\n",
      "│ count      ┆ 273961.0  │\n",
      "│ null_count ┆ 0.0       │\n",
      "│ mean       ┆ 80.893613 │\n",
      "│ std        ┆ 13.989593 │\n",
      "│ min        ┆ -0.299927 │\n",
      "│ 25%        ┆ 80.893616 │\n",
      "│ 50%        ┆ 85.0      │\n",
      "│ 75%        ┆ 85.0      │\n",
      "│ max        ┆ 100.0     │\n",
      "└────────────┴───────────┘\n"
     ]
    }
   ],
   "source": [
    "print(moving_avg.describe())\n",
    "\n",
    "print(series.describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fb2ac34ead0>"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 36876 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(range(len(moving_avg)),moving_avg, s=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fb2ac872440>"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 13684 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "subset = moving_avg[54500:55000]\n",
    "plt.scatter(range(len(subset)),subset, s=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see that we get artifacts of the moving averages because the data is not clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(800.0, 1000.0)"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 58204 characters]",
      "text/plain": [
       "<Figure size 1000x1600 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "start = 54500\n",
    "end = start + 5000\n",
    "\n",
    "fig, axs = plt.subplots(nrows=4, ncols=1, figsize=(10,16))\n",
    "numerical_values1 = df['PACK1_CRIDATA_BATT_VOL'].cast(pl.Float32).fill_null(strategy='mean')[start:end]\n",
    "numerical_values2 = convert_values(df['BATT_PACK_1_FAULT'])[start:end]\n",
    "numerical_values3 = df['PACK1_CRIDATA_AVG_CELL_TEMP'].cast(pl.Float32).fill_null(strategy='mean')[start:end]\n",
    "numerical_values4 = df['PACK1_CRIDATA_SOC'].cast(pl.Float32).fill_null(strategy='mean')[start:end]\n",
    "axs[0].scatter(range(len(numerical_values1)), numerical_values1, s=10)\n",
    "axs[1].scatter(range(len(numerical_values2)), numerical_values2, s=10)\n",
    "axs[2].scatter(range(len(numerical_values3)), numerical_values3, s=10)\n",
    "axs[3].scatter(range(len(numerical_values4)), numerical_values4, s=10)\n",
    "axs[0].set_ylim(800,1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fb2ac7adb40>"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 32720 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "series = df['PACK1_CRIDATA_SOC'].cast(pl.Float32).fill_null(strategy='mean')\n",
    "plt.scatter(range(len(series)), series)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fb2abbdf640>"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 15448 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# reset df\n",
    "series = df['PACK1_CRIDATA_SOC'].cast(pl.Float32).fill_null(-10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fb2ac59bd30>"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 25740 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# start = 50000\n",
    "# end = start + 10000\n",
    "start = 0\n",
    "end = len(df)\n",
    "window = series[start:end]\n",
    "plt.scatter(range(len(window)), window, s=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fb2abf70a60>"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 12908 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "start = 70400\n",
    "end = start + 500\n",
    "window = series[start:end]\n",
    "plt.scatter(range(len(window)), window, s=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Good news is that the data makes sense. It discharges in about 40 minutes, then charges back up in the same amount of time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fb2abe4d300>"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 12792 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "start = 50000\n",
    "end = start + 10000\n",
    "window = series[start:end]\n",
    "plt.scatter(range(len(window)), window, s=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## strategy\n",
    "\n",
    "We are going to identify periods of discharge/charge, then try to interpolate missing data in those intervals\n",
    "\n",
    "Large periods of missing data usually correspond to idle times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "window_size = 10\n",
    "# moving_avg = (df['PACK1_CRIDATA_SOC'].cast(pl.Float32)\n",
    "#                                      .fill_null(-10)\n",
    "#                                      .rolling_mean(window_size=window_size)\n",
    "#                                      .fill_null(-10))\n",
    "moving_avg = df['PACK1_CRIDATA_SOC'].cast(pl.Float32).fill_null(-10)\n",
    "\n",
    "# moving_std = (df['PACK1_CRIDATA_SOC'].cast(pl.Float32)\n",
    "#                                     .fill_null(-10)\n",
    "#                                     .rolling_std(window_size=window_size)\n",
    "#                                     .fill_null(10))\n",
    "\n",
    "downward_segments = []\n",
    "threshold_long = 10\n",
    "threshold_short = 0.1\n",
    "# smoothness = 3.0\n",
    "segment_start = None\n",
    "\n",
    "\n",
    "for i in range(len(df)):\n",
    "    if ((moving_avg[i] - moving_avg[i - 1] > threshold_short) and \n",
    "        (moving_avg[i] - moving_avg[i - 1] < threshold_long)):\n",
    "        # activate flag upon first contact\n",
    "        if segment_start is None:\n",
    "            segment_start = i - window_size + 1\n",
    "    # at end of downward movement, reset flag\n",
    "    elif segment_start is not None:\n",
    "        segment_end = i\n",
    "        downward_segments.append((segment_start, segment_end))\n",
    "        segment_start = None\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reset df\n",
    "series = df['PACK1_CRIDATA_SOC'].cast(pl.Float32).fill_null(-10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [],
   "source": [
    "time = range(len(df))\n",
    "for start, end in downward_segments:\n",
    "    plt.fill_betweenx(series, start, end, color='green', alpha=0.3)\n",
    "plt.xlim(0, len(df))\n",
    "plt.scatter(range(len(series)), series)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7f6dbf4bc970>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 17452 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "start_time = 70000 + 400; end_time = start_time + 500\n",
    "sampled_segments = [(segment_start, segment_end) for segment_start, segment_end in downward_segments if ((segment_start > start_time) and (segment_end < end_time))]\n",
    "for start, end in sampled_segments:\n",
    "    plt.fill_betweenx(series[start_time:end_time], start, end, color='green', alpha=0.3)\n",
    "# plt.xlim(0, len(df[start:end]))\n",
    "plt.scatter(range(start_time,end_time), series[start_time:end_time])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sampled_segments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# What are faults associated with?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import polars as pl\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from io import StringIO\n",
    "import math\n",
    "df = pl.read_csv('output.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['',\n",
       " 'time',\n",
       " 'AVG_PACK_SOC',\n",
       " 'AvgSOC',\n",
       " 'AvgSOH',\n",
       " 'BATT_PACK_1_DISCONN',\n",
       " 'BATT_PACK_1_FAULT',\n",
       " 'BATT_PACK_1_WARN',\n",
       " 'BATT_PACK_2_DISCONN',\n",
       " 'BATT_PACK_2_FAULT',\n",
       " 'BATT_PACK_2_WARN',\n",
       " 'BATT_PACK_3_DISCONN',\n",
       " 'BATT_PACK_3_FAULT',\n",
       " 'BATT_PACK_3_WARN',\n",
       " 'BATT_PACK_4_DISCONN',\n",
       " 'BATT_PACK_4_FAULT',\n",
       " 'BATT_PACK_4_WARN',\n",
       " 'BATT_PACK_5_DISCONN',\n",
       " 'BATT_PACK_5_FAULT',\n",
       " 'BATT_PACK_5_WARN',\n",
       " 'BATT_PACK_6_DISCONN',\n",
       " 'BATT_PACK_6_FAULT',\n",
       " 'BATT_PACK_6_WARN',\n",
       " 'BATT_PACK_7_DISCONN',\n",
       " 'BATT_PACK_7_FAULT',\n",
       " 'BATT_PACK_7_WARN',\n",
       " 'BATT_ROOM_TEMP',\n",
       " 'Current',\n",
       " 'MAX_CELL_TEMP',\n",
       " 'MAX_CELL_VOL',\n",
       " 'MAX_PACK_SOC',\n",
       " 'MAX_UNCONN_PACK_VOL',\n",
       " 'MIN_CELL_TEMP',\n",
       " 'MIN_CELL_VOL',\n",
       " 'MIN_PACK_SOC',\n",
       " 'MIN_UNCONN_PACK_VOL',\n",
       " 'NUMBER_OF_PACKS_CONN',\n",
       " 'NUMBER_OF_PACKS_FAULTED',\n",
       " 'PACK1_CRIDATA_AVG_CELL_TEMP',\n",
       " 'PACK1_CRIDATA_AVG_CELL_VOL',\n",
       " 'PACK1_CRIDATA_BATT_VOL',\n",
       " 'PACK1_CRIDATA_BUS_VOL',\n",
       " 'PACK1_CRIDATA_CHARGE_CURR_LIMIT',\n",
       " 'PACK1_CRIDATA_CURR',\n",
       " 'PACK1_CRIDATA_DISCHARGE_CURR_LIMIT',\n",
       " 'PACK1_CRIDATA_SOC',\n",
       " 'PACK1_CRIDATA_SOH',\n",
       " 'PACK2_CRIDATA_AVG_CELL_TEMP',\n",
       " 'PACK2_CRIDATA_AVG_CELL_VOL',\n",
       " 'PACK2_CRIDATA_BATT_VOL',\n",
       " 'PACK2_CRIDATA_BUS_VOL',\n",
       " 'PACK2_CRIDATA_CHARGE_CURR_LIMIT',\n",
       " 'PACK2_CRIDATA_CURR',\n",
       " 'PACK2_CRIDATA_DISCHARGE_CURR_LIMIT',\n",
       " 'PACK2_CRIDATA_SOC',\n",
       " 'PACK2_CRIDATA_SOH',\n",
       " 'PACK3_CRIDATA_AVG_CELL_TEMP',\n",
       " 'PACK3_CRIDATA_AVG_CELL_VOL',\n",
       " 'PACK3_CRIDATA_BATT_VOL',\n",
       " 'PACK3_CRIDATA_BUS_VOL',\n",
       " 'PACK3_CRIDATA_CHARGE_CURR_LIMIT',\n",
       " 'PACK3_CRIDATA_CURR',\n",
       " 'PACK3_CRIDATA_DISCHARGE_CURR_LIMIT',\n",
       " 'PACK3_CRIDATA_SOC',\n",
       " 'PACK3_CRIDATA_SOH',\n",
       " 'PACK4_CRIDATA_AVG_CELL_TEMP',\n",
       " 'PACK4_CRIDATA_AVG_CELL_VOL',\n",
       " 'PACK4_CRIDATA_BATT_VOL',\n",
       " 'PACK4_CRIDATA_BUS_VOL',\n",
       " 'PACK4_CRIDATA_CHARGE_CURR_LIMIT',\n",
       " 'PACK4_CRIDATA_CURR',\n",
       " 'PACK4_CRIDATA_DISCHARGE_CURR_LIMIT',\n",
       " 'PACK4_CRIDATA_SOC',\n",
       " 'PACK4_CRIDATA_SOH',\n",
       " 'PACK5_CRIDATA_AVG_CELL_TEMP',\n",
       " 'PACK5_CRIDATA_AVG_CELL_VOL',\n",
       " 'PACK5_CRIDATA_BATT_VOL',\n",
       " 'PACK5_CRIDATA_BUS_VOL',\n",
       " 'PACK5_CRIDATA_CHARGE_CURR_LIMIT',\n",
       " 'PACK5_CRIDATA_CURR',\n",
       " 'PACK5_CRIDATA_DISCHARGE_CURR_LIMIT',\n",
       " 'PACK5_CRIDATA_SOC',\n",
       " 'PACK5_CRIDATA_SOH',\n",
       " 'PACK6_CRIDATA_AVG_CELL_TEMP',\n",
       " 'PACK6_CRIDATA_AVG_CELL_VOL',\n",
       " 'PACK6_CRIDATA_BATT_VOL',\n",
       " 'PACK6_CRIDATA_BUS_VOL',\n",
       " 'PACK6_CRIDATA_CHARGE_CURR_LIMIT',\n",
       " 'PACK6_CRIDATA_CURR',\n",
       " 'PACK6_CRIDATA_DISCHARGE_CURR_LIMIT',\n",
       " 'PACK6_CRIDATA_SOC',\n",
       " 'PACK6_CRIDATA_SOH',\n",
       " 'PACK7_CRIDATA_AVG_CELL_TEMP',\n",
       " 'PACK7_CRIDATA_AVG_CELL_VOL',\n",
       " 'PACK7_CRIDATA_BATT_VOL',\n",
       " 'PACK7_CRIDATA_BUS_VOL',\n",
       " 'PACK7_CRIDATA_CHARGE_CURR_LIMIT',\n",
       " 'PACK7_CRIDATA_CURR',\n",
       " 'PACK7_CRIDATA_DISCHARGE_CURR_LIMIT',\n",
       " 'PACK7_CRIDATA_SOC',\n",
       " 'PACK7_CRIDATA_SOH',\n",
       " 'PACK_1_STATUS',\n",
       " 'PACK_2_STATUS',\n",
       " 'PACK_3_STATUS',\n",
       " 'PACK_4_STATUS',\n",
       " 'PACK_5_STATUS',\n",
       " 'PACK_6_STATUS',\n",
       " 'PACK_7_STATUS',\n",
       " 'RunningState',\n",
       " 'Voltage',\n",
       " 'valuestreamname']"
      ]
     },
     "execution_count": 209,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_values(values):\n",
    "    numerical_values = []\n",
    "    for value in values:\n",
    "        if value == 'False':\n",
    "            numerical_values.append(0)\n",
    "        elif value == 'True':\n",
    "            numerical_values.append(1)\n",
    "        else:\n",
    "            # numerical_values.append(np.nan)\n",
    "            numerical_values.append(-1)\n",
    "    return numerical_values\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fe32e9c24d0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 16872 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fault = convert_values(df['BATT_PACK_1_FAULT'])\n",
    "plt.scatter(range(len(fault)), fault)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f6d74a9b010>]"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 23028 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(df['Voltage'].cast(pl.Float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We see that faults are constrained to between 50000 and 150000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## overview"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'batt room temp')"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 181256 characters]",
      "text/plain": [
       "<Figure size 1000x3000 with 10 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# start = 115000\n",
    "# end = 140000\n",
    "start = 0\n",
    "end = len(df)\n",
    "#\n",
    "\n",
    "num_rows = 10\n",
    "fig, axs = plt.subplots(nrows=num_rows, ncols=1, figsize=(10,3 * num_rows))\n",
    "\n",
    "numerical_values0 = convert_values(df['BATT_PACK_1_FAULT'][start:end])\n",
    "numerical_values1 = df['PACK1_CRIDATA_BATT_VOL'].cast(pl.Float32).fill_null(-100)[start:end]\n",
    "numerical_values2 = df['PACK1_CRIDATA_AVG_CELL_VOL'].cast(pl.Float32).fill_null(-100)[start:end]\n",
    "numerical_values3 = df['PACK1_CRIDATA_AVG_CELL_TEMP'].cast(pl.Float32).fill_null(-100)[start:end]\n",
    "numerical_values4 = df['PACK1_CRIDATA_BUS_VOL'].cast(pl.Float32).fill_null(-100)[start:end]\n",
    "numerical_values5 = df['PACK1_CRIDATA_CHARGE_CURR_LIMIT'].cast(pl.Float32).fill_null(-100)[start:end]\n",
    "numerical_values6 = df['PACK1_CRIDATA_CURR'].cast(pl.Float32).fill_null(-100)[start:end]\n",
    "numerical_values7 = df['PACK1_CRIDATA_DISCHARGE_CURR_LIMIT'].cast(pl.Float32).fill_null(-100)[start:end]\n",
    "numerical_values8 = df['PACK1_CRIDATA_SOC'].cast(pl.Float32).fill_null(-100)[start:end]\n",
    "numerical_values9 = df['BATT_ROOM_TEMP'].cast(pl.Float32).fill_null(-100)[start:end]\n",
    "axs[0].scatter(range(len(numerical_values0)), numerical_values0, s=10)\n",
    "axs[0].set_title(\"fault\")\n",
    "axs[1].scatter(range(len(numerical_values1)), numerical_values1, s=10)\n",
    "axs[1].set_title(\"voltage\")\n",
    "axs[1].set_ylim(600,1000)\n",
    "\n",
    "axs[2].scatter(range(len(numerical_values2)), numerical_values2, s=10)\n",
    "axs[2].set_title(\"avg cell vol\")\n",
    "\n",
    "axs[3].scatter(range(len(numerical_values3)), numerical_values3, s=10)\n",
    "axs[3].set_title(\"avg cell temp\")\n",
    "axs[3].set_ylim(-20, 100)\n",
    "\n",
    "axs[4].scatter(range(len(numerical_values4)), numerical_values4, s=10)\n",
    "axs[4].set_title(\"bus vol\")\n",
    "\n",
    "axs[5].scatter(range(len(numerical_values5)), numerical_values5, s=10)\n",
    "axs[5].set_title(\"charge curr limit\")\n",
    "\n",
    "axs[6].scatter(range(len(numerical_values6)), numerical_values6, s=10)\n",
    "axs[6].set_title(\"curr\")\n",
    "\n",
    "axs[7].scatter(range(len(numerical_values7)), numerical_values7, s=10)\n",
    "axs[7].set_title(\"discharge curr limit\")\n",
    "\n",
    "axs[8].scatter(range(len(numerical_values8)), numerical_values8, s=10)\n",
    "axs[8].set_title(\"soc\")\n",
    "\n",
    "axs[9].scatter(range(len(numerical_values9)), numerical_values9, s=10)\n",
    "axs[9].set_title(\"batt room temp\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'batt room temp')"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 117592 characters]",
      "text/plain": [
       "<Figure size 1000x2100 with 7 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "start = 115000\n",
    "end = 140000\n",
    "# start = 0\n",
    "# end = len(df)\n",
    "\n",
    "num_rows = 7\n",
    "fig, axs = plt.subplots(nrows=num_rows, ncols=1, figsize=(10,3 * num_rows))\n",
    "\n",
    "numerical_values0 = convert_values(df['BATT_PACK_1_FAULT'][start:end])\n",
    "numerical_values1 = convert_values(df['BATT_PACK_1_WARN'][start:end])\n",
    "numerical_values2 = df['Current'].cast(pl.Float32).fill_null(-100)[start:end]\n",
    "numerical_values3 = df['PACK1_CRIDATA_AVG_CELL_TEMP'].cast(pl.Float32).fill_null(-100)[start:end]\n",
    "numerical_values6 = df['PACK1_CRIDATA_CURR'].cast(pl.Float32).fill_null(-100)[start:end]\n",
    "numerical_values8 = df['PACK1_CRIDATA_SOC'].cast(pl.Float32).fill_null(-100)[start:end]\n",
    "numerical_values9 = df['BATT_ROOM_TEMP'].cast(pl.Float32).fill_null(-100)[start:end]\n",
    "axs[0].scatter(range(len(numerical_values0)), numerical_values0, s=10)\n",
    "axs[0].set_title(\"fault\")\n",
    "\n",
    "axs[1].scatter(range(len(numerical_values1)), numerical_values1, s=10)\n",
    "axs[1].set_title(\"warn\")\n",
    "\n",
    "axs[2].scatter(range(len(numerical_values2)), numerical_values2, s=10)\n",
    "axs[2].set_title(\"current\")\n",
    "axs[2].set_ylim(-20, 30)\n",
    "\n",
    "axs[3].scatter(range(len(numerical_values3)), numerical_values3, s=10)\n",
    "axs[3].set_title(\"avg cell temp\")\n",
    "axs[3].set_ylim(0, 30)\n",
    "\n",
    "axs[4].scatter(range(len(numerical_values6)), numerical_values6, s=10)\n",
    "axs[4].set_title(\"pack 1 curr\")\n",
    "axs[4].set_ylim(-5, 5)\n",
    "\n",
    "axs[5].scatter(range(len(numerical_values8)), numerical_values8, s=10)\n",
    "axs[5].set_title(\"soc\")\n",
    "\n",
    "axs[6].scatter(range(len(numerical_values9)), numerical_values9, s=10)\n",
    "axs[6].set_title(\"batt room temp\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr > th,\n",
       ".dataframe > tbody > tr > td {\n",
       "  text-align: right;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (9, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>statistic</th><th>value</th></tr><tr><td>str</td><td>f64</td></tr></thead><tbody><tr><td>&quot;count&quot;</td><td>1000.0</td></tr><tr><td>&quot;null_count&quot;</td><td>0.0</td></tr><tr><td>&quot;mean&quot;</td><td>0.190974</td></tr><tr><td>&quot;std&quot;</td><td>0.914956</td></tr><tr><td>&quot;min&quot;</td><td>-1.19995</td></tr><tr><td>&quot;25%&quot;</td><td>-0.699951</td></tr><tr><td>&quot;50%&quot;</td><td>0.0</td></tr><tr><td>&quot;75%&quot;</td><td>1.09998</td></tr><tr><td>&quot;max&quot;</td><td>2.09998</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (9, 2)\n",
       "┌────────────┬───────────┐\n",
       "│ statistic  ┆ value     │\n",
       "│ ---        ┆ ---       │\n",
       "│ str        ┆ f64       │\n",
       "╞════════════╪═══════════╡\n",
       "│ count      ┆ 1000.0    │\n",
       "│ null_count ┆ 0.0       │\n",
       "│ mean       ┆ 0.190974  │\n",
       "│ std        ┆ 0.914956  │\n",
       "│ min        ┆ -1.19995  │\n",
       "│ 25%        ┆ -0.699951 │\n",
       "│ 50%        ┆ 0.0       │\n",
       "│ 75%        ┆ 1.09998   │\n",
       "│ max        ┆ 2.09998   │\n",
       "└────────────┴───────────┘"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start = 50000 + 70000 + 3000 \n",
    "end = start + 1000\n",
    "\n",
    "df['PACK1_CRIDATA_CURR'].cast(pl.Float32)[start:end].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr > th,\n",
       ".dataframe > tbody > tr > td {\n",
       "  text-align: right;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 2)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>statistic</th><th>value</th></tr><tr><td>str</td><td>i64</td></tr></thead><tbody><tr><td>&quot;count&quot;</td><td>273961</td></tr><tr><td>&quot;null_count&quot;</td><td>6346</td></tr><tr><td>&quot;sum&quot;</td><td>158353</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 2)\n",
       "┌────────────┬────────┐\n",
       "│ statistic  ┆ value  │\n",
       "│ ---        ┆ ---    │\n",
       "│ str        ┆ i64    │\n",
       "╞════════════╪════════╡\n",
       "│ count      ┆ 273961 │\n",
       "│ null_count ┆ 6346   │\n",
       "│ sum        ┆ 158353 │\n",
       "└────────────┴────────┘"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df.columns.index('PACK1_CRIDATA_BATT_VOL')\n",
    "# df.columns.index('PACK1_CRIDATA_CURR')\n",
    "df['RunningState'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'warn 2')"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 47108 characters]",
      "text/plain": [
       "<Figure size 1000x1200 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "start = 50000 + 70000 + 3000\n",
    "end = start + 1000\n",
    "\n",
    "num_rows = 4\n",
    "fig, axs = plt.subplots(nrows=num_rows, ncols=1, figsize=(10,3 * num_rows))\n",
    "\n",
    "numerical_values0 = convert_values(df['BATT_PACK_1_FAULT'][start:end])\n",
    "numerical_values1 = convert_values(df['BATT_PACK_1_WARN'][start:end])\n",
    "\n",
    "numerical_values2 = convert_values(df['BATT_PACK_7_FAULT'][start:end])\n",
    "numerical_values3 = convert_values(df['BATT_PACK_7_WARN'][start:end])\n",
    "\n",
    "axs[0].scatter(range(len(numerical_values0)), numerical_values0, s=10)\n",
    "axs[0].set_title(\"fault 1\")\n",
    "\n",
    "axs[1].scatter(range(len(numerical_values1)), numerical_values1, s=10)\n",
    "axs[1].set_title(\"warn 1\")\n",
    "\n",
    "axs[2].scatter(range(len(numerical_values2)), numerical_values2, s=10)\n",
    "axs[2].set_title(\"fault 2\")\n",
    "\n",
    "axs[3].scatter(range(len(numerical_values3)), numerical_values3, s=10)\n",
    "axs[3].set_title(\"warn 2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7f6d7ae55ae0>"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 18440 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(df['PACK1_CRIDATA_CURR'].cast(pl.Float64).fill_null(0))\n",
    "plt.scatter(range(len(numerical_values0)), numerical_values0, color=\"r\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'voltage')"
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 22784 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "stepper = 0\n",
    "start = 116300 + stepper\n",
    "end = start + 100 + stepper\n",
    "# start = 0\n",
    "# end = len(df)\n",
    "#\n",
    "\n",
    "num_rows = 2\n",
    "fig, axs = plt.subplots(nrows=num_rows, ncols=1, figsize=(10,3 * num_rows))\n",
    "\n",
    "numerical_values0 = convert_values(df['BATT_PACK_1_FAULT'][start:end])\n",
    "numerical_values1 = df['PACK1_CRIDATA_SOH'].cast(pl.Float32).fill_null(-100)[start:end]\n",
    "axs[0].scatter(range(len(numerical_values0)), numerical_values0, s=10)\n",
    "axs[0].set_title(\"fault\")\n",
    "axs[1].scatter(range(len(numerical_values1)), numerical_values1, s=10)\n",
    "axs[1].set_title(\"voltage\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# identify train and test regions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import polars as pl\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from io import StringIO\n",
    "import math\n",
    "df = pl.read_csv('output.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_values(values):\n",
    "    numerical_values = []\n",
    "    for value in values:\n",
    "        if value == 'False':\n",
    "            numerical_values.append(0)\n",
    "        elif value == 'True':\n",
    "            numerical_values.append(1)\n",
    "        else:\n",
    "            # numerical_values.append(np.nan)\n",
    "            numerical_values.append(-1)\n",
    "    return numerical_values\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fe32e9c24d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 16872 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fault = convert_values(df['BATT_PACK_1_FAULT'])\n",
    "plt.scatter(range(len(fault)), fault)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fe3301f9f30>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 12764 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "start = 170000\n",
    "end = len(df)\n",
    "window = fault[start:end]\n",
    "plt.scatter(range(len(window)), window)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'batt room temp')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 182076 characters]",
      "text/plain": [
       "<Figure size 1000x3000 with 10 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# start = 115000\n",
    "# end = 140000\n",
    "start = 170000\n",
    "end = len(df)\n",
    "#\n",
    "\n",
    "num_rows = 10\n",
    "fig, axs = plt.subplots(nrows=num_rows, ncols=1, figsize=(10,3 * num_rows))\n",
    "\n",
    "numerical_values0 = convert_values(df['BATT_PACK_1_FAULT'][start:end])\n",
    "numerical_values1 = df['PACK1_CRIDATA_BATT_VOL'].cast(pl.Float32).fill_null(strategy=\"mean\")[start:end]\n",
    "numerical_values2 = df['PACK1_CRIDATA_AVG_CELL_VOL'].cast(pl.Float32).fill_null(strategy=\"mean\")[start:end]\n",
    "numerical_values3 = df['PACK1_CRIDATA_AVG_CELL_TEMP'].cast(pl.Float32).fill_null(strategy=\"mean\")[start:end]\n",
    "numerical_values4 = df['PACK1_CRIDATA_BUS_VOL'].cast(pl.Float32).fill_null(strategy=\"mean\")[start:end]\n",
    "numerical_values5 = df['PACK1_CRIDATA_CHARGE_CURR_LIMIT'].cast(pl.Float32).fill_null(strategy=\"mean\")[start:end]\n",
    "numerical_values6 = df['PACK1_CRIDATA_CURR'].cast(pl.Float32).fill_null(strategy=\"mean\")[start:end]\n",
    "numerical_values7 = df['PACK1_CRIDATA_DISCHARGE_CURR_LIMIT'].cast(pl.Float32).fill_null(strategy=\"mean\")[start:end]\n",
    "numerical_values8 = df['PACK1_CRIDATA_SOC'].cast(pl.Float32).fill_null(strategy=\"mean\")[start:end]\n",
    "numerical_values9 = df['BATT_ROOM_TEMP'].cast(pl.Float32).fill_null(strategy=\"mean\")[start:end]\n",
    "axs[0].scatter(range(len(numerical_values0)), numerical_values0, s=10)\n",
    "axs[0].set_title(\"fault\")\n",
    "axs[1].scatter(range(len(numerical_values1)), numerical_values1, s=10)\n",
    "axs[1].set_title(\"voltage\")\n",
    "axs[1].set_ylim(600,1000)\n",
    "\n",
    "axs[2].scatter(range(len(numerical_values2)), numerical_values2, s=10)\n",
    "axs[2].set_title(\"avg cell vol\")\n",
    "\n",
    "axs[3].scatter(range(len(numerical_values3)), numerical_values3, s=10)\n",
    "axs[3].set_title(\"avg cell temp\")\n",
    "axs[3].set_ylim(-20, 100)\n",
    "\n",
    "axs[4].scatter(range(len(numerical_values4)), numerical_values4, s=10)\n",
    "axs[4].set_title(\"bus vol\")\n",
    "\n",
    "axs[5].scatter(range(len(numerical_values5)), numerical_values5, s=10)\n",
    "axs[5].set_title(\"charge curr limit\")\n",
    "\n",
    "axs[6].scatter(range(len(numerical_values6)), numerical_values6, s=10)\n",
    "axs[6].set_title(\"curr\")\n",
    "\n",
    "axs[7].scatter(range(len(numerical_values7)), numerical_values7, s=10)\n",
    "axs[7].set_title(\"discharge curr limit\")\n",
    "\n",
    "axs[8].scatter(range(len(numerical_values8)), numerical_values8, s=10)\n",
    "axs[8].set_title(\"soc\")\n",
    "\n",
    "axs[9].scatter(range(len(numerical_values9)), numerical_values9, s=10)\n",
    "axs[9].set_title(\"batt room temp\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "we can reasonably assume that the data in this interval is without anomalies\n",
    "\n",
    "The issue lies in removing the \"0\" data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'temp with zeros')"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 57848 characters]",
      "text/plain": [
       "<Figure size 1000x1200 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# start = 170000\n",
    "# end = len(df)\n",
    "start = 0\n",
    "\n",
    "end = len(df)\n",
    "\n",
    "num_rows = 4\n",
    "\n",
    "fig, axs = plt.subplots(nrows=num_rows, ncols=1, figsize=(10,3 * num_rows))\n",
    "\n",
    "\n",
    "numerical_values0 = convert_values((df['BATT_PACK_1_FAULT']\n",
    "                                    .filter(df['PACK1_CRIDATA_BATT_VOL'].cast(pl.Float32) != 0))\n",
    "                                    [start:end])\n",
    "numerical_values1 = (df['PACK1_CRIDATA_BATT_VOL']\n",
    "                     .cast(pl.Float32)\n",
    "                     .fill_null(strategy=\"mean\")\n",
    "                     .filter(df['PACK1_CRIDATA_BATT_VOL'].cast(pl.Float32) != 0))[start:end]\n",
    "\n",
    "numerical_values2 = (df['PACK1_CRIDATA_AVG_CELL_TEMP']\n",
    "                     .cast(pl.Float32)\n",
    "                     .fill_null(strategy=\"mean\")\n",
    "                     .filter(df['PACK1_CRIDATA_BATT_VOL'].cast(pl.Float32) != 0))[start:end]\n",
    "\n",
    "\n",
    "numerical_values3 = df['PACK1_CRIDATA_AVG_CELL_TEMP'].cast(pl.Float32).fill_null(-100)[start:end]\n",
    "\n",
    "axs[0].scatter(range(len(numerical_values0)), numerical_values0, s=10)\n",
    "axs[0].set_title(\"fault\")\n",
    "\n",
    "axs[1].scatter(range(len(numerical_values1)), numerical_values1, s=10)\n",
    "axs[1].set_title(\"voltage\")\n",
    "\n",
    "axs[2].scatter(range(len(numerical_values2)), numerical_values2, s=10)\n",
    "axs[2].set_title(\"temp\")\n",
    "\n",
    "axs[3].scatter(range(len(numerical_values3)), numerical_values3, s=10)\n",
    "axs[3].set_title(\"temp with zeros\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fe3243fa860>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 19748 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "numerical_values1 = (df['PACK1_CRIDATA_BATT_VOL']\n",
    "                     .cast(pl.Float32)\n",
    "                     .fill_null(strategy=\"mean\")\n",
    "                     .filter(df['PACK1_CRIDATA_BATT_VOL'].cast(pl.Float32) != 0))\n",
    "\n",
    "plt.scatter(range(len(numerical_values1)), numerical_values1, s=10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fe32601c5e0>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 19044 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "numerical_values1 = (df['PACK1_CRIDATA_BATT_VOL']\n",
    "                     .cast(pl.Float32)\n",
    "                     .fill_null(0))\n",
    "                     \n",
    "\n",
    "plt.scatter(range(len(numerical_values1)), numerical_values1, s=10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'batt room temp')"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 184884 characters]",
      "text/plain": [
       "<Figure size 1000x3000 with 10 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# start = 115000\n",
    "# end = 140000\n",
    "start = 0\n",
    "end = len(df)\n",
    "#\n",
    "\n",
    "num_rows = 10\n",
    "fig, axs = plt.subplots(nrows=num_rows, ncols=1, figsize=(10,3 * num_rows))\n",
    "\n",
    "filter_condition = (df['PACK1_CRIDATA_BATT_VOL']\n",
    "                    .cast(pl.Float32) != 0)\n",
    "\n",
    "numerical_values0 = convert_values(df['BATT_PACK_1_FAULT']\n",
    "                                   .filter(filter_condition)[start:end])\n",
    "numerical_values1 = (df['PACK1_CRIDATA_BATT_VOL']\n",
    "                     .filter(filter_condition)\n",
    "                     .cast(pl.Float32)\n",
    "                     .fill_null(strategy=\"mean\"))[start:end]\n",
    "numerical_values2 = (df['PACK1_CRIDATA_AVG_CELL_VOL']\n",
    "                     .filter(filter_condition)\n",
    "                     .cast(pl.Float32)\n",
    "                     .fill_null(strategy=\"mean\"))[start:end]\n",
    "numerical_values3 = (df['PACK1_CRIDATA_AVG_CELL_TEMP']\n",
    "                     .filter(filter_condition)\n",
    "                     .cast(pl.Float32)\n",
    "                     .fill_null(strategy=\"mean\"))[start:end]\n",
    "numerical_values4 = (df['PACK1_CRIDATA_BUS_VOL'].cast(pl.Float32).fill_null(strategy=\"mean\"))[start:end]\n",
    "numerical_values5 = (df['PACK1_CRIDATA_CHARGE_CURR_LIMIT'].cast(pl.Float32).fill_null(strategy=\"mean\"))[start:end]\n",
    "numerical_values6 = (df['PACK1_CRIDATA_CURR'].cast(pl.Float32).fill_null(strategy=\"mean\"))[start:end]\n",
    "numerical_values7 = (df['PACK1_CRIDATA_DISCHARGE_CURR_LIMIT'].cast(pl.Float32).fill_null(strategy=\"mean\"))[start:end]\n",
    "numerical_values8 = (df['PACK1_CRIDATA_SOC'].cast(pl.Float32).fill_null(strategy=\"mean\"))[start:end]\n",
    "numerical_values9 = (df['BATT_ROOM_TEMP'].cast(pl.Float32).fill_null(strategy=\"mean\"))[start:end]\n",
    "axs[0].scatter(range(len(numerical_values0)), numerical_values0, s=10)\n",
    "axs[0].set_title(\"fault\")\n",
    "axs[1].scatter(range(len(numerical_values1)), numerical_values1, s=10)\n",
    "axs[1].set_title(\"voltage\")\n",
    "\n",
    "axs[2].scatter(range(len(numerical_values2)), numerical_values2, s=10)\n",
    "axs[2].set_title(\"avg cell vol\")\n",
    "\n",
    "axs[3].scatter(range(len(numerical_values3)), numerical_values3, s=10)\n",
    "axs[3].set_title(\"avg cell temp\")\n",
    "\n",
    "axs[4].scatter(range(len(numerical_values4)), numerical_values4, s=10)\n",
    "axs[4].set_title(\"bus vol\")\n",
    "\n",
    "axs[5].scatter(range(len(numerical_values5)), numerical_values5, s=10)\n",
    "axs[5].set_title(\"charge curr limit\")\n",
    "\n",
    "axs[6].scatter(range(len(numerical_values6)), numerical_values6, s=10)\n",
    "axs[6].set_title(\"curr\")\n",
    "\n",
    "axs[7].scatter(range(len(numerical_values7)), numerical_values7, s=10)\n",
    "axs[7].set_title(\"discharge curr limit\")\n",
    "\n",
    "axs[8].scatter(range(len(numerical_values8)), numerical_values8, s=10)\n",
    "axs[8].set_title(\"soc\")\n",
    "\n",
    "axs[9].scatter(range(len(numerical_values9)), numerical_values9, s=10)\n",
    "axs[9].set_title(\"batt room temp\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
