{ "cells": [ { "cell_type": "markdown", "id": "3b43758b-7204-4cd1-b0dc-9a2ce7926ac7", "metadata": {}, "source": [ "# Ćwiczenie 1" ] }, { "cell_type": "markdown", "id": "c0538a1b-8b08-449c-87c1-8469e6e72197", "metadata": {}, "source": [ "## Wczytaj dane" ] }, { "cell_type": "code", "execution_count": 1, "id": "178ae645-cad9-491c-9a26-c173eda34a00", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 2, "id": "540f9f69-d932-4b5a-8d70-e18a08bdcf46", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
datestationPRCPSNOWSNWDTMAXTMINTOBSWESFinclement_weather
02018-01-01?0.00.0-inf5505.0-40.0NaNNaNNaN
12018-01-02GHCND:USC002809070.00.0-inf-8.3-16.1-12.2NaNFalse
22018-01-03GHCND:USC002809070.00.0-inf-4.4-13.9-13.3NaNFalse
32018-01-04?20.6229.0inf5505.0-40.0NaN19.3True
42018-01-05?0.3NaNNaN5505.0-40.0NaNNaNNaN
.................................
4762018-12-28GHCND:USC0028090711.70.0-inf6.1-1.75.0NaNFalse
4772018-12-29?21.3NaNNaN5505.0-40.0NaNNaNNaN
4782018-12-30?0.0NaNNaN5505.0-40.0NaNNaNNaN
4792018-12-31GHCND:USC002809070.00.0-inf3.3-3.3-2.8NaNFalse
4802018-12-31?0.00.0-inf5505.0-40.0NaNNaNNaN
\n", "

481 rows × 10 columns

\n", "
" ], "text/plain": [ " date station PRCP SNOW SNWD TMAX TMIN TOBS \\\n", "0 2018-01-01 ? 0.0 0.0 -inf 5505.0 -40.0 NaN \n", "1 2018-01-02 GHCND:USC00280907 0.0 0.0 -inf -8.3 -16.1 -12.2 \n", "2 2018-01-03 GHCND:USC00280907 0.0 0.0 -inf -4.4 -13.9 -13.3 \n", "3 2018-01-04 ? 20.6 229.0 inf 5505.0 -40.0 NaN \n", "4 2018-01-05 ? 0.3 NaN NaN 5505.0 -40.0 NaN \n", ".. ... ... ... ... ... ... ... ... \n", "476 2018-12-28 GHCND:USC00280907 11.7 0.0 -inf 6.1 -1.7 5.0 \n", "477 2018-12-29 ? 21.3 NaN NaN 5505.0 -40.0 NaN \n", "478 2018-12-30 ? 0.0 NaN NaN 5505.0 -40.0 NaN \n", "479 2018-12-31 GHCND:USC00280907 0.0 0.0 -inf 3.3 -3.3 -2.8 \n", "480 2018-12-31 ? 0.0 0.0 -inf 5505.0 -40.0 NaN \n", "\n", " WESF inclement_weather \n", "0 NaN NaN \n", "1 NaN False \n", "2 NaN False \n", "3 19.3 True \n", "4 NaN NaN \n", ".. ... ... \n", "476 NaN False \n", "477 NaN NaN \n", "478 NaN NaN \n", "479 NaN False \n", "480 NaN NaN \n", "\n", "[481 rows x 10 columns]" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = pd.read_csv(\"data/dirty_data.csv\")\n", "df = df.drop_duplicates().reset_index()\n", "del df['index']\n", "df.date = pd.to_datetime(df.date)\n", "df" ] }, { "cell_type": "markdown", "id": "1c107b82-f121-4bfb-8933-3b4882fb988f", "metadata": {}, "source": [ "## Clean inclement_weather" ] }, { "cell_type": "code", "execution_count": 3, "id": "62e61e9e-f06e-4124-9a89-1979dc071c47", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
datestationPRCPSNOWSNWDTMAXTMINTOBSWESFincl_weather_trueincl_weather_false
02018-01-01?0.00.0-inf5505.0-40.0NaNNaN00
12018-01-02GHCND:USC002809070.00.0-inf-8.3-16.1-12.2NaN01
22018-01-03GHCND:USC002809070.00.0-inf-4.4-13.9-13.3NaN01
32018-01-04?20.6229.0inf5505.0-40.0NaN19.310
42018-01-05?0.3NaNNaN5505.0-40.0NaNNaN00
....................................
4762018-12-28GHCND:USC0028090711.70.0-inf6.1-1.75.0NaN01
4772018-12-29?21.3NaNNaN5505.0-40.0NaNNaN00
4782018-12-30?0.0NaNNaN5505.0-40.0NaNNaN00
4792018-12-31GHCND:USC002809070.00.0-inf3.3-3.3-2.8NaN01
4802018-12-31?0.00.0-inf5505.0-40.0NaNNaN00
\n", "

481 rows × 11 columns

\n", "
" ], "text/plain": [ " date station PRCP SNOW SNWD TMAX TMIN TOBS \\\n", "0 2018-01-01 ? 0.0 0.0 -inf 5505.0 -40.0 NaN \n", "1 2018-01-02 GHCND:USC00280907 0.0 0.0 -inf -8.3 -16.1 -12.2 \n", "2 2018-01-03 GHCND:USC00280907 0.0 0.0 -inf -4.4 -13.9 -13.3 \n", "3 2018-01-04 ? 20.6 229.0 inf 5505.0 -40.0 NaN \n", "4 2018-01-05 ? 0.3 NaN NaN 5505.0 -40.0 NaN \n", ".. ... ... ... ... ... ... ... ... \n", "476 2018-12-28 GHCND:USC00280907 11.7 0.0 -inf 6.1 -1.7 5.0 \n", "477 2018-12-29 ? 21.3 NaN NaN 5505.0 -40.0 NaN \n", "478 2018-12-30 ? 0.0 NaN NaN 5505.0 -40.0 NaN \n", "479 2018-12-31 GHCND:USC00280907 0.0 0.0 -inf 3.3 -3.3 -2.8 \n", "480 2018-12-31 ? 0.0 0.0 -inf 5505.0 -40.0 NaN \n", "\n", " WESF incl_weather_true incl_weather_false \n", "0 NaN 0 0 \n", "1 NaN 0 1 \n", "2 NaN 0 1 \n", "3 19.3 1 0 \n", "4 NaN 0 0 \n", ".. ... ... ... \n", "476 NaN 0 1 \n", "477 NaN 0 0 \n", "478 NaN 0 0 \n", "479 NaN 0 1 \n", "480 NaN 0 0 \n", "\n", "[481 rows x 11 columns]" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Clean inclement_weather\n", "df = df.assign(\n", " incl_weather_true = lambda x: np.where(x.inclement_weather == True, 1, 0),\n", " incl_weather_false = lambda x: np.where(x.inclement_weather == False, 1, 0)\n", ")\n", "del df['inclement_weather']\n", "df" ] }, { "cell_type": "markdown", "id": "cf3c59c1-ae36-45bd-8b8f-de4f616a392a", "metadata": {}, "source": [ "## Remove SNWD" ] }, { "cell_type": "code", "execution_count": 4, "id": "e06e7a55-1837-456b-97ef-4c58276bd7b6", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
datestationPRCPSNOWTMAXTMINTOBSWESFincl_weather_trueincl_weather_falsesnow
02018-01-01?0.00.05505.0-40.0NaNNaN000
12018-01-02GHCND:USC002809070.00.0-8.3-16.1-12.2NaN010
22018-01-03GHCND:USC002809070.00.0-4.4-13.9-13.3NaN010
32018-01-04?20.6229.05505.0-40.0NaN19.3101
42018-01-05?0.3NaN5505.0-40.0NaNNaN000
....................................
4762018-12-28GHCND:USC0028090711.70.06.1-1.75.0NaN010
4772018-12-29?21.3NaN5505.0-40.0NaNNaN000
4782018-12-30?0.0NaN5505.0-40.0NaNNaN000
4792018-12-31GHCND:USC002809070.00.03.3-3.3-2.8NaN010
4802018-12-31?0.00.05505.0-40.0NaNNaN000
\n", "

481 rows × 11 columns

\n", "
" ], "text/plain": [ " date station PRCP SNOW TMAX TMIN TOBS WESF \\\n", "0 2018-01-01 ? 0.0 0.0 5505.0 -40.0 NaN NaN \n", "1 2018-01-02 GHCND:USC00280907 0.0 0.0 -8.3 -16.1 -12.2 NaN \n", "2 2018-01-03 GHCND:USC00280907 0.0 0.0 -4.4 -13.9 -13.3 NaN \n", "3 2018-01-04 ? 20.6 229.0 5505.0 -40.0 NaN 19.3 \n", "4 2018-01-05 ? 0.3 NaN 5505.0 -40.0 NaN NaN \n", ".. ... ... ... ... ... ... ... ... \n", "476 2018-12-28 GHCND:USC00280907 11.7 0.0 6.1 -1.7 5.0 NaN \n", "477 2018-12-29 ? 21.3 NaN 5505.0 -40.0 NaN NaN \n", "478 2018-12-30 ? 0.0 NaN 5505.0 -40.0 NaN NaN \n", "479 2018-12-31 GHCND:USC00280907 0.0 0.0 3.3 -3.3 -2.8 NaN \n", "480 2018-12-31 ? 0.0 0.0 5505.0 -40.0 NaN NaN \n", "\n", " incl_weather_true incl_weather_false snow \n", "0 0 0 0 \n", "1 0 1 0 \n", "2 0 1 0 \n", "3 1 0 1 \n", "4 0 0 0 \n", ".. ... ... ... \n", "476 0 1 0 \n", "477 0 0 0 \n", "478 0 0 0 \n", "479 0 1 0 \n", "480 0 0 0 \n", "\n", "[481 rows x 11 columns]" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# remove SNWD\n", "df = df.assign(\n", " snow = lambda x: np.where(x.SNWD == np.inf, 1, 0)\n", ")\n", "del df['SNWD']\n", "df" ] }, { "cell_type": "markdown", "id": "b0b5005b-ad90-4164-83ad-8e4030684820", "metadata": {}, "source": [ "## Replace NaN with 0" ] }, { "cell_type": "code", "execution_count": 5, "id": "389f0c22-b210-45f3-9627-02c06d255a69", "metadata": {}, "outputs": [], "source": [ "# Replace NaN with zero\n", "df['SNOW'] = df['SNOW'].fillna(0)\n", "df['WESF'] = df['WESF'].fillna(0)" ] }, { "cell_type": "markdown", "id": "ba7a7e21-0361-49c3-a7ca-eea9c9407758", "metadata": {}, "source": [ "## Split data into two different tables" ] }, { "cell_type": "code", "execution_count": 6, "id": "60362a3d-91ab-4666-bf26-afdeb3cd5c9c", "metadata": {}, "outputs": [], "source": [ "# Split data into temperature and snow\n", "df_snow = df[df['station']=='?']\n", "df_temp = df[df['station']!='?']" ] }, { "cell_type": "code", "execution_count": 7, "id": "a68dd0eb-6085-483b-99f3-a540a35515c2", "metadata": {}, "outputs": [], "source": [ "del df_snow['station']\n", "del df_snow['TMIN']\n", "del df_snow['TMAX']\n", "del df_snow['TOBS']\n", "# del df_snow['']\n", "del df_temp['station']\n", "del df_temp['WESF']" ] }, { "cell_type": "code", "execution_count": 8, "id": "bb4ecdab-2136-4970-a38d-6eaec8ec92ae", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
datestationPRCPSNOWTMAXTMINTOBSWESFincl_weather_trueincl_weather_falsesnow
02018-01-01?0.00.05505.0-40.0NaN0.0000
12018-01-02GHCND:USC002809070.00.0-8.3-16.1-12.20.0010
22018-01-03GHCND:USC002809070.00.0-4.4-13.9-13.30.0010
32018-01-04?20.6229.05505.0-40.0NaN19.3101
42018-01-05?0.30.05505.0-40.0NaN0.0000
....................................
4762018-12-28GHCND:USC0028090711.70.06.1-1.75.00.0010
4772018-12-29?21.30.05505.0-40.0NaN0.0000
4782018-12-30?0.00.05505.0-40.0NaN0.0000
4792018-12-31GHCND:USC002809070.00.03.3-3.3-2.80.0010
4802018-12-31?0.00.05505.0-40.0NaN0.0000
\n", "

481 rows × 11 columns

\n", "
" ], "text/plain": [ " date station PRCP SNOW TMAX TMIN TOBS WESF \\\n", "0 2018-01-01 ? 0.0 0.0 5505.0 -40.0 NaN 0.0 \n", "1 2018-01-02 GHCND:USC00280907 0.0 0.0 -8.3 -16.1 -12.2 0.0 \n", "2 2018-01-03 GHCND:USC00280907 0.0 0.0 -4.4 -13.9 -13.3 0.0 \n", "3 2018-01-04 ? 20.6 229.0 5505.0 -40.0 NaN 19.3 \n", "4 2018-01-05 ? 0.3 0.0 5505.0 -40.0 NaN 0.0 \n", ".. ... ... ... ... ... ... ... ... \n", "476 2018-12-28 GHCND:USC00280907 11.7 0.0 6.1 -1.7 5.0 0.0 \n", "477 2018-12-29 ? 21.3 0.0 5505.0 -40.0 NaN 0.0 \n", "478 2018-12-30 ? 0.0 0.0 5505.0 -40.0 NaN 0.0 \n", "479 2018-12-31 GHCND:USC00280907 0.0 0.0 3.3 -3.3 -2.8 0.0 \n", "480 2018-12-31 ? 0.0 0.0 5505.0 -40.0 NaN 0.0 \n", "\n", " incl_weather_true incl_weather_false snow \n", "0 0 0 0 \n", "1 0 1 0 \n", "2 0 1 0 \n", "3 1 0 1 \n", "4 0 0 0 \n", ".. ... ... ... \n", "476 0 1 0 \n", "477 0 0 0 \n", "478 0 0 0 \n", "479 0 1 0 \n", "480 0 0 0 \n", "\n", "[481 rows x 11 columns]" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df" ] }, { "cell_type": "code", "execution_count": 9, "id": "7c8199c4-9d1b-405a-9cda-b2aef1616999", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
datePRCPSNOWWESFincl_weather_trueincl_weather_falsesnow
count232232.000000232.000000232.000000232.000000232.0232.000000
mean2018-06-23 12:49:39.3103447044.1754312.9008620.4375000.0258620.00.025862
min2018-01-01 00:00:000.0000000.0000000.0000000.0000000.00.000000
25%2018-03-30 18:00:000.0000000.0000000.0000000.0000000.00.000000
50%2018-06-15 12:00:000.0000000.0000000.0000000.0000000.00.000000
75%2018-09-11 06:00:003.0750000.0000000.0000000.0000000.00.000000
max2018-12-31 00:00:0047.000000229.00000028.7000001.0000000.01.000000
stdNaN8.33315221.8248363.0072230.1590670.00.159067
\n", "
" ], "text/plain": [ " date PRCP SNOW WESF \\\n", "count 232 232.000000 232.000000 232.000000 \n", "mean 2018-06-23 12:49:39.310344704 4.175431 2.900862 0.437500 \n", "min 2018-01-01 00:00:00 0.000000 0.000000 0.000000 \n", "25% 2018-03-30 18:00:00 0.000000 0.000000 0.000000 \n", "50% 2018-06-15 12:00:00 0.000000 0.000000 0.000000 \n", "75% 2018-09-11 06:00:00 3.075000 0.000000 0.000000 \n", "max 2018-12-31 00:00:00 47.000000 229.000000 28.700000 \n", "std NaN 8.333152 21.824836 3.007223 \n", "\n", " incl_weather_true incl_weather_false snow \n", "count 232.000000 232.0 232.000000 \n", "mean 0.025862 0.0 0.025862 \n", "min 0.000000 0.0 0.000000 \n", "25% 0.000000 0.0 0.000000 \n", "50% 0.000000 0.0 0.000000 \n", "75% 0.000000 0.0 0.000000 \n", "max 1.000000 0.0 1.000000 \n", "std 0.159067 0.0 0.159067 " ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_snow.describe()" ] }, { "cell_type": "code", "execution_count": 10, "id": "7468e63c-c72b-4150-bd40-6f6e61f17af8", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
datePRCPSNOWTMAXTMINTOBSincl_weather_trueincl_weather_falsesnow
count249249.000000249.000000249.000000249.000000249.000000249.000000249.000000249.000000
mean2018-06-30 23:36:52.0481927685.5273092.90763116.0048196.3714868.7120480.0321290.9678710.032129
min2018-01-02 00:00:000.0000000.000000-11.700000-17.200000-16.1000000.0000000.0000000.000000
25%2018-03-27 00:00:000.0000000.0000006.700000-1.7000000.0000000.0000001.0000000.000000
50%2018-07-07 00:00:000.0000000.00000014.4000005.6000008.3000000.0000001.0000000.000000
75%2018-09-30 00:00:005.6000000.00000026.10000015.60000017.8000000.0000001.0000000.000000
max2018-12-31 00:00:0061.700000178.00000035.00000023.90000026.1000001.0000001.0000001.000000
stdNaN10.66519719.83204411.00061510.1578099.9364680.1766970.1766970.176697
\n", "
" ], "text/plain": [ " date PRCP SNOW TMAX \\\n", "count 249 249.000000 249.000000 249.000000 \n", "mean 2018-06-30 23:36:52.048192768 5.527309 2.907631 16.004819 \n", "min 2018-01-02 00:00:00 0.000000 0.000000 -11.700000 \n", "25% 2018-03-27 00:00:00 0.000000 0.000000 6.700000 \n", "50% 2018-07-07 00:00:00 0.000000 0.000000 14.400000 \n", "75% 2018-09-30 00:00:00 5.600000 0.000000 26.100000 \n", "max 2018-12-31 00:00:00 61.700000 178.000000 35.000000 \n", "std NaN 10.665197 19.832044 11.000615 \n", "\n", " TMIN TOBS incl_weather_true incl_weather_false \\\n", "count 249.000000 249.000000 249.000000 249.000000 \n", "mean 6.371486 8.712048 0.032129 0.967871 \n", "min -17.200000 -16.100000 0.000000 0.000000 \n", "25% -1.700000 0.000000 0.000000 1.000000 \n", "50% 5.600000 8.300000 0.000000 1.000000 \n", "75% 15.600000 17.800000 0.000000 1.000000 \n", "max 23.900000 26.100000 1.000000 1.000000 \n", "std 10.157809 9.936468 0.176697 0.176697 \n", "\n", " snow \n", "count 249.000000 \n", "mean 0.032129 \n", "min 0.000000 \n", "25% 0.000000 \n", "50% 0.000000 \n", "75% 0.000000 \n", "max 1.000000 \n", "std 0.176697 " ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_temp.describe()" ] }, { "cell_type": "markdown", "id": "ad025f7d-3200-47b1-8cc2-4a53faa6da3b", "metadata": {}, "source": [ "## Join the tables" ] }, { "cell_type": "code", "execution_count": 12, "id": "efa6a165-9988-420f-a2f0-10cbbcd5dc39", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
TMAXTMINTOBSWESFSNOWPRCPincl_weather_trueincl_weather_falsesnow
count249.000000249.000000249.000000249.000000249.000000249.000000249.000000249.000000249.000000
mean16.0048196.3714868.7120480.2301204.0803218.3602410.0481930.9678710.048193
std11.00061510.1578099.9364682.13245326.32553615.7189450.2493680.1766970.249368
min-11.700000-17.200000-16.1000000.0000000.0000000.0000000.0000000.0000000.000000
25%6.700000-1.7000000.0000000.0000000.0000000.0000000.0000001.0000000.000000
50%14.4000005.6000008.3000000.0000000.0000000.3000000.0000001.0000000.000000
75%26.10000015.60000017.8000000.0000000.0000008.6000000.0000001.0000000.000000
max35.00000023.90000026.10000028.700000279.00000079.2000002.0000001.0000002.000000
\n", "
" ], "text/plain": [ " TMAX TMIN TOBS WESF SNOW PRCP \\\n", "count 249.000000 249.000000 249.000000 249.000000 249.000000 249.000000 \n", "mean 16.004819 6.371486 8.712048 0.230120 4.080321 8.360241 \n", "std 11.000615 10.157809 9.936468 2.132453 26.325536 15.718945 \n", "min -11.700000 -17.200000 -16.100000 0.000000 0.000000 0.000000 \n", "25% 6.700000 -1.700000 0.000000 0.000000 0.000000 0.000000 \n", "50% 14.400000 5.600000 8.300000 0.000000 0.000000 0.300000 \n", "75% 26.100000 15.600000 17.800000 0.000000 0.000000 8.600000 \n", "max 35.000000 23.900000 26.100000 28.700000 279.000000 79.200000 \n", "\n", " incl_weather_true incl_weather_false snow \n", "count 249.000000 249.000000 249.000000 \n", "mean 0.048193 0.967871 0.048193 \n", "std 0.249368 0.176697 0.249368 \n", "min 0.000000 0.000000 0.000000 \n", "25% 0.000000 1.000000 0.000000 \n", "50% 0.000000 1.000000 0.000000 \n", "75% 0.000000 1.000000 0.000000 \n", "max 2.000000 1.000000 2.000000 " ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df_merged = df_temp.set_index('date').join(df_snow.set_index('date'), lsuffix=\"_temp\", rsuffix=\"_snow\")\n", "df_merged[\"SNOW\"] = df_merged.SNOW_temp.fillna(0) + df_merged.SNOW_snow.fillna(0)\n", "df_merged[\"PRCP\"] = df_merged.PRCP_temp.fillna(0) + df_merged.PRCP_snow.fillna(0)\n", "df_merged[\"incl_weather_true\"] = df_merged.incl_weather_true_temp.fillna(0) + df_merged.incl_weather_true_snow.fillna(0)\n", "df_merged[\"incl_weather_false\"] = df_merged.incl_weather_false_temp.fillna(0) + df_merged.incl_weather_false_snow.fillna(0)\n", "df_merged[\"snow\"] = df_merged.snow_temp.fillna(0) + df_merged.snow_snow.fillna(0)\n", "df_merged['WESF'] = df_merged['WESF'].fillna(0)\n", "\n", "del df_merged[\"PRCP_temp\"]\n", "del df_merged[\"PRCP_snow\"]\n", "del df_merged[\"incl_weather_false_temp\"]\n", "del df_merged[\"incl_weather_false_snow\"]\n", "del df_merged[\"incl_weather_true_temp\"]\n", "del df_merged[\"incl_weather_true_snow\"]\n", "del df_merged[\"SNOW_temp\"]\n", "del df_merged[\"SNOW_snow\"]\n", "del df_merged[\"snow_temp\"]\n", "del df_merged[\"snow_snow\"]\n", "\n", "\n", "df_merged.describe()" ] }, { "cell_type": "code", "execution_count": null, "id": "4d3b7d61-d247-49f3-a643-6fd924727cdd", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.13.2" } }, "nbformat": 4, "nbformat_minor": 5 }