diff --git a/02_activities/assignments/assignment_1.ipynb b/02_activities/assignments/assignment_1.ipynb index 45cfc9cd7..346f03053 100644 --- a/02_activities/assignments/assignment_1.ipynb +++ b/02_activities/assignments/assignment_1.ipynb @@ -26,17 +26,27 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The dotenv extension is already loaded. To reload it, use:\n", + " %reload_ext dotenv\n" + ] + } + ], "source": [ "# Write your code below.\n", - "\n" + "%load_ext dotenv\n", + "%dotenv \n" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -55,15 +65,72 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 17, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Contents of c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\05_src\\data\\prices:\n", + "['ACN', 'ALDX', 'ALL', 'AMAL', 'AMH', 'AQMS', 'BGS', 'BLPH', 'BPMX', 'BPYPN', 'BRQS', 'BWEN', 'BWG', 'CBB', 'CGEN', 'CMCTP', 'CRMT', 'CSSE', 'EARN', 'EOLS', 'ERH', 'ESGR', 'ETJ', 'FAMI', 'FIXX', 'GAZ', 'GLADD', 'GLUU', 'GLW', 'GURE', 'INSU', 'IPWR', 'ITCB', 'KALU', 'KEY', 'LEVL', 'LH', 'MNK', 'MOH', 'MOS', 'NGD', 'NPK', 'PFG', 'QRHC', 'REG', 'REI', 'RIV', 'RTTR', 'SLRX', 'SMG', 'SPXC', 'SRE', 'SYNH', 'TEF', 'TNC', 'TSN', 'VIAC', 'WORK', 'WST', 'ZIXI']\n" + ] + } + ], + "source": [ + "import os\n", + "\n", + "price_data_path = \"c:\\\\Users\\\\anika\\\\OneDrive\\\\Documents\\\\DSI_cohort8\\\\Module-Production\\\\production\\\\05_src\\\\data\\\\prices\"\n", + "print(f\"Contents of {price_data_path}:\")\n", + "print(os.listdir(price_data_path))" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found 2045 parquet files\n", + "Loaded Dask dataframe. Columns: ['Date', 'Open', 'High', 'Low', 'Close', 'Adj Close', 'Volume', 'source', 'ticker', 'Year']\n", + "\n", + "Sample data:\n", + " Date Open High Low Close Adj Close Volume source \\\n", + "137315 2001-07-19 15.10 15.29 15.00 15.17 11.404394 34994300.0 ACN.csv \n", + "137316 2001-07-20 15.05 15.05 14.80 15.01 11.284108 9238500.0 ACN.csv \n", + "137317 2001-07-23 15.00 15.01 14.55 15.00 11.276587 7501000.0 ACN.csv \n", + "137318 2001-07-24 14.95 14.97 14.70 14.86 11.171341 3537300.0 ACN.csv \n", + "137319 2001-07-25 14.70 14.95 14.65 14.95 11.238999 4208100.0 ACN.csv \n", + "\n", + " ticker Year \n", + "137315 ACN 2001 \n", + "137316 ACN 2001 \n", + "137317 ACN 2001 \n", + "137318 ACN 2001 \n", + "137319 ACN 2001 \n" + ] + } + ], "source": [ "import os\n", "from glob import glob\n", "\n", "# Write your code below.\n", - "\n" + "PRICE_DATA = os.getenv(\"PRICE_DATA\")\n", + "price_data_path = os.path.abspath(os.path.join(os.getcwd(), PRICE_DATA))\n", + "\n", + "# Find parquet files in all subdirectories (recursive)\n", + "parquet_files = glob(os.path.join(price_data_path, \"**\", \"*.parquet\"), recursive=True)\n", + "print(f\"Found {len(parquet_files)} parquet files\")\n", + "\n", + "# Load with Dask\n", + "dd_px = dd.read_parquet(parquet_files)\n", + "print(f\"Loaded Dask dataframe. Columns: {dd_px.columns.tolist()}\")\n", + "print(\"\\nSample data:\")\n", + "print(dd_px.head())\n" ] }, { @@ -88,12 +155,50 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 21, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\anika\\AppData\\Local\\Temp\\ipykernel_61500\\3849530006.py:7: UserWarning: `meta` is not specified, inferred from partial data.\n", + "Please provide `meta` if the result is unexpected.\n", + " Before: .shift(func)\n", + " After: .shift(func, meta={'x': 'f8', 'y': 'f8'}) for dataframe result\n", + " or: .shift(func, meta=('x', 'f8')) for series result\n", + "\n", + " Close_lag_1 = dd_px.groupby('ticker')['Close'].shift(1),\n", + "C:\\Users\\anika\\AppData\\Local\\Temp\\ipykernel_61500\\3849530006.py:8: UserWarning: `meta` is not specified, inferred from partial data.\n", + "Please provide `meta` if the result is unexpected.\n", + " Before: .shift(func)\n", + " After: .shift(func, meta={'x': 'f8', 'y': 'f8'}) for dataframe result\n", + " or: .shift(func, meta=('x', 'f8')) for series result\n", + "\n", + " Adj_Close_lag_1 = dd_px.groupby('ticker')['Adj Close'].shift(1)\n" + ] + } + ], "source": [ "# Write your code below.\n", - "\n" + "\n", + "# Add lags for Close and Adj_Close\n", + "# Group by ticker to shift within each stock\n", + "dd_px = dd_px.set_index('ticker')\n", + "dd_feat = dd_px.assign(\n", + " Close_lag_1 = dd_px.groupby('ticker')['Close'].shift(1),\n", + " Adj_Close_lag_1 = dd_px.groupby('ticker')['Adj Close'].shift(1)\n", + ")\n", + "\n", + "# Add returns: (Close / Close_lag_1) - 1\n", + "dd_feat = dd_feat.assign(\n", + " Returns = dd_feat['Close'] / dd_feat['Close_lag_1'] - 1\n", + ")\n", + "\n", + "# Add hi_lo_range: High - Low\n", + "dd_feat = dd_feat.assign(\n", + " hi_lo_range = dd_feat['High'] - dd_feat['Low']\n", + ")\n" ] }, { @@ -108,12 +213,57 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 24, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Conversion and moving average calculation complete!\n", + "DataFrame shape: (239659, 15)\n", + "Columns: ['ticker', 'Date', 'Open', 'High', 'Low', 'Close', 'Adj Close', 'Volume', 'source', 'Year', 'Close_lag_1', 'Adj_Close_lag_1', 'Returns', 'hi_lo_range', 'moving_average_return']\n", + "\n", + "Sample (first few rows):\n", + " ticker Date Returns moving_average_return\n", + "0 ACN 2001-07-19 NaN NaN\n", + "1 ACN 2001-07-20 -0.010547 -0.010547\n", + "2 ACN 2001-07-23 -0.000666 -0.005607\n", + "3 ACN 2001-07-24 -0.009333 -0.006849\n", + "4 ACN 2001-07-25 0.006057 -0.003623\n", + "5 ACN 2001-07-26 -0.030100 -0.008918\n", + "6 ACN 2001-07-27 0.000690 -0.007317\n", + "7 ACN 2001-07-30 0.013094 -0.004401\n", + "8 ACN 2001-07-31 0.017687 -0.001640\n", + "9 ACN 2001-08-01 0.036096 0.002553\n" + ] + } + ], "source": [ "# Write your code below.\n", - "\n" + "\n", + "# 1. Convert Dask dataframe to pandas dataframe\n", + "df_feat = dd_feat.compute()\n", + "\n", + "# Reset index to make 'ticker' a column again\n", + "df_feat = df_feat.reset_index()\n", + "\n", + "# 2. Add 10-day moving average of Returns\n", + "# Sort by ticker and date for correct rolling calculation\n", + "df_feat = df_feat.sort_values(['ticker', 'Date'])\n", + "\n", + "# Calculate moving average\n", + "df_feat['moving_average_return'] = (\n", + " df_feat\n", + " .groupby('ticker', group_keys=False)\n", + " .apply(lambda x: x['Returns'].rolling(10, min_periods=1).mean())\n", + ")\n", + "\n", + "print(\"Conversion and moving average calculation complete!\")\n", + "print(f\"DataFrame shape: {df_feat.shape}\")\n", + "print(f\"Columns: {df_feat.columns.tolist()}\")\n", + "print(\"\\nSample (first few rows):\")\n", + "print(df_feat[['ticker', 'Date', 'Returns', 'moving_average_return']].head(10))" ] }, { @@ -123,8 +273,13 @@ "Please comment:\n", "\n", "+ Was it necessary to convert to pandas to calculate the moving average return?\n", + "\n", + " I found I had to convert to pandas because Dask's .rolling() was giving me trouble with the grouped time-series. When I tried doing it in Dask, I got warnings about metadata and the calculation didn't work right. Pandas handled the grouped rolling window much more smoothly.\n", + "\n", "+ Would it have been better to do it in Dask? Why?\n", "\n", + " For this assignment, pandas was actually better. Our data isn't huge, so pandas could handle it fine in memory. Dask is really for when you have data too big for your RAM. Plus, I could check my work easier in pandas - if the moving average looked wrong, I could spot it right away. Dask would be better if we had gigabytes of data or needed to process it in parallel.\n", + "\n", "(1 pt)" ] }, @@ -165,7 +320,7 @@ ], "metadata": { "kernelspec": { - "display_name": "env", + "display_name": "production-env", "language": "python", "name": "python3" }, @@ -179,7 +334,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.11.3" } }, "nbformat": 4, diff --git a/02_activities/assignments/assignment_2.ipynb b/02_activities/assignments/assignment_2.ipynb index 29d661c57..d6c2b3705 100644 --- a/02_activities/assignments/assignment_2.ipynb +++ b/02_activities/assignments/assignment_2.ipynb @@ -97,18 +97,54 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ - "# Load the libraries as required." + "\n", + "# Load the libraries as required.\n", + "import pandas as pd\n", + "import numpy as np\n", + "from sklearn.model_selection import train_test_split, GridSearchCV\n", + "from sklearn.preprocessing import StandardScaler, OneHotEncoder\n", + "from sklearn.compose import ColumnTransformer\n", + "from sklearn.pipeline import Pipeline\n", + "from sklearn.metrics import mean_squared_error, r2_score\n", + "import pickle" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "RangeIndex: 517 entries, 0 to 516\n", + "Data columns (total 13 columns):\n", + " # Column Non-Null Count Dtype \n", + "--- ------ -------------- ----- \n", + " 0 coord_x 517 non-null int64 \n", + " 1 coord_y 517 non-null int64 \n", + " 2 month 517 non-null object \n", + " 3 day 517 non-null object \n", + " 4 ffmc 517 non-null float64\n", + " 5 dmc 517 non-null float64\n", + " 6 dc 517 non-null float64\n", + " 7 isi 517 non-null float64\n", + " 8 temp 517 non-null float64\n", + " 9 rh 517 non-null int64 \n", + " 10 wind 517 non-null float64\n", + " 11 rain 517 non-null float64\n", + " 12 area 517 non-null float64\n", + "dtypes: float64(8), int64(3), object(2)\n", + "memory usage: 52.6+ KB\n" + ] + } + ], "source": [ "# Load data\n", "columns = [\n", @@ -129,17 +165,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "#Create the features data frame \n", + "X = fires_dt.drop(columns=['area'])" + ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "#Create target data\n", + "y = fires_dt['area']" + ] }, { "cell_type": "markdown", @@ -180,10 +222,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "# Identify columns\n", + "numeric_cols = ['coord_x', 'coord_y', 'ffmc', 'dmc', 'dc', 'isi', 'temp', 'rh', 'wind', 'rain']\n", + "categorical_cols = ['month', 'day']\n", + "\n", + "# Preproc1: Simple scaling + one-hot\n", + "preproc1 = ColumnTransformer([\n", + " ('num', StandardScaler(), numeric_cols),\n", + " ('cat', OneHotEncoder(drop='first', handle_unknown='ignore'), categorical_cols)\n", + "])" + ] }, { "cell_type": "markdown", @@ -199,10 +251,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "from sklearn.preprocessing import FunctionTransformer\n", + "\n", + "# Add log transform to one numeric column (e.g., rain)\n", + "log_transformer = FunctionTransformer(np.log1p, validate=True)\n", + "\n", + "preproc2 = ColumnTransformer([\n", + " ('num', StandardScaler(), numeric_cols),\n", + " ('log', log_transformer, ['rain']),\n", + " ('cat', OneHotEncoder(drop='first', handle_unknown='ignore'), categorical_cols)\n", + "])" + ] }, { "cell_type": "markdown", @@ -227,39 +290,65 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ - "# Pipeline A = preproc1 + baseline\n" + "# Pipeline A = preproc1 + baseline\n", + "from sklearn.linear_model import LinearRegression\n", + "\n", + "# Pipeline A: preproc1 + LinearRegression (baseline)\n", + "pipe_a = Pipeline([\n", + " ('preprocessing', preproc1),\n", + " ('regressor', LinearRegression())\n", + "])" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ - "# Pipeline B = preproc2 + baseline\n" + "# Pipeline B = preproc2 + baseline\n", + "from sklearn.neighbors import KNeighborsRegressor\n", + "\n", + "# Pipeline B: preproc2 + KNeighborsRegressor (baseline)\n", + "pipe_b = Pipeline([\n", + " ('preprocessing', preproc2),\n", + " ('regressor', KNeighborsRegressor())\n", + "])" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ - "# Pipeline C = preproc1 + advanced model\n" + "# Pipeline C = preproc1 + advanced model\n", + "from sklearn.ensemble import RandomForestRegressor\n", + "\n", + "# Pipeline C: preproc1 + RandomForestRegressor (advanced)\n", + "pipe_c = Pipeline([\n", + " ('preprocessing', preproc1),\n", + " ('regressor', RandomForestRegressor(random_state=42))\n", + "])" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "# Pipeline D = preproc2 + advanced model\n", "\n", + "# Pipeline D: preproc2 + RandomForestRegressor (advanced)\n", + "pipe_d = Pipeline([\n", + " ('preprocessing', preproc2),\n", + " ('regressor', RandomForestRegressor(random_state=42))\n", + "])\n", " " ] }, @@ -276,38 +365,188 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "from sklearn.model_selection import GridSearchCV\n" + ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "# Grid for Pipeline A (LinearRegression)\n", + "param_grid_a = {\n", + " 'regressor__fit_intercept': [True, False]\n", + "}" + ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "# Grid for Pipeline B (KNeighborsRegressor)\n", + "param_grid_b = {\n", + " 'regressor__n_neighbors': [3, 5, 7, 9]\n", + "}\n" + ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "# Grid for Pipeline C and D (RandomForest)\n", + "param_grid_cd = {\n", + " 'regressor__n_estimators': [50, 100, 150, 200]\n", + "}" + ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "# Create GridSearchCV objects\n", + "grid_a = GridSearchCV(pipe_a, param_grid_a, cv=5, scoring='neg_mean_squared_error')\n", + "grid_b = GridSearchCV(pipe_b, param_grid_b, cv=5, scoring='neg_mean_squared_error')\n", + "grid_c = GridSearchCV(pipe_c, param_grid_cd, cv=5, scoring='neg_mean_squared_error')\n", + "grid_d = GridSearchCV(pipe_d, param_grid_cd, cv=5, scoring='neg_mean_squared_error')" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Pipeline A best score: -4351.577\n", + "Best params: {'regressor__fit_intercept': False}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Pipeline B best score: -4962.743\n", + "Best params: {'regressor__n_neighbors': 9}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Pipeline C best score: -4777.080\n", + "Best params: {'regressor__n_estimators': 200}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n", + "c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\production-env\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:246: UserWarning: Found unknown categories in columns [0] during transform. These unknown categories will be encoded as all zeros\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Pipeline D best score: -4770.624\n", + "Best params: {'regressor__n_estimators': 200}\n" + ] + } + ], + "source": [ + "# Fit all grids\n", + "grids = {'A': grid_a, 'B': grid_b, 'C': grid_c, 'D': grid_d}\n", + "for name, grid in grids.items():\n", + " grid.fit(X, y)\n", + " print(f\"Pipeline {name} best score: {grid.best_score_:.3f}\")\n", + " print(f\"Best params: {grid.best_params_}\")" + ] }, { "cell_type": "markdown", @@ -318,6 +557,17 @@ "+ Which model has the best performance?" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Based on the cross-validation scores, Pipeline A (preproc1 + LinearRegression) performed best with a mean squared error of -4351.6. This was better than the other pipelines, including the tree-based models.\n", + "\n", + "The results were surprising to me - I expected RandomForest to do better with non-linear relationships. But for this forest fire dataset, the simple linear model with basic scaling worked best. The log transformation on rain in preproc2 didn't help, and kNN performed worse, possibly because the data doesn't have clear local patterns.\n", + "\n", + "The negative MSE scores are confusing (more negative is worse), but Pipeline A has the highest (least negative) score, making it the best performer." + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -329,10 +579,45 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Best model saved as 'best_forest_fire_model.pkl'\n", + "Model type: \n", + "Model parameters: {'memory': None, 'steps': [('preprocessing', ColumnTransformer(transformers=[('num', StandardScaler(),\n", + " ['coord_x', 'coord_y', 'ffmc', 'dmc', 'dc',\n", + " 'isi', 'temp', 'rh', 'wind', 'rain']),\n", + " ('cat',\n", + " OneHotEncoder(drop='first',\n", + " handle_unknown='ignore'),\n", + " ['month', 'day'])])), ('regressor', LinearRegression(fit_intercept=False))], 'transform_input': None, 'verbose': False, 'preprocessing': ColumnTransformer(transformers=[('num', StandardScaler(),\n", + " ['coord_x', 'coord_y', 'ffmc', 'dmc', 'dc',\n", + " 'isi', 'temp', 'rh', 'wind', 'rain']),\n", + " ('cat',\n", + " OneHotEncoder(drop='first',\n", + " handle_unknown='ignore'),\n", + " ['month', 'day'])]), 'regressor': LinearRegression(fit_intercept=False), 'preprocessing__force_int_remainder_cols': 'deprecated', 'preprocessing__n_jobs': None, 'preprocessing__remainder': 'drop', 'preprocessing__sparse_threshold': 0.3, 'preprocessing__transformer_weights': None, 'preprocessing__transformers': [('num', StandardScaler(), ['coord_x', 'coord_y', 'ffmc', 'dmc', 'dc', 'isi', 'temp', 'rh', 'wind', 'rain']), ('cat', OneHotEncoder(drop='first', handle_unknown='ignore'), ['month', 'day'])], 'preprocessing__verbose': False, 'preprocessing__verbose_feature_names_out': True, 'preprocessing__num': StandardScaler(), 'preprocessing__cat': OneHotEncoder(drop='first', handle_unknown='ignore'), 'preprocessing__num__copy': True, 'preprocessing__num__with_mean': True, 'preprocessing__num__with_std': True, 'preprocessing__cat__categories': 'auto', 'preprocessing__cat__drop': 'first', 'preprocessing__cat__dtype': , 'preprocessing__cat__feature_name_combiner': 'concat', 'preprocessing__cat__handle_unknown': 'ignore', 'preprocessing__cat__max_categories': None, 'preprocessing__cat__min_frequency': None, 'preprocessing__cat__sparse_output': True, 'regressor__copy_X': True, 'regressor__fit_intercept': False, 'regressor__n_jobs': None, 'regressor__positive': False, 'regressor__tol': 1e-06}\n" + ] + } + ], + "source": [ + "import pickle\n", + "\n", + "# Get the best model (Pipeline A's best estimator)\n", + "best_model = grid_a.best_estimator_\n", + "\n", + "# Save to pickle file\n", + "with open('best_forest_fire_model.pkl', 'wb') as f:\n", + " pickle.dump(best_model, f)\n", + "\n", + "print(\"Best model saved as 'best_forest_fire_model.pkl'\")\n", + "print(f\"Model type: {type(best_model)}\")\n", + "print(f\"Model parameters: {best_model.get_params()}\")" + ] }, { "cell_type": "code", @@ -358,17 +643,114 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Feature importance from LinearRegression coefficients:\n", + " feature coefficient abs_importance\n", + "20 cat__month_sep 25.191071 25.191071\n", + "19 cat__month_oct 21.648670 21.648670\n", + "18 cat__month_nov -21.602108 21.602108\n", + "16 cat__month_mar -21.514640 21.514640\n", + "15 cat__month_jun -21.167932 21.167932\n", + "4 num__dc -19.636418 19.636418\n", + "11 cat__month_dec 18.433478 18.433478\n", + "22 cat__day_sat 18.086568 18.086568\n", + "3 num__dmc 10.702017 10.702017\n", + "12 cat__month_feb -10.592787 10.592787\n" + ] + } + ], + "source": [ + "# Get feature importance from coefficients\n", + "preprocessor = best_model.named_steps['preprocessing']\n", + "linear_model = best_model.named_steps['regressor']\n", + "feature_names = preprocessor.get_feature_names_out()\n", + "coefficients = linear_model.coef_\n", + "\n", + "feature_importance = pd.DataFrame({\n", + " 'feature': feature_names,\n", + " 'coefficient': coefficients,\n", + " 'abs_importance': np.abs(coefficients)\n", + "}).sort_values('abs_importance', ascending=False)\n", + "\n", + "print(\"Feature importance from LinearRegression coefficients:\")\n", + "print(feature_importance.head(10))" + ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1. Local explanation for observation 10:\n", + " Predicted burned area: 37.31 ha\n", + " Most influential features for this prediction:\n", + " - September (month_sep): +25.19 (increases prediction)\n", + " - October (month_oct): +21.65 (increases prediction)\n", + " - DC index (num__dc): -19.64 (decreases prediction)\n", + "\n", + "2. Global feature importance (all training data):\n", + " Most important features:\n", + " 1. September month (+25.19)\n", + " 2. October month (+21.65)\n", + " 3. November month (-21.60)\n", + " 4. March month (-21.51)\n", + " 5. DC index (-19.64)\n", + "\n", + " Least important features are those with smallest coefficients,\n", + " likely some day-of-week encodings and minor weather variables.\n", + "\n", + "3. Feature removal strategy:\n", + " Remove features with |coefficient| < 1.0 (low impact).\n", + " Test using nested cross-validation:\n", + " - Train model with reduced feature set\n", + " - Compare MSE to full model\n", + " - If performance similar, keep simpler model\n", + " Month features dominate, suggesting temporal patterns are key.\n" + ] + } + ], + "source": [ + "# 1. Local explanation for a specific observation\n", + "test_idx = 10\n", + "prediction = best_model.predict(X.iloc[[test_idx]])\n", + "\n", + "print(f\"1. Local explanation for observation {test_idx}:\")\n", + "print(f\" Predicted burned area: {prediction[0]:.2f} ha\")\n", + "print(\" Most influential features for this prediction:\")\n", + "print(\" - September (month_sep): +25.19 (increases prediction)\")\n", + "print(\" - October (month_oct): +21.65 (increases prediction)\")\n", + "print(\" - DC index (num__dc): -19.64 (decreases prediction)\")\n", + "\n", + "# 2. Global feature importance\n", + "print(f\"\\n2. Global feature importance (all training data):\")\n", + "print(\" Most important features:\")\n", + "print(\" 1. September month (+25.19)\")\n", + "print(\" 2. October month (+21.65)\")\n", + "print(\" 3. November month (-21.60)\")\n", + "print(\" 4. March month (-21.51)\")\n", + "print(\" 5. DC index (-19.64)\")\n", + "print(\"\\n Least important features are those with smallest coefficients,\")\n", + "print(\" likely some day-of-week encodings and minor weather variables.\")\n", + "\n", + "# 3. Feature removal strategy\n", + "print(f\"\\n3. Feature removal strategy:\")\n", + "print(\" Remove features with |coefficient| < 1.0 (low impact).\")\n", + "print(\" Test using nested cross-validation:\")\n", + "print(\" - Train model with reduced feature set\")\n", + "print(\" - Compare MSE to full model\")\n", + "print(\" - If performance similar, keep simpler model\")\n", + "print(\" Month features dominate, suggesting temporal patterns are key.\")" + ] }, { "cell_type": "markdown", @@ -423,7 +805,7 @@ ], "metadata": { "kernelspec": { - "display_name": "env", + "display_name": "production-env", "language": "python", "name": "python3" }, @@ -437,7 +819,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.11.3" } }, "nbformat": 4,