Skip to content

Assignment 2 #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
alf-99 wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Assignment 2 #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
35bdf0f
Complete assignment 1: Parquet feature engineering
Jan 19, 2026
740e51d
Complete: Forest Fire Regression Analysis Assignment
Jan 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
185 changes: 170 additions & 15 deletions 02_activities/assignments/assignment_1.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,17 +26,27 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 14,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"The dotenv extension is already loaded. To reload it, use:\n",
" %reload_ext dotenv\n"
]
}
],
"source": [
"# Write your code below.\n",
"\n"
"%load_ext dotenv\n",
"%dotenv \n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -55,15 +65,72 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 17,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Contents of c:\\Users\\anika\\OneDrive\\Documents\\DSI_cohort8\\Module-Production\\production\\05_src\\data\\prices:\n",
"['ACN', 'ALDX', 'ALL', 'AMAL', 'AMH', 'AQMS', 'BGS', 'BLPH', 'BPMX', 'BPYPN', 'BRQS', 'BWEN', 'BWG', 'CBB', 'CGEN', 'CMCTP', 'CRMT', 'CSSE', 'EARN', 'EOLS', 'ERH', 'ESGR', 'ETJ', 'FAMI', 'FIXX', 'GAZ', 'GLADD', 'GLUU', 'GLW', 'GURE', 'INSU', 'IPWR', 'ITCB', 'KALU', 'KEY', 'LEVL', 'LH', 'MNK', 'MOH', 'MOS', 'NGD', 'NPK', 'PFG', 'QRHC', 'REG', 'REI', 'RIV', 'RTTR', 'SLRX', 'SMG', 'SPXC', 'SRE', 'SYNH', 'TEF', 'TNC', 'TSN', 'VIAC', 'WORK', 'WST', 'ZIXI']\n"
]
}
],
"source": [
"import os\n",
"\n",
"price_data_path = \"c:\\\\Users\\\\anika\\\\OneDrive\\\\Documents\\\\DSI_cohort8\\\\Module-Production\\\\production\\\\05_src\\\\data\\\\prices\"\n",
"print(f\"Contents of {price_data_path}:\")\n",
"print(os.listdir(price_data_path))"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Found 2045 parquet files\n",
"Loaded Dask dataframe. Columns: ['Date', 'Open', 'High', 'Low', 'Close', 'Adj Close', 'Volume', 'source', 'ticker', 'Year']\n",
"\n",
"Sample data:\n",
" Date Open High Low Close Adj Close Volume source \\\n",
"137315 2001-07-19 15.10 15.29 15.00 15.17 11.404394 34994300.0 ACN.csv \n",
"137316 2001-07-20 15.05 15.05 14.80 15.01 11.284108 9238500.0 ACN.csv \n",
"137317 2001-07-23 15.00 15.01 14.55 15.00 11.276587 7501000.0 ACN.csv \n",
"137318 2001-07-24 14.95 14.97 14.70 14.86 11.171341 3537300.0 ACN.csv \n",
"137319 2001-07-25 14.70 14.95 14.65 14.95 11.238999 4208100.0 ACN.csv \n",
"\n",
" ticker Year \n",
"137315 ACN 2001 \n",
"137316 ACN 2001 \n",
"137317 ACN 2001 \n",
"137318 ACN 2001 \n",
"137319 ACN 2001 \n"
]
}
],
"source": [
"import os\n",
"from glob import glob\n",
"\n",
"# Write your code below.\n",
"\n"
"PRICE_DATA = os.getenv(\"PRICE_DATA\")\n",
"price_data_path = os.path.abspath(os.path.join(os.getcwd(), PRICE_DATA))\n",
"\n",
"# Find parquet files in all subdirectories (recursive)\n",
"parquet_files = glob(os.path.join(price_data_path, \"**\", \"*.parquet\"), recursive=True)\n",
"print(f\"Found {len(parquet_files)} parquet files\")\n",
"\n",
"# Load with Dask\n",
"dd_px = dd.read_parquet(parquet_files)\n",
"print(f\"Loaded Dask dataframe. Columns: {dd_px.columns.tolist()}\")\n",
"print(\"\\nSample data:\")\n",
"print(dd_px.head())\n"
]
},
{
Expand All @@ -88,12 +155,50 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 21,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\anika\\AppData\\Local\\Temp\\ipykernel_61500\\3849530006.py:7: UserWarning: `meta` is not specified, inferred from partial data.\n",
"Please provide `meta` if the result is unexpected.\n",
" Before: .shift(func)\n",
" After: .shift(func, meta={'x': 'f8', 'y': 'f8'}) for dataframe result\n",
" or: .shift(func, meta=('x', 'f8')) for series result\n",
"\n",
" Close_lag_1 = dd_px.groupby('ticker')['Close'].shift(1),\n",
"C:\\Users\\anika\\AppData\\Local\\Temp\\ipykernel_61500\\3849530006.py:8: UserWarning: `meta` is not specified, inferred from partial data.\n",
"Please provide `meta` if the result is unexpected.\n",
" Before: .shift(func)\n",
" After: .shift(func, meta={'x': 'f8', 'y': 'f8'}) for dataframe result\n",
" or: .shift(func, meta=('x', 'f8')) for series result\n",
"\n",
" Adj_Close_lag_1 = dd_px.groupby('ticker')['Adj Close'].shift(1)\n"
]
}
],
"source": [
"# Write your code below.\n",
"\n"
"\n",
"# Add lags for Close and Adj_Close\n",
"# Group by ticker to shift within each stock\n",
"dd_px = dd_px.set_index('ticker')\n",
"dd_feat = dd_px.assign(\n",
" Close_lag_1 = dd_px.groupby('ticker')['Close'].shift(1),\n",
" Adj_Close_lag_1 = dd_px.groupby('ticker')['Adj Close'].shift(1)\n",
")\n",
"\n",
"# Add returns: (Close / Close_lag_1) - 1\n",
"dd_feat = dd_feat.assign(\n",
" Returns = dd_feat['Close'] / dd_feat['Close_lag_1'] - 1\n",
")\n",
"\n",
"# Add hi_lo_range: High - Low\n",
"dd_feat = dd_feat.assign(\n",
" hi_lo_range = dd_feat['High'] - dd_feat['Low']\n",
")\n"
]
},
{
Expand All @@ -108,12 +213,57 @@
},
{
"cell_type": "code",
"execution_count": 25,
"execution_count": 24,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Conversion and moving average calculation complete!\n",
"DataFrame shape: (239659, 15)\n",
"Columns: ['ticker', 'Date', 'Open', 'High', 'Low', 'Close', 'Adj Close', 'Volume', 'source', 'Year', 'Close_lag_1', 'Adj_Close_lag_1', 'Returns', 'hi_lo_range', 'moving_average_return']\n",
"\n",
"Sample (first few rows):\n",
" ticker Date Returns moving_average_return\n",
"0 ACN 2001-07-19 NaN NaN\n",
"1 ACN 2001-07-20 -0.010547 -0.010547\n",
"2 ACN 2001-07-23 -0.000666 -0.005607\n",
"3 ACN 2001-07-24 -0.009333 -0.006849\n",
"4 ACN 2001-07-25 0.006057 -0.003623\n",
"5 ACN 2001-07-26 -0.030100 -0.008918\n",
"6 ACN 2001-07-27 0.000690 -0.007317\n",
"7 ACN 2001-07-30 0.013094 -0.004401\n",
"8 ACN 2001-07-31 0.017687 -0.001640\n",
"9 ACN 2001-08-01 0.036096 0.002553\n"
]
}
],
"source": [
"# Write your code below.\n",
"\n"
"\n",
"# 1. Convert Dask dataframe to pandas dataframe\n",
"df_feat = dd_feat.compute()\n",
"\n",
"# Reset index to make 'ticker' a column again\n",
"df_feat = df_feat.reset_index()\n",
"\n",
"# 2. Add 10-day moving average of Returns\n",
"# Sort by ticker and date for correct rolling calculation\n",
"df_feat = df_feat.sort_values(['ticker', 'Date'])\n",
"\n",
"# Calculate moving average\n",
"df_feat['moving_average_return'] = (\n",
" df_feat\n",
" .groupby('ticker', group_keys=False)\n",
" .apply(lambda x: x['Returns'].rolling(10, min_periods=1).mean())\n",
")\n",
"\n",
"print(\"Conversion and moving average calculation complete!\")\n",
"print(f\"DataFrame shape: {df_feat.shape}\")\n",
"print(f\"Columns: {df_feat.columns.tolist()}\")\n",
"print(\"\\nSample (first few rows):\")\n",
"print(df_feat[['ticker', 'Date', 'Returns', 'moving_average_return']].head(10))"
]
},
{
Expand All @@ -123,8 +273,13 @@
"Please comment:\n",
"\n",
"+ Was it necessary to convert to pandas to calculate the moving average return?\n",
"\n",
" I found I had to convert to pandas because Dask's .rolling() was giving me trouble with the grouped time-series. When I tried doing it in Dask, I got warnings about metadata and the calculation didn't work right. Pandas handled the grouped rolling window much more smoothly.\n",
"\n",
"+ Would it have been better to do it in Dask? Why?\n",
"\n",
" For this assignment, pandas was actually better. Our data isn't huge, so pandas could handle it fine in memory. Dask is really for when you have data too big for your RAM. Plus, I could check my work easier in pandas - if the moving average looked wrong, I could spot it right away. Dask would be better if we had gigabytes of data or needed to process it in parallel.\n",
"\n",
"(1 pt)"
]
},
Expand Down Expand Up @@ -165,7 +320,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "env",
"display_name": "production-env",
"language": "python",
"name": "python3"
},
Expand All @@ -179,7 +334,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.0"
"version": "3.11.3"
}
},
"nbformat": 4,
Expand Down
Loading