diff --git a/data/processed/census_income_SVM.csv b/data/processed/census_income_SVM_MLP.csv
similarity index 100%
rename from data/processed/census_income_SVM.csv
rename to data/processed/census_income_SVM_MLP.csv
diff --git a/notebooks/classification/preprocessing_ishwari.ipynb b/notebooks/classification/preprocessing_ishwari.ipynb
index bd1283ab19c4d624f214c7d60d1e417b18e14c97..7d7ed0956b9507e525c4625fd62cb1f40917f46b 100644
--- a/notebooks/classification/preprocessing_ishwari.ipynb
+++ b/notebooks/classification/preprocessing_ishwari.ipynb
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
- "execution_count": 961,
+ "execution_count": 172,
"metadata": {},
"outputs": [
{
@@ -161,7 +161,7 @@
"4 0 0 40 Cuba <=50K "
]
},
- "execution_count": 961,
+ "execution_count": 172,
"metadata": {},
"output_type": "execute_result"
}
@@ -175,7 +175,7 @@
},
{
"cell_type": "code",
- "execution_count": 962,
+ "execution_count": 173,
"metadata": {},
"outputs": [
{
@@ -213,7 +213,7 @@
},
{
"cell_type": "code",
- "execution_count": 963,
+ "execution_count": 174,
"metadata": {},
"outputs": [
{
@@ -245,7 +245,7 @@
},
{
"cell_type": "code",
- "execution_count": 964,
+ "execution_count": 175,
"metadata": {},
"outputs": [
{
@@ -288,7 +288,7 @@
},
{
"cell_type": "code",
- "execution_count": 965,
+ "execution_count": 176,
"metadata": {},
"outputs": [
{
@@ -329,7 +329,7 @@
},
{
"cell_type": "code",
- "execution_count": 966,
+ "execution_count": 177,
"metadata": {},
"outputs": [
{
@@ -347,7 +347,7 @@
},
{
"cell_type": "code",
- "execution_count": 967,
+ "execution_count": 178,
"metadata": {},
"outputs": [
{
@@ -368,7 +368,7 @@
},
{
"cell_type": "code",
- "execution_count": 968,
+ "execution_count": 179,
"metadata": {},
"outputs": [],
"source": [
@@ -394,7 +394,7 @@
},
{
"cell_type": "code",
- "execution_count": 969,
+ "execution_count": 180,
"metadata": {},
"outputs": [
{
@@ -522,7 +522,7 @@
"Income 0.227687 1.000000 "
]
},
- "execution_count": 969,
+ "execution_count": 180,
"metadata": {},
"output_type": "execute_result"
}
@@ -557,7 +557,7 @@
},
{
"cell_type": "code",
- "execution_count": 970,
+ "execution_count": 181,
"metadata": {},
"outputs": [
{
@@ -609,7 +609,7 @@
},
{
"cell_type": "code",
- "execution_count": 971,
+ "execution_count": 182,
"metadata": {},
"outputs": [
{
@@ -664,7 +664,7 @@
},
{
"cell_type": "code",
- "execution_count": 972,
+ "execution_count": 183,
"metadata": {},
"outputs": [
{
@@ -732,7 +732,7 @@
},
{
"cell_type": "code",
- "execution_count": 973,
+ "execution_count": 184,
"metadata": {},
"outputs": [],
"source": [
@@ -748,7 +748,7 @@
},
{
"cell_type": "code",
- "execution_count": 974,
+ "execution_count": 185,
"metadata": {},
"outputs": [],
"source": [
@@ -764,7 +764,7 @@
},
{
"cell_type": "code",
- "execution_count": 975,
+ "execution_count": 186,
"metadata": {},
"outputs": [],
"source": [
@@ -773,7 +773,7 @@
},
{
"cell_type": "code",
- "execution_count": 976,
+ "execution_count": 187,
"metadata": {},
"outputs": [
{
@@ -801,7 +801,7 @@
},
{
"cell_type": "code",
- "execution_count": 977,
+ "execution_count": 188,
"metadata": {},
"outputs": [
{
@@ -846,7 +846,7 @@
},
{
"cell_type": "code",
- "execution_count": 978,
+ "execution_count": 189,
"metadata": {},
"outputs": [],
"source": [
@@ -856,7 +856,7 @@
},
{
"cell_type": "code",
- "execution_count": 979,
+ "execution_count": 190,
"metadata": {},
"outputs": [
{
@@ -896,7 +896,7 @@
},
{
"cell_type": "code",
- "execution_count": 980,
+ "execution_count": 191,
"metadata": {},
"outputs": [
{
@@ -925,7 +925,7 @@
},
{
"cell_type": "code",
- "execution_count": 981,
+ "execution_count": 192,
"metadata": {},
"outputs": [
{
@@ -987,7 +987,7 @@
},
{
"cell_type": "code",
- "execution_count": 982,
+ "execution_count": 193,
"metadata": {},
"outputs": [],
"source": [
@@ -1006,7 +1006,7 @@
},
{
"cell_type": "code",
- "execution_count": 983,
+ "execution_count": 194,
"metadata": {},
"outputs": [],
"source": [
@@ -1026,42 +1026,6 @@
"df['education'] = df['education'].map(education_map)"
]
},
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "##### Preprocessing race column"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 984,
- "metadata": {},
- "outputs": [],
- "source": [
- "# # One-hot encoding the 'race' column\n",
- "# df = pd.get_dummies(df, columns=['race'], prefix='race')\n",
- "# df.to_csv('../../data/processed/processed_census_income_dataset.csv', index=False)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "##### Preprocessing relationship column"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 985,
- "metadata": {},
- "outputs": [],
- "source": [
- "# # One-hot encoding the 'relationship' column\n",
- "# df = pd.get_dummies(df, columns=['relationship'], prefix='rel')\n",
- "# df.to_csv('../../data/processed/processed_census_income_dataset.csv', index=False)"
- ]
- },
{
"cell_type": "markdown",
"metadata": {},
@@ -1071,7 +1035,7 @@
},
{
"cell_type": "code",
- "execution_count": 986,
+ "execution_count": 195,
"metadata": {},
"outputs": [],
"source": [
@@ -1098,7 +1062,7 @@
},
{
"cell_type": "code",
- "execution_count": 987,
+ "execution_count": 196,
"metadata": {},
"outputs": [],
"source": [
@@ -1117,7 +1081,7 @@
},
{
"cell_type": "code",
- "execution_count": 988,
+ "execution_count": 197,
"metadata": {},
"outputs": [
{
@@ -1151,7 +1115,7 @@
},
{
"cell_type": "code",
- "execution_count": 989,
+ "execution_count": 198,
"metadata": {},
"outputs": [],
"source": [
@@ -1175,7 +1139,7 @@
},
{
"cell_type": "code",
- "execution_count": 990,
+ "execution_count": 199,
"metadata": {},
"outputs": [],
"source": [
@@ -1186,7 +1150,7 @@
},
{
"cell_type": "code",
- "execution_count": 991,
+ "execution_count": 200,
"metadata": {},
"outputs": [],
"source": [
@@ -1229,7 +1193,7 @@
},
{
"cell_type": "code",
- "execution_count": 992,
+ "execution_count": 201,
"metadata": {},
"outputs": [],
"source": [
@@ -1240,7 +1204,7 @@
},
{
"cell_type": "code",
- "execution_count": 993,
+ "execution_count": 202,
"metadata": {},
"outputs": [],
"source": [
@@ -1250,7 +1214,7 @@
},
{
"cell_type": "code",
- "execution_count": 994,
+ "execution_count": 203,
"metadata": {},
"outputs": [],
"source": [
@@ -1283,14 +1247,13 @@
" return 'Other'\n",
"\n",
"\n",
- "# Apply mapping\n",
"df['region'] = df['native-country'].apply(map_country_to_region)\n",
"df.drop(['native-country'], axis=1, inplace=True)"
]
},
{
"cell_type": "code",
- "execution_count": 995,
+ "execution_count": 204,
"metadata": {},
"outputs": [],
"source": [
@@ -1301,7 +1264,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "#### Decision Tree specific Encoding"
+ "### Preprocessing for Decision Tree"
]
},
{
@@ -1310,7 +1273,7 @@
"source": [
"Decision Trees split data based on feature values. For categorical features, they only need a way to distinguish between categories — the model does **not assume any ordinal relationship**, so **label encoding** is appropriate.\n",
"\n",
- "Hence, we are appling `LabelEncoder` to the following categorical columns:\n",
+ "Hence, we are applying `LabelEncoder` to the following categorical columns:\n",
"- `workclass`\n",
"- `marital-status`\n",
"- `relationship`\n",
@@ -1323,7 +1286,7 @@
},
{
"cell_type": "code",
- "execution_count": 996,
+ "execution_count": 205,
"metadata": {},
"outputs": [],
"source": [
@@ -1343,7 +1306,7 @@
},
{
"cell_type": "code",
- "execution_count": 997,
+ "execution_count": 206,
"metadata": {},
"outputs": [
{
@@ -1409,7 +1372,7 @@
},
{
"cell_type": "code",
- "execution_count": 998,
+ "execution_count": 207,
"metadata": {},
"outputs": [],
"source": [
@@ -1421,21 +1384,30 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "#### Support Vector Machine (SVM) specific Encoding"
+ "### Preprocessing for Support Vector Machine (SVM) and Multi Layer Perceptron (MLP)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "Support Vector Machines (SVMs) are highly effective classifiers, but they are sensitive to the format and scale of input data. This sensitivity arises from the way SVMs compute the optimal hyperplane — using dot products and distances between feature vectors.\n",
+ "Support Vector Machines (SVMs) and Multilayer Perceptrons (MLPs) both require careful preprocessing due to their sensitivity to input feature scale and format.\n",
+ "\n",
+ "SVMs rely on dot products and distance-based computations to define decision boundaries. Without standardized features, larger-scale variables can dominate these calculations, leading to biased or unstable models.\n",
"\n",
- "Therefore, proper preprocessing is essential to ensure the model learns effectively and avoids bias from feature magnitude or misinterpreted categorical relationships.\n"
+ "MLPs, which learn via gradient descent and backpropagation, are similarly affected. Unscaled inputs can cause inefficient training, slow convergence, or issues like vanishing gradients — especially with deeper architectures.\n",
+ "\n",
+ "Both models also require categorical data to be encoded numerically. One-hot encoding is preferred over label encoding to avoid introducing false ordinal relationships.\n",
+ "\n",
+ "In summary, preprocessing for SVMs and MLPs typically involves:\n",
+ "- One-hot encoding of categorical variables\n",
+ "- Standardization of numerical features\n",
+ "- Target is binary\n"
]
},
{
"cell_type": "code",
- "execution_count": 999,
+ "execution_count": 208,
"metadata": {},
"outputs": [
{
@@ -1563,7 +1535,7 @@
"4 40 0 Prof-specialty Latin America "
]
},
- "execution_count": 999,
+ "execution_count": 208,
"metadata": {},
"output_type": "execute_result"
}
@@ -1572,7 +1544,7 @@
"from sklearn.preprocessing import StandardScaler\n",
"import pandas as pd\n",
"\n",
- "# Load the pre-encoded dataframe\n",
+ "# Loading the pre-encoded dataframe\n",
"df_svm = pd.read_csv('../../data/processed/before_model_specific_encoding.csv')\n",
"df_svm.head()\n"
]
@@ -1592,7 +1564,7 @@
},
{
"cell_type": "code",
- "execution_count": 1000,
+ "execution_count": 209,
"metadata": {},
"outputs": [
{
@@ -1818,7 +1790,7 @@
"[5 rows x 40 columns]"
]
},
- "execution_count": 1000,
+ "execution_count": 209,
"metadata": {},
"output_type": "execute_result"
}
@@ -1835,18 +1807,20 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "SVMs are sensitive to feature scales because they rely on computing distances and hyperplanes. If one feature has a larger numerical range than another, it can dominate the learning process unfairly.\n",
+ "SVMs rely on distance calculations and hyperplanes to separate classes, while MLPs depend on gradient-based optimization and weighted activations across layers. In both cases, if one feature has a much larger numerical range than others, it can dominate the learning process, leading to biased or inefficient training.\n",
"\n",
- "We used StandardScaler to standardize the following features:\n",
+ "To ensure balanced learning and faster convergence, we are standardizing the following numerical features using `StandardScaler`:\n",
"\n",
"- `age`\n",
"- `education`\n",
- "- `hours-per-week`"
+ "- `hours-per-week`\n",
+ "\n",
+ "This transformation scales each feature to have a mean of 0 and standard deviation of 1, which helps both models interpret and optimize feature weights effectively."
]
},
{
"cell_type": "code",
- "execution_count": 1001,
+ "execution_count": 210,
"metadata": {},
"outputs": [
{
@@ -2072,7 +2046,7 @@
"[5 rows x 40 columns]"
]
},
- "execution_count": 1001,
+ "execution_count": 210,
"metadata": {},
"output_type": "execute_result"
}
@@ -2086,7 +2060,7 @@
},
{
"cell_type": "code",
- "execution_count": 1002,
+ "execution_count": 211,
"metadata": {},
"outputs": [],
"source": [
@@ -2096,12 +2070,12 @@
},
{
"cell_type": "code",
- "execution_count": 1003,
+ "execution_count": 212,
"metadata": {},
"outputs": [],
"source": [
"# Saved the SVM preprocessed dataset\n",
- "df_svm.to_csv('../../data/processed/census_income_SVM.csv', index=False)"
+ "df_svm.to_csv('../../data/processed/census_income_SVM_MLP.csv', index=False)"
]
}
],