数据离散化
- 数据离散化就是分箱
- 一把你常用分箱方法是等频分箱或者等宽分箱
- 一般使用pd.cut或者pd.qcut函数
pandas.cut(x, bins, right=True, labels)
- x: 数据
- bins: 离散化的数目，或者切分的区间
- labels: 离散化后各个类别的标签
- right: 是否包含区间右边的值
import pandas as pd
import numpy as np
import osos.getcwd()'D:\\Jupyter\\notebook\\Python数据清洗实战\\数据'os.chdir('D:\\Jupyter\\notebook\\Python数据清洗实战\\数据')df = pd.read_csv('MotorcycleData.csv', encoding='gbk', na_values='Na')def f(x):
if '$' in str(x):
x = str(x).strip('$')
x = str(x).replace(',', '')
else:
x = str(x).replace(',', '')
return float(x)df['Price'] = df['Price'].apply(f)df['Mileage'] = df['Mileage'].apply(f)df.head(5)<div>
<style scoped>
.dataframe tbody tr th:only-of-type { vertical-align: middle;}.dataframe tbody tr th { vertical-align: top;}.dataframe thead th { text-align: right;}</style>
<table border=”1″ class=”dataframe”>
<thead>
<tr style="text-align: right;"> <th></th> <th>Condition</th> <th>Condition_Desc</th> <th>Price</th> <th>Location</th> <th>Model_Year</th> <th>Mileage</th> <th>Exterior_Color</th> <th>Make</th> <th>Warranty</th> <th>Model</th> <th>...</th> <th>Vehicle_Title</th> <th>OBO</th> <th>Feedback_Perc</th> <th>Watch_Count</th> <th>N_Reviews</th> <th>Seller_Status</th> <th>Vehicle_Tile</th> <th>Auction</th> <th>Buy_Now</th> <th>Bid_Count</th></tr></thead>
<tbody>
<tr> <th>0</th> <td>Used</td> <td>mint!!! very low miles</td> <td>11412.0</td> <td>McHenry, Illinois, United States</td> <td>2013.0</td> <td>16000.0</td> <td>Black</td> <td>Harley-Davidson</td> <td>Unspecified</td> <td>Touring</td> <td>...</td> <td>NaN</td> <td>FALSE</td> <td>8.1</td> <td>NaN</td> <td>2427</td> <td>Private Seller</td> <td>Clear</td> <td>True</td> <td>FALSE</td> <td>28.0</td></tr><tr> <th>1</th> <td>Used</td> <td>Perfect condition</td> <td>17200.0</td> <td>Fort Recovery, Ohio, United States</td> <td>2016.0</td> <td>60.0</td> <td>Black</td> <td>Harley-Davidson</td> <td>Vehicle has an existing warranty</td> <td>Touring</td> <td>...</td> <td>NaN</td> <td>FALSE</td> <td>100</td> <td>17</td> <td>657</td> <td>Private Seller</td> <td>Clear</td> <td>True</td> <td>TRUE</td> <td>0.0</td></tr><tr> <th>2</th> <td>Used</td> <td>NaN</td> <td>3872.0</td> <td>Chicago, Illinois, United States</td> <td>1970.0</td> <td>25763.0</td> <td>Silver/Blue</td> <td>BMW</td> <td>Vehicle does NOT have an existing warranty</td> <td>R-Series</td> <td>...</td> <td>NaN</td> <td>FALSE</td> <td>100</td> <td>NaN</td> <td>136</td> <td>NaN</td> <td>Clear</td> <td>True</td> <td>FALSE</td> <td>26.0</td></tr><tr> <th>3</th> <td>Used</td> <td>CLEAN TITLE READY TO RIDE HOME</td> <td>6575.0</td> <td>Green Bay, Wisconsin, United States</td> <td>2009.0</td> <td>33142.0</td> <td>Red</td> <td>Harley-Davidson</td> <td>NaN</td> <td>Touring</td> <td>...</td> <td>NaN</td> <td>FALSE</td> <td>100</td> <td>NaN</td> <td>2920</td> <td>Dealer</td> <td>Clear</td> <td>True</td> <td>FALSE</td> <td>11.0</td></tr><tr> <th>4</th> <td>Used</td> <td>NaN</td> <td>10000.0</td> <td>West Bend, Wisconsin, United States</td> <td>2012.0</td> <td>17800.0</td> <td>Blue</td> <td>Harley-Davidson</td> <td>NO WARRANTY</td> <td>Touring</td> <td>...</td> <td>NaN</td> <td>FALSE</td> <td>100</td> <td>13</td> <td>271</td> <td>OWNER</td> <td>Clear</td> <td>True</td> <td>TRUE</td> <td>0.0</td></tr></tbody>
</table>
<p>5 rows × 22 columns</p>
</div>
df['Price_bin'] = pd.cut(df['Price'], 5, labels=range(5))# 计算频数
df['Price_bin'].value_counts()0 67621 6592 503 204 2Name: Price_bin, dtype: int64%matplotlib inlinedf['Price_bin'].value_counts().plot(kind='bar')<matplotlib.axes._subplots.AxesSubplot at 0x1b35fba9048>df['Price_bin'].hist()<matplotlib.axes._subplots.AxesSubplot at 0x1b35f681278>w = [100, 1000, 5000, 10000, 20000, 100000]df['Price_bin'] = pd.cut(df['Price'], bins=w, labels=range(5))df[['Price', 'Price_bin']].head(5)<div>
<style scoped>
.dataframe tbody tr th:only-of-type { vertical-align: middle;}.dataframe tbody tr th { vertical-align: top;}.dataframe thead th { text-align: right;}</style>
<table border=”1″ class=”dataframe”>
<thead>
<tr style="text-align: right;"> <th></th> <th>Price</th> <th>Price_bin</th></tr></thead>
<tbody>
<tr> <th>0</th> <td>11412.0</td> <td>3</td></tr><tr> <th>1</th> <td>17200.0</td> <td>3</td></tr><tr> <th>2</th> <td>3872.0</td> <td>1</td></tr><tr> <th>3</th> <td>6575.0</td> <td>2</td></tr><tr> <th>4</th> <td>10000.0</td> <td>2</td></tr></tbody>
</table>
</div>
df['Price_bin'].hist()<matplotlib.axes._subplots.AxesSubplot at 0x1b35fb99898># 分位数
k = 5
w = [1.0 * i/k for i in range(k+1)]
w[0.0, 0.2, 0.4, 0.6, 0.8, 1.0]# 等频分成5段
df['Price_bin'] = pd.qcut(df['Price'], q=w, labels=range(5))df['Price_bin'].hist()<matplotlib.axes._subplots.AxesSubplot at 0x1b35fe2a080># 计算分位点
k = 5
w1 = df['Price'].quantile([1.0 * i/k for i in range(k+1)])w10.0 0.00.2 3500.00.4 6491.00.6 9777.00.8 14999.01.0 100000.0Name: Price, dtype: float64# 一般第一个分位点要比实际小
# 最后一个分位点要比实际大
w1[0] = w[0] * 0.95
w1[1.0] = w1[1.0] * 1.1w10.0 0.00.2 3500.00.4 6491.00.6 9777.00.8 14999.01.0 110000.0Name: Price, dtype: float64# 按照新的分段标准分割
df['Price_bin'] = pd.cut(df['Price'], bins=w1, labels=range(5))df['Price_bin'].hist()<matplotlib.axes._subplots.AxesSubplot at 0x1b35e53fa20>正文完