-
Notifications
You must be signed in to change notification settings - Fork 36
/
Copy pathkdd_preprocessing.py
42 lines (31 loc) · 1.05 KB
/
kdd_preprocessing.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
'''
corey and vishal monster creation
loading kdd99 cup data into pandas dataframe
'''
import pandas, re, numpy as np
def load_file(filename, num_cols, dtypes, delimiter='\t'):
data = None
try:
data = np.fromfile(filename + '.npy', dtype=dtypes)
except:
splitter = re.compile(delimiter)
def items(infile):
for line in infile:
for item in splitter.split(line):
yield item
with open(filename, 'r') as infile:
data = np.fromiter(items(infile), dtypes, -1)
data = data.reshape((-1, num_cols))
np.save(filename, data)
return pandas.DataFrame(data)
if __name__ == "__main__":
df = pd.read_csv('downsample.csv')
types = []
for i in df.columns:
try:
i = float(i)
types.append(type(i))
except ValueError:
types.append(type(i))
dtypes = np.dtype([('feature_'+str(i),types[i]) for i in range(len(types))])
load_file('kddcup.data',42, dtypes, delimiter=',')