import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.neighbors import KNeighborsClassifier
import pickle

data = pd.read_hdf("data/hy_round1_train_20200102.h5")
print(data.shape)
print(data.columns)
# 校验是否存在NaN
# print(np.all(pd.notnull(data)))

# 载入数据集
data = np.array(data)
# 量化标签
# print(np.where(data[:,-1] == "刺网"))
data[np.where(data[:,-1] == "刺网"),-1] = 0
data[np.where(data[:,-1] == "围网"),-1] = 1
data[np.where(data[:,-1] == "拖网"),-1] = 2
x_train, x_test, y_train, y_test = train_test_split(data[:,:-2],data[:,-1],test_size=0.25,random_state=42)
# 标准化（归一化存在离群点影响弊端）
transfer = StandardScaler()
x_train = transfer.fit_transform(x_train)
x_test = transfer.fit_transform(x_test)
y_train = y_train.astype("int")
y_test = y_test.astype("int")
print(y_train)

# 模型训练
estimator = KNeighborsClassifier(n_neighbors=5)
estimator.fit(x_train,y_train)

# 保存模型
f = open("model/hy_knn_model.pickle","wb")
pickle.dump(estimator, f)

# 模型评估
# 预测值
y_predict = estimator.predict(x_test)
print("模型预测结果：\n", y_predict)
print("真实值对比结果：\n", y_test == y_predict)

# 准确率
score = estimator.score(x_test,y_test)
print("模型准确率：",score)

from tensorflow import keras
from tensorflow.python.keras.layers import Dense, Dropout, Activation, Flatten, Conv1D, MaxPool1D
import matplotlib.pyplot as plt
import numpy as np
import os
 
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
 
# 数据集路径
dataset_path = 'dataset.npy'
# 接入softmax的全连接层维数
dense_num = 6
# 保存的模型文件路径
model_file = 'model/cnn_6traffic_model.h5'

# 数据集存储结构
# [training_images, training_labels, validation_images, validation_labels, testing_images, testing_labels]
data = np.load(dataset_path, allow_pickle=True)
 
x_train, y_train, x_test, y_test = np.array(data[0]), np.array(data[1]), np.array(data[2]), np.array(data[3])
print(x_train.shape, y_train.shape)
 
# 一维化
X_train = x_train.reshape(-1, 784, 1)
# print(X_train)
# X_train = X_train.astype('float32')
X_test = x_test.reshape(-1, 784, 1)
# X_test = X_test.astype('float32')
 
 
# 将像素值做归一化，也就是从0~255的取值压缩到0~1之间
# X_train /= 255
# X_test /= 255
 
# 构建模型
model = keras.models.Sequential()
 
# 卷积层1 + relu
# 25 卷积核的数量 即输出的维度
# 3 每个过滤器的长度
model.add(Conv1D(32, 3, activation='relu', input_shape=(784, 1), padding="same"))
# 池化层1
model.add(MaxPool1D(pool_size=3, strides=3))
 
# 卷积层2 + relu
model.add(Conv1D(64, 3, strides=1, activation='relu', padding='same'))
# 池化层2
model.add(MaxPool1D(pool_size=3, strides=3))
 
# 神经元随机失活
model.add(Dropout(0.25))
# 拉成一维数据
model.add(Flatten())
# 全连接层1
model.add(Dense(1024))
# 激活层
model.add(Activation('relu'))
 
# 随机失活
model.add(Dropout(0.4))
# 全连接层2
model.add(Dense(dense_num))
# Softmax评分
model.add(Activation('softmax'))
 
# 查看定义的模型
model.summary()
 
# 自定义优化器参数
# rmsprop = keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
 
# lr表示学习速率
# decay是学习速率的衰减系数(每个epoch衰减一次)
# momentum表示动量项
# Nesterov的值是False或者True，表示使不使用Nesterov momentum
sgd = keras.optimizers.SGD(lr=0.01, decay=1e-4, momentum=0.9, nesterov=True)
 
model.compile(optimizer='sgd', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
 
# 训练
history = model.fit(X_train, y_train, epochs=10, batch_size=1000,
                    verbose=1, validation_data=[X_test, y_test])
 
model.save(model_file)
print(history.params)

import numpy as np
import matplotlib.pyplot as plt

x = np.random.uniform(-10,10,size=100)
y = .4 * x + 3 + np.random.uniform(-1,1,size=100)

f = open('dataset/simulation_data_1d.txt','w')
for i in range(0,100):
    line = str(x[i]) + ',' + str(y[i])
    f.write(line)
    f.write('\n')

plt.plot(x,y,'b.')
plt.show()

import matplotlib.pyplot as plt
import random

data = open('dataset/simulation_data_1d.txt', 'r').read()
# print(data.split('\n'))

# 数据读取
x = []
y = []
for item in data.split('\n'):
    if item != '':
        item = item.split(',')
        x.append(float(item[0]))
        y.append(float(item[1]))


# 数据集构建
def seperateData(x, y, test_scale):
    # 将x，y进行矩阵组合
    data = []
    for _x in x:
        i = x.index(_x)
        data.append([])
        if type(_x) == 'list':
            for feature in range(len(_x)):
                data[i].append(feature)
            data[i].append(y[i])
        else:
            data[i].append(_x)
            data[i].append(y[i])

    # 切割数据集
    train = []
    test = []
    random.seed(3)
    for i in range(len(data)):
        count = random.random()
        if count > test_scale:
            train.append(data[i])
        elif count < test_scale:
            test.append(data[i])
    return train, test


train, test = seperateData(x, y, 0.2)

# 处理成能够用于模型训练的数据结构
X_train = []
Y_train = []
X_test = []
Y_test = []
for item in train:
    X_train.append(item[:-1])
    Y_train.append(item[-1])
for item in test:
    X_test.append(item[:-1])
    Y_test.append(item[-1])

Xsum = 0.0
Ysum = 0.0
XY = 0.0
X2sum = 0.0
n = len(Y_train)
# 推导的算法结果部分的实现
for i in range(n):
    Xsum += X_train[i][0]
    Ysum += Y_train[i]
    XY += X_train[i][0] * Y_train[i]
    X2sum += X_train[i][0] ** 2
k = (n * XY - Xsum * Ysum) / (n * X2sum  - Xsum ** 2)
b = (Ysum - k * Xsum) / n
print("拟合的曲线：y =",k,"* x +",b)

plt.plot(X_train, Y_train, 'b.')
plt.plot(X_test, Y_test, 'r*')
plt.xlabel("Trian_Area")
plt.ylabel("Trian_Price")

# 绘制拟合的曲线
Y = []
for item in X_test:
    Y.append(k * item[0] + b)
plt.plot(X_test, Y, 'g-')

plt.show()