⽆监督异常检测之卷积AE和卷积VAE
尝试⽤卷积AE和卷积VAE做⽆监督检测，思路如下：
1.先⽤正常样本训练AE或VAE
2.输⼊测试集给AE或VAE，获得重构的测试集数据。
3.计算重构的数据和原始数据的误差，如果误差⼤于某⼀个阈值，则此测试样本为⼀样。
对于数据集的描述如下：
本数据集⼀共有10100个样本，每个样本是1⾏48列的向量，为了让它变成矩阵，⾃⼰在末尾补了⼀个0，将其转变成7*7的矩阵。前8000个是正常样本。后2100个中，前300个是正常样本，之后的1800个中包括6种异常时间序列，每种异常时间序列包括300个样本。
VAE的代码如下：
#blog.csdn/wyx100/article/details/80647379
'''This script demonstrates how to build a variational autoencoder
with Keras and deconvolution layers.
使⽤Keras和反卷积层建⽴变分⾃编码器演⽰脚本
# Reference
- Auto-Encoding Variational Bayes
⾃动编码变分贝叶斯
arxiv/abs/1312.6114
日记大全100字
'''
from__future__import print_function
黑龙江分数线import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import norm
from pandas import read_csv
from keras.layers import Input, Den, Lambda, Flatten, Reshape
from keras.layers import Conv2D, Conv2DTranspo
dels import Model
from keras import backend as K
from keras import metrics
import xlwt
from keras.datats import mnist
from matplotlib import pyplot
import numpy
# input image dimensions
五彩缤纷的意思
# 输⼊图像维度
img_rows, img_cols, img_chns = 7, 7, 1
dimension_image=7
# number of convolutional filters to u
# 使⽤的卷积过滤器数量
filters = 64
# convolution kernel size
# 卷积核⼤⼩
num_conv = 3
batch_size = 50
if K.image_data_format() == 'channels_first':
original_img_size = (img_chns, img_rows, img_cols)
el:
original_img_size = (img_rows, img_cols, img_chns)
latent_dim = 2
intermediate_dim = 128
epsilon_std = 1.0
epochs = 100
x = Input(shape=original_img_size)
conv_1 = Conv2D(img_chns,
kernel_size=(2, 2),
padding='same', activation='relu')(x)
conv_2 = Conv2D(filters,
kernel_size=(2, 2),
padding='same', activation='relu',
strides=(2, 2))(conv_1)
conv_3 = Conv2D(filters,
kernel_size=num_conv,
padding='same', activation='relu',
strides=1)(conv_2)
conv_4 = Conv2D(filters,
kernel_size=num_conv,
padding='same', activation='relu',
strides=1)(conv_3)
flat = Flatten()(conv_4)
hidden = Den(intermediate_dim, activation='relu')(flat)
z_mean = Den(latent_dim)(hidden)
z_log_var = Den(latent_dim)(hidden)
def sampling(args):
z_mean, z_log_var = args
epsilon = K.random_normal(shape=(K.shape(z_mean)[0], latent_dim),
mean=0., stddev=epsilon_std)
return z_mean + K.exp(z_log_var) * epsilon
# note that "output_shape" isn't necessary with the TensorFlow backend
# so you could write `Lambda(sampling)([z_mean, z_log_var])`
# 注意，“output_shape”对于TensorFlow后端不是必需的。因此可以编写Lambda(sampling)([z_mean, z_log_var])` z = Lambda(sampling, output_shape=(latent_dim,))([z_mean, z_log_var])
# we instantiate the layers parately so as to reu them later
# 分别实例化这些层，以便在以后重⽤它们。
number=4
decoder_hid = Den(intermediate_dim, activation='relu')
decoder_upsample = Den(filters * number * number, activation='relu')
if K.image_data_format() == 'channels_first':
output_shape = (batch_size, filters, number, number)
el:
苦字组词output_shape = (batch_size, number, number, filters)
decoder_reshape = Reshape(output_shape[1:])
decoder_deconv_1 = Conv2DTranspo(filters,
kernel_size=num_conv,
padding='same',
strides=1,
activation='relu')
decoder_deconv_2 = Conv2DTranspo(filters,
kernel_size=num_conv,
padding='same',
strides=1,
activation='relu')
if K.image_data_format() == 'channels_first':
output_shape = (batch_size, filters, 13, 13)
el:
output_shape = (batch_size,13, 13, filters)
decoder_deconv_3_upsamp = Conv2DTranspo(filters,
kernel_size=(3, 3),
strides=(2, 2),
padding='valid',
activation='relu')
decoder_mean_squash = Conv2D(img_chns,
kernel_size=3,
padding='valid',
activation='sigmoid')
hid_decoded = decoder_hid(z)
up_decoded = decoder_upsample(hid_decoded)
reshape_decoded = decoder_reshape(up_decoded)
deconv_1_decoded = decoder_deconv_1(reshape_decoded)
deconv_2_decoded = decoder_deconv_2(deconv_1_decoded)
x_decoded_relu = decoder_deconv_3_upsamp(deconv_2_decoded)
x_decoded_mean_squash = decoder_mean_squash(x_decoded_relu)
# instantiate VAE model
# 实例化VAE模型
vae = Model(x, x_decoded_mean_squash)
# Compute VAE loss
# 计算VAE损失
xent_loss = img_rows * img_cols * metrics.binary_crosntropy(
K.flatten(x),
K.flatten(x_decoded_mean_squash))
慈善英文
寿终正寝什么意思kl_loss = - 0.5 * K.sum(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
vae_loss = K.mean(xent_loss + kl_loss)
vae.add_loss(vae_loss)
vae.summary()
datat = read_csv('randperm_zerone_Datat.csv')
values = datat.values
XY= values
n_train_hours1 =7000
n_train_hours3 =8000
x_train=XY[:n_train_hours1,:]
x_valid =XY[n_train_hours1:n_train_hours3, :]
x_test =XY[n_train_hours3:, :]
x_train=shape(-1,dimension_image,dimension_image,1)
x_valid=shape(-1,dimension_image,dimension_image,1)
x_test=shape(-1,dimension_image,dimension_image,1)
history=vae.fit(x_train,
shuffle=True,
epochs=epochs,
batch_size=batch_size,
validation_data=(x_valid, None))
pyplot.plot(history.history['loss'], label='train')
pyplot.plot(history.history['val_loss'], label='valid')
pyplot.legend()
pyplot.show()
# 建⽴⼀个潜在空间输⼊模型
encoder = Model(x, z_mean)
# 在潜在空间中显⽰数字类的2D图
x_test_encoded = encoder.predict(x_test, batch_size=batch_size)
plt.figure(figsize=(6, 6))
plt.scatter(x_test_encoded[:, 0], x_test_encoded[:, 1])
plt.show()
Reconstructed_train = vae.predict(x_train)
Reconstructed_valid = vae.predict(x_valid)
Reconstructed_test = vae.predict(x_test)
ReconstructedData1=np.vstack((Reconstructed_train,Reconstructed_valid))
ReconstructedData2=np.vstack((ReconstructedData1,Reconstructed_test))
shape((ReconstructedData2.shape[0], -1))
numpy.savetxt("ReconstructedData.csv", ReconstructedData3, delimiter=',')
AE代码如下
from keras.layers import Input, Den, Conv2D, MaxPooling2D, UpSampling2D
dels import Model
from keras import backend as K
import numpy as np
from pandas import read_csv
from matplotlib import pyplot
import numpy
三年级下册必背古诗dimension_image=7
input_img = Input(shape=(dimension_image, dimension_image, 1)) # adapt this if using `channels_fi
rst` image data format x = Conv2D(16, (3, 3), activation='relu', padding='same')(input_img)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(8, (3, 3), activation='relu', padding='same')(x)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(8, (3, 3), activation='relu', padding='same')(x)
encoded = MaxPooling2D((2, 2), padding='same')(x)
# at this point the reprentation is (4, 4, 8) i.e. 128-dimensional
x = Conv2D(8, (3, 3), activation='relu', padding='same')(encoded)
x = UpSampling2D((2, 2))(x)
x = Conv2D(8, (3, 3), activation='relu', padding='same')(x)
x = UpSampling2D((2, 2))(x)
x = Conv2D(16, (3, 3), activation='relu', padding='same')(x)
x = UpSampling2D((2, 2))(x)
decoded = Conv2D(1, (2, 2), activation='sigmoid')(x)
autoencoder = Model(input_img, decoded)
autoencoder.summary()
datat = read_csv('randperm_zerone_Datat.csv')
values = datat.values
XY= values
n_train_hours1 =7000
n_train_hours3 =8000
x_train=XY[:n_train_hours1,:]
x_valid =XY[n_train_hours1:n_train_hours3, :]
x_test =XY[n_train_hours3:, :]
x_train=shape(-1,dimension_image,dimension_image,1)
x_valid=shape(-1,dimension_image,dimension_image,1)
x_test=shape(-1,dimension_image,dimension_image,1)
history=autoencoder.fit(x_train, x_train,
epochs=200,
batch_size=32,
shuffle=True,
validation_data=(x_valid, x_valid))
橙汁的功效与作用pyplot.plot(history.history['loss'], label='train')
pyplot.plot(history.history['val_loss'], label='valid')
pyplot.legend()
pyplot.show()
Reconstructed_train = autoencoder.predict(x_train)
Reconstructed_valid = autoencoder.predict(x_valid)
Reconstructed_test = autoencoder.predict(x_test)
ReconstructedData1=np.vstack((Reconstructed_train,Reconstructed_valid)) ReconstructedData2=np.vstack((ReconstructedData1,Reconstructed_test)) shape((ReconstructedData2.shape[0], -1))
numpy.savetxt("ReconstructedData.csv", ReconstructedData3, delimiter=',')
⾄于数据集，正在上传到百度⽂库，以后更新
