labeled

ladderNet代码分析

importtensorflowastf

importinput_data

importmath

importos

importcsv

fromtqdmimporttqdm

layer_sizes=[784,1000,500,250,250,250,10]

L=len(layer_sizes)-1#numberoflayers

num_examples=60000

num_epochs=150

num_labeled=100

starter_learning_rate=0.02

decay_after=15#epochafterwhichtobeginlearningratedecay15个epoch之后LR降低

batch_size=100

num_iter=(num_examples/batch_size)*num_epochs#numberofloopiterations

inputs=older(32,shape=(None,layer_sizes[0]))

outputs=older(32)

defbi(inits,size,name):

le(inits*([size]),name=name)

defwi(shape,name):

le(_normal(shape,name=name))/(shape[0])

#把lay_sizes打包成元组列表[（784，1000），（1000,500）.........]

#shapes=zip(layer_sizes[:-1],layer_sizes[1:])#shapesoflinearlayers

shapes=list(zip(list(layer_sizes)[:-1],list(layer_sizes[1:])))

weights={'W':[wi(s,"W")forsinshapes],#Encoderweights

'V':[wi(s[::-1],"V")forsinshapes],#Decoderweights

#batchnormalizationparametertoshiftthenormalizedvalue

'beta':[bi(0.0,layer_sizes[l+1],"beta")forlinrange(L)],

#batchnormalizationparametertoscalethenormalizedvalue

'gamma':[bi(1.0,layer_sizes[l+1],"beta")forlinrange(L)]}

noi_std=0.3#scalingfactorfornoiudincorruptedencoder

#hyperparametersthatdenotetheimportanceofeachlayer每层的权重

denoising_cost=[1000.0,10.0,0.10,0.10,0.10,0.10,0.10]

join=lambdal,u:([l,u],0)#组合tensor

#制造数据，有监督⼤⼩是bacth_size，⽆监督⼤⼩是所有样本减去bacth_size

labeled=lambdax:(x,[0,0],[batch_size,-1])ifxisnotNoneelx

unlabeled=lambdax:(x,[batch_size,0],[-1,-1])ifxisnotNoneelx

split_lu=lambdax:(labeled(x),unlabeled(x))

training=older()

#tocalculatethemovingaveragesofmeanandvariance计算滑动平均

#滑动平均更新参数依据公式：shadow_variable（cur）=decay*shadow_variable（pre）+(1-decay)*variable

#其中，decay控制更新速率，⼀般取值0.9以上，值越⼤，更新越慢，值越稳定；

#variable是当前变量的值；shadow_variable（pre）为上⼀次更新参数值；shadow_variable（cur）当前更新参数值。

#variable是当前变量的值；shadow_variable（pre）为上⼀次更新参数值；shadow_variable（cur）当前更新参数值。

ewma=ntialMovingAverage(decay=0.99)

bn_assigns=[]#thisliststorestheupdatestobemadetoaveragemeanandvariance

#批归⼀化处理

#1.加快训练速度

#2.可以省去dropout，L1,L2等正则化处理⽅法

#3.提⾼模型训练精度

defbatch_normalization(batch,mean=None,var=None):

ifmeanisNoneorvarisNone:

mean,var=s(batch,axes=[0])#计算平均值和⽅差

return(batch-mean)/(var+nt(1e-10))

#averagemeanandvarianceofalllayers占位list[le()]

running_mean=[le(nt(0.0,shape=[l]),trainable=Fal)forlinlayer_sizes[1:]]

running_var=[le(nt(1.0,shape=[l]),trainable=Fal)forlinlayer_sizes[1:]]

defupdate_batch_normalization(batch,l):

"batchnormalize+updateaveragemeanandvarianceoflayerl"

mean,var=s(batch,axes=[0])

assign_mean=running_mean[l-1].assign(mean)#赋值

assign_var=running_var[l-1].assign(var)

bn_(([running_mean[l-1],running_var[l-1]]))

l_dependencies([assign_mean,assign_var]):

return(batch-mean)/(var+1e-10)

defencoder(inputs,noi_std):

h=inputs+_normal((inputs))*noi_std#addnoitoinput

d={}#tostorethepre-activation,activation,meanandvarianceforeachlayer

#Thedataforlabeledandunlabeledexamplesarestoredparately

d['labeled']={'z':{},'m':{},'v':{},'h':{}}

d['unlabeled']={'z':{},'m':{},'v':{},'h':{}}

d['labeled']['z'][0],d['unlabeled']['z'][0]=split_lu(h)

forlinrange(1,L+1):

print("Layer",l,":",layer_sizes[l-1],"->",layer_sizes[l])

d['labeled']['h'][l-1],d['unlabeled']['h'][l-1]=split_lu(h)

z_pre=(h,weights['W'][l-1])#pre-activation

z_pre_l,z_pre_u=split_lu(z_pre)#splitlabeledandunlabeledexamples

m,v=s(z_pre_u,axes=[0])

#iftraining:

deftraining_batch_norm():

#Trainingbatchnormalization

#batchnormalizationforlabeledandunlabeledexamplesisperformedparately

ifnoi_std>0:

#Corruptedencoder

#batchnormalization+noi

z=join(batch_normalization(z_pre_l),batch_normalization(z_pre_u,m,v))

z+=_normal((z_pre))*noi_std

el:

#Cleanencoder

#batchnormalization+updatetheaveragemeanandvarianceusingbatchmeanandvarianceoflabeledexamples

z=join(update_batch_normalization(z_pre_l,l),batch_normalization(z_pre_u,m,v))

returnz

#el:

defeval_batch_norm():

#Evaluationbatchnormalization

#obtainaveragemeanandvarianceanduittonormalizethebatch

mean=e(running_mean[l-1])#返回平均值是running_mean[l-1]的变量

var=e(running_var[l-1])#返回平均值是running_var[l-1]的变量

z=batch_normalization(z_pre,mean,var)

z=batch_normalization(z_pre,mean,var)

#Insteadoftheabovestatement,theuofthefollowing2statementscontainingatypo

#consistentlyproducesa0.2%higheraccuracyforunclearreasons.

#m_l,v_l=s(z_pre_l,axes=[0])

#z=join(batch_normalization(z_pre_l,m_l,mean,var),batch_normalization(z_pre_u,mean,var))

returnz

#performbatchnormalizationaccordingtovalueofboolean"training"placeholder:

#training==True-->training_batch_normel-->eval_batch_norm

z=(training,training_batch_norm,eval_batch_norm)

ifl==L:

#usoftmaxactivationinoutputlayer

h=x(weights['gamma'][l-1]*(z+weights["beta"][l-1]))

el:

#uReLUactivationinhiddenlayers

h=(z+weights["beta"][l-1])

d['labeled']['z'][l],d['unlabeled']['z'][l]=split_lu(z)#batch_normalization之后的输⼊

d['unlabeled']['m'][l],d['unlabeled']['v'][l]=m,v#savemeanandvarianceofunlabeledexamplesfordecoding

d['labeled']['h'][l],d['unlabeled']['h'][l]=split_lu(h)

returnh,d

print("===CorruptedEncoder===")

y_c,corr=encoder(inputs,noi_std)

print("===CleanEncoder===")

y,clean=encoder(inputs,0.0)#0.0->donotaddnoi

print("===Decoder===")

defg_gauss(z_c,u,size):

"gaussiandenoisingfunctionpropodintheoriginalpaper"

wi=lambdainits,name:le(inits*([size]),name=name)

a1=wi(0.,'a1')

a2=wi(1.,'a2')

a3=wi(0.,'a3')

a4=wi(0.,'a4')

a5=wi(0.,'a5')

a6=wi(0.,'a6')

a7=wi(1.,'a7')

a8=wi(0.,'a8')

a9=wi(0.,'a9')

a10=wi(0.,'a10')

mu=a1*d(a2*u+a3)+a4*u+a5

v=a6*d(a7*u+a8)+a9*u+a10

z_est=(z_c-mu)*v+mu

returnz_est

#Decoder

z_est={}

d_cost=[]#tostorethedenoisingcostofalllayers

forlinrange(L,-1,-1):#倒序到0

print("Layer",l,":",layer_sizes[l+1]ifl+1",layer_sizes[l],",denoisingcost:",denoising_cost[l])

z,z_c=clean['unlabeled']['z'][l],corr['unlabeled']['z'][l]

m,v=clean['unlabeled']['m'].get(l,0),clean['unlabeled']['v'].get(l,1-1e-10)

ifl==L:

u=unlabeled(y_c)

el:

u=(z_est[l+1],weights['V'][l])

u=batch_normalization(u)

z_est[l]=g_gauss(z_c,u,layer_sizes[l])

z_est_bn=(z_est[l]-m)/v

z_est_bn=(z_est[l]-m)/v

#appendthecostofthislayertod_cost

d_((_mean(_sum((z_est_bn-z),1))/layer_sizes[l])*denoising_cost[l])

#calculatetotalunsupervidcostbyaddingthedenoisingcostofalllayers

u_cost=_n(d_cost)

y_N=labeled(y_c)

cost=-_mean(_sum(outputs*(y_N),1))#supervidcost

loss=cost+u_cost#totalcost

pred_cost=-_mean(_sum(outputs*(y),1))#costudforprediction

#axis=1的时候，将每⼀⾏最⼤元素所在的索引记录下来，最后返回每⼀⾏最⼤元素所在的索引数组。

correct_prediction=((y,1),(outputs,1))#noofcorrectpredictions

accuracy=_mean((correct_prediction,"float"))*nt(100.0)#数据类型转换

learning_rate=le(starter_learning_rate,trainable=Fal)

train_step=timizer(learning_rate).minimize(loss)

#addtheupdatesofbatchnormalizationstatisticstotrain_step

bn_updates=(*bn_assigns)#*bn_assigns代表[]中所有元素

#l_dependencies指定某些操作执⾏的依赖关系

#control_dependencies(control_inputs)返回⼀个控制依赖的上下⽂管理器，

#使⽤with关键字可以让在这个上下⽂环境中的操作都在control_inputs执⾏

#l_dependencies([a,b,c]):

#d=...

#e=...

#`d`and`e`willonlyrunafter`a`,`b`,and`c`haveexecuted.

l_dependencies([train_step]):

train_step=(bn_updates)#()组合训练

print("===LoadingData===")

mnist=input__data_ts(r"D:QELDeepLearningmnist",n_labeled=num_labeled,one_hot=True)

saver=()

print("===StartingSession===")

ss=n()

i_iter=0

ckpt=_checkpoint_state('checkpoints/')#getlatestcheckpoint(ifany)

_checkpoint_path:

#ifcheckpointexists,restoretheparametersandtepoch_nandi_iter

e(ss,_checkpoint_path)

epoch_n=int(_checkpoint_('-')[1])

i_iter=(epoch_n+1)*(num_examples/batch_size)

print("RestoredEpoch",epoch_n)

el:

#checkpointsdirectoryifitdoesnotexist.

('checkpoints'):

rs('checkpoints')

init=_variables_initializer()

(init)

print("===Training===")

print("InitialAccuracy:",(accuracy,feed_dict={inputs:,outputs:,training:Fal}),"%")

#tqdm进度条显⽰

foriintqdm(range(int(i_iter),int(num_iter))):

#这⾥images有200个，labels有100个

images,labels=_batch(batch_size)

(train_step,feed_dict={inputs:images,outputs:labels,training:True})

if(i>1)and((i+1)%(num_iter/num_epochs)==0):

epoch_n=i/(num_examples/batch_size)

if(epoch_n+1)>=decay_after:

#decaylearningrate

#learning_rate=starter_learning_rate*((num_epochs-epoch_n)/(num_epochs-decay_after))

ratio=1.0*(num_epochs-(epoch_n+1))#epoch_n+1becaulearningrateistfornextepoch

ratio=max(0,ratio/(num_epochs-decay_after))

(learning_(starter_learning_rate*ratio))

(ss,'checkpoints/',int(epoch_n))

#print"Epoch",epoch_n,",Accuracy:",(accuracy,feed_dict={inputs:,outputs:,training:Fal}),"%"

#withopen('train_log','ab')astrain_log:

##writetestaccuracytofile"train_log"

#train_log_w=(train_log)

#log_i=[epoch_n]+([accuracy],feed_dict={inputs:,outputs:,training:Fal})

#train_log_ow(log_i)

print("FinalAccuracy:",(accuracy,feed_dict={inputs:,outputs:,training:Fal}),"%")

()

input_

"""FunctionsfordownloadingandreadingMNISTdata."""

from__future__importprint_function

importgzip

importos

t

importnumpy

SOURCE_URL='/exdb/mnist/'

defmaybe_download(filename,work_directory):

"""DownloadthedatafromYann'swebsite,unlessit'salreadyhere."""

(work_directory):

(work_directory)

filepath=(work_directory,filename)

(filepath):

#filepath,_=rieve(SOURCE_URL+filename,filepath)

filepath,_=rieve(SOURCE_URL+filename,filepath)

statinfo=(filepath)

print('Succesfullydownloaded',filename,_size,'bytes.')

returnfilepath

def_read32(bytestream):

dt=(32).newbyteorder('>')

ffer((4),dtype=dt)[0]

defextract_images(filename):

"""Extracttheimagesintoa4Duint8numpyarray[index,y,x,depth]."""

print('Extracting',filename)

(filename)asbytestream:

magic=_read32(bytestream)

ifmagic!=2051:

raiValueError(

'Invalidmagicnumber%dinMNISTimagefile:%s'%

(magic,filename))

num_images=_read32(bytestream)

rows=_read32(bytestream)

cols=_read32(bytestream)

buf=(rows*cols*num_images)

data=ffer(buf,dtype=8)

data=ffer(buf,dtype=8)

data=e(num_images,rows,cols,1)

returndata

defden_to_one_hot(labels_den,num_class=10):

"""Convertclasslabelsfromscalarstoone-hotvectors."""

num_labels=labels_[0]

index_offt=(num_labels)*num_class

labels_one_hot=((num_labels,num_class))

labels_one_[index_offt+labels_()]=1

returnlabels_one_hot

defextract_labels(filename,one_hot=Fal):

"""Extractthelabelsintoa1Duint8numpyarray[index]."""

print('Extracting',filename)

(filename)asbytestream:

magic=_read32(bytestream)

ifmagic!=2049:

raiValueError(

'Invalidmagicnumber%dinMNISTlabelfile:%s'%

(magic,filename))

num_items=_read32(bytestream)

buf=(num_items)

labels=ffer(buf,dtype=8)

ifone_hot:

returnden_to_one_hot(labels)

returnlabels

classDataSet(object):

def__init__(lf,images,labels,fake_data=Fal):

iffake_data:

lf._num_examples=10000

el:

[0]==[0],(

":%:%s"%(,

))

lf._num_examples=[0]

#Convertshapefrom[numexamples,rows,columns,depth]

#to[numexamples,rows*columns](assumingdepth==1)

[3]==1

images=e([0],

[1]*[2])

#Convertfrom[0,255]->[0.0,1.0].

images=(32)

images=ly(images,1.0/255.0)

lf._images=images

lf._labels=labels

lf._epochs_completed=0

lf._index_in_epoch=0

@property

defimages(lf):

returnlf._images

@property

deflabels(lf):

returnlf._labels

@property

defnum_examples(lf):

returnlf._num_examples

returnlf._num_examples

@property

defepochs_completed(lf):

returnlf._epochs_completed

defnext_batch(lf,batch_size,fake_data=Fal):

"""Returnthenext`batch_size`examplesfromthisdatat."""

iffake_data:

fake_image=[1.0for_inxrange(784)]

fake_label=0

return[fake_imagefor_inxrange(batch_size)],[

fake_labelfor_inxrange(batch_size)]

start=lf._index_in_epoch

lf._index_in_epoch+=batch_size

iflf._index_in_epoch>lf._num_examples:

#Finishedepoch

lf._epochs_completed+=1

#Shufflethedata

perm=(lf._num_examples)

e(perm)

lf._images=lf._images[perm]

lf._labels=lf._labels[perm]

#Startnextepoch

start=0

lf._index_in_epoch=batch_size

asrtbatch_size<=lf._num_examples

end=lf._index_in_epoch

returnlf._images[start:end],lf._labels[start:end]

classSemiDataSet(object):

def__init__(lf,images,labels,n_labeled):

lf.n_labeled=n_labeled

#UnlabledDataSet

led_ds=DataSet(images,labels)

#LabeledDataSet

_examples=led__examples

indices=(_examples)

shuffled_indices=ation(indices)

images=images[shuffled_indices]

labels=labels[shuffled_indices]

y=([(10)[l==1][0]forlinlabels])

idx=indices[y==0][:5]

n_class=()+1

n_from_each_class=n_labeled/n_class

i_labeled=[]

forcinrange(n_class):

i=indices[y==c][:int(n_from_each_class)]

i_labeled+=list(i)

l_images=images[i_labeled]

l_labels=labels[i_labeled]

d_ds=DataSet(l_images,l_labels)

defnext_batch(lf,batch_size):

unlabeled_images,_=led__batch(batch_size)

ifbatch_size>lf.n_labeled:

labeled_images,labels=d__batch(lf.n_labeled)

el:

labeled_images,labels=d__batch(batch_size)

images=([labeled_images,unlabeled_images])

returnimages,labels

defread_data_ts(train_dir,n_labeled=100,fake_data=Fal,one_hot=Fal):

classDataSets(object):

classDataSets(object):

pass

data_ts=DataSets()

iffake_data:

data_=DataSet([],[],fake_data=True)

data_tion=DataSet([],[],fake_data=True)

data_=DataSet([],[],fake_data=True)

returndata_ts

TRAIN_IMAGES=''

TRAIN_LABELS=''

TEST_IMAGES=''

TEST_LABELS=''

VALIDATION_SIZE=0

local_file=maybe_download(TRAIN_IMAGES,train_dir)

train_images=extract_images(local_file)

local_file=maybe_download(TRAIN_LABELS,train_dir)

train_labels=extract_labels(local_file,one_hot=one_hot)

local_file=maybe_download(TEST_IMAGES,train_dir)

test_images=extract_images(local_file)

local_file=maybe_download(TEST_LABELS,train_dir)

test_labels=extract_labels(local_file,one_hot=one_hot)

validation_images=train_images[:VALIDATION_SIZE]

validation_labels=train_labels[:VALIDATION_SIZE]

train_images=train_images[VALIDATION_SIZE:]

train_labels=train_labels[VALIDATION_SIZE:]

data_=SemiDataSet(train_images,train_labels,n_labeled)

data_tion=DataSet(validation_images,validation_labels)

data_=DataSet(test_images,test_labels)

returndata_ts