采访YoloV5系列（2）-数据集准备
Abstract
Yolo格式的数据集不同于coco和voc数据集的格式.它通过⽂本的格式保存了gt框的坐标，如果只将数据集⽤于基于box的⽬标检测的话. 这种格式的数据集显然是⼀种不错的选择，它可以很⽅便的被管理. 如果想要为这个数据集增添其他信息,⽐如说语义信息, 点云信息等…显然, 对于txt来说过于臃肿了, 还是⽤xml和json格式保存⽐较⽅便.
Introduction
1 yolo数据集介绍
yolo格式的数据集⽀持两种指定train,val,test划分的⽅式, ⼀种是基于⽂件的划分⽅式. ⼀种是基于⽂本的划分⽅式.
我们先来看第⼀种⽅式，基于⽂本进⾏划分如下⾯的tree图所⽰
⾸先会创建, ,等⽤来保存图⽚路径.这些图⽚可以默认放在根⽬录的images⽂件夹下.只需要将⽂件的路径保存到txt ⽂件中，便可以被yolov5读取到，即使是⼀个错误的路径也不⽤担⼼，
在运⾏代码中,yolov5会提⽰哪些图⽚⽂件是缺失的.如果使⽤的不是通⽤数据集.例如coco2017,voc2012等等，都建议使⽤这种⽅式灵活的管理数据集
优点：可以很好的管理train,val,test⽂件的划分.⼀张图⽚路径可以出现在多个txt⽂件中，减少了存放图⽚的空间.
缺点:⽂本⽂件路径是固定的，所以数据集的位置也需要相对固定
#基于⽂本划分数据集
datat
├── images
│├── 001.jpg
│├── 002.jpg
│├── 003.jpg
│└── 004.jpg
│
├── labels
│├──
│├──
│├──
│└──
│
└── ImageSets
├──
├──
├──
└──
基于⽂件夹划分的⽅式如下树状图所⽰
它不再需要txt⽂件指定数据集的划分了，只需要在images下⾯，新建train,test的⽂件夹.同时将对应的图⽚放进去就⾏了,⽐较⽅便. 但是其⽆法再对数据集进⾏灵活的划分了.同时，如果⼀张图⽚存在于train和val中，那么图⽚就需要两份放在各⾃的⽂件夹中，增加了内存空间.
通⽤数据集使⽤这种格式的数据集是⽐较友好的
#格基于⽂件划分数据集
datat
├── images
│└── train
││├── 001.jpg
││└── 002.jpg
││
│└── test
│├── 003.jpg
│└── 004.jpg
│
└── labels
├──train
│├──
│└──
│
└── test
├──
└──
2, 数据集转换⼯具
由于⼤部分的数据集都是voc和coco格式的，因此格式的转换让⼈头疼，在这⾥我提供了3套⼯具, 分别是voc转换coco格式⼯具, coco转换voc格式⼯具, voc转换yolo格式⼯具.
这些代码已经实操很多遍了，基本没有问题
使⽤这个数据集⼯具有⼏个点需要注意
1, 这些⼯具配置参数均在最前⾯
2，有些coco数据集⾥⾯写图⽚名字的时候加了.jpg，因此需要在⼯具中也加上.jpg，在使⽤过程中会发现这⼀问题
#coco数据集转voc数据集
import COCO
import skimage.io as io
import matplotlib.pyplot as plt
import pylab,os,cv2,shutil
from lxml import etree, objectify
from tqdm import tqdm
import random
from PIL import Image
import os
#修改位置
CKdir="./voc2017"
CK5cats =['person','bicycle','car','motorcycle','airplane','bus','train','truck','boat','traffic light',
'fire hydrant','stop sign','parking meter','bench','bird','cat','dog','hor','sheep','cow',
'elephant','bear','zebra','giraffe','backpack','umbrella','handbag','tie','suitca','frisbee',
'skis','snowboard','sports ball','kite','baball bat','baball glove','skateboard','surfboard',
'tennis racket','bottle','wine glass','cup','fork','knife','spoon','bowl','banana','apple',
'sandwich','orange','broccoli','carrot','hot dog','pizza','donut','cake','chair','couch',
'potted plant','bed','dining table','toilet','tv','laptop','mou','remote','keyboard','cell phone',
'microwave','oven','toaster','sink','refrigerator','book','clock','va','scissors','teddy bear',
'hair drier','toothbrush']
dataDir='./coco2017'
instance="instances_{}.json"
dataTypes=['val2017',"train2017"]
#========================================================================================
卫生服务站
CKimg_dir=os.path.join(CKdir,"JPEGImages")
CKanno_dir=os.path.join(CKdir,"Annotations")
#是否⾃⼰分割数据集
trans_by_lf=Fal
trans_by_lf=Fal
def mkr(dir):
if not ists(dir):
os.makedirs(dir)
def showimg(coco,dataType,img,CK5Ids):
global dataDir
I = io.imread('%s/%s/%s'%(dataDir, dataType, img['file_name']))
plt.imshow(I)
plt.axis('off')
annIds = AnnIds(imgIds=img['id'], catIds=CK5Ids, iscrowd=None) anns = coco.loadAnns(annIds)
coco.showAnns(anns)
plt.show()
def save_annotations(dataType,filename,objs):
annopath=os.path.join(CKanno_dir,filename[:-3]+"xml")
img_path=dataDir+"/"+dataType+"/"+filename
dst_path=CKimg_dir+"/"+filename
img=cv2.imread(img_path)
'''
#删掉不是RGB的图像
im=Image.open(img_path)
de!="RGB":
print(filename+" not a RGB image")
im.clo()
return
im.clo()
'''
E = objectify.ElementMaker(annotate=Fal)
anno_tree = E.annotation(
E.folder('1'),
E.filename(filename),
E.source(
E.databa('CKdemo'),
E.annotation('VOC'),
E.image('CK')
),
E.size(
E.width(img.shape[1]),
E.height(img.shape[0]),
E.depth(img.shape[2])
),
)
for obj in objs:
E2 = objectify.ElementMaker(annotate=Fal)
anno_tree2 = E2.object(
E.name(obj[0]),
E.po(),
E.difficult(0),
E.bndbox(
)
)
anno_tree.append(anno_tree2)
etree.ElementTree(anno_tree).write(annopath, pretty_print=True)
def showbycv(coco,dataType,img,class,CK5Ids):
global dataDir
filename= img['file_name']
filepath='%s/%s/%s'%(dataDir, dataType,filename)
I = cv2.imread(filepath)
annIds = AnnIds(imgIds=img['id'], catIds=CK5Ids, iscrowd=None) anns = coco.loadAnns(annIds)
objs=[]
for ann in anns:
name=class[ann['category_id']]
if name in CK5cats:
if'bbox'in ann:
bbox = ann['bbox']
xmin=(int)(bbox[0])
ymin=(int)(bbox[1])
xmax=(int)(bbox[2]+bbox[0])八府巡抚是几品官
ymax=(int)(bbox[3]+bbox[1])
obj=[name,1.0,xmin,ymin,xmax,ymax]
objs.append(obj)
cv2.putText(I,name,(xmin,ymin),3,1,(0,0,255))
save_annotations(dataType,filename,objs)
cv2.imwrite("tmp.jpg",I)
#cv2.imshow("img",I)
#cv2.waitKey(1)
return filename[:-4]
def catid2name(coco):
class=dict()
减肥午餐for cat in coco.datat['categories']:
class[cat['id']]=cat['name']
#print(str(cat['id'])+":"+cat['name'])
return class
def get_CK5():
imgids={}
for dataType in dataTypes:
annFile =('{}/annotations/'+instance).format(dataDir, dataType)
葱姜炒蛏子coco = COCO(annFile)
CK5Ids = CatIds(catNms=CK5cats)
print(len(CK5cats))
class=catid2name(coco)
print(CK5Ids,class)
img_ids = _img_ids()
print(len(img_ids))
for imgId in tqdm(img_ids):
img=coco.loadImgs(imgId)[0]
filename=showbycv(coco,dataType,img,class,CK5Ids)
def split():
imgnames={}
datat_dir=CKdir
list_imgs={}
for dataType in dataTypes:
imgnames[dataType]=os.listdir(os.path.join(dataDir,dataType)) print(len(imgnames[dataType]))
for dataType in dataTypes:
annFile =('{}/annotations/'+instance).format(dataDir, dataType)
coco = COCO(annFile)
cat_ids = _cat_ids(cat_names=CK5cats)
cat_ids = _cat_ids(cat_names=CK5cats)
img_ids = _img_ids()
for imgId in tqdm(img_ids):
img=coco.loadImgs(img_ids)[0]
filename=img["file_name"]
if filename not in imgnames[dataType]:
print("not found",filename)郭金辉
maindir=os.path.join(datat_dir,"ImageSets","Main")
mkr(maindir)
for dataType in dataTypes:
f=open(os.path.join(CKdir,"ImageSets","Main",dataType+".txt"),"w") for s in imgnames[dataType]:
f.write(s+"\n")
f.clo()
#split train and test for training
def split_traintest(trainratio=0.7,valratio=0.2,testratio=0.1):
datat_dir=CKdir
files=os.listdir(CKimg_dir)
trains=[]
vals=[]
trainvals=[]
tests=[]
random.shuffle(files)
for i in range(len(files)):
filepath=CKimg_dir+"/"+files[i][:-3]+"jpg"
if(i<trainratio*len(files)):
trains.append(files[i])
trainvals.append(files[i])
elif i<(trainratio+valratio)*len(files):
vals.append(files[i])
trainvals.append(files[i])
el:
tests.append(files[i])
#write files for voc
maindir=os.path.join(datat_dir,"ImageSets","Main")
mkr(maindir)为啥使用胃管活不长
with open(os.path.join(maindir,""),"w")as f:
for line in trains:
line=line[:line.rfind(".")]
f.write(line+"\n")
with open(os.path.join(maindir,""),"w")as f:
for line in vals:
line=line[:line.rfind(".")]
f.write(line+"\n")
with open(os.path.join(maindir,""),"w")as f:
for line in trainvals:
思念家乡的诗词line=line[:line.rfind(".")]
f.write(line+"\n")
with open(os.path.join(maindir,""),"w")as f:
for line in tests:
line=line[:line.rfind(".")]
f.write(line+"\n")
print("spliting done")
