pytorch部署新利器TorchServer pytorch的爬坑指南
上⼲货：
1.docker版本不能太低，我装的19.03.13
2.下载项⽬⽂件
git clone /pytorch/rve.git
cd rve/docker
3.构建docker image（cpu版本）
DOCKER_BUILDKIT=1 docker build --file Dockerfile -t torchrve:latest .
或者
docker pull pytorch/torchrve:latest
4.将训练好的模型及环境打包
趣味英语
# 加载模型
checkpoint = torch.load(checkpoint_dir)
bertconfig = BertConfig(vocab_size=int(vocab_size), num_hidden_layers=3)
model = Bert_Sentiment_Analysis(config=bertconfig)
# 校验
model.eval()
# 加载参数
model.load_state_dict(checkpoint["model_state_dict"])
# model input sample
texts_tokens_ = torch.randint(0,100,(1,127))
positional_enc = torch.randn((1,127,384), dtype=torch.float32)
# 打包并保存
traced_script_module = ace(model,(texts_tokens_, positional_enc))
traced_script_module.save("ntiment_test.pt")
5.完成handle.py⽂件对接torchServer
注意 preprocess⽅法接收的data[0].get(“data”)数据类型为bytes
# 模板
class ModelHandler(BaHandler):
"""
A custom model handler implementation.
"""
def__init__(lf):
lf._context =None
lf.initialized =Fal
def initialize(lf, context):
"""
Initialize model. This will be called during model loading time
:param context: Initial context contains model rver system properties.
:return:
"""
lf._context = context
lf.initialized =True
properties = context.system_properties
# load the model
lf.manifest = context.manifest
model_dir = ("model_dir")年休假规定
lf.device = torch.device("cuda:"+("gpu_id"))if torch.cuda.is_available()el"cpu")
lf.device = torch.device("cuda:"+("gpu_id"))if torch.cuda.is_available()el"cpu")
# Read model rialize/pt file
rialized_file = lf.manifest['model']['rializedFile']
model_pt_path = os.path.join(model_dir, rialized_file)
if not os.path.isfile(model_pt_path):
rai RuntimeError("Missing the model.pt file")
<(lf.device)
...
lf.initialized =True
def preprocess(lf, data:list):
"""
Transform raw input into model input data.
:param batch: list of raw requests, should match batch size
:return: list of preprocesd model input data
"""
# Take the input data and make it inference ready
text = data[0].get("data")or data[0].get("body")
# 异常判断
除法竖式怎么写
if text is None:
warnings.warn("data params is none")
rai Exception("no data")
el:
text = text.decode()
# 预处理, 获取batch
...
def inference(lf, texts_tokens_, positional_enc):
"""
Internal inference methods
:
param model_input: transformed model input data
:return: list of inference output in NDArray
"""
# Do some inference call to engine here and return output
predictions = lf.model.forward(texts_tokens_, positional_enc)说怎么组词
...
def postprocess(lf, inference_output):
"""
Return inference result.
:param inference_output: list of inference output
:return: list of predict results
"""不再犹豫简谱
# Take output from network and post-process to desired format
postprocess_output = inference_output
return postprocess_output
def handle(lf, data, context):
"""
Invoke by TorchServe for prediction request.
Do pre-processing of data, prediction using model and postprocessing of prediciton output
:param data: Input data for prediction
:param context: Initial context contains model rver system properties.
:return: prediction output
"""
lf.preprocess(data)
lf.inference()
lf.postprocess(model_output)
...
胸部疼痛怎么办rvice = ModelHandler()
def handle(data, context):
if not rvice.initialized:
rvice.initialize(context)
工作推进表
if data is None:
return None
return rvice.handle(data, context)
6.打包模型预测需要的⽂件
torch-model-archiver --model-name ntiment_test --version 1.0 --rialized-file /home/model-rver/model-store/ntiment_test.pt \
--export-path /home/model-rver/model-store \
--extra-files /home/model-rver/model-store/bert_word2idx.json \
--handler model_handler:handle -f
--model-name: 模型的名称，后来的接⼝名称和管理的模型名称都是这个
--rialized-file: 模型环境及代码及参数的打包⽂件
--export-path: 本次打包⽂件存放位置
--extra-files: handle.py中需要使⽤到的其他⽂件
--handler: 指定handler函数。（模型名：函数名）
-f 覆盖之前导出的同名打包⽂件
执⾏完会发现在/home/model-rver/model-store⽬下多了⼀个以.mar结尾的⽂件，这个就是我们要在模型服务中使⽤的最终的打包⽂件
7.启动docker服务
将.mar⽂件放到宿主机的/home/model-rver/model-store⽬录下
docker run --rm -it -p 3000:8080 -p 3001:8081 --name ntiment_test \
-v /home/model-rver/model-store:/home/model-rver/model-store \
torchrve:latest
# docker 可选参数 --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 \
# 切换到后台
ctrl + p
8.1模型管理相关接⼝
# 注冊模型并为模型分配资源
curl -v -X POST "localhost:3001/models?initial_workers=1&synchronous=fal&url=ntiment_test.mar&batch_size=8&max_batch_delay=200"
长春旅行# 修改分配worker数量
curl -v -X PUT "localhost:3001/models/ntiment_test?min_worker=3"
# 查看指定模型当前状态
curl 192.168.5.135:3001/models/ntiment_test
8.2模型预测接⼝（handler.py⽂件中的逻辑）
curl -X POST localhost:3000/predictions/ntiment_test -d "data=这也太难吃了把？再也不来了"
9.进⼊docker 镜像停⽌或运⾏程序
# 进⼊docker容器
docker exec -it [容器名称] /bin/bash
# 停⽌服务
torchrve --stop
# 启动服务
torchrve --start --ncs --model-store /home/model-rver/model-store --models ntiment_test.mar
