paddlex.cpp 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779
  1. // Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. #include <omp.h>
  15. #include <algorithm>
  16. #include <fstream>
  17. #include <cstring>
  18. #include "include/paddlex/paddlex.h"
  19. namespace PaddleX {
  20. void Model::create_predictor(const std::string& model_dir,
  21. bool use_gpu,
  22. bool use_trt,
  23. int gpu_id,
  24. std::string key,
  25. bool use_ir_optim) {
  26. paddle::AnalysisConfig config;
  27. std::string model_file = model_dir + OS_PATH_SEP + "__model__";
  28. std::string params_file = model_dir + OS_PATH_SEP + "__params__";
  29. std::string yaml_file = model_dir + OS_PATH_SEP + "model.yml";
  30. std::string yaml_input = "";
  31. #ifdef WITH_ENCRYPTION
  32. if (key != "") {
  33. model_file = model_dir + OS_PATH_SEP + "__model__.encrypted";
  34. params_file = model_dir + OS_PATH_SEP + "__params__.encrypted";
  35. yaml_file = model_dir + OS_PATH_SEP + "model.yml.encrypted";
  36. paddle_security_load_model(
  37. &config, key.c_str(), model_file.c_str(), params_file.c_str());
  38. yaml_input = decrypt_file(yaml_file.c_str(), key.c_str());
  39. }
  40. #endif
  41. if (yaml_input == "") {
  42. // 读取配置文件
  43. std::ifstream yaml_fin(yaml_file);
  44. yaml_fin.seekg(0, std::ios::end);
  45. size_t yaml_file_size = yaml_fin.tellg();
  46. yaml_input.assign(yaml_file_size, ' ');
  47. yaml_fin.seekg(0);
  48. yaml_fin.read(&yaml_input[0], yaml_file_size);
  49. }
  50. // 读取配置文件内容
  51. if (!load_config(yaml_input)) {
  52. std::cerr << "Parse file 'model.yml' failed!" << std::endl;
  53. exit(-1);
  54. }
  55. if (key == "") {
  56. config.SetModel(model_file, params_file);
  57. }
  58. if (use_gpu) {
  59. config.EnableUseGpu(100, gpu_id);
  60. } else {
  61. config.DisableGpu();
  62. }
  63. config.SwitchUseFeedFetchOps(false);
  64. config.SwitchSpecifyInputNames(true);
  65. // 开启图优化
  66. config.SwitchIrOptim(use_ir_optim);
  67. // 开启内存优化
  68. config.EnableMemoryOptim();
  69. if (use_trt) {
  70. config.EnableTensorRtEngine(
  71. 1 << 20 /* workspace_size*/,
  72. 32 /* max_batch_size*/,
  73. 20 /* min_subgraph_size*/,
  74. paddle::AnalysisConfig::Precision::kFloat32 /* precision*/,
  75. true /* use_static*/,
  76. false /* use_calib_mode*/);
  77. }
  78. predictor_ = std::move(CreatePaddlePredictor(config));
  79. }
  80. bool Model::load_config(const std::string& yaml_input) {
  81. YAML::Node config = YAML::Load(yaml_input);
  82. type = config["_Attributes"]["model_type"].as<std::string>();
  83. name = config["Model"].as<std::string>();
  84. std::string version = config["version"].as<std::string>();
  85. if (version[0] == '0') {
  86. std::cerr << "[Init] Version of the loaded model is lower than 1.0.0, "
  87. << "deployment cannot be done, please refer to "
  88. << "https://github.com/PaddlePaddle/PaddleX/blob/develop/docs"
  89. << "/tutorials/deploy/upgrade_version.md "
  90. << "to transfer version." << std::endl;
  91. return false;
  92. }
  93. bool to_rgb = true;
  94. if (config["TransformsMode"].IsDefined()) {
  95. std::string mode = config["TransformsMode"].as<std::string>();
  96. if (mode == "BGR") {
  97. to_rgb = false;
  98. } else if (mode != "RGB") {
  99. std::cerr << "[Init] Only 'RGB' or 'BGR' is supported for TransformsMode"
  100. << std::endl;
  101. return false;
  102. }
  103. }
  104. // 构建数据处理流
  105. transforms_.Init(config["Transforms"], to_rgb);
  106. // 读入label list
  107. labels.clear();
  108. for (const auto& item : config["_Attributes"]["labels"]) {
  109. int index = labels.size();
  110. labels[index] = item.as<std::string>();
  111. }
  112. return true;
  113. }
  114. bool Model::preprocess(const cv::Mat& input_im, ImageBlob* blob) {
  115. cv::Mat im = input_im.clone();
  116. if (!transforms_.Run(&im, blob)) {
  117. return false;
  118. }
  119. return true;
  120. }
  121. // use openmp
  122. bool Model::preprocess(const std::vector<cv::Mat>& input_im_batch,
  123. std::vector<ImageBlob>* blob_batch,
  124. int thread_num) {
  125. int batch_size = input_im_batch.size();
  126. bool success = true;
  127. thread_num = std::min(thread_num, batch_size);
  128. #pragma omp parallel for num_threads(thread_num)
  129. for (int i = 0; i < input_im_batch.size(); ++i) {
  130. cv::Mat im = input_im_batch[i].clone();
  131. if (!transforms_.Run(&im, &(*blob_batch)[i])) {
  132. success = false;
  133. }
  134. }
  135. return success;
  136. }
  137. bool Model::predict(const cv::Mat& im, ClsResult* result) {
  138. inputs_.clear();
  139. if (type == "detector") {
  140. std::cerr << "Loading model is a 'detector', DetResult should be passed to "
  141. "function predict()!"
  142. "to function predict()!" << std::endl;
  143. return false;
  144. }
  145. // 处理输入图像
  146. if (!preprocess(im, &inputs_)) {
  147. std::cerr << "Preprocess failed!" << std::endl;
  148. return false;
  149. }
  150. // 使用加载的模型进行预测
  151. auto in_tensor = predictor_->GetInputTensor("image");
  152. int h = inputs_.new_im_size_[0];
  153. int w = inputs_.new_im_size_[1];
  154. in_tensor->Reshape({1, 3, h, w});
  155. in_tensor->copy_from_cpu(inputs_.im_data_.data());
  156. predictor_->ZeroCopyRun();
  157. // 取出模型的输出结果
  158. auto output_names = predictor_->GetOutputNames();
  159. auto output_tensor = predictor_->GetOutputTensor(output_names[0]);
  160. std::vector<int> output_shape = output_tensor->shape();
  161. int size = 1;
  162. for (const auto& i : output_shape) {
  163. size *= i;
  164. }
  165. outputs_.resize(size);
  166. output_tensor->copy_to_cpu(outputs_.data());
  167. // 对模型输出结果进行后处理
  168. auto ptr = std::max_element(std::begin(outputs_), std::end(outputs_));
  169. result->category_id = std::distance(std::begin(outputs_), ptr);
  170. result->score = *ptr;
  171. result->category = labels[result->category_id];
  172. return true;
  173. }
  174. bool Model::predict(const std::vector<cv::Mat>& im_batch,
  175. std::vector<ClsResult>* results,
  176. int thread_num) {
  177. for (auto& inputs : inputs_batch_) {
  178. inputs.clear();
  179. }
  180. if (type == "detector") {
  181. std::cerr << "Loading model is a 'detector', DetResult should be passed to "
  182. "function predict()!" << std::endl;
  183. return false;
  184. } else if (type == "segmenter") {
  185. std::cerr << "Loading model is a 'segmenter', SegResult should be passed "
  186. "to function predict()!" << std::endl;
  187. return false;
  188. }
  189. inputs_batch_.assign(im_batch.size(), ImageBlob());
  190. // 处理输入图像
  191. if (!preprocess(im_batch, &inputs_batch_, thread_num)) {
  192. std::cerr << "Preprocess failed!" << std::endl;
  193. return false;
  194. }
  195. // 使用加载的模型进行预测
  196. int batch_size = im_batch.size();
  197. auto in_tensor = predictor_->GetInputTensor("image");
  198. int h = inputs_batch_[0].new_im_size_[0];
  199. int w = inputs_batch_[0].new_im_size_[1];
  200. in_tensor->Reshape({batch_size, 3, h, w});
  201. std::vector<float> inputs_data(batch_size * 3 * h * w);
  202. for (int i = 0; i < batch_size; ++i) {
  203. std::copy(inputs_batch_[i].im_data_.begin(),
  204. inputs_batch_[i].im_data_.end(),
  205. inputs_data.begin() + i * 3 * h * w);
  206. }
  207. in_tensor->copy_from_cpu(inputs_data.data());
  208. // in_tensor->copy_from_cpu(inputs_.im_data_.data());
  209. predictor_->ZeroCopyRun();
  210. // 取出模型的输出结果
  211. auto output_names = predictor_->GetOutputNames();
  212. auto output_tensor = predictor_->GetOutputTensor(output_names[0]);
  213. std::vector<int> output_shape = output_tensor->shape();
  214. int size = 1;
  215. for (const auto& i : output_shape) {
  216. size *= i;
  217. }
  218. outputs_.resize(size);
  219. output_tensor->copy_to_cpu(outputs_.data());
  220. // 对模型输出结果进行后处理
  221. (*results).clear();
  222. (*results).resize(batch_size);
  223. int single_batch_size = size / batch_size;
  224. for (int i = 0; i < batch_size; ++i) {
  225. auto start_ptr = std::begin(outputs_);
  226. auto end_ptr = std::begin(outputs_);
  227. std::advance(start_ptr, i * single_batch_size);
  228. std::advance(end_ptr, (i + 1) * single_batch_size);
  229. auto ptr = std::max_element(start_ptr, end_ptr);
  230. (*results)[i].category_id = std::distance(start_ptr, ptr);
  231. (*results)[i].score = *ptr;
  232. (*results)[i].category = labels[(*results)[i].category_id];
  233. }
  234. return true;
  235. }
  236. bool Model::predict(const cv::Mat& im, DetResult* result) {
  237. inputs_.clear();
  238. result->clear();
  239. if (type == "classifier") {
  240. std::cerr << "Loading model is a 'classifier', ClsResult should be passed "
  241. "to function predict()!" << std::endl;
  242. return false;
  243. } else if (type == "segmenter") {
  244. std::cerr << "Loading model is a 'segmenter', SegResult should be passed "
  245. "to function predict()!" << std::endl;
  246. return false;
  247. }
  248. // 处理输入图像
  249. if (!preprocess(im, &inputs_)) {
  250. std::cerr << "Preprocess failed!" << std::endl;
  251. return false;
  252. }
  253. int h = inputs_.new_im_size_[0];
  254. int w = inputs_.new_im_size_[1];
  255. auto im_tensor = predictor_->GetInputTensor("image");
  256. im_tensor->Reshape({1, 3, h, w});
  257. im_tensor->copy_from_cpu(inputs_.im_data_.data());
  258. if (name == "YOLOv3") {
  259. auto im_size_tensor = predictor_->GetInputTensor("im_size");
  260. im_size_tensor->Reshape({1, 2});
  261. im_size_tensor->copy_from_cpu(inputs_.ori_im_size_.data());
  262. } else if (name == "FasterRCNN" || name == "MaskRCNN") {
  263. auto im_info_tensor = predictor_->GetInputTensor("im_info");
  264. auto im_shape_tensor = predictor_->GetInputTensor("im_shape");
  265. im_info_tensor->Reshape({1, 3});
  266. im_shape_tensor->Reshape({1, 3});
  267. float ori_h = static_cast<float>(inputs_.ori_im_size_[0]);
  268. float ori_w = static_cast<float>(inputs_.ori_im_size_[1]);
  269. float new_h = static_cast<float>(inputs_.new_im_size_[0]);
  270. float new_w = static_cast<float>(inputs_.new_im_size_[1]);
  271. float im_info[] = {new_h, new_w, inputs_.scale};
  272. float im_shape[] = {ori_h, ori_w, 1.0};
  273. im_info_tensor->copy_from_cpu(im_info);
  274. im_shape_tensor->copy_from_cpu(im_shape);
  275. }
  276. // 使用加载的模型进行预测
  277. predictor_->ZeroCopyRun();
  278. std::vector<float> output_box;
  279. auto output_names = predictor_->GetOutputNames();
  280. auto output_box_tensor = predictor_->GetOutputTensor(output_names[0]);
  281. std::vector<int> output_box_shape = output_box_tensor->shape();
  282. int size = 1;
  283. for (const auto& i : output_box_shape) {
  284. size *= i;
  285. }
  286. output_box.resize(size);
  287. output_box_tensor->copy_to_cpu(output_box.data());
  288. if (size < 6) {
  289. std::cerr << "[WARNING] There's no object detected." << std::endl;
  290. return true;
  291. }
  292. int num_boxes = size / 6;
  293. // 解析预测框box
  294. for (int i = 0; i < num_boxes; ++i) {
  295. Box box;
  296. box.category_id = static_cast<int>(round(output_box[i * 6]));
  297. box.category = labels[box.category_id];
  298. box.score = output_box[i * 6 + 1];
  299. float xmin = output_box[i * 6 + 2];
  300. float ymin = output_box[i * 6 + 3];
  301. float xmax = output_box[i * 6 + 4];
  302. float ymax = output_box[i * 6 + 5];
  303. float w = xmax - xmin + 1;
  304. float h = ymax - ymin + 1;
  305. box.coordinate = {xmin, ymin, w, h};
  306. result->boxes.push_back(std::move(box));
  307. }
  308. // 实例分割需解析mask
  309. if (name == "MaskRCNN") {
  310. std::vector<float> output_mask;
  311. auto output_mask_tensor = predictor_->GetOutputTensor(output_names[1]);
  312. std::vector<int> output_mask_shape = output_mask_tensor->shape();
  313. int masks_size = 1;
  314. for (const auto& i : output_mask_shape) {
  315. masks_size *= i;
  316. }
  317. int mask_pixels = output_mask_shape[2] * output_mask_shape[3];
  318. int classes = output_mask_shape[1];
  319. output_mask.resize(masks_size);
  320. output_mask_tensor->copy_to_cpu(output_mask.data());
  321. result->mask_resolution = output_mask_shape[2];
  322. for (int i = 0; i < result->boxes.size(); ++i) {
  323. Box* box = &result->boxes[i];
  324. auto begin_mask =
  325. output_mask.begin() + (i * classes + box->category_id) * mask_pixels;
  326. auto end_mask = begin_mask + mask_pixels;
  327. box->mask.data.assign(begin_mask, end_mask);
  328. box->mask.shape = {static_cast<int>(box->coordinate[2]),
  329. static_cast<int>(box->coordinate[3])};
  330. }
  331. }
  332. return true;
  333. }
  334. bool Model::predict(const std::vector<cv::Mat>& im_batch,
  335. std::vector<DetResult>* results,
  336. int thread_num) {
  337. for (auto& inputs : inputs_batch_) {
  338. inputs.clear();
  339. }
  340. if (type == "classifier") {
  341. std::cerr << "Loading model is a 'classifier', ClsResult should be passed "
  342. "to function predict()!" << std::endl;
  343. return false;
  344. } else if (type == "segmenter") {
  345. std::cerr << "Loading model is a 'segmenter', SegResult should be passed "
  346. "to function predict()!" << std::endl;
  347. return false;
  348. }
  349. inputs_batch_.assign(im_batch.size(), ImageBlob());
  350. int batch_size = im_batch.size();
  351. // 处理输入图像
  352. if (!preprocess(im_batch, &inputs_batch_, thread_num)) {
  353. std::cerr << "Preprocess failed!" << std::endl;
  354. return false;
  355. }
  356. // 对RCNN类模型做批量padding
  357. if (batch_size > 1) {
  358. if (name == "FasterRCNN" || name == "MaskRCNN") {
  359. int max_h = -1;
  360. int max_w = -1;
  361. for (int i = 0; i < batch_size; ++i) {
  362. max_h = std::max(max_h, inputs_batch_[i].new_im_size_[0]);
  363. max_w = std::max(max_w, inputs_batch_[i].new_im_size_[1]);
  364. // std::cout << "(" << inputs_batch_[i].new_im_size_[0]
  365. // << ", " << inputs_batch_[i].new_im_size_[1]
  366. // << ")" << std::endl;
  367. }
  368. thread_num = std::min(thread_num, batch_size);
  369. #pragma omp parallel for num_threads(thread_num)
  370. for (int i = 0; i < batch_size; ++i) {
  371. int h = inputs_batch_[i].new_im_size_[0];
  372. int w = inputs_batch_[i].new_im_size_[1];
  373. int c = im_batch[i].channels();
  374. if (max_h != h || max_w != w) {
  375. std::vector<float> temp_buffer(c * max_h * max_w);
  376. float* temp_ptr = temp_buffer.data();
  377. float* ptr = inputs_batch_[i].im_data_.data();
  378. for (int cur_channel = c - 1; cur_channel >= 0; --cur_channel) {
  379. int ori_pos = cur_channel * h * w + (h - 1) * w;
  380. int des_pos = cur_channel * max_h * max_w + (h - 1) * max_w;
  381. int last_pos = cur_channel * h * w;
  382. for (; ori_pos >= last_pos; ori_pos -= w, des_pos -= max_w) {
  383. memcpy(temp_ptr + des_pos, ptr + ori_pos, w * sizeof(float));
  384. }
  385. }
  386. inputs_batch_[i].im_data_.swap(temp_buffer);
  387. inputs_batch_[i].new_im_size_[0] = max_h;
  388. inputs_batch_[i].new_im_size_[1] = max_w;
  389. }
  390. }
  391. }
  392. }
  393. int h = inputs_batch_[0].new_im_size_[0];
  394. int w = inputs_batch_[0].new_im_size_[1];
  395. auto im_tensor = predictor_->GetInputTensor("image");
  396. im_tensor->Reshape({batch_size, 3, h, w});
  397. std::vector<float> inputs_data(batch_size * 3 * h * w);
  398. for (int i = 0; i < batch_size; ++i) {
  399. std::copy(inputs_batch_[i].im_data_.begin(),
  400. inputs_batch_[i].im_data_.end(),
  401. inputs_data.begin() + i * 3 * h * w);
  402. }
  403. im_tensor->copy_from_cpu(inputs_data.data());
  404. if (name == "YOLOv3") {
  405. auto im_size_tensor = predictor_->GetInputTensor("im_size");
  406. im_size_tensor->Reshape({batch_size, 2});
  407. std::vector<int> inputs_data_size(batch_size * 2);
  408. for (int i = 0; i < batch_size; ++i) {
  409. std::copy(inputs_batch_[i].ori_im_size_.begin(),
  410. inputs_batch_[i].ori_im_size_.end(),
  411. inputs_data_size.begin() + 2 * i);
  412. }
  413. im_size_tensor->copy_from_cpu(inputs_data_size.data());
  414. } else if (name == "FasterRCNN" || name == "MaskRCNN") {
  415. auto im_info_tensor = predictor_->GetInputTensor("im_info");
  416. auto im_shape_tensor = predictor_->GetInputTensor("im_shape");
  417. im_info_tensor->Reshape({batch_size, 3});
  418. im_shape_tensor->Reshape({batch_size, 3});
  419. std::vector<float> im_info(3 * batch_size);
  420. std::vector<float> im_shape(3 * batch_size);
  421. for (int i = 0; i < batch_size; ++i) {
  422. float ori_h = static_cast<float>(inputs_batch_[i].ori_im_size_[0]);
  423. float ori_w = static_cast<float>(inputs_batch_[i].ori_im_size_[1]);
  424. float new_h = static_cast<float>(inputs_batch_[i].new_im_size_[0]);
  425. float new_w = static_cast<float>(inputs_batch_[i].new_im_size_[1]);
  426. im_info[i * 3] = new_h;
  427. im_info[i * 3 + 1] = new_w;
  428. im_info[i * 3 + 2] = inputs_batch_[i].scale;
  429. im_shape[i * 3] = ori_h;
  430. im_shape[i * 3 + 1] = ori_w;
  431. im_shape[i * 3 + 2] = 1.0;
  432. }
  433. im_info_tensor->copy_from_cpu(im_info.data());
  434. im_shape_tensor->copy_from_cpu(im_shape.data());
  435. }
  436. // 使用加载的模型进行预测
  437. predictor_->ZeroCopyRun();
  438. // 读取所有box
  439. std::vector<float> output_box;
  440. auto output_names = predictor_->GetOutputNames();
  441. auto output_box_tensor = predictor_->GetOutputTensor(output_names[0]);
  442. std::vector<int> output_box_shape = output_box_tensor->shape();
  443. int size = 1;
  444. for (const auto& i : output_box_shape) {
  445. size *= i;
  446. }
  447. output_box.resize(size);
  448. output_box_tensor->copy_to_cpu(output_box.data());
  449. if (size < 6) {
  450. std::cerr << "[WARNING] There's no object detected." << std::endl;
  451. return true;
  452. }
  453. auto lod_vector = output_box_tensor->lod();
  454. int num_boxes = size / 6;
  455. // 解析预测框box
  456. (*results).clear();
  457. (*results).resize(batch_size);
  458. for (int i = 0; i < lod_vector[0].size() - 1; ++i) {
  459. for (int j = lod_vector[0][i]; j < lod_vector[0][i + 1]; ++j) {
  460. Box box;
  461. box.category_id = static_cast<int>(round(output_box[j * 6]));
  462. box.category = labels[box.category_id];
  463. box.score = output_box[j * 6 + 1];
  464. float xmin = output_box[j * 6 + 2];
  465. float ymin = output_box[j * 6 + 3];
  466. float xmax = output_box[j * 6 + 4];
  467. float ymax = output_box[j * 6 + 5];
  468. float w = xmax - xmin + 1;
  469. float h = ymax - ymin + 1;
  470. box.coordinate = {xmin, ymin, w, h};
  471. (*results)[i].boxes.push_back(std::move(box));
  472. }
  473. }
  474. // 实例分割需解析mask
  475. if (name == "MaskRCNN") {
  476. std::vector<float> output_mask;
  477. auto output_mask_tensor = predictor_->GetOutputTensor(output_names[1]);
  478. std::vector<int> output_mask_shape = output_mask_tensor->shape();
  479. int masks_size = 1;
  480. for (const auto& i : output_mask_shape) {
  481. masks_size *= i;
  482. }
  483. int mask_pixels = output_mask_shape[2] * output_mask_shape[3];
  484. int classes = output_mask_shape[1];
  485. output_mask.resize(masks_size);
  486. output_mask_tensor->copy_to_cpu(output_mask.data());
  487. int mask_idx = 0;
  488. for (int i = 0; i < lod_vector[0].size() - 1; ++i) {
  489. (*results)[i].mask_resolution = output_mask_shape[2];
  490. for (int j = 0; j < (*results)[i].boxes.size(); ++j) {
  491. Box* box = &(*results)[i].boxes[j];
  492. int category_id = box->category_id;
  493. auto begin_mask = output_mask.begin() +
  494. (mask_idx * classes + category_id) * mask_pixels;
  495. auto end_mask = begin_mask + mask_pixels;
  496. box->mask.data.assign(begin_mask, end_mask);
  497. box->mask.shape = {static_cast<int>(box->coordinate[2]),
  498. static_cast<int>(box->coordinate[3])};
  499. mask_idx++;
  500. }
  501. }
  502. }
  503. return true;
  504. }
  505. bool Model::predict(const cv::Mat& im, SegResult* result) {
  506. result->clear();
  507. inputs_.clear();
  508. if (type == "classifier") {
  509. std::cerr << "Loading model is a 'classifier', ClsResult should be passed "
  510. "to function predict()!" << std::endl;
  511. return false;
  512. } else if (type == "detector") {
  513. std::cerr << "Loading model is a 'detector', DetResult should be passed to "
  514. "function predict()!" << std::endl;
  515. return false;
  516. }
  517. // 处理输入图像
  518. if (!preprocess(im, &inputs_)) {
  519. std::cerr << "Preprocess failed!" << std::endl;
  520. return false;
  521. }
  522. int h = inputs_.new_im_size_[0];
  523. int w = inputs_.new_im_size_[1];
  524. auto im_tensor = predictor_->GetInputTensor("image");
  525. im_tensor->Reshape({1, 3, h, w});
  526. im_tensor->copy_from_cpu(inputs_.im_data_.data());
  527. // 使用加载的模型进行预测
  528. predictor_->ZeroCopyRun();
  529. // 获取预测置信度,经过argmax后的labelmap
  530. auto output_names = predictor_->GetOutputNames();
  531. auto output_label_tensor = predictor_->GetOutputTensor(output_names[0]);
  532. std::vector<int> output_label_shape = output_label_tensor->shape();
  533. int size = 1;
  534. for (const auto& i : output_label_shape) {
  535. size *= i;
  536. result->label_map.shape.push_back(i);
  537. }
  538. result->label_map.data.resize(size);
  539. output_label_tensor->copy_to_cpu(result->label_map.data.data());
  540. // 获取预测置信度scoremap
  541. auto output_score_tensor = predictor_->GetOutputTensor(output_names[1]);
  542. std::vector<int> output_score_shape = output_score_tensor->shape();
  543. size = 1;
  544. for (const auto& i : output_score_shape) {
  545. size *= i;
  546. result->score_map.shape.push_back(i);
  547. }
  548. result->score_map.data.resize(size);
  549. output_score_tensor->copy_to_cpu(result->score_map.data.data());
  550. // 解析输出结果到原图大小
  551. std::vector<uint8_t> label_map(result->label_map.data.begin(),
  552. result->label_map.data.end());
  553. cv::Mat mask_label(result->label_map.shape[1],
  554. result->label_map.shape[2],
  555. CV_8UC1,
  556. label_map.data());
  557. cv::Mat mask_score(result->score_map.shape[2],
  558. result->score_map.shape[3],
  559. CV_32FC1,
  560. result->score_map.data.data());
  561. int idx = 1;
  562. int len_postprocess = inputs_.im_size_before_resize_.size();
  563. for (std::vector<std::string>::reverse_iterator iter =
  564. inputs_.reshape_order_.rbegin();
  565. iter != inputs_.reshape_order_.rend();
  566. ++iter) {
  567. if (*iter == "padding") {
  568. auto before_shape = inputs_.im_size_before_resize_[len_postprocess - idx];
  569. inputs_.im_size_before_resize_.pop_back();
  570. auto padding_w = before_shape[0];
  571. auto padding_h = before_shape[1];
  572. mask_label = mask_label(cv::Rect(0, 0, padding_h, padding_w));
  573. mask_score = mask_score(cv::Rect(0, 0, padding_h, padding_w));
  574. } else if (*iter == "resize") {
  575. auto before_shape = inputs_.im_size_before_resize_[len_postprocess - idx];
  576. inputs_.im_size_before_resize_.pop_back();
  577. auto resize_w = before_shape[0];
  578. auto resize_h = before_shape[1];
  579. cv::resize(mask_label,
  580. mask_label,
  581. cv::Size(resize_h, resize_w),
  582. 0,
  583. 0,
  584. cv::INTER_NEAREST);
  585. cv::resize(mask_score,
  586. mask_score,
  587. cv::Size(resize_h, resize_w),
  588. 0,
  589. 0,
  590. cv::INTER_LINEAR);
  591. }
  592. ++idx;
  593. }
  594. result->label_map.data.assign(mask_label.begin<uint8_t>(),
  595. mask_label.end<uint8_t>());
  596. result->label_map.shape = {mask_label.rows, mask_label.cols};
  597. result->score_map.data.assign(mask_score.begin<float>(),
  598. mask_score.end<float>());
  599. result->score_map.shape = {mask_score.rows, mask_score.cols};
  600. return true;
  601. }
  602. bool Model::predict(const std::vector<cv::Mat>& im_batch,
  603. std::vector<SegResult>* results,
  604. int thread_num) {
  605. for (auto& inputs : inputs_batch_) {
  606. inputs.clear();
  607. }
  608. if (type == "classifier") {
  609. std::cerr << "Loading model is a 'classifier', ClsResult should be passed "
  610. "to function predict()!" << std::endl;
  611. return false;
  612. } else if (type == "detector") {
  613. std::cerr << "Loading model is a 'detector', DetResult should be passed to "
  614. "function predict()!" << std::endl;
  615. return false;
  616. }
  617. // 处理输入图像
  618. inputs_batch_.assign(im_batch.size(), ImageBlob());
  619. if (!preprocess(im_batch, &inputs_batch_, thread_num)) {
  620. std::cerr << "Preprocess failed!" << std::endl;
  621. return false;
  622. }
  623. int batch_size = im_batch.size();
  624. (*results).clear();
  625. (*results).resize(batch_size);
  626. int h = inputs_batch_[0].new_im_size_[0];
  627. int w = inputs_batch_[0].new_im_size_[1];
  628. auto im_tensor = predictor_->GetInputTensor("image");
  629. im_tensor->Reshape({batch_size, 3, h, w});
  630. std::vector<float> inputs_data(batch_size * 3 * h * w);
  631. for (int i = 0; i < batch_size; ++i) {
  632. std::copy(inputs_batch_[i].im_data_.begin(),
  633. inputs_batch_[i].im_data_.end(),
  634. inputs_data.begin() + i * 3 * h * w);
  635. }
  636. im_tensor->copy_from_cpu(inputs_data.data());
  637. // im_tensor->copy_from_cpu(inputs_.im_data_.data());
  638. // 使用加载的模型进行预测
  639. predictor_->ZeroCopyRun();
  640. // 获取预测置信度,经过argmax后的labelmap
  641. auto output_names = predictor_->GetOutputNames();
  642. auto output_label_tensor = predictor_->GetOutputTensor(output_names[0]);
  643. std::vector<int> output_label_shape = output_label_tensor->shape();
  644. int size = 1;
  645. for (const auto& i : output_label_shape) {
  646. size *= i;
  647. }
  648. std::vector<int64_t> output_labels(size, 0);
  649. output_label_tensor->copy_to_cpu(output_labels.data());
  650. auto output_labels_iter = output_labels.begin();
  651. int single_batch_size = size / batch_size;
  652. for (int i = 0; i < batch_size; ++i) {
  653. (*results)[i].label_map.data.resize(single_batch_size);
  654. (*results)[i].label_map.shape.push_back(1);
  655. for (int j = 1; j < output_label_shape.size(); ++j) {
  656. (*results)[i].label_map.shape.push_back(output_label_shape[j]);
  657. }
  658. std::copy(output_labels_iter + i * single_batch_size,
  659. output_labels_iter + (i + 1) * single_batch_size,
  660. (*results)[i].label_map.data.data());
  661. }
  662. // 获取预测置信度scoremap
  663. auto output_score_tensor = predictor_->GetOutputTensor(output_names[1]);
  664. std::vector<int> output_score_shape = output_score_tensor->shape();
  665. size = 1;
  666. for (const auto& i : output_score_shape) {
  667. size *= i;
  668. }
  669. std::vector<float> output_scores(size, 0);
  670. output_score_tensor->copy_to_cpu(output_scores.data());
  671. auto output_scores_iter = output_scores.begin();
  672. int single_batch_score_size = size / batch_size;
  673. for (int i = 0; i < batch_size; ++i) {
  674. (*results)[i].score_map.data.resize(single_batch_score_size);
  675. (*results)[i].score_map.shape.push_back(1);
  676. for (int j = 1; j < output_score_shape.size(); ++j) {
  677. (*results)[i].score_map.shape.push_back(output_score_shape[j]);
  678. }
  679. std::copy(output_scores_iter + i * single_batch_score_size,
  680. output_scores_iter + (i + 1) * single_batch_score_size,
  681. (*results)[i].score_map.data.data());
  682. }
  683. // 解析输出结果到原图大小
  684. for (int i = 0; i < batch_size; ++i) {
  685. std::vector<uint8_t> label_map((*results)[i].label_map.data.begin(),
  686. (*results)[i].label_map.data.end());
  687. cv::Mat mask_label((*results)[i].label_map.shape[1],
  688. (*results)[i].label_map.shape[2],
  689. CV_8UC1,
  690. label_map.data());
  691. cv::Mat mask_score((*results)[i].score_map.shape[2],
  692. (*results)[i].score_map.shape[3],
  693. CV_32FC1,
  694. (*results)[i].score_map.data.data());
  695. int idx = 1;
  696. int len_postprocess = inputs_batch_[i].im_size_before_resize_.size();
  697. for (std::vector<std::string>::reverse_iterator iter =
  698. inputs_batch_[i].reshape_order_.rbegin();
  699. iter != inputs_batch_[i].reshape_order_.rend();
  700. ++iter) {
  701. if (*iter == "padding") {
  702. auto before_shape =
  703. inputs_batch_[i].im_size_before_resize_[len_postprocess - idx];
  704. inputs_batch_[i].im_size_before_resize_.pop_back();
  705. auto padding_w = before_shape[0];
  706. auto padding_h = before_shape[1];
  707. mask_label = mask_label(cv::Rect(0, 0, padding_h, padding_w));
  708. mask_score = mask_score(cv::Rect(0, 0, padding_h, padding_w));
  709. } else if (*iter == "resize") {
  710. auto before_shape =
  711. inputs_batch_[i].im_size_before_resize_[len_postprocess - idx];
  712. inputs_batch_[i].im_size_before_resize_.pop_back();
  713. auto resize_w = before_shape[0];
  714. auto resize_h = before_shape[1];
  715. cv::resize(mask_label,
  716. mask_label,
  717. cv::Size(resize_h, resize_w),
  718. 0,
  719. 0,
  720. cv::INTER_NEAREST);
  721. cv::resize(mask_score,
  722. mask_score,
  723. cv::Size(resize_h, resize_w),
  724. 0,
  725. 0,
  726. cv::INTER_LINEAR);
  727. }
  728. ++idx;
  729. }
  730. (*results)[i].label_map.data.assign(mask_label.begin<uint8_t>(),
  731. mask_label.end<uint8_t>());
  732. (*results)[i].label_map.shape = {mask_label.rows, mask_label.cols};
  733. (*results)[i].score_map.data.assign(mask_score.begin<float>(),
  734. mask_score.end<float>());
  735. (*results)[i].score_map.shape = {mask_score.rows, mask_score.cols};
  736. }
  737. return true;
  738. }
  739. } // namespace PaddleX