om_backend.cc 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578
  1. // Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. #include "ultra_infer/runtime/backends/om/om_backend.h"
  15. #include "acl/acl.h"
  16. #include <chrono>
  17. #include <sys/stat.h>
  18. namespace ultra_infer {
  19. bool OmBackend::aclInitFlag = false;
  20. OmBackend::~OmBackend() {
  21. FreeInputBuffer();
  22. FreeOutputBuffer();
  23. DestroyInput();
  24. DestroyOutput();
  25. DestroyResource();
  26. }
  27. TensorInfo OmBackend::GetInputInfo(int index) {
  28. FDASSERT(index < NumInputs(),
  29. "The index: %d should less than the number of inputs: %d.", index,
  30. NumInputs());
  31. return inputs_desc_[index];
  32. }
  33. std::vector<TensorInfo> OmBackend::GetInputInfos() { return inputs_desc_; }
  34. TensorInfo OmBackend::GetOutputInfo(int index) {
  35. FDASSERT(index < NumOutputs(),
  36. "The index: %d should less than the number of outputs %d.", index,
  37. NumOutputs());
  38. return outputs_desc_[index];
  39. }
  40. std::vector<TensorInfo> OmBackend::GetOutputInfos() { return outputs_desc_; }
  41. bool OmBackend::Init(const RuntimeOption &runtime_option) {
  42. // ACL init
  43. aclError ret = InitResource();
  44. if (ret != true) {
  45. FDERROR << "execute InitResource failed, errorCode = "
  46. << static_cast<int32_t>(ret);
  47. return false;
  48. }
  49. // model init;
  50. const char *omModelPath = (char *)runtime_option.model_file.data();
  51. FDINFO << "omModelPath = " << omModelPath;
  52. ret = LoadModel(omModelPath);
  53. if (ret != true) {
  54. FDERROR << "execute LoadModel failed";
  55. return false;
  56. }
  57. // build input/output info
  58. ret = CreateModelDesc();
  59. if (ret != true) {
  60. FDERROR << "execute CreateModelDesc failed";
  61. return false;
  62. }
  63. ret = CreateInput();
  64. if (ret != true) {
  65. FDERROR << "execute CreateInput failed";
  66. FreeInputBuffer();
  67. return false;
  68. }
  69. ret = CreateOutput();
  70. if (ret != true) {
  71. FDERROR << "execute CreateOutput failed";
  72. FreeInputBuffer();
  73. return false;
  74. }
  75. return true;
  76. }
  77. bool OmBackend::Infer(std::vector<FDTensor> &inputs,
  78. std::vector<FDTensor> *outputs, bool copy_to_fd) {
  79. // set context
  80. aclError aclRet = aclrtSetCurrentContext(context_);
  81. if (aclRet != ACL_SUCCESS) {
  82. FDERROR << "aclrtSetCurrentContext failed"
  83. << ", errorCode is " << static_cast<int32_t>(aclRet);
  84. return false;
  85. }
  86. // Judge whether the input and output size are the same
  87. if (inputs.size() != inputs_desc_.size()) {
  88. FDERROR << "[OmBackend] Size of the inputs(" << inputs.size()
  89. << ") should keep same with the inputs of this model("
  90. << inputs_desc_.size() << ")." << std::endl;
  91. FreeInputBuffer();
  92. return false;
  93. }
  94. // cp input tensor to inputBuffer
  95. for (size_t i = 0; i < inputs.size(); ++i) {
  96. if (inputs[i].Data() == nullptr) {
  97. FDERROR << "inputs[i].Data is NULL." << std::endl;
  98. return false;
  99. }
  100. size_t modelInputSize = aclmdlGetInputSizeByIndex(modelDesc_, i);
  101. aclRet = aclrtMemcpy(inputBuffer[i], modelInputSize, inputs[i].Data(),
  102. inputs[i].Nbytes(), ACL_MEMCPY_DEVICE_TO_DEVICE);
  103. if (aclRet != ACL_SUCCESS) {
  104. FDERROR << "memcpy d2d failed. buffer size is " << modelInputSize
  105. << ", inputs[i].Nbytes() is " << inputs[i].Nbytes()
  106. << ", errorCode is " << static_cast<int32_t>(aclRet);
  107. return false;
  108. }
  109. }
  110. bool ret = Execute();
  111. if (ret != true) {
  112. FDERROR << "execute inference failed";
  113. FreeInputBuffer();
  114. DestroyInput();
  115. DestroyOutput();
  116. return false;
  117. }
  118. // cp outputbuffer to outputs
  119. outputs->resize(outputs_desc_.size());
  120. std::vector<int64_t> temp_shape(4);
  121. for (size_t i = 0; i < outputs_desc_.size(); ++i) {
  122. temp_shape.resize(outputs_desc_[i].shape.size());
  123. for (int j = 0; j < outputs_desc_[i].shape.size(); ++j) {
  124. temp_shape[j] = outputs_desc_[i].shape[j];
  125. }
  126. (*outputs)[i].Resize(temp_shape, outputs_desc_[i].dtype,
  127. outputs_desc_[i].name);
  128. size_t modelOutputSize = aclmdlGetOutputSizeByIndex(modelDesc_, i);
  129. if (modelOutputSize != (*outputs)[i].Nbytes()) {
  130. FDERROR << "output size is not match, index: " << i
  131. << ", modelOutputSize:" << modelOutputSize
  132. << ", (*outputs)[i].Nbytes():" << (*outputs)[i].Nbytes();
  133. return false;
  134. }
  135. aclError aclRet = aclrtMemcpy(
  136. (*outputs)[i].MutableData(), (*outputs)[i].Nbytes(), outputBuffer[i],
  137. (*outputs)[i].Nbytes(), ACL_MEMCPY_DEVICE_TO_HOST);
  138. if (aclRet != ACL_SUCCESS) {
  139. FDERROR << "memcpy h2d failed. buffer size is " << (*outputs)[i].Nbytes()
  140. << ", errorCode is " << static_cast<int32_t>(aclRet);
  141. return false;
  142. }
  143. }
  144. return true;
  145. }
  146. bool OmBackend::InitResource() {
  147. // ACL init
  148. aclError ret;
  149. if (aclInitFlag == false) {
  150. ret = aclInit(NULL);
  151. if (ret != ACL_SUCCESS) {
  152. FDERROR << "acl init failed, errorCode = " << static_cast<int32_t>(ret);
  153. return false;
  154. }
  155. aclInitFlag = true;
  156. }
  157. // set device
  158. ret = aclrtSetDevice(deviceId_);
  159. if (ret != ACL_SUCCESS) {
  160. FDERROR << "acl set device" << deviceId_
  161. << " failed, errorCode = " << static_cast<int32_t>(ret);
  162. return false;
  163. }
  164. // create context (set current)
  165. ret = aclrtCreateContext(&context_, deviceId_);
  166. if (ret != ACL_SUCCESS) {
  167. FDERROR << "acl create context failed, deviceId" << deviceId_
  168. << ", errorCode = " << static_cast<int32_t>(ret);
  169. return false;
  170. }
  171. // create stream
  172. ret = aclrtCreateStream(&stream_);
  173. if (ret != ACL_SUCCESS) {
  174. FDERROR << "acl create stream failed, deviceId" << deviceId_
  175. << ", errorCode = " << static_cast<int32_t>(ret);
  176. return false;
  177. }
  178. // get run mode
  179. // runMode is ACL_HOST which represents app is running in host
  180. // runMode is ACL_DEVICE which represents app is running in device
  181. aclrtRunMode runMode;
  182. ret = aclrtGetRunMode(&runMode);
  183. if (ret != ACL_SUCCESS) {
  184. FDERROR << "acl get run mode failed, errorCode = "
  185. << static_cast<int32_t>(ret);
  186. return false;
  187. }
  188. return true;
  189. }
  190. bool OmBackend::LoadModel(const char *modelPath) {
  191. if (loadFlag_) {
  192. FDERROR << "model has already been loaded";
  193. return false;
  194. }
  195. aclError ret = aclmdlQuerySize(modelPath, &modelWorkSize_, &modelWeightSize_);
  196. if (ret != ACL_SUCCESS) {
  197. FDERROR << "query model false, model file is" << modelPath
  198. << ", errorCode is " << static_cast<int32_t>(ret);
  199. return false;
  200. }
  201. // using ACL_MEM_MALLOC_HUGE_FIRST to malloc memory, huge memory is preferred
  202. // to use and huge memory can improve performance.
  203. ret = aclrtMalloc(&modelWorkPtr_, modelWorkSize_, ACL_MEM_MALLOC_HUGE_FIRST);
  204. if (ret != ACL_SUCCESS) {
  205. FDERROR << "malloc buffer for work failed, require size is "
  206. << modelWorkSize_ << ", errorCode is " << static_cast<int32_t>(ret);
  207. return false;
  208. }
  209. // using ACL_MEM_MALLOC_HUGE_FIRST to malloc memory, huge memory is preferred
  210. // to use and huge memory can improve performance.
  211. ret = aclrtMalloc(&modelWeightPtr_, modelWeightSize_,
  212. ACL_MEM_MALLOC_HUGE_FIRST);
  213. if (ret != ACL_SUCCESS) {
  214. FDERROR << "malloc buffer for weight failed, require size is "
  215. << modelWeightSize_ << ", errorCode is "
  216. << static_cast<int32_t>(ret);
  217. return false;
  218. }
  219. ret = aclmdlLoadFromFileWithMem(modelPath, &modelId_, modelWorkPtr_,
  220. modelWorkSize_, modelWeightPtr_,
  221. modelWeightSize_);
  222. if (ret != ACL_SUCCESS) {
  223. FDERROR << "load model from file failed, model file is " << modelPath
  224. << ", errorCode is " << static_cast<int32_t>(ret);
  225. return false;
  226. }
  227. loadFlag_ = true;
  228. FDINFO << "load model " << modelPath << " success";
  229. return true;
  230. }
  231. bool OmBackend::Execute() {
  232. aclError ret = aclmdlExecute(modelId_, input_, output_);
  233. if (ret != ACL_SUCCESS) {
  234. FDERROR << "execute model failed, modelId is " << modelId_
  235. << ", errorCode is " << static_cast<int32_t>(ret);
  236. return false;
  237. }
  238. return true;
  239. }
  240. bool OmBackend::CreateModelDesc() {
  241. modelDesc_ = aclmdlCreateDesc();
  242. if (modelDesc_ == nullptr) {
  243. FDERROR << "create model description failed";
  244. return false;
  245. }
  246. aclError ret = aclmdlGetDesc(modelDesc_, modelId_);
  247. if (ret != ACL_SUCCESS) {
  248. FDERROR << "get model description failed, modelId is " << modelId_
  249. << ", errorCode is " << static_cast<int32_t>(ret);
  250. return false;
  251. }
  252. return true;
  253. }
  254. bool OmBackend::CreateInput() {
  255. // om used in this sample has only one input
  256. if (modelDesc_ == nullptr) {
  257. FDERROR << "no model description, create input failed";
  258. return false;
  259. }
  260. // input:aclmdlDataset
  261. input_ = aclmdlCreateDataset();
  262. if (input_ == nullptr) {
  263. FDERROR << "can't create dataset, create input failed";
  264. return false;
  265. }
  266. // get input nums
  267. size_t inputNum = aclmdlGetNumInputs(modelDesc_);
  268. inputs_desc_.resize(inputNum);
  269. inputBuffer.resize(inputNum, nullptr);
  270. // inputBuffer = {nullptr};
  271. for (size_t i = 0; i < inputNum; ++i) {
  272. // get input size
  273. size_t modelInputSize = aclmdlGetInputSizeByIndex(modelDesc_, i);
  274. aclError ret =
  275. aclrtMalloc(&inputBuffer[i], modelInputSize, ACL_MEM_MALLOC_HUGE_FIRST);
  276. if (ret != ACL_SUCCESS) {
  277. FDERROR << "can't malloc buffer, size is " << modelInputSize
  278. << ", errorCode is " << static_cast<int32_t>(ret);
  279. return false;
  280. }
  281. // inputData:aclDataBuffer
  282. aclDataBuffer *inputData =
  283. aclCreateDataBuffer(inputBuffer[i], modelInputSize);
  284. if (inputData == nullptr) {
  285. FDERROR << "can't create data buffer, create input failed";
  286. return false;
  287. }
  288. // add aclDataBuffer to input
  289. ret = aclmdlAddDatasetBuffer(input_, inputData);
  290. if (ret != ACL_SUCCESS) {
  291. FDERROR << "add input dataset buffer failed, errorCode is "
  292. << static_cast<int32_t>(ret);
  293. (void)aclDestroyDataBuffer(inputData);
  294. inputData = nullptr;
  295. return false;
  296. }
  297. // get name/shape/dtype of input to build inputs_desc_
  298. const char *name;
  299. name = aclmdlGetInputNameByIndex(modelDesc_, i);
  300. std::string temp_name = name;
  301. std::vector<int> temp_shape{};
  302. aclmdlIODims dims;
  303. ret = aclmdlGetInputDims(modelDesc_, i, &dims);
  304. if (ret != ACL_SUCCESS) {
  305. FDERROR << "get input tensor dims fail! ret=" << ret << std::endl;
  306. return false;
  307. }
  308. int n_dims = (int)dims.dimCount;
  309. temp_shape.resize(n_dims);
  310. for (int j = 0; j < n_dims; j++) {
  311. temp_shape[j] = (int)dims.dims[j];
  312. }
  313. aclDataType dtype = aclmdlGetInputDataType(modelDesc_, i);
  314. FDDataType temp_dtype;
  315. switch (dtype) {
  316. case ACL_BOOL:
  317. temp_dtype = FDDataType::BOOL;
  318. break;
  319. case ACL_UINT8:
  320. temp_dtype = FDDataType::UINT8;
  321. break;
  322. case ACL_INT8:
  323. temp_dtype = FDDataType::INT8;
  324. break;
  325. case ACL_INT16:
  326. temp_dtype = FDDataType::INT16;
  327. break;
  328. case ACL_INT32:
  329. temp_dtype = FDDataType::INT32;
  330. break;
  331. case ACL_INT64:
  332. temp_dtype = FDDataType::INT64;
  333. break;
  334. case ACL_FLOAT16:
  335. temp_dtype = FDDataType::FP16;
  336. break;
  337. case ACL_FLOAT:
  338. temp_dtype = FDDataType::FP32;
  339. break;
  340. case ACL_DOUBLE:
  341. temp_dtype = FDDataType::FP64;
  342. break;
  343. default:
  344. FDERROR << "unsupported input tensor dtype: " << (int)dtype;
  345. return false;
  346. }
  347. TensorInfo temp_input_info = {temp_name, temp_shape, temp_dtype};
  348. inputs_desc_[i] = temp_input_info;
  349. }
  350. return true;
  351. }
  352. bool OmBackend::CreateOutput() {
  353. if (modelDesc_ == nullptr) {
  354. FDERROR << "no model description, create ouput failed";
  355. return false;
  356. }
  357. output_ = aclmdlCreateDataset();
  358. if (output_ == nullptr) {
  359. FDERROR << "can't create dataset, create output failed";
  360. return false;
  361. }
  362. size_t outputSize = aclmdlGetNumOutputs(modelDesc_);
  363. outputs_desc_.resize(outputSize);
  364. outputBuffer.resize(outputSize, nullptr);
  365. for (size_t i = 0; i < outputSize; ++i) {
  366. size_t modelOutputSize = aclmdlGetOutputSizeByIndex(modelDesc_, i);
  367. aclError ret = aclrtMalloc(&outputBuffer[i], modelOutputSize,
  368. ACL_MEM_MALLOC_HUGE_FIRST);
  369. if (ret != ACL_SUCCESS) {
  370. FDERROR << "can't malloc buffer, size is " << modelOutputSize
  371. << ", errorCode is " << static_cast<int32_t>(ret);
  372. return false;
  373. }
  374. aclDataBuffer *outputData =
  375. aclCreateDataBuffer(outputBuffer[i], modelOutputSize);
  376. if (outputData == nullptr) {
  377. FDERROR << "can't create data buffer, create output failed";
  378. return false;
  379. }
  380. ret = aclmdlAddDatasetBuffer(output_, outputData);
  381. if (ret != ACL_SUCCESS) {
  382. FDERROR << "add output dataset buffer failed, errorCode is "
  383. << static_cast<int32_t>(ret);
  384. (void)aclDestroyDataBuffer(outputData);
  385. return false;
  386. }
  387. const char *name;
  388. name = aclmdlGetOutputNameByIndex(modelDesc_, i);
  389. std::string temp_name = name;
  390. std::vector<int> temp_shape{};
  391. aclmdlIODims dims;
  392. ret = aclmdlGetOutputDims(modelDesc_, i, &dims);
  393. if (ret != ACL_SUCCESS) {
  394. FDERROR << "get output tensor dims fail! ret=" << ret << std::endl;
  395. return false;
  396. }
  397. int n_dims = (int)dims.dimCount;
  398. temp_shape.resize(n_dims);
  399. for (int j = 0; j < n_dims; j++) {
  400. temp_shape[j] = (int)dims.dims[j];
  401. }
  402. aclDataType dtype = aclmdlGetOutputDataType(modelDesc_, i);
  403. FDDataType temp_dtype;
  404. switch (dtype) {
  405. case ACL_BOOL:
  406. temp_dtype = FDDataType::BOOL;
  407. break;
  408. case ACL_UINT8:
  409. temp_dtype = FDDataType::UINT8;
  410. break;
  411. case ACL_INT8:
  412. temp_dtype = FDDataType::INT8;
  413. break;
  414. case ACL_INT16:
  415. temp_dtype = FDDataType::INT16;
  416. break;
  417. case ACL_INT32:
  418. temp_dtype = FDDataType::INT32;
  419. break;
  420. case ACL_INT64:
  421. temp_dtype = FDDataType::INT64;
  422. break;
  423. case ACL_FLOAT16:
  424. temp_dtype = FDDataType::FP16;
  425. break;
  426. case ACL_FLOAT:
  427. temp_dtype = FDDataType::FP32;
  428. break;
  429. case ACL_DOUBLE:
  430. temp_dtype = FDDataType::FP64;
  431. break;
  432. default:
  433. FDERROR << "unsupported output tensor dtype: " << (int)dtype;
  434. return false;
  435. }
  436. TensorInfo temp_output_info = {temp_name, temp_shape, temp_dtype};
  437. outputs_desc_[i] = temp_output_info;
  438. }
  439. return true;
  440. }
  441. void OmBackend::FreeInputBuffer() {
  442. for (int i = 0; i < (int)inputs_desc_.size(); ++i) {
  443. if (inputBuffer[i] != nullptr) {
  444. (void)aclrtFree(inputBuffer[i]);
  445. inputBuffer[i] = nullptr;
  446. }
  447. }
  448. }
  449. void OmBackend::FreeOutputBuffer() {
  450. for (int i = 0; i < (int)outputs_desc_.size(); ++i) {
  451. if (outputBuffer[i] != nullptr) {
  452. (void)aclrtFree(outputBuffer[i]);
  453. outputBuffer[i] = nullptr;
  454. }
  455. }
  456. }
  457. void OmBackend::DestroyInput() {
  458. if (input_ == nullptr) {
  459. return;
  460. }
  461. for (size_t i = 0; i < aclmdlGetDatasetNumBuffers(input_); ++i) {
  462. aclDataBuffer *dataBuffer = aclmdlGetDatasetBuffer(input_, i);
  463. (void)aclDestroyDataBuffer(dataBuffer);
  464. }
  465. (void)aclmdlDestroyDataset(input_);
  466. input_ = nullptr;
  467. }
  468. void OmBackend::DestroyOutput() {
  469. if (output_ == nullptr) {
  470. return;
  471. }
  472. for (size_t i = 0; i < aclmdlGetDatasetNumBuffers(output_); ++i) {
  473. aclDataBuffer *dataBuffer = aclmdlGetDatasetBuffer(output_, i);
  474. void *data = aclGetDataBufferAddr(dataBuffer);
  475. (void)aclrtFree(data);
  476. (void)aclDestroyDataBuffer(dataBuffer);
  477. }
  478. (void)aclmdlDestroyDataset(output_);
  479. output_ = nullptr;
  480. }
  481. void OmBackend::DestroyResource() {
  482. // set context
  483. aclError ret = aclrtSetCurrentContext(context_);
  484. if (ret != ACL_SUCCESS) {
  485. FDERROR << "aclrtSetCurrentContext failed"
  486. << ", errorCode is " << static_cast<int32_t>(ret);
  487. return;
  488. }
  489. if (stream_ != nullptr) {
  490. ret = aclrtDestroyStream(stream_);
  491. if (ret != ACL_SUCCESS) {
  492. FDERROR << "destroy stream failed, errorCode = "
  493. << static_cast<int32_t>(ret);
  494. }
  495. stream_ = nullptr;
  496. }
  497. if (context_ != nullptr) {
  498. ret = aclrtDestroyContext(context_);
  499. if (ret != ACL_SUCCESS) {
  500. FDERROR << "destroy context failed, errorCode = "
  501. << static_cast<int32_t>(ret);
  502. }
  503. context_ = nullptr;
  504. }
  505. ret = aclrtResetDevice(deviceId_);
  506. if (ret != ACL_SUCCESS) {
  507. FDERROR << "reset device " << deviceId_
  508. << " failed, errorCode = " << static_cast<int32_t>(ret);
  509. }
  510. if (aclInitFlag == true) {
  511. ret = aclFinalize();
  512. if (ret != ACL_SUCCESS) {
  513. FDERROR << "finalize acl failed, errorCode = "
  514. << static_cast<int32_t>(ret);
  515. }
  516. aclInitFlag = false;
  517. }
  518. }
  519. } // namespace ultra_infer