LLVM 24.0.0git
NVVMIntrinsicUtils.h
Go to the documentation of this file.
1//===--- NVVMIntrinsicUtils.h -----------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file contains the definitions of the enumerations and flags
11/// associated with NVVM Intrinsics, along with some helper functions.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_IR_NVVMINTRINSICUTILS_H
16#define LLVM_IR_NVVMINTRINSICUTILS_H
17
18#include <stdint.h>
19
20#include "llvm/ADT/APFloat.h"
21#include "llvm/ADT/APInt.h"
22#include "llvm/ADT/StringRef.h"
23#include "llvm/IR/Constants.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/IntrinsicsNVPTX.h"
26#include "llvm/Support/raw_ostream.h"
27
28namespace llvm {
29namespace nvvm {
30
31// Reduction Ops supported with TMA Copy from Shared
32// to Global Memory for the "cp.reduce.async.bulk.tensor.*"
33// family of PTX instructions.
34enum class TMAReductionOp : uint8_t {
35 ADD = 0,
36 MIN = 1,
37 MAX = 2,
38 INC = 3,
39 DEC = 4,
40 AND = 5,
41 OR = 6,
42 XOR = 7,
43};
44
45inline StringRef getTMATensorReductionOpName(TMAReductionOp Op) {
46 switch (Op) {
47 case TMAReductionOp::ADD:
48 return "add";
49 case TMAReductionOp::MIN:
50 return "min";
51 case TMAReductionOp::MAX:
52 return "max";
53 case TMAReductionOp::INC:
54 return "inc";
55 case TMAReductionOp::DEC:
56 return "dec";
57 case TMAReductionOp::AND:
58 return "and";
59 case TMAReductionOp::OR:
60 return "or";
61 case TMAReductionOp::XOR:
62 return "xor";
63 }
64 llvm_unreachable("invalid TMA tensorreduction operation");
65}
66
67// Enum to represent the cta_group::1 and
68// cta_group::2 variants in TMA/TCGEN05 family of
69// PTX instructions.
70enum class CTAGroupKind : uint8_t {
71 CG_NONE = 0, // default with no cta_group modifier
72 CG_1 = 1, // cta_group::1 modifier
73 CG_2 = 2, // cta_group::2 modifier
74};
75
76enum class Tcgen05MMAKind : uint8_t {
77 F16 = 0,
78 TF32 = 1,
79 F8F6F4 = 2,
80 I8 = 3,
81 TI16 = 4,
82};
83
84enum class Tcgen05CollectorUsageOp : uint8_t {
85 DISCARD = 0,
86 LASTUSE = 1,
87 FILL = 2,
88 USE = 3,
89};
90
91enum class Tcgen05MMACollectorBBuffer : uint8_t {
92 B0 = 0,
93 B1 = 1,
94 B2 = 2,
95 B3 = 3,
96};
97
98enum class TensormapElemType : uint8_t {
99 U8 = 0,
100 U16 = 1,
101 U32 = 2,
102 S32 = 3,
103 U64 = 4,
104 S64 = 5,
105 F16 = 6,
106 F32 = 7,
107 F32_FTZ = 8,
108 F64 = 9,
109 BF16 = 10,
110 TF32 = 11,
111 TF32_FTZ = 12,
112 B4x16 = 13,
113 B4x16_p64 = 14,
114 B6x16_p32 = 15,
115};
116
122
130
137
142
143LLVM_ABI void printTcgen05MMAKind(raw_ostream &OS, const Constant *ImmArgVal);
144
145LLVM_ABI void printTMAReductionOp(raw_ostream &OS, const Constant *ImmArgVal);
146
147LLVM_ABI void printTcgen05CollectorUsageOp(raw_ostream &OS,
148 const Constant *ImmArgVal);
149
150LLVM_ABI void printTcgen05MMACollectorBBuffer(raw_ostream &OS,
151 const Constant *ImmArgVal);
152
153LLVM_ABI void printTensormapElemType(raw_ostream &OS,
154 const Constant *ImmArgVal);
155LLVM_ABI void printTensormapInterleaveLayout(raw_ostream &OS,
156 const Constant *ImmArgVal);
157LLVM_ABI void printTensormapSwizzleMode(raw_ostream &OS,
158 const Constant *ImmArgVal);
159LLVM_ABI void printTensormapSwizzleAtomicity(raw_ostream &OS,
160 const Constant *ImmArgVal);
161LLVM_ABI void printTensormapFillMode(raw_ostream &OS,
162 const Constant *ImmArgVal);
163
164inline bool FPToIntegerIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID) {
165 switch (IntrinsicID) {
166 case Intrinsic::nvvm_f2i_rm_ftz:
167 case Intrinsic::nvvm_f2i_rn_ftz:
168 case Intrinsic::nvvm_f2i_rp_ftz:
169 case Intrinsic::nvvm_f2i_rz_ftz:
170
171 case Intrinsic::nvvm_f2ui_rm_ftz:
172 case Intrinsic::nvvm_f2ui_rn_ftz:
173 case Intrinsic::nvvm_f2ui_rp_ftz:
174 case Intrinsic::nvvm_f2ui_rz_ftz:
175
176 case Intrinsic::nvvm_f2ll_rm_ftz:
177 case Intrinsic::nvvm_f2ll_rn_ftz:
178 case Intrinsic::nvvm_f2ll_rp_ftz:
179 case Intrinsic::nvvm_f2ll_rz_ftz:
180
181 case Intrinsic::nvvm_f2ull_rm_ftz:
182 case Intrinsic::nvvm_f2ull_rn_ftz:
183 case Intrinsic::nvvm_f2ull_rp_ftz:
184 case Intrinsic::nvvm_f2ull_rz_ftz:
185 return true;
186
187 case Intrinsic::nvvm_f2i_rm:
188 case Intrinsic::nvvm_f2i_rn:
189 case Intrinsic::nvvm_f2i_rp:
190 case Intrinsic::nvvm_f2i_rz:
191
192 case Intrinsic::nvvm_f2ui_rm:
193 case Intrinsic::nvvm_f2ui_rn:
194 case Intrinsic::nvvm_f2ui_rp:
195 case Intrinsic::nvvm_f2ui_rz:
196
197 case Intrinsic::nvvm_d2i_rm:
198 case Intrinsic::nvvm_d2i_rn:
199 case Intrinsic::nvvm_d2i_rp:
200 case Intrinsic::nvvm_d2i_rz:
201
202 case Intrinsic::nvvm_d2ui_rm:
203 case Intrinsic::nvvm_d2ui_rn:
204 case Intrinsic::nvvm_d2ui_rp:
205 case Intrinsic::nvvm_d2ui_rz:
206
207 case Intrinsic::nvvm_f2ll_rm:
208 case Intrinsic::nvvm_f2ll_rn:
209 case Intrinsic::nvvm_f2ll_rp:
210 case Intrinsic::nvvm_f2ll_rz:
211
212 case Intrinsic::nvvm_f2ull_rm:
213 case Intrinsic::nvvm_f2ull_rn:
214 case Intrinsic::nvvm_f2ull_rp:
215 case Intrinsic::nvvm_f2ull_rz:
216
217 case Intrinsic::nvvm_d2ll_rm:
218 case Intrinsic::nvvm_d2ll_rn:
219 case Intrinsic::nvvm_d2ll_rp:
220 case Intrinsic::nvvm_d2ll_rz:
221
222 case Intrinsic::nvvm_d2ull_rm:
223 case Intrinsic::nvvm_d2ull_rn:
224 case Intrinsic::nvvm_d2ull_rp:
225 case Intrinsic::nvvm_d2ull_rz:
226 return false;
227 }
228 llvm_unreachable("Checking FTZ flag for invalid f2i/d2i intrinsic");
229}
230
231inline bool FPToIntegerIntrinsicResultIsSigned(Intrinsic::ID IntrinsicID) {
232 switch (IntrinsicID) {
233 // f2i
234 case Intrinsic::nvvm_f2i_rm:
235 case Intrinsic::nvvm_f2i_rm_ftz:
236 case Intrinsic::nvvm_f2i_rn:
237 case Intrinsic::nvvm_f2i_rn_ftz:
238 case Intrinsic::nvvm_f2i_rp:
239 case Intrinsic::nvvm_f2i_rp_ftz:
240 case Intrinsic::nvvm_f2i_rz:
241 case Intrinsic::nvvm_f2i_rz_ftz:
242 // d2i
243 case Intrinsic::nvvm_d2i_rm:
244 case Intrinsic::nvvm_d2i_rn:
245 case Intrinsic::nvvm_d2i_rp:
246 case Intrinsic::nvvm_d2i_rz:
247 // f2ll
248 case Intrinsic::nvvm_f2ll_rm:
249 case Intrinsic::nvvm_f2ll_rm_ftz:
250 case Intrinsic::nvvm_f2ll_rn:
251 case Intrinsic::nvvm_f2ll_rn_ftz:
252 case Intrinsic::nvvm_f2ll_rp:
253 case Intrinsic::nvvm_f2ll_rp_ftz:
254 case Intrinsic::nvvm_f2ll_rz:
255 case Intrinsic::nvvm_f2ll_rz_ftz:
256 // d2ll
257 case Intrinsic::nvvm_d2ll_rm:
258 case Intrinsic::nvvm_d2ll_rn:
259 case Intrinsic::nvvm_d2ll_rp:
260 case Intrinsic::nvvm_d2ll_rz:
261 return true;
262
263 // f2ui
264 case Intrinsic::nvvm_f2ui_rm:
265 case Intrinsic::nvvm_f2ui_rm_ftz:
266 case Intrinsic::nvvm_f2ui_rn:
267 case Intrinsic::nvvm_f2ui_rn_ftz:
268 case Intrinsic::nvvm_f2ui_rp:
269 case Intrinsic::nvvm_f2ui_rp_ftz:
270 case Intrinsic::nvvm_f2ui_rz:
271 case Intrinsic::nvvm_f2ui_rz_ftz:
272 // d2ui
273 case Intrinsic::nvvm_d2ui_rm:
274 case Intrinsic::nvvm_d2ui_rn:
275 case Intrinsic::nvvm_d2ui_rp:
276 case Intrinsic::nvvm_d2ui_rz:
277 // f2ull
278 case Intrinsic::nvvm_f2ull_rm:
279 case Intrinsic::nvvm_f2ull_rm_ftz:
280 case Intrinsic::nvvm_f2ull_rn:
281 case Intrinsic::nvvm_f2ull_rn_ftz:
282 case Intrinsic::nvvm_f2ull_rp:
283 case Intrinsic::nvvm_f2ull_rp_ftz:
284 case Intrinsic::nvvm_f2ull_rz:
285 case Intrinsic::nvvm_f2ull_rz_ftz:
286 // d2ull
287 case Intrinsic::nvvm_d2ull_rm:
288 case Intrinsic::nvvm_d2ull_rn:
289 case Intrinsic::nvvm_d2ull_rp:
290 case Intrinsic::nvvm_d2ull_rz:
291 return false;
292 }
293 llvm_unreachable(
294 "Checking invalid f2i/d2i intrinsic for signed int conversion");
295}
296
297inline bool FPToIntegerIntrinsicNaNZero(Intrinsic::ID IntrinsicID) {
298 switch (IntrinsicID) {
299 // f2i
300 case Intrinsic::nvvm_f2i_rm:
301 case Intrinsic::nvvm_f2i_rn:
302 case Intrinsic::nvvm_f2i_rp:
303 case Intrinsic::nvvm_f2i_rz:
304 case Intrinsic::nvvm_f2i_rm_ftz:
305 case Intrinsic::nvvm_f2i_rn_ftz:
306 case Intrinsic::nvvm_f2i_rp_ftz:
307 case Intrinsic::nvvm_f2i_rz_ftz:
308 // f2ui
309 case Intrinsic::nvvm_f2ui_rm:
310 case Intrinsic::nvvm_f2ui_rn:
311 case Intrinsic::nvvm_f2ui_rp:
312 case Intrinsic::nvvm_f2ui_rz:
313 case Intrinsic::nvvm_f2ui_rm_ftz:
314 case Intrinsic::nvvm_f2ui_rn_ftz:
315 case Intrinsic::nvvm_f2ui_rp_ftz:
316 case Intrinsic::nvvm_f2ui_rz_ftz:
317 return true;
318 // d2i
319 case Intrinsic::nvvm_d2i_rm:
320 case Intrinsic::nvvm_d2i_rn:
321 case Intrinsic::nvvm_d2i_rp:
322 case Intrinsic::nvvm_d2i_rz:
323 // d2ui
324 case Intrinsic::nvvm_d2ui_rm:
325 case Intrinsic::nvvm_d2ui_rn:
326 case Intrinsic::nvvm_d2ui_rp:
327 case Intrinsic::nvvm_d2ui_rz:
328 // f2ll
329 case Intrinsic::nvvm_f2ll_rm:
330 case Intrinsic::nvvm_f2ll_rn:
331 case Intrinsic::nvvm_f2ll_rp:
332 case Intrinsic::nvvm_f2ll_rz:
333 case Intrinsic::nvvm_f2ll_rm_ftz:
334 case Intrinsic::nvvm_f2ll_rn_ftz:
335 case Intrinsic::nvvm_f2ll_rp_ftz:
336 case Intrinsic::nvvm_f2ll_rz_ftz:
337 // f2ull
338 case Intrinsic::nvvm_f2ull_rm:
339 case Intrinsic::nvvm_f2ull_rn:
340 case Intrinsic::nvvm_f2ull_rp:
341 case Intrinsic::nvvm_f2ull_rz:
342 case Intrinsic::nvvm_f2ull_rm_ftz:
343 case Intrinsic::nvvm_f2ull_rn_ftz:
344 case Intrinsic::nvvm_f2ull_rp_ftz:
345 case Intrinsic::nvvm_f2ull_rz_ftz:
346 // d2ll
347 case Intrinsic::nvvm_d2ll_rm:
348 case Intrinsic::nvvm_d2ll_rn:
349 case Intrinsic::nvvm_d2ll_rp:
350 case Intrinsic::nvvm_d2ll_rz:
351 // d2ull
352 case Intrinsic::nvvm_d2ull_rm:
353 case Intrinsic::nvvm_d2ull_rn:
354 case Intrinsic::nvvm_d2ull_rp:
355 case Intrinsic::nvvm_d2ull_rz:
356 return false;
357 }
358 llvm_unreachable("Checking NaN result for invalid f2i/d2i intrinsic");
359}
360
361inline APFloat::roundingMode
362GetFPToIntegerRoundingMode(Intrinsic::ID IntrinsicID) {
363 switch (IntrinsicID) {
364 // RM:
365 case Intrinsic::nvvm_f2i_rm:
366 case Intrinsic::nvvm_f2ui_rm:
367 case Intrinsic::nvvm_f2i_rm_ftz:
368 case Intrinsic::nvvm_f2ui_rm_ftz:
369 case Intrinsic::nvvm_d2i_rm:
370 case Intrinsic::nvvm_d2ui_rm:
371
372 case Intrinsic::nvvm_f2ll_rm:
373 case Intrinsic::nvvm_f2ull_rm:
374 case Intrinsic::nvvm_f2ll_rm_ftz:
375 case Intrinsic::nvvm_f2ull_rm_ftz:
376 case Intrinsic::nvvm_d2ll_rm:
377 case Intrinsic::nvvm_d2ull_rm:
378 return APFloat::rmTowardNegative;
379
380 // RN:
381 case Intrinsic::nvvm_f2i_rn:
382 case Intrinsic::nvvm_f2ui_rn:
383 case Intrinsic::nvvm_f2i_rn_ftz:
384 case Intrinsic::nvvm_f2ui_rn_ftz:
385 case Intrinsic::nvvm_d2i_rn:
386 case Intrinsic::nvvm_d2ui_rn:
387
388 case Intrinsic::nvvm_f2ll_rn:
389 case Intrinsic::nvvm_f2ull_rn:
390 case Intrinsic::nvvm_f2ll_rn_ftz:
391 case Intrinsic::nvvm_f2ull_rn_ftz:
392 case Intrinsic::nvvm_d2ll_rn:
393 case Intrinsic::nvvm_d2ull_rn:
394 return APFloat::rmNearestTiesToEven;
395
396 // RP:
397 case Intrinsic::nvvm_f2i_rp:
398 case Intrinsic::nvvm_f2ui_rp:
399 case Intrinsic::nvvm_f2i_rp_ftz:
400 case Intrinsic::nvvm_f2ui_rp_ftz:
401 case Intrinsic::nvvm_d2i_rp:
402 case Intrinsic::nvvm_d2ui_rp:
403
404 case Intrinsic::nvvm_f2ll_rp:
405 case Intrinsic::nvvm_f2ull_rp:
406 case Intrinsic::nvvm_f2ll_rp_ftz:
407 case Intrinsic::nvvm_f2ull_rp_ftz:
408 case Intrinsic::nvvm_d2ll_rp:
409 case Intrinsic::nvvm_d2ull_rp:
410 return APFloat::rmTowardPositive;
411
412 // RZ:
413 case Intrinsic::nvvm_f2i_rz:
414 case Intrinsic::nvvm_f2ui_rz:
415 case Intrinsic::nvvm_f2i_rz_ftz:
416 case Intrinsic::nvvm_f2ui_rz_ftz:
417 case Intrinsic::nvvm_d2i_rz:
418 case Intrinsic::nvvm_d2ui_rz:
419
420 case Intrinsic::nvvm_f2ll_rz:
421 case Intrinsic::nvvm_f2ull_rz:
422 case Intrinsic::nvvm_f2ll_rz_ftz:
423 case Intrinsic::nvvm_f2ull_rz_ftz:
424 case Intrinsic::nvvm_d2ll_rz:
425 case Intrinsic::nvvm_d2ull_rz:
426 return APFloat::rmTowardZero;
427 }
428 llvm_unreachable("Checking rounding mode for invalid f2i/d2i intrinsic");
429}
430
431inline bool FMinFMaxShouldFTZ(Intrinsic::ID IntrinsicID) {
432 switch (IntrinsicID) {
433 case Intrinsic::nvvm_fmax_ftz_f:
434 case Intrinsic::nvvm_fmax_ftz_nan_f:
435 case Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f:
436 case Intrinsic::nvvm_fmax_ftz_xorsign_abs_f:
437
438 case Intrinsic::nvvm_fmin_ftz_f:
439 case Intrinsic::nvvm_fmin_ftz_nan_f:
440 case Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f:
441 case Intrinsic::nvvm_fmin_ftz_xorsign_abs_f:
442 return true;
443
444 case Intrinsic::nvvm_fmax_d:
445 case Intrinsic::nvvm_fmax_f:
446 case Intrinsic::nvvm_fmax_nan_f:
447 case Intrinsic::nvvm_fmax_nan_xorsign_abs_f:
448 case Intrinsic::nvvm_fmax_xorsign_abs_f:
449
450 case Intrinsic::nvvm_fmin_d:
451 case Intrinsic::nvvm_fmin_f:
452 case Intrinsic::nvvm_fmin_nan_f:
453 case Intrinsic::nvvm_fmin_nan_xorsign_abs_f:
454 case Intrinsic::nvvm_fmin_xorsign_abs_f:
455 return false;
456 }
457 llvm_unreachable("Checking FTZ flag for invalid fmin/fmax intrinsic");
458}
459
460inline bool FMinFMaxPropagatesNaNs(Intrinsic::ID IntrinsicID) {
461 switch (IntrinsicID) {
462 case Intrinsic::nvvm_fmax_ftz_nan_f:
463 case Intrinsic::nvvm_fmax_nan_f:
464 case Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f:
465 case Intrinsic::nvvm_fmax_nan_xorsign_abs_f:
466
467 case Intrinsic::nvvm_fmin_ftz_nan_f:
468 case Intrinsic::nvvm_fmin_nan_f:
469 case Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f:
470 case Intrinsic::nvvm_fmin_nan_xorsign_abs_f:
471 return true;
472
473 case Intrinsic::nvvm_fmax_d:
474 case Intrinsic::nvvm_fmax_f:
475 case Intrinsic::nvvm_fmax_ftz_f:
476 case Intrinsic::nvvm_fmax_ftz_xorsign_abs_f:
477 case Intrinsic::nvvm_fmax_xorsign_abs_f:
478
479 case Intrinsic::nvvm_fmin_d:
480 case Intrinsic::nvvm_fmin_f:
481 case Intrinsic::nvvm_fmin_ftz_f:
482 case Intrinsic::nvvm_fmin_ftz_xorsign_abs_f:
483 case Intrinsic::nvvm_fmin_xorsign_abs_f:
484 return false;
485 }
486 llvm_unreachable("Checking NaN flag for invalid fmin/fmax intrinsic");
487}
488
489inline bool FMinFMaxIsXorSignAbs(Intrinsic::ID IntrinsicID) {
490 switch (IntrinsicID) {
491 case Intrinsic::nvvm_fmax_ftz_nan_xorsign_abs_f:
492 case Intrinsic::nvvm_fmax_ftz_xorsign_abs_f:
493 case Intrinsic::nvvm_fmax_nan_xorsign_abs_f:
494 case Intrinsic::nvvm_fmax_xorsign_abs_f:
495
496 case Intrinsic::nvvm_fmin_ftz_nan_xorsign_abs_f:
497 case Intrinsic::nvvm_fmin_ftz_xorsign_abs_f:
498 case Intrinsic::nvvm_fmin_nan_xorsign_abs_f:
499 case Intrinsic::nvvm_fmin_xorsign_abs_f:
500 return true;
501
502 case Intrinsic::nvvm_fmax_d:
503 case Intrinsic::nvvm_fmax_f:
504 case Intrinsic::nvvm_fmax_ftz_f:
505 case Intrinsic::nvvm_fmax_ftz_nan_f:
506 case Intrinsic::nvvm_fmax_nan_f:
507
508 case Intrinsic::nvvm_fmin_d:
509 case Intrinsic::nvvm_fmin_f:
510 case Intrinsic::nvvm_fmin_ftz_f:
511 case Intrinsic::nvvm_fmin_ftz_nan_f:
512 case Intrinsic::nvvm_fmin_nan_f:
513 return false;
514 }
515 llvm_unreachable("Checking XorSignAbs flag for invalid fmin/fmax intrinsic");
516}
517
518inline bool UnaryMathIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID) {
519 switch (IntrinsicID) {
520 case Intrinsic::nvvm_ceil_ftz_f:
521 case Intrinsic::nvvm_fabs_ftz:
522 case Intrinsic::nvvm_floor_ftz_f:
523 case Intrinsic::nvvm_round_ftz_f:
524 case Intrinsic::nvvm_saturate_ftz_f:
525 case Intrinsic::nvvm_sqrt_rn_ftz_f:
526 return true;
527 case Intrinsic::nvvm_ceil_f:
528 case Intrinsic::nvvm_ceil_d:
529 case Intrinsic::nvvm_fabs:
530 case Intrinsic::nvvm_floor_f:
531 case Intrinsic::nvvm_floor_d:
532 case Intrinsic::nvvm_round_f:
533 case Intrinsic::nvvm_round_d:
534 case Intrinsic::nvvm_saturate_d:
535 case Intrinsic::nvvm_saturate_f:
536 case Intrinsic::nvvm_sqrt_f:
537 case Intrinsic::nvvm_sqrt_rn_d:
538 case Intrinsic::nvvm_sqrt_rn_f:
539 return false;
540 }
541 llvm_unreachable("Checking FTZ flag for invalid unary intrinsic");
542}
543
544inline bool RCPShouldFTZ(Intrinsic::ID IntrinsicID) {
545 switch (IntrinsicID) {
546 case Intrinsic::nvvm_rcp_rm_ftz_f:
547 case Intrinsic::nvvm_rcp_rn_ftz_f:
548 case Intrinsic::nvvm_rcp_rp_ftz_f:
549 case Intrinsic::nvvm_rcp_rz_ftz_f:
550 return true;
551 case Intrinsic::nvvm_rcp_rm_d:
552 case Intrinsic::nvvm_rcp_rm_f:
553 case Intrinsic::nvvm_rcp_rn_d:
554 case Intrinsic::nvvm_rcp_rn_f:
555 case Intrinsic::nvvm_rcp_rp_d:
556 case Intrinsic::nvvm_rcp_rp_f:
557 case Intrinsic::nvvm_rcp_rz_d:
558 case Intrinsic::nvvm_rcp_rz_f:
559 return false;
560 }
561 llvm_unreachable("Checking FTZ flag for invalid rcp intrinsic");
562}
563
564inline APFloat::roundingMode GetRCPRoundingMode(Intrinsic::ID IntrinsicID) {
565 switch (IntrinsicID) {
566 case Intrinsic::nvvm_rcp_rm_f:
567 case Intrinsic::nvvm_rcp_rm_d:
568 case Intrinsic::nvvm_rcp_rm_ftz_f:
569 return APFloat::rmTowardNegative;
570
571 case Intrinsic::nvvm_rcp_rn_f:
572 case Intrinsic::nvvm_rcp_rn_d:
573 case Intrinsic::nvvm_rcp_rn_ftz_f:
574 return APFloat::rmNearestTiesToEven;
575
576 case Intrinsic::nvvm_rcp_rp_f:
577 case Intrinsic::nvvm_rcp_rp_d:
578 case Intrinsic::nvvm_rcp_rp_ftz_f:
579 return APFloat::rmTowardPositive;
580
581 case Intrinsic::nvvm_rcp_rz_f:
582 case Intrinsic::nvvm_rcp_rz_d:
583 case Intrinsic::nvvm_rcp_rz_ftz_f:
584 return APFloat::rmTowardZero;
585 }
586 llvm_unreachable("Checking rounding mode for invalid rcp intrinsic");
587}
588
589inline DenormalMode GetNVVMDenormMode(bool ShouldFTZ) {
590 if (ShouldFTZ)
591 return DenormalMode::getPreserveSign();
592 return DenormalMode::getIEEE();
593}
594
595inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
596 switch (IntrinsicID) {
597 case Intrinsic::nvvm_add_rm_ftz_f:
598 case Intrinsic::nvvm_add_rn_ftz_f:
599 case Intrinsic::nvvm_add_rp_ftz_f:
600 case Intrinsic::nvvm_add_rz_ftz_f:
601 return true;
602
603 case Intrinsic::nvvm_add_rm_f:
604 case Intrinsic::nvvm_add_rn_f:
605 case Intrinsic::nvvm_add_rp_f:
606 case Intrinsic::nvvm_add_rz_f:
607 case Intrinsic::nvvm_add_rm_d:
608 case Intrinsic::nvvm_add_rn_d:
609 case Intrinsic::nvvm_add_rp_d:
610 case Intrinsic::nvvm_add_rz_d:
611 return false;
612 }
613 llvm_unreachable("Checking FTZ flag for invalid NVVM add intrinsic");
614}
615
616inline APFloat::roundingMode GetFAddRoundingMode(Intrinsic::ID IntrinsicID) {
617 switch (IntrinsicID) {
618 case Intrinsic::nvvm_add_rm_f:
619 case Intrinsic::nvvm_add_rm_d:
620 case Intrinsic::nvvm_add_rm_ftz_f:
621 return APFloat::rmTowardNegative;
622 case Intrinsic::nvvm_add_rn_f:
623 case Intrinsic::nvvm_add_rn_d:
624 case Intrinsic::nvvm_add_rn_ftz_f:
625 return APFloat::rmNearestTiesToEven;
626 case Intrinsic::nvvm_add_rp_f:
627 case Intrinsic::nvvm_add_rp_d:
628 case Intrinsic::nvvm_add_rp_ftz_f:
629 return APFloat::rmTowardPositive;
630 case Intrinsic::nvvm_add_rz_f:
631 case Intrinsic::nvvm_add_rz_d:
632 case Intrinsic::nvvm_add_rz_ftz_f:
633 return APFloat::rmTowardZero;
634 }
635 llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM add");
636}
637
638inline bool FMulShouldFTZ(Intrinsic::ID IntrinsicID) {
639 switch (IntrinsicID) {
640 case Intrinsic::nvvm_mul_rm_ftz_f:
641 case Intrinsic::nvvm_mul_rn_ftz_f:
642 case Intrinsic::nvvm_mul_rp_ftz_f:
643 case Intrinsic::nvvm_mul_rz_ftz_f:
644 return true;
645
646 case Intrinsic::nvvm_mul_rm_f:
647 case Intrinsic::nvvm_mul_rn_f:
648 case Intrinsic::nvvm_mul_rp_f:
649 case Intrinsic::nvvm_mul_rz_f:
650 case Intrinsic::nvvm_mul_rm_d:
651 case Intrinsic::nvvm_mul_rn_d:
652 case Intrinsic::nvvm_mul_rp_d:
653 case Intrinsic::nvvm_mul_rz_d:
654 return false;
655 }
656 llvm_unreachable("Checking FTZ flag for invalid NVVM mul intrinsic");
657}
658
659inline APFloat::roundingMode GetFMulRoundingMode(Intrinsic::ID IntrinsicID) {
660 switch (IntrinsicID) {
661 case Intrinsic::nvvm_mul_rm_f:
662 case Intrinsic::nvvm_mul_rm_d:
663 case Intrinsic::nvvm_mul_rm_ftz_f:
664 return APFloat::rmTowardNegative;
665 case Intrinsic::nvvm_mul_rn_f:
666 case Intrinsic::nvvm_mul_rn_d:
667 case Intrinsic::nvvm_mul_rn_ftz_f:
668 return APFloat::rmNearestTiesToEven;
669 case Intrinsic::nvvm_mul_rp_f:
670 case Intrinsic::nvvm_mul_rp_d:
671 case Intrinsic::nvvm_mul_rp_ftz_f:
672 return APFloat::rmTowardPositive;
673 case Intrinsic::nvvm_mul_rz_f:
674 case Intrinsic::nvvm_mul_rz_d:
675 case Intrinsic::nvvm_mul_rz_ftz_f:
676 return APFloat::rmTowardZero;
677 }
678 llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM mul");
679}
680
681inline bool FDivShouldFTZ(Intrinsic::ID IntrinsicID) {
682 switch (IntrinsicID) {
683 case Intrinsic::nvvm_div_rm_ftz_f:
684 case Intrinsic::nvvm_div_rn_ftz_f:
685 case Intrinsic::nvvm_div_rp_ftz_f:
686 case Intrinsic::nvvm_div_rz_ftz_f:
687 return true;
688
689 case Intrinsic::nvvm_div_rm_f:
690 case Intrinsic::nvvm_div_rn_f:
691 case Intrinsic::nvvm_div_rp_f:
692 case Intrinsic::nvvm_div_rz_f:
693 case Intrinsic::nvvm_div_rm_d:
694 case Intrinsic::nvvm_div_rn_d:
695 case Intrinsic::nvvm_div_rp_d:
696 case Intrinsic::nvvm_div_rz_d:
697 return false;
698 }
699 llvm_unreachable("Checking FTZ flag for invalid NVVM div intrinsic");
700}
701
702inline APFloat::roundingMode GetFDivRoundingMode(Intrinsic::ID IntrinsicID) {
703 switch (IntrinsicID) {
704 case Intrinsic::nvvm_div_rm_f:
705 case Intrinsic::nvvm_div_rm_d:
706 case Intrinsic::nvvm_div_rm_ftz_f:
707 return APFloat::rmTowardNegative;
708 case Intrinsic::nvvm_div_rn_f:
709 case Intrinsic::nvvm_div_rn_d:
710 case Intrinsic::nvvm_div_rn_ftz_f:
711 return APFloat::rmNearestTiesToEven;
712 case Intrinsic::nvvm_div_rp_f:
713 case Intrinsic::nvvm_div_rp_d:
714 case Intrinsic::nvvm_div_rp_ftz_f:
715 return APFloat::rmTowardPositive;
716 case Intrinsic::nvvm_div_rz_f:
717 case Intrinsic::nvvm_div_rz_d:
718 case Intrinsic::nvvm_div_rz_ftz_f:
719 return APFloat::rmTowardZero;
720 }
721 llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM div");
722}
723
724inline bool FMAShouldFTZ(Intrinsic::ID IntrinsicID) {
725 switch (IntrinsicID) {
726 case Intrinsic::nvvm_fma_rm_ftz_f:
727 case Intrinsic::nvvm_fma_rn_ftz_f:
728 case Intrinsic::nvvm_fma_rp_ftz_f:
729 case Intrinsic::nvvm_fma_rz_ftz_f:
730 return true;
731
732 case Intrinsic::nvvm_fma_rm_f:
733 case Intrinsic::nvvm_fma_rn_f:
734 case Intrinsic::nvvm_fma_rp_f:
735 case Intrinsic::nvvm_fma_rz_f:
736 case Intrinsic::nvvm_fma_rm_d:
737 case Intrinsic::nvvm_fma_rn_d:
738 case Intrinsic::nvvm_fma_rp_d:
739 case Intrinsic::nvvm_fma_rz_d:
740 return false;
741 }
742 llvm_unreachable("Checking FTZ flag for invalid NVVM fma intrinsic");
743}
744
745inline APFloat::roundingMode GetFMARoundingMode(Intrinsic::ID IntrinsicID) {
746 switch (IntrinsicID) {
747 case Intrinsic::nvvm_fma_rm_f:
748 case Intrinsic::nvvm_fma_rm_d:
749 case Intrinsic::nvvm_fma_rm_ftz_f:
750 return APFloat::rmTowardNegative;
751 case Intrinsic::nvvm_fma_rn_f:
752 case Intrinsic::nvvm_fma_rn_d:
753 case Intrinsic::nvvm_fma_rn_ftz_f:
754 return APFloat::rmNearestTiesToEven;
755 case Intrinsic::nvvm_fma_rp_f:
756 case Intrinsic::nvvm_fma_rp_d:
757 case Intrinsic::nvvm_fma_rp_ftz_f:
758 return APFloat::rmTowardPositive;
759 case Intrinsic::nvvm_fma_rz_f:
760 case Intrinsic::nvvm_fma_rz_d:
761 case Intrinsic::nvvm_fma_rz_ftz_f:
762 return APFloat::rmTowardZero;
763 }
764 llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM fma");
765}
766
767} // namespace nvvm
768} // namespace llvm
769#endif // LLVM_IR_NVVMINTRINSICUTILS_H
APFloat.h
This file declares a class to represent arbitrary precision floating point values and provide a varie...
APInt.h
This file implements a class to represent arbitrary precision integral constant values and operations...
LLVM_ABI
#define LLVM_ABI
Definition Compiler.h:215
Constants.h
This file contains the declarations for the subclasses of Constant, which represent the different fla...
llvm::APFloatBase::rmTowardZero
static constexpr roundingMode rmTowardZero
Definition APFloat.h:357
llvm::APFloatBase::roundingMode
llvm::RoundingMode roundingMode
IEEE-754R 4.3: Rounding-direction attributes.
Definition APFloat.h:351
llvm::APFloatBase::rmTowardNegative
static constexpr roundingMode rmTowardNegative
Definition APFloat.h:356
llvm::APFloatBase::rmNearestTiesToEven
static constexpr roundingMode rmNearestTiesToEven
Definition APFloat.h:353
llvm::APFloatBase::rmTowardPositive
static constexpr roundingMode rmTowardPositive
Definition APFloat.h:355
llvm::Constant
This is an important base class in LLVM.
Definition Constant.h:43
llvm::StringRef
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
llvm::raw_ostream
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
uint8_t
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164
llvm::nvvm::GetFMARoundingMode
APFloat::roundingMode GetFMARoundingMode(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:745
llvm::nvvm::printTcgen05MMACollectorBBuffer
LLVM_ABI void printTcgen05MMACollectorBBuffer(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:73
llvm::nvvm::printTensormapSwizzleMode
LLVM_ABI void printTensormapSwizzleMode(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:126
llvm::nvvm::GetNVVMDenormMode
DenormalMode GetNVVMDenormMode(bool ShouldFTZ)
Definition NVVMIntrinsicUtils.h:589
llvm::nvvm::printTMAReductionOp
LLVM_ABI void printTMAReductionOp(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:19
llvm::nvvm::FPToIntegerIntrinsicNaNZero
bool FPToIntegerIntrinsicNaNZero(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:297
llvm::nvvm::printTensormapInterleaveLayout
LLVM_ABI void printTensormapInterleaveLayout(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:108
llvm::nvvm::GetFDivRoundingMode
APFloat::roundingMode GetFDivRoundingMode(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:702
llvm::nvvm::printTensormapSwizzleAtomicity
LLVM_ABI void printTensormapSwizzleAtomicity(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:140
llvm::nvvm::FPToIntegerIntrinsicResultIsSigned
bool FPToIntegerIntrinsicResultIsSigned(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:231
llvm::nvvm::GetFPToIntegerRoundingMode
APFloat::roundingMode GetFPToIntegerRoundingMode(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:362
llvm::nvvm::RCPShouldFTZ
bool RCPShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:544
llvm::nvvm::getTMATensorReductionOpName
StringRef getTMATensorReductionOpName(TMAReductionOp Op)
Definition NVVMIntrinsicUtils.h:45
llvm::nvvm::FPToIntegerIntrinsicShouldFTZ
bool FPToIntegerIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:164
llvm::nvvm::FDivShouldFTZ
bool FDivShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:681
llvm::nvvm::FAddShouldFTZ
bool FAddShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:595
llvm::nvvm::FMinFMaxIsXorSignAbs
bool FMinFMaxIsXorSignAbs(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:489
llvm::nvvm::GetFMulRoundingMode
APFloat::roundingMode GetFMulRoundingMode(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:659
llvm::nvvm::printTcgen05MMAKind
LLVM_ABI void printTcgen05MMAKind(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:29
llvm::nvvm::printTcgen05CollectorUsageOp
LLVM_ABI void printTcgen05CollectorUsageOp(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:52
llvm::nvvm::UnaryMathIntrinsicShouldFTZ
bool UnaryMathIntrinsicShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:518
llvm::nvvm::FMinFMaxShouldFTZ
bool FMinFMaxShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:431
llvm::nvvm::GetFAddRoundingMode
APFloat::roundingMode GetFAddRoundingMode(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:616
llvm::nvvm::FMAShouldFTZ
bool FMAShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:724
llvm::nvvm::FMulShouldFTZ
bool FMulShouldFTZ(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:638
llvm::nvvm::printTensormapFillMode
LLVM_ABI void printTensormapFillMode(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:154
llvm::nvvm::GetRCPRoundingMode
APFloat::roundingMode GetRCPRoundingMode(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:564
llvm::nvvm::FMinFMaxPropagatesNaNs
bool FMinFMaxPropagatesNaNs(Intrinsic::ID IntrinsicID)
Definition NVVMIntrinsicUtils.h:460
llvm::nvvm::printTensormapElemType
LLVM_ABI void printTensormapElemType(raw_ostream &OS, const Constant *ImmArgVal)
Definition NVVMIntrinsicUtils.cpp:94
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition FunctionInfo.h:25
llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:25
raw_ostream.h
llvm::DenormalMode
Represent subnormal handling kind for floating point instruction inputs and outputs.
Definition FloatingPointMode.h:71
llvm::DenormalMode::getPreserveSign
static constexpr DenormalMode getPreserveSign()
Definition FloatingPointMode.h:119
llvm::DenormalMode::getIEEE
static constexpr DenormalMode getIEEE()
Definition FloatingPointMode.h:115
Generated on for LLVM by doxygen 1.14.0