Grcov report - args.rs

1

//! Instruction operand sub-components (aka "parts"): definitions and printing.

2

3

use super::regs::{self};

4

use super::EmitState;

5

use crate::ir::condcodes::{FloatCC, IntCC};

6

use crate::ir::{MemFlags, Type};

7

use crate::isa::x64::inst::regs::pretty_print_reg;

8

use crate::isa::x64::inst::Inst;

9

use crate::machinst::*;

10

use regalloc2::VReg;

11

use smallvec::{smallvec, SmallVec};

12

use std::fmt;

13

use std::string::String;

14

15

pub use crate::isa::x64::lower::isle::generated_code::DivSignedness;

16

17

/// An extenstion trait for converting `Writable{Xmm,Gpr}` to `Writable<Reg>`.

18

pub trait ToWritableReg {

19

    /// Convert `Writable{Xmm,Gpr}` to `Writable<Reg>`.

20

    fn to_writable_reg(&self) -> Writable<Reg>;

21

22

23

/// An extension trait for converting `Writable<Reg>` to `Writable{Xmm,Gpr}`.

24

pub trait FromWritableReg: Sized {

25

    /// Convert `Writable<Reg>` to `Writable{Xmm,Gpr}`.

26

    fn from_writable_reg(w: Writable<Reg>) -> Option<Self>;

27

28

29

/// A macro for defining a newtype of `Reg` that enforces some invariant about

30

/// the wrapped `Reg` (such as that it is of a particular register class).

31

macro_rules! newtype_of_reg {

32

33

        $newtype_reg:ident,

34

        $newtype_writable_reg:ident,

35

        $newtype_option_writable_reg:ident,

36

        reg_mem: ($($newtype_reg_mem:ident $(aligned:$aligned:ident)?),*),

37

        reg_mem_imm: ($($newtype_reg_mem_imm:ident $(aligned:$aligned_imm:ident)?),*),

38

        $newtype_imm8_reg:ident,

39

        |$check_reg:ident| $check:expr

40

    ) => {

41

        /// A newtype wrapper around `Reg`.

42

        #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]

43

        pub struct $newtype_reg(Reg);

44

45

        impl PartialEq<Reg> for $newtype_reg {

46

            fn eq(&self, other: &Reg) -> bool {

47

                self.0 == *other

48

49

50

51

        impl From<$newtype_reg> for Reg {

52

            fn from(r: $newtype_reg) -> Self {

53

r.0

54

55

56

57

        impl $newtype_reg {

58

            /// Create this newtype from the given register, or return `None` if the register

59

            /// is not a valid instance of this newtype.

60

            pub fn new($check_reg: Reg) -> Option<Self> {

61

                if $check {

62

                    Some(Self($check_reg))

63

                } else {

64

                    None

65

66

67

68

            /// Get this newtype's underlying `Reg`.

69

            pub fn to_reg(self) -> Reg {

70

                self.0

71

72

73

74

        // Convenience impl so that people working with this newtype can use it

75

        // "just like" a plain `Reg`.

76

//

77

        // NB: We cannot implement `DerefMut` because that would let people do

78

        // nasty stuff like `*my_gpr.deref_mut() = some_xmm_reg`, breaking the

79

        // invariants that `Gpr` provides.

80

        impl std::ops::Deref for $newtype_reg {

81

            type Target = Reg;

82

83

            fn deref(&self) -> &Reg {

84

                &self.0

85

86

87

88

        /// Writable Gpr.

89

        pub type $newtype_writable_reg = Writable<$newtype_reg>;

90

91

        #[allow(dead_code)] // Used by some newtypes and not others.

92

        /// Optional writable Gpr.

93

        pub type $newtype_option_writable_reg = Option<Writable<$newtype_reg>>;

94

95

        impl ToWritableReg for $newtype_writable_reg {

96

            fn to_writable_reg(&self) -> Writable<Reg> {

97

                Writable::from_reg(self.to_reg().to_reg())

98

99

100

101

        impl FromWritableReg for $newtype_writable_reg {

102

            fn from_writable_reg(w: Writable<Reg>) -> Option<Self> {

103

                Some(Writable::from_reg($newtype_reg::new(w.to_reg())?))

104

105

106

107

$(

108

            /// A newtype wrapper around `RegMem` for general-purpose registers.

109

            #[derive(Clone, Debug)]

110

            pub struct $newtype_reg_mem(RegMem);

111

112

            impl From<$newtype_reg_mem> for RegMem {

113

                fn from(rm: $newtype_reg_mem) -> Self {

114

                    rm.0

115

116

117

118

            impl From<$newtype_reg> for $newtype_reg_mem {

119

                fn from(r: $newtype_reg) -> Self {

120

                    $newtype_reg_mem(RegMem::reg(r.into()))

121

122

123

124

            impl $newtype_reg_mem {

125

                /// Construct a `RegMem` newtype from the given `RegMem`, or return

126

                /// `None` if the `RegMem` is not a valid instance of this `RegMem`

127

                /// newtype.

128

                pub fn new(rm: RegMem) -> Option<Self> {

129

                    match rm {

130

                        RegMem::Mem { addr } => {

131

                            let mut _allow = true;

132

$(

133

                                if $aligned {

134

                                    _allow = addr.aligned();

135

136

)?

137

                            if _allow {

138

                                Some(Self(RegMem::Mem { addr }))

139

                            } else {

140

                                None

141

142

143

                        RegMem::Reg { reg: $check_reg } if $check => Some(Self(rm)),

144

                        RegMem::Reg { reg: _ } => None,

145

146

147

148

                /// Convert this newtype into its underlying `RegMem`.

149

                pub fn to_reg_mem(self) -> RegMem {

150

                    self.0

151

152

153

                #[allow(dead_code)] // Used by some newtypes and not others.

154

                pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(

155

                    &self,

156

                    collector: &mut OperandCollector<'_, F>,

157

) {

158

                    self.0.get_operands(collector);

159

160

161

            impl PrettyPrint for $newtype_reg_mem {

162

                fn pretty_print(&self, size: u8, allocs: &mut AllocationConsumer<'_>) -> String {

163

                    self.0.pretty_print(size, allocs)

164

165

166

)*

167

168

$(

169

            /// A newtype wrapper around `RegMemImm`.

170

            #[derive(Clone, Debug)]

171

            pub struct $newtype_reg_mem_imm(RegMemImm);

172

173

            impl From<$newtype_reg_mem_imm> for RegMemImm {

174

                fn from(rmi: $newtype_reg_mem_imm) -> RegMemImm {

175

                    rmi.0

176

177

178

179

            impl From<$newtype_reg> for $newtype_reg_mem_imm {

180

                fn from(r: $newtype_reg) -> Self {

181

                    $newtype_reg_mem_imm(RegMemImm::reg(r.into()))

182

183

184

185

            impl $newtype_reg_mem_imm {

186

                /// Construct this newtype from the given `RegMemImm`, or return

187

                /// `None` if the `RegMemImm` is not a valid instance of this

188

                /// newtype.

189

                pub fn new(rmi: RegMemImm) -> Option<Self> {

190

                    match rmi {

191

                        RegMemImm::Imm { .. } => Some(Self(rmi)),

192

                        RegMemImm::Mem { addr } => {

193

                            let mut _allow = true;

194

$(

195

                                if $aligned_imm {

196

                                    _allow = addr.aligned();

197

198

)?

199

                            if _allow {

200

                                Some(Self(RegMemImm::Mem { addr }))

201

                            } else {

202

                                None

203

204

205

                        RegMemImm::Reg { reg: $check_reg } if $check => Some(Self(rmi)),

206

                        RegMemImm::Reg { reg: _ } => None,

207

208

209

210

                /// Convert this newtype into its underlying `RegMemImm`.

211

                #[allow(dead_code)] // Used by some newtypes and not others.

212

                pub fn to_reg_mem_imm(self) -> RegMemImm {

213

                    self.0

214

215

216

                #[allow(dead_code)] // Used by some newtypes and not others.

217

                pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(

218

                    &self,

219

                    collector: &mut OperandCollector<'_, F>,

220

) {

221

                    self.0.get_operands(collector);

222

223

224

225

            impl PrettyPrint for $newtype_reg_mem_imm {

226

                fn pretty_print(&self, size: u8, allocs: &mut AllocationConsumer<'_>) -> String {

227

                    self.0.pretty_print(size, allocs)

228

229

230

)*

231

232

        /// A newtype wrapper around `Imm8Reg`.

233

        #[derive(Clone, Debug)]

234

        #[allow(dead_code)] // Used by some newtypes and not others.

235

        pub struct $newtype_imm8_reg(Imm8Reg);

236

237

        impl From<$newtype_reg> for $newtype_imm8_reg {

238

            fn from(r: $newtype_reg) -> Self {

239

                Self(Imm8Reg::Reg { reg: r.to_reg() })

240

241

242

243

        impl $newtype_imm8_reg {

244

            /// Construct this newtype from the given `Imm8Reg`, or return

245

            /// `None` if the `Imm8Reg` is not a valid instance of this newtype.

246

            #[allow(dead_code)] // Used by some newtypes and not others.

247

            pub fn new(imm8_reg: Imm8Reg) -> Option<Self> {

248

                match imm8_reg {

249

                    Imm8Reg::Imm8 { .. } => Some(Self(imm8_reg)),

250

                    Imm8Reg::Reg { reg: $check_reg } if $check => Some(Self(imm8_reg)),

251

                    Imm8Reg::Reg { reg: _ } => None,

252

253

254

255

            /// Convert this newtype into its underlying `Imm8Reg`.

256

            #[allow(dead_code)] // Used by some newtypes and not others.

257

            pub fn to_imm8_reg(self) -> Imm8Reg {

258

                self.0

259

260

261

};

262

263

264

// Define a newtype of `Reg` for general-purpose registers.

265

newtype_of_reg!(

266

    Gpr,

267

    WritableGpr,

268

    OptionWritableGpr,

269

    reg_mem: (GprMem),

270

    reg_mem_imm: (GprMemImm),

271

    Imm8Gpr,

272

    |reg| reg.class() == RegClass::Int

273

);

274

275

// Define a newtype of `Reg` for XMM registers.

276

newtype_of_reg!(

277

    Xmm,

278

    WritableXmm,

279

    OptionWritableXmm,

280

    reg_mem: (XmmMem, XmmMemAligned aligned:true),

281

    reg_mem_imm: (XmmMemImm, XmmMemAlignedImm aligned:true),

282

    Imm8Xmm,

283

    |reg| reg.class() == RegClass::Float

284

);

285

286

// N.B.: `Amode` is defined in `inst.isle`. We add some convenience

287

// constructors here.

288

289

// Re-export the type from the ISLE generated code.

290

pub use crate::isa::x64::lower::isle::generated_code::Amode;

291

292

impl Amode {

293

    /// Create an immediate sign-extended and register addressing mode.

294

    pub fn imm_reg(simm32: u32, base: Reg) -> Self {

295

        debug_assert!(base.class() == RegClass::Int);

296

        Self::ImmReg {

297

            simm32,

298

            base,

299

            flags: MemFlags::trusted(),

300

301

302

303

    /// Create a sign-extended-32-to-64 with register and shift addressing mode.

304

    pub fn imm_reg_reg_shift(simm32: u32, base: Gpr, index: Gpr, shift: u8) -> Self {

305

        debug_assert!(base.class() == RegClass::Int);

306

        debug_assert!(index.class() == RegClass::Int);

307

        debug_assert!(shift <= 3);

308

        Self::ImmRegRegShift {

309

            simm32,

310

            base,

311

            index,

312

            shift,

313

            flags: MemFlags::trusted(),

314

315

316

317

    pub(crate) fn rip_relative(target: MachLabel) -> Self {

318

        Self::RipRelative { target }

319

320

321

    pub(crate) fn with_flags(&self, flags: MemFlags) -> Self {

322

        match self {

323

            &Self::ImmReg { simm32, base, .. } => Self::ImmReg {

324

                simm32,

325

                base,

326

                flags,

327

},

328

            &Self::ImmRegRegShift {

329

                simm32,

330

                base,

331

                index,

332

                shift,

333

..

334

            } => Self::ImmRegRegShift {

335

                simm32,

336

                base,

337

                index,

338

                shift,

339

                flags,

340

},

341

            _ => panic!("Amode {:?} cannot take memflags", self),

342

343

344

345

    /// Add the registers mentioned by `self` to `collector`.

346

    pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(

347

        &self,

348

        collector: &mut OperandCollector<'_, F>,

349

) {

350

        match self {

351

            Amode::ImmReg { base, .. } => {

352

                if *base != regs::rbp() && *base != regs::rsp() {

353

                    collector.reg_use(*base);

354

355

356

            Amode::ImmRegRegShift { base, index, .. } => {

357

                debug_assert_ne!(base.to_reg(), regs::rbp());

358

                debug_assert_ne!(base.to_reg(), regs::rsp());

359

                collector.reg_use(base.to_reg());

360

                debug_assert_ne!(index.to_reg(), regs::rbp());

361

                debug_assert_ne!(index.to_reg(), regs::rsp());

362

                collector.reg_use(index.to_reg());

363

364

            Amode::RipRelative { .. } => {

365

                // RIP isn't involved in regalloc.

366

367

368

369

370

    /// Same as `get_operands`, but add the registers in the "late" phase.

371

    pub(crate) fn get_operands_late<F: Fn(VReg) -> VReg>(

372

        &self,

373

        collector: &mut OperandCollector<'_, F>,

374

) {

375

        match self {

376

            Amode::ImmReg { base, .. } => {

377

                collector.reg_late_use(*base);

378

379

            Amode::ImmRegRegShift { base, index, .. } => {

380

                collector.reg_late_use(base.to_reg());

381

                collector.reg_late_use(index.to_reg());

382

383

            Amode::RipRelative { .. } => {

384

                // RIP isn't involved in regalloc.

385

386

387

388

389

    pub(crate) fn get_flags(&self) -> MemFlags {

390

        match self {

391

            Amode::ImmReg { flags, .. } | Amode::ImmRegRegShift { flags, .. } => *flags,

392

            Amode::RipRelative { .. } => MemFlags::trusted(),

393

394

395

396

    pub(crate) fn can_trap(&self) -> bool {

397

        !self.get_flags().notrap()

398

399

400

    pub(crate) fn with_allocs(&self, allocs: &mut AllocationConsumer<'_>) -> Self {

401

        // The order in which we consume allocs here must match the

402

        // order in which we produce operands in get_operands() above.

403

        match self {

404

            &Amode::ImmReg {

405

                simm32,

406

                base,

407

                flags,

408

            } => {

409

                let base = if base == regs::rsp() || base == regs::rbp() {

410

                    base

411

                } else {

412

                    allocs.next(base)

413

};

414

                Amode::ImmReg {

415

                    simm32,

416

                    flags,

417

                    base,

418

419

420

            &Amode::ImmRegRegShift {

421

                simm32,

422

                base,

423

                index,

424

                shift,

425

                flags,

426

            } => Amode::ImmRegRegShift {

427

                simm32,

428

                shift,

429

                flags,

430

                base: Gpr::new(allocs.next(*base)).unwrap(),

431

                index: Gpr::new(allocs.next(*index)).unwrap(),

432

},

433

            &Amode::RipRelative { target } => Amode::RipRelative { target },

434

435

436

437

    /// Offset the amode by a fixed offset.

438

    pub(crate) fn offset(&self, offset: u32) -> Self {

439

        let mut ret = self.clone();

440

        match &mut ret {

441

            &mut Amode::ImmReg { ref mut simm32, .. } => *simm32 += offset,

442

            &mut Amode::ImmRegRegShift { ref mut simm32, .. } => *simm32 += offset,

443

            _ => panic!("Cannot offset amode: {:?}", self),

444

445

ret

446

447

448

    pub(crate) fn aligned(&self) -> bool {

449

        self.get_flags().aligned()

450

451

452

453

impl PrettyPrint for Amode {

454

    fn pretty_print(&self, _size: u8, allocs: &mut AllocationConsumer<'_>) -> String {

455

        match self {

456

            Amode::ImmReg { simm32, base, .. } => {

457

                // Note: size is always 8; the address is 64 bits,

458

                // even if the addressed operand is smaller.

459

                format!("{}({})", *simm32 as i32, pretty_print_reg(*base, 8, allocs))

460

461

            Amode::ImmRegRegShift {

462

                simm32,

463

                base,

464

                index,

465

                shift,

466

..

467

            } => format!(

468

                "{}({},{},{})",

469

                *simm32 as i32,

470

                pretty_print_reg(base.to_reg(), 8, allocs),

471

                pretty_print_reg(index.to_reg(), 8, allocs),

472

                1 << shift

473

),

474

            Amode::RipRelative { ref target } => format!("label{}(%rip)", target.get()),

475

476

477

478

479

/// A Memory Address. These denote a 64-bit value only.

480

/// Used for usual addressing modes as well as addressing modes used during compilation, when the

481

/// moving SP offset is not known.

482

#[derive(Clone, Debug)]

483

pub enum SyntheticAmode {

484

    /// A real amode.

485

    Real(Amode),

486

487

    /// A (virtual) offset to the "nominal SP" value, which will be recomputed as we push and pop

488

    /// within the function.

489

    NominalSPOffset {

490

        /// The nominal stack pointer value.

491

        simm32: u32,

492

},

493

494

    /// A virtual offset to a constant that will be emitted in the constant section of the buffer.

495

    ConstantOffset(VCodeConstant),

496

497

498

impl SyntheticAmode {

499

    /// Create a real addressing mode.

500

    pub fn real(amode: Amode) -> Self {

501

        Self::Real(amode)

502

503

504

    pub(crate) fn nominal_sp_offset(simm32: u32) -> Self {

505

        SyntheticAmode::NominalSPOffset { simm32 }

506

507

508

    /// Add the registers mentioned by `self` to `collector`.

509

    pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(

510

        &self,

511

        collector: &mut OperandCollector<'_, F>,

512

) {

513

        match self {

514

            SyntheticAmode::Real(addr) => addr.get_operands(collector),

515

            SyntheticAmode::NominalSPOffset { .. } => {

516

                // Nothing to do; the base is SP and isn't involved in regalloc.

517

518

            SyntheticAmode::ConstantOffset(_) => {}

519

520

521

522

    /// Same as `get_operands`, but add the register in the "late" phase.

523

    pub(crate) fn get_operands_late<F: Fn(VReg) -> VReg>(

524

        &self,

525

        collector: &mut OperandCollector<'_, F>,

526

) {

527

        match self {

528

            SyntheticAmode::Real(addr) => addr.get_operands_late(collector),

529

            SyntheticAmode::NominalSPOffset { .. } => {

530

                // Nothing to do; the base is SP and isn't involved in regalloc.

531

532

            SyntheticAmode::ConstantOffset(_) => {}

533

534

535

536

    pub(crate) fn finalize(&self, state: &mut EmitState, buffer: &MachBuffer<Inst>) -> Amode {

537

        match self {

538

            SyntheticAmode::Real(addr) => addr.clone(),

539

            SyntheticAmode::NominalSPOffset { simm32 } => {

540

                let off = *simm32 as i64 + state.virtual_sp_offset;

541

                // TODO will require a sequence of add etc.

542

                assert!(

543

                    off <= u32::max_value() as i64,

544

                    "amode finalize: add sequence NYI"

545

);

546

                Amode::imm_reg(off as u32, regs::rsp())

547

548

            SyntheticAmode::ConstantOffset(c) => {

549

                Amode::rip_relative(buffer.get_label_for_constant(*c))

550

551

552

553

554

    pub(crate) fn with_allocs(&self, allocs: &mut AllocationConsumer<'_>) -> Self {

555

        match self {

556

            SyntheticAmode::Real(addr) => SyntheticAmode::Real(addr.with_allocs(allocs)),

557

            &SyntheticAmode::NominalSPOffset { .. } | &SyntheticAmode::ConstantOffset { .. } => {

558

                self.clone()

559

560

561

562

563

    pub(crate) fn aligned(&self) -> bool {

564

        match self {

565

            SyntheticAmode::Real(addr) => addr.aligned(),

566

            SyntheticAmode::NominalSPOffset { .. } | SyntheticAmode::ConstantOffset { .. } => true,

567

568

569

570

571

impl Into<SyntheticAmode> for Amode {

572

    fn into(self) -> SyntheticAmode {

573

        SyntheticAmode::Real(self)

574

575

576

577

impl Into<SyntheticAmode> for VCodeConstant {

578

    fn into(self) -> SyntheticAmode {

579

        SyntheticAmode::ConstantOffset(self)

580

581

582

583

impl PrettyPrint for SyntheticAmode {

584

    fn pretty_print(&self, _size: u8, allocs: &mut AllocationConsumer<'_>) -> String {

585

        match self {

586

            // See note in `Amode` regarding constant size of `8`.

587

            SyntheticAmode::Real(addr) => addr.pretty_print(8, allocs),

588

            SyntheticAmode::NominalSPOffset { simm32 } => {

589

                format!("rsp({} + virtual offset)", *simm32 as i32)

590

591

            SyntheticAmode::ConstantOffset(c) => format!("const({})", c.as_u32()),

592

593

594

595

596

/// An operand which is either an integer Register, a value in Memory or an Immediate.  This can

597

/// denote an 8, 16, 32 or 64 bit value.  For the Immediate form, in the 8- and 16-bit case, only

598

/// the lower 8 or 16 bits of `simm32` is relevant.  In the 64-bit case, the value denoted by

599

/// `simm32` is its sign-extension out to 64 bits.

600

#[derive(Clone, Debug)]

601

pub enum RegMemImm {

602

    /// A register operand.

603

    Reg {

604

        /// The underlying register.

605

        reg: Reg,

606

},

607

    /// A memory operand.

608

    Mem {

609

        /// The memory address.

610

        addr: SyntheticAmode,

611

},

612

    /// An immediate operand.

613

    Imm {

614

        /// The immediate value.

615

        simm32: u32,

616

},

617

618

619

impl RegMemImm {

620

    /// Create a register operand.

621

    pub fn reg(reg: Reg) -> Self {

622

        debug_assert!(reg.class() == RegClass::Int || reg.class() == RegClass::Float);

623

        Self::Reg { reg }

624

625

626

    /// Create a memory operand.

627

    pub fn mem(addr: impl Into<SyntheticAmode>) -> Self {

628

        Self::Mem { addr: addr.into() }

629

630

631

    /// Create an immediate operand.

632

    pub fn imm(simm32: u32) -> Self {

633

        Self::Imm { simm32 }

634

635

636

    /// Asserts that in register mode, the reg class is the one that's expected.

637

    pub(crate) fn assert_regclass_is(&self, expected_reg_class: RegClass) {

638

        if let Self::Reg { reg } = self {

639

            debug_assert_eq!(reg.class(), expected_reg_class);

640

641

642

643

    /// Add the regs mentioned by `self` to `collector`.

644

    pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(

645

        &self,

646

        collector: &mut OperandCollector<'_, F>,

647

) {

648

        match self {

649

            Self::Reg { reg } => collector.reg_use(*reg),

650

            Self::Mem { addr } => addr.get_operands(collector),

651

            Self::Imm { .. } => {}

652

653

654

655

    pub(crate) fn with_allocs(&self, allocs: &mut AllocationConsumer<'_>) -> Self {

656

        match self {

657

            Self::Reg { reg } => Self::Reg {

658

                reg: allocs.next(*reg),

659

},

660

            Self::Mem { addr } => Self::Mem {

661

                addr: addr.with_allocs(allocs),

662

},

663

            Self::Imm { .. } => self.clone(),

664

665

666

667

668

impl From<RegMem> for RegMemImm {

669

    fn from(rm: RegMem) -> RegMemImm {

670

        match rm {

671

            RegMem::Reg { reg } => RegMemImm::Reg { reg },

672

            RegMem::Mem { addr } => RegMemImm::Mem { addr },

673

674

675

676

677

impl PrettyPrint for RegMemImm {

678

    fn pretty_print(&self, size: u8, allocs: &mut AllocationConsumer<'_>) -> String {

679

        match self {

680

            Self::Reg { reg } => pretty_print_reg(*reg, size, allocs),

681

            Self::Mem { addr } => addr.pretty_print(size, allocs),

682

            Self::Imm { simm32 } => format!("${}", *simm32 as i32),

683

684

685

686

687

/// An operand which is either an 8-bit integer immediate or a register.

688

#[derive(Clone, Debug)]

689

pub enum Imm8Reg {

690

    /// 8-bit immediate operand.

691

    Imm8 {

692

        /// The 8-bit immediate value.

693

        imm: u8,

694

},

695

    /// A register operand.

696

    Reg {

697

        /// The underlying register.

698

        reg: Reg,

699

},

700

701

702

impl From<u8> for Imm8Reg {

703

    fn from(imm: u8) -> Self {

704

        Self::Imm8 { imm }

705

706

707

708

impl From<Reg> for Imm8Reg {

709

    fn from(reg: Reg) -> Self {

710

        Self::Reg { reg }

711

712

713

714

/// An operand which is either an integer Register or a value in Memory.  This can denote an 8, 16,

715

/// 32, 64, or 128 bit value.

716

#[derive(Clone, Debug)]

717

pub enum RegMem {

718

    /// A register operand.

719

    Reg {

720

        /// The underlying register.

721

        reg: Reg,

722

},

723

    /// A memory operand.

724

    Mem {

725

        /// The memory address.

726

        addr: SyntheticAmode,

727

},

728

729

730

impl RegMem {

731

    /// Create a register operand.

732

    pub fn reg(reg: Reg) -> Self {

733

        debug_assert!(reg.class() == RegClass::Int || reg.class() == RegClass::Float);

734

        Self::Reg { reg }

735

736

737

    /// Create a memory operand.

738

    pub fn mem(addr: impl Into<SyntheticAmode>) -> Self {

739

        Self::Mem { addr: addr.into() }

740

741

    /// Asserts that in register mode, the reg class is the one that's expected.

742

    pub(crate) fn assert_regclass_is(&self, expected_reg_class: RegClass) {

743

        if let Self::Reg { reg } = self {

744

            debug_assert_eq!(reg.class(), expected_reg_class);

745

746

747

    /// Add the regs mentioned by `self` to `collector`.

748

    pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(

749

        &self,

750

        collector: &mut OperandCollector<'_, F>,

751

) {

752

        match self {

753

            RegMem::Reg { reg } => collector.reg_use(*reg),

754

            RegMem::Mem { addr, .. } => addr.get_operands(collector),

755

756

757

758

    pub(crate) fn with_allocs(&self, allocs: &mut AllocationConsumer<'_>) -> Self {

759

        match self {

760

            RegMem::Reg { reg } => RegMem::Reg {

761

                reg: allocs.next(*reg),

762

},

763

            RegMem::Mem { addr } => RegMem::Mem {

764

                addr: addr.with_allocs(allocs),

765

},

766

767

768

769

770

impl From<Reg> for RegMem {

771

    fn from(reg: Reg) -> RegMem {

772

        RegMem::Reg { reg }

773

774

775

776

impl From<Writable<Reg>> for RegMem {

777

    fn from(r: Writable<Reg>) -> Self {

778

        RegMem::reg(r.to_reg())

779

780

781

782

impl PrettyPrint for RegMem {

783

    fn pretty_print(&self, size: u8, allocs: &mut AllocationConsumer<'_>) -> String {

784

        match self {

785

            RegMem::Reg { reg } => pretty_print_reg(*reg, size, allocs),

786

            RegMem::Mem { addr, .. } => addr.pretty_print(size, allocs),

787

788

789

790

791

/// Some basic ALU operations.

792

#[derive(Copy, Clone, PartialEq)]

793

pub enum AluRmiROpcode {

794

    /// Add operation.

795

    Add,

796

    /// Add with carry.

797

    Adc,

798

    /// Integer subtraction.

799

    Sub,

800

    /// Integer subtraction with borrow.

801

    Sbb,

802

    /// Bitwise AND operation.

803

    And,

804

    /// Bitwise inclusive OR.

805

Or,

806

    /// Bitwise exclusive OR.

807

    Xor,

808

    /// The signless, non-extending (N x N -> N, for N in {32,64}) variant.

809

    Mul,

810

811

812

impl fmt::Debug for AluRmiROpcode {

813

    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {

814

        let name = match self {

815

            AluRmiROpcode::Add => "add",

816

            AluRmiROpcode::Adc => "adc",

817

            AluRmiROpcode::Sub => "sub",

818

            AluRmiROpcode::Sbb => "sbb",

819

            AluRmiROpcode::And => "and",

820

            AluRmiROpcode::Or => "or",

821

            AluRmiROpcode::Xor => "xor",

822

            AluRmiROpcode::Mul => "imul",

823

};

824

        write!(fmt, "{}", name)

825

826

827

828

impl fmt::Display for AluRmiROpcode {

829

    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {

830

        fmt::Debug::fmt(self, f)

831

832

833

834

/// ALU operations that don't accept intermediates.

835

#[derive(Copy, Clone, PartialEq)]

836

pub enum AluRmROpcode {

837

    /// And with negated second operand.

838

    Andn,

839

840

841

impl AluRmROpcode {

842

    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {

843

        match self {

844

            AluRmROpcode::Andn => smallvec![InstructionSet::BMI1],

845

846

847

848

849

impl fmt::Debug for AluRmROpcode {

850

    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {

851

        let name = match self {

852

            AluRmROpcode::Andn => "andn",

853

};

854

        write!(fmt, "{}", name)

855

856

857

858

impl fmt::Display for AluRmROpcode {

859

    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {

860

        fmt::Debug::fmt(self, f)

861

862

863

864

#[derive(Clone, PartialEq)]

865

/// Unary operations requiring register or memory and register operands.

866

pub enum UnaryRmROpcode {

867

    /// Bit-scan reverse.

868

    Bsr,

869

    /// Bit-scan forward.

870

    Bsf,

871

    /// Counts leading zeroes (Leading Zero CouNT).

872

    Lzcnt,

873

    /// Counts trailing zeroes (Trailing Zero CouNT).

874

    Tzcnt,

875

    /// Counts the number of ones (POPulation CouNT).

876

    Popcnt,

877

878

879

impl UnaryRmROpcode {

880

    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {

881

        match self {

882

            UnaryRmROpcode::Bsr | UnaryRmROpcode::Bsf => smallvec![],

883

            UnaryRmROpcode::Lzcnt => smallvec![InstructionSet::Lzcnt],

884

            UnaryRmROpcode::Tzcnt => smallvec![InstructionSet::BMI1],

885

            UnaryRmROpcode::Popcnt => smallvec![InstructionSet::Popcnt],

886

887

888

889

890

impl fmt::Debug for UnaryRmROpcode {

891

    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {

892

        match self {

893

            UnaryRmROpcode::Bsr => write!(fmt, "bsr"),

894

            UnaryRmROpcode::Bsf => write!(fmt, "bsf"),

895

            UnaryRmROpcode::Lzcnt => write!(fmt, "lzcnt"),

896

            UnaryRmROpcode::Tzcnt => write!(fmt, "tzcnt"),

897

            UnaryRmROpcode::Popcnt => write!(fmt, "popcnt"),

898

899

900

901

902

impl fmt::Display for UnaryRmROpcode {

903

    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {

904

        fmt::Debug::fmt(self, f)

905

906

907

908

#[derive(Clone, Copy, PartialEq)]

909

/// Comparison operations.

910

pub enum CmpOpcode {

911

    /// CMP instruction: compute `a - b` and set flags from result.

912

    Cmp,

913

    /// TEST instruction: compute `a & b` and set flags from result.

914

    Test,

915

916

917

#[derive(Debug)]

918

pub(crate) enum InstructionSet {

919

    SSE,

920

    SSE2,

921

    SSSE3,

922

    SSE41,

923

    SSE42,

924

    Popcnt,

925

    Lzcnt,

926

    BMI1,

927

    #[allow(dead_code)] // never constructed (yet).

928

    BMI2,

929

    FMA,

930

    AVX,

931

    AVX2,

932

    AVX512BITALG,

933

    AVX512DQ,

934

    AVX512F,

935

    AVX512VBMI,

936

    AVX512VL,

937

938

939

/// Some SSE operations requiring 2 operands r/m and r.

940

#[derive(Clone, Copy, PartialEq)]

941

#[allow(dead_code)] // some variants here aren't used just yet

942

#[allow(missing_docs)]

943

pub enum SseOpcode {

944

    Addps,

945

    Addpd,

946

    Addss,

947

    Addsd,

948

    Andps,

949

    Andpd,

950

    Andnps,

951

    Andnpd,

952

    Blendvpd,

953

    Blendvps,

954

    Comiss,

955

    Comisd,

956

    Cmpps,

957

    Cmppd,

958

    Cmpss,

959

    Cmpsd,

960

    Cvtdq2ps,

961

    Cvtdq2pd,

962

    Cvtpd2ps,

963

    Cvtps2pd,

964

    Cvtsd2ss,

965

    Cvtsd2si,

966

    Cvtsi2ss,

967

    Cvtsi2sd,

968

    Cvtss2si,

969

    Cvtss2sd,

970

    Cvttpd2dq,

971

    Cvttps2dq,

972

    Cvttss2si,

973

    Cvttsd2si,

974

    Divps,

975

    Divpd,

976

    Divss,

977

    Divsd,

978

    Insertps,

979

    Maxps,

980

    Maxpd,

981

    Maxss,

982

    Maxsd,

983

    Minps,

984

    Minpd,

985

    Minss,

986

    Minsd,

987

    Movaps,

988

    Movapd,

989

    Movd,

990

    Movdqa,

991

    Movdqu,

992

    Movlhps,

993

    Movmskps,

994

    Movmskpd,

995

    Movq,

996

    Movss,

997

    Movsd,

998

    Movups,

999

    Movupd,

1000

    Mulps,

1001

    Mulpd,

1002

    Mulss,

1003

    Mulsd,

1004

    Orps,

1005

    Orpd,

1006

    Pabsb,

1007

    Pabsw,

1008

    Pabsd,

1009

    Packssdw,

1010

    Packsswb,

1011

    Packusdw,

1012

    Packuswb,

1013

    Paddb,

1014

    Paddd,

1015

    Paddq,

1016

    Paddw,

1017

    Paddsb,

1018

    Paddsw,

1019

    Paddusb,

1020

    Paddusw,

1021

    Palignr,

1022

    Pand,

1023

    Pandn,

1024

    Pavgb,

1025

    Pavgw,

1026

    Pblendvb,

1027

    Pcmpeqb,

1028

    Pcmpeqw,

1029

    Pcmpeqd,

1030

    Pcmpeqq,

1031

    Pcmpgtb,

1032

    Pcmpgtw,

1033

    Pcmpgtd,

1034

    Pcmpgtq,

1035

    Pextrb,

1036

    Pextrw,

1037

    Pextrd,

1038

    Pextrq,

1039

    Pinsrb,

1040

    Pinsrw,

1041

    Pinsrd,

1042

    Pmaddubsw,

1043

    Pmaddwd,

1044

    Pmaxsb,

1045

    Pmaxsw,

1046

    Pmaxsd,

1047

    Pmaxub,

1048

    Pmaxuw,

1049

    Pmaxud,

1050

    Pminsb,

1051

    Pminsw,

1052

    Pminsd,

1053

    Pminub,

1054

    Pminuw,

1055

    Pminud,

1056

    Pmovmskb,

1057

    Pmovsxbd,

1058

    Pmovsxbw,

1059

    Pmovsxbq,

1060

    Pmovsxwd,

1061

    Pmovsxwq,

1062

    Pmovsxdq,

1063

    Pmovzxbd,

1064

    Pmovzxbw,

1065

    Pmovzxbq,

1066

    Pmovzxwd,

1067

    Pmovzxwq,

1068

    Pmovzxdq,

1069

    Pmuldq,

1070

    Pmulhw,

1071

    Pmulhuw,

1072

    Pmulhrsw,

1073

    Pmulld,

1074

    Pmullw,

1075

    Pmuludq,

1076

    Por,

1077

    Pshufb,

1078

    Pshufd,

1079

    Psllw,

1080

    Pslld,

1081

    Psllq,

1082

    Psraw,

1083

    Psrad,

1084

    Psrlw,

1085

    Psrld,

1086

    Psrlq,

1087

    Psubb,

1088

    Psubd,

1089

    Psubq,

1090

    Psubw,

1091

    Psubsb,

1092

    Psubsw,

1093

    Psubusb,

1094

    Psubusw,

1095

    Ptest,

1096

    Punpckhbw,

1097

    Punpckhwd,

1098

    Punpcklbw,

1099

    Punpcklwd,

1100

    Pxor,

1101

    Rcpss,

1102

    Roundps,

1103

    Roundpd,

1104

    Roundss,

1105

    Roundsd,

1106

    Rsqrtss,

1107

    Shufps,

1108

    Sqrtps,

1109

    Sqrtpd,

1110

    Sqrtss,

1111

    Sqrtsd,

1112

    Subps,

1113

    Subpd,

1114

    Subss,

1115

    Subsd,

1116

    Ucomiss,

1117

    Ucomisd,

1118

    Unpcklps,

1119

    Xorps,

1120

    Xorpd,

1121

    Phaddw,

1122

    Phaddd,

1123

    Punpckhdq,

1124

    Punpckldq,

1125

    Punpckhqdq,

1126

    Punpcklqdq,

1127

    Pshuflw,

1128

    Pshufhw,

1129

    Pblendw,

1130

    Movddup,

1131

1132

1133

impl SseOpcode {

1134

    /// Which `InstructionSet` is the first supporting this opcode?

1135

    pub(crate) fn available_from(&self) -> InstructionSet {

1136

        use InstructionSet::*;

1137

        match self {

1138

            SseOpcode::Addps

1139

            | SseOpcode::Addss

1140

            | SseOpcode::Andps

1141

            | SseOpcode::Andnps

1142

            | SseOpcode::Comiss

1143

            | SseOpcode::Cmpps

1144

            | SseOpcode::Cmpss

1145

            | SseOpcode::Cvtsi2ss

1146

            | SseOpcode::Cvtss2si

1147

            | SseOpcode::Cvttss2si

1148

            | SseOpcode::Divps

1149

            | SseOpcode::Divss

1150

            | SseOpcode::Maxps

1151

            | SseOpcode::Maxss

1152

            | SseOpcode::Minps

1153

            | SseOpcode::Minss

1154

            | SseOpcode::Movaps

1155

            | SseOpcode::Movlhps

1156

            | SseOpcode::Movmskps

1157

            | SseOpcode::Movss

1158

            | SseOpcode::Movups

1159

            | SseOpcode::Mulps

1160

            | SseOpcode::Mulss

1161

            | SseOpcode::Orps

1162

            | SseOpcode::Rcpss

1163

            | SseOpcode::Rsqrtss

1164

            | SseOpcode::Shufps

1165

            | SseOpcode::Sqrtps

1166

            | SseOpcode::Sqrtss

1167

            | SseOpcode::Subps

1168

            | SseOpcode::Subss

1169

            | SseOpcode::Ucomiss

1170

            | SseOpcode::Unpcklps

1171

            | SseOpcode::Xorps => SSE,

1172

1173

            SseOpcode::Addpd

1174

            | SseOpcode::Addsd

1175

            | SseOpcode::Andpd

1176

            | SseOpcode::Andnpd

1177

            | SseOpcode::Cmppd

1178

            | SseOpcode::Cmpsd

1179

            | SseOpcode::Comisd

1180

            | SseOpcode::Cvtdq2ps

1181

            | SseOpcode::Cvtdq2pd

1182

            | SseOpcode::Cvtpd2ps

1183

            | SseOpcode::Cvtps2pd

1184

            | SseOpcode::Cvtsd2ss

1185

            | SseOpcode::Cvtsd2si

1186

            | SseOpcode::Cvtsi2sd

1187

            | SseOpcode::Cvtss2sd

1188

            | SseOpcode::Cvttpd2dq

1189

            | SseOpcode::Cvttps2dq

1190

            | SseOpcode::Cvttsd2si

1191

            | SseOpcode::Divpd

1192

            | SseOpcode::Divsd

1193

            | SseOpcode::Maxpd

1194

            | SseOpcode::Maxsd

1195

            | SseOpcode::Minpd

1196

            | SseOpcode::Minsd

1197

            | SseOpcode::Movapd

1198

            | SseOpcode::Movd

1199

            | SseOpcode::Movmskpd

1200

            | SseOpcode::Movq

1201

            | SseOpcode::Movsd

1202

            | SseOpcode::Movupd

1203

            | SseOpcode::Movdqa

1204

            | SseOpcode::Movdqu

1205

            | SseOpcode::Mulpd

1206

            | SseOpcode::Mulsd

1207

            | SseOpcode::Orpd

1208

            | SseOpcode::Packssdw

1209

            | SseOpcode::Packsswb

1210

            | SseOpcode::Packuswb

1211

            | SseOpcode::Paddb

1212

            | SseOpcode::Paddd

1213

            | SseOpcode::Paddq

1214

            | SseOpcode::Paddw

1215

            | SseOpcode::Paddsb

1216

            | SseOpcode::Paddsw

1217

            | SseOpcode::Paddusb

1218

            | SseOpcode::Paddusw

1219

            | SseOpcode::Pand

1220

            | SseOpcode::Pandn

1221

            | SseOpcode::Pavgb

1222

            | SseOpcode::Pavgw

1223

            | SseOpcode::Pcmpeqb

1224

            | SseOpcode::Pcmpeqw

1225

            | SseOpcode::Pcmpeqd

1226

            | SseOpcode::Pcmpgtb

1227

            | SseOpcode::Pcmpgtw

1228

            | SseOpcode::Pcmpgtd

1229

            | SseOpcode::Pextrw

1230

            | SseOpcode::Pinsrw

1231

            | SseOpcode::Pmaddubsw

1232

            | SseOpcode::Pmaddwd

1233

            | SseOpcode::Pmaxsw

1234

            | SseOpcode::Pmaxub

1235

            | SseOpcode::Pminsw

1236

            | SseOpcode::Pminub

1237

            | SseOpcode::Pmovmskb

1238

            | SseOpcode::Pmulhw

1239

            | SseOpcode::Pmulhuw

1240

            | SseOpcode::Pmullw

1241

            | SseOpcode::Pmuludq

1242

            | SseOpcode::Por

1243

            | SseOpcode::Pshufd

1244

            | SseOpcode::Psllw

1245

            | SseOpcode::Pslld

1246

            | SseOpcode::Psllq

1247

            | SseOpcode::Psraw

1248

            | SseOpcode::Psrad

1249

            | SseOpcode::Psrlw

1250

            | SseOpcode::Psrld

1251

            | SseOpcode::Psrlq

1252

            | SseOpcode::Psubb

1253

            | SseOpcode::Psubd

1254

            | SseOpcode::Psubq

1255

            | SseOpcode::Psubw

1256

            | SseOpcode::Psubsb

1257

            | SseOpcode::Psubsw

1258

            | SseOpcode::Psubusb

1259

            | SseOpcode::Psubusw

1260

            | SseOpcode::Punpckhbw

1261

            | SseOpcode::Punpckhwd

1262

            | SseOpcode::Punpcklbw

1263

            | SseOpcode::Punpcklwd

1264

            | SseOpcode::Pxor

1265

            | SseOpcode::Sqrtpd

1266

            | SseOpcode::Sqrtsd

1267

            | SseOpcode::Subpd

1268

            | SseOpcode::Subsd

1269

            | SseOpcode::Ucomisd

1270

            | SseOpcode::Xorpd

1271

            | SseOpcode::Punpckldq

1272

            | SseOpcode::Punpckhdq

1273

            | SseOpcode::Punpcklqdq

1274

            | SseOpcode::Punpckhqdq

1275

            | SseOpcode::Pshuflw

1276

            | SseOpcode::Pshufhw => SSE2,

1277

1278

            SseOpcode::Pabsb

1279

            | SseOpcode::Pabsw

1280

            | SseOpcode::Pabsd

1281

            | SseOpcode::Palignr

1282

            | SseOpcode::Pmulhrsw

1283

            | SseOpcode::Pshufb

1284

            | SseOpcode::Phaddw

1285

            | SseOpcode::Phaddd

1286

            | SseOpcode::Movddup => SSSE3,

1287

1288

            SseOpcode::Blendvpd

1289

            | SseOpcode::Blendvps

1290

            | SseOpcode::Insertps

1291

            | SseOpcode::Packusdw

1292

            | SseOpcode::Pblendvb

1293

            | SseOpcode::Pcmpeqq

1294

            | SseOpcode::Pextrb

1295

            | SseOpcode::Pextrd

1296

            | SseOpcode::Pextrq

1297

            | SseOpcode::Pinsrb

1298

            | SseOpcode::Pinsrd

1299

            | SseOpcode::Pmaxsb

1300

            | SseOpcode::Pmaxsd

1301

            | SseOpcode::Pmaxuw

1302

            | SseOpcode::Pmaxud

1303

            | SseOpcode::Pminsb

1304

            | SseOpcode::Pminsd

1305

            | SseOpcode::Pminuw

1306

            | SseOpcode::Pminud

1307

            | SseOpcode::Pmovsxbd

1308

            | SseOpcode::Pmovsxbw

1309

            | SseOpcode::Pmovsxbq

1310

            | SseOpcode::Pmovsxwd

1311

            | SseOpcode::Pmovsxwq

1312

            | SseOpcode::Pmovsxdq

1313

            | SseOpcode::Pmovzxbd

1314

            | SseOpcode::Pmovzxbw

1315

            | SseOpcode::Pmovzxbq

1316

            | SseOpcode::Pmovzxwd

1317

            | SseOpcode::Pmovzxwq

1318

            | SseOpcode::Pmovzxdq

1319

            | SseOpcode::Pmuldq

1320

            | SseOpcode::Pmulld

1321

            | SseOpcode::Ptest

1322

            | SseOpcode::Roundps

1323

            | SseOpcode::Roundpd

1324

            | SseOpcode::Roundss

1325

            | SseOpcode::Roundsd

1326

            | SseOpcode::Pblendw => SSE41,

1327

1328

            SseOpcode::Pcmpgtq => SSE42,

1329

1330

1331

1332

    /// Returns the src operand size for an instruction.

1333

    pub(crate) fn src_size(&self) -> u8 {

1334

        match self {

1335

            SseOpcode::Movd => 4,

1336

            _ => 8,

1337

1338

1339

1340

1341

impl fmt::Debug for SseOpcode {

1342

    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {

1343

        let name = match self {

1344

            SseOpcode::Addps => "addps",

1345

            SseOpcode::Addpd => "addpd",

1346

            SseOpcode::Addss => "addss",

1347

            SseOpcode::Addsd => "addsd",

1348

            SseOpcode::Andpd => "andpd",

1349

            SseOpcode::Andps => "andps",

1350

            SseOpcode::Andnps => "andnps",

1351

            SseOpcode::Andnpd => "andnpd",

1352

            SseOpcode::Blendvpd => "blendvpd",

1353

            SseOpcode::Blendvps => "blendvps",

1354

            SseOpcode::Cmpps => "cmpps",

1355

            SseOpcode::Cmppd => "cmppd",

1356

            SseOpcode::Cmpss => "cmpss",

1357

            SseOpcode::Cmpsd => "cmpsd",

1358

            SseOpcode::Comiss => "comiss",

1359

            SseOpcode::Comisd => "comisd",

1360

            SseOpcode::Cvtdq2ps => "cvtdq2ps",

1361

            SseOpcode::Cvtdq2pd => "cvtdq2pd",

1362

            SseOpcode::Cvtpd2ps => "cvtpd2ps",

1363

            SseOpcode::Cvtps2pd => "cvtps2pd",

1364

            SseOpcode::Cvtsd2ss => "cvtsd2ss",

1365

            SseOpcode::Cvtsd2si => "cvtsd2si",

1366

            SseOpcode::Cvtsi2ss => "cvtsi2ss",

1367

            SseOpcode::Cvtsi2sd => "cvtsi2sd",

1368

            SseOpcode::Cvtss2si => "cvtss2si",

1369

            SseOpcode::Cvtss2sd => "cvtss2sd",

1370

            SseOpcode::Cvttpd2dq => "cvttpd2dq",

1371

            SseOpcode::Cvttps2dq => "cvttps2dq",

1372

            SseOpcode::Cvttss2si => "cvttss2si",

1373

            SseOpcode::Cvttsd2si => "cvttsd2si",

1374

            SseOpcode::Divps => "divps",

1375

            SseOpcode::Divpd => "divpd",

1376

            SseOpcode::Divss => "divss",

1377

            SseOpcode::Divsd => "divsd",

1378

            SseOpcode::Insertps => "insertps",

1379

            SseOpcode::Maxps => "maxps",

1380

            SseOpcode::Maxpd => "maxpd",

1381

            SseOpcode::Maxss => "maxss",

1382

            SseOpcode::Maxsd => "maxsd",

1383

            SseOpcode::Minps => "minps",

1384

            SseOpcode::Minpd => "minpd",

1385

            SseOpcode::Minss => "minss",

1386

            SseOpcode::Minsd => "minsd",

1387

            SseOpcode::Movaps => "movaps",

1388

            SseOpcode::Movapd => "movapd",

1389

            SseOpcode::Movd => "movd",

1390

            SseOpcode::Movdqa => "movdqa",

1391

            SseOpcode::Movdqu => "movdqu",

1392

            SseOpcode::Movlhps => "movlhps",

1393

            SseOpcode::Movmskps => "movmskps",

1394

            SseOpcode::Movmskpd => "movmskpd",

1395

            SseOpcode::Movq => "movq",

1396

            SseOpcode::Movss => "movss",

1397

            SseOpcode::Movsd => "movsd",

1398

            SseOpcode::Movups => "movups",

1399

            SseOpcode::Movupd => "movupd",

1400

            SseOpcode::Mulps => "mulps",

1401

            SseOpcode::Mulpd => "mulpd",

1402

            SseOpcode::Mulss => "mulss",

1403

            SseOpcode::Mulsd => "mulsd",

1404

            SseOpcode::Orpd => "orpd",

1405

            SseOpcode::Orps => "orps",

1406

            SseOpcode::Pabsb => "pabsb",

1407

            SseOpcode::Pabsw => "pabsw",

1408

            SseOpcode::Pabsd => "pabsd",

1409

            SseOpcode::Packssdw => "packssdw",

1410

            SseOpcode::Packsswb => "packsswb",

1411

            SseOpcode::Packusdw => "packusdw",

1412

            SseOpcode::Packuswb => "packuswb",

1413

            SseOpcode::Paddb => "paddb",

1414

            SseOpcode::Paddd => "paddd",

1415

            SseOpcode::Paddq => "paddq",

1416

            SseOpcode::Paddw => "paddw",

1417

            SseOpcode::Paddsb => "paddsb",

1418

            SseOpcode::Paddsw => "paddsw",

1419

            SseOpcode::Paddusb => "paddusb",

1420

            SseOpcode::Paddusw => "paddusw",

1421

            SseOpcode::Palignr => "palignr",

1422

            SseOpcode::Pand => "pand",

1423

            SseOpcode::Pandn => "pandn",

1424

            SseOpcode::Pavgb => "pavgb",

1425

            SseOpcode::Pavgw => "pavgw",

1426

            SseOpcode::Pblendvb => "pblendvb",

1427

            SseOpcode::Pcmpeqb => "pcmpeqb",

1428

            SseOpcode::Pcmpeqw => "pcmpeqw",

1429

            SseOpcode::Pcmpeqd => "pcmpeqd",

1430

            SseOpcode::Pcmpeqq => "pcmpeqq",

1431

            SseOpcode::Pcmpgtb => "pcmpgtb",

1432

            SseOpcode::Pcmpgtw => "pcmpgtw",

1433

            SseOpcode::Pcmpgtd => "pcmpgtd",

1434

            SseOpcode::Pcmpgtq => "pcmpgtq",

1435

            SseOpcode::Pextrb => "pextrb",

1436

            SseOpcode::Pextrw => "pextrw",

1437

            SseOpcode::Pextrd => "pextrd",

1438

            SseOpcode::Pextrq => "pextrq",

1439

            SseOpcode::Pinsrb => "pinsrb",

1440

            SseOpcode::Pinsrw => "pinsrw",

1441

            SseOpcode::Pinsrd => "pinsrd",

1442

            SseOpcode::Pmaddubsw => "pmaddubsw",

1443

            SseOpcode::Pmaddwd => "pmaddwd",

1444

            SseOpcode::Pmaxsb => "pmaxsb",

1445

            SseOpcode::Pmaxsw => "pmaxsw",

1446

            SseOpcode::Pmaxsd => "pmaxsd",

1447

            SseOpcode::Pmaxub => "pmaxub",

1448

            SseOpcode::Pmaxuw => "pmaxuw",

1449

            SseOpcode::Pmaxud => "pmaxud",

1450

            SseOpcode::Pminsb => "pminsb",

1451

            SseOpcode::Pminsw => "pminsw",

1452

            SseOpcode::Pminsd => "pminsd",

1453

            SseOpcode::Pminub => "pminub",

1454

            SseOpcode::Pminuw => "pminuw",

1455

            SseOpcode::Pminud => "pminud",

1456

            SseOpcode::Pmovmskb => "pmovmskb",

1457

            SseOpcode::Pmovsxbd => "pmovsxbd",

1458

            SseOpcode::Pmovsxbw => "pmovsxbw",

1459

            SseOpcode::Pmovsxbq => "pmovsxbq",

1460

            SseOpcode::Pmovsxwd => "pmovsxwd",

1461

            SseOpcode::Pmovsxwq => "pmovsxwq",

1462

            SseOpcode::Pmovsxdq => "pmovsxdq",

1463

            SseOpcode::Pmovzxbd => "pmovzxbd",

1464

            SseOpcode::Pmovzxbw => "pmovzxbw",

1465

            SseOpcode::Pmovzxbq => "pmovzxbq",

1466

            SseOpcode::Pmovzxwd => "pmovzxwd",

1467

            SseOpcode::Pmovzxwq => "pmovzxwq",

1468

            SseOpcode::Pmovzxdq => "pmovzxdq",

1469

            SseOpcode::Pmuldq => "pmuldq",

1470

            SseOpcode::Pmulhw => "pmulhw",

1471

            SseOpcode::Pmulhuw => "pmulhuw",

1472

            SseOpcode::Pmulhrsw => "pmulhrsw",

1473

            SseOpcode::Pmulld => "pmulld",

1474

            SseOpcode::Pmullw => "pmullw",

1475

            SseOpcode::Pmuludq => "pmuludq",

1476

            SseOpcode::Por => "por",

1477

            SseOpcode::Pshufb => "pshufb",

1478

            SseOpcode::Pshufd => "pshufd",

1479

            SseOpcode::Psllw => "psllw",

1480

            SseOpcode::Pslld => "pslld",

1481

            SseOpcode::Psllq => "psllq",

1482

            SseOpcode::Psraw => "psraw",

1483

            SseOpcode::Psrad => "psrad",

1484

            SseOpcode::Psrlw => "psrlw",

1485

            SseOpcode::Psrld => "psrld",

1486

            SseOpcode::Psrlq => "psrlq",

1487

            SseOpcode::Psubb => "psubb",

1488

            SseOpcode::Psubd => "psubd",

1489

            SseOpcode::Psubq => "psubq",

1490

            SseOpcode::Psubw => "psubw",

1491

            SseOpcode::Psubsb => "psubsb",

1492

            SseOpcode::Psubsw => "psubsw",

1493

            SseOpcode::Psubusb => "psubusb",

1494

            SseOpcode::Psubusw => "psubusw",

1495

            SseOpcode::Ptest => "ptest",

1496

            SseOpcode::Punpckhbw => "punpckhbw",

1497

            SseOpcode::Punpckhwd => "punpckhwd",

1498

            SseOpcode::Punpcklbw => "punpcklbw",

1499

            SseOpcode::Punpcklwd => "punpcklwd",

1500

            SseOpcode::Pxor => "pxor",

1501

            SseOpcode::Rcpss => "rcpss",

1502

            SseOpcode::Roundps => "roundps",

1503

            SseOpcode::Roundpd => "roundpd",

1504

            SseOpcode::Roundss => "roundss",

1505

            SseOpcode::Roundsd => "roundsd",

1506

            SseOpcode::Rsqrtss => "rsqrtss",

1507

            SseOpcode::Shufps => "shufps",

1508

            SseOpcode::Sqrtps => "sqrtps",

1509

            SseOpcode::Sqrtpd => "sqrtpd",

1510

            SseOpcode::Sqrtss => "sqrtss",

1511

            SseOpcode::Sqrtsd => "sqrtsd",

1512

            SseOpcode::Subps => "subps",

1513

            SseOpcode::Subpd => "subpd",

1514

            SseOpcode::Subss => "subss",

1515

            SseOpcode::Subsd => "subsd",

1516

            SseOpcode::Ucomiss => "ucomiss",

1517

            SseOpcode::Ucomisd => "ucomisd",

1518

            SseOpcode::Unpcklps => "unpcklps",

1519

            SseOpcode::Xorps => "xorps",

1520

            SseOpcode::Xorpd => "xorpd",

1521

            SseOpcode::Phaddw => "phaddw",

1522

            SseOpcode::Phaddd => "phaddd",

1523

            SseOpcode::Punpckldq => "punpckldq",

1524

            SseOpcode::Punpckhdq => "punpckhdq",

1525

            SseOpcode::Punpcklqdq => "punpcklqdq",

1526

            SseOpcode::Punpckhqdq => "punpckhqdq",

1527

            SseOpcode::Pshuflw => "pshuflw",

1528

            SseOpcode::Pshufhw => "pshufhw",

1529

            SseOpcode::Pblendw => "pblendw",

1530

            SseOpcode::Movddup => "movddup",

1531

};

1532

        write!(fmt, "{}", name)

1533

1534

1535

1536

impl fmt::Display for SseOpcode {

1537

    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {

1538

        fmt::Debug::fmt(self, f)

1539

1540

1541

1542

pub use crate::isa::x64::lower::isle::generated_code::AvxOpcode;

1543

1544

impl AvxOpcode {

1545

    /// Which `InstructionSet`s support the opcode?

1546

    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {

1547

        match self {

1548

            AvxOpcode::Vfmadd213ss

1549

            | AvxOpcode::Vfmadd213sd

1550

            | AvxOpcode::Vfmadd213ps

1551

            | AvxOpcode::Vfmadd213pd

1552

            | AvxOpcode::Vfmadd132ss

1553

            | AvxOpcode::Vfmadd132sd

1554

            | AvxOpcode::Vfmadd132ps

1555

            | AvxOpcode::Vfmadd132pd

1556

            | AvxOpcode::Vfnmadd213ss

1557

            | AvxOpcode::Vfnmadd213sd

1558

            | AvxOpcode::Vfnmadd213ps

1559

            | AvxOpcode::Vfnmadd213pd

1560

            | AvxOpcode::Vfnmadd132ss

1561

            | AvxOpcode::Vfnmadd132sd

1562

            | AvxOpcode::Vfnmadd132ps

1563

            | AvxOpcode::Vfnmadd132pd => smallvec![InstructionSet::FMA],

1564

            AvxOpcode::Vminps

1565

            | AvxOpcode::Vminpd

1566

            | AvxOpcode::Vmaxps

1567

            | AvxOpcode::Vmaxpd

1568

            | AvxOpcode::Vandnps

1569

            | AvxOpcode::Vandnpd

1570

            | AvxOpcode::Vpandn

1571

            | AvxOpcode::Vcmpps

1572

            | AvxOpcode::Vcmppd

1573

            | AvxOpcode::Vpsrlw

1574

            | AvxOpcode::Vpsrld

1575

            | AvxOpcode::Vpsrlq

1576

            | AvxOpcode::Vpaddb

1577

            | AvxOpcode::Vpaddw

1578

            | AvxOpcode::Vpaddd

1579

            | AvxOpcode::Vpaddq

1580

            | AvxOpcode::Vpaddsb

1581

            | AvxOpcode::Vpaddsw

1582

            | AvxOpcode::Vpaddusb

1583

            | AvxOpcode::Vpaddusw

1584

            | AvxOpcode::Vpsubb

1585

            | AvxOpcode::Vpsubw

1586

            | AvxOpcode::Vpsubd

1587

            | AvxOpcode::Vpsubq

1588

            | AvxOpcode::Vpsubsb

1589

            | AvxOpcode::Vpsubsw

1590

            | AvxOpcode::Vpsubusb

1591

            | AvxOpcode::Vpsubusw

1592

            | AvxOpcode::Vpavgb

1593

            | AvxOpcode::Vpavgw

1594

            | AvxOpcode::Vpand

1595

            | AvxOpcode::Vandps

1596

            | AvxOpcode::Vandpd

1597

            | AvxOpcode::Vpor

1598

            | AvxOpcode::Vorps

1599

            | AvxOpcode::Vorpd

1600

            | AvxOpcode::Vpxor

1601

            | AvxOpcode::Vxorps

1602

            | AvxOpcode::Vxorpd

1603

            | AvxOpcode::Vpmullw

1604

            | AvxOpcode::Vpmulld

1605

            | AvxOpcode::Vpmulhw

1606

            | AvxOpcode::Vpmulhd

1607

            | AvxOpcode::Vpmulhrsw

1608

            | AvxOpcode::Vpmulhuw

1609

            | AvxOpcode::Vpmuldq

1610

            | AvxOpcode::Vpmuludq

1611

            | AvxOpcode::Vpunpckhwd

1612

            | AvxOpcode::Vpunpcklwd

1613

            | AvxOpcode::Vunpcklps

1614

            | AvxOpcode::Vaddps

1615

            | AvxOpcode::Vaddpd

1616

            | AvxOpcode::Vsubps

1617

            | AvxOpcode::Vsubpd

1618

            | AvxOpcode::Vmulps

1619

            | AvxOpcode::Vmulpd

1620

            | AvxOpcode::Vdivps

1621

            | AvxOpcode::Vdivpd

1622

            | AvxOpcode::Vpcmpeqb

1623

            | AvxOpcode::Vpcmpeqw

1624

            | AvxOpcode::Vpcmpeqd

1625

            | AvxOpcode::Vpcmpeqq

1626

            | AvxOpcode::Vpcmpgtb

1627

            | AvxOpcode::Vpcmpgtw

1628

            | AvxOpcode::Vpcmpgtd

1629

            | AvxOpcode::Vpcmpgtq

1630

            | AvxOpcode::Vblendvps

1631

            | AvxOpcode::Vblendvpd

1632

            | AvxOpcode::Vpblendvb

1633

            | AvxOpcode::Vmovlhps

1634

            | AvxOpcode::Vpminsb

1635

            | AvxOpcode::Vpminsw

1636

            | AvxOpcode::Vpminsd

1637

            | AvxOpcode::Vpminub

1638

            | AvxOpcode::Vpminuw

1639

            | AvxOpcode::Vpminud

1640

            | AvxOpcode::Vpmaxsb

1641

            | AvxOpcode::Vpmaxsw

1642

            | AvxOpcode::Vpmaxsd

1643

            | AvxOpcode::Vpmaxub

1644

            | AvxOpcode::Vpmaxuw

1645

            | AvxOpcode::Vpmaxud

1646

            | AvxOpcode::Vpunpcklbw

1647

            | AvxOpcode::Vpunpckhbw

1648

            | AvxOpcode::Vpacksswb

1649

            | AvxOpcode::Vpackssdw

1650

            | AvxOpcode::Vpackuswb

1651

            | AvxOpcode::Vpackusdw

1652

            | AvxOpcode::Vpalignr

1653

            | AvxOpcode::Vpinsrb

1654

            | AvxOpcode::Vpinsrw

1655

            | AvxOpcode::Vpinsrd

1656

            | AvxOpcode::Vpinsrq

1657

            | AvxOpcode::Vpmaddwd

1658

            | AvxOpcode::Vpmaddubsw

1659

            | AvxOpcode::Vinsertps

1660

            | AvxOpcode::Vpshufb

1661

            | AvxOpcode::Vshufps

1662

            | AvxOpcode::Vpsllw

1663

            | AvxOpcode::Vpslld

1664

            | AvxOpcode::Vpsllq

1665

            | AvxOpcode::Vpsraw

1666

            | AvxOpcode::Vpsrad

1667

            | AvxOpcode::Vpmovsxbw

1668

            | AvxOpcode::Vpmovzxbw

1669

            | AvxOpcode::Vpmovsxwd

1670

            | AvxOpcode::Vpmovzxwd

1671

            | AvxOpcode::Vpmovsxdq

1672

            | AvxOpcode::Vpmovzxdq

1673

            | AvxOpcode::Vaddss

1674

            | AvxOpcode::Vaddsd

1675

            | AvxOpcode::Vmulss

1676

            | AvxOpcode::Vmulsd

1677

            | AvxOpcode::Vsubss

1678

            | AvxOpcode::Vsubsd

1679

            | AvxOpcode::Vdivss

1680

            | AvxOpcode::Vdivsd

1681

            | AvxOpcode::Vpabsb

1682

            | AvxOpcode::Vpabsw

1683

            | AvxOpcode::Vpabsd

1684

            | AvxOpcode::Vminss

1685

            | AvxOpcode::Vminsd

1686

            | AvxOpcode::Vmaxss

1687

            | AvxOpcode::Vmaxsd

1688

            | AvxOpcode::Vsqrtps

1689

            | AvxOpcode::Vsqrtpd

1690

            | AvxOpcode::Vroundpd

1691

            | AvxOpcode::Vroundps

1692

            | AvxOpcode::Vcvtdq2pd

1693

            | AvxOpcode::Vcvtdq2ps

1694

            | AvxOpcode::Vcvtpd2ps

1695

            | AvxOpcode::Vcvtps2pd

1696

            | AvxOpcode::Vcvttpd2dq

1697

            | AvxOpcode::Vcvttps2dq

1698

            | AvxOpcode::Vphaddw

1699

            | AvxOpcode::Vphaddd

1700

            | AvxOpcode::Vpunpckldq

1701

            | AvxOpcode::Vpunpckhdq

1702

            | AvxOpcode::Vpunpcklqdq

1703

            | AvxOpcode::Vpunpckhqdq

1704

            | AvxOpcode::Vpshuflw

1705

            | AvxOpcode::Vpshufhw

1706

            | AvxOpcode::Vpshufd

1707

            | AvxOpcode::Vmovss

1708

            | AvxOpcode::Vmovsd

1709

            | AvxOpcode::Vmovups

1710

            | AvxOpcode::Vmovupd

1711

            | AvxOpcode::Vmovdqu

1712

            | AvxOpcode::Vpextrb

1713

            | AvxOpcode::Vpextrw

1714

            | AvxOpcode::Vpextrd

1715

            | AvxOpcode::Vpextrq

1716

            | AvxOpcode::Vpblendw

1717

            | AvxOpcode::Vmovddup

1718

            | AvxOpcode::Vbroadcastss

1719

            | AvxOpcode::Vmovd

1720

            | AvxOpcode::Vmovq

1721

            | AvxOpcode::Vmovmskps

1722

            | AvxOpcode::Vmovmskpd

1723

            | AvxOpcode::Vpmovmskb

1724

            | AvxOpcode::Vcvtsi2ss

1725

            | AvxOpcode::Vcvtsi2sd

1726

            | AvxOpcode::Vcvtss2sd

1727

            | AvxOpcode::Vcvtsd2ss

1728

            | AvxOpcode::Vsqrtss

1729

            | AvxOpcode::Vsqrtsd

1730

            | AvxOpcode::Vroundss

1731

            | AvxOpcode::Vroundsd => {

1732

                smallvec![InstructionSet::AVX]

1733

1734

1735

            AvxOpcode::Vpbroadcastb | AvxOpcode::Vpbroadcastw | AvxOpcode::Vpbroadcastd => {

1736

                smallvec![InstructionSet::AVX2]

1737

1738

1739

1740

1741

1742

impl fmt::Display for AvxOpcode {

1743

    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {

1744

        format!("{self:?}").to_lowercase().fmt(f)

1745

1746

1747

1748

#[derive(Clone, PartialEq)]

1749

#[allow(missing_docs)]

1750

pub enum Avx512Opcode {

1751

    Vcvtudq2ps,

1752

    Vpabsq,

1753

    Vpermi2b,

1754

    Vpmullq,

1755

    Vpopcntb,

1756

1757

1758

impl Avx512Opcode {

1759

    /// Which `InstructionSet`s support the opcode?

1760

    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {

1761

        match self {

1762

            Avx512Opcode::Vcvtudq2ps => {

1763

                smallvec![InstructionSet::AVX512F, InstructionSet::AVX512VL]

1764

1765

            Avx512Opcode::Vpabsq => smallvec![InstructionSet::AVX512F, InstructionSet::AVX512VL],

1766

            Avx512Opcode::Vpermi2b => {

1767

                smallvec![InstructionSet::AVX512VL, InstructionSet::AVX512VBMI]

1768

1769

            Avx512Opcode::Vpmullq => smallvec![InstructionSet::AVX512VL, InstructionSet::AVX512DQ],

1770

            Avx512Opcode::Vpopcntb => {

1771

                smallvec![InstructionSet::AVX512VL, InstructionSet::AVX512BITALG]

1772

1773

1774

1775

1776

1777

impl fmt::Debug for Avx512Opcode {

1778

    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {

1779

        let name = match self {

1780

            Avx512Opcode::Vcvtudq2ps => "vcvtudq2ps",

1781

            Avx512Opcode::Vpabsq => "vpabsq",

1782

            Avx512Opcode::Vpermi2b => "vpermi2b",

1783

            Avx512Opcode::Vpmullq => "vpmullq",

1784

            Avx512Opcode::Vpopcntb => "vpopcntb",

1785

};

1786

        write!(fmt, "{}", name)

1787

1788

1789

1790

impl fmt::Display for Avx512Opcode {

1791

    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {

1792

        fmt::Debug::fmt(self, f)

1793

1794

1795

1796

/// This defines the ways a value can be extended: either signed- or zero-extension, or none for

1797

/// types that are not extended. Contrast with [ExtMode], which defines the widths from and to which

1798

/// values can be extended.

1799

#[allow(dead_code)]

1800

#[derive(Clone, PartialEq)]

1801

pub enum ExtKind {

1802

    /// No extension.

1803

    None,

1804

    /// Sign-extend.

1805

    SignExtend,

1806

    /// Zero-extend.

1807

    ZeroExtend,

1808

1809

1810

/// These indicate ways of extending (widening) a value, using the Intel

1811

/// naming: B(yte) = u8, W(ord) = u16, L(ong)word = u32, Q(uad)word = u64

1812

#[derive(Clone, PartialEq)]

1813

pub enum ExtMode {

1814

    /// Byte -> Longword.

1815

BL,

1816

    /// Byte -> Quadword.

1817

BQ,

1818

    /// Word -> Longword.

1819

WL,

1820

    /// Word -> Quadword.

1821

WQ,

1822

    /// Longword -> Quadword.

1823

LQ,

1824

1825

1826

impl ExtMode {

1827

    /// Calculate the `ExtMode` from passed bit lengths of the from/to types.

1828

    pub(crate) fn new(from_bits: u16, to_bits: u16) -> Option<ExtMode> {

1829

        match (from_bits, to_bits) {

1830

            (1, 8) | (1, 16) | (1, 32) | (8, 16) | (8, 32) => Some(ExtMode::BL),

1831

            (1, 64) | (8, 64) => Some(ExtMode::BQ),

1832

            (16, 32) => Some(ExtMode::WL),

1833

            (16, 64) => Some(ExtMode::WQ),

1834

            (32, 64) => Some(ExtMode::LQ),

1835

            _ => None,

1836

1837

1838

1839

    /// Return the source register size in bytes.

1840

    pub(crate) fn src_size(&self) -> u8 {

1841

        match self {

1842

            ExtMode::BL | ExtMode::BQ => 1,

1843

            ExtMode::WL | ExtMode::WQ => 2,

1844

            ExtMode::LQ => 4,

1845

1846

1847

1848

    /// Return the destination register size in bytes.

1849

    pub(crate) fn dst_size(&self) -> u8 {

1850

        match self {

1851

            ExtMode::BL | ExtMode::WL => 4,

1852

            ExtMode::BQ | ExtMode::WQ | ExtMode::LQ => 8,

1853

1854

1855

1856

1857

impl fmt::Debug for ExtMode {

1858

    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {

1859

        let name = match self {

1860

            ExtMode::BL => "bl",

1861

            ExtMode::BQ => "bq",

1862

            ExtMode::WL => "wl",

1863

            ExtMode::WQ => "wq",

1864

            ExtMode::LQ => "lq",

1865

};

1866

        write!(fmt, "{}", name)

1867

1868

1869

1870

impl fmt::Display for ExtMode {

1871

    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {

1872

        fmt::Debug::fmt(self, f)

1873

1874

1875

1876

/// These indicate the form of a scalar shift/rotate: left, signed right, unsigned right.

1877

#[derive(Clone, Copy)]

1878

pub enum ShiftKind {

1879

    /// Left shift.

1880

    ShiftLeft,

1881

    /// Inserts zeros in the most significant bits.

1882

    ShiftRightLogical,

1883

    /// Replicates the sign bit in the most significant bits.

1884

    ShiftRightArithmetic,

1885

    /// Left rotation.

1886

    RotateLeft,

1887

    /// Right rotation.

1888

    RotateRight,

1889

1890

1891

impl fmt::Debug for ShiftKind {

1892

    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {

1893

        let name = match self {

1894

            ShiftKind::ShiftLeft => "shl",

1895

            ShiftKind::ShiftRightLogical => "shr",

1896

            ShiftKind::ShiftRightArithmetic => "sar",

1897

            ShiftKind::RotateLeft => "rol",

1898

            ShiftKind::RotateRight => "ror",

1899

};

1900

        write!(fmt, "{}", name)

1901

1902

1903

1904

impl fmt::Display for ShiftKind {

1905

    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {

1906

        fmt::Debug::fmt(self, f)

1907

1908

1909

1910

/// These indicate condition code tests.  Not all are represented since not all are useful in

1911

/// compiler-generated code.

1912

#[derive(Copy, Clone)]

1913

#[repr(u8)]

1914

pub enum CC {

1915

    ///  overflow

1916

    O = 0,

1917

    /// no overflow

1918

    NO = 1,

1919

1920

    /// < unsigned

1921

    B = 2,

1922

    /// >= unsigned

1923

    NB = 3,

1924

1925

    /// zero

1926

    Z = 4,

1927

    /// not-zero

1928

    NZ = 5,

1929

1930

    /// <= unsigned

1931

    BE = 6,

1932

    /// > unsigned

1933

    NBE = 7,

1934

1935

    /// negative

1936

    S = 8,

1937

    /// not-negative

1938

    NS = 9,

1939

1940

    /// < signed

1941

    L = 12,

1942

    /// >= signed

1943

    NL = 13,

1944

1945

    /// <= signed

1946

    LE = 14,

1947

    /// > signed

1948

    NLE = 15,

1949

1950

    /// parity

1951

    P = 10,

1952

1953

    /// not parity

1954

    NP = 11,

1955

1956

1957

impl CC {

1958

    pub(crate) fn from_intcc(intcc: IntCC) -> Self {

1959

        match intcc {

1960

            IntCC::Equal => CC::Z,

1961

            IntCC::NotEqual => CC::NZ,

1962

            IntCC::SignedGreaterThanOrEqual => CC::NL,

1963

            IntCC::SignedGreaterThan => CC::NLE,

1964

            IntCC::SignedLessThanOrEqual => CC::LE,

1965

            IntCC::SignedLessThan => CC::L,

1966

            IntCC::UnsignedGreaterThanOrEqual => CC::NB,

1967

            IntCC::UnsignedGreaterThan => CC::NBE,

1968

            IntCC::UnsignedLessThanOrEqual => CC::BE,

1969

            IntCC::UnsignedLessThan => CC::B,

1970

1971

1972

1973

    pub(crate) fn invert(&self) -> Self {

1974

        match self {

1975

            CC::O => CC::NO,

1976

            CC::NO => CC::O,

1977

1978

            CC::B => CC::NB,

1979

            CC::NB => CC::B,

1980

1981

            CC::Z => CC::NZ,

1982

            CC::NZ => CC::Z,

1983

1984

            CC::BE => CC::NBE,

1985

            CC::NBE => CC::BE,

1986

1987

            CC::S => CC::NS,

1988

            CC::NS => CC::S,

1989

1990

            CC::L => CC::NL,

1991

            CC::NL => CC::L,

1992

1993

            CC::LE => CC::NLE,

1994

            CC::NLE => CC::LE,

1995

1996

            CC::P => CC::NP,

1997

            CC::NP => CC::P,

1998

1999

2000

2001

    pub(crate) fn get_enc(self) -> u8 {

2002

        self as u8

2003

2004

2005

2006

impl fmt::Debug for CC {

2007

    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {

2008

        let name = match self {

2009

            CC::O => "o",

2010

            CC::NO => "no",

2011

            CC::B => "b",

2012

            CC::NB => "nb",

2013

            CC::Z => "z",

2014

            CC::NZ => "nz",

2015

            CC::BE => "be",

2016

            CC::NBE => "nbe",

2017

            CC::S => "s",

2018

            CC::NS => "ns",

2019

            CC::L => "l",

2020

            CC::NL => "nl",

2021

            CC::LE => "le",

2022

            CC::NLE => "nle",

2023

            CC::P => "p",

2024

            CC::NP => "np",

2025

};

2026

        write!(fmt, "{}", name)

2027

2028

2029

2030

impl fmt::Display for CC {

2031

    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {

2032

        fmt::Debug::fmt(self, f)

2033

2034

2035

2036

/// Encode the ways that floats can be compared. This is used in float comparisons such as `cmpps`,

2037

/// e.g.; it is distinguished from other float comparisons (e.g. `ucomiss`) in that those use EFLAGS

2038

/// whereas [FcmpImm] is used as an immediate.

2039

#[derive(Clone, Copy)]

2040

pub enum FcmpImm {

2041

    /// Equal comparison.

2042

    Equal = 0x00,

2043

    /// Less than comparison.

2044

    LessThan = 0x01,

2045

    /// Less than or equal comparison.

2046

    LessThanOrEqual = 0x02,

2047

    /// Unordered.

2048

    Unordered = 0x03,

2049

    /// Not equal comparison.

2050

    NotEqual = 0x04,

2051

    /// Unordered of greater than or equal comparison.

2052

    UnorderedOrGreaterThanOrEqual = 0x05,

2053

    /// Unordered or greater than comparison.

2054

    UnorderedOrGreaterThan = 0x06,

2055

    /// Ordered.

2056

    Ordered = 0x07,

2057

2058

2059

impl FcmpImm {

2060

    pub(crate) fn encode(self) -> u8 {

2061

        self as u8

2062

2063

2064

2065

impl From<FloatCC> for FcmpImm {

2066

    fn from(cond: FloatCC) -> Self {

2067

        match cond {

2068

            FloatCC::Equal => FcmpImm::Equal,

2069

            FloatCC::LessThan => FcmpImm::LessThan,

2070

            FloatCC::LessThanOrEqual => FcmpImm::LessThanOrEqual,

2071

            FloatCC::Unordered => FcmpImm::Unordered,

2072

            FloatCC::NotEqual => FcmpImm::NotEqual,

2073

            FloatCC::UnorderedOrGreaterThanOrEqual => FcmpImm::UnorderedOrGreaterThanOrEqual,

2074

            FloatCC::UnorderedOrGreaterThan => FcmpImm::UnorderedOrGreaterThan,

2075

            FloatCC::Ordered => FcmpImm::Ordered,

2076

            _ => panic!("unable to create comparison predicate for {}", cond),

2077

2078

2079

2080

2081

/// Encode the rounding modes used as part of the Rounding Control field.

2082

/// Note, these rounding immediates only consider the rounding control field

2083

/// (i.e. the rounding mode) which only take up the first two bits when encoded.

2084

/// However the rounding immediate which this field helps make up, also includes

2085

/// bits 3 and 4 which define the rounding select and precision mask respectively.

2086

/// These two bits are not defined here and are implictly set to zero when encoded.

2087

#[derive(Clone, Copy)]

2088

pub enum RoundImm {

2089

    /// Round to nearest mode.

2090

    RoundNearest = 0x00,

2091

    /// Round down mode.

2092

    RoundDown = 0x01,

2093

    /// Round up mode.

2094

    RoundUp = 0x02,

2095

    /// Round to zero mode.

2096

    RoundZero = 0x03,

2097

2098

2099

impl RoundImm {

2100

    pub(crate) fn encode(self) -> u8 {

2101

        self as u8

2102

2103

2104

2105

/// An operand's size in bits.

2106

#[derive(Clone, Copy, PartialEq)]

2107

pub enum OperandSize {

2108

    /// 8-bit.

2109

    Size8,

2110

    /// 16-bit.

2111

    Size16,

2112

    /// 32-bit.

2113

    Size32,

2114

    /// 64-bit.

2115

    Size64,

2116

2117

2118

impl OperandSize {

2119

    pub(crate) fn from_bytes(num_bytes: u32) -> Self {

2120

        match num_bytes {

2121

            1 => OperandSize::Size8,

2122

            2 => OperandSize::Size16,

2123

            4 => OperandSize::Size32,

2124

            8 => OperandSize::Size64,

2125

            _ => unreachable!("Invalid OperandSize: {}", num_bytes),

2126

2127

2128

2129

    // Computes the OperandSize for a given type.

2130

    // For vectors, the OperandSize of the lanes is returned.

2131

    pub(crate) fn from_ty(ty: Type) -> Self {

2132

        Self::from_bytes(ty.lane_type().bytes())

2133

2134

2135

    // Check that the value of self is one of the allowed sizes.

2136

    pub(crate) fn is_one_of(&self, sizes: &[Self]) -> bool {

2137

        sizes.iter().any(|val| *self == *val)

2138

2139

2140

    pub(crate) fn to_bytes(&self) -> u8 {

2141

        match self {

2142

            Self::Size8 => 1,

2143

            Self::Size16 => 2,

2144

            Self::Size32 => 4,

2145

            Self::Size64 => 8,

2146

2147

2148

2149

    pub(crate) fn to_bits(&self) -> u8 {

2150

        self.to_bytes() * 8

2151

2152

2153

2154

/// An x64 memory fence kind.

2155

#[derive(Clone)]

2156

#[allow(dead_code)]

2157

pub enum FenceKind {

2158

    /// `mfence` instruction ("Memory Fence")

2159

    MFence,

2160

    /// `lfence` instruction ("Load Fence")

2161

    LFence,

2162

    /// `sfence` instruction ("Store Fence")

2163

    SFence,

2164